@based/schema 5.0.0-alpha.2 → 5.0.0-alpha.21

package/dist/def/selvaBuffer.js

@@ -0,0 +1,145 @@
+import { ALIAS, ALIASES, BINARY, EMPTY_MICRO_BUFFER, CARDINALITY, MICRO_BUFFER, REFERENCE, REFERENCES, STRING, TEXT, VECTOR, WEAK_REFERENCE, WEAK_REFERENCES, JSON, COLVEC, } from './types.js';
+const selvaFieldType = {
+    NULL: 0,
+    MICRO_BUFFER: 1,
+    STRING: 2,
+    TEXT: 3,
+    REFERENCE: 4,
+    REFERENCES: 5,
+    WEAK_REFERENCE: 6,
+    WEAK_REFERENCES: 7,
+    ALIAS: 8,
+    ALIASES: 9,
+    COLVEC: 10,
+};
+const selvaTypeMap = new Uint8Array(32); // 1.2x faster than JS array
+selvaTypeMap[MICRO_BUFFER] = selvaFieldType.MICRO_BUFFER;
+selvaTypeMap[VECTOR] = selvaFieldType.MICRO_BUFFER;
+selvaTypeMap[BINARY] = selvaFieldType.STRING;
+selvaTypeMap[CARDINALITY] = selvaFieldType.STRING;
+selvaTypeMap[JSON] = selvaFieldType.STRING;
+selvaTypeMap[STRING] = selvaFieldType.STRING;
+selvaTypeMap[TEXT] = selvaFieldType.TEXT;
+selvaTypeMap[REFERENCE] = selvaFieldType.REFERENCE;
+selvaTypeMap[REFERENCES] = selvaFieldType.REFERENCES;
+selvaTypeMap[WEAK_REFERENCE] = selvaFieldType.WEAK_REFERENCE;
+selvaTypeMap[WEAK_REFERENCES] = selvaFieldType.WEAK_REFERENCES;
+selvaTypeMap[ALIAS] = selvaFieldType.ALIAS;
+selvaTypeMap[ALIASES] = selvaFieldType.ALIASES;
+selvaTypeMap[COLVEC] = selvaFieldType.COLVEC;
+const EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT = 0x01;
+const EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80;
+function blockCapacity(blockCapacity) {
+    const buf = new Uint8Array(Uint32Array.BYTES_PER_ELEMENT);
+    const view = new DataView(buf.buffer);
+    view.setUint32(0, blockCapacity, true);
+    return buf;
+}
+function sepPropCount(props) {
+    return props.filter((prop) => prop.separate).length;
+}
+function makeEdgeConstraintFlags(prop, inverseProp) {
+    return ((prop.dependent ? EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT : 0x00) |
+        (prop.typeIndex === REFERENCE && inverseProp && inverseProp.typeIndex === REFERENCES
+            ? EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP
+            : 0x00));
+}
+const propDefBuffer = (schema, prop, isEdge) => {
+    const type = prop.typeIndex;
+    const selvaType = selvaTypeMap[type];
+    if (prop.len && (type === MICRO_BUFFER || type === VECTOR)) {
+        const buf = new Uint8Array(3);
+        const view = new DataView(buf.buffer);
+        buf[0] = selvaType;
+        view.setUint16(1, prop.len, true);
+        return [...buf];
+    }
+    else if (prop.len && (type === COLVEC)) {
+        const buf = new Uint8Array(5);
+        const view = new DataView(buf.buffer);
+        buf[0] = selvaType;
+        view.setUint16(1, prop.len, true);
+        view.setUint16(3, 4, true); // TODO Other types than f32
+        return [...buf];
+    }
+    else if (type === REFERENCE || type === REFERENCES) {
+        const buf = new Uint8Array(9);
+        const view = new DataView(buf.buffer);
+        const dstType = schema[prop.inverseTypeName];
+        let eschema = [];
+        // @ts-ignore
+        buf[0] = selvaType + 2 * !!isEdge; // field type
+        buf[1] = makeEdgeConstraintFlags(prop, dstType.props[prop.inversePropName]); // flags
+        view.setUint16(2, dstType.id, true); // dst_node_type
+        view.setUint32(5, 0, true); // schema_len
+        if (!isEdge) {
+            prop.inverseTypeId = dstType.id;
+            prop.inversePropNumber = dstType.props[prop.inversePropName].prop;
+            buf[4] = prop.inversePropNumber;
+            if (prop.edges) {
+                const edgesS = Object.values(prop.edges);
+                if (edgesS.length) {
+                    const props = edgesS
+                        .filter((v) => v.separate === true)
+                        .sort((a, b) => (a.prop > b.prop ? 1 : -1));
+                    const p = [
+                        {
+                            ...EMPTY_MICRO_BUFFER,
+                            len: prop.edgeMainLen || 1, // allow zero here... else useless padding
+                            __isEdgeDef: true,
+                        },
+                        // or handle this here...
+                        ...props,
+                    ];
+                    eschema = p
+                        .map((prop) => propDefBuffer(schema, prop, true))
+                        .flat(1);
+                    eschema.unshift(0, 0, 0, 0, sepPropCount(p), 0);
+                    view.setUint32(5, eschema.length, true);
+                }
+            }
+        }
+        return [...buf, ...eschema];
+    }
+    else if (type === STRING ||
+        type === BINARY ||
+        type === CARDINALITY ||
+        type === JSON) {
+        return [selvaType, prop.len < 50 ? prop.len : 0];
+    }
+    {
+        return [selvaType];
+    }
+};
+// TODO rewrite
+export function schemaToSelvaBuffer(schema) {
+    return Object.values(schema).map((t, i) => {
+        const props = Object.values(t.props);
+        const rest = [];
+        const nrFields = 1 + sepPropCount(props);
+        let refFields = 0;
+        if (nrFields >= 250) {
+            throw new Error('Too many fields');
+        }
+        for (const f of props) {
+            if (f.separate) {
+                if (f.typeIndex === REFERENCE || f.typeIndex === REFERENCES) {
+                    refFields++;
+                }
+                rest.push(f);
+            }
+        }
+        rest.sort((a, b) => a.prop - b.prop);
+        return Uint8Array.from([
+            ...blockCapacity(t.blockCapacity),
+            nrFields,
+            1 + refFields,
+            ...propDefBuffer(schema, {
+                ...EMPTY_MICRO_BUFFER,
+                len: t.mainLen === 0 ? 1 : t.mainLen,
+            }),
+            ...rest.map((f) => propDefBuffer(schema, f)).flat(1),
+        ]).buffer;
+    });
+}
+//# sourceMappingURL=selvaBuffer.js.map

package/dist/def/timestamp.d.ts

@@ -0,0 +1,2 @@
+export declare const convertToTimestamp: (value: string | Date | number) => number;
+//# sourceMappingURL=timestamp.d.ts.map

package/dist/def/timestamp.js

@@ -0,0 +1,67 @@
+const timeToNumber = (ex) => {
+    if (ex === 's') {
+        return 1000;
+    }
+    if (ex === 'm') {
+        return 1000 * 60;
+    }
+    if (ex === 'h') {
+        return 1000 * 60 * 60;
+    }
+    if (ex === 'd') {
+        return 1000 * 60 * 60 * 24;
+    }
+    if (ex === 'y') {
+        return 31556952000;
+    }
+    return 1;
+};
+export const convertToTimestamp = (value) => {
+    if (value instanceof Date) {
+        return value.valueOf();
+    }
+    if (typeof value === 'string') {
+        if (value === 'now') {
+            return Date.now();
+        }
+        const y = value.replace(/([+-])/g, ' $1 ');
+        const arr = y.split(/ +/);
+        let newValue = 0;
+        let now;
+        let op = 1;
+        for (const seg of arr) {
+            if (seg === '-') {
+                op = -1;
+            }
+            else if (seg === '+') {
+                op = 1;
+            }
+            else {
+                var v = 0;
+                if (seg === 'now') {
+                    if (!now) {
+                        now = Date.now();
+                    }
+                    v = now;
+                }
+                else if (/[smhdy]$/.test(seg)) {
+                    const ex = seg[seg.length - 1];
+                    const number = parseInt(seg, 10);
+                    v = number * timeToNumber(ex);
+                }
+                else if (seg) {
+                    v = new Date(seg).valueOf();
+                }
+                if (op === -1) {
+                    newValue -= v;
+                }
+                else {
+                    newValue += v;
+                }
+            }
+        }
+        return newValue;
+    }
+    return value;
+};
+//# sourceMappingURL=timestamp.js.map

package/dist/def/typeDef.d.ts

@@ -0,0 +1,13 @@
+import { SchemaObject, StrictSchemaType, SchemaLocales } from '../index.js';
+import { SchemaTypeDef, SchemaTypesParsed } from './types.js';
+import { StrictSchema } from '../types.js';
+export declare const updateTypeDefs: (schema: StrictSchema) => {
+    schemaTypesParsed: {
+        [key: string]: SchemaTypeDef;
+    };
+    schemaTypesParsedById: {
+        [id: number]: SchemaTypeDef;
+    };
+};
+export declare const createSchemaTypeDef: (typeName: string, type: StrictSchemaType | SchemaObject, parsed: SchemaTypesParsed, locales: Partial<SchemaLocales>, result?: Partial<SchemaTypeDef>, path?: string[], top?: boolean) => SchemaTypeDef;
+//# sourceMappingURL=typeDef.d.ts.map

package/dist/def/typeDef.js

@@ -0,0 +1,231 @@
+import { isPropType, getPropType, } from '../index.js';
+import { setByPath } from '@saulx/utils';
+import { TYPE_INDEX_MAP, REFERENCES, REFERENCE, NUMBER, BLOCK_CAPACITY_MAX, BLOCK_CAPACITY_DEFAULT, BLOCK_CAPACITY_MIN, } from './types.js';
+import { DEFAULT_MAP } from './defaultMap.js';
+import { makeSeparateTextSort } from './makeSeparateTextSort.js';
+import { makeSeparateSort } from './makeSeparateSort.js';
+import { getPropLen, isSeparate, parseMinMaxStep } from './utils.js';
+import { addEdges } from './addEdges.js';
+import { createEmptyDef } from './createEmptyDef.js';
+import { fillEmptyMain, isZeroes } from './fillEmptyMain.js';
+import { defaultValidation, VALIDATION_MAP } from './validation.js';
+export const updateTypeDefs = (schema) => {
+    const schemaTypesParsed = {};
+    const schemaTypesParsedById = {};
+    for (const typeName in schemaTypesParsed) {
+        if (typeName in schema.types) {
+            continue;
+        }
+        const id = schemaTypesParsed[typeName].id;
+        delete schemaTypesParsed[typeName];
+        delete schemaTypesParsedById[id];
+    }
+    for (const typeName in schema.types) {
+        const type = schema.types[typeName];
+        if (!type.id) {
+            throw new Error('NEED ID ON TYPE');
+        }
+        const def = createSchemaTypeDef(typeName, type, schemaTypesParsed, schema.locales ?? {
+            en: {},
+        });
+        schemaTypesParsed[typeName] = def;
+        schemaTypesParsedById[type.id] = def;
+    }
+    return { schemaTypesParsed, schemaTypesParsedById };
+};
+export const createSchemaTypeDef = (typeName, type, parsed, locales, result = createEmptyDef(typeName, type, locales), path = [], top = true) => {
+    if (top) {
+        if (result.id == 0) {
+            if ('id' in type) {
+                result.id = type.id;
+            }
+            else {
+                throw new Error(`Invalid schema type id ${result.type}`);
+            }
+        }
+        if (result.blockCapacity == 0) {
+            if ('blockCapacity' in type) {
+                if (typeof type.blockCapacity !== 'number' || type.blockCapacity < BLOCK_CAPACITY_MIN || type.blockCapacity > BLOCK_CAPACITY_MAX) {
+                    throw new Error('Invalid blockCapacity');
+                }
+                result.blockCapacity = type.blockCapacity;
+            }
+            else {
+                result.blockCapacity = typeName === '_root' ? BLOCK_CAPACITY_MAX : BLOCK_CAPACITY_DEFAULT;
+            }
+        }
+        if (result.insertOnly == false && 'insertOnly' in type) {
+            result.insertOnly = !!type.insertOnly;
+        }
+        if (result.partial == false && 'partial' in type) {
+            result.partial = !!type.partial;
+        }
+    }
+    result.locales = locales;
+    result.localeSize = Object.keys(locales).length;
+    result.idUint8[0] = result.id & 255;
+    result.idUint8[1] = result.id >> 8;
+    const target = type.props;
+    for (const key in target) {
+        // Create prop def
+        const schemaProp = target[key];
+        const propPath = [...path, key];
+        const propType = getPropType(schemaProp);
+        if (propType === 'object') {
+            createSchemaTypeDef(typeName, schemaProp, parsed, locales, result, propPath, false);
+        }
+        else {
+            const len = getPropLen(schemaProp);
+            if (isPropType('string', schemaProp) ||
+                isPropType('alias', schemaProp) ||
+                isPropType('cardinality', schemaProp)) {
+                if (typeof schemaProp === 'object') {
+                    if (!(schemaProp.maxBytes < 61) ||
+                        !('max' in schemaProp && schemaProp.max < 31)) {
+                        result.separateSortProps++;
+                    }
+                }
+                else {
+                    result.separateSortProps++;
+                }
+            }
+            else if (isPropType('text', schemaProp)) {
+                result.separateSortText++;
+            }
+            else if (isPropType('colvec', schemaProp)) {
+                if (!result.insertOnly) {
+                    throw new Error('colvec requires insertOnly');
+                }
+            }
+            const isseparate = isSeparate(schemaProp, len);
+            const typeIndex = TYPE_INDEX_MAP[propType];
+            const prop = {
+                typeIndex,
+                __isPropDef: true,
+                separate: isseparate,
+                path: propPath,
+                start: 0,
+                validation: schemaProp.validation ??
+                    VALIDATION_MAP[typeIndex] ??
+                    defaultValidation,
+                len,
+                default: schemaProp.default ?? DEFAULT_MAP[typeIndex],
+                prop: isseparate ? ++result.cnt : 0,
+            };
+            if (schemaProp.max !== undefined) {
+                prop.max = parseMinMaxStep(schemaProp.max);
+            }
+            if (schemaProp.min !== undefined) {
+                prop.min = parseMinMaxStep(schemaProp.min);
+            }
+            if (schemaProp.step !== undefined) {
+                prop.step = parseMinMaxStep(schemaProp.step);
+            }
+            if (prop.typeIndex !== NUMBER && prop.step === undefined) {
+                prop.step = 1;
+            }
+            if (isPropType('enum', schemaProp)) {
+                prop.enum = Array.isArray(schemaProp) ? schemaProp : schemaProp.enum;
+                prop.reverseEnum = {};
+                for (let i = 0; i < prop.enum.length; i++) {
+                    prop.reverseEnum[prop.enum[i]] = i;
+                }
+            }
+            else if (isPropType('references', schemaProp)) {
+                if (result.partial) {
+                    throw new Error('references is not supported with partial');
+                }
+                prop.inversePropName = schemaProp.items.prop;
+                prop.inverseTypeName = schemaProp.items.ref;
+                prop.dependent = schemaProp.items.dependent;
+                addEdges(prop, schemaProp.items);
+            }
+            else if (isPropType('reference', schemaProp)) {
+                if (result.partial) {
+                    throw new Error('reference is not supported with partial');
+                }
+                prop.inversePropName = schemaProp.prop;
+                prop.inverseTypeName = schemaProp.ref;
+                prop.dependent = schemaProp.dependent;
+                addEdges(prop, schemaProp);
+            }
+            else if (typeof schemaProp === 'object') {
+                if (isPropType('string', schemaProp) ||
+                    isPropType('text', schemaProp)) {
+                    prop.compression =
+                        'compression' in schemaProp && schemaProp.compression === 'none'
+                            ? 0
+                            : 1;
+                }
+                else if (isPropType('timestamp', schemaProp) && 'on' in schemaProp) {
+                    if (schemaProp.on[0] === 'c') {
+                        result.createTs ??= [];
+                        result.createTs.push(prop);
+                    }
+                    else if (schemaProp.on[0] === 'u') {
+                        result.createTs ??= [];
+                        result.createTs.push(prop);
+                        result.updateTs ??= [];
+                        result.updateTs.push(prop);
+                    }
+                }
+            }
+            result.props[propPath.join('.')] = prop;
+            if (isseparate) {
+                result.separate.push(prop);
+            }
+        }
+    }
+    if (top) {
+        // Put top level together
+        const vals = Object.values(result.props);
+        vals.sort((a, b) => {
+            if (b.separate &&
+                (a.typeIndex === REFERENCES || a.typeIndex === REFERENCE)) {
+                return -1;
+            }
+            return a.prop - b.prop;
+        });
+        let lastProp = 0;
+        for (const p of vals) {
+            if (p.separate) {
+                p.prop = ++lastProp;
+            }
+        }
+        let len = 2;
+        for (const f of vals) {
+            if (f.separate) {
+                len += 2;
+                setByPath(result.tree, f.path, f);
+            }
+            else {
+                if (!result.mainLen) {
+                    len += 2;
+                }
+                len += 1;
+                f.start = result.mainLen;
+                result.mainLen += f.len;
+                setByPath(result.tree, f.path, f);
+            }
+        }
+        result.mainEmpty = fillEmptyMain(vals, result.mainLen);
+        result.mainEmptyAllZeroes = isZeroes(result.mainEmpty);
+        if (result.separateSortText > 0) {
+            makeSeparateTextSort(result);
+        }
+        if (result.separateSortProps > 0) {
+            makeSeparateSort(result);
+        }
+        for (const p in result.props) {
+            const x = result.props[p];
+            if (!x.separate) {
+                result.main[x.start] = x;
+            }
+            else {
+                result.reverseProps[x.prop] = x;
+            }
+        }
+    }
+    return result;
+};
+//# sourceMappingURL=typeDef.js.map

package/dist/def/types.d.ts

@@ -0,0 +1,186 @@
+import type { LangCode, SchemaLocales } from '../index.js';
+import { Validation } from './validation.js';
+export declare const NULL = 0;
+export declare const TIMESTAMP = 1;
+export declare const NUMBER = 4;
+export declare const CARDINALITY = 5;
+export declare const INT8 = 20;
+export declare const UINT8 = 6;
+export declare const INT16 = 21;
+export declare const UINT16 = 22;
+export declare const INT32 = 23;
+export declare const UINT32 = 7;
+export declare const BOOLEAN = 9;
+export declare const ENUM = 10;
+export declare const STRING = 11;
+export declare const TEXT = 12;
+export declare const REFERENCE = 13;
+export declare const REFERENCES = 14;
+export declare const WEAK_REFERENCE = 15;
+export declare const WEAK_REFERENCES = 16;
+export declare const MICRO_BUFFER = 17;
+export declare const ALIAS = 18;
+export declare const ALIASES = 19;
+export declare const BINARY = 25;
+export declare const ID = 26;
+export declare const VECTOR = 27;
+export declare const JSON = 28;
+export declare const OBJECT = 29;
+export declare const COLVEC = 30;
+export declare const TYPE_INDEX_MAP: {
+    alias: number;
+    aliases: number;
+    microbuffer: number;
+    references: number;
+    reference: number;
+    timestamp: number;
+    boolean: number;
+    number: number;
+    string: number;
+    text: number;
+    uint16: number;
+    uint32: number;
+    int16: number;
+    int32: number;
+    uint8: number;
+    enum: number;
+    int8: number;
+    id: number;
+    binary: number;
+    vector: number;
+    cardinality: number;
+    json: number;
+    object: number;
+    colvec: number;
+};
+export declare const enum numberTypes {
+    number = 4,
+    uint16 = 22,
+    uint32 = 7,
+    int16 = 21,
+    int32 = 23,
+    uint8 = 6,
+    int8 = 20,
+    cardinality = 5
+}
+export declare function isNumberType(type: TypeIndex): boolean;
+export type InternalSchemaProp = keyof typeof TYPE_INDEX_MAP;
+export type TypeIndex = (typeof TYPE_INDEX_MAP)[InternalSchemaProp];
+export type PropDef = {
+    __isPropDef: true;
+    prop: number;
+    typeIndex: TypeIndex;
+    separate: boolean;
+    path: string[];
+    start: number;
+    len: number;
+    inverseTypeName?: string;
+    inversePropName?: string;
+    compression?: 0 | 1;
+    inverseTypeId?: number;
+    inversePropNumber?: number;
+    enum?: any[];
+    dependent?: boolean;
+    validation: Validation;
+    default: any;
+    edgeMainLen?: 0;
+    reverseEnum?: {
+        [key: string]: number;
+    };
+    edgesSeperateCnt?: number;
+    edges?: {
+        [key: string]: PropDefEdge;
+    };
+    reverseSeperateEdges?: {
+        [prop: string]: PropDefEdge;
+    };
+    reverseMainEdges?: {
+        [start: string]: PropDefEdge;
+    };
+    edgeMainEmpty?: Uint8Array;
+    __isEdge?: boolean;
+    max?: any;
+    min?: any;
+    step?: any;
+};
+export type PropDefEdge = Partial<PropDef> & {
+    __isPropDef: true;
+    typeIndex: TypeIndex;
+    len: number;
+    prop: number;
+    name: string;
+    edgesTotalLen?: number;
+    __isEdge: true;
+};
+export type PropDefAggregate = Partial<PropDef> & {
+    __isPropDef: true;
+    typeIndex: TypeIndex;
+    len: number;
+    prop: number;
+    name: string;
+};
+export type SchemaPropTree = {
+    [key: string]: SchemaPropTree | PropDef;
+};
+export type SchemaSortUndefinedHandler = {
+    size: number;
+    buffer: Uint8Array;
+    bufferTmp: Uint8Array;
+    props: PropDef[];
+};
+export declare const BLOCK_CAPACITY_MIN = 1025;
+export declare const BLOCK_CAPACITY_MAX = 2147483647;
+export declare const BLOCK_CAPACITY_DEFAULT = 100000;
+export type SchemaTypeDef = {
+    cnt: number;
+    checksum: number;
+    type: string;
+    lastId: number;
+    blockCapacity: number;
+    mainLen: number;
+    insertOnly: boolean;
+    partial: boolean;
+    buf: Uint8Array;
+    propNames: Uint8Array;
+    props: {
+        [path: string]: PropDef;
+    };
+    reverseProps: {
+        [field: string]: PropDef;
+    };
+    id: number;
+    idUint8: Uint8Array;
+    separate: PropDef[];
+    main: {
+        [start: string]: PropDef;
+    };
+    mainEmpty: Uint8Array;
+    mainEmptyAllZeroes: boolean;
+    tree: SchemaPropTree;
+    separateSortProps: number;
+    separateSortText: number;
+    hasSeperateSort: boolean;
+    seperateSort: SchemaSortUndefinedHandler;
+    hasSeperateTextSort: boolean;
+    seperateTextSort: SchemaSortUndefinedHandler & {
+        noUndefined: Uint8Array;
+        localeStringToIndex: Map<string, Uint8Array>;
+        localeToIndex: Map<LangCode, number>;
+    };
+    createTs?: PropDef[];
+    updateTs?: PropDef[];
+    locales: Partial<SchemaLocales>;
+    localeSize: number;
+};
+export declare const SIZE_MAP: Record<InternalSchemaProp, number>;
+export declare const REVERSE_SIZE_MAP: Record<TypeIndex, number>;
+export declare const REVERSE_TYPE_INDEX_MAP: Record<TypeIndex, InternalSchemaProp>;
+export declare const ID_FIELD_DEF: PropDef;
+export declare const EMPTY_MICRO_BUFFER: PropDef;
+export declare const getPropTypeName: (propType: TypeIndex) => InternalSchemaProp;
+export declare const isPropDef: (prop: any) => prop is PropDef;
+export type SchemaTypesParsed = {
+    [key: string]: SchemaTypeDef;
+};
+export type SchemaTypesParsedById = Record<number, SchemaTypeDef>;
+//# sourceMappingURL=types.d.ts.map