@based/db 0.0.1

package/dist/src/server/schema/selvaBuffer.js

@@ -0,0 +1,102 @@
+import { EMPTY_MICRO_BUFFER, MICRO_BUFFER, REFERENCE, REFERENCES, STRING, BINARY, CREATED, UPDATED, TIMESTAMP, NULL, NUMBER, HLL, INT8, UINT8, INT16, UINT16, INT32, UINT32, INT64, BOOLEAN, ENUM, TEXT, WEAK_REFERENCE, WEAK_REFERENCES, ALIAS, ALIASES, } from './types.js';
+const selvaTypeMap = [];
+selvaTypeMap[NULL] = 0;
+selvaTypeMap[TIMESTAMP] = 1;
+selvaTypeMap[CREATED] = 1;
+selvaTypeMap[UPDATED] = 1;
+selvaTypeMap[NUMBER] = 4;
+selvaTypeMap[HLL] = 11;
+selvaTypeMap[INT8] = 20;
+selvaTypeMap[UINT8] = 6;
+selvaTypeMap[INT16] = 21;
+selvaTypeMap[UINT16] = 22;
+selvaTypeMap[INT32] = 23;
+selvaTypeMap[UINT32] = 7;
+selvaTypeMap[INT64] = 24;
+selvaTypeMap[BOOLEAN] = 9;
+selvaTypeMap[ENUM] = 10;
+selvaTypeMap[STRING] = 11;
+selvaTypeMap[TEXT] = 12;
+selvaTypeMap[REFERENCE] = 13;
+selvaTypeMap[REFERENCES] = 14;
+selvaTypeMap[WEAK_REFERENCE] = 15;
+selvaTypeMap[WEAK_REFERENCES] = 16;
+selvaTypeMap[MICRO_BUFFER] = 17;
+selvaTypeMap[ALIAS] = 18;
+selvaTypeMap[ALIASES] = 19;
+selvaTypeMap[BINARY] = 11;
+function sepPropCount(props) {
+    return props.filter((prop) => prop.separate).length;
+}
+const propDefBuffer = (schema, prop, isEdge) => {
+    const type = prop.typeIndex;
+    const selvaType = selvaTypeMap[type];
+    if (prop.len && type === MICRO_BUFFER) {
+        const buf = Buffer.allocUnsafe(3);
+        buf[0] = selvaType;
+        buf.writeUint16LE(prop.len, 1);
+        return [...buf.values()];
+    }
+    else if (type === REFERENCE || type === REFERENCES) {
+        const buf = Buffer.allocUnsafe(8);
+        const dstType = schema[prop.inverseTypeName];
+        let eschema = [];
+        // @ts-ignore
+        buf[0] = selvaType + 2 * !!isEdge;
+        buf.writeUInt16LE(dstType.id, 1);
+        buf.writeUint32LE(0, 4);
+        if (!isEdge) {
+            prop.inverseTypeId = dstType.id;
+            prop.inversePropNumber = dstType.props[prop.inversePropName].prop;
+            buf[3] = prop.inversePropNumber;
+            if (prop.edges) {
+                const props = Object.values(prop.edges);
+                eschema = props
+                    .map((prop) => propDefBuffer(schema, prop, true))
+                    .flat(1);
+                eschema.unshift(...[0, 0, 0, 0], sepPropCount(props), 0);
+                buf.writeUint32LE(eschema.length, 4);
+            }
+        }
+        return [...buf.values(), ...eschema];
+    }
+    else if (type === STRING || type == BINARY) {
+        return [selvaType, prop.len < 50 ? prop.len : 0];
+    }
+    else {
+        return [selvaType];
+    }
+};
+function makeBlockCapacityBuffer(blockCapacity) {
+    const buf = Buffer.allocUnsafe(4);
+    buf.writeInt32LE(blockCapacity);
+    return buf;
+}
+// todo rewrite
+export function schemaToSelvaBuffer(schema) {
+    return Object.values(schema).map((t, i) => {
+        const props = Object.values(t.props);
+        const rest = [];
+        let refFields = 0;
+        for (const f of props) {
+            if (f.separate) {
+                if (f.typeIndex === 13 || f.typeIndex === 14) {
+                    refFields++;
+                }
+                rest.push(f);
+            }
+        }
+        rest.sort((a, b) => a.prop - b.prop);
+        return Buffer.from([
+            ...makeBlockCapacityBuffer(t.blockCapacity).values(),
+            1 + sepPropCount(props),
+            1 + refFields,
+            ...propDefBuffer(schema, {
+                ...EMPTY_MICRO_BUFFER,
+                len: t.mainLen === 0 ? 1 : t.mainLen,
+            }),
+            ...rest.map((f) => propDefBuffer(schema, f)).flat(1),
+        ]);
+    });
+}
+//# sourceMappingURL=selvaBuffer.js.map

package/dist/src/server/schema/typeDef.d.ts

@@ -0,0 +1,6 @@
+import { SchemaObject, StrictSchemaType } from '@based/schema';
+import { SchemaTypeDef } from './types.js';
+import { DbClient } from '../../client/index.js';
+import { DbServer } from '../index.js';
+export declare const updateTypeDefs: (db: DbClient | DbServer) => void;
+export declare const createSchemaTypeDef: (typeName: string, type: StrictSchemaType | SchemaObject, parsed: DbClient["schemaTypesParsed"], result?: Partial<SchemaTypeDef>, path?: string[], top?: boolean) => SchemaTypeDef;

package/dist/src/server/schema/typeDef.js

@@ -0,0 +1,315 @@
+import { isPropType, getPropType, } from '@based/schema';
+import { setByPath } from '@saulx/utils';
+import { hashObjectIgnoreKeyOrder } from '@saulx/hash';
+import { SIZE_MAP, TYPE_INDEX_MAP, } from './types.js';
+import { genId } from './utils.js';
+import { DEFAULT_BLOCK_CAPACITY } from '../start.js';
+const addEdges = (prop, refProp) => {
+    let edgesCnt = 0;
+    for (const key in refProp) {
+        if (key[0] === '$') {
+            if (!prop.edges) {
+                prop.edges = {};
+                prop.reverseEdges = {};
+                prop.edgesTotalLen = 0;
+            }
+            edgesCnt++;
+            const edgeType = getPropType(refProp[key]);
+            const edge = {
+                __isPropDef: true,
+                __isEdge: true,
+                prop: edgesCnt,
+                name: key,
+                typeIndex: TYPE_INDEX_MAP[edgeType],
+                len: SIZE_MAP[edgeType],
+                separate: true,
+                path: [...prop.path, key],
+            };
+            if (edge.len == 0) {
+                prop.edgesTotalLen = 0;
+            }
+            else {
+                // [field] [size] [data]
+                prop.edgesTotalLen += 1 + 2 + edge.len; // field len
+            }
+            if (edge.typeIndex === 10) {
+                edge.enum = Array.isArray(refProp[key])
+                    ? refProp[key]
+                    : refProp[key].enum;
+                edge.reverseEnum = {};
+                for (let i = 0; i < edge.enum.length; i++) {
+                    edge.reverseEnum[edge.enum[i]] = i;
+                }
+            }
+            else if (edge.typeIndex === 14) {
+                edge.inverseTypeName = refProp[key].items.ref;
+            }
+            else if (edge.typeIndex === 13) {
+                edge.inverseTypeName = refProp[key].ref;
+            }
+            prop.edges[key] = edge;
+            prop.reverseEdges[edge.prop] = edge;
+        }
+    }
+};
+export const updateTypeDefs = (db) => {
+    for (const field in db.schemaTypesParsed) {
+        if (field in db.schema.types) {
+            continue;
+        }
+        const id = db.schemaTypesParsed[field].id;
+        delete db.schemaTypesParsed[field];
+        delete db.schemaTypesParsedById[id];
+    }
+    for (const field in db.schema.types) {
+        const type = db.schema.types[field];
+        if (db.schemaTypesParsed[field] &&
+            db.schemaTypesParsed[field].checksum === hashObjectIgnoreKeyOrder(type) // bit weird..
+        ) {
+            continue;
+        }
+        else {
+            if (!type.id) {
+                type.id = genId(db);
+            }
+            const def = createSchemaTypeDef(field, type, db.schemaTypesParsed);
+            def.blockCapacity =
+                field === '_root' ? 2147483647 : DEFAULT_BLOCK_CAPACITY; // TODO this should come from somewhere else
+            db.schemaTypesParsed[field] = def;
+            db.schemaTypesParsedById[type.id] = def;
+        }
+    }
+};
+export const createSchemaTypeDef = (typeName, type, parsed, result = {
+    cnt: 0,
+    checksum: hashObjectIgnoreKeyOrder(type),
+    type: typeName,
+    props: {},
+    reverseProps: {},
+    idUint8: new Uint8Array([0, 0]),
+    id: 0,
+    mainLen: 0,
+    separate: [],
+    tree: {},
+    total: 0,
+    lastId: 0,
+    stringPropsSize: 0,
+    stringPropsLoop: [],
+    main: {},
+}, path = [], top = true) => {
+    if (result.id == 0 && top) {
+        if ('id' in type) {
+            result.id = type.id;
+        }
+        else {
+            throw new Error(`Invalid schema type id ${result.type}`);
+        }
+    }
+    result.idUint8[0] = result.id & 255;
+    result.idUint8[1] = result.id >> 8;
+    const encoder = new TextEncoder();
+    const target = type.props;
+    let stringFields = 0;
+    for (const key in target) {
+        const schemaProp = target[key];
+        const propPath = [...path, key];
+        const propType = getPropType(schemaProp);
+        if (propType === 'object') {
+            createSchemaTypeDef(typeName, schemaProp, parsed, result, propPath, false);
+        }
+        else {
+            let len = SIZE_MAP[propType];
+            if (isPropType('string', schemaProp) ||
+                isPropType('alias', schemaProp) ||
+                isPropType('binary', schemaProp)) {
+                if (typeof schemaProp === 'object') {
+                    if (schemaProp.maxBytes < 61) {
+                        len = schemaProp.maxBytes + 1;
+                    }
+                    else if ('max' in schemaProp && schemaProp.max < 31) {
+                        len = schemaProp.max * 2 + 1;
+                    }
+                    else {
+                        stringFields++;
+                    }
+                }
+                else {
+                    stringFields++;
+                }
+            }
+            else if (isPropType('text', schemaProp)) {
+                stringFields++;
+            }
+            const isseparate = len === 0;
+            if (isseparate) {
+                result.cnt++;
+            }
+            const prop = {
+                typeIndex: TYPE_INDEX_MAP[propType],
+                __isPropDef: true,
+                separate: isseparate,
+                path: propPath,
+                start: 0,
+                len,
+                prop: isseparate ? result.cnt : 0,
+            };
+            if (isPropType('enum', schemaProp)) {
+                prop.enum = Array.isArray(schemaProp) ? schemaProp : schemaProp.enum;
+                prop.reverseEnum = {};
+                for (let i = 0; i < prop.enum.length; i++) {
+                    prop.reverseEnum[prop.enum[i]] = i;
+                }
+            }
+            else if (isPropType('references', schemaProp)) {
+                prop.inversePropName = schemaProp.items.prop;
+                prop.inverseTypeName = schemaProp.items.ref;
+                addEdges(prop, schemaProp.items);
+            }
+            else if (isPropType('reference', schemaProp)) {
+                prop.inversePropName = schemaProp.prop;
+                prop.inverseTypeName = schemaProp.ref;
+                addEdges(prop, schemaProp);
+            }
+            else if (typeof schemaProp === 'object') {
+                if (isPropType('string', schemaProp) ||
+                    isPropType('text', schemaProp)) {
+                    prop.compression =
+                        'compression' in schemaProp && schemaProp.compression === 'none'
+                            ? 0
+                            : 1;
+                }
+                else if (isPropType('timestamp', schemaProp) && 'on' in schemaProp) {
+                    if (schemaProp.on[0] === 'c') {
+                        result.createTs ??= [];
+                        result.createTs.push(prop);
+                    }
+                    else if (schemaProp.on[0] === 'u') {
+                        result.createTs ??= [];
+                        result.createTs.push(prop);
+                        result.updateTs ??= [];
+                        result.updateTs.push(prop);
+                    }
+                }
+            }
+            result.props[propPath.join('.')] = prop;
+            if (isseparate) {
+                result.separate.push(prop);
+            }
+        }
+    }
+    if (top) {
+        const vals = Object.values(result.props);
+        vals.sort((a, b) => {
+            if (b.separate && (a.typeIndex === 14 || a.typeIndex === 13)) {
+                return -1;
+            }
+            return a.prop - b.prop;
+        });
+        let lastProp = 0;
+        for (const p of vals) {
+            if (p.separate) {
+                lastProp++;
+                p.prop = lastProp;
+            }
+        }
+        let len = 2;
+        for (const f of vals) {
+            if (f.separate) {
+                len += 2;
+                setByPath(result.tree, f.path, f);
+            }
+            else {
+                if (!result.mainLen) {
+                    len += 2;
+                }
+                len += 1;
+                f.start = result.mainLen;
+                result.mainLen += f.len;
+                setByPath(result.tree, f.path, f);
+            }
+        }
+        const mainFields = [];
+        const restFields = [];
+        for (const f of vals) {
+            if (f.separate) {
+                restFields.push(f);
+            }
+            else {
+                mainFields.push(f);
+            }
+        }
+        result.buf = Buffer.allocUnsafe(len);
+        result.buf[0] = result.idUint8[0];
+        result.buf[1] = result.idUint8[1];
+        const fieldNames = [];
+        const tNameBuf = encoder.encode(typeName);
+        fieldNames.push(tNameBuf);
+        let fieldNameLen = tNameBuf.byteLength + 1;
+        let i = 2;
+        if (result.mainLen) {
+            result.buf[i] = 0;
+            for (const f of vals) {
+                if (!f.separate) {
+                    i++;
+                    result.buf[i] = f.typeIndex;
+                    const name = encoder.encode(f.path.join('.'));
+                    fieldNames.push(name);
+                    fieldNameLen += name.byteLength + 1;
+                }
+            }
+            i++;
+            result.buf[i] = 0;
+        }
+        for (const f of vals) {
+            if (f.separate) {
+                i++;
+                result.buf[i] = f.prop;
+                i++;
+                result.buf[i] = f.typeIndex;
+                const name = encoder.encode(f.path.join('.'));
+                fieldNames.push(name);
+                fieldNameLen += name.byteLength + 1;
+            }
+        }
+        result.propNames = Buffer.allocUnsafe(fieldNameLen);
+        let lastWritten = 0;
+        for (const f of fieldNames) {
+            result.propNames[lastWritten] = f.byteLength;
+            result.propNames.set(f, lastWritten + 1);
+            lastWritten += f.byteLength + 1;
+        }
+        // result.responseCtx = new BasedNode(result as SchemaTypeDef, parsed)
+        if (stringFields > 0) {
+            result.hasStringProp = true;
+            let max = 0;
+            for (const f of result.separate) {
+                if (f.typeIndex === 11) {
+                    if (f.prop > max) {
+                        max = f.prop;
+                    }
+                }
+            }
+            result.stringProps = Buffer.allocUnsafe(max + 1);
+            for (const f of result.separate) {
+                if (f.typeIndex === 11) {
+                    result.stringProps[f.prop] = 1;
+                    result.stringPropsLoop.push(f);
+                    result.stringPropsSize++;
+                }
+            }
+            result.stringPropsCurrent = Buffer.allocUnsafe(max + 1);
+            result.stringProps.copy(result.stringPropsCurrent);
+        }
+        for (const p in result.props) {
+            const x = result.props[p];
+            if (!x.separate) {
+                result.main[x.start] = x;
+            }
+            else {
+                result.reverseProps[x.prop] = x;
+            }
+        }
+    }
+    return result;
+};
+//# sourceMappingURL=typeDef.js.map

package/dist/src/server/schema/types.d.ts

@@ -0,0 +1,127 @@
+export declare const NULL = 0;
+export declare const TIMESTAMP = 1;
+export declare const CREATED = 2;
+export declare const UPDATED = 3;
+export declare const NUMBER = 4;
+export declare const HLL = 5;
+export declare const INT8 = 20;
+export declare const UINT8 = 6;
+export declare const INT16 = 21;
+export declare const UINT16 = 22;
+export declare const INT32 = 23;
+export declare const UINT32 = 7;
+export declare const INT64 = 24;
+export declare const BOOLEAN = 9;
+export declare const ENUM = 10;
+export declare const STRING = 11;
+export declare const TEXT = 12;
+export declare const REFERENCE = 13;
+export declare const REFERENCES = 14;
+export declare const WEAK_REFERENCE = 15;
+export declare const WEAK_REFERENCES = 16;
+export declare const MICRO_BUFFER = 17;
+export declare const ALIAS = 18;
+export declare const ALIASES = 19;
+export declare const BINARY = 25;
+export declare const TYPE_INDEX_MAP: {
+    alias: number;
+    aliases: number;
+    microbuffer: number;
+    references: number;
+    reference: number;
+    timestamp: number;
+    boolean: number;
+    created: number;
+    updated: number;
+    number: number;
+    string: number;
+    text: number;
+    uint16: number;
+    uint32: number;
+    int16: number;
+    int32: number;
+    uint8: number;
+    enum: number;
+    int8: number;
+    id: number;
+    binary: number;
+    hll: number;
+};
+export type InternalSchemaProp = keyof typeof TYPE_INDEX_MAP;
+export type TypeIndex = (typeof TYPE_INDEX_MAP)[InternalSchemaProp];
+export type PropDef = {
+    __isPropDef: true;
+    prop: number;
+    typeIndex: TypeIndex;
+    separate: boolean;
+    path: string[];
+    start: number;
+    len: number;
+    inverseTypeName?: string;
+    inversePropName?: string;
+    compression?: 0 | 1;
+    inverseTypeId?: number;
+    inversePropNumber?: number;
+    enum?: any[];
+    reverseEnum?: {
+        [key: string]: number;
+    };
+    edgesTotalLen?: number;
+    edges?: {
+        [key: string]: PropDefEdge;
+    };
+    reverseEdges?: {
+        [prop: string]: PropDefEdge;
+    };
+    __isEdge?: boolean;
+};
+export type PropDefEdge = Partial<PropDef> & {
+    __isPropDef: true;
+    typeIndex: TypeIndex;
+    len: number;
+    prop: number;
+    name: string;
+    edgesTotalLen?: number;
+    __isEdge: true;
+};
+export type SchemaPropTree = {
+    [key: string]: SchemaPropTree | PropDef;
+};
+export type SchemaTypeDef = {
+    cnt: number;
+    checksum: number;
+    total: number;
+    type: string;
+    lastId: number;
+    blockCapacity: number;
+    mainLen: number;
+    buf: Buffer;
+    propNames: Buffer;
+    props: {
+        [path: string]: PropDef;
+    };
+    reverseProps: {
+        [field: string]: PropDef;
+    };
+    id: number;
+    idUint8: Uint8Array;
+    separate: PropDef[];
+    main: {
+        [start: string]: PropDef;
+    };
+    tree: SchemaPropTree;
+    hasStringProp: boolean;
+    stringPropsSize: number;
+    stringProps: Buffer;
+    stringPropsCurrent: Buffer;
+    stringPropsLoop: PropDef[];
+    createTs?: PropDef[];
+    updateTs?: PropDef[];
+};
+export declare const SIZE_MAP: Record<InternalSchemaProp, number>;
+export declare let REVERSE_SIZE_MAP: Record<TypeIndex, number>;
+export declare const REVERSE_TYPE_INDEX_MAP: Record<TypeIndex, InternalSchemaProp>;
+export declare const ID_FIELD_DEF: PropDef;
+export declare const EMPTY_MICRO_BUFFER: PropDef;
+export declare const getPropTypeName: (propType: TypeIndex) => InternalSchemaProp;
+export declare const isPropDef: (prop: any) => prop is PropDef;

package/dist/src/server/schema/types.js

@@ -0,0 +1,113 @@
+export const NULL = 0;
+export const TIMESTAMP = 1;
+export const CREATED = 2;
+export const UPDATED = 3;
+export const NUMBER = 4;
+export const HLL = 5;
+export const INT8 = 20;
+export const UINT8 = 6;
+export const INT16 = 21;
+export const UINT16 = 22;
+export const INT32 = 23;
+export const UINT32 = 7;
+export const INT64 = 24;
+export const BOOLEAN = 9;
+export const ENUM = 10;
+export const STRING = 11;
+export const TEXT = 12;
+export const REFERENCE = 13;
+export const REFERENCES = 14;
+export const WEAK_REFERENCE = 15;
+export const WEAK_REFERENCES = 16;
+export const MICRO_BUFFER = 17;
+export const ALIAS = 18;
+export const ALIASES = 19;
+export const BINARY = 25;
+export const TYPE_INDEX_MAP = {
+    alias: ALIAS,
+    aliases: ALIASES,
+    microbuffer: MICRO_BUFFER,
+    references: REFERENCES,
+    reference: REFERENCE,
+    timestamp: TIMESTAMP,
+    boolean: BOOLEAN,
+    created: CREATED,
+    updated: UPDATED,
+    number: NUMBER,
+    string: STRING,
+    text: TEXT,
+    uint16: UINT16,
+    uint32: UINT32,
+    int16: INT16,
+    int32: INT32,
+    uint8: UINT8,
+    enum: ENUM,
+    int8: INT8,
+    id: NULL,
+    binary: BINARY,
+    hll: HLL,
+};
+export const SIZE_MAP = {
+    timestamp: 8, // 64bit
+    created: 8,
+    updated: 8,
+    // double-precision 64-bit binary format IEEE 754 value
+    number: 8, // 64bit
+    int8: 1,
+    uint8: 1,
+    int16: 2,
+    uint16: 2,
+    int32: 4,
+    uint32: 4,
+    boolean: 1,
+    reference: 0, // separate
+    enum: 1, // enum
+    string: 0, // separate
+    text: 0, // separate
+    hll: 0, // separate
+    references: 0, // separate
+    microbuffer: 0, // separate
+    alias: 0,
+    aliases: 0,
+    id: 4,
+    binary: 0,
+};
+const reverseMap = {};
+for (const k in TYPE_INDEX_MAP) {
+    reverseMap[TYPE_INDEX_MAP[k]] = k;
+}
+export let REVERSE_SIZE_MAP;
+// @ts-ignore
+REVERSE_SIZE_MAP = {};
+for (const k in SIZE_MAP) {
+    REVERSE_SIZE_MAP[TYPE_INDEX_MAP[k]] = SIZE_MAP[k];
+}
+export const REVERSE_TYPE_INDEX_MAP = reverseMap;
+export const ID_FIELD_DEF = {
+    typeIndex: TYPE_INDEX_MAP['id'],
+    separate: true,
+    path: ['id'],
+    start: 0,
+    prop: 255,
+    len: 4,
+    __isPropDef: true,
+};
+export const EMPTY_MICRO_BUFFER = {
+    typeIndex: TYPE_INDEX_MAP['microbuffer'],
+    separate: true,
+    path: [''],
+    start: 0,
+    prop: 0,
+    len: 1,
+    __isPropDef: true,
+};
+export const getPropTypeName = (propType) => {
+    return REVERSE_TYPE_INDEX_MAP[propType];
+};
+export const isPropDef = (prop) => {
+    if ('__isPropDef' in prop && prop.__isPropDef === true) {
+        return true;
+    }
+    return false;
+};
+//# sourceMappingURL=types.js.map

package/dist/src/server/schema/utils.d.ts

@@ -0,0 +1,6 @@
+import { DbClient } from '../../client/index.js';
+import { DbServer } from '../index.js';
+import { PropDef, PropDefEdge } from './types.js';
+export declare const propIsSigned: (prop: PropDef | PropDefEdge) => boolean;
+export declare const genRootId: () => number;
+export declare const genId: (db: DbServer | DbClient) => number;

package/dist/src/server/schema/utils.js

@@ -0,0 +1,21 @@
+import { INT16, INT32, INT64, INT8 } from './types.js';
+export const propIsSigned = (prop) => {
+    if (prop.typeIndex === INT16 ||
+        prop.typeIndex === INT32 ||
+        prop.typeIndex === INT64 ||
+        prop.typeIndex === INT8) {
+        return true;
+    }
+    return false;
+};
+const genIdFromInt = (n) => {
+    return n;
+};
+export const genRootId = () => {
+    return genIdFromInt(1);
+};
+export const genId = (db) => {
+    db.schema.lastId++;
+    return genIdFromInt(db.schema.lastId);
+};
+//# sourceMappingURL=utils.js.map

package/dist/src/server/start.d.ts

@@ -0,0 +1,6 @@
+import { DbServer } from './index.js';
+import './worker.js';
+export declare const DEFAULT_BLOCK_CAPACITY = 100000;
+export declare function start(db: DbServer, opts: {
+    clean?: boolean;
+}): Promise<void>;