@based/db 0.1.5 → 0.2.1

package/dist/src/server/index.js

@@ -5,7 +5,6 @@ import { ID_FIELD_DEF } from '@based/schema/def';
 import { start } from './start.js';
 import { destructureTreeKey, makeTreeKeyFromNodeId } from './tree.js';
 import { save } from './save.js';
-import { setTimeout } from 'node:timers/promises';
 import { migrate } from './migrate/index.js';
 import { debugServer } from '../utils.js';
 import { readUint16, readUint64 } from '@based/utils';
@@ -27,6 +26,15 @@ class SortIndex {
 export class DbServer extends DbShared {
     modifyDirtyRanges;
     dbCtxExternal; // pointer to zig dbCtx
+    subscriptions = {
+        subInterval: 200,
+        active: 0,
+        updateHandler: null,
+        ids: new Map(),
+        fullType: new Map(),
+        updateId: 1,
+        now: { listeners: new Set(), lastUpdated: 1 },
+    };
     migrating = null;
     saveInProgress = false;
     fileSystemPath;
@@ -382,6 +390,8 @@ export class DbServer extends DbShared {
         if (this.stopped) {
             return;
         }
+        clearTimeout(this.subscriptions.updateHandler);
+        this.subscriptions.updateHandler = null;
         this.stopped = true;
         this.unlistenExit();
         if (this.cleanupTimer) {
@@ -402,7 +412,6 @@ export class DbServer extends DbShared {
             this.workers = [];
             native.stop(this.dbCtxExternal);
             this.dbCtxExternal = null;
-            await setTimeout(100);
         }
         catch (e) {
             this.stopped = false;

package/dist/src/server/schema.js

@@ -1,4 +1,4 @@
-import { schemaToSelvaBuffer, updateTypeDefs } from '@based/schema/def';
+import { updateTypeDefs } from '@based/schema/def';
 import { serialize } from '@based/schema';
 import { join } from 'node:path';
 import { writeFile } from 'node:fs/promises';
@@ -8,6 +8,7 @@ import { saveSync } from './save.js';
 import { writeCreate } from '../client/modify/create/index.js';
 import { Ctx } from '../client/modify/Ctx.js';
 import { consume } from '../client/modify/drain.js';
+import { schemaToSelvaBuffer } from './schemaSelvaBuffer.js';
 export const setSchemaOnServer = (server, schema) => {
     const { schemaTypesParsed, schemaTypesParsedById } = updateTypeDefs(schema);
     server.schema = schema;
@@ -40,7 +41,7 @@ export const setNativeSchema = (server, schema) => {
         const type = server.schemaTypesParsed[types[i]];
         maxTid = Math.max(maxTid, type.id);
         try {
-            native.setSchemaType(type.id, new Uint8Array(s[i]), server.dbCtxExternal);
+            native.setSchemaType(server.dbCtxExternal, type.id, new Uint8Array(s[i]));
         }
         catch (err) {
             throw new Error(`Cannot update schema on selva (native) ${type.type} ${err.message}`);

package/dist/src/server/schemaSelvaBuffer.d.ts

@@ -0,0 +1,4 @@
+import { SchemaTypeDef } from '@based/schema/def';
+export declare function schemaToSelvaBuffer(schema: {
+    [key: string]: SchemaTypeDef;
+}): ArrayBuffer[];

package/dist/src/server/schemaSelvaBuffer.js

@@ -0,0 +1,182 @@
+import { ENCODER, writeDoubleLE, writeUint16, writeUint32, writeUint64 } from '@based/utils';
+import { ALIAS, ALIASES, BINARY, EMPTY_MICRO_BUFFER, CARDINALITY, MICRO_BUFFER, REFERENCE, REFERENCES, STRING, TEXT, VECTOR, JSON, COLVEC, VECTOR_BASE_TYPE_SIZE_MAP, INT8, UINT8, BOOLEAN, INT16, UINT16, INT32, UINT32, NUMBER, TIMESTAMP, ENUM, } from '@based/schema/def';
+import { NOT_COMPRESSED } from '@based/protocol';
+import native from '../native.js';
+const selvaFieldType = {
+    NULL: 0,
+    MICRO_BUFFER: 1,
+    STRING: 2,
+    TEXT: 3,
+    REFERENCE: 4,
+    REFERENCES: 5,
+    ALIAS: 8,
+    ALIASES: 9,
+    COLVEC: 10,
+};
+const selvaTypeMap = new Uint8Array(32); // 1.2x faster than JS array
+selvaTypeMap[MICRO_BUFFER] = selvaFieldType.MICRO_BUFFER;
+selvaTypeMap[VECTOR] = selvaFieldType.MICRO_BUFFER;
+selvaTypeMap[BINARY] = selvaFieldType.STRING;
+selvaTypeMap[CARDINALITY] = selvaFieldType.STRING;
+selvaTypeMap[JSON] = selvaFieldType.STRING;
+selvaTypeMap[STRING] = selvaFieldType.STRING;
+selvaTypeMap[TEXT] = selvaFieldType.TEXT;
+selvaTypeMap[REFERENCE] = selvaFieldType.REFERENCE;
+selvaTypeMap[REFERENCES] = selvaFieldType.REFERENCES;
+selvaTypeMap[ALIAS] = selvaFieldType.ALIAS;
+selvaTypeMap[ALIASES] = selvaFieldType.ALIASES;
+selvaTypeMap[COLVEC] = selvaFieldType.COLVEC;
+const EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT = 0x01;
+function blockCapacity(blockCapacity) {
+    const buf = new Uint8Array(Uint32Array.BYTES_PER_ELEMENT);
+    const view = new DataView(buf.buffer);
+    view.setUint32(0, blockCapacity, true);
+    return buf;
+}
+function sepPropCount(props) {
+    return props.filter((prop) => prop.separate).length;
+}
+function makeEdgeConstraintFlags(prop) {
+    let flags = 0;
+    flags |= prop.dependent ? EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT : 0x00;
+    return flags;
+}
+const propDefBuffer = (schema, prop) => {
+    const type = prop.typeIndex;
+    const selvaType = selvaTypeMap[type];
+    if (prop.len && (type === MICRO_BUFFER || type === VECTOR)) {
+        const buf = new Uint8Array(4);
+        const view = new DataView(buf.buffer);
+        buf[0] = selvaType;
+        view.setUint16(1, prop.len, true);
+        if (prop.default) {
+            buf[3] = 1; // has default
+            return [...buf, ...prop.default];
+        }
+        else {
+            buf[3] = 0; // has default
+            return [...buf];
+        }
+    }
+    else if (prop.len && type === COLVEC) {
+        const buf = new Uint8Array(5);
+        const view = new DataView(buf.buffer);
+        buf[0] = selvaType;
+        const baseSize = VECTOR_BASE_TYPE_SIZE_MAP[prop.vectorBaseType];
+        view.setUint16(1, prop.len / baseSize, true); // elements
+        view.setUint16(3, baseSize, true); // element size
+        return [...buf];
+    }
+    else if (type === REFERENCE || type === REFERENCES) {
+        const buf = new Uint8Array(7);
+        const view = new DataView(buf.buffer);
+        const dstType = schema[prop.inverseTypeName];
+        buf[0] = selvaType; // field type
+        buf[1] = makeEdgeConstraintFlags(prop); // flags
+        view.setUint16(2, dstType.id, true); // dst_node_type
+        buf[4] = prop.inversePropNumber; // inverse_field
+        view.setUint16(5, prop.edgeNodeTypeId ?? 0, true); // meta_node_type
+        return [...buf];
+    }
+    else if (type === STRING ||
+        type === BINARY ||
+        type === CARDINALITY ||
+        type === JSON) {
+        return [selvaType, prop.len < 50 ? prop.len : 0];
+    }
+    {
+        return [selvaType];
+    }
+};
+// TODO rewrite
+export function schemaToSelvaBuffer(schema) {
+    return Object.values(schema).map((t) => {
+        const props = Object.values(t.props);
+        const rest = [];
+        const nrFields = 1 + sepPropCount(props);
+        let refFields = 0;
+        let virtualFields = 0;
+        if (nrFields >= 250) {
+            throw new Error('Too many fields');
+        }
+        const main = {
+            ...EMPTY_MICRO_BUFFER,
+            len: t.mainLen === 0 ? 1 : t.mainLen,
+        };
+        for (const f of props) {
+            if (f.separate) {
+                if (f.typeIndex === REFERENCE || f.typeIndex === REFERENCES) {
+                    refFields++;
+                }
+                else if (f.typeIndex === ALIAS ||
+                    f.typeIndex === ALIASES ||
+                    f.typeIndex === COLVEC) {
+                    // We assume that these are always the last props!
+                    virtualFields++;
+                }
+                rest.push(f);
+            }
+            else {
+                if (f.default) {
+                    if (!main.default) {
+                        main.default = new Uint8Array(main.len);
+                    }
+                    const buf = main.default;
+                    switch (f.typeIndex) {
+                        case INT8:
+                        case UINT8:
+                        case BOOLEAN:
+                        case ENUM:
+                            main.default[f.start] = f.default;
+                            break;
+                        case INT16:
+                        case UINT16:
+                            writeUint16(buf, f.default, f.start);
+                            break;
+                        case INT32:
+                        case UINT32:
+                            writeUint32(buf, f.default, f.start);
+                            break;
+                        case NUMBER:
+                            writeDoubleLE(buf, f.default, f.start);
+                            //const view = new DataView(
+                            //  buf.buffer,
+                            //  f.start,
+                            //  8,
+                            //)
+                            //view.setFloat64(0, f.default, true)
+                            break;
+                        case TIMESTAMP:
+                            writeUint64(buf, f.default, f.start);
+                            break;
+                        case BINARY:
+                        case STRING:
+                            if (f.default instanceof Uint8Array) {
+                                buf.set(f.default, f.start);
+                            }
+                            else {
+                                const value = f.default.normalize('NFKD');
+                                buf[f.start] = 0; // lang
+                                buf[f.start + 1] = NOT_COMPRESSED;
+                                const { written: l } = ENCODER.encodeInto(value, buf.subarray(f.start + 2));
+                                let crc = native.crc32(buf.subarray(f.start + 2, f.start + 2 + l));
+                                writeUint32(buf, crc, f.start + 2 + l);
+                            }
+                            break;
+                    }
+                }
+            }
+        }
+        rest.sort((a, b) => a.prop - b.prop);
+        return Uint8Array.from([
+            ...blockCapacity(t.blockCapacity), // u32 blockCapacity
+            nrFields, // u8 nrFields
+            1 + refFields, // u8 nrFixedFields
+            virtualFields, // u8 nrVirtualFields
+            6, // u8 version (generally follows the sdb version)
+            ...propDefBuffer(schema, main),
+            ...rest.map((f) => propDefBuffer(schema, f)).flat(1),
+        ]).buffer;
+    });
+}
+//# sourceMappingURL=schemaSelvaBuffer.js.map

package/dist/src/server/subscription.d.ts

@@ -0,0 +1,32 @@
+import { OnError } from '../client/query/subscription/types.js';
+import { DbServer } from '../index.js';
+type OnData = (res: Uint8Array) => void;
+export type SubscriptionFullType = {
+    listeners: Set<() => void>;
+};
+export type SubscriptionNow = {
+    lastEval: number;
+    next: number;
+    listeners: Set<() => void>;
+};
+export type SubscriptionId = {
+    types?: Uint16Array;
+    ids: Map<number, Set<() => void>>;
+    typesListener?: () => void;
+    nowListener?: () => void;
+};
+export type Subscriptions = {
+    updateId: number;
+    active: number;
+    updateHandler: ReturnType<typeof setTimeout>;
+    ids: Map<number, SubscriptionId>;
+    fullType: Map<number, SubscriptionFullType>;
+    now: {
+        listeners: Set<() => void>;
+        lastUpdated: number;
+    };
+    subInterval: number;
+};
+export declare const startUpdateHandler: (server: DbServer) => void;
+export declare const registerSubscription: (server: DbServer, query: Uint8Array, sub: Uint8Array, onData: OnData, onError: OnError, subInterval?: number) => () => void;
+export {};

package/dist/src/server/subscription.js

@@ -0,0 +1,288 @@
+import { readInt64, readUint16, readUint32, writeInt64 } from '@based/utils';
+import { SubscriptionType, } from '../client/query/subscription/types.js';
+import native from '../native.js';
+import { MAX_ID } from '@based/schema';
+import { styleText } from 'util';
+export const startUpdateHandler = (server) => {
+    // skip next if queries are sitll in progress can add a number for each staged sub
+    // combine this with handled modify
+    const scheduleUpdate = () => {
+        if (server.stopped) {
+            return;
+        }
+        server.subscriptions.updateId++;
+        if (server.subscriptions.updateId > MAX_ID) {
+            server.subscriptions.updateId = 1;
+        }
+        // can do seperate timing for id / type
+        // scince multi queries are much heavier ofc
+        const markedIdSubs = native.getMarkedIdSubscriptions(server.dbCtxExternal);
+        if (markedIdSubs) {
+            const buffer = new Uint8Array(markedIdSubs);
+            for (let i = 0; i < buffer.byteLength; i += 8) {
+                const id = readUint32(buffer, i);
+                const subId = readUint32(buffer, i + 4);
+                const subContainer = server.subscriptions.ids.get(subId);
+                const ids = subContainer.ids.get(id);
+                if (ids) {
+                    for (const fn of ids) {
+                        fn();
+                    }
+                }
+            }
+        }
+        const markedMultiSubs = native.getMarkedMultiSubscriptions(server.dbCtxExternal);
+        if (markedMultiSubs) {
+            const buffer = new Uint8Array(markedMultiSubs);
+            for (let i = 0; i < buffer.byteLength; i += 2) {
+                const typeId = readUint16(buffer, i);
+                const subs = server.subscriptions.fullType.get(typeId);
+                if (subs) {
+                    for (const fn of subs.listeners) {
+                        fn();
+                    }
+                }
+            }
+        }
+        if (server.subscriptions.updateId - server.subscriptions.now.lastUpdated >
+            Math.max(1000 / server.subscriptions.subInterval, 1) // 1 time per second
+        ) {
+            server.subscriptions.now.lastUpdated = server.subscriptions.updateId;
+            for (const fn of server.subscriptions.now.listeners) {
+                fn();
+            }
+        }
+        server.subscriptions.updateHandler = setTimeout(scheduleUpdate, server.subscriptions.subInterval);
+    };
+    server.subscriptions.updateHandler = setTimeout(scheduleUpdate, server.subscriptions.subInterval);
+};
+const addToMultiSub = (server, typeId, runQuery) => {
+    let fullType;
+    let listeners;
+    if (!server.subscriptions.fullType.has(typeId)) {
+        listeners = new Set();
+        fullType = {
+            listeners,
+        };
+        server.subscriptions.fullType.set(typeId, fullType);
+        native.addMultiSubscription(server.dbCtxExternal, typeId);
+    }
+    else {
+        fullType = server.subscriptions.fullType.get(typeId);
+        listeners = fullType.listeners;
+    }
+    listeners.add(runQuery);
+};
+const removeFromMultiSub = (server, typeId, runQuery) => {
+    const typeSub = server.subscriptions.fullType.get(typeId);
+    if (!typeSub) {
+        return;
+    }
+    typeSub.listeners.delete(runQuery);
+    if (typeSub.listeners.size === 0) {
+        native.removeMultiSubscription(server.dbCtxExternal, typeId);
+        server.subscriptions.fullType.delete(typeId);
+    }
+};
+const replaceNowValues = (query, now) => {
+    const dateNow = Date.now();
+    for (let i = 0; i < now.byteLength; i += 16) {
+        const offset = readInt64(now, i + 4);
+        const byteIndex = readUint32(now, i + 12);
+        writeInt64(query, dateNow + offset, byteIndex);
+    }
+};
+let total = 0;
+let exectime = 0;
+let int;
+export const registerSubscription = (server, query, sub, onData, onError, subInterval) => {
+    // this can change dynamicly
+    if (subInterval) {
+        server.subscriptions.subInterval = subInterval;
+    }
+    if (!int)
+        int = setInterval(() => {
+            console.log('EXECED', total, exectime / total, 'ms exec time');
+            if (server.stopped) {
+                clearInterval(int);
+            }
+        }, 1e3);
+    let killed = false;
+    // now maybe just once per second? (for now)
+    if (server.subscriptions.active === 0) {
+        startUpdateHandler(server);
+    }
+    let lastUpdated = 0;
+    let now;
+    const runQuery = () => {
+        if (lastUpdated !== server.subscriptions.updateId) {
+            lastUpdated = server.subscriptions.updateId;
+            if (now) {
+                replaceNowValues(query, now);
+            }
+            let d = Date.now();
+            server.getQueryBuf(query).then((res) => {
+                if (killed) {
+                    return;
+                }
+                total++;
+                exectime += Date.now() - d;
+                if (res.byteLength >= 4) {
+                    onData(res);
+                }
+                else if (res.byteLength === 1 && res[0] === 0) {
+                    server.emit('info', `Subscribe schema mismatch - should resolve after update`);
+                    return;
+                }
+                else {
+                    let name = styleText('red', `QueryError[]\n`);
+                    name += `  Incorrect buffer received in subscription (maybe server not started ${res.byteLength}) bytes\n`;
+                    onError(new Error(name));
+                }
+            });
+        }
+        else {
+            // console.log('Allready fired block')
+        }
+    };
+    server.subscriptions.active++;
+    if (sub[0] === SubscriptionType.singleId) {
+        const subId = readUint32(sub, 1);
+        const id = readUint32(sub, 7);
+        const headerLen = 16;
+        let subContainer;
+        let listeners;
+        if (!server.subscriptions.ids.get(subId)) {
+            subContainer = { ids: new Map() };
+            server.subscriptions.ids.set(subId, subContainer);
+            const typesLen = readUint16(sub, 12);
+            const nowLen = readUint16(sub, 14);
+            const fLen = sub[11];
+            if (typesLen != 0) {
+                // double check if this is alignment correct with the byteOffset else copy
+                const byteOffset = sub.byteOffset + headerLen + fLen;
+                if (byteOffset % 2 === 0) {
+                    subContainer.types = new Uint16Array(sub.buffer, byteOffset, typesLen);
+                }
+                else {
+                    subContainer.types = new Uint16Array(sub.slice(headerLen + fLen, headerLen + fLen + typesLen * 2));
+                }
+                subContainer.typesListener = () => {
+                    for (const set of subContainer.ids.values()) {
+                        for (const fn of set) {
+                            fn();
+                        }
+                    }
+                };
+                for (const typeId of subContainer.types) {
+                    addToMultiSub(server, typeId, subContainer.typesListener);
+                }
+            }
+            if (nowLen != 0) {
+                // when this is the case do a completely different strategy
+                // keep track of last update on sub id container
+                // and get the date allways (as a seperate query)
+                // when getting the date mark next in line
+                // have to make a copy (subArray is weak)
+                now = sub.slice(headerLen + fLen + typesLen * 2);
+                subContainer.nowListener = () => {
+                    // per id want to have a last eval and needs next eval
+                    for (const set of subContainer.ids.values()) {
+                        for (const fn of set) {
+                            fn();
+                        }
+                    }
+                };
+                server.subscriptions.now.listeners.add(subContainer.nowListener);
+            }
+        }
+        else {
+            subContainer = server.subscriptions.ids.get(subId);
+        }
+        if (!subContainer.ids.has(id)) {
+            listeners = new Set();
+            subContainer.ids.set(id, listeners);
+            native.addIdSubscription(server.dbCtxExternal, sub);
+        }
+        else {
+            listeners = subContainer.ids.get(id);
+        }
+        listeners.add(runQuery);
+        process.nextTick(() => {
+            runQuery();
+        });
+        return () => {
+            killed = true;
+            listeners.delete(runQuery);
+            if (listeners.size === 0) {
+                native.removeIdSubscription(server.dbCtxExternal, sub);
+                subContainer.ids.delete(id);
+            }
+            if (subContainer.ids.size === 0) {
+                if (subContainer.types) {
+                    for (const typeId of subContainer.types) {
+                        removeFromMultiSub(server, typeId, subContainer.typesListener);
+                    }
+                }
+                if (now) {
+                    server.subscriptions.now.listeners.delete(subContainer.nowListener);
+                }
+                server.subscriptions.ids.delete(subId);
+            }
+            server.subscriptions.active--;
+            if (server.subscriptions.active === 0) {
+                clearTimeout(server.subscriptions.updateHandler);
+                server.subscriptions.updateHandler = null;
+            }
+        };
+    }
+    else if (sub[0] === SubscriptionType.fullType) {
+        const headerLen = 8;
+        const typeId = readUint16(sub, 1);
+        addToMultiSub(server, typeId, runQuery);
+        const typesLen = readUint16(sub, 3);
+        let types;
+        if (typesLen != 0) {
+            // double check if this is alignment correct with the byteOffset else copy
+            const byteOffset = sub.byteOffset + headerLen;
+            if (byteOffset % 2 === 0) {
+                types = new Uint16Array(sub.buffer, byteOffset, typesLen);
+            }
+            else {
+                types = new Uint16Array(sub.slice(headerLen, headerLen + typesLen * 2));
+            }
+            for (const typeId of types) {
+                addToMultiSub(server, typeId, runQuery);
+            }
+        }
+        if (readUint16(sub, 5) != 0) {
+            // have to make a copy (subArray is weak)
+            now = sub.slice(headerLen + typesLen * 2);
+            server.subscriptions.now.listeners.add(runQuery);
+        }
+        process.nextTick(() => {
+            runQuery();
+        });
+        return () => {
+            killed = true;
+            if (now) {
+                server.subscriptions.now.listeners.delete(runQuery);
+            }
+            removeFromMultiSub(server, typeId, runQuery);
+            if (types) {
+                for (const typeId of types) {
+                    removeFromMultiSub(server, typeId, runQuery);
+                }
+            }
+            server.subscriptions.active--;
+            if (server.subscriptions.active === 0) {
+                clearTimeout(server.subscriptions.updateHandler);
+                server.subscriptions.updateHandler = null;
+            }
+        };
+    }
+    else {
+        throw new Error('Unhandled subscription!');
+    }
+};
+//# sourceMappingURL=subscription.js.map

package/dist/src/utils.js

@@ -1,5 +1,4 @@
-import { inspect } from 'node:util';
-import picocolors from 'picocolors';
+import { inspect, styleText } from 'node:util';
 export const debugMode = (target, getInfo = null) => {
     const opts = { showHidden: false, depth: null, colors: true };
     const info = (v) => (typeof v === 'object' ? inspect(v, opts) : v);
@@ -29,7 +28,7 @@ export const debugMode = (target, getInfo = null) => {
             let cnt = 0;
             const color = colors[colorKey++ % colors.length];
             target[key] = function () {
-                const arr = [picocolors[color](`[${key}:${++cnt}]`)];
+                const arr = [styleText(color, `[${key}:${++cnt}]`)];
                 const add = getInfo?.(key);
                 if (add)
                     arr.push(add);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.1.5",
+  "version": "0.2.1",
   "type": "module",
   "main": "./dist/src/index.js",
   "scripts": {
@@ -42,11 +42,10 @@
   ],
   "dependencies": {
     "@based/hash": "1.1.0",
-    "@based/schema": "5.0.4",
-    "@based/utils": "1.1.1",
-    "@based/protocol": "0.0.5",
-    "exit-hook": "^4.0.0",
-    "picocolors": "^1.1.0"
+    "@based/schema": "5.1.1",
+    "@based/utils": "1.2.0",
+    "@based/protocol": "0.1.1",
+    "exit-hook": "^4.0.0"
   },
   "optionalDependencies": {
     "@based/locale-x86-64-gnu": "*"