@based/db 0.1.4 → 0.2.0

package/dist/src/server/index.js

@@ -5,7 +5,6 @@ import { ID_FIELD_DEF } from '@based/schema/def';
 import { start } from './start.js';
 import { destructureTreeKey, makeTreeKeyFromNodeId } from './tree.js';
 import { save } from './save.js';
-import { setTimeout } from 'node:timers/promises';
 import { migrate } from './migrate/index.js';
 import { debugServer } from '../utils.js';
 import { readUint16, readUint64 } from '@based/utils';
@@ -27,6 +26,15 @@ class SortIndex {
 export class DbServer extends DbShared {
     modifyDirtyRanges;
     dbCtxExternal; // pointer to zig dbCtx
+    subscriptions = {
+        subInterval: 200,
+        active: 0,
+        updateHandler: null,
+        ids: new Map(),
+        fullType: new Map(),
+        updateId: 1,
+        now: { listeners: new Set(), lastUpdated: 1 },
+    };
     migrating = null;
     saveInProgress = false;
     fileSystemPath;
@@ -382,6 +390,8 @@ export class DbServer extends DbShared {
         if (this.stopped) {
             return;
         }
+        clearTimeout(this.subscriptions.updateHandler);
+        this.subscriptions.updateHandler = null;
         this.stopped = true;
         this.unlistenExit();
         if (this.cleanupTimer) {
@@ -402,7 +412,6 @@ export class DbServer extends DbShared {
             this.workers = [];
             native.stop(this.dbCtxExternal);
             this.dbCtxExternal = null;
-            await setTimeout(100);
         }
         catch (e) {
             this.stopped = false;

package/dist/src/server/migrate/worker.js

@@ -10,101 +10,104 @@ if (isMainThread) {
     console.warn('running worker.ts in mainthread');
 }
 else if (workerData?.isDbMigrateWorker) {
-    const { from, to, fromSchema, toSchema, channel, workerState, transformFns } = workerData;
-    const fromCtx = native.externalFromInt(from);
-    const toCtx = native.externalFromInt(to);
-    native.createThreadCtx(fromCtx, native.getThreadId());
-    native.createThreadCtx(toCtx, native.getThreadId());
-    const fromDb = new BasedDb({ path: null });
-    const toDb = new BasedDb({ path: null });
-    fromDb.server.dbCtxExternal = fromCtx;
-    toDb.server.dbCtxExternal = toCtx;
-    setSchemaOnServer(fromDb.server, deSerialize(fromSchema));
-    setSchemaOnServer(toDb.server, deSerialize(toSchema));
-    setLocalClientSchema(fromDb.client, fromDb.server.schema);
-    setLocalClientSchema(toDb.client, toDb.server.schema);
-    try {
-        const map = {};
-        for (const type in fromDb.server.schemaTypesParsed) {
-            const { id, props } = fromDb.server.schemaTypesParsed[type];
-            const include = [];
-            const includeRaw = [];
-            for (const path in props) {
-                const prop = props[path];
-                if (prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) {
-                    include.push(`${path}.id`);
-                    if (prop.edges) {
-                        for (const key in prop.edges) {
-                            const edge = prop.edges[key];
-                            if (edge.typeIndex === REFERENCE ||
-                                edge.typeIndex === REFERENCES) {
-                                include.push(`${path}.${key}.id`);
-                            }
-                            else if (edge.typeIndex === CARDINALITY) {
-                                includeRaw.push(`${path}.${key}`);
-                            }
-                            else {
-                                include.push(`${path}.${key}`);
+    ;
+    (async () => {
+        const { from, to, fromSchema, toSchema, channel, workerState, transformFns, } = workerData;
+        const fromCtx = native.externalFromInt(from);
+        const toCtx = native.externalFromInt(to);
+        native.createThreadCtx(fromCtx, native.getThreadId());
+        native.createThreadCtx(toCtx, native.getThreadId());
+        const fromDb = new BasedDb({ path: null });
+        const toDb = new BasedDb({ path: null });
+        fromDb.server.dbCtxExternal = fromCtx;
+        toDb.server.dbCtxExternal = toCtx;
+        setSchemaOnServer(fromDb.server, deSerialize(fromSchema));
+        setSchemaOnServer(toDb.server, deSerialize(toSchema));
+        setLocalClientSchema(fromDb.client, fromDb.server.schema);
+        setLocalClientSchema(toDb.client, toDb.server.schema);
+        try {
+            const map = {};
+            for (const type in fromDb.server.schemaTypesParsed) {
+                const { id, props } = fromDb.server.schemaTypesParsed[type];
+                const include = [];
+                const includeRaw = [];
+                for (const path in props) {
+                    const prop = props[path];
+                    if (prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) {
+                        include.push(`${path}.id`);
+                        if (prop.edges) {
+                            for (const key in prop.edges) {
+                                const edge = prop.edges[key];
+                                if (edge.typeIndex === REFERENCE ||
+                                    edge.typeIndex === REFERENCES) {
+                                    include.push(`${path}.${key}.id`);
+                                }
+                                else if (edge.typeIndex === CARDINALITY) {
+                                    includeRaw.push(`${path}.${key}`);
+                                }
+                                else {
+                                    include.push(`${path}.${key}`);
+                                }
                             }
                         }
                     }
+                    else if (prop.typeIndex === CARDINALITY) {
+                        includeRaw.push(path);
+                    }
+                    else {
+                        include.push(path);
+                    }
                 }
-                else if (prop.typeIndex === CARDINALITY) {
-                    includeRaw.push(path);
-                }
-                else {
-                    include.push(path);
-                }
+                map[id] = { type, include, includeRaw };
             }
-            map[id] = { type, include, includeRaw };
-        }
-        for (const type in transformFns) {
-            const fnOrNull = transformFns[type];
-            transformFns[type] = eval(`(${fnOrNull})`);
-        }
-        while (true) {
-            let msg;
-            while ((msg = receiveMessageOnPort(channel))) {
-                const leafData = msg.message;
-                const { type, include, includeRaw } = map[leafData.typeId];
-                const typeTransformFn = transformFns[type];
-                const query = fromDb
-                    .query(type)
-                    .range(leafData.start - 1, leafData.end)
-                    .include(include);
-                for (const rawProp of includeRaw) {
-                    query.include(rawProp, { raw: true });
-                }
-                const nodes = query._getSync(fromCtx);
-                if (typeTransformFn) {
-                    for (const node of nodes) {
-                        const res = typeTransformFn(node);
-                        if (res === null) {
-                            continue;
-                        }
-                        if (Array.isArray(res)) {
-                            toDb.create(res[0], res[1] || node, { unsafe: true });
-                        }
-                        else {
-                            toDb.create(type, res || node, { unsafe: true });
+            for (const type in transformFns) {
+                const fnOrNull = transformFns[type];
+                transformFns[type] = eval(`(${fnOrNull})`);
+            }
+            while (true) {
+                let msg;
+                while ((msg = receiveMessageOnPort(channel))) {
+                    const leafData = msg.message;
+                    const { type, include, includeRaw } = map[leafData.typeId];
+                    const typeTransformFn = transformFns[type];
+                    const query = fromDb
+                        .query(type)
+                        .range(leafData.start - 1, leafData.end)
+                        .include(include);
+                    for (const rawProp of includeRaw) {
+                        query.include(rawProp, { raw: true });
+                    }
+                    const nodes = query._getSync(fromCtx);
+                    if (typeTransformFn) {
+                        for (const node of nodes) {
+                            const res = typeTransformFn(node);
+                            if (res === null) {
+                                continue;
+                            }
+                            if (Array.isArray(res)) {
+                                toDb.create(res[0], res[1] || node, { unsafe: true });
+                            }
+                            else {
+                                toDb.create(type, res || node, { unsafe: true });
+                            }
                         }
                     }
-                }
-                else if (type in toDb.server.schemaTypesParsed) {
-                    for (const node of nodes) {
-                        toDb.create(type, node, { unsafe: true });
+                    else if (type in toDb.server.schemaTypesParsed) {
+                        for (const node of nodes) {
+                            toDb.create(type, node, { unsafe: true });
+                        }
                     }
                 }
+                await toDb.drain();
+                native.membarSyncWrite();
+                setToSleep(workerState);
             }
-            await toDb.drain();
-            native.membarSyncWrite();
-            setToSleep(workerState);
         }
-    }
-    catch (e) {
-        console.error(e);
-        throw e;
-    }
+        catch (e) {
+            console.error(e);
+            throw e;
+        }
+    })();
 }
 else {
     console.info('incorrect worker db migrate');

package/dist/src/server/schema.js

@@ -1,4 +1,4 @@
-import { schemaToSelvaBuffer, updateTypeDefs } from '@based/schema/def';
+import { updateTypeDefs } from '@based/schema/def';
 import { serialize } from '@based/schema';
 import { join } from 'node:path';
 import { writeFile } from 'node:fs/promises';
@@ -8,6 +8,7 @@ import { saveSync } from './save.js';
 import { writeCreate } from '../client/modify/create/index.js';
 import { Ctx } from '../client/modify/Ctx.js';
 import { consume } from '../client/modify/drain.js';
+import { schemaToSelvaBuffer } from './schemaSelvaBuffer.js';
 export const setSchemaOnServer = (server, schema) => {
     const { schemaTypesParsed, schemaTypesParsedById } = updateTypeDefs(schema);
     server.schema = schema;
@@ -40,7 +41,7 @@ export const setNativeSchema = (server, schema) => {
         const type = server.schemaTypesParsed[types[i]];
         maxTid = Math.max(maxTid, type.id);
         try {
-            native.setSchemaType(type.id, new Uint8Array(s[i]), server.dbCtxExternal);
+            native.setSchemaType(server.dbCtxExternal, type.id, new Uint8Array(s[i]));
         }
         catch (err) {
             throw new Error(`Cannot update schema on selva (native) ${type.type} ${err.message}`);

package/dist/src/server/schemaSelvaBuffer.d.ts

@@ -0,0 +1,4 @@
+import { SchemaTypeDef } from '@based/schema/def';
+export declare function schemaToSelvaBuffer(schema: {
+    [key: string]: SchemaTypeDef;
+}): ArrayBuffer[];

package/dist/src/server/schemaSelvaBuffer.js

@@ -0,0 +1,182 @@
+import { ENCODER, writeDoubleLE, writeUint16, writeUint32, writeUint64 } from '@based/utils';
+import { ALIAS, ALIASES, BINARY, EMPTY_MICRO_BUFFER, CARDINALITY, MICRO_BUFFER, REFERENCE, REFERENCES, STRING, TEXT, VECTOR, JSON, COLVEC, VECTOR_BASE_TYPE_SIZE_MAP, INT8, UINT8, BOOLEAN, INT16, UINT16, INT32, UINT32, NUMBER, TIMESTAMP, ENUM, } from '@based/schema/def';
+import { NOT_COMPRESSED } from '@based/protocol';
+import native from '../native.js';
+const selvaFieldType = {
+    NULL: 0,
+    MICRO_BUFFER: 1,
+    STRING: 2,
+    TEXT: 3,
+    REFERENCE: 4,
+    REFERENCES: 5,
+    ALIAS: 8,
+    ALIASES: 9,
+    COLVEC: 10,
+};
+const selvaTypeMap = new Uint8Array(32); // 1.2x faster than JS array
+selvaTypeMap[MICRO_BUFFER] = selvaFieldType.MICRO_BUFFER;
+selvaTypeMap[VECTOR] = selvaFieldType.MICRO_BUFFER;
+selvaTypeMap[BINARY] = selvaFieldType.STRING;
+selvaTypeMap[CARDINALITY] = selvaFieldType.STRING;
+selvaTypeMap[JSON] = selvaFieldType.STRING;
+selvaTypeMap[STRING] = selvaFieldType.STRING;
+selvaTypeMap[TEXT] = selvaFieldType.TEXT;
+selvaTypeMap[REFERENCE] = selvaFieldType.REFERENCE;
+selvaTypeMap[REFERENCES] = selvaFieldType.REFERENCES;
+selvaTypeMap[ALIAS] = selvaFieldType.ALIAS;
+selvaTypeMap[ALIASES] = selvaFieldType.ALIASES;
+selvaTypeMap[COLVEC] = selvaFieldType.COLVEC;
+const EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT = 0x01;
+function blockCapacity(blockCapacity) {
+    const buf = new Uint8Array(Uint32Array.BYTES_PER_ELEMENT);
+    const view = new DataView(buf.buffer);
+    view.setUint32(0, blockCapacity, true);
+    return buf;
+}
+function sepPropCount(props) {
+    return props.filter((prop) => prop.separate).length;
+}
+function makeEdgeConstraintFlags(prop) {
+    let flags = 0;
+    flags |= prop.dependent ? EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT : 0x00;
+    return flags;
+}
+const propDefBuffer = (schema, prop) => {
+    const type = prop.typeIndex;
+    const selvaType = selvaTypeMap[type];
+    if (prop.len && (type === MICRO_BUFFER || type === VECTOR)) {
+        const buf = new Uint8Array(4);
+        const view = new DataView(buf.buffer);
+        buf[0] = selvaType;
+        view.setUint16(1, prop.len, true);
+        if (prop.default) {
+            buf[3] = 1; // has default
+            return [...buf, ...prop.default];
+        }
+        else {
+            buf[3] = 0; // has default
+            return [...buf];
+        }
+    }
+    else if (prop.len && type === COLVEC) {
+        const buf = new Uint8Array(5);
+        const view = new DataView(buf.buffer);
+        buf[0] = selvaType;
+        const baseSize = VECTOR_BASE_TYPE_SIZE_MAP[prop.vectorBaseType];
+        view.setUint16(1, prop.len / baseSize, true); // elements
+        view.setUint16(3, baseSize, true); // element size
+        return [...buf];
+    }
+    else if (type === REFERENCE || type === REFERENCES) {
+        const buf = new Uint8Array(7);
+        const view = new DataView(buf.buffer);
+        const dstType = schema[prop.inverseTypeName];
+        buf[0] = selvaType; // field type
+        buf[1] = makeEdgeConstraintFlags(prop); // flags
+        view.setUint16(2, dstType.id, true); // dst_node_type
+        buf[4] = prop.inversePropNumber; // inverse_field
+        view.setUint16(5, prop.edgeNodeTypeId ?? 0, true); // meta_node_type
+        return [...buf];
+    }
+    else if (type === STRING ||
+        type === BINARY ||
+        type === CARDINALITY ||
+        type === JSON) {
+        return [selvaType, prop.len < 50 ? prop.len : 0];
+    }
+    {
+        return [selvaType];
+    }
+};
+// TODO rewrite
+export function schemaToSelvaBuffer(schema) {
+    return Object.values(schema).map((t) => {
+        const props = Object.values(t.props);
+        const rest = [];
+        const nrFields = 1 + sepPropCount(props);
+        let refFields = 0;
+        let virtualFields = 0;
+        if (nrFields >= 250) {
+            throw new Error('Too many fields');
+        }
+        const main = {
+            ...EMPTY_MICRO_BUFFER,
+            len: t.mainLen === 0 ? 1 : t.mainLen,
+        };
+        for (const f of props) {
+            if (f.separate) {
+                if (f.typeIndex === REFERENCE || f.typeIndex === REFERENCES) {
+                    refFields++;
+                }
+                else if (f.typeIndex === ALIAS ||
+                    f.typeIndex === ALIASES ||
+                    f.typeIndex === COLVEC) {
+                    // We assume that these are always the last props!
+                    virtualFields++;
+                }
+                rest.push(f);
+            }
+            else {
+                if (f.default) {
+                    if (!main.default) {
+                        main.default = new Uint8Array(main.len);
+                    }
+                    const buf = main.default;
+                    switch (f.typeIndex) {
+                        case INT8:
+                        case UINT8:
+                        case BOOLEAN:
+                        case ENUM:
+                            main.default[f.start] = f.default;
+                            break;
+                        case INT16:
+                        case UINT16:
+                            writeUint16(buf, f.default, f.start);
+                            break;
+                        case INT32:
+                        case UINT32:
+                            writeUint32(buf, f.default, f.start);
+                            break;
+                        case NUMBER:
+                            writeDoubleLE(buf, f.default, f.start);
+                            //const view = new DataView(
+                            //  buf.buffer,
+                            //  f.start,
+                            //  8,
+                            //)
+                            //view.setFloat64(0, f.default, true)
+                            break;
+                        case TIMESTAMP:
+                            writeUint64(buf, f.default, f.start);
+                            break;
+                        case BINARY:
+                        case STRING:
+                            if (f.default instanceof Uint8Array) {
+                                buf.set(f.default, f.start);
+                            }
+                            else {
+                                const value = f.default.normalize('NFKD');
+                                buf[f.start] = 0; // lang
+                                buf[f.start + 1] = NOT_COMPRESSED;
+                                const { written: l } = ENCODER.encodeInto(value, buf.subarray(f.start + 2));
+                                let crc = native.crc32(buf.subarray(f.start + 2, f.start + 2 + l));
+                                writeUint32(buf, crc, f.start + 2 + l);
+                            }
+                            break;
+                    }
+                }
+            }
+        }
+        rest.sort((a, b) => a.prop - b.prop);
+        return Uint8Array.from([
+            ...blockCapacity(t.blockCapacity), // u32 blockCapacity
+            nrFields, // u8 nrFields
+            1 + refFields, // u8 nrFixedFields
+            virtualFields, // u8 nrVirtualFields
+            6, // u8 version (generally follows the sdb version)
+            ...propDefBuffer(schema, main),
+            ...rest.map((f) => propDefBuffer(schema, f)).flat(1),
+        ]).buffer;
+    });
+}
+//# sourceMappingURL=schemaSelvaBuffer.js.map

package/dist/src/server/subscription.d.ts

@@ -0,0 +1,32 @@
+import { OnError } from '../client/query/subscription/types.js';
+import { DbServer } from '../index.js';
+type OnData = (res: Uint8Array) => void;
+export type SubscriptionFullType = {
+    listeners: Set<() => void>;
+};
+export type SubscriptionNow = {
+    lastEval: number;
+    next: number;
+    listeners: Set<() => void>;
+};
+export type SubscriptionId = {
+    types?: Uint16Array;
+    ids: Map<number, Set<() => void>>;
+    typesListener?: () => void;
+    nowListener?: () => void;
+};
+export type Subscriptions = {
+    updateId: number;
+    active: number;
+    updateHandler: ReturnType<typeof setTimeout>;
+    ids: Map<number, SubscriptionId>;
+    fullType: Map<number, SubscriptionFullType>;
+    now: {
+        listeners: Set<() => void>;
+        lastUpdated: number;
+    };
+    subInterval: number;
+};
+export declare const startUpdateHandler: (server: DbServer) => void;
+export declare const registerSubscription: (server: DbServer, query: Uint8Array, sub: Uint8Array, onData: OnData, onError: OnError, subInterval?: number) => () => void;
+export {};