@based/db 0.0.1

package/dist/src/server/index.js

@@ -0,0 +1,407 @@
+import native from '../native.js';
+import { rm, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { getPropType } from '@based/schema';
+import { genRootId } from './schema/utils.js';
+import { updateTypeDefs } from './schema/typeDef.js';
+import { schemaToSelvaBuffer } from './schema/selvaBuffer.js';
+import { start } from './start.js';
+import { foreachDirtyBlock, makeCsmtKey } from './tree.js';
+import { save } from './save.js';
+import { Worker, MessageChannel } from 'node:worker_threads';
+import { fileURLToPath } from 'node:url';
+import { setTimeout } from 'node:timers/promises';
+import { migrate } from './migrate/index.js';
+const SCHEMA_FILE = 'schema.json';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const workerPath = join(__dirname, 'worker.js');
+export class DbWorker {
+    constructor(address, db) {
+        const { port1, port2 } = new MessageChannel();
+        this.db = db;
+        this.channel = port1;
+        this.worker = new Worker(workerPath, {
+            workerData: {
+                channel: port2,
+                address,
+            },
+            transferList: [port2],
+        });
+        port1.on('message', (buf) => {
+            this.resolvers.shift()(buf);
+            this.db.onQueryEnd();
+        });
+    }
+    db;
+    channel;
+    worker;
+    resolvers = [];
+    callback = (resolve) => {
+        this.db.processingQueries++;
+        this.resolvers.push(resolve);
+    };
+    updateCtx(address) {
+        this.channel.postMessage(address);
+        return new Promise(this.callback);
+    }
+    getQueryBuf(buf) {
+        this.channel.postMessage(buf);
+        return new Promise(this.callback);
+    }
+}
+export class DbServer {
+    modifyBuf;
+    dbCtxExternal; // pointer to zig dbCtx
+    schema = {
+        lastId: 1, // we reserve one for root props
+        types: {},
+    };
+    migrating = null;
+    schemaTypesParsed = {};
+    schemaTypesParsedById = {};
+    fileSystemPath;
+    maxModifySize;
+    merkleTree;
+    dirtyRanges = new Set();
+    csmtHashFun = native.createHash();
+    workers = [];
+    availableWorkerIndex = -1;
+    processingQueries = 0;
+    modifyQueue = [];
+    queryQueue = new Map();
+    stopped;
+    onSchemaChange;
+    constructor({ path, maxModifySize = 100 * 1e3 * 1e3, onSchemaChange, }) {
+        this.maxModifySize = maxModifySize;
+        this.fileSystemPath = path;
+        this.sortIndexes = {};
+        this.onSchemaChange = onSchemaChange;
+    }
+    start(opts) {
+        return start(this, opts);
+    }
+    save() {
+        return save(this);
+    }
+    createCsmtHashFun = () => {
+        // We can just reuse it as long as we only have one tree.
+        this.csmtHashFun.reset();
+        return this.csmtHashFun;
+    };
+    sortIndexes;
+    createSortIndex(type, field) {
+        const t = this.schemaTypesParsed[type];
+        const prop = t.props[field];
+        let types = this.sortIndexes[t.id];
+        if (!types) {
+            types = this.sortIndexes[t.id] = {};
+        }
+        let fields = types[prop.prop];
+        if (!fields) {
+            fields = types[prop.prop] = {};
+        }
+        let sortIndex = fields[prop.start];
+        if (sortIndex) {
+            return sortIndex;
+        }
+        const buf = Buffer.allocUnsafe(8);
+        // size [2 type] [1 field] [2 start] [2 len]
+        buf.writeUint16LE(t.id, 0);
+        buf[2] = prop.prop;
+        buf.writeUint16LE(prop.start, 3);
+        buf.writeUint16LE(prop.len, 5);
+        buf[7] = prop.typeIndex;
+        sortIndex = native.createSortIndex(buf, this.dbCtxExternal);
+        fields[prop.start] = sortIndex;
+        return sortIndex;
+    }
+    destroySortIndex(type, field) {
+        const t = this.schemaTypesParsed[type];
+        const prop = t.props[field];
+        let types = this.sortIndexes[t.id];
+        if (!type) {
+            return;
+        }
+        let fields = types[prop.prop];
+        if (!fields) {
+            fields = types[prop.prop] = {};
+        }
+        let sortIndex = fields[prop.start];
+        if (sortIndex) {
+            const buf = Buffer.allocUnsafe(5);
+            buf.writeUint16LE(t.id, 0);
+            buf[2] = prop.prop;
+            buf.writeUint16LE(prop.start, 3);
+            native.destroySortIndex(buf, this.dbCtxExternal);
+            delete fields[prop.start];
+        }
+    }
+    hasSortIndex(typeId, field, start) {
+        let types = this.sortIndexes[typeId];
+        if (!types) {
+            types = this.sortIndexes[typeId] = {};
+        }
+        let fields = types[field];
+        if (!fields) {
+            fields = types[field] = {};
+        }
+        let sortIndex = fields[start];
+        if (sortIndex) {
+            return true;
+        }
+        return false;
+    }
+    migrateSchema(schema, transform) {
+        return migrate(this, schema, transform);
+    }
+    createSortIndexBuffer(typeId, field, start) {
+        const buf = Buffer.allocUnsafe(8);
+        buf.writeUint16LE(typeId, 0);
+        buf[2] = field;
+        buf.writeUint16LE(start, 3);
+        let typeDef;
+        let prop;
+        for (const t in this.schemaTypesParsed) {
+            typeDef = this.schemaTypesParsed[t];
+            if (typeDef.id == typeId) {
+                for (const p in typeDef.props) {
+                    const propDef = typeDef.props[p];
+                    if (propDef.prop == field && propDef.start == start) {
+                        prop = propDef;
+                        break;
+                    }
+                }
+                break;
+            }
+        }
+        if (!typeDef) {
+            throw new Error(`Cannot find type id on db from query for sort ${typeId}`);
+        }
+        if (!prop) {
+            throw new Error(`Cannot find prop on db from query for sort ${field}`);
+        }
+        buf.writeUint16LE(prop.len, 5);
+        buf[7] = prop.typeIndex;
+        // put in modify stuff
+        const sortIndex = native.createSortIndex(buf, this.dbCtxExternal);
+        const types = this.sortIndexes[typeId];
+        const fields = types[field];
+        fields[start] = sortIndex;
+        return sortIndex;
+    }
+    updateMerkleTree() {
+        foreachDirtyBlock(this, (mtKey, typeId, start, end) => {
+            const oldLeaf = this.merkleTree.search(mtKey);
+            const hash = Buffer.allocUnsafe(16);
+            native.getNodeRangeHash(typeId, start, end, hash, this.dbCtxExternal);
+            if (oldLeaf) {
+                if (oldLeaf.hash.equals(hash)) {
+                    return;
+                }
+                try {
+                    this.merkleTree.delete(mtKey);
+                }
+                catch (err) { }
+            }
+            const data = {
+                file: '', // not saved yet
+                typeId,
+                start,
+                end,
+            };
+            this.merkleTree.insert(mtKey, hash, data);
+        });
+    }
+    putSchema(strictSchema, fromStart = false, transformFns) {
+        if (!fromStart && Object.keys(this.schema.types).length > 0) {
+            return this.migrateSchema(strictSchema, transformFns);
+        }
+        const { lastId } = this.schema;
+        this.schema = {
+            lastId,
+            ...strictSchema,
+        };
+        if (strictSchema.props) {
+            this.schema.types ??= {};
+            const props = { ...strictSchema.props };
+            for (const key in props) {
+                const prop = props[key];
+                const propType = getPropType(prop);
+                let refProp;
+                if (propType === 'reference') {
+                    refProp = prop;
+                }
+                else if (propType === 'references') {
+                    refProp = prop.items;
+                }
+                if (refProp) {
+                    const type = this.schema.types[refProp.ref];
+                    const inverseKey = '_' + key;
+                    this.schema.types[refProp.ref] = {
+                        ...type,
+                        props: {
+                            ...type.props,
+                            [inverseKey]: {
+                                items: {
+                                    ref: '_root',
+                                    prop: key,
+                                },
+                            },
+                        },
+                    };
+                    refProp.prop = inverseKey;
+                }
+            }
+            // @ts-ignore This creates an internal type to use for root props
+            this.schema.types._root = {
+                id: genRootId(),
+                props,
+            };
+            delete this.schema.props;
+        }
+        updateTypeDefs(this);
+        if (!fromStart) {
+            writeFile(join(this.fileSystemPath, SCHEMA_FILE), JSON.stringify(this.schema)).catch((err) => console.error(SCHEMA_FILE, err));
+            let types = Object.keys(this.schemaTypesParsed);
+            const s = schemaToSelvaBuffer(this.schemaTypesParsed);
+            for (let i = 0; i < s.length; i++) {
+                //  TYPE SELVA user Uint8Array(6) [ 1, 17, 23, 0, 11, 0 ]
+                const type = this.schemaTypesParsed[types[i]];
+                // TODO should not crash!
+                try {
+                    native.updateSchemaType(type.id, s[i], this.dbCtxExternal);
+                }
+                catch (err) {
+                    console.error('Cannot update schema on selva', type.type, err, s[i]);
+                }
+            }
+            if (strictSchema.props) {
+                // insert a root node
+                const data = [2, 1, 0, 0, 0, 9, 1, 0, 0, 0, 7, 1, 0, 1];
+                const blockKey = makeCsmtKey(1, 0);
+                const buf = Buffer.alloc(data.length + 2 + 8 + 4);
+                // add content
+                buf.set(data);
+                // add typesLen
+                buf.writeDoubleLE(0, data.length);
+                // add dirty key
+                buf.writeDoubleLE(blockKey, data.length + 2);
+                // add dataLen
+                buf.writeUint32LE(data.length, buf.length - 4);
+                this.modify(buf);
+            }
+        }
+        this.onSchemaChange?.(this.schema);
+        return this.schema;
+    }
+    modify(buf) {
+        const offsets = {};
+        const dataLen = buf.readUint32LE(buf.length - 4);
+        let typesSize = buf.readUint16LE(dataLen);
+        let i = dataLen + 2;
+        while (typesSize--) {
+            const typeId = buf.readUint16LE(i);
+            i += 2;
+            const startId = buf.readUint32LE(i);
+            const def = this.schemaTypesParsedById[typeId];
+            const offset = def.lastId - startId;
+            buf.writeUint32LE(offset, i);
+            i += 4;
+            const lastId = buf.readUint32LE(i);
+            i += 4;
+            def.lastId = lastId + offset;
+            offsets[typeId] = offset;
+        }
+        if (this.processingQueries) {
+            this.modifyQueue.push(Buffer.from(buf));
+        }
+        else {
+            this.#modify(buf);
+        }
+        return offsets;
+    }
+    #modify(buf) {
+        const end = buf.length - 4;
+        const dataLen = buf.readUint32LE(end);
+        const typesSize = buf.readUint16LE(dataLen);
+        const typesLen = typesSize * 10;
+        const types = buf.subarray(dataLen + 2, dataLen + typesLen + 2);
+        const data = buf.subarray(0, dataLen);
+        let i = dataLen + 2 + typesLen;
+        while (i < end) {
+            const key = buf.readDoubleLE(i);
+            this.dirtyRanges.add(key);
+            i += 8;
+        }
+        native.modify(data, types, this.dbCtxExternal);
+    }
+    getQueryBuf(buf) {
+        if (this.modifyQueue.length) {
+            return new Promise((resolve) => {
+                this.queryQueue.set(resolve, buf);
+            });
+        }
+        else {
+            const queryType = buf[0];
+            if (queryType == 2) {
+                const s = 13 + buf.readUint16LE(11);
+                const sortLen = buf.readUint16LE(s);
+                if (sortLen) {
+                    const typeId = buf.readUint16LE(1);
+                    const sort = buf.slice(s + 2, s + 2 + sortLen);
+                    const field = sort[1];
+                    const start = sort.readUint16LE(2 + 1);
+                    if (!this.hasSortIndex(typeId, field, start)) {
+                        if (this.processingQueries) {
+                            return new Promise((resolve) => {
+                                this.queryQueue.set(resolve, buf);
+                            });
+                        }
+                        this.createSortIndexBuffer(typeId, field, start);
+                    }
+                }
+            }
+            else if (queryType == 1) {
+                // This will be more advanced - sometimes has indexes / sometimes not
+            }
+            this.availableWorkerIndex =
+                (this.availableWorkerIndex + 1) % this.workers.length;
+            return this.workers[this.availableWorkerIndex].getQueryBuf(buf);
+        }
+    }
+    onQueryEnd() {
+        this.processingQueries--;
+        if (this.processingQueries === 0) {
+            if (this.modifyQueue.length) {
+                for (const buf of this.modifyQueue) {
+                    this.#modify(buf);
+                }
+                this.modifyQueue = [];
+            }
+            if (this.queryQueue.size) {
+                for (const [resolve, buf] of this.queryQueue) {
+                    resolve(this.getQueryBuf(buf));
+                }
+                this.queryQueue.clear();
+            }
+        }
+    }
+    async stop(noSave) {
+        if (this.stopped) {
+            return;
+        }
+        if (!noSave) {
+            await this.save();
+        }
+        await Promise.all(this.workers.map(({ worker }) => worker.terminate()));
+        this.workers = [];
+        native.stop(this.dbCtxExternal);
+        await setTimeout();
+    }
+    async destroy() {
+        await this.stop(true);
+        await rm(this.fileSystemPath, { recursive: true }).catch((err) => console.warn('Error removing dump folder', err.message));
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/src/server/migrate/index.d.ts

@@ -0,0 +1,7 @@
+import { StrictSchema } from '@based/schema';
+import './worker.js';
+import { DbServer } from '../index.js';
+type TransformFn = (node: Record<string, any>) => Record<string, any> | [string, Record<string, any>];
+export type TransformFns = Record<string, TransformFn>;
+export declare const migrate: (fromDbServer: DbServer, toSchema: StrictSchema, transform?: TransformFns) => Promise<DbServer["schema"]>;
+export {};

package/dist/src/server/migrate/index.js

@@ -0,0 +1,108 @@
+import { BasedDb } from '../../index.js';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { Worker, MessageChannel } from 'node:worker_threads';
+import native from '../../native.js';
+import './worker.js';
+import { foreachDirtyBlock } from '../tree.js';
+let migrationCnt = 0;
+const parseTransform = (transform) => {
+    const res = {};
+    if (typeof transform === 'object' && transform !== null) {
+        for (const type in transform) {
+            const fn = transform[type];
+            if (typeof fn === 'function') {
+                let src = fn.toString();
+                const trimmedForCheckOnly = src.replace(/\s+/g, '');
+                if (trimmedForCheckOnly.startsWith(type + '(')) {
+                    src = 'function ' + src;
+                }
+                res[type] = src;
+            }
+        }
+    }
+    return res;
+};
+export const migrate = async (fromDbServer, toSchema, transform) => {
+    const migrationId = migrationCnt++;
+    fromDbServer.migrating = migrationId;
+    const abort = () => fromDbServer.migrating !== migrationId;
+    const toDb = new BasedDb({
+        path: join(tmpdir(), (~~Math.random()).toString(36)),
+    });
+    await toDb.start({ clean: true });
+    if (abort()) {
+        await toDb.destroy();
+        return fromDbServer.schema;
+    }
+    toSchema = await toDb.putSchema(toSchema);
+    const fromSchema = fromDbServer.schema;
+    const fromCtx = fromDbServer.dbCtxExternal;
+    const toCtx = toDb.server.dbCtxExternal;
+    const { port1, port2 } = new MessageChannel();
+    const atomics = new Int32Array(new SharedArrayBuffer(4));
+    const fromAddress = native.intFromExternal(fromCtx);
+    const toAddress = native.intFromExternal(toCtx);
+    const transformFns = parseTransform(transform);
+    atomics[0] = 1;
+    const worker = new Worker('./dist/src/server/migrate/worker.js', {
+        workerData: {
+            from: fromAddress,
+            to: toAddress,
+            fromSchema,
+            toSchema,
+            channel: port2,
+            atomics,
+            transformFns,
+        },
+        transferList: [port2],
+    });
+    worker.on('error', console.error);
+    let i = 0;
+    let ranges = [];
+    fromDbServer.updateMerkleTree();
+    fromDbServer.dirtyRanges.clear();
+    fromDbServer.merkleTree.visitLeafNodes((leaf) => {
+        ranges.push(leaf.data);
+    });
+    await Atomics.waitAsync(atomics, 0, 1).value;
+    while (i < ranges.length) {
+        // block modifies
+        fromDbServer.processingQueries++;
+        const leafData = ranges[i++];
+        port1.postMessage(leafData);
+        // wake up the worker
+        atomics[0] = 1;
+        Atomics.notify(atomics, 0);
+        // wait until it's done
+        await Atomics.waitAsync(atomics, 0, 1).value;
+        // exec queued modifies
+        fromDbServer.onQueryEnd();
+        if (abort()) {
+            break;
+        }
+        if (i === ranges.length && fromDbServer.dirtyRanges.size) {
+            ranges = [];
+            i = 0;
+            foreachDirtyBlock(fromDbServer, (_mtKey, typeId, start, end) => {
+                ranges.push({
+                    typeId,
+                    start,
+                    end,
+                });
+            });
+            fromDbServer.dirtyRanges.clear();
+        }
+    }
+    if (!abort()) {
+        fromDbServer.putSchema(toSchema, true);
+        fromDbServer.dbCtxExternal = toCtx;
+        toDb.server.dbCtxExternal = fromCtx;
+    }
+    const promises = fromDbServer.workers.map((worker) => worker.updateCtx(toAddress));
+    promises.push(toDb.destroy(), worker.terminate());
+    await Promise.all(promises);
+    fromDbServer.onSchemaChange?.(fromDbServer.schema);
+    return fromDbServer.schema;
+};
+//# sourceMappingURL=index.js.map

package/dist/src/server/migrate/worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/server/migrate/worker.js

@@ -0,0 +1,81 @@
+import { isMainThread, receiveMessageOnPort, workerData, } from 'node:worker_threads';
+import native from '../../native.js';
+import { BasedDb } from '../../index.js';
+import { REFERENCE, REFERENCES } from '../schema/types.js';
+if (isMainThread) {
+    console.warn('running worker.ts in mainthread');
+}
+else {
+    const { from, to, fromSchema, toSchema, channel, atomics, transformFns } = workerData;
+    const fromCtx = native.externalFromInt(from);
+    const toCtx = native.externalFromInt(to);
+    const path = null;
+    const fromDb = new BasedDb({ path });
+    const toDb = new BasedDb({ path });
+    fromDb.server.dbCtxExternal = fromCtx;
+    toDb.server.dbCtxExternal = toCtx;
+    await fromDb.putSchema(fromSchema, true);
+    await toDb.putSchema(toSchema, true);
+    const map = {};
+    for (const type in fromDb.client.schemaTypesParsed) {
+        const { id, props } = fromDb.client.schemaTypesParsed[type];
+        const include = Object.keys(props);
+        let i = include.length;
+        while (i--) {
+            const path = include[i];
+            if (props[path].typeIndex === REFERENCE ||
+                props[path].typeIndex === REFERENCES) {
+                include[i] = `${path}.id`;
+            }
+        }
+        map[id] = { type, include };
+    }
+    for (const type in transformFns) {
+        const fnOrNull = transformFns[type];
+        transformFns[type] = eval(`(${fnOrNull})`);
+    }
+    while (true) {
+        let msg;
+        while ((msg = receiveMessageOnPort(channel))) {
+            const leafData = msg.message;
+            const { type, include } = map[leafData.typeId];
+            const typeTransformFn = transformFns[type];
+            if (typeTransformFn) {
+                const nodes = fromDb
+                    .query(type)
+                    .include(include)
+                    .range(leafData.start - 1, leafData.end - leafData.start + 1)
+                    ._getSync(fromCtx);
+                for (const node of nodes) {
+                    const res = typeTransformFn(node);
+                    if (res === null) {
+                        continue;
+                    }
+                    if (Array.isArray(res)) {
+                        toDb.create(res[0], res[1] || node, true);
+                    }
+                    else {
+                        toDb.create(type, res || node, true);
+                    }
+                }
+            }
+            else if (type in toDb.client.schemaTypesParsed) {
+                const nodes = fromDb
+                    .query(type)
+                    .include(include)
+                    .range(leafData.start - 1, leafData.end - leafData.start + 1)
+                    ._getSync(fromCtx);
+                for (const node of nodes) {
+                    toDb.create(type, node, true);
+                }
+            }
+        }
+        await toDb.drain();
+        // await setTimeout()
+        // put it to sleep
+        atomics[0] = 0;
+        Atomics.notify(atomics, 0);
+        Atomics.wait(atomics, 0, 0);
+    }
+}
+//# sourceMappingURL=worker.js.map

package/dist/src/server/save.d.ts

@@ -0,0 +1,2 @@
+import { DbServer } from './index.js';
+export declare function save(db: DbServer): Promise<void>;

package/dist/src/server/save.js

@@ -0,0 +1,66 @@
+import native from '../native.js';
+import { writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { destructureCsmtKey, foreachDirtyBlock } from './tree.js';
+const WRITELOG_FILE = 'writelog.json';
+const COMMON_SDB_FILE = 'common.sdb';
+const block_sdb_file = (typeId, start, end) => `${typeId}_${start}_${end}.sdb`;
+export async function save(db) {
+    let err;
+    const ts = Date.now();
+    err = native.saveCommon(join(db.fileSystemPath, COMMON_SDB_FILE), db.dbCtxExternal);
+    if (err) {
+        console.error(`Save common failed: ${err}`);
+    }
+    foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
+        const file = block_sdb_file(typeId, start, end);
+        const path = join(db.fileSystemPath, file);
+        const hash = Buffer.allocUnsafe(16);
+        err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hash);
+        if (err) {
+            console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
+            return; // TODO What to do with the merkle tree in db situation?
+        }
+        const data = {
+            file,
+            typeId,
+            start,
+            end,
+        };
+        try {
+            db.merkleTree.delete(mtKey);
+        }
+        catch (err) {
+            // console.error({ err })
+        }
+        db.merkleTree.insert(mtKey, hash, data);
+    });
+    db.dirtyRanges.clear();
+    const types = {};
+    for (const key in db.schemaTypesParsed) {
+        const def = db.schemaTypesParsed[key];
+        types[def.id] = { lastId: def.lastId, blockCapacity: def.blockCapacity };
+    }
+    const dumps = {};
+    for (const key in db.schemaTypesParsed) {
+        const def = db.schemaTypesParsed[key];
+        dumps[def.id] = [];
+    }
+    db.merkleTree.visitLeafNodes((leaf) => {
+        const [typeId] = destructureCsmtKey(leaf.key);
+        const data = leaf.data;
+        dumps[typeId].push({ ...data, hash: leaf.hash.toString('hex') });
+    });
+    const data = {
+        ts,
+        types,
+        commonDump: COMMON_SDB_FILE,
+        rangeDumps: dumps,
+    };
+    const mtRoot = db.merkleTree.getRoot();
+    if (mtRoot) {
+        data.hash = mtRoot.hash.toString('hex');
+    }
+    await writeFile(join(db.fileSystemPath, WRITELOG_FILE), JSON.stringify(data));
+}
+//# sourceMappingURL=save.js.map

package/dist/src/server/schema/schema.d.ts

@@ -0,0 +1,3 @@
+export * from './types.js';
+export * from './typeDef.js';
+export * from './selvaBuffer.js';

package/dist/src/server/schema/schema.js

@@ -0,0 +1,4 @@
+export * from './types.js';
+export * from './typeDef.js';
+export * from './selvaBuffer.js';
+//# sourceMappingURL=schema.js.map

package/dist/src/server/schema/selvaBuffer.d.ts

@@ -0,0 +1,4 @@
+import { SchemaTypeDef } from './types.js';
+export declare function schemaToSelvaBuffer(schema: {
+    [key: string]: SchemaTypeDef;
+}): Buffer[];