@based/db 0.0.58 → 0.0.59

package/dist/src/hooks.js

@@ -14,12 +14,11 @@ export const getDefaultHooks = (server, subInterval = 200) => {
                     onData(res);
                 }
                 else if (res.byteLength === 1 && res[0] === 0) {
-                    console.info('schema mismatch, should resolve after update');
-                    // ignore update and stop polling
+                    server.emit('info', `[${displayTarget(q.def)}] Subscribe schema mismatch - should resolve after update`);
                     return;
                 }
                 else {
-                    const def = this.def;
+                    const def = q.def;
                     let name = picocolors.red(`QueryError[${displayTarget(def)}]\n`);
                     name += `  Incorrect buffer received in subscription (maybe server not started ${res.byteLength}) bytes\n`;
                     onError(new Error(name));

package/dist/src/index.d.ts

@@ -4,6 +4,7 @@ import { DbServer } from './server/index.js';
 import { DbClient } from './client/index.js';
 import { getDefaultHooks } from './hooks.js';
 import { Emitter } from './shared/Emitter.js';
+import { BasedDbOpts } from './types.js';
 export * from './client/modify/modify.js';
 export { compress, decompress };
 export { ModifyCtx };
@@ -24,12 +25,7 @@ export declare class BasedDb extends Emitter {
     server: DbServer;
     fileSystemPath: string;
     maxModifySize: number;
-    constructor(opts: {
-        path: string;
-        maxModifySize?: number;
-        debug?: boolean | 'server' | 'client';
-        saveIntervalInSeconds?: number;
-    });
+    constructor(opts: BasedDbOpts);
     create: DbClient['create'];
     copy: DbClient['copy'];
     update: DbClient['update'];

package/dist/src/server/DbWorker.d.ts

@@ -1,11 +1,12 @@
 import { Worker, MessagePort } from 'node:worker_threads';
 import { DbServer } from './index.js';
 export declare class DbWorker {
-    constructor(address: BigInt, db: DbServer);
+    constructor(address: BigInt, db: DbServer, workerIndex: number);
     db: DbServer;
     channel: MessagePort;
     worker: Worker;
     resolvers: any[];
+    readyPromise: Promise<true>;
     callback: (resolve: any) => void;
     updateCtx(address: BigInt): Promise<void>;
     getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;

package/dist/src/server/DbWorker.js

@@ -5,7 +5,7 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const workerPath = join(__dirname, 'worker.js');
 export class DbWorker {
-    constructor(address, db) {
+    constructor(address, db, workerIndex) {
         const { port1, port2 } = new MessageChannel();
         this.db = db;
         this.channel = port1;
@@ -17,6 +17,30 @@ export class DbWorker {
             },
             transferList: [port2],
         });
+        this.readyPromise = new Promise((resolve) => {
+            const onReady = (msg) => {
+                if (msg === 'READY') {
+                    this.worker.off('message', onReady);
+                    resolve(true);
+                }
+            };
+            this.worker.on('message', onReady);
+        });
+        this.worker.on('error', (err) => {
+            console.error('error in query worker:', err);
+            this.worker.terminate();
+        });
+        this.worker.on('exit', (code) => {
+            if (!this.db.stopped) {
+                console.info('unexpected exit query worker with code:', code);
+                const err = new Error('Worker could not process query');
+                for (const resolve of this.resolvers) {
+                    resolve(err);
+                }
+                this.resolvers = [];
+                this.db.workers[workerIndex] = new DbWorker(address, db, workerIndex);
+            }
+        });
         port1.on('message', (buf) => {
             this.resolvers.shift()(new Uint8Array(buf));
             this.db.onQueryEnd();
@@ -26,6 +50,7 @@ export class DbWorker {
     channel;
     worker;
     resolvers = [];
+    readyPromise;
     callback = (resolve) => {
         this.db.processingQueries++;
         this.resolvers.push(resolve);

package/dist/src/server/index.d.ts

@@ -1,5 +1,6 @@
 import { LangName, StrictSchema } from '@based/schema';
 import { createTree } from './csmt/index.js';
+import { StartOpts } from './start.js';
 import { CsmtNodeRange } from './tree.js';
 import { TransformFns } from './migrate/index.js';
 import exitHook from 'exit-hook';
@@ -17,6 +18,7 @@ export declare class DbServer extends DbShared {
     modifyDirtyRanges: Float64Array;
     dbCtxExternal: any;
     migrating: number;
+    saveInProgress: boolean;
     fileSystemPath: string;
     merkleTree: ReturnType<typeof createTree<CsmtNodeRange>>;
     dirtyRanges: Set<number>;
@@ -34,15 +36,13 @@ export declare class DbServer extends DbShared {
     unlistenExit: ReturnType<typeof exitHook>;
     saveIntervalInSeconds?: number;
     saveInterval?: NodeJS.Timeout;
+    delayInMs?: number;
     constructor({ path, debug, saveIntervalInSeconds, }: {
         path: string;
         debug?: boolean;
         saveIntervalInSeconds?: number;
     });
-    start(opts?: {
-        clean?: boolean;
-        hosted?: boolean;
-    }): Promise<void>;
+    start(opts?: StartOpts): Promise<void>;
     save(opts?: {
         forceFullDump?: boolean;
     }): Promise<void>;

package/dist/src/server/index.js

@@ -28,6 +28,7 @@ export class DbServer extends DbShared {
     modifyDirtyRanges;
     dbCtxExternal; // pointer to zig dbCtx
     migrating = null;
+    saveInProgress = false;
     fileSystemPath;
     merkleTree;
     dirtyRanges = new Set();
@@ -41,6 +42,7 @@ export class DbServer extends DbShared {
     unlistenExit;
     saveIntervalInSeconds;
     saveInterval;
+    delayInMs;
     constructor({ path, debug, saveIntervalInSeconds, }) {
         super();
         this.fileSystemPath = path;

package/dist/src/server/migrate/index.js

@@ -9,7 +9,6 @@ import { setToAwake, waitUntilSleeping } from './utils.js';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const workerPath = join(__dirname, 'worker.js');
-let migrationCnt = 0;
 const parseTransform = (transform) => {
     const res = {};
     if (typeof transform === 'object' && transform !== null) {
@@ -37,13 +36,20 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
             server.emit('info', `migration killed something went wrong ${migrationId}`);
             return true;
         }
-        server.emit('info', `abort migration - migrating: ${server.migrating} abort: ${migrationId}`);
-        return server.migrating !== migrationId;
+        const newMigrationInProgress = server.migrating !== migrationId;
+        if (newMigrationInProgress) {
+            server.emit('info', `abort migration - migrating: ${server.migrating} abort: ${migrationId}`);
+        }
+        return newMigrationInProgress;
     };
     const tmpDb = new BasedDb({
         path: null,
     });
-    await tmpDb.start({ clean: true });
+    await tmpDb.start({
+        clean: true,
+        delayInMs: server.delayInMs,
+        queryThreads: 0,
+    });
     if (abort()) {
         await tmpDb.destroy();
         return;

package/dist/src/server/save.js

@@ -27,95 +27,128 @@ export function save(db, sync = false, forceFullDump = false, skipMigrationCheck
         return;
     }
     if (db.migrating && !skipMigrationCheck) {
+        db.emit('info', 'Block save db is migrating');
         return;
     }
-    let err;
-    const ts = Date.now();
-    err = native.saveCommon(join(db.fileSystemPath, COMMON_SDB_FILE), db.dbCtxExternal);
-    if (err) {
-        console.error(`Save common failed: ${err}`);
+    if (db.saveInProgress) {
+        db.emit('info', 'Already have a save in progress cancel save');
+        return;
     }
-    if (forceFullDump) {
-        // We just rebuild the whole tree
-        initCsmt(db);
-        for (const key in db.schemaTypesParsed) {
-            const def = db.schemaTypesParsed[key];
-            foreachBlock(db, def, (start, end, _hash) => {
-                const typeId = def.id;
-                const mtKey = makeCsmtKey(typeId, start);
+    let ts = Date.now();
+    db.saveInProgress = true;
+    try {
+        let err;
+        err = native.saveCommon(join(db.fileSystemPath, COMMON_SDB_FILE), db.dbCtxExternal);
+        if (err) {
+            console.error(`Save common failed: ${err}`);
+            // Return ?
+        }
+        if (forceFullDump) {
+            // We just rebuild the whole tree
+            initCsmt(db);
+            for (const key in db.schemaTypesParsed) {
+                const def = db.schemaTypesParsed[key];
+                foreachBlock(db, def, (start, end, _hash) => {
+                    const typeId = def.id;
+                    const mtKey = makeCsmtKey(typeId, start);
+                    const hash = new Uint8Array(16);
+                    const file = saveRange(db, typeId, start, end, hash);
+                    if (file === null) {
+                        throw new Error('full dump failed');
+                    }
+                    else {
+                        const data = {
+                            file,
+                            typeId,
+                            start,
+                            end,
+                        };
+                        db.merkleTree.insert(mtKey, hash, data);
+                    }
+                });
+            }
+        }
+        else {
+            foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
                 const hash = new Uint8Array(16);
                 const file = saveRange(db, typeId, start, end, hash);
                 if (file === null) {
-                    throw new Error('full dump failed');
+                    // The previous state should remain in the merkle tree for
+                    // load and sync purposes.
+                    return;
                 }
                 else {
-                    const data = {
-                        file,
-                        typeId,
-                        start,
-                        end,
-                    };
-                    db.merkleTree.insert(mtKey, hash, data);
+                    const oldLeaf = db.merkleTree.search(mtKey);
+                    // If (file.length === 0) then the range is empty but that's fine,
+                    // we'll keep them around for now to maintain the order of the tree.
+                    if (oldLeaf) {
+                        oldLeaf.data.file = file;
+                        db.merkleTree.update(mtKey, hash);
+                    }
+                    else {
+                        const data = {
+                            file,
+                            typeId,
+                            start,
+                            end,
+                        };
+                        db.merkleTree.insert(mtKey, hash, data);
+                    }
                 }
             });
         }
-    }
-    else {
-        foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
-            const hash = new Uint8Array(16);
-            const file = saveRange(db, typeId, start, end, hash);
-            if (file === null) {
-                // The previous state should remain in the merkle tree for
-                // load and sync purposes.
-                return;
-            }
-            else {
-                const oldLeaf = db.merkleTree.search(mtKey);
-                // If (file.length === 0) then the range is empty but that's fine,
-                // we'll keep them around for now to maintain the order of the tree.
-                if (oldLeaf) {
-                    oldLeaf.data.file = file;
-                    db.merkleTree.update(mtKey, hash);
-                }
-                else {
-                    const data = {
-                        file,
-                        typeId,
-                        start,
-                        end,
-                    };
-                    db.merkleTree.insert(mtKey, hash, data);
-                }
+        db.dirtyRanges.clear();
+        const types = {};
+        const rangeDumps = {};
+        for (const key in db.schemaTypesParsed) {
+            const { id, lastId, blockCapacity } = db.schemaTypesParsed[key];
+            types[id] = { lastId, blockCapacity };
+            rangeDumps[id] = [];
+        }
+        db.merkleTree.visitLeafNodes((leaf) => {
+            const [typeId, start] = destructureCsmtKey(leaf.key);
+            if (start == specialBlock)
+                return; // skip the type specialBlock
+            const data = leaf.data;
+            if (start != data.start) {
+                console.error(`csmtKey start and range start mismatch: ${start} != ${data.start}`);
             }
+            rangeDumps[typeId].push({ ...data, hash: bufToHex(leaf.hash) });
         });
+        const data = {
+            ts,
+            types,
+            commonDump: COMMON_SDB_FILE,
+            rangeDumps,
+            hash: bufToHex(db.merkleTree.getRoot()?.hash ?? new Uint8Array(0)),
+        };
+        const filePath = join(db.fileSystemPath, WRITELOG_FILE);
+        const content = JSON.stringify(data);
+        db.emit('info', `Save done took ${Date.now() - ts}ms`);
+        if (sync) {
+            db.saveInProgress = false;
+            return writeFileSync(filePath, content);
+        }
+        else {
+            return new Promise((resolve, reject) => {
+                return writeFile(filePath, content)
+                    .then((v) => {
+                    db.saveInProgress = false;
+                    resolve(v);
+                })
+                    .catch((err) => {
+                    console.error('Save: writing writeLog failed');
+                    db.emit('info', `Save: writing writeLog failed ${err.message}`);
+                    db.saveInProgress = false;
+                    reject(err);
+                });
+            });
+        }
     }
-    db.dirtyRanges.clear();
-    const types = {};
-    const rangeDumps = {};
-    for (const key in db.schemaTypesParsed) {
-        const { id, lastId, blockCapacity } = db.schemaTypesParsed[key];
-        types[id] = { lastId, blockCapacity };
-        rangeDumps[id] = [];
+    catch (err) {
+        db.emit('info', `Save failed ${err.message}`);
+        db.saveInProgress = false;
+        throw err;
     }
-    db.merkleTree.visitLeafNodes((leaf) => {
-        const [typeId, start] = destructureCsmtKey(leaf.key);
-        if (start == specialBlock)
-            return; // skip the type specialBlock
-        const data = leaf.data;
-        if (start != data.start) {
-            console.error(`csmtKey start and range start mismatch: ${start} != ${data.start}`);
-        }
-        rangeDumps[typeId].push({ ...data, hash: bufToHex(leaf.hash) });
-    });
-    const data = {
-        ts,
-        types,
-        commonDump: COMMON_SDB_FILE,
-        rangeDumps,
-        hash: bufToHex(db.merkleTree.getRoot()?.hash ?? new Uint8Array(0)),
-    };
-    const filePath = join(db.fileSystemPath, WRITELOG_FILE);
-    const content = JSON.stringify(data);
-    return sync ? writeFileSync(filePath, content) : writeFile(filePath, content);
 }
 //# sourceMappingURL=save.js.map

package/dist/src/server/start.d.ts

@@ -1,5 +1,8 @@
 import { DbServer } from './index.js';
-export declare function start(db: DbServer, opts: {
+export type StartOpts = {
     clean?: boolean;
     hosted?: boolean;
-}): Promise<void>;
+    delayInMs?: number;
+    queryThreads?: number;
+};
+export declare function start(db: DbServer, opts: StartOpts): Promise<void>;

package/dist/src/server/start.js

@@ -7,7 +7,7 @@ import { availableParallelism } from 'node:os';
 import exitHook from 'exit-hook';
 import { save } from './save.js';
 import { DEFAULT_BLOCK_CAPACITY } from '@based/schema/def';
-import { bufToHex } from '@saulx/utils';
+import { bufToHex, wait } from '@saulx/utils';
 import { SCHEMA_FILE, WRITELOG_FILE } from '../types.js';
 import { setSchemaOnServer } from './schema.js';
 export async function start(db, opts) {
@@ -46,9 +46,8 @@ export async function start(db, opts) {
         }
         const schema = await readFile(join(path, SCHEMA_FILE));
         if (schema) {
-            // Prop need to not call setting in selva
-            setSchemaOnServer(db, JSON.parse(schema.toString()));
-            // setNativeSchema might still be nessecary...
+            const s = JSON.parse(schema.toString());
+            setSchemaOnServer(db, s);
         }
     }
     catch (err) {
@@ -58,7 +57,6 @@ export async function start(db, opts) {
     for (const key in csmtTypes) {
         const def = csmtTypes[key];
         const [total, lastId] = native.getTypeInfo(def.id, db.dbCtxExternal);
-        def.total = total;
         def.lastId = writelog?.types[def.id]?.lastId || lastId;
         def.blockCapacity =
             writelog?.types[def.id]?.blockCapacity || DEFAULT_BLOCK_CAPACITY;
@@ -100,11 +98,11 @@ export async function start(db, opts) {
         }
     }
     // start workers
-    let i = availableParallelism();
+    const queryThreads = opts?.queryThreads ?? availableParallelism();
     const address = native.intFromExternal(db.dbCtxExternal);
-    db.workers = new Array(i);
-    while (i--) {
-        db.workers[i] = new DbWorker(address, db);
+    db.workers = [];
+    for (let i = 0; i < queryThreads; i++) {
+        db.workers.push(new DbWorker(address, db, i));
     }
     if (!opts?.hosted) {
         db.unlistenExit = exitHook((signal) => {
@@ -114,12 +112,16 @@ export async function start(db, opts) {
             // This is needed because there is no way to set the process signal mask
             // in Node.js.
             signals.forEach((sig) => process.on(sig, blockSig));
-            console.log(`Exiting with signal: ${signal}`);
+            db.emit('info', `Exiting with signal: ${signal}`);
             save(db, true);
-            console.log('Successfully saved.');
+            db.emit('info', 'Successfully saved.');
             signals.forEach((sig) => process.off(sig, blockSig));
         });
     }
+    const d = performance.now();
+    await Promise.all(db.workers.map(({ readyPromise }) => readyPromise));
+    db.emit('info', `Starting workers took ${d}ms`);
+    // use timeout
     if (db.saveIntervalInSeconds > 0) {
         db.saveInterval ??= setInterval(() => {
             save(db);
@@ -128,5 +130,9 @@ export async function start(db, opts) {
     if (db.schema) {
         db.emit('schema', db.schema);
     }
+    if (opts?.delayInMs) {
+        this.delayInMs = opts.delayInMs;
+        await wait(opts.delayInMs);
+    }
 }
 //# sourceMappingURL=start.js.map

package/dist/src/server/worker.js

@@ -1,4 +1,4 @@
-import { isMainThread, receiveMessageOnPort, workerData, } from 'node:worker_threads';
+import { isMainThread, parentPort, workerData } from 'node:worker_threads';
 import native from '../native.js';
 if (isMainThread) {
     console.warn('running query worker.ts in mainthread - incorrect');
@@ -7,7 +7,6 @@ else if (workerData?.isDbWorker) {
     let { address, channel } = workerData;
     let dbCtx = native.externalFromInt(address);
     native.workerCtxInit();
-    // const transferList = new Array(1)
     const handleMsg = (msg) => {
         try {
             if (typeof msg === 'bigint') {
@@ -18,7 +17,6 @@ else if (workerData?.isDbWorker) {
             }
             else {
                 const arrayBuf = native.getQueryBuf(msg, dbCtx);
-                // transferList[0] = arrayBuf
                 channel.postMessage(arrayBuf, [arrayBuf]);
             }
         }
@@ -27,10 +25,7 @@ else if (workerData?.isDbWorker) {
         }
     };
     channel.on('message', handleMsg);
-    const msg = receiveMessageOnPort(channel);
-    if (msg) {
-        handleMsg(msg.message);
-    }
+    parentPort.postMessage('READY');
 }
 else {
     console.info('incorrect worker db query');

package/dist/src/types.d.ts

@@ -1,3 +1,9 @@
 export declare const SCHEMA_FILE = "schema.json";
 export declare const WRITELOG_FILE = "writelog.json";
 export declare const COMMON_SDB_FILE = "common.sdb";
+export type BasedDbOpts = {
+    path: string;
+    maxModifySize?: number;
+    debug?: boolean | 'server' | 'client';
+    saveIntervalInSeconds?: number;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.58",
+  "version": "0.0.59",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",