@based/db 0.0.68 → 0.0.70

package/dist/src/server/save.js

@@ -3,7 +3,7 @@ import { isMainThread } from 'node:worker_threads';
 import { writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { VerifTree, destructureTreeKey, } from './tree.js';
-import { saveBlock, foreachBlock, foreachDirtyBlock, } from './blocks.js';
+import { saveBlock, foreachBlock, foreachDirtyBlock, saveBlocks, } from './blocks.js';
 import { writeFileSync } from 'node:fs';
 import { bufToHex } from '@saulx/utils';
 import { COMMON_SDB_FILE, WRITELOG_FILE } from '../types.js';
@@ -14,22 +14,56 @@ function hasPartialTypes(db) {
     }
     return res;
 }
-export function save(db, sync = false, forceFullDump = false, skipMigrationCheck = false) {
-    if (!(isMainThread && (db.dirtyRanges.size || forceFullDump))) {
-        return;
+function inhibitSave(db, { skipDirtyCheck, forceFullDump, skipMigrationCheck }) {
+    // RFE isMainThread needed??
+    if (!(isMainThread && (skipDirtyCheck || db.dirtyRanges.size || forceFullDump))) {
+        return true;
     }
     if (forceFullDump && hasPartialTypes(db)) {
         db.emit('error', 'forceFullDump is not allowed with partial types');
-        return;
+        return true;
     }
     if (db.migrating && !skipMigrationCheck) {
         db.emit('info', 'Block save db is migrating');
-        return;
+        return true;
     }
     if (db.saveInProgress) {
         db.emit('info', 'Already have a save in progress cancel save');
-        return;
+        return true;
+    }
+    return false;
+}
+function makeWritelog(db, ts) {
+    const types = {};
+    const rangeDumps = {};
+    for (const key in db.schemaTypesParsed) {
+        const { id, lastId, blockCapacity } = db.schemaTypesParsed[key];
+        types[id] = { lastId, blockCapacity };
+        rangeDumps[id] = [];
     }
+    db.verifTree.foreachBlock((block) => {
+        const [typeId, start] = destructureTreeKey(block.key);
+        const def = db.schemaTypesParsedById[typeId];
+        const end = start + def.blockCapacity - 1;
+        const data = {
+            file: db.verifTree.getBlockFile(block),
+            hash: bufToHex(block.hash),
+            start,
+            end,
+        };
+        rangeDumps[typeId].push(data);
+    });
+    return {
+        ts,
+        types,
+        commonDump: COMMON_SDB_FILE,
+        rangeDumps,
+        hash: bufToHex(db.verifTree.hash), // TODO `hash('hex')`
+    };
+}
+export function saveSync(db, opts = {}) {
+    if (inhibitSave(db, opts))
+        return;
     let ts = Date.now();
     db.saveInProgress = true;
     try {
@@ -39,7 +73,7 @@ export function save(db, sync = false, forceFullDump = false, skipMigrationCheck
             db.emit('error', `Save common failed: ${err}`);
             // Return ?
         }
-        if (forceFullDump) {
+        if (opts.forceFullDump) {
             // reset the state just in case
             db.verifTree = new VerifTree(db.schemaTypesParsed);
             // We use db.verifTree.types instead of db.schemaTypesParsed because it's
@@ -53,58 +87,81 @@ export function save(db, sync = false, forceFullDump = false, skipMigrationCheck
             foreachDirtyBlock(db, (_mtKey, typeId, start, end) => saveBlock(db, typeId, start, end));
         }
         db.dirtyRanges.clear();
-        const types = {};
-        const rangeDumps = {};
-        for (const key in db.schemaTypesParsed) {
-            const { id, lastId, blockCapacity } = db.schemaTypesParsed[key];
-            types[id] = { lastId, blockCapacity };
-            rangeDumps[id] = [];
-        }
-        db.verifTree.foreachBlock((block) => {
-            const [typeId, start] = destructureTreeKey(block.key);
-            const def = db.schemaTypesParsedById[typeId];
-            const end = start + def.blockCapacity - 1;
-            const data = {
-                file: db.verifTree.getBlockFile(block),
-                hash: bufToHex(block.hash),
-                start,
-                end,
-            };
-            rangeDumps[typeId].push(data);
-        });
-        const data = {
-            ts,
-            types,
-            commonDump: COMMON_SDB_FILE,
-            rangeDumps,
-            hash: bufToHex(db.verifTree.hash), // TODO `hash('hex')`
-        };
-        const filePath = join(db.fileSystemPath, WRITELOG_FILE);
+        const data = makeWritelog(db, ts);
         const content = JSON.stringify(data);
-        db.emit('info', `Save done took ${Date.now() - ts}ms`);
-        if (sync) {
-            db.saveInProgress = false;
-            return writeFileSync(filePath, content);
+        db.emit('info', `Save took ${Date.now() - ts}ms`);
+        db.saveInProgress = false;
+        return writeFileSync(join(db.fileSystemPath, WRITELOG_FILE), content);
+    }
+    catch (err) {
+        db.emit('error', `Save failed ${err.message}`);
+        db.saveInProgress = false;
+        throw err;
+    }
+}
+export async function save(db, opts = {}) {
+    if (inhibitSave(db, opts))
+        return;
+    let ts = Date.now();
+    db.saveInProgress = true;
+    try {
+        let err;
+        err = native.saveCommon(join(db.fileSystemPath, COMMON_SDB_FILE), db.dbCtxExternal);
+        if (err) {
+            db.emit('error', `Save common failed: ${err}`);
+            // Return ?
+        }
+        const blocks = [];
+        if (opts.forceFullDump) {
+            // reset the state just in case
+            db.verifTree = new VerifTree(db.schemaTypesParsed);
+            // We use db.verifTree.types instead of db.schemaTypesParsed because it's
+            // ordered.
+            for (const { typeId } of db.verifTree.types()) {
+                const def = db.schemaTypesParsedById[typeId];
+                foreachBlock(db, def, (start, end, _hash) => {
+                    const typeId = def.id;
+                    const file = VerifTree.blockSdbFile(typeId, start, end);
+                    const filepath = join(db.fileSystemPath, file);
+                    blocks.push({
+                        filepath,
+                        typeId,
+                        start,
+                    });
+                });
+            }
         }
         else {
-            return new Promise((resolve, reject) => {
-                return writeFile(filePath, content)
-                    .then((v) => {
-                    db.saveInProgress = false;
-                    resolve(v);
-                })
-                    .catch((err) => {
-                    db.emit('error', `Save: writing writeLog failed ${err.message}`);
-                    db.saveInProgress = false;
-                    reject(err);
+            foreachDirtyBlock(db, (_mtKey, typeId, start, end) => {
+                const file = VerifTree.blockSdbFile(typeId, start, end);
+                const filepath = join(db.fileSystemPath, file);
+                blocks.push({
+                    filepath,
+                    typeId,
+                    start
                 });
             });
         }
+        db.dirtyRanges.clear();
+        await saveBlocks(db, blocks);
+        try {
+            // Note that we assume here that verifTree didn't change before we call
+            // makeWritelog(). This is true as long as db.saveInProgress protects
+            // the verifTree from changes.
+            const data = makeWritelog(db, ts);
+            await writeFile(join(db.fileSystemPath, WRITELOG_FILE), JSON.stringify(data));
+        }
+        catch (err) {
+            db.emit('error', `Save: writing writeLog failed ${err.message}`);
+        }
+        db.emit('info', `Save took ${Date.now() - ts}ms`);
     }
     catch (err) {
         db.emit('error', `Save failed ${err.message}`);
-        db.saveInProgress = false;
         throw err;
     }
+    finally {
+        db.saveInProgress = false;
+    }
 }
 //# sourceMappingURL=save.js.map

package/dist/src/server/schema.d.ts

@@ -1,6 +1,6 @@
+import { StrictSchema } from '@based/schema';
 import { DbServer } from './index.js';
 import { DbSchema } from '../schema.js';
-import { StrictSchema } from '@based/schema';
 export declare const setSchemaOnServer: (server: DbServer, schema: DbSchema) => void;
 export declare const writeSchemaFile: (server: DbServer, schema: DbSchema) => Promise<void>;
 export declare const setNativeSchema: (server: DbServer, schema: DbSchema) => void;

package/dist/src/server/schema.js

@@ -1,12 +1,13 @@
 import { schemaToSelvaBuffer, updateTypeDefs } from '@based/schema/def';
+import { deepCopy, writeUint64 } from '@saulx/utils';
+import { getPropType, serialize } from '@based/schema';
 import { join } from 'node:path';
 import { writeFile } from 'node:fs/promises';
 import native from '../native.js';
 import { makeTreeKey } from './tree.js';
-import { deepCopy, writeUint64 } from '@saulx/utils';
 import { SCHEMA_FILE } from '../types.js';
+import { saveSync } from './save.js';
 import { hash } from '@saulx/hash';
-import { getPropType, serialize } from '@based/schema';
 export const setSchemaOnServer = (server, schema) => {
     const { schemaTypesParsed, schemaTypesParsedById } = updateTypeDefs(schema);
     server.schema = schema;
@@ -55,6 +56,9 @@ export const setNativeSchema = (server, schema) => {
         //server.verifTree = new VerifTree(.schemaTypesParsed)
     }
     server.verifTree.updateTypes(server.schemaTypesParsed);
+    if (server.fileSystemPath) {
+        saveSync(server, { skipDirtyCheck: true });
+    }
 };
 export const strictSchemaToDbSchema = (schema) => {
     // @ts-ignore

package/dist/src/server/start.js

@@ -7,11 +7,11 @@ import { join } from 'node:path';
 import { VerifTree, makeTreeKey } from './tree.js';
 import { foreachBlock } from './blocks.js';
 import exitHook from 'exit-hook';
-import { save } from './save.js';
+import { save, saveSync } from './save.js';
 import { deSerialize } from '@based/schema';
 import { BLOCK_CAPACITY_DEFAULT } from '@based/schema/def';
 import { bufToHex, equals, hexToBuf, wait } from '@saulx/utils';
-import { SCHEMA_FILE, WRITELOG_FILE } from '../types.js';
+import { SCHEMA_FILE, WRITELOG_FILE, SCHEMA_FILE_DEPRECATED } from '../types.js';
 import { setSchemaOnServer } from './schema.js';
 function startWorkers(db, opts) {
     const queryThreads = opts?.queryThreads ?? availableParallelism();
@@ -42,11 +42,17 @@ export async function start(db, opts) {
             console.error(e.message);
             throw e;
         }
-        const schema = await readFile(join(path, SCHEMA_FILE));
+        const schema = await readFile(join(path, SCHEMA_FILE)).catch(noop);
         if (schema) {
             const s = deSerialize(schema);
             setSchemaOnServer(db, s);
         }
+        else {
+            const schemaJson = await readFile(join(path, SCHEMA_FILE_DEPRECATED));
+            if (schemaJson) {
+                setSchemaOnServer(db, JSON.parse(schemaJson.toString()));
+            }
+        }
         // Load all range dumps
         for (const typeId in writelog.rangeDumps) {
             const dumps = writelog.rangeDumps[typeId];
@@ -111,7 +117,7 @@ export async function start(db, opts) {
             // in Node.js.
             signals.forEach((sig) => process.on(sig, blockSig));
             db.emit('info', `Exiting with signal: ${signal}`);
-            save(db, true);
+            saveSync(db);
             db.emit('info', 'Successfully saved.');
             signals.forEach((sig) => process.off(sig, blockSig));
         });
@@ -121,7 +127,7 @@ export async function start(db, opts) {
     // use timeout
     if (db.saveIntervalInSeconds > 0) {
         db.saveInterval ??= setInterval(() => {
-            void save(db);
+            save(db);
         }, db.saveIntervalInSeconds * 1e3);
     }
     if (db.schema) {

package/dist/src/server/tree.d.ts

@@ -6,8 +6,19 @@ export declare const makeTreeKeyFromNodeId: (typeId: number, blockCapacity: numb
 type Hash = Uint8Array;
 export type VerifBlock = {
     key: number;
+    /**
+     * Last acquired hash of the block.
+     * This is normally updated at load and save time but never during read/modify ops.
+     */
     hash: Hash;
+    /**
+     * If false the block is offloaded to fs;
+     * true doesn't necessarily mean that the block still exists because it could have been deleted.
+     */
     inmem: boolean;
+    /**
+     * If set, the block is being loaded and it can be awaited with this promise.
+     */
     loadPromise: null | Promise<void>;
 };
 export declare class VerifTree {

package/dist/src/server/tree.js

@@ -26,7 +26,6 @@ export class VerifTree {
             obj[typeId] = {
                 typeId,
                 blockCapacity: def.blockCapacity,
-                hash: new Uint8Array(HASH_SIZE),
                 blocks: [],
             };
             return obj;

package/dist/src/server/workers/DbWorker.d.ts

@@ -1,12 +1,13 @@
+import { MessagePort } from 'node:worker_threads';
 import { DbServer } from '../index.js';
 export declare abstract class DbWorker {
     constructor(address: BigInt, db: DbServer, onExit: (code: number) => void, workerName: string);
     protected db: DbServer;
-    private channel;
+    protected channel: MessagePort;
     private worker;
     protected resolvers: ((x: any) => any)[];
     readyPromise: Promise<true>;
-    terminate(): Promise<number>;
+    terminate(): Promise<void>;
     abstract handleMsg(buf: any): void;
     protected callback: (resolve: (x: any) => any) => void;
     /**

package/dist/src/server/workers/DbWorker.js

@@ -53,7 +53,8 @@ export class DbWorker {
     resolvers = [];
     readyPromise;
     async terminate() {
-        return this.worker.terminate();
+        // TODO do we want to force this.worker.terminate() after a timeout?
+        await this.call(0n);
     }
     callback = (resolve) => {
         this.resolvers.push(resolve);

package/dist/src/server/workers/io_worker.js

@@ -1,14 +1,8 @@
 import { registerMsgHandler } from './worker.js';
-import { ENCODER, writeInt32 } from '@saulx/utils';
+import { writeInt32 } from '@saulx/utils';
 import native from '../../native.js';
 function loadBlock(dbCtx, filepath) {
-    try {
-        native.loadBlock(filepath, dbCtx);
-    }
-    catch (e) {
-        // need to get rid of the shared buffer
-        return new Uint8Array(ENCODER.encode(e.toString())).buffer;
-    }
+    native.loadBlock(filepath, dbCtx);
     return null;
 }
 function unloadBlock(dbCtx, filepath, typeId, start) {
@@ -25,15 +19,26 @@ function unloadBlock(dbCtx, filepath, typeId, start) {
     return buf;
 }
 registerMsgHandler((dbCtx, msg) => {
-    if (typeof msg?.type === 'string') {
-        const job = msg;
-        if (job.type === 'load') {
-            return loadBlock(dbCtx, job.filepath);
-        }
-        else if (job.type === 'unload') {
-            return unloadBlock(dbCtx, job.filepath, job.typeId, job.start);
-        }
+    if (typeof msg?.type !== 'string') {
+        throw new Error('Invalid message');
     }
-    return null;
+    const job = msg;
+    if (job.type === 'save') {
+        const LEN = 20;
+        return job.blocks.reduce((buf, block, index) => {
+            const errCodeBuf = new Uint8Array(buf, index * LEN, 4);
+            const hash = new Uint8Array(buf, index * LEN + 4, 16);
+            const err = native.saveBlock(block.filepath, block.typeId, block.start, dbCtx, hash);
+            writeInt32(errCodeBuf, err, 0);
+            return buf;
+        }, new ArrayBuffer(job.blocks.length * LEN));
+    }
+    else if (job.type === 'load') {
+        return loadBlock(dbCtx, job.filepath);
+    }
+    else if (job.type === 'unload') {
+        return unloadBlock(dbCtx, job.filepath, job.typeId, job.start);
+    }
+    throw new Error(`Unsupported type: "${msg.type}"`);
 });
 //# sourceMappingURL=io_worker.js.map

package/dist/src/server/workers/io_worker_types.d.ts

@@ -1,3 +1,11 @@
+export type IoJobSave = {
+    type: 'save';
+    blocks: {
+        filepath: string;
+        typeId: number;
+        start: number;
+    }[];
+};
 type IoJobLoad = {
     type: 'load';
     filepath: string;
@@ -8,5 +16,5 @@ type IoJobUnload = {
     typeId: number;
     start: number;
 };
-export type IoJob = IoJobLoad | IoJobUnload;
+export type IoJob = IoJobSave | IoJobLoad | IoJobUnload;
 export {};

package/dist/src/server/workers/worker.js

@@ -1,4 +1,5 @@
 import { isMainThread, parentPort, workerData } from 'node:worker_threads';
+import { nextTick } from 'node:process';
 import native from '../../native.js';
 let dbCtx;
 let wCtx; // This must be held until the worker exits otherwise the ctx will be autofreed instantly
@@ -21,9 +22,16 @@ export function registerMsgHandler(onMsg) {
     const handleMsg = (msg) => {
         try {
             if (typeof msg === 'bigint') {
-                // it's a ctx address
-                dbCtx = native.externalFromInt(msg);
-                channel.postMessage(null);
+                if (msg === 0n) {
+                    // terminate
+                    nextTick(() => typeof self === 'undefined' ? process.exit() : self.close());
+                    channel.postMessage(null);
+                }
+                else {
+                    // it's a ctx address
+                    dbCtx = native.externalFromInt(msg);
+                    channel.postMessage(null);
+                }
             }
             else {
                 // a message to the worker handler

package/dist/src/types.d.ts

@@ -1,3 +1,4 @@
+export declare const SCHEMA_FILE_DEPRECATED = "schema.json";
 export declare const SCHEMA_FILE = "schema.bin";
 export declare const WRITELOG_FILE = "writelog.json";
 export declare const COMMON_SDB_FILE = "common.sdb";

package/dist/src/types.js

@@ -1,3 +1,4 @@
+export const SCHEMA_FILE_DEPRECATED = 'schema.json';
 export const SCHEMA_FILE = 'schema.bin';
 export const WRITELOG_FILE = 'writelog.json';
 export const COMMON_SDB_FILE = 'common.sdb';

package/dist/src/utils.js

@@ -42,5 +42,5 @@ export const debugMode = (target, getInfo = null) => {
         }
     }
 };
-export const debugServer = (server) => debugMode(server, () => `p: ${server.processingQueries} m: ${server.modifyQueue.length} q: ${server.queryQueue.size}`);
+export const debugServer = (server) => debugMode(server, () => `p: ${server.activeReaders} m: ${server.modifyQueue.length} q: ${server.queryQueue.size}`);
 //# sourceMappingURL=utils.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.68",
+  "version": "0.0.70",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",
@@ -20,6 +20,7 @@
     "test": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale ./scripts/lldb-node ./scripts/test.js",
     "test-gdb": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale gdb -ex run --args node ./scripts/test.js",
     "test-fast": "LOCPATH=../locale/locale-x86_64-gnu/locale node ./scripts/test.js",
+    "test-fast-linux_aarch64-schema": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; npm run test-fast schemadebug'",
     "test-fast-linux_aarch64": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; npm run test-fast'",
     "test-fast-linux_aarch64-gdb": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; LOCPATH=../locale/locale-x86_64-gnu/locale gdb -ex run --args node ./scripts/test.js'",
     "test-fast-linux_aarch64-valgrind": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; LOCPATH=../locale/locale-aarch64-gnu/locale valgrind --leak-check=full node ./scripts/test.js references:update2'",
@@ -38,7 +39,7 @@
     "basedDbNative.cjs"
   ],
   "dependencies": {
-    "@based/schema": "5.0.0-alpha.23",
+    "@based/schema": "5.0.0-alpha.26",
     "@saulx/hash": "^3.0.0",
     "@saulx/utils": "^6.7.2",
     "exit-hook": "^4.0.0",
@@ -55,6 +56,7 @@
     "axios": "^1.7.9",
     "rimraf": "^3.0.2",
     "tar": "^7.4.3",
-    "typescript": "^5.6.3"
+    "typescript": "^5.6.3",
+    "fs-extra": "^11.1.1"
   }
 }