@based/db 0.0.66 → 0.0.68

package/dist/src/server/blocks.js

@@ -0,0 +1,112 @@
+import native from '../native.js';
+import { join } from 'node:path';
+import { equals } from '@saulx/utils';
+import { VerifTree, destructureTreeKey, makeTreeKey, } from './tree.js';
+/**
+ * Save a block.
+ */
+export function saveBlock(db, typeId, start, end) {
+    const hash = new Uint8Array(16);
+    const mtKey = makeTreeKey(typeId, start);
+    const file = VerifTree.blockSdbFile(typeId, start, end);
+    const path = join(db.fileSystemPath, file);
+    const err = native.saveBlock(path, typeId, start, db.dbCtxExternal, hash);
+    if (err == -8) {
+        // TODO ENOENT
+        db.verifTree.remove(mtKey);
+    }
+    else if (err) {
+        // TODO print the error string
+        console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
+    }
+    else {
+        db.verifTree.update(mtKey, hash);
+    }
+}
+/**
+ * Load a block (typically of a partial type) back to memory.
+ */
+export async function loadBlock(db, def, start) {
+    const key = makeTreeKey(def.id, start);
+    const block = db.verifTree.getBlock(key);
+    if (!block) {
+        throw new Error(`No such block: ${key}`);
+    }
+    if (block.loadPromise) {
+        return block.loadPromise;
+    }
+    const prevHash = block.hash;
+    const filename = db.verifTree.getBlockFile(block);
+    const p = db.ioWorker.loadBlock(join(db.fileSystemPath, filename));
+    block.loadPromise = p;
+    await p;
+    delete block.loadPromise;
+    // Update and verify the hash
+    const hash = new Uint8Array(16);
+    const end = start + def.blockCapacity - 1;
+    const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
+    if (res) {
+        const key = makeTreeKey(def.id, start);
+        db.verifTree.update(key, hash);
+        if (!equals(prevHash, hash)) {
+            throw new Error('Block hash mismatch');
+        }
+    }
+}
+/**
+ * Save a block and remove it from memory.
+ */
+export async function unloadBlock(db, def, start) {
+    const typeId = def.id;
+    const end = start + def.blockCapacity - 1;
+    const key = makeTreeKey(typeId, start);
+    const block = db.verifTree.getBlock(key);
+    if (!block) {
+        throw new Error(`No such block: ${key}`);
+    }
+    const filepath = join(db.fileSystemPath, VerifTree.blockSdbFile(typeId, start, end));
+    try {
+        const hash = await db.ioWorker.unloadBlock(filepath, typeId, start);
+        native.delBlock(db.dbCtxExternal, typeId, start);
+        db.verifTree.update(key, hash, false);
+    }
+    catch (e) {
+        // TODO Proper logging
+        // TODO err == -8 == SELVA_ENOENT => db.verifTree.remove(key) ??
+        console.error(`Save ${typeId}:${start}-${end} failed`);
+        console.error(e);
+    }
+}
+/**
+ * Execute cb() for each block in memory.
+ */
+export function foreachBlock(db, def, cb, includeEmptyBlocks = false) {
+    const step = def.blockCapacity;
+    for (let start = 1; start <= def.lastId; start += step) {
+        const end = start + step - 1;
+        const hash = new Uint8Array(16);
+        const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
+        if (res || includeEmptyBlocks) {
+            cb(start, end, hash);
+        }
+    }
+}
+/**
+ * Execute cb() for each dirty block.
+ * A dirty block is one that is changed in memory but not yet persisted in the
+ * file system.
+ */
+export function foreachDirtyBlock(db, cb) {
+    const typeIdMap = {};
+    for (const typeName in db.schemaTypesParsed) {
+        const type = db.schemaTypesParsed[typeName];
+        const typeId = type.id;
+        typeIdMap[typeId] = type;
+    }
+    for (const mtKey of db.dirtyRanges) {
+        const [typeId, start] = destructureTreeKey(mtKey);
+        const end = start + typeIdMap[typeId].blockCapacity - 1;
+        cb(mtKey, typeId, start, end);
+    }
+}
+//# sourceMappingURL=blocks.js.map

package/dist/src/server/dbHash.d.ts

@@ -1,5 +1,5 @@
 export default function createHash(): {
-    update: (buf: Uint8Array) => any;
+    update: (buf: Uint8Array) => /*elided*/ any;
     digest: (encoding?: "hex") => Uint8Array | string;
     reset: () => void;
 };

package/dist/src/server/index.d.ts

@@ -1,11 +1,11 @@
 import { LangName, StrictSchema } from '@based/schema';
-import { createTree } from './csmt/index.js';
 import { StartOpts } from './start.js';
-import { CsmtNodeRange } from './tree.js';
+import { VerifTree } from './tree.js';
 import { TransformFns } from './migrate/index.js';
 import exitHook from 'exit-hook';
 import { SchemaChecksum } from '../schema.js';
-import { DbWorker } from './DbWorker.js';
+import { IoWorker } from './IoWorker.js';
+import { QueryWorker } from './QueryWorker.js';
 import { DbShared } from '../shared/DbBase.js';
 declare class SortIndex {
     constructor(buf: Uint8Array, dbCtxExternal: any);
@@ -21,14 +21,10 @@ export declare class DbServer extends DbShared {
     migrating: number;
     saveInProgress: boolean;
     fileSystemPath: string;
-    merkleTree: ReturnType<typeof createTree<CsmtNodeRange>>;
+    verifTree: VerifTree;
     dirtyRanges: Set<number>;
-    csmtHashFun: {
-        update: (buf: Uint8Array) => any;
-        digest: (encoding?: "hex") => Uint8Array | string;
-        reset: () => void;
-    };
-    workers: DbWorker[];
+    ioWorker: IoWorker;
+    workers: QueryWorker[];
     availableWorkerIndex: number;
     processingQueries: number;
     modifyQueue: Uint8Array[];
@@ -47,11 +43,8 @@ export declare class DbServer extends DbShared {
     save(opts?: {
         forceFullDump?: boolean;
     }): Promise<void>;
-    createCsmtHashFun: () => {
-        update: (buf: Uint8Array) => any;
-        digest: (encoding?: "hex") => Uint8Array | string;
-        reset: () => void;
-    };
+    loadBlock(typeName: string, nodeId: number): Promise<void>;
+    unloadBlock(typeName: string, nodeId: number): Promise<void>;
     sortIndexes: {
         [type: number]: {
             [field: number]: {
@@ -69,7 +62,7 @@ export declare class DbServer extends DbShared {
     createSortIndexBuffer(typeId: number, field: number, start: number, lang: number): SortIndex;
     setSchema(strictSchema: StrictSchema, transformFns?: TransformFns): Promise<SchemaChecksum>;
     modify(bufWithHash: Uint8Array): Record<number, number> | null;
-    addToQueryQueue(resolve: any, buf: any): Promise<Uint8Array>;
+    addToQueryQueue(resolve: any, buf: any): Promise<Uint8Array<ArrayBuffer>>;
     getQueryBuf(buf: Uint8Array, fromQueue?: boolean): Promise<Uint8Array>;
     onQueryEnd(): any;
     stop(noSave?: boolean): Promise<void>;

package/dist/src/server/index.js

@@ -1,9 +1,8 @@
 import native from '../native.js';
-import createDbHash from './dbHash.js';
 import { rm } from 'node:fs/promises';
 import { langCodesMap } from '@based/schema';
 import { start } from './start.js';
-import { makeCsmtKeyFromNodeId } from './tree.js';
+import { destructureTreeKey, makeTreeKeyFromNodeId } from './tree.js';
 import { save } from './save.js';
 import { setTimeout } from 'node:timers/promises';
 import { migrate } from './migrate/index.js';
@@ -14,6 +13,7 @@ import { strictSchemaToDbSchema } from './schema.js';
 import { DbShared } from '../shared/DbBase.js';
 import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from './schema.js';
 import { resizeModifyDirtyRanges } from './resizeModifyDirtyRanges.js';
+import { loadBlock, unloadBlock } from './blocks.js';
 const emptyUint8Array = new Uint8Array(0);
 class SortIndex {
     constructor(buf, dbCtxExternal) {
@@ -31,12 +31,12 @@ export class DbServer extends DbShared {
     migrating = null;
     saveInProgress = false;
     fileSystemPath;
-    merkleTree;
+    verifTree;
     dirtyRanges = new Set();
-    csmtHashFun = createDbHash();
+    ioWorker;
     workers = [];
     availableWorkerIndex = -1;
-    processingQueries = 0;
+    processingQueries = 0; // semaphore for locking writes
     modifyQueue = [];
     queryQueue = new Map();
     stopped; // = true does not work
@@ -60,11 +60,34 @@ export class DbServer extends DbShared {
     save(opts) {
         return save(this, false, opts?.forceFullDump ?? false);
     }
-    createCsmtHashFun = () => {
-        // We can just reuse it as long as we only have one tree.
-        this.csmtHashFun.reset();
-        return this.csmtHashFun;
-    };
+    async loadBlock(typeName, nodeId) {
+        const def = this.schemaTypesParsed[typeName];
+        if (!def) {
+            throw new Error('Type not found');
+        }
+        const typeId = def.id;
+        const key = makeTreeKeyFromNodeId(typeId, def.blockCapacity, nodeId);
+        const [, start] = destructureTreeKey(key);
+        const block = this.verifTree.getBlock(key);
+        if (!block) {
+            throw new Error('Block not found');
+        }
+        await loadBlock(this, def, start);
+    }
+    async unloadBlock(typeName, nodeId) {
+        const def = this.schemaTypesParsed[typeName];
+        if (!def) {
+            throw new Error('Type not found');
+        }
+        const typeId = def.id;
+        const key = makeTreeKeyFromNodeId(typeId, def.blockCapacity, nodeId);
+        const [, start] = destructureTreeKey(key);
+        const block = this.verifTree.getBlock(key);
+        if (!block) {
+            throw new Error('Block not found');
+        }
+        await unloadBlock(this, def, start);
+    }
     sortIndexes;
     cleanupTimer;
     cleanup() {
@@ -299,7 +322,7 @@ export class DbServer extends DbShared {
         while (typesSize--) {
             const typeId = readUint16(buf, i);
             const def = this.schemaTypesParsedById[typeId];
-            const key = makeCsmtKeyFromNodeId(def.id, def.blockCapacity, def.lastId);
+            const key = makeTreeKeyFromNodeId(def.id, def.blockCapacity, def.lastId);
             this.dirtyRanges.add(key);
             i += 10;
         }
@@ -307,7 +330,7 @@ export class DbServer extends DbShared {
         while (i < end) {
             // const key = view.getFloat64(i, true)
             // These node ranges may not actually exist
-            //this.dirtyRanges.add(key)
+            // this.dirtyRanges.add(key)
             i += 8;
         }
         resizeModifyDirtyRanges(this);
@@ -334,7 +357,6 @@ export class DbServer extends DbShared {
             resolve(new Error('Query queue exceeded'));
             return;
         }
-        // TODO should we check here as well? Already will check in DbWorker
         const schemaChecksum = readUint64(buf, buf.byteLength - 8);
         if (schemaChecksum !== this.schema?.hash) {
             return Promise.resolve(new Uint8Array(1));
@@ -389,7 +411,6 @@ export class DbServer extends DbShared {
         }
     }
     onQueryEnd() {
-        this.processingQueries--;
         if (this.processingQueries === 0) {
             if (this.modifyQueue.length) {
                 const modifyQueue = this.modifyQueue;
@@ -432,7 +453,8 @@ export class DbServer extends DbShared {
             if (!noSave) {
                 await this.save();
             }
-            await Promise.all(this.workers.map(({ worker }) => worker.terminate()));
+            await this.ioWorker.terminate();
+            await Promise.all(this.workers.map((worker) => worker.terminate()));
             this.workers = [];
             native.stop(this.dbCtxExternal);
             await setTimeout(100);

package/dist/src/server/migrate/index.d.ts

@@ -1,5 +1,10 @@
 import { DbServer } from '../index.js';
 import { DbSchema } from '../../schema.js';
+export type MigrateRange = {
+    typeId: number;
+    start: number;
+    end: number;
+};
 type TransformFn = (node: Record<string, any>) => Record<string, any> | [string, Record<string, any>];
 export type TransformFns = Record<string, TransformFn>;
 export declare const migrate: (server: DbServer, fromSchema: DbSchema, toSchema: DbSchema, transform?: TransformFns) => Promise<void>;

package/dist/src/server/migrate/index.js

@@ -2,7 +2,8 @@ import { BasedDb, save } from '../../index.js';
 import { dirname, join } from 'path';
 import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threads';
 import native from '../../native.js';
-import { destructureCsmtKey, foreachDirtyBlock, specialBlock } from '../tree.js';
+import { destructureTreeKey } from '../tree.js';
+import { foreachDirtyBlock } from '../blocks.js';
 import { fileURLToPath } from 'url';
 import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from '../schema.js';
 import { setToAwake, waitUntilSleeping } from './utils.js';
@@ -90,11 +91,11 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
     let i = 0;
     let rangesToMigrate = [];
     await save(server, false, false, true);
-    server.merkleTree.visitLeafNodes((leaf) => {
-        const [_typeId, start] = destructureCsmtKey(leaf.key);
-        if (start == specialBlock)
-            return; // skip the type specialBlock
-        rangesToMigrate.push(leaf.data);
+    server.verifTree.foreachBlock((block) => {
+        const [typeId, start] = destructureTreeKey(block.key);
+        const def = server.schemaTypesParsedById[typeId];
+        const end = start + def.blockCapacity - 1;
+        rangesToMigrate.push({ typeId, start, end });
     });
     await waitUntilSleeping(workerState);
     while (i < rangesToMigrate.length) {
@@ -108,12 +109,13 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
         setToAwake(workerState, true);
         await waitUntilSleeping(workerState);
         // exec queued modifies
+        server.processingQueries--;
         server.onQueryEnd();
         if (i === rangesToMigrate.length) {
             if (server.dirtyRanges.size) {
                 rangesToMigrate = [];
                 i = 0;
-                void foreachDirtyBlock(server, (_mtKey, typeId, start, end) => {
+                foreachDirtyBlock(server, (_mtKey, typeId, start, end) => {
                     rangesToMigrate.push({
                         typeId,
                         start,
@@ -140,6 +142,7 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
     // pass last node IDS { type: lastId }
     setSchemaOnServer(server, toSchema);
     for (const key in schemaTypesParsed) {
+        // maybe only send the lastId
         const def = server.schemaTypesParsed[key];
         def.lastId = schemaTypesParsed[key].lastId;
     }

package/dist/src/server/save.d.ts

@@ -1,4 +1,10 @@
 import { DbServer } from './index.js';
+type RangeDump = {
+    file: string;
+    hash: string;
+    start: number;
+    end: number;
+};
 export type Writelog = {
     ts: number;
     types: {
@@ -10,12 +16,8 @@ export type Writelog = {
     hash: string;
     commonDump: string;
     rangeDumps: {
-        [t: number]: {
-            file: string;
-            hash: string;
-            start: number;
-            end: number;
-        }[];
+        [t: number]: RangeDump[];
     };
 };
 export declare function save<T extends boolean>(db: DbServer, sync?: T, forceFullDump?: boolean, skipMigrationCheck?: boolean): T extends true ? void : Promise<void>;
+export {};

package/dist/src/server/save.js

@@ -2,30 +2,26 @@ import native from '../native.js';
 import { isMainThread } from 'node:worker_threads';
 import { writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, initCsmt, makeCsmtKey, specialBlock, } from './tree.js';
+import { VerifTree, destructureTreeKey, } from './tree.js';
+import { saveBlock, foreachBlock, foreachDirtyBlock, } from './blocks.js';
 import { writeFileSync } from 'node:fs';
 import { bufToHex } from '@saulx/utils';
 import { COMMON_SDB_FILE, WRITELOG_FILE } from '../types.js';
-const block_sdb_file = (typeId, start, end) => `${typeId}_${start}_${end}.sdb`;
-function saveBlock(db, typeId, start, end, hashOut) {
-    const file = block_sdb_file(typeId, start, end);
-    const path = join(db.fileSystemPath, file);
-    const err = native.saveBlock(path, typeId, start, db.dbCtxExternal, hashOut);
-    if (err == -8) {
-        // TODO ENOENT
-        return ''; // empty range
+function hasPartialTypes(db) {
+    let res = false;
+    for (let id in db.schemaTypesParsedById) {
+        res = res || db.schemaTypesParsedById[id].partial;
     }
-    else if (err) {
-        // TODO print the error string
-        console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
-        return null;
-    }
-    return file;
+    return res;
 }
 export function save(db, sync = false, forceFullDump = false, skipMigrationCheck = false) {
     if (!(isMainThread && (db.dirtyRanges.size || forceFullDump))) {
         return;
     }
+    if (forceFullDump && hasPartialTypes(db)) {
+        db.emit('error', 'forceFullDump is not allowed with partial types');
+        return;
+    }
     if (db.migrating && !skipMigrationCheck) {
         db.emit('info', 'Block save db is migrating');
         return;
@@ -40,62 +36,21 @@ export function save(db, sync = false, forceFullDump = false, skipMigrationCheck
         let err;
         err = native.saveCommon(join(db.fileSystemPath, COMMON_SDB_FILE), db.dbCtxExternal);
         if (err) {
-            console.error(`Save common failed: ${err}`);
+            db.emit('error', `Save common failed: ${err}`);
             // Return ?
         }
         if (forceFullDump) {
-            // We just rebuild the whole tree
-            initCsmt(db);
-            for (const key in db.schemaTypesParsed) {
-                const def = db.schemaTypesParsed[key];
-                foreachBlock(db, def, (start, end, _hash) => {
-                    const typeId = def.id;
-                    const mtKey = makeCsmtKey(typeId, start);
-                    const hash = new Uint8Array(16);
-                    const file = saveBlock(db, typeId, start, end, hash);
-                    if (file === null) {
-                        throw new Error('full dump failed');
-                    }
-                    else {
-                        const data = {
-                            file,
-                            typeId,
-                            start,
-                            end,
-                        };
-                        db.merkleTree.insert(mtKey, hash, data);
-                    }
-                });
+            // reset the state just in case
+            db.verifTree = new VerifTree(db.schemaTypesParsed);
+            // We use db.verifTree.types instead of db.schemaTypesParsed because it's
+            // ordered.
+            for (const { typeId } of db.verifTree.types()) {
+                const def = db.schemaTypesParsedById[typeId];
+                foreachBlock(db, def, (start, end, _hash) => saveBlock(db, def.id, start, end));
             }
         }
         else {
-            void foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
-                const hash = new Uint8Array(16);
-                const file = saveBlock(db, typeId, start, end, hash);
-                if (file === null) {
-                    // The previous state should remain in the merkle tree for
-                    // load and sync purposes.
-                    return;
-                }
-                else {
-                    const oldLeaf = db.merkleTree.search(mtKey);
-                    // If (file.length === 0) then the range is empty but that's fine,
-                    // we'll keep them around for now to maintain the order of the tree.
-                    if (oldLeaf) {
-                        oldLeaf.data.file = file;
-                        db.merkleTree.update(mtKey, hash);
-                    }
-                    else {
-                        const data = {
-                            file,
-                            typeId,
-                            start,
-                            end,
-                        };
-                        db.merkleTree.insert(mtKey, hash, data);
-                    }
-                }
-            });
+            foreachDirtyBlock(db, (_mtKey, typeId, start, end) => saveBlock(db, typeId, start, end));
         }
         db.dirtyRanges.clear();
         const types = {};
@@ -105,22 +60,24 @@ export function save(db, sync = false, forceFullDump = false, skipMigrationCheck
             types[id] = { lastId, blockCapacity };
             rangeDumps[id] = [];
         }
-        db.merkleTree.visitLeafNodes((leaf) => {
-            const [typeId, start] = destructureCsmtKey(leaf.key);
-            if (start == specialBlock)
-                return; // skip the type specialBlock
-            const data = leaf.data;
-            if (start != data.start) {
-                console.error(`csmtKey start and range start mismatch: ${start} != ${data.start}`);
-            }
-            rangeDumps[typeId].push({ ...data, hash: bufToHex(leaf.hash) });
+        db.verifTree.foreachBlock((block) => {
+            const [typeId, start] = destructureTreeKey(block.key);
+            const def = db.schemaTypesParsedById[typeId];
+            const end = start + def.blockCapacity - 1;
+            const data = {
+                file: db.verifTree.getBlockFile(block),
+                hash: bufToHex(block.hash),
+                start,
+                end,
+            };
+            rangeDumps[typeId].push(data);
         });
         const data = {
             ts,
             types,
             commonDump: COMMON_SDB_FILE,
             rangeDumps,
-            hash: bufToHex(db.merkleTree.getRoot()?.hash ?? new Uint8Array(0)),
+            hash: bufToHex(db.verifTree.hash), // TODO `hash('hex')`
         };
         const filePath = join(db.fileSystemPath, WRITELOG_FILE);
         const content = JSON.stringify(data);
@@ -137,8 +94,7 @@ export function save(db, sync = false, forceFullDump = false, skipMigrationCheck
                     resolve(v);
                 })
                     .catch((err) => {
-                    console.error('Save: writing writeLog failed');
-                    db.emit('info', `Save: writing writeLog failed ${err.message}`);
+                    db.emit('error', `Save: writing writeLog failed ${err.message}`);
                     db.saveInProgress = false;
                     reject(err);
                 });
@@ -146,7 +102,7 @@ export function save(db, sync = false, forceFullDump = false, skipMigrationCheck
         }
     }
     catch (err) {
-        db.emit('info', `Save failed ${err.message}`);
+        db.emit('error', `Save failed ${err.message}`);
         db.saveInProgress = false;
         throw err;
     }

package/dist/src/server/schema.js

@@ -2,11 +2,11 @@ import { schemaToSelvaBuffer, updateTypeDefs } from '@based/schema/def';
 import { join } from 'node:path';
 import { writeFile } from 'node:fs/promises';
 import native from '../native.js';
-import { initCsmt, makeCsmtKey } from './tree.js';
+import { makeTreeKey } from './tree.js';
 import { deepCopy, writeUint64 } from '@saulx/utils';
 import { SCHEMA_FILE } from '../types.js';
 import { hash } from '@saulx/hash';
-import { getPropType } from '@based/schema';
+import { getPropType, serialize } from '@based/schema';
 export const setSchemaOnServer = (server, schema) => {
     const { schemaTypesParsed, schemaTypesParsedById } = updateTypeDefs(schema);
     server.schema = schema;
@@ -16,7 +16,7 @@ export const setSchemaOnServer = (server, schema) => {
 export const writeSchemaFile = async (server, schema) => {
     const schemaFilePath = join(server.fileSystemPath, SCHEMA_FILE);
     try {
-        await writeFile(schemaFilePath, JSON.stringify(schema));
+        await writeFile(schemaFilePath, serialize(schema));
     }
     catch (err) {
         throw new Error(`Error writing schema to a file path ${schemaFilePath}}`);
@@ -38,7 +38,7 @@ export const setNativeSchema = (server, schema) => {
     if (schema.types._root) {
         // TODO fix server add it in schema at least
         const data = [2, 1, 0, 0, 0, 1, 9, 1, 0, 0, 0, 7, 1, 0, 1];
-        const blockKey = makeCsmtKey(1, 1);
+        const blockKey = makeTreeKey(1, 1);
         const buf = new Uint8Array(8 + data.length + 2 + 8 + 4);
         const view = new DataView(buf.buffer, 0, buf.byteLength);
         // set schema hash
@@ -52,8 +52,9 @@ export const setNativeSchema = (server, schema) => {
         // add dataLen
         view.setUint32(buf.length - 4, data.length, true);
         server.modify(buf);
-        initCsmt(server);
+        //server.verifTree = new VerifTree(.schemaTypesParsed)
     }
+    server.verifTree.updateTypes(server.schemaTypesParsed);
 };
 export const strictSchemaToDbSchema = (schema) => {
     // @ts-ignore