@based/db 0.0.19 → 0.0.21

package/dist/src/server/start.js

@@ -3,13 +3,14 @@ import { DbWorker } from './index.js';
 import native from '../native.js';
 import { rm, mkdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { createTree } from './csmt/index.js';
+import { createTree, hashEq } from './csmt/index.js';
 import { foreachBlock } from './tree.js';
 import { availableParallelism } from 'node:os';
 import exitHook from 'exit-hook';
 import './worker.js';
 import { save } from './save.js';
 import { DEFAULT_BLOCK_CAPACITY } from '@based/schema/def';
+import { bufToHex, hexToBuf } from '../utils.js';
 const SCHEMA_FILE = 'schema.json';
 const WRITELOG_FILE = 'writelog.json';
 const makeCsmtKey = (typeId, start) => typeId * 4294967296 + start;
@@ -28,15 +29,23 @@ export async function start(db, opts) {
     try {
         writelog = JSON.parse((await readFile(join(path, WRITELOG_FILE))).toString());
         // Load the common dump
-        native.loadCommon(join(path, writelog.commonDump), db.dbCtxExternal);
+        try {
+            native.loadCommon(join(path, writelog.commonDump), db.dbCtxExternal);
+        }
+        catch (e) {
+            console.log(e.message);
+            throw e;
+        }
         // Load all range dumps
         for (const typeId in writelog.rangeDumps) {
             const dumps = writelog.rangeDumps[typeId];
             for (const dump of dumps) {
                 const fname = dump.file;
-                const err = native.loadRange(join(path, fname), db.dbCtxExternal);
-                if (err) {
-                    console.log(`Failed to load a range. file: "${fname}": ${err}`);
+                try {
+                    native.loadRange(join(path, fname), db.dbCtxExternal);
+                }
+                catch (e) {
+                    console.log(e.message);
                 }
             }
         }
@@ -46,13 +55,22 @@ export async function start(db, opts) {
             db.putSchema(JSON.parse(schema.toString()), true);
         }
     }
-    catch (err) { }
+    catch (err) {
+        // TODO In some cases we really should give up!
+    }
     // The merkle tree should be empty at start.
     if (!db.merkleTree || db.merkleTree.getRoot()) {
         db.merkleTree = createTree(db.createCsmtHashFun);
     }
-    for (const key in db.schemaTypesParsed) {
-        const def = db.schemaTypesParsed[key];
+    // FDN-791 CSMT is unstable (not history independent)
+    // For now we just sort the types to ensure that we always
+    // load in the same order.
+    const types = Object.keys(db.schemaTypesParsed).sort((a, b) => db.schemaTypesParsed[a].id - db.schemaTypesParsed[b].id).reduce((obj, key) => {
+        obj[key] = db.schemaTypesParsed[key];
+        return obj;
+    }, {});
+    for (const key in types) {
+        const def = types[key];
         const [total, lastId] = native.getTypeInfo(def.id, db.dbCtxExternal);
         def.total = total;
         def.lastId = writelog?.types[def.id]?.lastId || lastId;
@@ -71,10 +89,10 @@ export async function start(db, opts) {
         });
     }
     if (writelog?.hash) {
-        const oldHash = Buffer.from(writelog.hash, 'hex');
+        const oldHash = hexToBuf(writelog.hash);
         const newHash = db.merkleTree.getRoot()?.hash;
-        if (!oldHash.equals(newHash)) {
-            console.error(`WARN: CSMT hash mismatch: ${writelog.hash} != ${newHash.toString('hex')}`);
+        if (!hashEq(oldHash, newHash)) {
+            console.error(`WARN: CSMT hash mismatch: ${writelog.hash} != ${bufToHex(newHash)}`);
         }
     }
     // start workers
@@ -84,10 +102,19 @@ export async function start(db, opts) {
     while (i--) {
         db.workers[i] = new DbWorker(address, db);
     }
-    db.unlistenExit = exitHook((signal) => {
-        console.log(`Exiting with signal: ${signal}`);
-        save(db, true);
-        console.log('Successfully saved.');
-    });
+    if (!opts?.hosted) {
+        db.unlistenExit = exitHook(async (signal) => {
+            const blockSig = () => { };
+            const signals = ['SIGINT', 'SIGTERM', 'SIGHUP'];
+            // A really dumb way to block signals temporarily while saving.
+            // This is needed because there is no way to set the process signal mask
+            // in Node.js.
+            signals.forEach((sig) => process.on(sig, blockSig));
+            console.log(`Exiting with signal: ${signal}`);
+            save(db, true);
+            console.log('Successfully saved.');
+            signals.forEach((sig) => process.off(sig, blockSig));
+        });
+    }
 }
 //# sourceMappingURL=start.js.map

package/dist/src/server/tree.d.ts

@@ -9,5 +9,5 @@ export type CsmtNodeRange = {
 export declare const makeCsmtKey: (typeId: number, start: number) => number;
 export declare const destructureCsmtKey: (key: number) => number[];
 export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
-export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Buffer) => void): Promise<void>;
+export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): Promise<void>;
 export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;

package/dist/src/server/tree.js

@@ -12,7 +12,7 @@ export async function foreachBlock(db, def, cb) {
     const step = def.blockCapacity;
     for (let start = 1; start <= def.lastId; start += step) {
         const end = start + step - 1;
-        const hash = Buffer.allocUnsafe(16);
+        const hash = new Uint8Array(16);
         native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
         cb(start, end, hash);
     }
@@ -24,7 +24,22 @@ export async function foreachDirtyBlock(db, cb) {
         const typeId = type.id;
         typeIdMap[typeId] = type;
     }
-    for (const mtKey of db.dirtyRanges) {
+    // FDN-791 CSMT is unstable (not history independent)
+    // For now we just sort the dirty data by type and start
+    // to make the insertion order more deterministic.
+    // This doesn't solve the issue completely because
+    // we might mess the order later with updates.
+    const dirty = [...db.dirtyRanges].sort((a, b) => {
+        const [aTypeId, aStart] = destructureCsmtKey(a);
+        const [bTypeId, bStart] = destructureCsmtKey(b);
+        const aId = typeIdMap[aTypeId].id;
+        const bId = typeIdMap[bTypeId].id;
+        const td = aId - bId;
+        if (td != 0)
+            return td;
+        return aStart - bStart;
+    });
+    for (const mtKey of dirty) {
         const [typeId, start] = destructureCsmtKey(mtKey);
         const end = start + typeIdMap[typeId].blockCapacity - 1;
         cb(mtKey, typeId, start, end);

package/dist/src/server/worker.js

@@ -1,12 +1,13 @@
 import { isMainThread, receiveMessageOnPort, workerData, } from 'node:worker_threads';
 import native from '../native.js';
+let workerCtx;
 if (isMainThread) {
     console.warn('running query worker.ts in mainthread');
 }
 else {
     let { address, channel } = workerData;
     let dbCtx = native.externalFromInt(address);
-    native.workerCtxInit();
+    workerCtx = native.workerCtxInit();
     const transferList = new Array(1);
     const handleMsg = (msg) => {
         try {

package/dist/src/utils.d.ts

@@ -0,0 +1,4 @@
+export declare const equals: (aB: Uint8Array, bB: Uint8Array) => boolean;
+export declare const bufToHex: (a: Uint8Array) => string;
+export declare const hexToBuf: (s: string) => Uint8Array;
+export declare const base64encode: (a: Uint8Array, lineMax?: number) => string;

package/dist/src/utils.js

@@ -0,0 +1,84 @@
+import native from './native.js';
+export const equals = (aB, bB) => {
+    const len = aB.byteLength;
+    if (len != bB.byteLength) {
+        return false;
+    }
+    let i = 0;
+    if (len < 50) {
+        while (i < len) {
+            if (aB[i] != bB[i]) {
+                return false;
+            }
+            i++;
+        }
+        return true;
+    }
+    else {
+        while (i < 4) {
+            if (aB[i] != bB[i]) {
+                return false;
+            }
+            i++;
+        }
+        return native.equals(aB, bB);
+    }
+};
+const charMap = new TextEncoder().encode("0123456789abcdef");
+const DECODER = new TextDecoder();
+// Uint8Array.fromHex() and Uint8Array.toHex() are not available in V8
+// https://issues.chromium.org/issues/42204568
+export const bufToHex = (a) => {
+    const tmp = new Uint8Array(2 * a.byteLength);
+    const n = charMap.length;
+    let j = 0;
+    for (let i = 0; i < a.byteLength; i++) {
+        tmp[j++] = charMap[a[i] >>> 4 % n];
+        tmp[j++] = charMap[a[i] & 0x0f % n];
+    }
+    return DECODER.decode(tmp);
+};
+const intMap = {
+    '0': 0,
+    '1': 0x1,
+    '2': 0x2,
+    '3': 0x3,
+    '4': 0x4,
+    '5': 0x5,
+    '6': 0x6,
+    '7': 0x7,
+    '8': 0x8,
+    '9': 0x9,
+    'a': 0xa,
+    'b': 0xb,
+    'c': 0xc,
+    'd': 0xd,
+    'e': 0xe,
+    'f': 0xf,
+};
+// Uint8Array.fromHex() and Uint8Array.toHex() are not available in V8
+// https://issues.chromium.org/issues/42204568
+export const hexToBuf = (s) => {
+    const len = (s.length >>> 1) << 1;
+    const buf = new Uint8Array(len / 2);
+    for (let i = 0; i < buf.byteLength; i++) {
+        const x = s.charAt(2 * i);
+        const y = s.charAt(2 * i + 1);
+        buf[i] = (intMap[x] << 4) + intMap[y];
+    }
+    return buf;
+};
+function base64OutLen(n, lineMax) {
+    let olen;
+    /* This version would be with padding but we don't pad */
+    //olen = n * 4 / 3 + 4; /* 3-byte blocks to 4-byte */
+    olen = ((4 * n / 3) + 3) & ~3;
+    olen += lineMax > 0 ? olen / lineMax : 0; // line feeds
+    return olen;
+}
+export const base64encode = (a, lineMax = 72) => {
+    // TODO Could fallback to @saulx/utils if native is not available
+    const tmp = new Uint8Array(base64OutLen(a.byteLength, lineMax));
+    return DECODER.decode(native.base64encode(tmp, a, lineMax));
+};
+//# sourceMappingURL=utils.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.19",
+  "version": "0.0.21",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",
@@ -12,9 +12,9 @@
     "clean-selva": "make -C clibs clean",
     "clean": "rimraf {.turbo,dist,node_modules,.zig-cache,.zig-out,tmp} && npm run clean-selva",
     "playground-nodebug": "npm run build && node ./dist/test/shared/playground.js",
-    "playground": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale lldb -b -o run --one-line-on-crash bt -o \"script import os; os._exit(0)\" node ./dist/test/shared/playground.js",
+    "playground": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale ./scripts/lldb-node ./dist/test/shared/playground.js",
     "playground-fast": "node ./dist/test/shared/playground.js",
-    "test": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale lldb -b -o run --one-line-on-crash bt -o \"script import os; os._exit(0)\" node ./scripts/test.js",
+    "test": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale ./scripts/lldb-node ./scripts/test.js",
     "test-gdb": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale gdb -ex run --args node ./scripts/test.js",
     "test-fast": "LOCPATH=../locale/locale-x86_64-gnu/locale node ./scripts/test.js",
     "test-zig": "npm run build-zig -- debug && tsc && npm run test-fast",
@@ -31,7 +31,7 @@
     "basedDbNative.cjs"
   ],
   "dependencies": {
-    "@based/schema": "5.0.0-alpha.4",
+    "@based/schema": "5.0.0-alpha.5",
     "@saulx/hash": "^3.0.0",
     "@saulx/utils": "^4.3.2",
     "exit-hook": "^4.0.0",