@based/db 0.0.20 → 0.0.21

package/dist/src/client/string.js

@@ -1,7 +1,9 @@
 import native from '../native.js';
 import { readUint32 } from './bitWise.js';
+import makeTmpBuffer from './tmpBuffer.js';
 const DECODER = new TextDecoder('utf-8');
-// add encoder
+const ENCODER = new TextEncoder();
+const { getUint8Array: getTmpBuffer } = makeTmpBuffer(4096); // the usual page size?
 // type 0 = no compression; 1 = deflate
 // [lang] [type] [uncompressed size 4] [compressed string] [crc32]
 // var cnt = 0
@@ -11,24 +13,28 @@ const DECODER = new TextDecoder('utf-8');
 export const write = (buf, value, offset, noCompression, lang) => {
     value = value.normalize('NFKD');
     buf[offset] = lang || 0;
+    const { written: l } = ENCODER.encodeInto(value, buf.subarray(offset + 2));
+    let crc = native.crc32(buf.subarray(offset + 2, offset + 2 + l));
     // 50 maybe if lvl 1
     if (value.length > 200 && !noCompression) {
-        const s = Buffer.byteLength(value, 'utf8');
-        buf.write(value, offset + 6 + s, 'utf8');
-        let crc = native.crc32(buf.subarray(offset + 6 + s, offset + 6 + 2 * s));
-        const size = native.compress(buf, offset + 6, s);
+        buf.copyWithin(offset + 6 + l, offset + 2, offset + 2 + l);
+        const size = native.compress(buf, offset + 6, l);
         if (size === 0) {
             buf[offset + 1] = 0; // not compressed
-            const len = buf.write(value, offset + 2, 'utf8');
-            buf[offset + len + 2] = crc;
-            buf[offset + len + 3] = crc >>>= 8;
-            buf[offset + len + 4] = crc >>>= 8;
-            buf[offset + len + 5] = crc >>>= 8;
-            return len + 6;
+            ENCODER.encodeInto(value, buf.subarray(offset + 2));
+            buf[offset + l + 2] = crc;
+            buf[offset + l + 3] = crc >>>= 8;
+            buf[offset + l + 4] = crc >>>= 8;
+            buf[offset + l + 5] = crc >>>= 8;
+            return l + 6;
         }
         else {
+            let len = l;
             buf[offset + 1] = 1; // compressed
-            buf.writeUInt32LE(s, offset + 2);
+            buf[offset + 2] = len;
+            buf[offset + 3] = len >>>= 8;
+            buf[offset + 4] = len >>>= 8;
+            buf[offset + 5] = len >>>= 8;
             buf[offset + size + 6] = crc;
             buf[offset + size + 7] = crc >>>= 8;
             buf[offset + size + 8] = crc >>>= 8;
@@ -38,23 +44,19 @@ export const write = (buf, value, offset, noCompression, lang) => {
     }
     else {
         buf[offset + 1] = 0; // not compressed
-        const len = buf.write(value, offset + 2, 'utf8');
-        let crc = native.crc32(buf.subarray(offset + 2, offset + len + 2));
-        buf[offset + len + 2] = crc;
-        buf[offset + len + 3] = crc >>>= 8;
-        buf[offset + len + 4] = crc >>>= 8;
-        buf[offset + len + 5] = crc >>>= 8;
-        return len + 6;
+        buf[offset + l + 2] = crc;
+        buf[offset + l + 3] = crc >>>= 8;
+        buf[offset + l + 4] = crc >>>= 8;
+        buf[offset + l + 5] = crc >>>= 8;
+        return l + 6;
     }
 };
-let tmpCompressBlock;
 export const compress = (str) => {
-    if (!tmpCompressBlock || tmpCompressBlock.byteLength < str.length * 3) {
-        tmpCompressBlock = Buffer.allocUnsafe(str.length * 3);
-    }
-    const s = write(tmpCompressBlock, str, 0, false);
-    const nBuffer = Buffer.allocUnsafe(s);
-    tmpCompressBlock.copy(nBuffer, 0, 0, s);
+    const len = ENCODER.encode(str).byteLength;
+    const tmpCompressBlock = getTmpBuffer(len * 3);
+    const l = write(tmpCompressBlock, str, 0, false);
+    const nBuffer = new Uint8Array(l);
+    nBuffer.set(tmpCompressBlock.subarray(0, l));
     return nBuffer;
 };
 export const decompress = (val) => {
@@ -64,9 +66,10 @@ export const read = (val, offset, len) => {
     const type = val[offset + 1];
     if (type == 1) {
         const origSize = readUint32(val, offset + 2);
-        const newBuffer = Buffer.allocUnsafe(origSize);
-        native.decompress(Buffer.from(val), newBuffer, offset + 6, len - 6);
-        return newBuffer.toString('utf8');
+        const newBuffer = getTmpBuffer(origSize);
+        // deflate in browser for this...
+        native.decompress(val, newBuffer, offset + 6, len - 6);
+        return DECODER.decode(newBuffer);
     }
     else if (type == 0) {
         return DECODER.decode(val.subarray(offset + 2, len + offset - 4));

package/dist/src/client/tmpBuffer.d.ts

@@ -0,0 +1,3 @@
+export default function makeTmpBuffer(initialSize: number): {
+    getUint8Array: (size: number) => Uint8Array;
+};

package/dist/src/client/tmpBuffer.js

@@ -0,0 +1,20 @@
+export default function makeTmpBuffer(initialSize) {
+    // @ts-ignore
+    let tmpBuffer = new ArrayBuffer(initialSize, { maxByteLength: initialSize });
+    return {
+        getUint8Array: (size) => {
+            const opts = {
+                maxByteLength: Math.min(Math.round(1.5 * size), 274877906944)
+            };
+            // @ts-ignore
+            if (tmpBuffer.maxByteLength < size) {
+                // @ts-ignore
+                tmpBuffer = new ArrayBuffer(size, opts);
+            }
+            // @ts-ignore
+            tmpBuffer.resize(size);
+            return new Uint8Array(tmpBuffer);
+        }
+    };
+}
+//# sourceMappingURL=tmpBuffer.js.map

package/dist/src/client/xxHash64.d.ts

@@ -1 +1 @@
-export declare const xxHash64: (buf: Buffer, target?: Buffer, index?: number) => Buffer;
+export declare const xxHash64: (buf: Buffer | Uint8Array, target?: Uint8Array | Buffer, index?: number) => Uint8Array | Buffer;

package/dist/src/client/xxHash64.js

@@ -1,7 +1,7 @@
 import native from '../native.js';
 export const xxHash64 = (buf, target, index) => {
     if (!target) {
-        target = Buffer.allocUnsafe(8);
+        target = new Uint8Array(8);
         index = 0;
     }
     native.xxHash64(buf, target, index);

package/dist/src/index.d.ts

@@ -8,6 +8,8 @@ export { ModifyCtx };
 export { DbClient, DbServer };
 export { xxHash64 } from './client/xxHash64.js';
 export { crc32 } from './client/crc32.js';
+export * from './client/query/serialize.js';
+export * from './utils.js';
 export declare class BasedDb {
     #private;
     client: DbClient;

package/dist/src/index.js

@@ -9,6 +9,8 @@ export { ModifyCtx }; // TODO move this somewhere
 export { DbClient, DbServer };
 export { xxHash64 } from './client/xxHash64.js';
 export { crc32 } from './client/crc32.js';
+export * from './client/query/serialize.js';
+export * from './utils.js';
 export class BasedDb {
     client;
     server;

package/dist/src/native.d.ts

@@ -1,8 +1,7 @@
 declare const _default: {
-    historyAppend(history: any, typeId: number, nodeId: number): any;
+    historyAppend(history: any, typeId: number, nodeId: number, dbCtx: any): any;
     historyCreate(pathname: string, mainLen: number): any;
     workerCtxInit: () => void;
-    markMerkleBlock: (buf: Buffer) => any;
     externalFromInt(address: BigInt): any;
     intFromExternal(external: any): BigInt;
     modify: (data: Buffer, types: Buffer, dbCtx: any) => any;
@@ -10,22 +9,24 @@ declare const _default: {
     start: (id: number) => any;
     stop: (dbCtx: any) => any;
     saveCommon: (path: string, dbCtx: any) => number;
-    saveRange: (path: string, typeCode: number, start: number, end: number, dbCtx: any, hashOut: Buffer) => number;
-    loadCommon: (path: string, dbCtx: any) => number;
-    loadRange: (path: string, dbCtx: any) => number;
-    updateSchemaType: (prefix: number, buf: Buffer, dbCtx: any) => any;
+    saveRange: (path: string, typeCode: number, start: number, end: number, dbCtx: any, hashOut: Uint8Array) => number;
+    loadCommon: (path: string, dbCtx: any) => void;
+    loadRange: (path: string, dbCtx: any) => void;
+    updateSchemaType: (prefix: number, buf: Uint8Array, dbCtx: any) => any;
     getTypeInfo: (typeId: number, dbCtx: any) => any;
-    getNodeRangeHash: (typeId: number, start: number, end: number, bufOut: Buffer, dbCtx: any) => any;
+    getNodeRangeHash: (typeId: number, start: number, end: number, bufOut: Uint8Array, dbCtx: any) => any;
     createHash: () => {
-        update: (buf: Buffer) => any;
-        digest: (encoding?: BufferEncoding) => Buffer | string;
+        update: (buf: Buffer | Uint8Array) => any;
+        digest: (encoding?: "hex") => Uint8Array | string;
         reset: () => void;
     };
-    compress: (buf: Buffer, offset: number, stringSize: number) => any;
-    decompress: (input: Buffer, output: Buffer, offset: number, len: number) => any;
-    crc32: (buf: Buffer) => any;
+    compress: (buf: Buffer | Uint8Array, offset: number, stringSize: number) => any;
+    decompress: (input: Buffer | Uint8Array, output: Buffer | Uint8Array, offset: number, len: number) => any;
+    crc32: (buf: Buffer | Uint8Array) => any;
     createSortIndex: (buf: Buffer, dbCtx: any) => any;
     destroySortIndex: (buf: Buffer, dbCtx: any) => any;
-    xxHash64: (buf: Buffer, target: Buffer, index: number) => any;
+    xxHash64: (buf: Buffer | Uint8Array, target: Buffer | Uint8Array, index: number) => any;
+    base64encode: (dst: Uint8Array, src: Uint8Array, lineMax: number) => Uint8Array;
+    equals: (a: Uint8Array, b: Uint8Array) => boolean;
 };
 export default _default;

package/dist/src/native.js

@@ -1,20 +1,26 @@
 // @ts-ignore
 import db from '../../basedDbNative.cjs';
-var compressor = null;
-var decompressor = null;
+import { bufToHex } from './utils.js';
+const selvaIoErrlog = new Uint8Array(256);
+const textEncoder = new TextEncoder();
+var compressor = db.createCompressor();
+var decompressor = db.createDecompressor();
+function SelvaIoErrlogToString(buf) {
+    let i;
+    let len = (i = buf.indexOf(0)) >= 0 ? i : buf.byteLength;
+    return new TextDecoder().decode(selvaIoErrlog.slice(0, len));
+}
 export default {
-    historyAppend(history, typeId, nodeId) {
-        return db.historyCreate(history, typeId, nodeId);
+    historyAppend(history, typeId, nodeId, dbCtx) {
+        return db.historyAppend(history, typeId, nodeId, dbCtx);
     },
     historyCreate(pathname, mainLen) {
-        return db.historyCreate(Buffer.from(pathname), mainLen + 16 - (mainLen % 16));
+        const pathBuf = textEncoder.encode(pathname + '\0');
+        return db.historyCreate(pathBuf, mainLen + 16 - (mainLen % 16));
     },
     workerCtxInit: () => {
         return db.workerCtxInit();
     },
-    markMerkleBlock: (buf) => {
-        // pstart,
-    },
     externalFromInt(address) {
         return db.externalFromInt(address);
     },
@@ -35,20 +41,26 @@ export default {
         return db.stop(dbCtx);
     },
     saveCommon: (path, dbCtx) => {
-        const buf = Buffer.concat([Buffer.from(path), Buffer.from([0])]);
-        return db.saveCommon(buf, dbCtx);
+        const pathBuf = textEncoder.encode(path + '\0');
+        return db.saveCommon(pathBuf, dbCtx);
     },
     saveRange: (path, typeCode, start, end, dbCtx, hashOut) => {
-        const buf = Buffer.concat([Buffer.from(path), Buffer.from([0])]);
-        return db.saveRange(buf, typeCode, start, end, dbCtx, hashOut);
+        const pathBuf = textEncoder.encode(path + '\0');
+        return db.saveRange(pathBuf, typeCode, start, end, dbCtx, hashOut);
     },
     loadCommon: (path, dbCtx) => {
-        const buf = Buffer.concat([Buffer.from(path), Buffer.from([0])]);
-        return db.loadCommon(buf, dbCtx);
+        const pathBuf = textEncoder.encode(path + '\0');
+        const err = db.loadCommon(pathBuf, dbCtx, selvaIoErrlog);
+        if (err) {
+            throw new Error(`Failed to load common. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
+        }
     },
     loadRange: (path, dbCtx) => {
-        const buf = Buffer.concat([Buffer.from(path), Buffer.from([0])]);
-        return db.loadRange(buf, dbCtx);
+        const pathBuf = textEncoder.encode(path + '\0');
+        const err = db.loadRange(pathBuf, dbCtx, selvaIoErrlog);
+        if (err) {
+            throw new Error(`Failed to load a range. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
+        }
     },
     updateSchemaType: (prefix, buf, dbCtx) => {
         return db.updateSchema(prefix, buf, dbCtx);
@@ -67,9 +79,14 @@ export default {
                 return hash;
             },
             digest: (encoding) => {
-                const buf = Buffer.allocUnsafe(16);
+                const buf = new Uint8Array(16);
                 db.hashDigest(state, buf);
-                return encoding ? buf.toString(encoding) : buf;
+                if (encoding === 'hex') {
+                    return bufToHex(buf);
+                }
+                else {
+                    return buf;
+                }
             },
             reset: () => {
                 db.hashReset(state);
@@ -77,17 +94,10 @@ export default {
         };
         return hash;
     },
-    // needs to pass dbCtx: any
     compress: (buf, offset, stringSize) => {
-        if (compressor === null) {
-            compressor = db.createCompressor();
-        }
         return db.compress(compressor, buf, offset, stringSize);
     },
     decompress: (input, output, offset, len) => {
-        if (decompressor === null) {
-            decompressor = db.createDecompressor();
-        }
         return db.decompress(decompressor, input, output, offset, len);
     },
     crc32: (buf) => {
@@ -102,5 +112,11 @@ export default {
     xxHash64: (buf, target, index) => {
         return db.xxHash64(buf, target, index);
     },
+    base64encode: (dst, src, lineMax) => {
+        return db.base64encode(dst, src, lineMax);
+    },
+    equals: (a, b) => {
+        return !!db.equals(a, b);
+    },
 };
 //# sourceMappingURL=native.js.map

package/dist/src/server/csmt/draw-dot.js

@@ -1,5 +1,6 @@
+import { base64encode } from '../../utils.js';
 function makeLabel(node) {
-    return `${node.key}\n${node.hash.toString('base64').substring(0, 5)}`;
+    return `${node.key}\n${base64encode(node.hash, 0).substring(0, 5)}`;
 }
 export default function draw(csmt) {
     const root = csmt.getRoot();

package/dist/src/server/csmt/memebership-proof.d.ts

@@ -3,5 +3,5 @@ export declare enum Direction {
     Left = "L",
     Right = "R"
 }
-export type Proof = [TreeKey | null, TreeKey | null] | [TreeKey | Buffer, TreeKey | Direction][];
+export type Proof = [TreeKey | null, TreeKey | null] | [TreeKey | Uint8Array, TreeKey | Direction][];
 export default function membershipProof(root: TreeNode | null, k: TreeKey): Proof;

package/dist/src/server/csmt/tree.d.ts

@@ -1,2 +1,3 @@
-import { Csmt } from './types.js';
+import { Hash, Csmt } from './types.js';
+export declare function hashEq(a: Hash, b: Hash): boolean;
 export declare function createTree(createHash: () => any): Csmt;

package/dist/src/server/csmt/tree.js

@@ -1,12 +1,13 @@
 import { TreeKeyNil } from './types.js';
 import { distance, min, max } from './tree-utils.js';
 import membershipProof from './memebership-proof.js';
+import { equals } from '../../utils.js';
+export function hashEq(a, b) {
+    return equals(a, b);
+}
 export function createTree(createHash) {
     let root = null;
-    const emptyHash = genHash(Buffer.from(''));
-    function genHash(s) {
-        return createHash().update(s).digest();
-    }
+    const emptyHash = createHash().digest();
     function genNodeHash(lHash, rHash) {
         return createHash().update(lHash).update(rHash).digest();
     }
@@ -123,7 +124,7 @@ export function createTree(createHash) {
         if (nodeA &&
             nodeB &&
             nodeA.key === nodeB.key &&
-            nodeA.hash.compare(nodeB.hash) === 0) {
+            hashEq(nodeA.hash, nodeB.hash)) {
             return;
         }
         // No hash match
@@ -134,7 +135,7 @@ export function createTree(createHash) {
         // Check if this is a leaf that is missing from the right tree
         if (nodeA && !leftA && !rightA) {
             const bHash = diffB.get(nodeA.key);
-            if (bHash && bHash.compare(nodeA.hash) === 0) {
+            if (bHash && hashEq(bHash, nodeA.hash)) {
                 // The same leaf appears to exist in both trees
                 diffB.delete(nodeA.key);
             }
@@ -146,7 +147,7 @@ export function createTree(createHash) {
         // Check if this is a leaf that is missing from the left tree
         if (nodeB && !leftB && !rightB) {
             const aHash = diffA.get(nodeB.key);
-            if (aHash && aHash.compare(nodeB.hash) === 0) {
+            if (aHash && hashEq(aHash, nodeB.hash)) {
                 diffA.delete(nodeB.key);
             }
             else {
@@ -198,8 +199,8 @@ export function createTree(createHash) {
     return {
         getRoot: () => root,
         insert: (k, h, data = null) => {
-            if (!(h instanceof Buffer)) {
-                throw new TypeError('`h` must be a Buffer');
+            if (!(h instanceof Uint8Array)) { // TODO can we extract the name somehow from Hash?
+                throw new TypeError('`h` must be a Uint8Array'); // TODO can we extract the name somehow from Hash?
             }
             const newLeaf = createLeaf(k, h, data);
             root = root ? insert(root, newLeaf) : newLeaf;

package/dist/src/server/csmt/types.d.ts

@@ -1,14 +1,15 @@
 import { Proof } from './memebership-proof.js';
 export type TreeKey = number;
 export declare const TreeKeyNil = 0;
+export type Hash = Uint8Array;
 export interface TreeNode {
-    hash: Buffer;
+    hash: Hash;
     key: TreeKey;
     data?: any;
     left: TreeNode | null;
     right: TreeNode | null;
 }
-export type KeyHashPair = [TreeKey, Buffer];
+export type KeyHashPair = [TreeKey, Hash];
 export interface TreeDiff {
     left: KeyHashPair[];
     right: KeyHashPair[];
@@ -21,7 +22,7 @@ export interface Csmt {
     /**
      * Insert a new key-hash pair.
      */
-    insert: (k: TreeKey, h: Buffer, data?: any) => void;
+    insert: (k: TreeKey, h: Hash, data?: any) => void;
     /**
      * Delete a key-hash pair from the tree.
      */

package/dist/src/server/index.d.ts

@@ -36,8 +36,8 @@ export declare class DbServer {
     merkleTree: ReturnType<typeof createTree>;
     dirtyRanges: Set<number>;
     csmtHashFun: {
-        update: (buf: Buffer) => any;
-        digest: (encoding?: BufferEncoding) => Buffer | string;
+        update: (buf: Buffer | Uint8Array) => any;
+        digest: (encoding?: "hex") => Uint8Array | string;
         reset: () => void;
     };
     workers: DbWorker[];
@@ -55,11 +55,12 @@ export declare class DbServer {
     });
     start(opts?: {
         clean?: boolean;
+        hosted?: boolean;
     }): Promise<void>;
     save(): void | Promise<void>;
     createCsmtHashFun: () => {
-        update: (buf: Buffer) => any;
-        digest: (encoding?: BufferEncoding) => Buffer | string;
+        update: (buf: Buffer | Uint8Array) => any;
+        digest: (encoding?: "hex") => Uint8Array | string;
         reset: () => void;
     };
     sortIndexes: {

package/dist/src/server/index.js

@@ -2,9 +2,8 @@ import native from '../native.js';
 import { rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { getPropType, langCodesMap, } from '@based/schema';
-import { 
-// genRootId,
-updateTypeDefs, schemaToSelvaBuffer, } from '@based/schema/def';
+import { updateTypeDefs, schemaToSelvaBuffer, } from '@based/schema/def';
+import { hashEq } from './csmt/index.js';
 import { start } from './start.js';
 import { foreachDirtyBlock, makeCsmtKey, makeCsmtKeyFromNodeId, } from './tree.js';
 import { save } from './save.js';
@@ -251,7 +250,7 @@ export class DbServer {
             const hash = Buffer.allocUnsafe(16);
             native.getNodeRangeHash(typeId, start, end, hash, this.dbCtxExternal);
             if (oldLeaf) {
-                if (oldLeaf.hash.equals(hash)) {
+                if (hashEq(oldLeaf.hash, hash)) {
                     return;
                 }
                 try {
@@ -331,7 +330,7 @@ export class DbServer {
                 const type = this.schemaTypesParsed[types[i]];
                 // TODO should not crash!
                 try {
-                    native.updateSchemaType(type.id, s[i], this.dbCtxExternal);
+                    native.updateSchemaType(type.id, new Uint8Array(s[i]), this.dbCtxExternal);
                 }
                 catch (err) {
                     console.error('Cannot update schema on selva', type.type, err, s[i]);
@@ -465,7 +464,7 @@ export class DbServer {
         }
         this.stopped = true;
         clearTimeout(this.cleanupTimer);
-        this.unlistenExit();
+        this?.unlistenExit();
         try {
             if (!noSave) {
                 await this.save();

package/dist/src/server/save.js

@@ -3,6 +3,7 @@ import { writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { destructureCsmtKey, foreachDirtyBlock } from './tree.js';
 import { writeFileSync } from 'node:fs';
+import { bufToHex } from '../utils.js';
 const WRITELOG_FILE = 'writelog.json';
 const COMMON_SDB_FILE = 'common.sdb';
 const block_sdb_file = (typeId, start, end) => `${typeId}_${start}_${end}.sdb`;
@@ -19,7 +20,7 @@ export function save(db, sync = false) {
     foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
         const file = block_sdb_file(typeId, start, end);
         const path = join(db.fileSystemPath, file);
-        const hash = Buffer.allocUnsafe(16);
+        const hash = new Uint8Array(16);
         err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hash);
         if (err) {
             console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
@@ -53,7 +54,7 @@ export function save(db, sync = false) {
     db.merkleTree.visitLeafNodes((leaf) => {
         const [typeId] = destructureCsmtKey(leaf.key);
         const data = leaf.data;
-        dumps[typeId].push({ ...data, hash: leaf.hash.toString('hex') });
+        dumps[typeId].push({ ...data, hash: bufToHex(leaf.hash) });
     });
     const data = {
         ts,
@@ -63,7 +64,7 @@ export function save(db, sync = false) {
     };
     const mtRoot = db.merkleTree.getRoot();
     if (mtRoot) {
-        data.hash = mtRoot.hash.toString('hex');
+        data.hash = bufToHex(mtRoot.hash);
     }
     const filePath = join(db.fileSystemPath, WRITELOG_FILE);
     const content = JSON.stringify(data);

package/dist/src/server/start.d.ts

@@ -2,4 +2,5 @@ import { DbServer } from './index.js';
 import './worker.js';
 export declare function start(db: DbServer, opts: {
     clean?: boolean;
+    hosted?: boolean;
 }): Promise<void>;

package/dist/src/server/start.js

@@ -3,13 +3,14 @@ import { DbWorker } from './index.js';
 import native from '../native.js';
 import { rm, mkdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { createTree } from './csmt/index.js';
+import { createTree, hashEq } from './csmt/index.js';
 import { foreachBlock } from './tree.js';
 import { availableParallelism } from 'node:os';
 import exitHook from 'exit-hook';
 import './worker.js';
 import { save } from './save.js';
 import { DEFAULT_BLOCK_CAPACITY } from '@based/schema/def';
+import { bufToHex, hexToBuf } from '../utils.js';
 const SCHEMA_FILE = 'schema.json';
 const WRITELOG_FILE = 'writelog.json';
 const makeCsmtKey = (typeId, start) => typeId * 4294967296 + start;
@@ -28,15 +29,23 @@ export async function start(db, opts) {
     try {
         writelog = JSON.parse((await readFile(join(path, WRITELOG_FILE))).toString());
         // Load the common dump
-        native.loadCommon(join(path, writelog.commonDump), db.dbCtxExternal);
+        try {
+            native.loadCommon(join(path, writelog.commonDump), db.dbCtxExternal);
+        }
+        catch (e) {
+            console.log(e.message);
+            throw e;
+        }
         // Load all range dumps
         for (const typeId in writelog.rangeDumps) {
             const dumps = writelog.rangeDumps[typeId];
             for (const dump of dumps) {
                 const fname = dump.file;
-                const err = native.loadRange(join(path, fname), db.dbCtxExternal);
-                if (err) {
-                    console.log(`Failed to load a range. file: "${fname}": ${err}`);
+                try {
+                    native.loadRange(join(path, fname), db.dbCtxExternal);
+                }
+                catch (e) {
+                    console.log(e.message);
                 }
             }
         }
@@ -46,13 +55,22 @@ export async function start(db, opts) {
             db.putSchema(JSON.parse(schema.toString()), true);
         }
     }
-    catch (err) { }
+    catch (err) {
+        // TODO In some cases we really should give up!
+    }
     // The merkle tree should be empty at start.
     if (!db.merkleTree || db.merkleTree.getRoot()) {
         db.merkleTree = createTree(db.createCsmtHashFun);
     }
-    for (const key in db.schemaTypesParsed) {
-        const def = db.schemaTypesParsed[key];
+    // FDN-791 CSMT is unstable (not history independent)
+    // For now we just sort the types to ensure that we always
+    // load in the same order.
+    const types = Object.keys(db.schemaTypesParsed).sort((a, b) => db.schemaTypesParsed[a].id - db.schemaTypesParsed[b].id).reduce((obj, key) => {
+        obj[key] = db.schemaTypesParsed[key];
+        return obj;
+    }, {});
+    for (const key in types) {
+        const def = types[key];
         const [total, lastId] = native.getTypeInfo(def.id, db.dbCtxExternal);
         def.total = total;
         def.lastId = writelog?.types[def.id]?.lastId || lastId;
@@ -71,10 +89,10 @@ export async function start(db, opts) {
         });
     }
     if (writelog?.hash) {
-        const oldHash = Buffer.from(writelog.hash, 'hex');
+        const oldHash = hexToBuf(writelog.hash);
         const newHash = db.merkleTree.getRoot()?.hash;
-        if (!oldHash.equals(newHash)) {
-            console.error(`WARN: CSMT hash mismatch: ${writelog.hash} != ${newHash.toString('hex')}`);
+        if (!hashEq(oldHash, newHash)) {
+            console.error(`WARN: CSMT hash mismatch: ${writelog.hash} != ${bufToHex(newHash)}`);
         }
     }
     // start workers
@@ -84,10 +102,19 @@ export async function start(db, opts) {
     while (i--) {
         db.workers[i] = new DbWorker(address, db);
     }
-    db.unlistenExit = exitHook((signal) => {
-        console.log(`Exiting with signal: ${signal}`);
-        save(db, true);
-        console.log('Successfully saved.');
-    });
+    if (!opts?.hosted) {
+        db.unlistenExit = exitHook(async (signal) => {
+            const blockSig = () => { };
+            const signals = ['SIGINT', 'SIGTERM', 'SIGHUP'];
+            // A really dumb way to block signals temporarily while saving.
+            // This is needed because there is no way to set the process signal mask
+            // in Node.js.
+            signals.forEach((sig) => process.on(sig, blockSig));
+            console.log(`Exiting with signal: ${signal}`);
+            save(db, true);
+            console.log('Successfully saved.');
+            signals.forEach((sig) => process.off(sig, blockSig));
+        });
+    }
 }
 //# sourceMappingURL=start.js.map

package/dist/src/server/tree.d.ts

@@ -9,5 +9,5 @@ export type CsmtNodeRange = {
 export declare const makeCsmtKey: (typeId: number, start: number) => number;
 export declare const destructureCsmtKey: (key: number) => number[];
 export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
-export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Buffer) => void): Promise<void>;
+export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): Promise<void>;
 export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;