@based/db 0.0.24 → 0.0.26

package/dist/src/server/index.js

@@ -12,7 +12,8 @@ import { Worker, MessageChannel } from 'node:worker_threads';
 import { fileURLToPath } from 'node:url';
 import { setTimeout } from 'node:timers/promises';
 import { migrate } from './migrate/index.js';
-const SCHEMA_FILE = 'schema.json';
+export const SCHEMA_FILE = 'schema.json';
+export const WRITELOG_FILE = 'writelog.json';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const workerPath = join(__dirname, 'worker.js');
@@ -45,6 +46,9 @@ export class DbWorker {
         });
         port1.on('message', (buf) => {
             // TODO FIX TYPES CHECK IF THIS MAKES A COPY
+            // It's a copy, if you don't want a copy you'd need to make it an explicit view
+            // to the underlying buffer:
+            // new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
             this.resolvers.shift()(new Uint8Array(buf));
             this.db.onQueryEnd();
         });
@@ -67,7 +71,7 @@ export class DbWorker {
     }
 }
 export class DbServer {
-    modifyBuf;
+    modifyDirtyRanges;
     dbCtxExternal; // pointer to zig dbCtx
     schema = {
         lastId: 1, // we reserve one for root props
@@ -95,11 +99,26 @@ export class DbServer {
         this.sortIndexes = {};
         this.onSchemaChange = onSchemaChange;
     }
+    #resizeModifyDirtyRanges() {
+        let maxNrChanges = 0;
+        for (const typeId in this.schemaTypesParsedById) {
+            const def = this.schemaTypesParsedById[typeId];
+            const lastId = def.lastId;
+            const blockCapacity = def.blockCapacity;
+            const tmp = lastId - +!(lastId % def.blockCapacity);
+            const lastBlock = Math.ceil((((tmp / blockCapacity) | 0) * blockCapacity + 1) / blockCapacity);
+            maxNrChanges += lastBlock;
+        }
+        if (!this.modifyDirtyRanges || this.modifyDirtyRanges.length < maxNrChanges) {
+            const min = Math.max(maxNrChanges * 1.2, 1024) | 0;
+            this.modifyDirtyRanges = new Float64Array(min);
+        }
+    }
     start(opts) {
         return start(this, opts);
     }
-    save() {
-        return save(this);
+    save(opts) {
+        return save(this, false, opts?.forceFullDump ?? false);
     }
     createCsmtHashFun = () => {
         // We can just reuse it as long as we only have one tree.
@@ -420,7 +439,14 @@ export class DbServer {
             this.dirtyRanges.add(key);
             i += 8;
         }
-        native.modify(data, types, this.dbCtxExternal);
+        this.#resizeModifyDirtyRanges();
+        native.modify(data, types, this.dbCtxExternal, this.modifyDirtyRanges);
+        for (let key of this.modifyDirtyRanges) {
+            if (key === 0) {
+                break;
+            }
+            this.dirtyRanges.add(key);
+        }
     }
     getQueryBuf(buf) {
         if (this.modifyQueue.length) {

package/dist/src/server/save.d.ts

@@ -1,2 +1,21 @@
 import { DbServer } from './index.js';
-export declare function save<T extends boolean>(db: DbServer, sync?: T): T extends true ? void : Promise<void>;
+export type Writelog = {
+    ts: number;
+    types: {
+        [t: number]: {
+            lastId: number;
+            blockCapacity: number;
+        };
+    };
+    hash: string;
+    commonDump: string;
+    rangeDumps: {
+        [t: number]: {
+            file: string;
+            hash: string;
+            start: number;
+            end: number;
+        }[];
+    };
+};
+export declare function save<T extends boolean>(db: DbServer, sync?: T, forceFullDump?: boolean): T extends true ? void : Promise<void>;

package/dist/src/server/save.js

@@ -1,14 +1,27 @@
 import native from '../native.js';
+import { isMainThread } from 'node:worker_threads';
 import { writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { destructureCsmtKey, foreachDirtyBlock } from './tree.js';
+import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, makeCsmtKey } from './tree.js';
+import { WRITELOG_FILE } from './index.js';
 import { writeFileSync } from 'node:fs';
 import { bufToHex } from '../utils.js';
-const WRITELOG_FILE = 'writelog.json';
+import { createTree } from './csmt/tree.js';
 const COMMON_SDB_FILE = 'common.sdb';
 const block_sdb_file = (typeId, start, end) => `${typeId}_${start}_${end}.sdb`;
-export function save(db, sync = false) {
-    if (!db.dirtyRanges.size) {
+function saveRange(db, typeId, start, end, hashOut) {
+    const file = block_sdb_file(typeId, start, end);
+    const path = join(db.fileSystemPath, file);
+    const err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hashOut);
+    if (err) {
+        // TODO print the error string
+        console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
+        return null;
+    }
+    return file;
+}
+export function save(db, sync = false, forceFullDump = false) {
+    if (!(isMainThread && (db.dirtyRanges.size || forceFullDump))) {
         return;
     }
     let err;
@@ -17,29 +30,55 @@ export function save(db, sync = false) {
     if (err) {
         console.error(`Save common failed: ${err}`);
     }
-    foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
-        const file = block_sdb_file(typeId, start, end);
-        const path = join(db.fileSystemPath, file);
-        const hash = new Uint8Array(16);
-        err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hash);
-        if (err) {
-            console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
-            return; // TODO What to do with the merkle tree in db situation?
-        }
-        const data = {
-            file,
-            typeId,
-            start,
-            end,
-        };
-        try {
-            db.merkleTree.delete(mtKey);
-        }
-        catch (err) {
-            // console.error({ err })
+    if (forceFullDump) {
+        // We just rebuild the whole tree
+        db.merkleTree = createTree(db.createCsmtHashFun); // TODO This could be somewhere else.
+        for (const key in db.schemaTypesParsed) {
+            const def = db.schemaTypesParsed[key];
+            foreachBlock(db, def, (start, end, _hash) => {
+                const typeId = def.id;
+                const hash = new Uint8Array(16);
+                const file = saveRange(db, typeId, start, end, hash);
+                if (!file) {
+                    // The previous state should remain in the merkle tree for
+                    // load and sync purposes.
+                    return;
+                }
+                const mtKey = makeCsmtKey(typeId, start);
+                const data = {
+                    file,
+                    typeId,
+                    start,
+                    end,
+                };
+                db.merkleTree.insert(mtKey, hash, data);
+            });
         }
-        db.merkleTree.insert(mtKey, hash, data);
-    });
+    }
+    else {
+        foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
+            const hash = new Uint8Array(16);
+            const file = saveRange(db, typeId, start, end, hash);
+            if (!file) {
+                // The previous state should remain in the merkle tree for
+                // load and sync purposes.
+                return;
+            }
+            const data = {
+                file,
+                typeId,
+                start,
+                end,
+            };
+            try {
+                db.merkleTree.delete(mtKey);
+            }
+            catch (err) {
+                // console.error({ err })
+            }
+            db.merkleTree.insert(mtKey, hash, data);
+        });
+    }
     db.dirtyRanges.clear();
     const types = {};
     const rangeDumps = {};
@@ -61,11 +100,8 @@ export function save(db, sync = false) {
         types,
         commonDump: COMMON_SDB_FILE,
         rangeDumps,
+        hash: bufToHex(db.merkleTree.getRoot()?.hash ?? new Uint8Array(0)),
     };
-    const mtRoot = db.merkleTree.getRoot();
-    if (mtRoot) {
-        data.hash = bufToHex(mtRoot.hash);
-    }
     const filePath = join(db.fileSystemPath, WRITELOG_FILE);
     const content = JSON.stringify(data);
     return sync ? writeFileSync(filePath, content) : writeFile(filePath, content);

package/dist/src/server/start.js

@@ -1,19 +1,16 @@
 import { stringHash } from '@saulx/hash';
-import { DbWorker } from './index.js';
+import { DbWorker, SCHEMA_FILE, WRITELOG_FILE } from './index.js';
 import native from '../native.js';
 import { rm, mkdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { createTree, hashEq } from './csmt/index.js';
-import { foreachBlock } from './tree.js';
+import { foreachBlock, makeCsmtKey } from './tree.js';
 import { availableParallelism } from 'node:os';
 import exitHook from 'exit-hook';
 import './worker.js';
 import { save } from './save.js';
 import { DEFAULT_BLOCK_CAPACITY } from '@based/schema/def';
 import { bufToHex, hexToBuf } from '../utils.js';
-const SCHEMA_FILE = 'schema.json';
-const WRITELOG_FILE = 'writelog.json';
-const makeCsmtKey = (typeId, start) => typeId * 4294967296 + start;
 export async function start(db, opts) {
     const path = db.fileSystemPath;
     const id = stringHash(path) >>> 0;
@@ -22,8 +19,6 @@ export async function start(db, opts) {
         await rm(path, { recursive: true, force: true }).catch(noop);
     }
     await mkdir(path, { recursive: true }).catch(noop);
-    // not doing db yet
-    // db.modifyBuf = new SharedArrayBuffer(db.maxModifySize)
     db.dbCtxExternal = native.start(id);
     let writelog = null;
     try {
@@ -94,7 +89,7 @@ export async function start(db, opts) {
         const oldHash = hexToBuf(writelog.hash);
         const newHash = db.merkleTree.getRoot()?.hash;
         if (!hashEq(oldHash, newHash)) {
-            console.error(`WARN: CSMT hash mismatch: ${writelog.hash} != ${bufToHex(newHash)}`);
+            console.error(`WARN: CSMT hash mismatch. expected: ${writelog.hash} actual: ${bufToHex(newHash)}`);
         }
     }
     // start workers
@@ -105,7 +100,7 @@ export async function start(db, opts) {
         db.workers[i] = new DbWorker(address, db);
     }
     if (!opts?.hosted) {
-        db.unlistenExit = exitHook(async (signal) => {
+        db.unlistenExit = exitHook((signal) => {
             const blockSig = () => { };
             const signals = ['SIGINT', 'SIGTERM', 'SIGHUP'];
             // A really dumb way to block signals temporarily while saving.

package/dist/src/server/tree.d.ts

@@ -9,5 +9,5 @@ export type CsmtNodeRange = {
 export declare const destructureCsmtKey: (key: number) => number[];
 export declare const makeCsmtKey: (typeId: number, start: number) => number;
 export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
-export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): Promise<void>;
+export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): void;
 export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;

package/dist/src/server/tree.js

@@ -8,7 +8,7 @@ export const makeCsmtKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
     const tmp = nodeId - +!(nodeId % blockCapacity);
     return typeId * 4294967296 + ((tmp / blockCapacity) | 0) * blockCapacity + 1;
 };
-export async function foreachBlock(db, def, cb) {
+export function foreachBlock(db, def, cb) {
     const step = def.blockCapacity;
     for (let start = 1; start <= def.lastId; start += step) {
         const end = start + step - 1;

package/dist/src/utils.d.ts

@@ -5,3 +5,9 @@ export declare function concatUint8Arr(bufs: Uint8Array[], totalByteLength?: num
 export declare const bufToHex: (a: Uint8Array) => string;
 export declare const hexToBuf: (s: string) => Uint8Array;
 export declare const base64encode: (a: Uint8Array, lineMax?: number) => string;
+export declare const readDoubleLE: (val: Uint8Array, offset: number) => number;
+export declare const readFloatLE: (val: Uint8Array, offset: number) => number;
+export declare const readUint32: (val: Uint8Array, offset: number) => number;
+export declare const readInt32: (val: Uint8Array, offset: number) => number;
+export declare const readInt16: (val: Uint8Array, offset: number) => number;
+export declare const readUint16: (val: Uint8Array, offset: number) => number;

package/dist/src/utils.js

@@ -1,5 +1,5 @@
 import { encodeBase64 } from '@saulx/utils';
-const native = (typeof window === 'undefined') ? (await import('./native.js')).default : null;
+const native = typeof window === 'undefined' ? (await import('./native.js')).default : null;
 export const DECODER = new TextDecoder('utf-8');
 export const ENCODER = new TextEncoder();
 export const equals = (aB, bB) => {
@@ -28,7 +28,8 @@ export const equals = (aB, bB) => {
     }
 };
 export function concatUint8Arr(bufs, totalByteLength) {
-    totalByteLength = totalByteLength ?? bufs.reduce((acc, cur) => acc + cur.byteLength, 0);
+    totalByteLength =
+        totalByteLength ?? bufs.reduce((acc, cur) => acc + cur.byteLength, 0);
     const res = new Uint8Array(totalByteLength);
     let off = 0;
     for (let i = 0; i < bufs.length; i++) {
@@ -62,12 +63,12 @@ const intMap = {
     '7': 0x7,
     '8': 0x8,
     '9': 0x9,
-    'a': 0xa,
-    'b': 0xb,
-    'c': 0xc,
-    'd': 0xd,
-    'e': 0xe,
-    'f': 0xf,
+    a: 0xa,
+    b: 0xb,
+    c: 0xc,
+    d: 0xd,
+    e: 0xe,
+    f: 0xf,
 };
 // Uint8Array.fromHex() and Uint8Array.toHex() are not available in V8
 // https://issues.chromium.org/issues/42204568
@@ -85,7 +86,7 @@ function base64OutLen(n, lineMax) {
     let olen;
     /* This version would be with padding but we don't pad */
     //olen = n * 4 / 3 + 4; /* 3-byte blocks to 4-byte */
-    olen = ((4 * n / 3) + 3) & ~3;
+    olen = ((4 * n) / 3 + 3) & ~3;
     olen += lineMax > 0 ? olen / lineMax : 0; // line feeds
     return olen;
 }
@@ -99,4 +100,75 @@ export const base64encode = (a, lineMax = 72) => {
         return DECODER.decode(native.base64encode(tmp, a, lineMax));
     }
 };
+export const readDoubleLE = (val, offset) => {
+    const low = (val[offset] |
+        (val[offset + 1] << 8) |
+        (val[offset + 2] << 16) |
+        (val[offset + 3] << 24)) >>>
+        0;
+    const high = (val[offset + 4] |
+        (val[offset + 5] << 8) |
+        (val[offset + 6] << 16) |
+        (val[offset + 7] << 24)) >>>
+        0;
+    const sign = high >>> 31 ? -1 : 1;
+    let exponent = (high >>> 20) & 0x7ff;
+    let fraction = (high & 0xfffff) * 2 ** 32 + low;
+    if (exponent === 0x7ff) {
+        if (fraction === 0)
+            return sign * Infinity;
+        return NaN;
+    }
+    if (exponent === 0) {
+        if (fraction === 0)
+            return sign * 0;
+        exponent = 1;
+    }
+    else {
+        fraction += 2 ** 52;
+    }
+    return sign * fraction * 2 ** (exponent - 1075);
+};
+export const readFloatLE = (val, offset) => {
+    const bits = val[offset] |
+        (val[offset + 1] << 8) |
+        (val[offset + 2] << 16) |
+        (val[offset + 3] << 24);
+    const sign = bits >>> 31 ? -1 : 1;
+    let exponent = (bits >>> 23) & 0xff;
+    let fraction = bits & 0x7fffff;
+    if (exponent === 0xff) {
+        if (fraction === 0)
+            return sign * Infinity;
+        return NaN;
+    }
+    if (exponent === 0) {
+        if (fraction === 0)
+            return sign * 0;
+        exponent = 1;
+    }
+    else {
+        fraction |= 0x800000;
+    }
+    return sign * fraction * 2 ** (exponent - 150);
+};
+export const readUint32 = (val, offset) => {
+    return ((val[offset] |
+        (val[offset + 1] << 8) |
+        (val[offset + 2] << 16) |
+        (val[offset + 3] << 24)) >>>
+        0);
+};
+export const readInt32 = (val, offset) => {
+    return (val[offset] |
+        (val[offset + 1] << 8) |
+        (val[offset + 2] << 16) |
+        (val[offset + 3] << 24));
+};
+export const readInt16 = (val, offset) => {
+    return ((val[offset] | (val[offset + 1] << 8)) << 16) >> 16;
+};
+export const readUint16 = (val, offset) => {
+    return (val[offset] | (val[offset + 1] << 8)) >>> 0;
+};
 //# sourceMappingURL=utils.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.24",
+  "version": "0.0.26",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",
@@ -32,9 +32,9 @@
     "basedDbNative.cjs"
   ],
   "dependencies": {
-    "@based/schema": "5.0.0-alpha.7",
+    "@based/schema": "5.0.0-alpha.8",
     "@saulx/hash": "^3.0.0",
-    "@saulx/utils": "^4.3.2",
+    "@saulx/utils": "^6.1.1",
     "exit-hook": "^4.0.0",
     "picocolors": "^1.1.0",
     "@based/crc32c": "^1.0.0"
@@ -50,4 +50,4 @@
     "tar": "^7.4.3",
     "typescript": "^5.6.3"
   }
-}
+}