@based/db 0.0.28 → 0.0.30

package/dist/src/index.js

@@ -2,8 +2,8 @@ import { compress, decompress } from './client/string.js';
 import { ModifyCtx } from './client/flushModify.js';
 import { DbServer } from './server/index.js';
 import { DbClient } from './client/index.js';
-import picocolors from 'picocolors';
 import { wait } from '@saulx/utils';
+import { debugMode, debugServer } from './utils.js';
 export * from './client/modify/modify.js';
 export { compress, decompress };
 export { ModifyCtx }; // TODO move this somewhere
@@ -23,16 +23,11 @@ export class BasedDb {
     constructor(opts) {
         this.#init(opts);
         if (opts.debug) {
-            for (const key in this) {
-                const fn = this[key];
-                if (typeof fn === 'function') {
-                    // @ts-ignore
-                    this[key] = function () {
-                        const str = [`[${key}]`, ...arguments].join(' ');
-                        console.info(picocolors.dim(str));
-                        return fn.apply(this, arguments);
-                    };
-                }
+            if (opts.debug === 'server') {
+                debugServer(this.server);
+            }
+            else {
+                debugMode(this);
             }
         }
     }
@@ -49,7 +44,10 @@ export class BasedDb {
         const client = new DbClient({
             maxModifySize,
             hooks: {
-                flushTime: 0,
+                subscribe(q, onData, onError) {
+                    console.warn('Subscription not supported without based-server!');
+                    return () => { };
+                },
                 setSchema(schema, fromStart) {
                     return Promise.resolve(server.setSchema(schema, fromStart));
                 },
@@ -59,8 +57,6 @@ export class BasedDb {
                         offsets,
                     });
                 },
-                flushReady: () => { },
-                flushIsReady: new Promise(() => { }),
                 getQueryBuf(buf) {
                     return Promise.resolve(server.getQueryBuf(buf));
                 },
@@ -127,7 +123,7 @@ export class BasedDb {
         await this.isModified();
         // Tmp fix: Gives node time to GC existing buffers else it can incorrectly re-asign to mem
         // Todo: clear all active queries, queues ETC
-        await wait(Math.max(this.client.hooks.flushTime + 10, 10));
+        await wait(Math.max(this.client.flushTime + 10, 10));
         this.client.destroy();
         await this.server.destroy();
     }

package/dist/src/native.d.ts

@@ -1,4 +1,4 @@
-declare const _default: {
+declare const native: {
     historyAppend(history: any, typeId: number, nodeId: number, dbCtx: any): any;
     historyCreate(pathname: string, mainLen: number): any;
     workerCtxInit: () => void;
@@ -21,7 +21,7 @@ declare const _default: {
     createSortIndex: (buf: Uint8Array, dbCtx: any) => any;
     destroySortIndex: (buf: Uint8Array, dbCtx: any) => any;
     xxHash64: (buf: Uint8Array, target: Uint8Array, index: number) => any;
-    base64encode: (dst: Uint8Array, src: Uint8Array, lineMax: number) => Uint8Array;
     equals: (a: Uint8Array, b: Uint8Array) => boolean;
+    expire: (dbCtx: any) => void;
 };
-export default _default;
+export default native;

package/dist/src/native.js

@@ -11,7 +11,7 @@ function SelvaIoErrlogToString(buf) {
     let len = (i = buf.indexOf(0)) >= 0 ? i : buf.byteLength;
     return DECODER.decode(selvaIoErrlog.slice(0, len));
 }
-export default {
+const native = {
     historyAppend(history, typeId, nodeId, dbCtx) {
         return db.historyAppend(history, typeId, nodeId, dbCtx);
     },
@@ -90,11 +90,13 @@ export default {
     xxHash64: (buf, target, index) => {
         return db.xxHash64(buf, target, index);
     },
-    base64encode: (dst, src, lineMax) => {
-        return db.base64encode(dst, src, lineMax);
-    },
     equals: (a, b) => {
         return !!db.equals(a, b);
     },
+    expire: (dbCtx) => {
+        db.expire(dbCtx);
+    },
 };
+global.__basedDb__native__ = native;
+export default native;
 //# sourceMappingURL=native.js.map

package/dist/src/server/csmt/draw-dot.js

@@ -1,6 +1,6 @@
-import { base64encode } from '../../utils.js';
+import { encodeBase64 } from '@saulx/utils';
 function makeLabel(node) {
-    return `${node.key}\n${base64encode(node.hash, 0).substring(0, 5)}`;
+    return `${node.key}\n${encodeBase64(node.hash).substring(0, 5)}`;
 }
 export default function draw(csmt) {
     const root = csmt.getRoot();

package/dist/src/server/csmt/tree.js

@@ -12,9 +12,7 @@ export function createTree(createHash) {
         return createHash().update(lHash).update(rHash).digest();
     }
     function createNode(left, right) {
-        const hash = left && right
-            ? genNodeHash(left.hash, right.hash)
-            : emptyHash;
+        const hash = left && right ? genNodeHash(left.hash, right.hash) : emptyHash;
         return {
             hash,
             key: max(left, right),
@@ -56,7 +54,7 @@ export function createTree(createHash) {
                 return createNode(newLeaf, node);
             }
             else {
-                throw new Error('Key exist');
+                throw new Error(`k=${k} exists`);
             }
         }
         const lDist = distance(k, (left && left.key) || TreeKeyNil);
@@ -120,6 +118,47 @@ export function createTree(createHash) {
             }
         }
     }
+    function updateNodeHash(node) {
+        if (node.left && node.right) {
+            node.hash = genNodeHash(node.left.hash, node.right.hash);
+        }
+    }
+    function updateHash(node, k, hash) {
+        if (!node)
+            return;
+        const { left, right } = node;
+        if (k === node.key && !left && !right) {
+            node.hash = hash;
+        }
+        else {
+            if (left && left.key === k) {
+                updateHash(left, k, hash);
+                updateNodeHash(left);
+                updateNodeHash(node);
+                return;
+            }
+            if (right && right.key === k) {
+                updateHash(right, k, hash);
+                updateNodeHash(right);
+                updateNodeHash(node);
+                return;
+            }
+            const lDist = distance(k, (left && left.key) || TreeKeyNil);
+            const rDist = distance(k, (right && right.key) || TreeKeyNil);
+            if (left && lDist <= rDist) {
+                updateHash(left, k, hash);
+                updateNodeHash(left);
+                updateNodeHash(node);
+                return;
+            }
+            if (right && rDist <= lDist) {
+                updateHash(right, k, hash);
+                updateNodeHash(right);
+                updateNodeHash(node);
+                return;
+            }
+        }
+    }
     function diffAB(diffA, diffB, nodeA, nodeB) {
         if (nodeA &&
             nodeB &&
@@ -190,13 +229,20 @@ export function createTree(createHash) {
         if (!node || (k === node.key && !node.left && !node.right))
             return node;
         const { left, right } = node;
-        if (left && k <= left.key)
+        if (left && left.key === k)
+            return search(left, k);
+        if (right && right.key === k)
+            return search(right, k);
+        const lDist = distance(k, (left && left.key) || TreeKeyNil);
+        const rDist = distance(k, (right && right.key) || TreeKeyNil);
+        if (left && lDist <= rDist)
             return search(left, k);
-        if (right && k <= right.key)
+        if (right && rDist <= lDist)
             return search(right, k);
         return null;
     }
     return {
+        emptyHash,
         getRoot: () => root,
         insert: (k, h, data = null) => {
             if (!(h instanceof Uint8Array)) {
@@ -205,6 +251,11 @@ export function createTree(createHash) {
             const newLeaf = createLeaf(k, h, data);
             root = root ? insert(root, newLeaf) : newLeaf;
         },
+        update: (k, h) => {
+            if (root) {
+                updateHash(root, k, h);
+            }
+        },
         delete: (k) => {
             if (root) {
                 root = deleteNode(root, k);

package/dist/src/server/csmt/types.d.ts

@@ -15,6 +15,7 @@ export interface TreeDiff {
     right: KeyHashPair[];
 }
 export interface Csmt {
+    emptyHash: Uint8Array;
     /**
      * Get the root node.
      */
@@ -27,6 +28,10 @@ export interface Csmt {
      * Delete a key-hash pair from the tree.
      */
     delete: (k: TreeKey) => void;
+    /**
+     * Update node hash.
+     */
+    update: (k: TreeKey, h: Hash) => void;
     /**
      * Compute the diff between this and a given tree.
      */

package/dist/src/server/index.d.ts

@@ -50,10 +50,11 @@ export declare class DbServer {
     stopped: boolean;
     onSchemaChange: OnSchemaChange;
     unlistenExit: ReturnType<typeof exitHook>;
-    constructor({ path, maxModifySize, onSchemaChange, }: {
+    constructor({ path, maxModifySize, onSchemaChange, debug, }: {
         path: string;
         maxModifySize?: number;
         onSchemaChange?: OnSchemaChange;
+        debug?: boolean;
     });
     start(opts?: {
         clean?: boolean;
@@ -85,14 +86,14 @@ export declare class DbServer {
         lastId: number;
     }>;
     createSortIndexBuffer(typeId: number, field: number, start: number, lang: number): SortIndex;
-    updateMerkleTree(): void;
     setSchema(strictSchema: StrictSchema, fromStart?: boolean, transformFns?: TransformFns): (StrictSchema & {
         lastId: number;
     }) | Promise<StrictSchema & {
         lastId: number;
     }>;
     modify(buf: Uint8Array): Record<number, number>;
-    getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
+    addToQueryQueue(resolve: any, buf: any): void;
+    getQueryBuf(buf: Uint8Array, fromQueue?: boolean): Promise<Uint8Array>;
     onQueryEnd(): void;
     stop(noSave?: boolean): Promise<void>;
     destroy(): Promise<void>;

package/dist/src/server/index.js

@@ -4,14 +4,14 @@ import { rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { getPropType, langCodesMap, } from '@based/schema';
 import { updateTypeDefs, schemaToSelvaBuffer, } from '@based/schema/def';
-import { hashEq } from './csmt/index.js';
 import { start } from './start.js';
-import { foreachDirtyBlock, makeCsmtKey, makeCsmtKeyFromNodeId, } from './tree.js';
+import { initCsmt, makeCsmtKey, makeCsmtKeyFromNodeId } from './tree.js';
 import { save } from './save.js';
 import { Worker, MessageChannel } from 'node:worker_threads';
 import { fileURLToPath } from 'node:url';
 import { setTimeout } from 'node:timers/promises';
 import { migrate } from './migrate/index.js';
+import { debugServer } from '../utils.js';
 export const SCHEMA_FILE = 'schema.json';
 export const WRITELOG_FILE = 'writelog.json';
 const __filename = fileURLToPath(import.meta.url);
@@ -45,10 +45,6 @@ export class DbWorker {
             transferList: [port2],
         });
         port1.on('message', (buf) => {
-            // TODO FIX TYPES CHECK IF THIS MAKES A COPY
-            // It's a copy, if you don't want a copy you'd need to make it an explicit view
-            // to the underlying buffer:
-            // new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
             this.resolvers.shift()(new Uint8Array(buf));
             this.db.onQueryEnd();
         });
@@ -93,11 +89,14 @@ export class DbServer {
     stopped;
     onSchemaChange;
     unlistenExit;
-    constructor({ path, maxModifySize = 100 * 1e3 * 1e3, onSchemaChange, }) {
+    constructor({ path, maxModifySize = 100 * 1e3 * 1e3, onSchemaChange, debug, }) {
         this.maxModifySize = maxModifySize;
         this.fileSystemPath = path;
         this.sortIndexes = {};
         this.onSchemaChange = onSchemaChange;
+        if (debug) {
+            debugServer(this);
+        }
     }
     #resizeModifyDirtyRanges() {
         let maxNrChanges = 0;
@@ -141,7 +140,7 @@ export class DbServer {
                             for (const lang in this.sortIndexes[type][field][start]) {
                                 const sortIndex = this.sortIndexes[type][field][start][lang];
                                 sortIndex.cnt /= 2;
-                                if (sortIndex.cnt < 1) {
+                                if (sortIndex.cnt < 1 && !this.processingQueries) {
                                     native.destroySortIndex(sortIndex.buf, this.dbCtxExternal);
                                     delete this.sortIndexes[type][field][start][lang];
                                 }
@@ -179,7 +178,7 @@ export class DbServer {
         if (sortIndex) {
             return sortIndex;
         }
-        const buf = new Uint8Array(8);
+        const buf = new Uint8Array(9);
         // size [2 type] [1 field]  [2 start] [2 len] [propIndex] [lang]
         // call createSortBuf here
         buf[0] = t.id;
@@ -208,7 +207,6 @@ export class DbServer {
         }
         let sortIndex = fields[prop.start];
         if (sortIndex) {
-            // [2 type] [1 field] [2 start] [1 lang]
             const buf = new Uint8Array(6);
             buf[0] = t.id;
             buf[1] = t.id >>> 8;
@@ -279,29 +277,6 @@ export class DbServer {
         fields[start][lang] = sortIndex;
         return sortIndex;
     }
-    updateMerkleTree() {
-        foreachDirtyBlock(this, (mtKey, typeId, start, end) => {
-            const oldLeaf = this.merkleTree.search(mtKey);
-            const hash = new Uint8Array(16);
-            native.getNodeRangeHash(typeId, start, end, hash, this.dbCtxExternal);
-            if (oldLeaf) {
-                if (hashEq(oldLeaf.hash, hash)) {
-                    return;
-                }
-                try {
-                    this.merkleTree.delete(mtKey);
-                }
-                catch (err) { }
-            }
-            const data = {
-                file: '', // not saved yet
-                typeId,
-                start,
-                end,
-            };
-            this.merkleTree.insert(mtKey, hash, data);
-        });
-    }
     setSchema(strictSchema, fromStart = false, transformFns) {
         if (!fromStart && Object.keys(this.schema.types).length > 0) {
             return this.migrateSchema(strictSchema, transformFns);
@@ -373,6 +348,7 @@ export class DbServer {
             }
             if (strictSchema.props) {
                 // insert a root node
+                // TODO fix this add it in schema at least
                 const data = [2, 1, 0, 0, 0, 1, 9, 1, 0, 0, 0, 7, 1, 0, 1];
                 const blockKey = makeCsmtKey(1, 1);
                 const buf = new Uint8Array(data.length + 2 + 8 + 4);
@@ -387,6 +363,7 @@ export class DbServer {
                 view.setUint32(buf.length - 4, data.length, true);
                 this.modify(buf);
             }
+            initCsmt(this);
         }
         this.onSchemaChange?.(this.schema);
         return this.schema;
@@ -401,7 +378,16 @@ export class DbServer {
             i += 2;
             const startId = readUint32LE(buf, i);
             const def = this.schemaTypesParsedById[typeId];
-            const offset = def.lastId - startId;
+            let offset = def.lastId - startId;
+            if (offset < 0) {
+                console.log('-----------------');
+                console.log(def.type, {
+                    offset,
+                    serverId: def.lastId,
+                    clientId: startId,
+                });
+                offset = 0;
+            }
             buf[i++] = offset;
             buf[i++] = offset >>> 8;
             buf[i++] = offset >>> 16;
@@ -411,6 +397,7 @@ export class DbServer {
             def.lastId = lastId + offset;
             offsets[typeId] = offset;
         }
+        // console.log('modify', this.processingQueries)
         if (this.processingQueries) {
             this.modifyQueue.push(new Uint8Array(buf));
         }
@@ -449,10 +436,17 @@ export class DbServer {
             this.dirtyRanges.add(key);
         }
     }
-    getQueryBuf(buf) {
+    addToQueryQueue(resolve, buf) {
+        if (this.queryQueue.size === 16777216) {
+            resolve(new Error('Query queue exceeded'));
+            return;
+        }
+        this.queryQueue.set(resolve, buf);
+    }
+    getQueryBuf(buf, fromQueue = false) {
         if (this.modifyQueue.length) {
             return new Promise((resolve) => {
-                this.queryQueue.set(resolve, buf);
+                this.addToQueryQueue(resolve, buf);
             });
         }
         else {
@@ -469,7 +463,7 @@ export class DbServer {
                     if (!sortIndex) {
                         if (this.processingQueries) {
                             return new Promise((resolve) => {
-                                this.queryQueue.set(resolve, buf);
+                                this.addToQueryQueue(resolve, buf);
                             });
                         }
                         sortIndex = this.createSortIndexBuffer(typeId, field, start, sort[sort.byteLength - 1]);
@@ -482,6 +476,9 @@ export class DbServer {
             else if (queryType == 1) {
                 // This will be more advanced - sometimes has indexes / sometimes not
             }
+            if (!fromQueue) {
+                native.expire(this.dbCtxExternal);
+            }
             this.availableWorkerIndex =
                 (this.availableWorkerIndex + 1) % this.workers.length;
             return this.workers[this.availableWorkerIndex].getQueryBuf(buf);
@@ -491,16 +488,19 @@ export class DbServer {
         this.processingQueries--;
         if (this.processingQueries === 0) {
             if (this.modifyQueue.length) {
-                for (const buf of this.modifyQueue) {
+                const modifyQueue = this.modifyQueue;
+                this.modifyQueue = [];
+                for (const buf of modifyQueue) {
                     this.#modify(buf);
                 }
-                this.modifyQueue = [];
             }
             if (this.queryQueue.size) {
-                for (const [resolve, buf] of this.queryQueue) {
-                    resolve(this.getQueryBuf(buf));
+                const queryQueue = this.queryQueue;
+                this.queryQueue = new Map();
+                native.expire(this.dbCtxExternal);
+                for (const [resolve, buf] of queryQueue) {
+                    resolve(this.getQueryBuf(buf, true));
                 }
-                this.queryQueue.clear();
             }
         }
     }
@@ -510,7 +510,7 @@ export class DbServer {
         }
         this.stopped = true;
         clearTimeout(this.cleanupTimer);
-        this?.unlistenExit();
+        this.unlistenExit();
         try {
             if (!noSave) {
                 await this.save();
@@ -518,7 +518,7 @@ export class DbServer {
             await Promise.all(this.workers.map(({ worker }) => worker.terminate()));
             this.workers = [];
             native.stop(this.dbCtxExternal);
-            await setTimeout();
+            await setTimeout(20);
         }
         catch (e) {
             this.stopped = false;

package/dist/src/server/migrate/index.js

@@ -4,7 +4,7 @@ import { tmpdir } from 'os';
 import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threads';
 import native from '../../native.js';
 import './worker.js';
-import { foreachDirtyBlock } from '../tree.js';
+import { destructureCsmtKey, foreachDirtyBlock, specialBlock } from '../tree.js';
 import { SCHEMA_FILE } from '../index.js';
 import { fileURLToPath } from 'url';
 import { deepMerge } from '@saulx/utils';
@@ -35,7 +35,7 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
     fromDbServer.migrating = migrationId;
     const abort = () => fromDbServer.migrating !== migrationId;
     const toDb = new BasedDb({
-        path: join(tmpdir(), (~~Math.random()).toString(36)),
+        path: join(tmpdir(), (~~(Math.random() * 1e9)).toString(36)),
     });
     await toDb.start({ clean: true });
     if (abort()) {
@@ -43,6 +43,10 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
         return fromDbServer.schema;
     }
     toSchema = await toDb.setSchema(toSchema);
+    if (abort()) {
+        await toDb.destroy();
+        return fromDbServer.schema;
+    }
     const fromSchema = fromDbServer.schema;
     const fromCtx = fromDbServer.dbCtxExternal;
     const toCtx = toDb.server.dbCtxExternal;
@@ -67,14 +71,18 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
     worker.on('error', console.error);
     let i = 0;
     let ranges = [];
-    // fromDbServer.updateMerkleTree()
-    // fromDbServer.dirtyRanges.clear()
     await fromDbServer.save();
     fromDbServer.merkleTree.visitLeafNodes((leaf) => {
+        const [_typeId, start] = destructureCsmtKey(leaf.key);
+        if (start == specialBlock)
+            return; // skip the type specialBlock
         ranges.push(leaf.data);
     });
     await Atomics.waitAsync(atomics, 0, 1).value;
     while (i < ranges.length) {
+        if (abort()) {
+            break;
+        }
         // block modifies
         fromDbServer.processingQueries++;
         const leafData = ranges[i++];
@@ -86,38 +94,48 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
         await Atomics.waitAsync(atomics, 0, 1).value;
         // exec queued modifies
         fromDbServer.onQueryEnd();
-        if (abort()) {
-            break;
-        }
-        if (i === ranges.length && fromDbServer.dirtyRanges.size) {
-            ranges = [];
-            i = 0;
-            foreachDirtyBlock(fromDbServer, (_mtKey, typeId, start, end) => {
-                ranges.push({
-                    typeId,
-                    start,
-                    end,
+        if (i === ranges.length) {
+            if (fromDbServer.dirtyRanges.size) {
+                ranges = [];
+                i = 0;
+                foreachDirtyBlock(fromDbServer, (_mtKey, typeId, start, end) => {
+                    ranges.push({
+                        typeId,
+                        start,
+                        end,
+                    });
                 });
-            });
-            fromDbServer.dirtyRanges.clear();
+                fromDbServer.dirtyRanges.clear();
+            }
         }
     }
-    if (!abort()) {
-        let msg;
-        let schema;
-        let schemaTypesParsed;
-        while ((msg = receiveMessageOnPort(port1))) {
-            ;
-            [schema, schemaTypesParsed] = msg.message;
-        }
-        fromDbServer.schema = deepMerge(toDb.server.schema, schema);
-        fromDbServer.schemaTypesParsed = deepMerge(toDb.server.schemaTypesParsed, schemaTypesParsed);
-        fromDbServer.dbCtxExternal = toCtx;
-        toDb.server.dbCtxExternal = fromCtx;
+    if (abort()) {
+        await Promise.all([toDb.destroy(), worker.terminate()]);
+        return fromDbServer.schema;
+    }
+    let msg;
+    let schema;
+    let schemaTypesParsed;
+    while ((msg = receiveMessageOnPort(port1))) {
+        ;
+        [schema, schemaTypesParsed] = msg.message;
     }
+    fromDbServer.dbCtxExternal = toCtx;
+    fromDbServer.sortIndexes = {};
+    fromDbServer.schema = deepMerge(toDb.server.schema, schema);
+    fromDbServer.schemaTypesParsed = deepMerge(toDb.server.schemaTypesParsed, schemaTypesParsed);
+    fromDbServer.schemaTypesParsedById = {};
+    for (const key in fromDbServer.schemaTypesParsed) {
+        const def = fromDbServer.schemaTypesParsed[key];
+        fromDbServer.schemaTypesParsedById[def.id] = def;
+    }
+    toDb.server.dbCtxExternal = fromCtx;
     const promises = fromDbServer.workers.map((worker) => worker.updateCtx(toAddress));
     promises.push(toDb.destroy(), worker.terminate(), fromDbServer.save({ forceFullDump: true }), writeFile(join(fromDbServer.fileSystemPath, SCHEMA_FILE), JSON.stringify(fromDbServer.schema)));
     await Promise.all(promises);
+    if (abort()) {
+        return fromDbServer.schema;
+    }
     fromDbServer.onSchemaChange?.(fromDbServer.schema);
     return fromDbServer.schema;
 };

package/dist/src/server/migrate/worker.js

@@ -66,7 +66,7 @@ else {
                 const nodes = fromDb
                     .query(type)
                     .include(include)
-                    .range(leafData.start - 1, leafData.end - leafData.start + 1)
+                    .range(leafData.start - 1, leafData.end - 1)
                     ._getSync(fromCtx);
                 for (const node of nodes) {
                     const res = typeTransformFn(node);
@@ -85,7 +85,7 @@ else {
                 const nodes = fromDb
                     .query(type)
                     .include(include)
-                    .range(leafData.start - 1, leafData.end - leafData.start + 1)
+                    .range(leafData.start - 1, leafData.end - 1)
                     ._getSync(fromCtx);
                 for (const node of nodes) {
                     toDb.create(type, node, { unsafe: true });