@based/db 0.0.48 → 0.0.50

package/dist/src/client/index.js

@@ -10,7 +10,8 @@ import { deleteFn } from './modify/delete.js';
 import { wait } from '@saulx/utils';
 import { hash } from '@saulx/hash';
 import { expire } from './modify/expire.js';
-import { debugMode, schemaLooseEqual } from '../utils.js';
+import { debugMode } from '../utils.js';
+import { parseSchema, schemaLooseEqual } from '../schema.js';
 const makeFlushIsReady = (dbClient) => {
     dbClient.flushIsReady = new Promise((resolve) => {
         dbClient.flushReady = () => {
@@ -48,8 +49,9 @@ export class DbClient {
     schemaPromise;
     async setSchema(schema, fromStart, transformFns) {
         const strictSchema = fromStart ? schema : parse(schema).schema;
+        const parsedSchema = parseSchema(strictSchema);
         // this one excludes all the ids
-        if (schemaLooseEqual(strictSchema, this.schema)) {
+        if (schemaLooseEqual(parsedSchema, this.schema)) {
             return this.schema;
         }
         // drain current things

package/dist/src/client/modify/references/references.js

@@ -1,7 +1,7 @@
 import { REFERENCES } from '@based/schema/def';
 import { ModifyError, ModifyState } from '../ModifyRes.js';
 import { setCursor } from '../setCursor.js';
-import { DELETE, EDGE_INDEX_REALID, EDGE_INDEX_TMPID, EDGE_NOINDEX_REALID, EDGE_NOINDEX_TMPID, NOEDGE_INDEX_REALID, NOEDGE_INDEX_TMPID, NOEDGE_NOINDEX_REALID, NOEDGE_NOINDEX_TMPID, RANGE_ERR, REF_OP_UPDATE, } from '../types.js';
+import { DELETE, EDGE_INDEX_REALID, EDGE_INDEX_TMPID, EDGE_NOINDEX_REALID, EDGE_NOINDEX_TMPID, NOEDGE_INDEX_REALID, NOEDGE_INDEX_TMPID, NOEDGE_NOINDEX_REALID, NOEDGE_NOINDEX_TMPID, RANGE_ERR, REF_OP_OVERWRITE, REF_OP_PUT_ADD, REF_OP_PUT_OVERWRITE, REF_OP_UPDATE, } from '../types.js';
 import { writeEdges } from './edge.js';
 export function writeReferences(value, ctx, schema, def, res, mod) {
     if (typeof value !== 'object') {
@@ -266,8 +266,8 @@ function putRefs(def, ctx, modifyOp, refs, op) {
     ctx.buf[ctx.len++] = size >>>= 8;
     ctx.buf[ctx.len++] = size >>>= 8;
     ctx.buf[ctx.len++] = size >>>= 8;
-    ctx.buf[ctx.len++] = op === 0 ? 3 : 4;
-    ctx.len = (ctx.len + 3) & ~3;
+    ctx.buf[ctx.len++] =
+        op === REF_OP_OVERWRITE ? REF_OP_PUT_OVERWRITE : REF_OP_PUT_ADD;
     let i = 0;
     for (; i < refs.length; i++) {
         let ref = refs[i];

package/dist/src/client/modify/types.d.ts

@@ -37,4 +37,6 @@ export declare const EDGE_INDEX_TMPID = 6;
 export declare const NOEDGE_INDEX_TMPID = 7;
 export declare const REF_OP_UPDATE = 1;
 export declare const REF_OP_OVERWRITE = 0;
+export declare const REF_OP_PUT_OVERWRITE = 3;
+export declare const REF_OP_PUT_ADD = 4;
 export type REF_OP = typeof REF_OP_OVERWRITE | typeof REF_OP_UPDATE;

package/dist/src/client/modify/types.js

@@ -25,4 +25,6 @@ export const EDGE_INDEX_TMPID = 6;
 export const NOEDGE_INDEX_TMPID = 7;
 export const REF_OP_UPDATE = 1;
 export const REF_OP_OVERWRITE = 0;
+export const REF_OP_PUT_OVERWRITE = 3;
+export const REF_OP_PUT_ADD = 4;
 //# sourceMappingURL=types.js.map

package/dist/src/client/query/BasedDbQuery.js

@@ -12,6 +12,8 @@ import { convertFilter } from './filter/convertFilter.js';
 import { validateLocale, validateRange } from './validation.js';
 import { DEF_RANGE_PROP_LIMIT } from './thresholds.js';
 import { concatUint8Arr } from '@saulx/utils';
+import { displayTarget } from './display.js';
+import picocolors from 'picocolors';
 export class QueryBranch {
     db;
     def;
@@ -325,6 +327,10 @@ export class BasedDbQuery extends QueryBranch {
         const res = await this.db.hooks.getQueryBuf(buf);
         if (res.byteLength === 1) {
             if (res[0] === 0) {
+                if (this.db.schema?.hash !== this.def.schemaChecksum) {
+                    this.reBuildQuery();
+                    return this.#getInternal(resolve, reject);
+                }
                 reject(new Error('schema mismatch'));
             }
             else {
@@ -353,7 +359,19 @@ export class BasedDbQuery extends QueryBranch {
         return this;
     }
     subscribe(onData, onError) {
-        return subscribe(this, onData, onError ??
+        return subscribe(this, (res) => {
+            try {
+                onData(res);
+            }
+            catch (err) {
+                // const t = displayTarget(this.def)
+                const def = this.def;
+                let name = picocolors.red(`QueryError[${displayTarget(def)}]\n`);
+                name += `  Error executing onData handler in subscription\n`;
+                name += `  ${err.message}\n`;
+                console.error(name);
+            }
+        }, onError ??
             ((err) => {
                 console.error(err);
             }));

package/dist/src/schema.d.ts

@@ -0,0 +1,3 @@
+import { StrictSchema } from '@based/schema';
+export declare const schemaLooseEqual: (a: any, b: any, key?: string) => boolean;
+export declare const parseSchema: (strictSchema: StrictSchema) => StrictSchema;

package/dist/src/schema.js

@@ -0,0 +1,94 @@
+import { getPropType } from '@based/schema';
+const exclude = new Set(['id', 'lastId', 'hash']);
+export const schemaLooseEqual = (a, b, key) => {
+    if (a === b) {
+        return true;
+    }
+    const typeofA = typeof a;
+    if (typeofA !== 'object') {
+        return exclude.has(key);
+    }
+    const typeofB = typeof b;
+    if (typeofA !== typeofB) {
+        return exclude.has(key);
+    }
+    if (a === null || b === null) {
+        return false;
+    }
+    if (a.constructor !== b.constructor) {
+        return false;
+    }
+    if (Array.isArray(a)) {
+        let i = a.length;
+        if (i !== b.length) {
+            return false;
+        }
+        while (i--) {
+            if (!schemaLooseEqual(a[i], b[i])) {
+                return false;
+            }
+        }
+    }
+    else {
+        for (const k in a) {
+            if (!schemaLooseEqual(a[k], b[k], k)) {
+                return false;
+            }
+        }
+        for (const k in b) {
+            if (k in a) {
+                continue;
+            }
+            if (!schemaLooseEqual(a[k], b[k], k)) {
+                return false;
+            }
+        }
+    }
+    return true;
+};
+export const parseSchema = (strictSchema) => {
+    let parsedSchema = strictSchema;
+    if (strictSchema.props) {
+        parsedSchema = {
+            ...strictSchema,
+            types: { ...parsedSchema.types },
+        };
+        const props = { ...strictSchema.props };
+        for (const key in props) {
+            const prop = props[key];
+            const propType = getPropType(prop);
+            let refProp;
+            if (propType === 'reference') {
+                refProp = prop;
+            }
+            else if (propType === 'references') {
+                refProp = prop.items;
+            }
+            if (refProp) {
+                const type = parsedSchema.types[refProp.ref];
+                const inverseKey = '_' + key;
+                parsedSchema.types[refProp.ref] = {
+                    ...type,
+                    props: {
+                        ...type.props,
+                        [inverseKey]: {
+                            items: {
+                                ref: '_root',
+                                prop: key,
+                            },
+                        },
+                    },
+                };
+                refProp.prop = inverseKey;
+            }
+        }
+        // @ts-ignore This creates an internal type to use for root props
+        parsedSchema.types._root = {
+            id: 1,
+            props,
+        };
+        delete parsedSchema.props;
+    }
+    return parsedSchema;
+};
+//# sourceMappingURL=schema.js.map

package/dist/src/server/index.js

@@ -2,7 +2,7 @@ import native from '../native.js';
 import createDbHash from './dbHash.js';
 import { rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
-import { getPropType, langCodesMap, } from '@based/schema';
+import { langCodesMap } from '@based/schema';
 import { updateTypeDefs, schemaToSelvaBuffer, } from '@based/schema/def';
 import { start } from './start.js';
 import { initCsmt, makeCsmtKey, makeCsmtKeyFromNodeId, } from './tree.js';
@@ -11,10 +11,11 @@ import { Worker, MessageChannel } from 'node:worker_threads';
 import { fileURLToPath } from 'node:url';
 import { setTimeout } from 'node:timers/promises';
 import { migrate } from './migrate/index.js';
-import { debugServer, schemaLooseEqual } from '../utils.js';
+import { debugServer } from '../utils.js';
 import { readUint16, readUint32, readUint64, writeUint64 } from '@saulx/utils';
-import { hash } from '@saulx/hash';
 import { QueryType } from '../client/query/types.js';
+import { parseSchema, schemaLooseEqual } from '../schema.js';
+import { hash } from '@saulx/hash';
 export const SCHEMA_FILE = 'schema.json';
 export const WRITELOG_FILE = 'writelog.json';
 const __filename = fileURLToPath(import.meta.url);
@@ -281,64 +282,26 @@ export class DbServer {
         return sortIndex;
     }
     setSchema(strictSchema, fromStart = false, transformFns) {
+        const parsedSchema = parseSchema(strictSchema);
         if (!fromStart && Object.keys(this.schema.types).length > 0) {
-            if (schemaLooseEqual(strictSchema, this.schema)) {
+            if (schemaLooseEqual(parsedSchema, this.schema)) {
                 return this.schema;
             }
             return this.migrateSchema(strictSchema, transformFns);
         }
-        const { lastId } = this.schema;
         this.schema = {
-            lastId,
-            ...strictSchema,
+            lastId: this.schema.lastId,
+            ...parsedSchema,
         };
-        if (strictSchema.props) {
-            this.schema.types ??= {};
-            const props = { ...strictSchema.props };
-            for (const key in props) {
-                const prop = props[key];
-                const propType = getPropType(prop);
-                let refProp;
-                if (propType === 'reference') {
-                    refProp = prop;
-                }
-                else if (propType === 'references') {
-                    refProp = prop.items;
-                }
-                if (refProp) {
-                    const type = this.schema.types[refProp.ref];
-                    const inverseKey = '_' + key;
-                    this.schema.types[refProp.ref] = {
-                        ...type,
-                        props: {
-                            ...type.props,
-                            [inverseKey]: {
-                                items: {
-                                    ref: '_root',
-                                    prop: key,
-                                },
-                            },
-                        },
-                    };
-                    refProp.prop = inverseKey;
-                }
-            }
-            // @ts-ignore This creates an internal type to use for root props
-            this.schema.types._root = {
-                id: 1,
-                props,
-            };
-            delete this.schema.props;
-        }
         for (const field in this.schema.types) {
             if (!('id' in this.schema.types[field])) {
                 this.schema.lastId++;
                 this.schema.types[field].id = this.schema.lastId;
             }
         }
+        const { hash: _, ...rest } = this.schema;
+        this.schema.hash = hash(rest);
         updateTypeDefs(this.schema, this.schemaTypesParsed, this.schemaTypesParsedById);
-        const { hash: _, ...schemaWithoutHash } = this.schema;
-        this.schema.hash = hash(schemaWithoutHash);
         if (!fromStart) {
             writeFile(join(this.fileSystemPath, SCHEMA_FILE), JSON.stringify(this.schema)).catch((err) => console.error('!!!', SCHEMA_FILE, err));
             let types = Object.keys(this.schemaTypesParsed);
@@ -354,7 +317,7 @@ export class DbServer {
                     console.error('Cannot update schema on selva', type.type, err, s[i]);
                 }
             }
-            if (strictSchema.props) {
+            if (strictSchema.props || strictSchema.types?._root) {
                 // insert a root node
                 // TODO fix this add it in schema at least
                 const data = [2, 1, 0, 0, 0, 1, 9, 1, 0, 0, 0, 7, 1, 0, 1];

package/dist/src/server/start.js

@@ -2,13 +2,12 @@ import { DbWorker, SCHEMA_FILE, WRITELOG_FILE } from './index.js';
 import native from '../native.js';
 import { rm, mkdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { hashEq } from './csmt/index.js';
-import { foreachBlock, initCsmt, makeCsmtKey } from './tree.js';
+import { destructureCsmtKey, foreachBlock, initCsmt, makeCsmtKey, specialBlock } from './tree.js';
 import { availableParallelism } from 'node:os';
 import exitHook from 'exit-hook';
 import { save } from './save.js';
 import { DEFAULT_BLOCK_CAPACITY } from '@based/schema/def';
-import { bufToHex, hexToBuf } from '@saulx/utils';
+import { bufToHex } from '@saulx/utils';
 export async function start(db, opts) {
     const path = db.fileSystemPath;
     const noop = () => { };
@@ -74,10 +73,27 @@ export async function start(db, opts) {
         }, true);
     }
     if (writelog?.hash) {
-        const oldHash = hexToBuf(writelog.hash);
-        const newHash = db.merkleTree.getRoot()?.hash;
-        if (!hashEq(oldHash, newHash)) {
-            console.error(`WARN: CSMT hash mismatch. expected: ${writelog.hash} actual: ${bufToHex(newHash)}`);
+        // FIXME FDN-1301
+        //const oldHash = hexToBuf(writelog.hash)
+        //const newHash = db.merkleTree.getRoot()?.hash
+        //if (!hashEq(oldHash, newHash)) {
+        //  console.error(
+        //    `WARN: CSMT hash mismatch. expected: ${writelog.hash} actual: ${bufToHex(newHash)}`,
+        //  )
+        //}
+        const oldHashSet = new Set();
+        const newHashSet = new Set();
+        for (let k in writelog.rangeDumps)
+            writelog.rangeDumps[k].forEach(({ hash }) => oldHashSet.add(hash));
+        db.merkleTree.visitLeafNodes(({ key, hash }) => {
+            const [_typeId, start] = destructureCsmtKey(key);
+            if (start == specialBlock)
+                return; // skip the type specialBlock
+            newHashSet.add(bufToHex(hash));
+        });
+        const setEq = (a, b) => a.size === b.size && [...a].every(value => b.has(value));
+        if (!setEq(oldHashSet, newHashSet)) {
+            console.error(`WARN: CSMT hash mismatch.`);
         }
     }
     // start workers

package/dist/src/utils.d.ts

@@ -3,4 +3,3 @@ export declare const DECODER: TextDecoder;
 export declare const ENCODER: TextEncoder;
 export declare const debugMode: (target: any, getInfo?: any) => void;
 export declare const debugServer: (server: DbServer) => void;
-export declare const schemaLooseEqual: (a: any, b: any, key?: string) => boolean;

package/dist/src/utils.js

@@ -43,51 +43,4 @@ export const debugMode = (target, getInfo = null) => {
     }
 };
 export const debugServer = (server) => debugMode(server, () => `p: ${server.processingQueries} m: ${server.modifyQueue.length} q: ${server.queryQueue.size}`);
-const exclude = new Set(['id', 'lastId', 'hash']);
-export const schemaLooseEqual = (a, b, key) => {
-    if (a === b) {
-        return true;
-    }
-    const typeofA = typeof a;
-    if (typeofA !== 'object') {
-        return exclude.has(key);
-    }
-    const typeofB = typeof b;
-    if (typeofA !== typeofB) {
-        return exclude.has(key);
-    }
-    if (a === null || b === null) {
-        return false;
-    }
-    if (a.constructor !== b.constructor) {
-        return false;
-    }
-    if (Array.isArray(a)) {
-        let i = a.length;
-        if (i !== b.length) {
-            return false;
-        }
-        while (i--) {
-            if (!schemaLooseEqual(a[i], b[i])) {
-                return false;
-            }
-        }
-    }
-    else {
-        for (const k in a) {
-            if (!schemaLooseEqual(a[k], b[k], k)) {
-                return false;
-            }
-        }
-        for (const k in b) {
-            if (k in a) {
-                continue;
-            }
-            if (!schemaLooseEqual(a[k], b[k], k)) {
-                return false;
-            }
-        }
-    }
-    return true;
-};
 //# sourceMappingURL=utils.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.48",
+  "version": "0.0.50",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",