@based/db 0.0.52 → 0.0.56

package/dist/src/index.js

@@ -5,9 +5,10 @@ import { DbClient } from './client/index.js';
 import { wait } from '@saulx/utils';
 import { debugMode, debugServer } from './utils.js';
 import { getDefaultHooks } from './hooks.js';
+import { Emitter } from './shared/Emitter.js';
 export * from './client/modify/modify.js';
 export { compress, decompress };
-export { ModifyCtx }; // TODO move this somewhere
+export { ModifyCtx };
 export { DbClient, DbServer };
 export { xxHash64 } from './client/xxHash64.js';
 export { crc32 } from './client/crc32.js';
@@ -18,14 +19,27 @@ export * from './client/query/query.js';
 export * from './client/query/BasedDbQuery.js';
 export * from './client/query/BasedIterable.js';
 export * from './server/save.js';
+export * from './hooks.js';
 export { getDefaultHooks };
-export class BasedDb {
+export class BasedDb extends Emitter {
     client;
     server;
     fileSystemPath;
     maxModifySize;
     constructor(opts) {
-        this.#init(opts);
+        super();
+        this.fileSystemPath = opts.path;
+        this.maxModifySize = opts.maxModifySize;
+        const server = new DbServer({
+            path: opts.path,
+            saveIntervalInSeconds: opts.saveIntervalInSeconds,
+        });
+        const client = new DbClient({
+            maxModifySize: opts.maxModifySize,
+            hooks: getDefaultHooks(server),
+        });
+        this.server = server;
+        this.client = client;
         if (opts.debug) {
             if (opts.debug === 'client') {
                 debugServer(this.server);
@@ -38,25 +52,6 @@ export class BasedDb {
             }
         }
     }
-    #init({ path, maxModifySize, saveIntervalInSeconds, }) {
-        this.fileSystemPath = path;
-        this.maxModifySize = maxModifySize;
-        const server = new DbServer({
-            path,
-            maxModifySize,
-            saveIntervalInSeconds,
-            onSchemaChange(schema) {
-                client.putLocalSchema(schema);
-            },
-        });
-        const client = new DbClient({
-            maxModifySize,
-            hooks: getDefaultHooks(server),
-        });
-        this.server = server;
-        this.client = client;
-    }
-    // client
     create = function () {
         return this.client.create.apply(this.client, arguments);
     };
@@ -78,6 +73,9 @@ export class BasedDb {
     query = function () {
         return this.client.query.apply(this.client, arguments);
     };
+    schemaIsSet = function () {
+        return this.client.schemaIsSet.apply(this.client, arguments);
+    };
     setSchema = function () {
         return this.client.setSchema.apply(this.client, arguments);
     };
@@ -101,15 +99,9 @@ export class BasedDb {
         await this.isModified();
         return this.server.save.apply(this.server, arguments);
     };
-    migrateSchema = function () {
-        return this.server.migrateSchema.apply(this.server, arguments);
-    };
     isModified = function () {
         return this.client.isModified.apply(this.client, arguments);
     };
-    schemaIsSet = function () {
-        return this.client.schemaIsSet.apply(this.client, arguments);
-    };
     async destroy() {
         await this.isModified();
         // Tmp fix: Gives node time to GC existing buffers else it can incorrectly re-asign to mem
@@ -118,20 +110,11 @@ export class BasedDb {
         this.client.destroy();
         await this.server.destroy();
     }
-    async wipe() {
-        const opts = {
-            maxModifySize: this.maxModifySize,
-            path: this.fileSystemPath,
-        };
-        await this.destroy();
-        this.#init(opts);
-        await this.start({ clean: true });
-    }
-    on = function () {
+    on() {
         return this.client.on.apply(this.client, arguments);
-    };
-    off = function () {
+    }
+    off() {
         return this.client.on.apply(this.client, arguments);
-    };
+    }
 }
 //# sourceMappingURL=index.js.map

package/dist/src/schema.d.ts

@@ -1,3 +1,6 @@
 import { StrictSchema } from '@based/schema';
-export declare const schemaLooseEqual: (a: any, b: any, key?: string) => boolean;
-export declare const parseSchema: (strictSchema: StrictSchema) => StrictSchema;
+export type DbSchema = StrictSchema & {
+    lastId: number;
+    hash: number;
+};
+export type SchemaChecksum = number;

package/dist/src/schema.js

@@ -1,94 +1,2 @@
-import { getPropType } from '@based/schema';
-import { deepCopy } from '@saulx/utils';
-const exclude = new Set(['id', 'lastId', 'hash']);
-export const schemaLooseEqual = (a, b, key) => {
-    if (a === b) {
-        return true;
-    }
-    const typeofA = typeof a;
-    if (typeofA !== 'object') {
-        return exclude.has(key);
-    }
-    const typeofB = typeof b;
-    if (typeofA !== typeofB) {
-        return exclude.has(key);
-    }
-    if (a === null || b === null) {
-        return false;
-    }
-    if (a.constructor !== b.constructor) {
-        return false;
-    }
-    if (Array.isArray(a)) {
-        let i = a.length;
-        if (i !== b.length) {
-            return false;
-        }
-        while (i--) {
-            if (!schemaLooseEqual(a[i], b[i])) {
-                return false;
-            }
-        }
-    }
-    else {
-        for (const k in a) {
-            if (!schemaLooseEqual(a[k], b[k], k)) {
-                return false;
-            }
-        }
-        for (const k in b) {
-            if (k in a) {
-                continue;
-            }
-            if (!schemaLooseEqual(a[k], b[k], k)) {
-                return false;
-            }
-        }
-    }
-    return true;
-};
-export const parseSchema = (strictSchema) => {
-    let parsedSchema = strictSchema;
-    if (strictSchema.props) {
-        // TODO do this more precise
-        parsedSchema = deepCopy(strictSchema);
-        for (const key in parsedSchema.props) {
-            const prop = parsedSchema.props[key];
-            const propType = getPropType(prop);
-            let refProp;
-            if (propType === 'reference') {
-                refProp = prop;
-            }
-            else if (propType === 'references') {
-                refProp = prop.items;
-                prop.items = refProp;
-            }
-            if (refProp) {
-                const type = parsedSchema.types[refProp.ref];
-                const inverseKey = '_' + key;
-                parsedSchema.types[refProp.ref] = {
-                    ...type,
-                    props: {
-                        ...type.props,
-                        [inverseKey]: {
-                            items: {
-                                ref: '_root',
-                                prop: key,
-                            },
-                        },
-                    },
-                };
-                refProp.prop = inverseKey;
-            }
-        }
-        parsedSchema.types ??= {};
-        // @ts-ignore This creates an internal type to use for root props
-        parsedSchema.types._root = {
-            id: 1,
-            props: parsedSchema.props,
-        };
-        delete parsedSchema.props;
-    }
-    return parsedSchema;
-};
+export {};
 //# sourceMappingURL=schema.js.map

package/dist/src/server/DbWorker.d.ts

@@ -0,0 +1,12 @@
+import { Worker, MessagePort } from 'node:worker_threads';
+import { DbServer } from './index.js';
+export declare class DbWorker {
+    constructor(address: BigInt, db: DbServer);
+    db: DbServer;
+    channel: MessagePort;
+    worker: Worker;
+    resolvers: any[];
+    callback: (resolve: any) => void;
+    updateCtx(address: BigInt): Promise<void>;
+    getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
+}

package/dist/src/server/DbWorker.js

@@ -0,0 +1,42 @@
+import { MessageChannel, Worker } from 'node:worker_threads';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const workerPath = join(__dirname, 'worker.js');
+export class DbWorker {
+    constructor(address, db) {
+        const { port1, port2 } = new MessageChannel();
+        this.db = db;
+        this.channel = port1;
+        this.worker = new Worker(workerPath, {
+            workerData: {
+                isDbWorker: true,
+                channel: port2,
+                address,
+            },
+            transferList: [port2],
+        });
+        port1.on('message', (buf) => {
+            this.resolvers.shift()(new Uint8Array(buf));
+            this.db.onQueryEnd();
+        });
+    }
+    db;
+    channel;
+    worker;
+    resolvers = [];
+    callback = (resolve) => {
+        this.db.processingQueries++;
+        this.resolvers.push(resolve);
+    };
+    updateCtx(address) {
+        this.channel.postMessage(address);
+        return new Promise(this.callback);
+    }
+    getQueryBuf(buf) {
+        this.channel.postMessage(buf);
+        return new Promise(this.callback);
+    }
+}
+//# sourceMappingURL=DbWorker.js.map

package/dist/src/server/index.d.ts

@@ -1,42 +1,23 @@
 import { LangName, StrictSchema } from '@based/schema';
-import { SchemaTypesParsed, SchemaTypesParsedById } from '@based/schema/def';
 import { createTree } from './csmt/index.js';
 import { CsmtNodeRange } from './tree.js';
-import { Worker, MessagePort } from 'node:worker_threads';
 import { TransformFns } from './migrate/index.js';
 import exitHook from 'exit-hook';
-export declare const SCHEMA_FILE = "schema.json";
-export declare const WRITELOG_FILE = "writelog.json";
+import { SchemaChecksum } from '../schema.js';
+import { DbWorker } from './DbWorker.js';
+import { DbShared } from '../shared/DbBase.js';
 declare class SortIndex {
     constructor(buf: Uint8Array, dbCtxExternal: any);
     buf: Uint8Array;
     idx: any;
     cnt: number;
 }
-export declare class DbWorker {
-    constructor(address: BigInt, db: DbServer);
-    db: DbServer;
-    channel: MessagePort;
-    worker: Worker;
-    resolvers: any[];
-    callback: (resolve: any) => void;
-    updateCtx(address: BigInt): Promise<void>;
-    getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
-}
-type OnSchemaChange = (schema: DbServer['schema']) => void;
-export declare class DbServer {
+export declare class DbServer extends DbShared {
     #private;
     modifyDirtyRanges: Float64Array;
     dbCtxExternal: any;
-    schema: StrictSchema & {
-        lastId: number;
-        hash?: number;
-    };
     migrating: number;
-    schemaTypesParsed: SchemaTypesParsed;
-    schemaTypesParsedById: SchemaTypesParsedById;
     fileSystemPath: string;
-    maxModifySize: number;
     merkleTree: ReturnType<typeof createTree<CsmtNodeRange>>;
     dirtyRanges: Set<number>;
     csmtHashFun: {
@@ -50,14 +31,11 @@ export declare class DbServer {
     modifyQueue: Uint8Array[];
     queryQueue: Map<Function, Uint8Array>;
     stopped: boolean;
-    onSchemaChange: OnSchemaChange;
     unlistenExit: ReturnType<typeof exitHook>;
     saveIntervalInSeconds?: number;
     saveInterval?: NodeJS.Timeout;
-    constructor({ path, maxModifySize, onSchemaChange, debug, saveIntervalInSeconds, }: {
+    constructor({ path, debug, saveIntervalInSeconds, }: {
         path: string;
-        maxModifySize?: number;
-        onSchemaChange?: OnSchemaChange;
         debug?: boolean;
         saveIntervalInSeconds?: number;
     });
@@ -87,18 +65,8 @@ export declare class DbServer {
     createSortIndex(type: string, field: string, lang?: LangName): SortIndex;
     destroySortIndex(type: string, field: string, lang?: LangName): any;
     getSortIndex(typeId: number, field: number, start: number, lang: number): SortIndex;
-    migrateSchema(schema: StrictSchema, transform?: Record<string, (node: Record<string, any>) => Record<string, any> | [string, Record<string, any>]>): Promise<StrictSchema & {
-        lastId: number;
-        hash?: number;
-    }>;
     createSortIndexBuffer(typeId: number, field: number, start: number, lang: number): SortIndex;
-    setSchema(strictSchema: StrictSchema, fromStart?: boolean, transformFns?: TransformFns): (StrictSchema & {
-        lastId: number;
-        hash?: number;
-    }) | Promise<StrictSchema & {
-        lastId: number;
-        hash?: number;
-    }>;
+    setSchema(strictSchema: StrictSchema, transformFns?: TransformFns): Promise<SchemaChecksum>;
     modify(buf: Uint8Array): Record<number, number> | null;
     addToQueryQueue(resolve: any, buf: any): void;
     getQueryBuf(buf: Uint8Array, fromQueue?: boolean): Promise<Uint8Array>;

package/dist/src/server/index.js

@@ -1,26 +1,19 @@
 import native from '../native.js';
 import createDbHash from './dbHash.js';
-import { rm, writeFile } from 'node:fs/promises';
-import { dirname, join } from 'node:path';
+import { rm } from 'node:fs/promises';
 import { langCodesMap } from '@based/schema';
-import { updateTypeDefs, schemaToSelvaBuffer, } from '@based/schema/def';
 import { start } from './start.js';
-import { initCsmt, makeCsmtKey, makeCsmtKeyFromNodeId, } from './tree.js';
+import { makeCsmtKeyFromNodeId } from './tree.js';
 import { save } from './save.js';
-import { Worker, MessageChannel } from 'node:worker_threads';
-import { fileURLToPath } from 'node:url';
 import { setTimeout } from 'node:timers/promises';
 import { migrate } from './migrate/index.js';
 import { debugServer } from '../utils.js';
-import { readUint16, readUint32, readUint64, writeUint64 } from '@saulx/utils';
+import { readUint16, readUint32, readUint64 } from '@saulx/utils';
 import { QueryType } from '../client/query/types.js';
-import { parseSchema, schemaLooseEqual } from '../schema.js';
-import { hash } from '@saulx/hash';
-export const SCHEMA_FILE = 'schema.json';
-export const WRITELOG_FILE = 'writelog.json';
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-const workerPath = join(__dirname, 'worker.js');
+import { strictSchemaToDbSchema } from './schema.js';
+import { DbShared } from '../shared/DbBase.js';
+import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from './schema.js';
+import { resizeModifyDirtyRanges } from './resizeModifyDirtyRanges.js';
 const emptyUint8Array = new Uint8Array(0);
 class SortIndex {
     constructor(buf, dbCtxExternal) {
@@ -31,53 +24,11 @@ class SortIndex {
     idx;
     cnt = 0;
 }
-export class DbWorker {
-    constructor(address, db) {
-        const { port1, port2 } = new MessageChannel();
-        this.db = db;
-        this.channel = port1;
-        this.worker = new Worker(workerPath, {
-            workerData: {
-                isDbWorker: true,
-                channel: port2,
-                address,
-            },
-            transferList: [port2],
-        });
-        port1.on('message', (buf) => {
-            this.resolvers.shift()(new Uint8Array(buf));
-            this.db.onQueryEnd();
-        });
-    }
-    db;
-    channel;
-    worker;
-    resolvers = [];
-    callback = (resolve) => {
-        this.db.processingQueries++;
-        this.resolvers.push(resolve);
-    };
-    updateCtx(address) {
-        this.channel.postMessage(address);
-        return new Promise(this.callback);
-    }
-    getQueryBuf(buf) {
-        this.channel.postMessage(buf);
-        return new Promise(this.callback);
-    }
-}
-export class DbServer {
+export class DbServer extends DbShared {
     modifyDirtyRanges;
     dbCtxExternal; // pointer to zig dbCtx
-    schema = {
-        lastId: 1, // we reserve one for root props
-        types: {},
-    };
     migrating = null;
-    schemaTypesParsed = {};
-    schemaTypesParsedById = {};
     fileSystemPath;
-    maxModifySize;
     merkleTree;
     dirtyRanges = new Set();
     csmtHashFun = createDbHash();
@@ -87,36 +38,18 @@ export class DbServer {
     modifyQueue = [];
     queryQueue = new Map();
     stopped; // = true does not work
-    onSchemaChange;
     unlistenExit;
     saveIntervalInSeconds;
     saveInterval;
-    constructor({ path, maxModifySize = 100 * 1e3 * 1e3, onSchemaChange, debug, saveIntervalInSeconds, }) {
-        this.maxModifySize = maxModifySize;
+    constructor({ path, debug, saveIntervalInSeconds, }) {
+        super();
         this.fileSystemPath = path;
         this.sortIndexes = {};
-        this.onSchemaChange = onSchemaChange;
         this.saveIntervalInSeconds = saveIntervalInSeconds;
         if (debug) {
             debugServer(this);
         }
     }
-    #resizeModifyDirtyRanges() {
-        let maxNrChanges = 0;
-        for (const typeId in this.schemaTypesParsedById) {
-            const def = this.schemaTypesParsedById[typeId];
-            const lastId = def.lastId;
-            const blockCapacity = def.blockCapacity;
-            const tmp = lastId - +!(lastId % def.blockCapacity);
-            const lastBlock = Math.ceil((((tmp / blockCapacity) | 0) * blockCapacity + 1) / blockCapacity);
-            maxNrChanges += lastBlock;
-        }
-        if (!this.modifyDirtyRanges ||
-            this.modifyDirtyRanges.length < maxNrChanges) {
-            const min = Math.max(maxNrChanges * 1.2, 1024) | 0;
-            this.modifyDirtyRanges = new Float64Array(min);
-        }
-    }
     start(opts) {
         this.stopped = false;
         return start(this, opts);
@@ -238,9 +171,6 @@ export class DbServer {
         }
         return fields[lang];
     }
-    migrateSchema(schema, transform) {
-        return migrate(this, schema, transform);
-    }
     createSortIndexBuffer(typeId, field, start, lang) {
         const buf = new Uint8Array(9);
         buf[0] = typeId;
@@ -281,71 +211,34 @@ export class DbServer {
         fields[start][lang] = sortIndex;
         return sortIndex;
     }
-    setSchema(strictSchema, fromStart = false, transformFns) {
-        const parsedSchema = parseSchema(strictSchema);
-        if (!fromStart && Object.keys(this.schema.types).length > 0) {
-            if (schemaLooseEqual(parsedSchema, this.schema)) {
-                return this.schema;
-            }
-            try {
-                return this.migrateSchema(strictSchema, transformFns);
-            }
-            catch (e) {
-                console.error('error migrating schema:', e);
-                return this.schema;
-            }
-        }
-        this.schema = {
-            lastId: this.schema.lastId,
-            ...parsedSchema,
-        };
-        for (const field in this.schema.types) {
-            if (!('id' in this.schema.types[field])) {
-                this.schema.lastId++;
-                this.schema.types[field].id = this.schema.lastId;
-            }
-        }
-        const { hash: _, ...rest } = this.schema;
-        this.schema.hash = hash(rest);
-        updateTypeDefs(this.schema, this.schemaTypesParsed, this.schemaTypesParsedById);
-        if (!fromStart) {
-            writeFile(join(this.fileSystemPath, SCHEMA_FILE), JSON.stringify(this.schema)).catch((err) => console.error('!!!', SCHEMA_FILE, err));
-            let types = Object.keys(this.schemaTypesParsed);
-            const s = schemaToSelvaBuffer(this.schemaTypesParsed);
-            for (let i = 0; i < s.length; i++) {
-                //  TYPE SELVA user Uint8Array(6) [ 1, 17, 23, 0, 11, 0 ]
-                const type = this.schemaTypesParsed[types[i]];
-                // TODO should not crash!
-                try {
-                    native.updateSchemaType(type.id, new Uint8Array(s[i]), this.dbCtxExternal);
-                }
-                catch (err) {
-                    console.error('Cannot update schema on selva', type.type, err, s[i]);
-                }
+    async setSchema(strictSchema, transformFns) {
+        const schema = strictSchemaToDbSchema(strictSchema);
+        if (schema.hash === this.schema?.hash) {
+            // Todo something for sending back to actual client
+            return schema.hash;
+        }
+        if (this.schema) {
+            // skip if allrdy doing the same
+            if (schema.hash === this.migrating) {
+                await this.once('schema');
+                return this.schema.hash;
             }
-            if (strictSchema.props || strictSchema.types?._root) {
-                // insert a root node
-                // TODO fix this add it in schema at least
-                const data = [2, 1, 0, 0, 0, 1, 9, 1, 0, 0, 0, 7, 1, 0, 1];
-                const blockKey = makeCsmtKey(1, 1);
-                const buf = new Uint8Array(8 + data.length + 2 + 8 + 4);
-                const view = new DataView(buf.buffer, 0, buf.byteLength);
-                // set schema hash
-                writeUint64(buf, this.schema.hash, 0);
-                // add content
-                buf.set(data, 8);
-                // add typesLen
-                view.setFloat64(8 + data.length, 0, true);
-                // add dirty key
-                view.setFloat64(8 + data.length + 2, blockKey, true);
-                // add dataLen
-                view.setUint32(buf.length - 4, data.length, true);
-                this.modify(buf);
-            }
-            initCsmt(this);
-        }
-        this.onSchemaChange?.(this.schema);
-        return this.schema;
+            await migrate(this, this.schema, schema, transformFns);
+            // if (this.schema.hash !== schema.hash) {
+            //   // process.nextTick(() => this.emit('schema', this.schema))
+            //   await this.once('schema')
+            // }
+            // Handle this later if it gets changed back to the same schema do false
+            // console.log(this.schema.hash == schema.hash)
+            return this.schema.hash;
+        }
+        setSchemaOnServer(this, schema);
+        await writeSchemaFile(this, schema);
+        await setNativeSchema(this, schema);
+        process.nextTick(() => {
+            this.emit('schema', this.schema);
+        });
+        return schema.hash;
     }
     modify(buf) {
         const schemaHash = readUint64(buf, 0);
@@ -409,7 +302,7 @@ export class DbServer {
             //this.dirtyRanges.add(key)
             i += 8;
         }
-        this.#resizeModifyDirtyRanges();
+        resizeModifyDirtyRanges(this);
         native.modify(data, types, this.dbCtxExternal, this.modifyDirtyRanges);
         for (let key of this.modifyDirtyRanges) {
             if (key === 0) {
@@ -419,7 +312,7 @@ export class DbServer {
         }
     }
     #expire() {
-        this.#resizeModifyDirtyRanges();
+        resizeModifyDirtyRanges(this);
         native.modify(emptyUint8Array, emptyUint8Array, this.dbCtxExternal, this.modifyDirtyRanges);
         for (let key of this.modifyDirtyRanges) {
             if (key === 0) {

package/dist/src/server/migrate/index.d.ts

@@ -1,6 +1,6 @@
-import { StrictSchema } from '@based/schema';
 import { DbServer } from '../index.js';
+import { DbSchema } from '../../schema.js';
 type TransformFn = (node: Record<string, any>) => Record<string, any> | [string, Record<string, any>];
 export type TransformFns = Record<string, TransformFn>;
-export declare const migrate: (fromDbServer: DbServer, toSchema: StrictSchema, transform?: TransformFns) => Promise<DbServer["schema"]>;
+export declare const migrate: (server: DbServer, fromSchema: DbSchema, toSchema: DbSchema, transform?: TransformFns) => Promise<void>;
 export {};