@based/db 0.0.68 → 0.0.70

package/dist/src/client/query/read/read.js

@@ -69,6 +69,9 @@ const readAggregate = (q, result, offset, len) => {
     return results;
 };
 const addField = (p, value, item, defaultOnly = false, lang = 0) => {
+    if (p.transform) {
+        value = p.transform('read', value);
+    }
     let i = p.__isEdge === true ? 1 : 0;
     // TODO OPTMIZE
     const path = lang ? [...p.path, inverseLangMap.get(lang)] : p.path;
@@ -215,21 +218,17 @@ const handleUndefinedProps = (id, q, item) => {
                     }
                 }
             }
-            // else if (prop.typeIndex === BINARY) {
-            //   addField(prop, prop.default, item)
-            // }
             else if (prop.typeIndex === TEXT ||
                 prop.typeIndex === STRING ||
                 prop.typeIndex === ALIAS) {
                 addField(prop, '', item);
             }
-            // else if (prop.typeIndex === JSON) {
-            //   // addField(prop, null, item)
-            // } else {
-            //   // if (prop.default !== undefined) {
-            //   //   addField(prop, '', item)
-            //   // }
-            // }
+            else if (prop.typeIndex === JSON) {
+                addField(prop, null, item);
+            }
+            else if (prop.typeIndex === BINARY) {
+                addField(prop, new Uint8Array(), item);
+            }
         }
     }
 };

package/dist/src/native.d.ts

@@ -24,6 +24,7 @@ declare const native: {
     equals: (a: Uint8Array, b: Uint8Array) => boolean;
     membarSyncRead: () => void;
     membarSyncWrite: () => void;
+    selvaStrerror: (err: number) => any;
     colvecTest: (dbCtx: any, typeId: number, field: number, nodeId: number, len: number) => any;
 };
 export default native;

package/dist/src/native.js

@@ -100,6 +100,9 @@ const native = {
     membarSyncWrite: () => {
         db.membarSyncWrite();
     },
+    selvaStrerror: (err) => {
+        return db.selvaStrerror(err);
+    },
     colvecTest: (dbCtx, typeId, field, nodeId, len) => {
         return db.colvecTest(dbCtx, typeId, field, nodeId, len);
     }

package/dist/src/server/IoWorker.d.ts

@@ -3,6 +3,22 @@ import { DbServer } from './index.js';
 export declare class IoWorker extends DbWorker {
     constructor(address: BigInt, db: DbServer);
     handleMsg(_buf: any): void;
+    private cb;
+    /**
+     * Save given blocks and return errors and hashes in an array.
+     * @returns [[4 bytes err], [16 bytes hash]][] with the same length as blocks.
+     */
+    saveBlocks(blocks: {
+        filepath: string;
+        typeId: number;
+        start: number;
+    }[]): Promise<Uint8Array>;
     loadBlock(filepath: string): Promise<void>;
+    /**
+     * Save a block and discard it from memory "atomically".
+     * Note that this worker doesn't give any protection from other threads
+     * accessing the block concurrently, and it must be coordinated in the
+     * main thread.
+     */
     unloadBlock(filepath: string, typeId: number, start: number): Promise<Uint8Array>;
 }

package/dist/src/server/IoWorker.js

@@ -1,5 +1,6 @@
 import { DbWorker } from './workers/DbWorker.js';
 import { DECODER, readInt32 } from '@saulx/utils';
+import native from '../native.js';
 export class IoWorker extends DbWorker {
     constructor(address, db) {
         const onExit = (_code) => {
@@ -9,27 +10,55 @@ export class IoWorker extends DbWorker {
     }
     handleMsg(_buf) {
     }
+    cb = (resolve) => {
+        this.db.activeReaders++;
+        this.resolvers.push((r) => {
+            this.db.activeReaders--;
+            resolve(r);
+            //this.db.onQueryEnd()
+        });
+    };
+    /**
+     * Save given blocks and return errors and hashes in an array.
+     * @returns [[4 bytes err], [16 bytes hash]][] with the same length as blocks.
+     */
+    async saveBlocks(blocks) {
+        const job = {
+            type: 'save',
+            blocks,
+        };
+        this.channel.postMessage(job);
+        const resBufP = new Promise(this.cb);
+        resBufP.then(() => this.db.onQueryEnd());
+        return resBufP;
+    }
     async loadBlock(filepath) {
         const job = {
             type: 'load',
-            filepath
+            filepath,
         };
         const resBuf = await this.call(job);
         if (resBuf.length) {
             throw new Error(DECODER.decode(resBuf));
         }
     }
+    /**
+     * Save a block and discard it from memory "atomically".
+     * Note that this worker doesn't give any protection from other threads
+     * accessing the block concurrently, and it must be coordinated in the
+     * main thread.
+     */
     async unloadBlock(filepath, typeId, start) {
         const job = {
             type: 'unload',
             filepath,
             typeId,
-            start
+            start,
         };
         const resBuf = await this.call(job);
         const err = readInt32(resBuf, 0);
         if (err) {
-            throw new Error(`selva error: ${err}`);
+            throw new Error(native.selvaStrerror(err));
         }
         // Note that this shares the original buffer which may not be 100% optimal,
         // as the first 4 bytes are no longer needed.

package/dist/src/server/QueryWorker.js

@@ -8,11 +8,11 @@ export class QueryWorker extends DbWorker {
         super(address, db, onExit, 'query_worker.js');
     }
     handleMsg(_buf) {
-        this.db.processingQueries--;
+        this.db.activeReaders--;
         this.db.onQueryEnd();
     }
     callback = (resolve) => {
-        this.db.processingQueries++;
+        this.db.activeReaders++;
         this.resolvers.push(resolve);
     };
     getQueryBuf(buf) {

package/dist/src/server/blocks.d.ts

@@ -1,11 +1,13 @@
 import { SchemaTypeDef } from '@based/schema/def';
 import { DbServer } from './index.js';
+import { IoJobSave } from './workers/io_worker_types.js';
 /**
  * Save a block.
  */
 export declare function saveBlock(db: DbServer, typeId: number, start: number, end: number): void;
+export declare function saveBlocks(db: DbServer, blocks: IoJobSave['blocks']): Promise<void>;
 /**
- * Load a block (typically of a partial type) back to memory.
+ * Load an existing block (typically of a partial type) back to memory.
  */
 export declare function loadBlock(db: DbServer, def: SchemaTypeDef, start: number): Promise<void>;
 /**

package/dist/src/server/blocks.js

@@ -1,7 +1,8 @@
 import native from '../native.js';
 import { join } from 'node:path';
-import { equals } from '@saulx/utils';
+import { equals, readInt32 } from '@saulx/utils';
 import { VerifTree, destructureTreeKey, makeTreeKey, } from './tree.js';
+const SELVA_ENOENT = -8;
 /**
  * Save a block.
  */
@@ -11,20 +12,43 @@ export function saveBlock(db, typeId, start, end) {
     const file = VerifTree.blockSdbFile(typeId, start, end);
     const path = join(db.fileSystemPath, file);
     const err = native.saveBlock(path, typeId, start, db.dbCtxExternal, hash);
-    if (err == -8) {
-        // TODO ENOENT
+    if (err == SELVA_ENOENT) {
+        // Generally we don't nor can't remove blocks from verifTree before we
+        // attempt to access them.
         db.verifTree.remove(mtKey);
     }
     else if (err) {
-        // TODO print the error string
-        console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
+        db.emit('error', `Save ${typeId}:${start}-${end} failed: ${native.selvaStrerror(err)}`);
     }
     else {
         db.verifTree.update(mtKey, hash);
     }
 }
+export async function saveBlocks(db, blocks) {
+    const res = await db.ioWorker.saveBlocks(blocks);
+    if (res.byteOffset !== 0)
+        throw new Error('Unexpected offset');
+    // if (res.byteLength / 20 !== blocks.length) throw new Error('Invalid res size')
+    for (let i = 0; i < blocks.length; i++) {
+        const block = blocks[i];
+        const key = makeTreeKey(block.typeId, block.start);
+        const err = readInt32(res, i * 20);
+        const hash = new Uint8Array(res.buffer, i * 20 + 4, 16);
+        if (err === SELVA_ENOENT) {
+            // Generally we don't nor can't remove blocks from verifTree before we
+            // attempt to access them.
+            db.verifTree.remove(key);
+        }
+        else if (err) {
+            db.emit('error', `Save ${block.typeId}:${block.start} failed: ${native.selvaStrerror(err)}`);
+        }
+        else {
+            db.verifTree.update(key, hash);
+        }
+    }
+}
 /**
- * Load a block (typically of a partial type) back to memory.
+ * Load an existing block (typically of a partial type) back to memory.
  */
 export async function loadBlock(db, def, start) {
     const key = makeTreeKey(def.id, start);
@@ -72,7 +96,7 @@ export async function unloadBlock(db, def, start) {
     }
     catch (e) {
         // TODO Proper logging
-        // TODO err == -8 == SELVA_ENOENT => db.verifTree.remove(key) ??
+        // TODO SELVA_ENOENT => db.verifTree.remove(key) ??
         console.error(`Save ${typeId}:${start}-${end} failed`);
         console.error(e);
     }

package/dist/src/server/index.d.ts

@@ -26,7 +26,7 @@ export declare class DbServer extends DbShared {
     ioWorker: IoWorker;
     workers: QueryWorker[];
     availableWorkerIndex: number;
-    processingQueries: number;
+    activeReaders: number;
     modifyQueue: Uint8Array[];
     queryQueue: Map<Function, Uint8Array>;
     stopped: boolean;

package/dist/src/server/index.js

@@ -31,12 +31,12 @@ export class DbServer extends DbShared {
     migrating = null;
     saveInProgress = false;
     fileSystemPath;
-    verifTree;
+    verifTree; // should be updated only when saving/loading
     dirtyRanges = new Set();
     ioWorker;
     workers = [];
     availableWorkerIndex = -1;
-    processingQueries = 0; // semaphore for locking writes
+    activeReaders = 0; // processing queries or other DB reads
     modifyQueue = [];
     queryQueue = new Map();
     stopped; // = true does not work
@@ -58,7 +58,7 @@ export class DbServer extends DbShared {
         return start(this, opts);
     }
     save(opts) {
-        return save(this, false, opts?.forceFullDump ?? false);
+        return save(this, opts);
     }
     async loadBlock(typeName, nodeId) {
         const def = this.schemaTypesParsed[typeName];
@@ -68,10 +68,6 @@ export class DbServer extends DbShared {
         const typeId = def.id;
         const key = makeTreeKeyFromNodeId(typeId, def.blockCapacity, nodeId);
         const [, start] = destructureTreeKey(key);
-        const block = this.verifTree.getBlock(key);
-        if (!block) {
-            throw new Error('Block not found');
-        }
         await loadBlock(this, def, start);
     }
     async unloadBlock(typeName, nodeId) {
@@ -103,7 +99,7 @@ export class DbServer extends DbShared {
                             for (const lang in this.sortIndexes[type][field][start]) {
                                 const sortIndex = this.sortIndexes[type][field][start][lang];
                                 sortIndex.cnt /= 2;
-                                if (sortIndex.cnt < 1 && !this.processingQueries) {
+                                if (sortIndex.cnt < 1 && !this.activeReaders) {
                                     native.destroySortIndex(sortIndex.buf, this.dbCtxExternal);
                                     delete this.sortIndexes[type][field][start][lang];
                                 }
@@ -299,7 +295,7 @@ export class DbServer extends DbShared {
             def.lastId = lastId + offset;
             offsets[typeId] = offset;
         }
-        if (this.processingQueries) {
+        if (this.activeReaders) {
             this.modifyQueue.push(new Uint8Array(bufWithHash));
         }
         else {
@@ -387,7 +383,7 @@ export class DbServer extends DbShared {
                     const start = readUint16(sort, 3);
                     let sortIndex = this.getSortIndex(typeId, field, start, 0);
                     if (!sortIndex) {
-                        if (this.processingQueries) {
+                        if (this.activeReaders) {
                             return new Promise((resolve) => {
                                 this.addToQueryQueue(resolve, buf);
                             });
@@ -411,7 +407,7 @@ export class DbServer extends DbShared {
         }
     }
     onQueryEnd() {
-        if (this.processingQueries === 0) {
+        if (this.activeReaders === 0) {
             if (this.modifyQueue.length) {
                 const modifyQueue = this.modifyQueue;
                 this.modifyQueue = [];
@@ -454,6 +450,7 @@ export class DbServer extends DbShared {
                 await this.save();
             }
             await this.ioWorker.terminate();
+            this.ioWorker = null;
             await Promise.all(this.workers.map((worker) => worker.terminate()));
             this.workers = [];
             native.stop(this.dbCtxExternal);

package/dist/src/server/migrate/index.js

@@ -7,6 +7,7 @@ import { foreachDirtyBlock } from '../blocks.js';
 import { fileURLToPath } from 'url';
 import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from '../schema.js';
 import { setToAwake, waitUntilSleeping } from './utils.js';
+import { serialize } from '@based/schema';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const workerPath = join(__dirname, 'worker.js');
@@ -74,8 +75,14 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
             isDbMigrateWorker: true,
             from: fromAddress,
             to: toAddress,
-            fromSchema,
-            toSchema,
+            fromSchema: serialize(fromSchema, {
+                stripMetaInformation: true,
+                stripTransform: true,
+            }),
+            toSchema: serialize(toSchema, {
+                stripMetaInformation: true,
+                stripTransform: true,
+            }),
             channel: port2,
             workerState,
             transformFns,
@@ -83,14 +90,17 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
         transferList: [port2],
     });
     // handle?
-    worker.on('error', (err) => {
-        killed = true;
-        console.error(`Error in migration ${err.message}`);
+    const errorPromise = new Promise((resolve) => {
+        worker.once('error', (err) => {
+            killed = true;
+            console.error(`Error in migration ${err.message}, aborting migration`);
+            resolve();
+        });
     });
     // Block handling
     let i = 0;
     let rangesToMigrate = [];
-    await save(server, false, false, true);
+    await save(server, { skipMigrationCheck: true });
     server.verifTree.foreachBlock((block) => {
         const [typeId, start] = destructureTreeKey(block.key);
         const def = server.schemaTypesParsedById[typeId];
@@ -103,13 +113,13 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
             break;
         }
         // block modifies
-        server.processingQueries++;
+        server.activeReaders++;
         const leafData = rangesToMigrate[i++];
         port1.postMessage(leafData);
         setToAwake(workerState, true);
-        await waitUntilSleeping(workerState);
+        await Promise.race([errorPromise, waitUntilSleeping(workerState)]);
         // exec queued modifies
-        server.processingQueries--;
+        server.activeReaders--;
         server.onQueryEnd();
         if (i === rangesToMigrate.length) {
             if (server.dirtyRanges.size) {
@@ -149,14 +159,18 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
     // -----------------------------------------
     tmpDb.server.dbCtxExternal = fromCtx;
     // TODO makes this SYNC
-    const promises = server.workers.map((worker) => worker.updateCtx(toAddress));
+    const promises = [server.ioWorker, ...server.workers].map((worker) => worker.updateCtx(toAddress));
     promises.push(tmpDb.destroy(), worker.terminate());
     await Promise.all(promises);
     if (abort()) {
         return;
     }
     native.membarSyncRead();
-    await save(server, false, true, true);
+    await save(server, {
+        forceFullDump: true,
+        skipDirtyCheck: true,
+        skipMigrationCheck: true,
+    });
     await writeSchemaFile(server, toSchema);
     server.migrating = 0;
     process.nextTick(() => server.emit('schema', server.schema));

package/dist/src/server/migrate/worker.js

@@ -6,6 +6,7 @@ import { isTypedArray } from 'node:util/types';
 import { setSchemaOnServer } from '../schema.js';
 import { setToSleep } from './utils.js';
 import { setLocalClientSchema } from '../../client/setLocalClientSchema.js';
+import { deSerialize } from '@based/schema';
 if (isMainThread) {
     console.warn('running worker.ts in mainthread');
 }
@@ -41,80 +42,87 @@ else if (workerData?.isDbMigrateWorker) {
     };
     fromDb.server.dbCtxExternal = fromCtx;
     toDb.server.dbCtxExternal = toCtx;
-    setSchemaOnServer(fromDb.server, fromSchema);
-    setSchemaOnServer(toDb.server, toSchema);
+    setSchemaOnServer(fromDb.server, deSerialize(fromSchema));
+    setSchemaOnServer(toDb.server, deSerialize(toSchema));
     setLocalClientSchema(fromDb.client, fromDb.server.schema);
     setLocalClientSchema(toDb.client, toDb.server.schema);
-    const map = {};
-    for (const type in fromDb.server.schemaTypesParsed) {
-        const { id, props } = fromDb.server.schemaTypesParsed[type];
-        const include = Object.keys(props);
-        let i = include.length;
-        while (i--) {
-            const path = include[i];
-            if (props[path].typeIndex === REFERENCE ||
-                props[path].typeIndex === REFERENCES) {
-                include[i] = `${path}.id`;
-                if (props[path].edges) {
-                    for (const key in props[path].edges) {
-                        const prop = props[path].edges[key];
-                        if (prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) {
-                            include.push(`${path}.${key}.id`);
-                        }
-                        else {
-                            include.push(`${path}.${key}`);
+    try {
+        const map = {};
+        for (const type in fromDb.server.schemaTypesParsed) {
+            const { id, props } = fromDb.server.schemaTypesParsed[type];
+            const include = Object.keys(props);
+            let i = include.length;
+            while (i--) {
+                const path = include[i];
+                if (props[path].typeIndex === REFERENCE ||
+                    props[path].typeIndex === REFERENCES) {
+                    include[i] = `${path}.id`;
+                    if (props[path].edges) {
+                        for (const key in props[path].edges) {
+                            const prop = props[path].edges[key];
+                            if (prop.typeIndex === REFERENCE ||
+                                prop.typeIndex === REFERENCES) {
+                                include.push(`${path}.${key}.id`);
+                            }
+                            else {
+                                include.push(`${path}.${key}`);
+                            }
                         }
                     }
                 }
             }
+            map[id] = { type, include };
         }
-        map[id] = { type, include };
-    }
-    for (const type in transformFns) {
-        const fnOrNull = transformFns[type];
-        transformFns[type] = eval(`(${fnOrNull})`);
-    }
-    while (true) {
-        let msg;
-        while ((msg = receiveMessageOnPort(channel))) {
-            const leafData = msg.message;
-            const { type, include } = map[leafData.typeId];
-            const typeTransformFn = transformFns[type];
-            if (typeTransformFn) {
-                const nodes = fromDb
-                    .query(type)
-                    .include(include)
-                    .range(leafData.start - 1, leafData.end)
-                    ._getSync(fromCtx);
-                for (const node of nodes) {
-                    const res = typeTransformFn(node);
-                    if (res === null) {
-                        continue;
-                    }
-                    if (Array.isArray(res)) {
-                        toDb.create(res[0], res[1] || node, { unsafe: true });
-                    }
-                    else {
-                        toDb.create(type, res || node, { unsafe: true });
+        for (const type in transformFns) {
+            const fnOrNull = transformFns[type];
+            transformFns[type] = eval(`(${fnOrNull})`);
+        }
+        while (true) {
+            let msg;
+            while ((msg = receiveMessageOnPort(channel))) {
+                const leafData = msg.message;
+                const { type, include } = map[leafData.typeId];
+                const typeTransformFn = transformFns[type];
+                if (typeTransformFn) {
+                    const nodes = fromDb
+                        .query(type)
+                        .include(include)
+                        .range(leafData.start - 1, leafData.end)
+                        ._getSync(fromCtx);
+                    for (const node of nodes) {
+                        const res = typeTransformFn(node);
+                        if (res === null) {
+                            continue;
+                        }
+                        if (Array.isArray(res)) {
+                            toDb.create(res[0], res[1] || node, { unsafe: true });
+                        }
+                        else {
+                            toDb.create(type, res || node, { unsafe: true });
+                        }
                     }
                 }
-            }
-            else if (type in toDb.server.schemaTypesParsed) {
-                const nodes = fromDb
-                    .query(type)
-                    .include(include)
-                    .range(leafData.start - 1, leafData.end)
-                    ._getSync(fromCtx);
-                for (const node of nodes) {
-                    toDb.create(type, node, { unsafe: true });
+                else if (type in toDb.server.schemaTypesParsed) {
+                    const nodes = fromDb
+                        .query(type)
+                        .include(include)
+                        .range(leafData.start - 1, leafData.end)
+                        ._getSync(fromCtx);
+                    for (const node of nodes) {
+                        toDb.create(type, node, { unsafe: true });
+                    }
                 }
             }
+            await toDb.drain();
+            native.membarSyncWrite();
+            // WE ARE ONLY GOING TO SEND { type: lastNodeId }
+            channel.postMessage(cp(toDb.server.schemaTypesParsed));
+            setToSleep(workerState);
         }
-        await toDb.drain();
-        native.membarSyncWrite();
-        // WE ARE ONLY GOING TO SEND { type: lastNodeId }
-        channel.postMessage(cp(toDb.server.schemaTypesParsed));
-        setToSleep(workerState);
+    }
+    catch (e) {
+        console.error(e);
+        throw e;
     }
 }
 else {

package/dist/src/server/save.d.ts

@@ -19,5 +19,11 @@ export type Writelog = {
         [t: number]: RangeDump[];
     };
 };
-export declare function save<T extends boolean>(db: DbServer, sync?: T, forceFullDump?: boolean, skipMigrationCheck?: boolean): T extends true ? void : Promise<void>;
+type SaveOpts = {
+    forceFullDump?: boolean;
+    skipDirtyCheck?: boolean;
+    skipMigrationCheck?: boolean;
+};
+export declare function saveSync(db: DbServer, opts?: SaveOpts): void;
+export declare function save(db: DbServer, opts?: SaveOpts): Promise<void>;
 export {};