@based/db 0.0.24 → 0.0.25

package/dist/lib/darwin_aarch64/include/selva/selva_string.h

@@ -123,7 +123,7 @@ int selva_string_init_crc(struct selva_string *s, const char *str, size_t len, u
 [[nodiscard]]
 SELVA_EXPORT
 struct selva_string *selva_string_create(const char *str, size_t len, enum selva_string_flags flags)
-    __attribute__((access(read_only, 1, 2)));
+    __attribute__((access(read_only, 1)));
 
 /**
  * Create a new string with a user provided CRC.
@@ -205,7 +205,7 @@ int selva_string_truncate(struct selva_string *s, size_t newlen)
  */
 SELVA_EXPORT
 int selva_string_append(struct selva_string *s, const char *str, size_t len)
-    __attribute__((access(read_only, 2, 3)));
+    __attribute__((access(read_only, 2)));
 
 /**
  * Replace current value of the string s with str.

package/dist/lib/linux_aarch64/include/selva/selva_string.h

@@ -123,7 +123,7 @@ int selva_string_init_crc(struct selva_string *s, const char *str, size_t len, u
 [[nodiscard]]
 SELVA_EXPORT
 struct selva_string *selva_string_create(const char *str, size_t len, enum selva_string_flags flags)
-    __attribute__((access(read_only, 1, 2)));
+    __attribute__((access(read_only, 1)));
 
 /**
  * Create a new string with a user provided CRC.
@@ -205,7 +205,7 @@ int selva_string_truncate(struct selva_string *s, size_t newlen)
  */
 SELVA_EXPORT
 int selva_string_append(struct selva_string *s, const char *str, size_t len)
-    __attribute__((access(read_only, 2, 3)));
+    __attribute__((access(read_only, 2)));
 
 /**
  * Replace current value of the string s with str.

package/dist/lib/linux_x86_64/include/selva/selva_string.h

@@ -123,7 +123,7 @@ int selva_string_init_crc(struct selva_string *s, const char *str, size_t len, u
 [[nodiscard]]
 SELVA_EXPORT
 struct selva_string *selva_string_create(const char *str, size_t len, enum selva_string_flags flags)
-    __attribute__((access(read_only, 1, 2)));
+    __attribute__((access(read_only, 1)));
 
 /**
  * Create a new string with a user provided CRC.
@@ -205,7 +205,7 @@ int selva_string_truncate(struct selva_string *s, size_t newlen)
  */
 SELVA_EXPORT
 int selva_string_append(struct selva_string *s, const char *str, size_t len)
-    __attribute__((access(read_only, 2, 3)));
+    __attribute__((access(read_only, 2)));
 
 /**
  * Replace current value of the string s with str.

package/dist/src/client/modify/fixed.js

@@ -56,7 +56,7 @@ map[ENUM] = (ctx, val, def) => {
         return RANGE_ERR;
     }
     if (val === null) {
-        ctx.buf[ctx.len++] = 1;
+        ctx.buf[ctx.len++] = 0;
     }
     else if (val in def.reverseEnum) {
         ctx.buf[ctx.len++] = def.reverseEnum[val] + 1;

package/dist/src/client/modify/modify.js

@@ -24,10 +24,13 @@ function _modify(ctx, res, obj, schema, mod, tree, overwrite, unsafe) {
         }
         let err;
         if (isPropDef(def)) {
+            const val = obj[key];
+            if (val === undefined) {
+                continue;
+            }
             if (res.subMarkers) {
                 checkSubscriptionMarkers(ctx.db, res.subMarkers, def);
             }
-            const val = obj[key];
             const type = def.typeIndex;
             if (def.separate) {
                 if (type === STRING) {
@@ -81,8 +84,8 @@ function _modify(ctx, res, obj, schema, mod, tree, overwrite, unsafe) {
                     err = writeFixedValue(ctx, val, def, ctx.lastMain + def.start);
                 }
             }
-            else if (typeof val === 'object') {
-                if (val !== null && 'increment' in val) {
+            else if (typeof val === 'object' && val !== null) {
+                if ('increment' in val) {
                     let increment = val.increment;
                     if (increment === 0) {
                         continue;

package/dist/src/client/query/BasedIterable.d.ts

@@ -14,7 +14,7 @@ export declare class BasedQueryResponse {
     [inspect.custom](depth: number): string;
     debug(): this;
     node(index?: number): any;
-    [Symbol.iterator](): Generator<Item, void, unknown>;
+    [Symbol.iterator](): Generator<Partial<Item>, void, unknown>;
     inspect(depth?: number): this;
     forEach(fn: (item: any, key: number) => void): void;
     map(fn: (item: any, key: number) => any): any[];

package/dist/src/client/query/BasedIterable.js

@@ -59,9 +59,15 @@ export class BasedQueryResponse {
         while (i < result.byteLength - 4) {
             let id = readUint32(result, i);
             i += 4;
-            const item = {
-                id,
-            };
+            let item;
+            if (this.def.aggregation == -999 /* AggFn.NONE */) {
+                item = {};
+            }
+            else {
+                item = {
+                    id,
+                };
+            }
             if (this.def.search) {
                 item.$searchScore = readFloatLE(result, i);
                 i += 4;

package/dist/src/client/query/read/read.js

@@ -396,7 +396,7 @@ export const resultToObject = (q, result, end, offset = 0) => {
     while (i < end) {
         const id = readUint32(result, i);
         i += 4;
-        var item;
+        let item;
         if (q.aggregation == -999 /* AggFn.NONE */) {
             item = {};
         }

package/dist/src/server/index.d.ts

@@ -4,6 +4,8 @@ import { createTree } from './csmt/index.js';
 import { Worker, MessagePort } from 'node:worker_threads';
 import { TransformFns } from './migrate/index.js';
 import exitHook from 'exit-hook';
+export declare const SCHEMA_FILE = "schema.json";
+export declare const WRITELOG_FILE = "writelog.json";
 declare class SortIndex {
     constructor(buf: Uint8Array, dbCtxExternal: any);
     buf: Uint8Array;
@@ -57,7 +59,9 @@ export declare class DbServer {
         clean?: boolean;
         hosted?: boolean;
     }): Promise<void>;
-    save(): void | Promise<void>;
+    save(opts?: {
+        forceFullDump?: boolean;
+    }): Promise<void>;
     createCsmtHashFun: () => {
         update: (buf: Uint8Array) => any;
         digest: (encoding?: "hex") => Uint8Array | string;

package/dist/src/server/index.js

@@ -12,7 +12,8 @@ import { Worker, MessageChannel } from 'node:worker_threads';
 import { fileURLToPath } from 'node:url';
 import { setTimeout } from 'node:timers/promises';
 import { migrate } from './migrate/index.js';
-const SCHEMA_FILE = 'schema.json';
+export const SCHEMA_FILE = 'schema.json';
+export const WRITELOG_FILE = 'writelog.json';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const workerPath = join(__dirname, 'worker.js');
@@ -98,8 +99,8 @@ export class DbServer {
     start(opts) {
         return start(this, opts);
     }
-    save() {
-        return save(this);
+    save(opts) {
+        return save(this, false, opts?.forceFullDump ?? false);
     }
     createCsmtHashFun = () => {
         // We can just reuse it as long as we only have one tree.

package/dist/src/server/save.d.ts

@@ -1,2 +1,21 @@
 import { DbServer } from './index.js';
-export declare function save<T extends boolean>(db: DbServer, sync?: T): T extends true ? void : Promise<void>;
+export type Writelog = {
+    ts: number;
+    types: {
+        [t: number]: {
+            lastId: number;
+            blockCapacity: number;
+        };
+    };
+    hash: string;
+    commonDump: string;
+    rangeDumps: {
+        [t: number]: {
+            file: string;
+            hash: string;
+            start: number;
+            end: number;
+        }[];
+    };
+};
+export declare function save<T extends boolean>(db: DbServer, sync?: T, forceFullDump?: boolean): T extends true ? void : Promise<void>;

package/dist/src/server/save.js

@@ -1,14 +1,16 @@
 import native from '../native.js';
+import { isMainThread } from 'node:worker_threads';
 import { writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { destructureCsmtKey, foreachDirtyBlock } from './tree.js';
+import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, makeCsmtKey } from './tree.js';
+import { WRITELOG_FILE } from './index.js';
 import { writeFileSync } from 'node:fs';
 import { bufToHex } from '../utils.js';
-const WRITELOG_FILE = 'writelog.json';
+import { createTree } from './csmt/tree.js';
 const COMMON_SDB_FILE = 'common.sdb';
 const block_sdb_file = (typeId, start, end) => `${typeId}_${start}_${end}.sdb`;
-export function save(db, sync = false) {
-    if (!db.dirtyRanges.size) {
+export function save(db, sync = false, forceFullDump = false) {
+    if (!(isMainThread && (db.dirtyRanges.size || forceFullDump))) {
         return;
     }
     let err;
@@ -17,29 +19,58 @@ export function save(db, sync = false) {
     if (err) {
         console.error(`Save common failed: ${err}`);
     }
-    foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
-        const file = block_sdb_file(typeId, start, end);
-        const path = join(db.fileSystemPath, file);
-        const hash = new Uint8Array(16);
-        err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hash);
-        if (err) {
-            console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
-            return; // TODO What to do with the merkle tree in db situation?
+    if (forceFullDump) {
+        // We just rebuild the whole tree
+        db.merkleTree = createTree(db.createCsmtHashFun); // TODO This could be somewhere else.
+        for (const key in db.schemaTypesParsed) {
+            const def = db.schemaTypesParsed[key];
+            foreachBlock(db, def, (start, end, _hash) => {
+                const typeId = def.id;
+                const file = block_sdb_file(typeId, start, end);
+                const hash = new Uint8Array(16); // TODO One is unnecessary, probably the arg of this cb
+                err = native.saveRange(join(db.fileSystemPath, file), typeId, start, end, db.dbCtxExternal, hash);
+                if (err) {
+                    console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
+                    return; // TODO What to do with the merkle tree in db situation?
+                }
+                const mtKey = makeCsmtKey(typeId, start);
+                const data = {
+                    file,
+                    typeId,
+                    start,
+                    end,
+                };
+                db.merkleTree.insert(mtKey, hash, data);
+            });
         }
-        const data = {
-            file,
-            typeId,
-            start,
-            end,
-        };
-        try {
-            db.merkleTree.delete(mtKey);
-        }
-        catch (err) {
-            // console.error({ err })
-        }
-        db.merkleTree.insert(mtKey, hash, data);
-    });
+    }
+    else {
+        foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
+            const file = block_sdb_file(typeId, start, end);
+            const path = join(db.fileSystemPath, file);
+            const hash = new Uint8Array(16);
+            err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hash);
+            if (err) {
+                console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
+                // The previous state should remain in the merkle tree for
+                // load and sync purposes.
+                return;
+            }
+            const data = {
+                file,
+                typeId,
+                start,
+                end,
+            };
+            try {
+                db.merkleTree.delete(mtKey);
+            }
+            catch (err) {
+                // console.error({ err })
+            }
+            db.merkleTree.insert(mtKey, hash, data);
+        });
+    }
     db.dirtyRanges.clear();
     const types = {};
     const rangeDumps = {};
@@ -61,11 +92,8 @@ export function save(db, sync = false) {
         types,
         commonDump: COMMON_SDB_FILE,
         rangeDumps,
+        hash: bufToHex(db.merkleTree.getRoot()?.hash ?? new Uint8Array(0)),
     };
-    const mtRoot = db.merkleTree.getRoot();
-    if (mtRoot) {
-        data.hash = bufToHex(mtRoot.hash);
-    }
     const filePath = join(db.fileSystemPath, WRITELOG_FILE);
     const content = JSON.stringify(data);
     return sync ? writeFileSync(filePath, content) : writeFile(filePath, content);

package/dist/src/server/start.js

@@ -1,19 +1,16 @@
 import { stringHash } from '@saulx/hash';
-import { DbWorker } from './index.js';
+import { DbWorker, SCHEMA_FILE, WRITELOG_FILE } from './index.js';
 import native from '../native.js';
 import { rm, mkdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { createTree, hashEq } from './csmt/index.js';
-import { foreachBlock } from './tree.js';
+import { foreachBlock, makeCsmtKey } from './tree.js';
 import { availableParallelism } from 'node:os';
 import exitHook from 'exit-hook';
 import './worker.js';
 import { save } from './save.js';
 import { DEFAULT_BLOCK_CAPACITY } from '@based/schema/def';
 import { bufToHex, hexToBuf } from '../utils.js';
-const SCHEMA_FILE = 'schema.json';
-const WRITELOG_FILE = 'writelog.json';
-const makeCsmtKey = (typeId, start) => typeId * 4294967296 + start;
 export async function start(db, opts) {
     const path = db.fileSystemPath;
     const id = stringHash(path) >>> 0;
@@ -94,7 +91,7 @@ export async function start(db, opts) {
         const oldHash = hexToBuf(writelog.hash);
         const newHash = db.merkleTree.getRoot()?.hash;
         if (!hashEq(oldHash, newHash)) {
-            console.error(`WARN: CSMT hash mismatch: ${writelog.hash} != ${bufToHex(newHash)}`);
+            console.error(`WARN: CSMT hash mismatch. expected: ${writelog.hash} actual: ${bufToHex(newHash)}`);
         }
     }
     // start workers
@@ -105,7 +102,7 @@ export async function start(db, opts) {
         db.workers[i] = new DbWorker(address, db);
     }
     if (!opts?.hosted) {
-        db.unlistenExit = exitHook(async (signal) => {
+        db.unlistenExit = exitHook((signal) => {
             const blockSig = () => { };
             const signals = ['SIGINT', 'SIGTERM', 'SIGHUP'];
             // A really dumb way to block signals temporarily while saving.

package/dist/src/server/tree.d.ts

@@ -9,5 +9,5 @@ export type CsmtNodeRange = {
 export declare const destructureCsmtKey: (key: number) => number[];
 export declare const makeCsmtKey: (typeId: number, start: number) => number;
 export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
-export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): Promise<void>;
+export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): void;
 export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;

package/dist/src/server/tree.js

@@ -8,7 +8,7 @@ export const makeCsmtKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
     const tmp = nodeId - +!(nodeId % blockCapacity);
     return typeId * 4294967296 + ((tmp / blockCapacity) | 0) * blockCapacity + 1;
 };
-export async function foreachBlock(db, def, cb) {
+export function foreachBlock(db, def, cb) {
     const step = def.blockCapacity;
     for (let start = 1; start <= def.lastId; start += step) {
         const end = start + step - 1;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.24",
+  "version": "0.0.25",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",