@based/db 0.0.26 → 0.0.28

package/dist/src/client/modify/json.js

@@ -1,5 +1,19 @@
 import { writeBinary } from './binary.js';
+import { ModifyError } from './ModifyRes.js';
 export function writeJson(value, ctx, schema, t, parentId, modifyOp) {
-    return writeBinary(value === null ? null : JSON.stringify(value), ctx, schema, t, parentId, modifyOp);
+    try {
+        if (value === null) {
+            return writeBinary(null, ctx, schema, t, parentId, modifyOp);
+        }
+        else {
+            if (!t.validation(value, t)) {
+                return new ModifyError(t, value);
+            }
+            return writeBinary(JSON.stringify(value), ctx, schema, t, parentId, modifyOp);
+        }
+    }
+    catch (err) {
+        return new ModifyError(t, value);
+    }
 }
 //# sourceMappingURL=json.js.map

package/dist/src/client/modify/references/edge.js

@@ -133,7 +133,7 @@ export function writeEdges(t, ref, ctx) {
                         return RANGE_ERR;
                     }
                     ctx.buf[ctx.len++] = CARDINALITY;
-                    writeHllBuf(value, ctx, t, size);
+                    writeHllBuf(value, ctx, edge, size);
                 }
             }
             else {

package/dist/src/client/modify/references/references.js

@@ -27,7 +27,14 @@ export function writeReferences(value, ctx, schema, def, res, mod) {
         else if (key === 'delete') {
             err = deleteRefs(def, ctx, schema, mod, val, res.tmpId);
         }
-        else if (key === 'set') {
+        else if (key === 'update') {
+            // and add add: []
+            // replace this with update
+            err = updateRefs(def, ctx, schema, mod, val, res.tmpId, 1);
+        }
+        else if (key === 'add') {
+            // and add add: []
+            // replace this with update
             err = updateRefs(def, ctx, schema, mod, val, res.tmpId, 1);
         }
         else if (key === 'upsert') {
@@ -173,6 +180,9 @@ function appendRefs(def, ctx, modifyOp, refs, op, remaining) {
         else {
             return new ModifyError(def, refs);
         }
+        if (!def.validation(id, def)) {
+            return new ModifyError(def, refs);
+        }
         if (hasEdges) {
             if (index === undefined) {
                 if (ctx.len + 9 > ctx.max) {
@@ -261,17 +271,22 @@ function putRefs(def, ctx, modifyOp, refs, op) {
     for (; i < refs.length; i++) {
         let ref = refs[i];
         if (typeof ref === 'number') {
-            ctx.buf[ctx.len++] = ref;
-            ctx.buf[ctx.len++] = ref >>>= 8;
-            ctx.buf[ctx.len++] = ref >>>= 8;
-            ctx.buf[ctx.len++] = ref >>>= 8;
+            if (!def.validation(ref, def)) {
+                break;
+            }
+            else {
+                ctx.buf[ctx.len++] = ref;
+                ctx.buf[ctx.len++] = ref >>>= 8;
+                ctx.buf[ctx.len++] = ref >>>= 8;
+                ctx.buf[ctx.len++] = ref >>>= 8;
+            }
         }
         else if (ref instanceof ModifyState) {
             if (ref.error) {
                 return ref.error;
             }
             ref = ref.getId();
-            if (!ref) {
+            if (!def.validation(ref, def)) {
                 break;
             }
             ctx.buf[ctx.len++] = ref;

package/dist/src/client/modify/string.js

@@ -8,11 +8,7 @@ import { write } from '../string.js';
 // add compression handling for edge fields
 export function writeString(lang, value, ctx, def, t, parentId, modifyOp) {
     const isBuffer = value instanceof Uint8Array;
-    if (typeof value !== 'string' && value !== null && !isBuffer) {
-        return new ModifyError(t, value);
-    }
-    const len = value?.length;
-    if (!len) {
+    if (value === '' || value === null) {
         if (modifyOp === UPDATE) {
             if (ctx.len + 11 /* SIZE.DEFAULT_CURSOR */ + 1 > ctx.max) {
                 return RANGE_ERR;
@@ -22,11 +18,14 @@ export function writeString(lang, value, ctx, def, t, parentId, modifyOp) {
         }
     }
     else {
+        if (!t.validation(value, t)) {
+            return new ModifyError(t, value);
+        }
         let size = isBuffer
             ? value.byteLength
             : ENCODER.encode(value).byteLength + 6;
         if (ctx.len + 11 /* SIZE.DEFAULT_CURSOR */ + 11 + size > ctx.max) {
-            // +10 OR +11, teh original check was +20 but
+            // +10 OR +11, the original check was +20 but
             // there are 10 addtional bytes in this scope
             // 5 compression size
             return RANGE_ERR;

package/dist/src/client/modify/text.js

@@ -4,16 +4,6 @@ import { writeString } from './string.js';
 import { ModifyError } from './ModifyRes.js';
 import { setCursor } from './setCursor.js';
 export function writeText(value, ctx, def, t, res, parentId, modifyOp) {
-    const isBuffer = value instanceof Uint8Array;
-    if (typeof value !== 'string' &&
-        value !== null &&
-        !isBuffer &&
-        value &&
-        typeof value !== 'object') {
-        return new ModifyError(t, value);
-    }
-    // const len = value?.length
-    // think about this
     if (value === null && !res.locale) {
         if (modifyOp === UPDATE) {
             if (ctx.len + 11 /* SIZE.DEFAULT_CURSOR */ + 1 > ctx.max) {
@@ -31,7 +21,6 @@ export function writeText(value, ctx, def, t, res, parentId, modifyOp) {
             // @ts-ignore
             value = '';
         }
-        // @ts-ignore
         const err = writeString(res.locale, value, ctx, def, t, res.tmpId, modifyOp);
         if (modifyOp === CREATE) {
             const index = t.prop * (def.localeSize + 1);

package/dist/src/client/modify/vector.js

@@ -18,11 +18,11 @@ export function writeVector(value, ctx, schema, t, parentId, modifyOp) {
         size = 0;
     }
     else {
-        if (!value) {
-            return new ModifyError(t, value);
-        }
         size = value.byteLength + 4;
     }
+    if (!t.validation(value, t)) {
+        return new ModifyError(t, value);
+    }
     if (size === 0) {
         if (modifyOp === UPDATE) {
             if (ctx.len + 11 /* SIZE.DEFAULT_CURSOR */ + 1 > ctx.max) {

package/dist/src/client/query/BasedDbQuery.js

@@ -162,7 +162,12 @@ export class QueryBranch {
                 });
             }
             else if (Array.isArray(f)) {
-                includeFields(this.def, f);
+                if (f.length === 0) {
+                    includeFields(this.def, ['id']);
+                }
+                else {
+                    includeFields(this.def, f);
+                }
             }
             else if (f !== undefined) {
                 throw new Error('Invalid include statement: expected props, refs and edges (string or array) or function');

package/dist/src/client/query/filter/parseFilterValue.js

@@ -1,4 +1,4 @@
-import { TIMESTAMP, CREATED, UPDATED, ENUM, BOOLEAN, STRING, BINARY, TEXT, } from '@based/schema/def';
+import { TIMESTAMP, ENUM, BOOLEAN, STRING, BINARY, TEXT, } from '@based/schema/def';
 import { crc32 } from '../../crc32.js';
 import { convertToTimestamp } from '../../timestamp.js';
 import { ENCODER } from '../../../utils.js';
@@ -39,9 +39,7 @@ export const parseFilterValue = (prop, value) => {
     else if (prop.typeIndex === ENUM) {
         return prop.reverseEnum[value] + 1;
     }
-    else if (prop.typeIndex === TIMESTAMP ||
-        prop.typeIndex === CREATED ||
-        prop.typeIndex === UPDATED) {
+    else if (prop.typeIndex === TIMESTAMP) {
         const v = convertToTimestamp(value);
         if (typeof v !== 'number') {
             throw new Error(`Incorrect value for timestamp ${prop.path.join('.')}`);

package/dist/src/client/query/include/walk.js

@@ -20,6 +20,7 @@ export const walkDefs = (db, def, f) => {
                     def.edges.lang = def.lang;
                 }
                 const edgeProp = def.edges.props[p];
+                // console.log(p, !!edgeProp, Object.keys(def.edges.props), def.target)
                 if (edgeProp.typeIndex === REFERENCE ||
                     edgeProp.typeIndex === REFERENCES) {
                     const refDef = createOrGetRefQueryDef(db, def.edges, edgeProp);

package/dist/src/client/query/read/read.js

@@ -296,7 +296,7 @@ export const readAllFields = (q, result, offset, end, item, id) => {
             return i - offset - 4 - (q.search ? 4 : 0);
         }
         else if (index === READ_AGGREGATION) {
-            // TODO: To change to a map and also to get the aggregate field name from a query function parameter
+            // TODO: Change to a map and also to get the aggregate field name from a query function parameter
             const propAgg = {
                 name: 'count',
                 path: ['count'],
@@ -314,6 +314,9 @@ export const readAllFields = (q, result, offset, end, item, id) => {
         }
         else {
             const prop = q.schema.reverseProps[index];
+            if (!prop) {
+                console.log({ prop: !!prop }, index);
+            }
             if (prop.typeIndex === CARDINALITY) {
                 q.include.propsRead[index] = id;
                 const size = readUint32(result, i);
@@ -419,6 +422,10 @@ export const resultToObject = (q, result, end, offset = 0) => {
         items.push(item);
     }
     if ('id' in q.target || 'alias' in q.target) {
+        if (q.type === QueryDefType.Root && q.target.type === '_root') {
+            // Todo can be optimized
+            delete items[0].id;
+        }
         return items[0];
     }
     return items;

package/dist/src/client/query/validation.js

@@ -418,6 +418,7 @@ export const EMPTY_ALIAS_PROP_DEF = {
     typeIndex: ALIAS,
     __isPropDef: true,
     separate: true,
+    validation: () => true,
     len: 0,
     start: 0,
     default: DEFAULT_MAP[ALIAS],
@@ -428,6 +429,7 @@ export const ERROR_STRING = {
     typeIndex: STRING,
     __isPropDef: true,
     separate: true,
+    validation: () => true,
     len: 0,
     start: 0,
     default: DEFAULT_MAP[STRING],
@@ -438,6 +440,7 @@ export const ERROR_VECTOR = {
     typeIndex: VECTOR,
     __isPropDef: true,
     separate: true,
+    validation: () => true,
     len: 0,
     start: 0,
     default: DEFAULT_MAP[VECTOR],

package/dist/src/index.d.ts

@@ -10,6 +10,9 @@ export { xxHash64 } from './client/xxHash64.js';
 export { crc32 } from './client/crc32.js';
 export * from './client/query/serialize.js';
 export * from './utils.js';
+export * from './client/query/query.js';
+export * from './client/query/BasedDbQuery.js';
+export * from './client/query/BasedIterable.js';
 export declare class BasedDb {
     #private;
     client: DbClient;

package/dist/src/index.js

@@ -12,6 +12,9 @@ export { xxHash64 } from './client/xxHash64.js';
 export { crc32 } from './client/crc32.js';
 export * from './client/query/serialize.js';
 export * from './utils.js';
+export * from './client/query/query.js';
+export * from './client/query/BasedDbQuery.js';
+export * from './client/query/BasedIterable.js';
 export class BasedDb {
     client;
     server;
@@ -102,11 +105,13 @@ export class BasedDb {
     start = function () {
         return this.server.start.apply(this.server, arguments);
     };
-    stop = function () {
+    stop = async function () {
+        await this.isModified();
         this.client.stop();
         return this.server.stop.apply(this.server, arguments);
     };
-    save = function () {
+    save = async function () {
+        await this.isModified();
         return this.server.save.apply(this.server, arguments);
     };
     migrateSchema = function () {
@@ -119,6 +124,7 @@ export class BasedDb {
         return this.client.schemaIsSet.apply(this.client, arguments);
     };
     async destroy() {
+        await this.isModified();
         // Tmp fix: Gives node time to GC existing buffers else it can incorrectly re-asign to mem
         // Todo: clear all active queries, queues ETC
         await wait(Math.max(this.client.hooks.flushTime + 10, 10));

package/dist/src/server/index.js

@@ -109,7 +109,8 @@ export class DbServer {
             const lastBlock = Math.ceil((((tmp / blockCapacity) | 0) * blockCapacity + 1) / blockCapacity);
             maxNrChanges += lastBlock;
         }
-        if (!this.modifyDirtyRanges || this.modifyDirtyRanges.length < maxNrChanges) {
+        if (!this.modifyDirtyRanges ||
+            this.modifyDirtyRanges.length < maxNrChanges) {
             const min = Math.max(maxNrChanges * 1.2, 1024) | 0;
             this.modifyDirtyRanges = new Float64Array(min);
         }
@@ -356,7 +357,7 @@ export class DbServer {
         }
         updateTypeDefs(this.schema, this.schemaTypesParsed, this.schemaTypesParsedById);
         if (!fromStart) {
-            writeFile(join(this.fileSystemPath, SCHEMA_FILE), JSON.stringify(this.schema)).catch((err) => console.error(SCHEMA_FILE, err));
+            writeFile(join(this.fileSystemPath, SCHEMA_FILE), JSON.stringify(this.schema)).catch((err) => console.error('!!!', SCHEMA_FILE, err));
             let types = Object.keys(this.schemaTypesParsed);
             const s = schemaToSelvaBuffer(this.schemaTypesParsed);
             for (let i = 0; i < s.length; i++) {
@@ -526,7 +527,13 @@ export class DbServer {
     }
     async destroy() {
         await this.stop(true);
-        await rm(this.fileSystemPath, { recursive: true }).catch((err) => console.warn('Error removing dump folder', this.fileSystemPath, err.message));
+        await rm(this.fileSystemPath, { recursive: true }).catch((err) => {
+            // console.warn(
+            //   'Error removing dump folder',
+            //   this.fileSystemPath,
+            //   err.message,
+            // ),
+        });
     }
 }
 //# sourceMappingURL=index.js.map

package/dist/src/server/migrate/index.js

@@ -5,7 +5,10 @@ import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threa
 import native from '../../native.js';
 import './worker.js';
 import { foreachDirtyBlock } from '../tree.js';
+import { SCHEMA_FILE } from '../index.js';
 import { fileURLToPath } from 'url';
+import { deepMerge } from '@saulx/utils';
+import { writeFile } from 'fs/promises';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const workerPath = join(__dirname, 'worker.js');
@@ -64,8 +67,9 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
     worker.on('error', console.error);
     let i = 0;
     let ranges = [];
-    fromDbServer.updateMerkleTree();
-    fromDbServer.dirtyRanges.clear();
+    // fromDbServer.updateMerkleTree()
+    // fromDbServer.dirtyRanges.clear()
+    await fromDbServer.save();
     fromDbServer.merkleTree.visitLeafNodes((leaf) => {
         ranges.push(leaf.data);
     });
@@ -106,13 +110,13 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
             ;
             [schema, schemaTypesParsed] = msg.message;
         }
-        fromDbServer.schema = schema;
-        fromDbServer.schemaTypesParsed = schemaTypesParsed;
+        fromDbServer.schema = deepMerge(toDb.server.schema, schema);
+        fromDbServer.schemaTypesParsed = deepMerge(toDb.server.schemaTypesParsed, schemaTypesParsed);
         fromDbServer.dbCtxExternal = toCtx;
         toDb.server.dbCtxExternal = fromCtx;
     }
     const promises = fromDbServer.workers.map((worker) => worker.updateCtx(toAddress));
-    promises.push(toDb.destroy(), worker.terminate());
+    promises.push(toDb.destroy(), worker.terminate(), fromDbServer.save({ forceFullDump: true }), writeFile(join(fromDbServer.fileSystemPath, SCHEMA_FILE), JSON.stringify(fromDbServer.schema)));
     await Promise.all(promises);
     fromDbServer.onSchemaChange?.(fromDbServer.schema);
     return fromDbServer.schema;

package/dist/src/server/migrate/worker.js

@@ -2,6 +2,7 @@ import { isMainThread, receiveMessageOnPort, workerData, } from 'node:worker_thr
 import native from '../../native.js';
 import { BasedDb } from '../../index.js';
 import { REFERENCE, REFERENCES } from '@based/schema/def';
+import { isTypedArray } from 'node:util/types';
 if (isMainThread) {
     console.warn('running worker.ts in mainthread');
 }
@@ -12,6 +13,27 @@ else {
     const path = null;
     const fromDb = new BasedDb({ path });
     const toDb = new BasedDb({ path });
+    const cp = (obj) => {
+        let copy;
+        for (const key in obj) {
+            const val = obj[key];
+            if (typeof val === 'number') {
+                // only copy numbers
+                copy ??= Array.isArray(obj) ? [] : {};
+                copy[key] = val;
+            }
+            else if (typeof val === 'object' &&
+                val !== null &&
+                !isTypedArray(val)) {
+                const res = cp(val);
+                if (res) {
+                    copy ??= Array.isArray(obj) ? [] : {};
+                    copy[key] = cp(val);
+                }
+            }
+        }
+        return copy;
+    };
     fromDb.server.dbCtxExternal = fromCtx;
     toDb.server.dbCtxExternal = toCtx;
     await fromDb.setSchema(fromSchema, true);
@@ -71,7 +93,10 @@ else {
             }
         }
         await toDb.drain();
-        channel.postMessage([toDb.server.schema, toDb.server.schemaTypesParsed]);
+        channel.postMessage([
+            cp(toDb.server.schema),
+            cp(toDb.server.schemaTypesParsed),
+        ]);
         // put it to sleep
         atomics[0] = 0;
         Atomics.notify(atomics, 0);

package/dist/src/server/save.js

@@ -2,7 +2,7 @@ import native from '../native.js';
 import { isMainThread } from 'node:worker_threads';
 import { writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, makeCsmtKey } from './tree.js';
+import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, makeCsmtKey, } from './tree.js';
 import { WRITELOG_FILE } from './index.js';
 import { writeFileSync } from 'node:fs';
 import { bufToHex } from '../utils.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.26",
+  "version": "0.0.28",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",
@@ -32,7 +32,7 @@
     "basedDbNative.cjs"
   ],
   "dependencies": {
-    "@based/schema": "5.0.0-alpha.8",
+    "@based/schema": "5.0.0-alpha.9",
     "@saulx/hash": "^3.0.0",
     "@saulx/utils": "^6.1.1",
     "exit-hook": "^4.0.0",
@@ -43,6 +43,7 @@
     "@based/locale-x86-64-gnu": "*"
   },
   "devDependencies": {
+    "jsondiffpatch": "^0.7.3",
     "@based/crc32c": "^1.0.0",
     "@types/node": "^22.5.3",
     "axios": "^1.7.9",
@@ -50,4 +51,4 @@
     "tar": "^7.4.3",
     "typescript": "^5.6.3"
   }
-}
+}

package/dist/src/client/bitWise.d.ts

@@ -1,6 +0,0 @@
-export declare const readDoubleLE: (val: Uint8Array, offset: number) => number;
-export declare const readFloatLE: (val: Uint8Array, offset: number) => number;
-export declare const readUint32: (val: Uint8Array, offset: number) => number;
-export declare const readInt32: (val: Uint8Array, offset: number) => number;
-export declare const readInt16: (val: Uint8Array, offset: number) => number;
-export declare const readUint16: (val: Uint8Array, offset: number) => number;

package/dist/src/client/bitWise.js

@@ -1,72 +0,0 @@
-export const readDoubleLE = (val, offset) => {
-    const low = (val[offset] |
-        (val[offset + 1] << 8) |
-        (val[offset + 2] << 16) |
-        (val[offset + 3] << 24)) >>>
-        0;
-    const high = (val[offset + 4] |
-        (val[offset + 5] << 8) |
-        (val[offset + 6] << 16) |
-        (val[offset + 7] << 24)) >>>
-        0;
-    const sign = high >>> 31 ? -1 : 1;
-    let exponent = (high >>> 20) & 0x7ff;
-    let fraction = (high & 0xfffff) * 2 ** 32 + low;
-    if (exponent === 0x7ff) {
-        if (fraction === 0)
-            return sign * Infinity;
-        return NaN;
-    }
-    if (exponent === 0) {
-        if (fraction === 0)
-            return sign * 0;
-        exponent = 1;
-    }
-    else {
-        fraction += 2 ** 52;
-    }
-    return sign * fraction * 2 ** (exponent - 1075);
-};
-export const readFloatLE = (val, offset) => {
-    const bits = val[offset] |
-        (val[offset + 1] << 8) |
-        (val[offset + 2] << 16) |
-        (val[offset + 3] << 24);
-    const sign = bits >>> 31 ? -1 : 1;
-    let exponent = (bits >>> 23) & 0xff;
-    let fraction = bits & 0x7fffff;
-    if (exponent === 0xff) {
-        if (fraction === 0)
-            return sign * Infinity;
-        return NaN;
-    }
-    if (exponent === 0) {
-        if (fraction === 0)
-            return sign * 0;
-        exponent = 1;
-    }
-    else {
-        fraction |= 0x800000;
-    }
-    return sign * fraction * 2 ** (exponent - 150);
-};
-export const readUint32 = (val, offset) => {
-    return ((val[offset] |
-        (val[offset + 1] << 8) |
-        (val[offset + 2] << 16) |
-        (val[offset + 3] << 24)) >>>
-        0);
-};
-export const readInt32 = (val, offset) => {
-    return (val[offset] |
-        (val[offset + 1] << 8) |
-        (val[offset + 2] << 16) |
-        (val[offset + 3] << 24));
-};
-export const readInt16 = (val, offset) => {
-    return ((val[offset] | (val[offset + 1] << 8)) << 16) >> 16;
-};
-export const readUint16 = (val, offset) => {
-    return (val[offset] | (val[offset + 1] << 8)) >>> 0;
-};
-//# sourceMappingURL=bitWise.js.map

package/dist/src/client/operations.d.ts

@@ -1,32 +0,0 @@
-import { PropDef, SchemaTypeDef } from '@based/schema/def';
-import { DbClient } from './index.js';
-import { ModifyState } from './modify/ModifyRes.js';
-export declare class ModifyCtx {
-    constructor(db: DbClient);
-    len: number;
-    id: number;
-    hasSortField: number;
-    hasSortText: number;
-    queue: Map<(payload: any) => void, ModifyState>;
-    ctx: {
-        offsets?: Record<number, number>;
-    };
-    payload: Uint8Array;
-    max: number;
-    buf: Uint8Array;
-    field: number;
-    prefix0: number;
-    prefix1: number;
-    lastMain: number;
-    mergeMain: (PropDef | any)[] | null;
-    mergeMainSize: number;
-    db: DbClient;
-    dirtyRanges: Set<number>;
-    dirtyTypes: Map<number, number>;
-    markNodeDirty(schema: SchemaTypeDef, nodeId: number): void;
-    markTypeDirty(schema: SchemaTypeDef): void;
-    updateMax(): void;
-    getData(lastIds: Record<number, number>): Uint8Array;
-}
-export declare const flushBuffer: (db: DbClient) => Promise<void>;
-export declare const startDrain: (db: DbClient) => void;