@based/db 0.0.28 → 0.0.29

package/dist/src/client/query/toBuffer.js

@@ -13,6 +13,7 @@ const byteSize = (arr) => {
 export function defToBuffer(db, def) {
     const result = [];
     const include = includeToBuffer(db, def);
+    // ---------------------------------------
     def.references.forEach((ref) => {
         include.push(...defToBuffer(db, ref));
         if (ref.errors) {
@@ -32,27 +33,21 @@ export function defToBuffer(db, def) {
         edgesSize = byteSize(edges);
     }
     const size = (edges ? edgesSize + 3 : 0) + byteSize(include);
+    // ---------------------------------------
+    // move down and will handle size after store the size Var
+    // only for references | edges
     if (def.type === QueryDefType.Root) {
-        let filter;
-        let filterSize = 0;
         let search;
         let searchSize = 0;
+        const filterSize = def.filter.size || 0;
         if (def.search) {
             search = searchToBuffer(def.search);
             searchSize = def.search.size;
         }
-        if (def.filter.size) {
-            filter = filterToBuffer(def.filter);
-            filterSize = filter.byteLength;
-        }
-        // [type,type]
-        // [q type] 0 == id, 1 === ids, 2 === type only
         if (def.target.resolvedAlias) {
-            // put this somehwere else at some point
             const alias = def.target.resolvedAlias;
             const aliasStr = ENCODER.encode(alias.value);
             const aliasLen = aliasStr.byteLength;
-            // filter is nice for things like access
             const buf = new Uint8Array(8 + filterSize + aliasLen);
             buf[0] = 3;
             buf[1] = def.schema.idUint8[0];
@@ -64,14 +59,11 @@ export function defToBuffer(db, def) {
             buf[6 + aliasLen] = filterSize;
             buf[7 + aliasLen] = filterSize >>> 8;
             if (filterSize) {
-                buf.set(filter, 8 + aliasLen);
+                buf.set(filterToBuffer(def.filter), 8 + aliasLen);
             }
             result.push(buf);
         }
         else if (def.target.id) {
-            // type 0
-            // 0: 4 [id]
-            // 0: 2 [filterSize]
             const buf = new Uint8Array(9 + filterSize);
             buf[0] = 0;
             buf[1] = def.schema.idUint8[0];
@@ -83,7 +75,7 @@ export function defToBuffer(db, def) {
             buf[7] = filterSize;
             buf[8] = filterSize >>> 8;
             if (filterSize) {
-                buf.set(filter, 9);
+                buf.set(filterToBuffer(def.filter), 9);
             }
             result.push(buf);
         }
@@ -95,15 +87,10 @@ export function defToBuffer(db, def) {
                 sortSize = sort.byteLength;
             }
             let aggregation;
-            aggregation = createAggFlagBuffer(def.aggregation);
+            if (def.aggregation) {
+                aggregation = createAggFlagBuffer(def.aggregation);
+            }
             if (def.target.ids) {
-                // type 1
-                // 1: 4 + ids * 4 [ids len] [id,id,id]
-                // 1: 8 [offset, limit]
-                // 1: 2 [filter size]
-                // ?filter
-                // 1: 2 [sort size]
-                // ?sort
                 if (!sortSize &&
                     (def.range.offset || def.range.limit < def.target.ids.length)) {
                     def.target.ids = def.target.ids.slice(def.range.offset, def.range.offset + def.range.limit);
@@ -129,7 +116,8 @@ export function defToBuffer(db, def) {
                 buf[idsSize + 15] = filterSize;
                 buf[idsSize + 16] = filterSize >>> 8;
                 if (filterSize) {
-                    buf.set(filter, idsSize + 17);
+                    // just write it here dont do set
+                    buf.set(filterToBuffer(def.filter), idsSize + 17);
                 }
                 buf[17 + filterSize + idsSize] = sortSize;
                 buf[18 + filterSize + idsSize] = sortSize >>> 8;
@@ -145,13 +133,6 @@ export function defToBuffer(db, def) {
                 result.push(buf);
             }
             else {
-                // type 2
-                // 2: 0
-                // 2: 8 [offset, limit]
-                // 2: 2 [filter size]
-                // ?filter
-                // 2: 2 [sort size]
-                // ?sort
                 const buf = new Uint8Array(18 + filterSize + sortSize + searchSize);
                 buf[0] = 2;
                 buf[1] = def.schema.idUint8[0];
@@ -167,7 +148,7 @@ export function defToBuffer(db, def) {
                 buf[11] = filterSize;
                 buf[12] = filterSize >>> 8;
                 if (filterSize) {
-                    buf.set(filter, 13);
+                    buf.set(filterToBuffer(def.filter), 13);
                 }
                 buf[13 + filterSize] = sortSize;
                 buf[14 + filterSize] = sortSize >>> 8;
@@ -187,19 +168,12 @@ export function defToBuffer(db, def) {
         }
     }
     else if (def.type === QueryDefType.References) {
-        // TODO filter edge
-        let filter;
-        if (def.filter.size) {
-            filter = filterToBuffer(def.filter);
-        }
+        const filterSize = def.filter.size || 0;
         let sort;
         if (def.sort) {
             sort = createSortBuffer(def.sort);
         }
-        // ADD IT
-        // TODO: ADD RANGE [offset,limit] (8 bytes)
         const sortSize = sort?.byteLength ?? 0;
-        const filterSize = filter?.byteLength ?? 0;
         const modsSize = filterSize + sortSize;
         const meta = new Uint8Array(modsSize + 10 + 8);
         const sz = size + 7 + modsSize + 8;
@@ -218,8 +192,8 @@ export function defToBuffer(db, def) {
         meta[12] = def.range.limit >>> 8;
         meta[13] = def.range.limit >>> 16;
         meta[14] = def.range.limit >>> 24;
-        if (filter) {
-            meta.set(filter, 15);
+        if (filterSize) {
+            meta.set(filterToBuffer(def.filter), 15);
         }
         if (sort) {
             meta.set(sort, 15 + filterSize);

package/dist/src/client/query/types.d.ts

@@ -27,9 +27,10 @@ export type Target = {
     };
 };
 export declare const isRefDef: (def: QueryDef) => def is QueryDefRest;
+export type FilterCondition = Uint8Array;
 export type QueryDefFilter = {
     size: number;
-    conditions: Map<number, Uint8Array[]>;
+    conditions: Map<number, FilterCondition[]>;
     exists?: {
         prop: PropDef | PropDefEdge;
         negate: boolean;
@@ -37,7 +38,7 @@ export type QueryDefFilter = {
     references?: Map<number, QueryDefFilter>;
     fromRef?: PropDef;
     schema?: SchemaTypeDef;
-    edges?: Map<number, Uint8Array[]>;
+    edges?: Map<number, FilterCondition[]>;
     or?: QueryDefFilter;
     and?: QueryDefFilter;
 };

package/dist/src/client/query/validation.d.ts

@@ -61,9 +61,7 @@ export declare const searchIncorrecQueryValue: (def: QueryDef, payload: any) =>
 export declare const searchIncorrectType: (def: QueryDef, payload: PropDef | PropDefEdge) => void;
 export declare const searchDoesNotExist: (def: QueryDef, field: string, isVector: boolean) => PropDef;
 export declare const validateRange: (def: QueryDef, offset: number, limit: number) => boolean;
-export declare const isValidId: (id: number) => boolean;
-export declare const isValidString: (v: any) => boolean;
-export declare const validateVal: (def: QueryDef, f: Filter, validate: (v: any) => boolean) => boolean;
+export declare const validateVal: (def: QueryDef, f: Filter, validate: (v: any, p: PropDef | PropDefEdge) => boolean) => boolean;
 export declare const validateFilter: (def: QueryDef, prop: PropDef | PropDefEdge, f: Filter) => boolean;
 export declare const validateType: (db: DbClient, def: QueryDef, type: string) => SchemaTypeDef;
 export declare const filterOperatorDoesNotExist: (def: QueryDef, field: string) => void;

package/dist/src/client/query/validation.js

@@ -1,9 +1,9 @@
 import picocolors from 'picocolors';
 import { ALIAS, BINARY, BOOLEAN, REFERENCE, REFERENCES, REVERSE_TYPE_INDEX_MAP, STRING, TEXT, TIMESTAMP, VECTOR, propIsNumerical, createEmptyDef, DEFAULT_MAP, } from '@based/schema/def';
 import { EQUAL, EXISTS, HAS, isNumerical, LIKE, operatorReverseMap, VECTOR_FNS, } from './filter/types.js';
-import { MAX_ID, MAX_ID_VALUE, MAX_IDS_PER_QUERY, MIN_ID_VALUE, } from './thresholds.js';
+import { MAX_IDS_PER_QUERY } from './thresholds.js';
 import { displayTarget, safeStringify } from './display.js';
-import { langCodesMap } from '@based/schema';
+import { isValidId, isValidString, langCodesMap, MAX_ID, } from '@based/schema';
 export const ERR_TARGET_INVAL_TYPE = 1;
 export const ERR_TARGET_INVAL_ALIAS = 2;
 export const ERR_TARGET_EXCEED_MAX_IDS = 3;
@@ -85,18 +85,6 @@ export const validateRange = (def, offset, limit) => {
     }
     return r;
 };
-export const isValidId = (id) => {
-    if (typeof id != 'number' || id < MIN_ID_VALUE || id > MAX_ID_VALUE) {
-        return false;
-    }
-    return true;
-};
-export const isValidString = (v) => {
-    const isVal = typeof v === 'string' ||
-        v instanceof Uint8Array ||
-        ArrayBuffer.isView(v);
-    return isVal;
-};
 export const validateVal = (def, f, validate) => {
     if (def.skipValidation) {
         return false;
@@ -104,13 +92,13 @@ export const validateVal = (def, f, validate) => {
     const value = f[2];
     if (Array.isArray(value)) {
         for (const v of value) {
-            if (!validate(v)) {
+            if (!validate(v, f[2])) {
                 def.errors.push({ code: ERR_FILTER_INVALID_VAL, payload: f });
                 return true;
             }
         }
     }
-    else if (!validate(value)) {
+    else if (!validate(value, f[2])) {
         def.errors.push({
             code: ERR_FILTER_INVALID_VAL,
             payload: f,
@@ -147,7 +135,7 @@ export const validateFilter = (def, prop, f) => {
             });
             return true;
         }
-        if (validateVal(def, f, isValidId)) {
+        if (validateVal(def, f, prop.validation)) {
             return true;
         }
     }
@@ -287,7 +275,7 @@ export const validateSort = (def, field, orderInput) => {
         }
     }
     const type = propDef.typeIndex;
-    if (type === REFERENCES || type === REFERENCE) {
+    if (type === REFERENCES || type === REFERENCE || type === VECTOR) {
         def.errors.push({
             code: ERR_SORT_TYPE,
             payload: propDef,

package/dist/src/client/string.d.ts

@@ -1,4 +1,6 @@
 import { LangCode } from '@based/schema';
+export declare const COMPRESSED = 1;
+export declare const NOT_COMPRESSED = 0;
 export declare const write: (buf: Uint8Array, value: string, offset: number, noCompression: boolean, lang?: LangCode) => number;
 export declare const compress: (str: string) => Uint8Array;
 export declare const decompress: (val: Uint8Array) => string;

package/dist/src/client/string.js

@@ -3,12 +3,8 @@ import { readUint32 } from './../utils.js';
 import makeTmpBuffer from './tmpBuffer.js';
 import { DECODER, ENCODER } from '../utils.js';
 const { getUint8Array: getTmpBuffer } = makeTmpBuffer(4096); // the usual page size?
-// type 0 = no compression; 1 = deflate
-// [lang] [type] [uncompressed size 4] [compressed string] [crc32]
-// var cnt = 0
-// var bytesSaved = 0
-// make this into a package
-// write the type Byte
+export const COMPRESSED = 1;
+export const NOT_COMPRESSED = 0;
 export const write = (buf, value, offset, noCompression, lang) => {
     value = value.normalize('NFKD');
     buf[offset] = lang || 0;
@@ -19,7 +15,7 @@ export const write = (buf, value, offset, noCompression, lang) => {
         buf.copyWithin(offset + 6 + l, offset + 2, offset + 2 + l);
         const size = native.compress(buf, offset + 6, l);
         if (size === 0) {
-            buf[offset + 1] = 0; // not compressed
+            buf[offset + 1] = NOT_COMPRESSED;
             ENCODER.encodeInto(value, buf.subarray(offset + 2));
             buf[offset + l + 2] = crc;
             buf[offset + l + 3] = crc >>>= 8;
@@ -29,7 +25,7 @@ export const write = (buf, value, offset, noCompression, lang) => {
         }
         else {
             let len = l;
-            buf[offset + 1] = 1; // compressed
+            buf[offset + 1] = COMPRESSED;
             buf[offset + 2] = len;
             buf[offset + 3] = len >>>= 8;
             buf[offset + 4] = len >>>= 8;
@@ -38,11 +34,11 @@ export const write = (buf, value, offset, noCompression, lang) => {
             buf[offset + size + 7] = crc >>>= 8;
             buf[offset + size + 8] = crc >>>= 8;
             buf[offset + size + 9] = crc >>>= 8;
-            return size + 10; // 0 C 4 4
+            return size + 10;
         }
     }
     else {
-        buf[offset + 1] = 0; // not compressed
+        buf[offset + 1] = NOT_COMPRESSED;
         buf[offset + l + 2] = crc;
         buf[offset + l + 3] = crc >>>= 8;
         buf[offset + l + 4] = crc >>>= 8;
@@ -63,14 +59,14 @@ export const decompress = (val) => {
 };
 export const read = (val, offset, len) => {
     const type = val[offset + 1];
-    if (type == 1) {
+    if (type == COMPRESSED) {
         const origSize = readUint32(val, offset + 2);
         const newBuffer = getTmpBuffer(origSize);
-        // deflate in browser for this...
+        // Browser fallback required for this
         native.decompress(val, newBuffer, offset + 6, len - 6);
         return DECODER.decode(newBuffer);
     }
-    else if (type == 0) {
+    else if (type == NOT_COMPRESSED) {
         return DECODER.decode(val.subarray(offset + 2, len + offset - 4));
     }
     return '';

package/dist/src/index.d.ts

@@ -22,7 +22,7 @@ export declare class BasedDb {
     constructor(opts: {
         path: string;
         maxModifySize?: number;
-        debug?: boolean;
+        debug?: boolean | 'server';
     });
     create: DbClient['create'];
     copy: DbClient['copy'];

package/dist/src/index.js

@@ -2,8 +2,8 @@ import { compress, decompress } from './client/string.js';
 import { ModifyCtx } from './client/flushModify.js';
 import { DbServer } from './server/index.js';
 import { DbClient } from './client/index.js';
-import picocolors from 'picocolors';
 import { wait } from '@saulx/utils';
+import { debugMode, debugServer } from './utils.js';
 export * from './client/modify/modify.js';
 export { compress, decompress };
 export { ModifyCtx }; // TODO move this somewhere
@@ -23,16 +23,11 @@ export class BasedDb {
     constructor(opts) {
         this.#init(opts);
         if (opts.debug) {
-            for (const key in this) {
-                const fn = this[key];
-                if (typeof fn === 'function') {
-                    // @ts-ignore
-                    this[key] = function () {
-                        const str = [`[${key}]`, ...arguments].join(' ');
-                        console.info(picocolors.dim(str));
-                        return fn.apply(this, arguments);
-                    };
-                }
+            if (opts.debug === 'server') {
+                debugServer(this.server);
+            }
+            else {
+                debugMode(this);
             }
         }
     }
@@ -49,7 +44,6 @@ export class BasedDb {
         const client = new DbClient({
             maxModifySize,
             hooks: {
-                flushTime: 0,
                 setSchema(schema, fromStart) {
                     return Promise.resolve(server.setSchema(schema, fromStart));
                 },
@@ -59,8 +53,6 @@ export class BasedDb {
                         offsets,
                     });
                 },
-                flushReady: () => { },
-                flushIsReady: new Promise(() => { }),
                 getQueryBuf(buf) {
                     return Promise.resolve(server.getQueryBuf(buf));
                 },
@@ -127,7 +119,7 @@ export class BasedDb {
         await this.isModified();
         // Tmp fix: Gives node time to GC existing buffers else it can incorrectly re-asign to mem
         // Todo: clear all active queries, queues ETC
-        await wait(Math.max(this.client.hooks.flushTime + 10, 10));
+        await wait(Math.max(this.client.flushTime + 10, 10));
         this.client.destroy();
         await this.server.destroy();
     }

package/dist/src/native.d.ts

@@ -21,7 +21,7 @@ declare const _default: {
     createSortIndex: (buf: Uint8Array, dbCtx: any) => any;
     destroySortIndex: (buf: Uint8Array, dbCtx: any) => any;
     xxHash64: (buf: Uint8Array, target: Uint8Array, index: number) => any;
-    base64encode: (dst: Uint8Array, src: Uint8Array, lineMax: number) => Uint8Array;
     equals: (a: Uint8Array, b: Uint8Array) => boolean;
+    expire: (dbCtx: any) => void;
 };
 export default _default;

package/dist/src/native.js

@@ -90,11 +90,11 @@ export default {
     xxHash64: (buf, target, index) => {
         return db.xxHash64(buf, target, index);
     },
-    base64encode: (dst, src, lineMax) => {
-        return db.base64encode(dst, src, lineMax);
-    },
     equals: (a, b) => {
         return !!db.equals(a, b);
     },
+    expire: (dbCtx) => {
+        db.expire(dbCtx);
+    },
 };
 //# sourceMappingURL=native.js.map

package/dist/src/server/csmt/draw-dot.js

@@ -1,6 +1,6 @@
-import { base64encode } from '../../utils.js';
+import { encodeBase64 } from '@saulx/utils';
 function makeLabel(node) {
-    return `${node.key}\n${base64encode(node.hash, 0).substring(0, 5)}`;
+    return `${node.key}\n${encodeBase64(node.hash).substring(0, 5)}`;
 }
 export default function draw(csmt) {
     const root = csmt.getRoot();

package/dist/src/server/csmt/tree.js

@@ -12,9 +12,7 @@ export function createTree(createHash) {
         return createHash().update(lHash).update(rHash).digest();
     }
     function createNode(left, right) {
-        const hash = left && right
-            ? genNodeHash(left.hash, right.hash)
-            : emptyHash;
+        const hash = left && right ? genNodeHash(left.hash, right.hash) : emptyHash;
         return {
             hash,
             key: max(left, right),
@@ -56,7 +54,7 @@ export function createTree(createHash) {
                 return createNode(newLeaf, node);
             }
             else {
-                throw new Error('Key exist');
+                throw new Error(`k=${k} exists`);
             }
         }
         const lDist = distance(k, (left && left.key) || TreeKeyNil);
@@ -120,6 +118,47 @@ export function createTree(createHash) {
             }
         }
     }
+    function updateNodeHash(node) {
+        if (node.left && node.right) {
+            node.hash = genNodeHash(node.left.hash, node.right.hash);
+        }
+    }
+    function updateHash(node, k, hash) {
+        if (!node)
+            return;
+        const { left, right } = node;
+        if (k === node.key && !left && !right) {
+            node.hash = hash;
+        }
+        else {
+            if (left && left.key === k) {
+                updateHash(left, k, hash);
+                updateNodeHash(left);
+                updateNodeHash(node);
+                return;
+            }
+            if (right && right.key === k) {
+                updateHash(right, k, hash);
+                updateNodeHash(right);
+                updateNodeHash(node);
+                return;
+            }
+            const lDist = distance(k, (left && left.key) || TreeKeyNil);
+            const rDist = distance(k, (right && right.key) || TreeKeyNil);
+            if (left && lDist <= rDist) {
+                updateHash(left, k, hash);
+                updateNodeHash(left);
+                updateNodeHash(node);
+                return;
+            }
+            if (right && rDist <= lDist) {
+                updateHash(right, k, hash);
+                updateNodeHash(right);
+                updateNodeHash(node);
+                return;
+            }
+        }
+    }
     function diffAB(diffA, diffB, nodeA, nodeB) {
         if (nodeA &&
             nodeB &&
@@ -190,13 +229,20 @@ export function createTree(createHash) {
         if (!node || (k === node.key && !node.left && !node.right))
             return node;
         const { left, right } = node;
-        if (left && k <= left.key)
+        if (left && left.key === k)
+            return search(left, k);
+        if (right && right.key === k)
+            return search(right, k);
+        const lDist = distance(k, (left && left.key) || TreeKeyNil);
+        const rDist = distance(k, (right && right.key) || TreeKeyNil);
+        if (left && lDist <= rDist)
             return search(left, k);
-        if (right && k <= right.key)
+        if (right && rDist <= lDist)
             return search(right, k);
         return null;
     }
     return {
+        emptyHash,
         getRoot: () => root,
         insert: (k, h, data = null) => {
             if (!(h instanceof Uint8Array)) {
@@ -205,6 +251,11 @@ export function createTree(createHash) {
             const newLeaf = createLeaf(k, h, data);
             root = root ? insert(root, newLeaf) : newLeaf;
         },
+        update: (k, h) => {
+            if (root) {
+                updateHash(root, k, h);
+            }
+        },
         delete: (k) => {
             if (root) {
                 root = deleteNode(root, k);

package/dist/src/server/csmt/types.d.ts

@@ -15,6 +15,7 @@ export interface TreeDiff {
     right: KeyHashPair[];
 }
 export interface Csmt {
+    emptyHash: Uint8Array;
     /**
      * Get the root node.
      */
@@ -27,6 +28,10 @@ export interface Csmt {
      * Delete a key-hash pair from the tree.
      */
     delete: (k: TreeKey) => void;
+    /**
+     * Update node hash.
+     */
+    update: (k: TreeKey, h: Hash) => void;
     /**
      * Compute the diff between this and a given tree.
      */

package/dist/src/server/index.d.ts

@@ -50,10 +50,11 @@ export declare class DbServer {
     stopped: boolean;
     onSchemaChange: OnSchemaChange;
     unlistenExit: ReturnType<typeof exitHook>;
-    constructor({ path, maxModifySize, onSchemaChange, }: {
+    constructor({ path, maxModifySize, onSchemaChange, debug, }: {
         path: string;
         maxModifySize?: number;
         onSchemaChange?: OnSchemaChange;
+        debug?: boolean;
     });
     start(opts?: {
         clean?: boolean;
@@ -85,14 +86,14 @@ export declare class DbServer {
         lastId: number;
     }>;
     createSortIndexBuffer(typeId: number, field: number, start: number, lang: number): SortIndex;
-    updateMerkleTree(): void;
     setSchema(strictSchema: StrictSchema, fromStart?: boolean, transformFns?: TransformFns): (StrictSchema & {
         lastId: number;
     }) | Promise<StrictSchema & {
         lastId: number;
     }>;
     modify(buf: Uint8Array): Record<number, number>;
-    getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
+    addToQueryQueue(resolve: any, buf: any): void;
+    getQueryBuf(buf: Uint8Array, fromQueue?: boolean): Promise<Uint8Array>;
     onQueryEnd(): void;
     stop(noSave?: boolean): Promise<void>;
     destroy(): Promise<void>;