@based/db 0.0.15 → 0.0.16

package/dist/lib/darwin_aarch64/include/selva/hll.h

@@ -8,13 +8,14 @@
 #include <stdbool.h>
 #include "cdefs.h"
 
+#define HLL_INIT_SIZE 8
 
 SELVA_EXPORT
 void hll_init(struct selva_string *hllss, uint8_t precision, bool is_sparse);
 SELVA_EXPORT
 void hll_add(struct selva_string *hllss, uint64_t element);
 SELVA_EXPORT
-double hll_count(struct selva_string *hllss);
+uint8_t *hll_count(struct selva_string *hllss);
 SELVA_EXPORT
 struct selva_string hll_array_union(struct selva_string *hll_array, size_t count);
 

package/dist/lib/linux_aarch64/include/selva/hll.h

@@ -8,13 +8,14 @@
 #include <stdbool.h>
 #include "cdefs.h"
 
+#define HLL_INIT_SIZE 8
 
 SELVA_EXPORT
 void hll_init(struct selva_string *hllss, uint8_t precision, bool is_sparse);
 SELVA_EXPORT
 void hll_add(struct selva_string *hllss, uint64_t element);
 SELVA_EXPORT
-double hll_count(struct selva_string *hllss);
+uint8_t *hll_count(struct selva_string *hllss);
 SELVA_EXPORT
 struct selva_string hll_array_union(struct selva_string *hll_array, size_t count);
 

package/dist/lib/linux_x86_64/include/selva/hll.h

@@ -8,13 +8,14 @@
 #include <stdbool.h>
 #include "cdefs.h"
 
+#define HLL_INIT_SIZE 8
 
 SELVA_EXPORT
 void hll_init(struct selva_string *hllss, uint8_t precision, bool is_sparse);
 SELVA_EXPORT
 void hll_add(struct selva_string *hllss, uint64_t element);
 SELVA_EXPORT
-double hll_count(struct selva_string *hllss);
+uint8_t *hll_count(struct selva_string *hllss);
 SELVA_EXPORT
 struct selva_string hll_array_union(struct selva_string *hll_array, size_t count);
 

package/dist/src/client/bitWise.js

@@ -79,4 +79,5 @@ export const concatUint8Arr = (arrs) => {
     });
     return result;
 };
+const textDecoder = new TextDecoder('utf-8');
 //# sourceMappingURL=bitWise.js.map

package/dist/src/client/modify/cardinality.js

@@ -17,25 +17,24 @@ export function writeHll(value, ctx, def, t, parentId, modifyOp) {
 }
 function addHll(value, ctx, def, t, parentId, modifyOp) {
     const len = value.length;
-    let size = 2 + len * 8;
+    let size = 4 + len * 8;
     if (ctx.len + size + 11 > ctx.max) {
         return RANGE_ERR;
     }
     setCursor(ctx, def, t.prop, t.typeIndex, parentId, modifyOp);
     ctx.buf[ctx.len++] = modifyOp;
-    ctx.buf.writeUint16LE(len, ctx.len);
-    ctx.len += 2;
+    ctx.buf.writeUint32LE(len, ctx.len);
+    ctx.len += 4;
     for (let val of value) {
-        let b;
         if (typeof val === 'string') {
-            b = Buffer.from(val);
+            xxHash64(Buffer.from(val), ctx.buf, ctx.len);
         }
-        else if (!(val instanceof Buffer)) {
-            b = val;
+        else if (val instanceof Buffer && val.byteLength === 8) {
+            ctx.buf.set(val, ctx.len);
+        }
+        else {
             return new ModifyError(t, val);
         }
-        const hash = xxHash64(b);
-        ctx.buf.writeBigUInt64LE(hash, ctx.len);
         ctx.len += 8;
     }
 }

package/dist/src/client/query/BasedDbQuery.d.ts

@@ -14,9 +14,9 @@ export declare class QueryBranch<T> {
     constructor(db: DbClient, def: QueryDef);
     sort(field: string, order?: 'asc' | 'desc'): T;
     filter<O extends Operator>(field: string, operator?: O | boolean, value?: any, opts?: FilterOpts<O>): T;
+    filterBatch(f: FilterAst): this;
     search(query: string, ...fields: Search[]): T;
     search(query: ArrayBufferView, field: string, opts?: Omit<FilterOpts, 'lowerCase'>): T;
-    filterBatch(f: FilterAst): this;
     or(fn: FilterBranchFn): T;
     or(field: string, operator?: Operator | boolean, value?: any, opts?: FilterOpts): T;
     range(offset: number, limit: number): T;

package/dist/src/client/query/BasedDbQuery.js

@@ -1,4 +1,4 @@
-import { createQueryDef, QueryDefType, filter, sort, defToBuffer, filterOr, convertFilter, isAlias, includeField, includeFields, } from './query.js';
+import { createQueryDef, QueryDefType, filter, sort, defToBuffer, filterOr, isAlias, includeField, includeFields, } from './query.js';
 import { BasedQueryResponse } from './BasedIterable.js';
 import { createOrGetEdgeRefQueryDef, createOrGetRefQueryDef, } from './include/utils.js';
 import { FilterBranch } from './filter/FilterBranch.js';
@@ -8,6 +8,7 @@ import { REFERENCE, REFERENCES } from '../../server/schema/types.js';
 import { subscribe } from './subscription/index.js';
 import { registerQuery } from './registerQuery.js';
 import { langCodesMap } from '@based/schema';
+import { convertFilter } from './filter/convertFilter.js';
 export class QueryBranch {
     db;
     def;
@@ -22,6 +23,15 @@ export class QueryBranch {
     }
     filter(field, operator, value, opts) {
         const f = convertFilter(this.def, field, operator, value, opts);
+        if (!f) {
+            // @ts-ignore
+            return this;
+        }
+        filter(this.db, this.def, f, this.def.filter);
+        // @ts-ignore
+        return this;
+    }
+    filterBatch(f) {
         filter(this.db, this.def, f, this.def.filter);
         // @ts-ignore
         return this;
@@ -81,11 +91,6 @@ export class QueryBranch {
         // @ts-ignore
         return this;
     }
-    filterBatch(f) {
-        filter(this.db, this.def, f, this.def.filter);
-        // @ts-ignore
-        return this;
-    }
     or(field, operator, value, opts) {
         if (typeof field === 'function') {
             const f = new FilterBranch(this.db, filterOr(this.db, this.def, [], this.def.filter), this.def);
@@ -94,7 +99,9 @@ export class QueryBranch {
         }
         else {
             const f = convertFilter(this.def, field, operator, value, opts);
-            filterOr(this.db, this.def, f, this.def.filter);
+            if (f) {
+                filterOr(this.db, this.def, f, this.def.filter);
+            }
         }
         // @ts-ignore
         return this;

package/dist/src/client/query/display.js

@@ -1,5 +1,5 @@
 import picocolors from 'picocolors';
-import { BINARY, NUMBER, REFERENCE, REFERENCES, STRING, TEXT, TIMESTAMP, } from '../../server/schema/types.js';
+import { BINARY, CARDINALITY, NUMBER, REFERENCE, REFERENCES, STRING, TEXT, TIMESTAMP, } from '../../server/schema/types.js';
 const decimals = (v) => ~~(v * 100) / 100;
 const sizeCalc = (size) => {
     if (size > 1e6) {
@@ -59,6 +59,9 @@ const prettyPrintVal = (v, type) => {
             return `"${v}"`;
         }
     }
+    if (type === CARDINALITY) {
+        return `${picocolors.blue(v)} ${picocolors.italic(picocolors.dim('unique'))}`;
+    }
     if (type === TIMESTAMP) {
         if (v === 0) {
             return `0 ${picocolors.italic(picocolors.dim('No date'))}`;
@@ -100,8 +103,10 @@ const inspectObject = (object, q, path, level, isLast, isFirst, isObject, depth)
             // skip
         }
         else if (key === 'id') {
-            // @ts-ignore
-            str += v + picocolors.italic(picocolors.dim(` ${q.target.type}`));
+            str +=
+                picocolors.blue(v) +
+                    // @ts-ignore
+                    picocolors.italic(picocolors.dim(` ${q.target.type}`));
             str += ',\n';
         }
         else if (!def) {
@@ -146,6 +151,9 @@ const inspectObject = (object, q, path, level, isLast, isFirst, isObject, depth)
                 }
                 str += prettyPrintVal(v, def.typeIndex);
             }
+            else if (def.typeIndex === CARDINALITY) {
+                str += prettyPrintVal(v, def.typeIndex);
+            }
             else if (def.typeIndex === TIMESTAMP) {
                 str += prettyPrintVal(v, def.typeIndex);
             }

package/dist/src/client/query/filter/FilterBranch.js

@@ -1,4 +1,5 @@
-import { convertFilter, filter, filterOr } from './filter.js';
+import { filter, filterOr } from './filter.js';
+import { convertFilter } from './convertFilter.js';
 export class FilterBranch {
     constructor(db, filterBranch, def) {
         this.def = def;
@@ -16,13 +17,17 @@ export class FilterBranch {
         }
         else {
             const f = convertFilter(this.def, field, operator, value, opts);
-            filterOr(this.db, this.def, f, this.filterBranch);
+            if (f) {
+                filterOr(this.db, this.def, f, this.filterBranch);
+            }
         }
         return this;
     }
     filter(field, operator, value) {
         const f = convertFilter(this.def, field, operator, value);
-        filter(this.db, this.def, f, this.filterBranch);
+        if (f) {
+            filter(this.db, this.def, f, this.filterBranch);
+        }
         return this;
     }
 }

package/dist/src/client/query/filter/convertFilter.d.ts

@@ -0,0 +1,4 @@
+import { QueryDef } from '../types.js';
+import { Operator } from './filter.js';
+import { FilterOpts, FilterAst } from './types.js';
+export declare const convertFilter: (def: QueryDef, field: string, operator?: Operator | boolean, value?: any, opts?: FilterOpts | undefined) => FilterAst;

package/dist/src/client/query/filter/convertFilter.js

@@ -0,0 +1,56 @@
+import { toFilterCtx } from './types.js';
+const normalizeNeedle = (s) => {
+    return s
+        .normalize('NFKD')
+        .split('')
+        .filter((ch) => ch.charCodeAt(0) <= 127)
+        .join('');
+};
+export const convertFilter = (def, field, operator, value, opts) => {
+    if (operator === undefined) {
+        operator = '=';
+        value = true;
+    }
+    else if (typeof operator === 'boolean') {
+        value = operator;
+        operator = '=';
+    }
+    if (value === '' || value === undefined) {
+        return;
+    }
+    if (operator === '!..') {
+        return [
+            [field, toFilterCtx(def, '>', opts), value[1]],
+            [field, toFilterCtx(def, '<', opts), value[0]],
+        ];
+    }
+    else if (operator === '..') {
+        return [
+            [field, toFilterCtx(def, '>', opts), value[0]],
+            [field, toFilterCtx(def, '<', opts), value[1]],
+        ];
+    }
+    else {
+        if (operator == 'like') {
+            if (value == null) {
+                throw new Error('Value is required');
+            }
+            if (value?.normalize) {
+                value = normalizeNeedle(value);
+            }
+            else if (Array.isArray(value)) {
+                if (value[0]?.normalize) {
+                    value = value.map(normalizeNeedle);
+                }
+                else if (value[0]?.BYTES_PER_ELEMENT > 1) {
+                    value = value.map((v) => v.buffer);
+                }
+            }
+            else if (value?.BYTES_PER_ELEMENT > 1) {
+                value = value.buffer;
+            }
+        }
+        return [[field, toFilterCtx(def, operator, opts), value]];
+    }
+};
+//# sourceMappingURL=convertFilter.js.map

package/dist/src/client/query/filter/filter.d.ts

@@ -1,10 +1,9 @@
 import { QueryDef, QueryDefFilter } from '../types.js';
 import { SchemaTypeDef } from '../../../server/schema/schema.js';
-import { FilterOpts, Operator } from './types.js';
+import { Operator } from './types.js';
 import { Filter, FilterAst } from './types.js';
 import { DbClient } from '../../index.js';
 export { Operator, Filter };
 export declare const filterRaw: (db: DbClient, filter: Filter, schema: SchemaTypeDef, conditions: QueryDefFilter, def: QueryDef) => number;
 export declare const filter: (db: DbClient, def: QueryDef, filterAst: FilterAst, conditions: QueryDefFilter) => void;
 export declare const filterOr: (db: DbClient, def: QueryDef, filterAst: FilterAst[], conditions: QueryDefFilter) => QueryDefFilter;
-export declare const convertFilter: (def: QueryDef, field: string, operator?: Operator | boolean, value?: any, opts?: FilterOpts) => FilterAst;

package/dist/src/client/query/filter/filter.js

@@ -1,6 +1,5 @@
 import { isPropDef, ID_FIELD_DEF, TEXT, REFERENCE, } from '../../../server/schema/schema.js';
 import { primitiveFilter } from './primitiveFilter.js';
-import { toFilterCtx } from './types.js';
 import { IsFilter } from './types.js';
 import { langCodesMap } from '@based/schema';
 import { filterFieldDoesNotExist, filterInvalidLang } from '../validation.js';
@@ -22,7 +21,8 @@ const referencesFilter = (db, filter, schema, conditions, def) => {
                     const edgeDef = edges[p];
                     if (edgeDef) {
                         conditions.edges ??= new Map();
-                        size += 3 + primitiveFilter(edgeDef, filter, conditions, def.lang);
+                        size +=
+                            3 + primitiveFilter(def, edgeDef, filter, conditions, def.lang);
                     }
                 }
             }
@@ -71,16 +71,16 @@ export const filterRaw = (db, filter, schema, conditions, def) => {
                     filterInvalidLang(def, field);
                     return 0;
                 }
-                return primitiveFilter(fieldDef, filter, conditions, code);
+                return primitiveFilter(def, fieldDef, filter, conditions, code);
             }
         }
         if (field === 'id') {
             fieldDef = ID_FIELD_DEF;
-            return primitiveFilter(fieldDef, filter, conditions, def.lang);
+            return primitiveFilter(def, fieldDef, filter, conditions, def.lang);
         }
         return referencesFilter(db, filter, schema, conditions, def);
     }
-    return primitiveFilter(fieldDef, filter, conditions, def.lang);
+    return primitiveFilter(def, fieldDef, filter, conditions, def.lang);
 };
 export const filter = (db, def, filterAst, conditions) => {
     for (const f of filterAst) {
@@ -104,55 +104,4 @@ export const filterOr = (db, def, filterAst, conditions) => {
     conditions.size += conditions.or.size;
     return conditions.or;
 };
-const normalizeNeedle = (s) => {
-    return s
-        .normalize('NFKD')
-        .split('')
-        .filter((ch) => ch.charCodeAt(0) <= 127)
-        .join('');
-};
-export const convertFilter = (def, field, operator, value, opts) => {
-    if (operator === undefined) {
-        operator = '=';
-        value = true;
-    }
-    else if (typeof operator === 'boolean') {
-        value = operator;
-        operator = '=';
-    }
-    if (operator === '!..') {
-        return [
-            [field, toFilterCtx(def, '>', opts), value[1]],
-            [field, toFilterCtx(def, '<', opts), value[0]],
-        ];
-    }
-    else if (operator === '..') {
-        return [
-            [field, toFilterCtx(def, '>', opts), value[0]],
-            [field, toFilterCtx(def, '<', opts), value[1]],
-        ];
-    }
-    else {
-        if (operator == 'like') {
-            if (value == null) {
-                throw new Error('Value is required');
-            }
-            if (value?.normalize) {
-                value = normalizeNeedle(value);
-            }
-            else if (Array.isArray(value)) {
-                if (value[0]?.normalize) {
-                    value = value.map(normalizeNeedle);
-                }
-                else if (value[0]?.BYTES_PER_ELEMENT > 1) {
-                    value = value.map((v) => v.buffer);
-                }
-            }
-            else if (value?.BYTES_PER_ELEMENT > 1) {
-                value = value.buffer;
-            }
-        }
-        return [[field, toFilterCtx(def, operator, opts), value]];
-    }
-};
 //# sourceMappingURL=filter.js.map

package/dist/src/client/query/filter/primitiveFilter.d.ts

@@ -1,5 +1,5 @@
 import { PropDef, PropDefEdge } from '../../../server/schema/types.js';
-import { QueryDefFilter } from '../types.js';
+import { QueryDef, QueryDefFilter } from '../types.js';
 import { Filter } from './types.js';
 import { LangCode } from '@based/schema';
-export declare const primitiveFilter: (prop: PropDef | PropDefEdge, filter: Filter, conditions: QueryDefFilter, lang: LangCode) => number;
+export declare const primitiveFilter: (def: QueryDef, prop: PropDef | PropDefEdge, filter: Filter, conditions: QueryDefFilter, lang: LangCode) => number;

package/dist/src/client/query/filter/primitiveFilter.js

@@ -3,7 +3,11 @@ import { EQUAL, isNumerical } from './types.js';
 import { createVariableFilterBuffer } from './createVariableFilterBuffer.js';
 import { createFixedFilterBuffer } from './createFixedFilterBuffer.js';
 import { createReferenceFilter } from './createReferenceFilter.js';
-export const primitiveFilter = (prop, filter, conditions, lang) => {
+import { validateFilter } from '../validation.js';
+export const primitiveFilter = (def, prop, filter, conditions, lang) => {
+    if (validateFilter(def, prop, filter)) {
+        return;
+    }
     let [, ctx, value] = filter;
     const fieldIndexChar = prop.prop;
     let buf;

package/dist/src/client/query/filter/types.d.ts

@@ -4,8 +4,8 @@ export type Filter = [fieldStr: string, ctx: FilterCtx, value: any];
 export type FilterBranchFn = (filterBranch: FilterBranch) => void;
 export type FilterAst = (Filter | FilterAst)[];
 export declare const IsFilter: (f: FilterAst) => f is Filter;
-export declare const validOperators: readonly ["=", "has", "!has", "<", ">", "!=", "like", ">=", "<=", "..", "!.."];
 export type Operator = '=' | 'has' | '!has' | '<' | '>' | '!=' | 'like' | '>=' | '<=' | '..' | '!..' | 'like';
+export declare const VECTOR_FNS: string[];
 export type FilterOpts<O = Operator> = {
     lowerCase?: boolean;
     fn?: 'dotProduct' | 'manhattanDistance' | 'cosineSimilarity' | 'euclideanDistance';
@@ -54,3 +54,4 @@ export declare const VECTOR_COSTINE_SIMILARITY = 2;
 export declare const VECTOR_EUCLIDEAN_DIST = 3;
 export declare const getVectorFn: (optsFn?: FilterOpts["fn"]) => 0 | 1 | 2 | 3;
 export declare const toFilterCtx: (def: QueryDef, op: Operator, opts?: FilterOpts) => FilterCtx;
+export declare const operatorReverseMap: Record<OPERATOR, string>;

package/dist/src/client/query/filter/types.js

@@ -5,18 +5,11 @@ export const IsFilter = (f) => {
     }
     return false;
 };
-export const validOperators = [
-    '=',
-    'has',
-    '!has',
-    '<',
-    '>',
-    '!=',
-    'like',
-    '>=',
-    '<=',
-    '..',
-    '!..',
+export const VECTOR_FNS = [
+    'dotProduct',
+    'manhattanDistance',
+    'cosineSimilarity',
+    'euclideanDistance',
 ];
 // -------------------------------------------
 // operations shared
@@ -119,4 +112,22 @@ export const toFilterCtx = (def, op, opts = {}) => {
     }
     filterOperatorDoesNotExist(def, op);
 };
+export const operatorReverseMap = {
+    [EQUAL]: '=',
+    [HAS]: 'has',
+    [ENDS_WITH]: 'endsWith',
+    [STARTS_WITH]: 'startsWith',
+    [GREATER_THAN]: '>',
+    [SMALLER_THAN]: '<',
+    [GREATER_THAN_INCLUSIVE]: '>=',
+    [SMALLER_THAN_INCLUSIVE]: '<=',
+    [RANGE]: '..',
+    [RANGE_EXCLUDE]: '!..',
+    [EQUAL_LOWER_CASE]: '= (lowerCase)',
+    [HAS_TO_LOWER_CASE]: 'has (lowerCase)',
+    [STARTS_WITH_LOWER_CASE]: 'startsWith (lowerCase)',
+    [ENDS_WITH_LOWER_CASE]: 'endsWith (lowerCase)',
+    [LIKE]: 'like',
+    [EQUAL_CRC32]: '= (crc32)',
+};
 //# sourceMappingURL=types.js.map

package/dist/src/client/query/read/read.js

@@ -1,7 +1,8 @@
-import { ALIAS, ALIASES, BINARY, BOOLEAN, ENUM, INT16, INT32, INT8, NUMBER, STRING, TEXT, TIMESTAMP, UINT16, UINT32, UINT8, VECTOR, JSON, } from '../../../server/schema/types.js';
+import { ALIAS, ALIASES, BINARY, BOOLEAN, ENUM, INT16, INT32, INT8, NUMBER, STRING, TEXT, TIMESTAMP, UINT16, UINT32, UINT8, VECTOR, JSON, CARDINALITY, } from '../../../server/schema/types.js';
 import { read, readUtf8 } from '../../string.js';
 import { readDoubleLE, readFloatLE, readInt16, readInt32, readUint16, readUint32, } from '../../bitWise.js';
 import { inverseLangMap } from '@based/schema';
+import { READ_EDGE, READ_ID, READ_REFERENCE, READ_REFERENCES } from './types.js';
 const addField = (p, value, item, defaultOnly = false, lang = 0) => {
     let i = p.__isEdge === true ? 1 : 0;
     // TODO OPTMIZE
@@ -58,19 +59,15 @@ const getEmptyField = (p, item) => {
     return select;
 };
 const readMainValue = (prop, result, index, item) => {
-    // 1: timestamp, 4: number
     if (prop.typeIndex === TIMESTAMP || prop.typeIndex === NUMBER) {
         addField(prop, readDoubleLE(result, index), item);
     }
-    // 5: uint32
     else if (prop.typeIndex === UINT32) {
         addField(prop, readUint32(result, index), item);
     }
-    // 9: boolean
     else if (prop.typeIndex === BOOLEAN) {
         addField(prop, Boolean(result[index]), item);
     }
-    // 10: Enum
     else if (prop.typeIndex === ENUM) {
         if (result[index] === 0) {
             addField(prop, undefined, item);
@@ -79,9 +76,7 @@ const readMainValue = (prop, result, index, item) => {
             addField(prop, prop.enum[result[index] - 1], item);
         }
     }
-    // 11: string
     else if (prop.typeIndex === STRING) {
-        // Also delete this default then (same as other string)
         const len = result[index];
         if (len !== 0) {
             const str = readUtf8(result, index + 1, len);
@@ -91,32 +86,25 @@ const readMainValue = (prop, result, index, item) => {
             addField(prop, '', item);
         }
     }
-    // ?: json
     else if (prop.typeIndex === JSON) {
-        addField(prop, global.JSON.parse(Buffer.from(result.subarray(index + 1, index + 1 + result[index])).toString()), item);
+        addField(prop, global.JSON.parse(readUtf8(result, index + 1, index + 1 + result[index])), item);
     }
-    // 25: binary
     else if (prop.typeIndex === BINARY) {
         addField(prop, result.subarray(index + 1, index + 1 + result[index]), item);
     }
-    // 18: int8
     else if (prop.typeIndex === INT8) {
         const signedVal = (result[index] << 24) >> 24;
         addField(prop, signedVal, item);
     }
-    // 19: uint8
     else if (prop.typeIndex === UINT8) {
         addField(prop, result[index], item);
     }
-    // 20: int16
     else if (prop.typeIndex === INT16) {
         addField(prop, readInt16(result, index), item);
     }
-    // 21: uint16
     else if (prop.typeIndex === UINT16) {
         addField(prop, readUint16(result, index), item);
     }
-    // 22: int32
     else if (prop.typeIndex === INT32) {
         addField(prop, readInt32(result, index), item);
     }
@@ -143,7 +131,10 @@ const handleUndefinedProps = (id, q, item) => {
     for (const k in q.include.propsRead) {
         if (q.include.propsRead[k] !== id) {
             const prop = q.schema.reverseProps[k];
-            if (prop.typeIndex === TEXT && q.lang == 0) {
+            if (prop.typeIndex === CARDINALITY) {
+                addField(prop, 0, item);
+            }
+            else if (prop.typeIndex === TEXT && q.lang == 0) {
                 const lan = getEmptyField(prop, item);
                 const lang = q.include.langTextFields.get(prop.prop).codes;
                 if (lang.has(0)) {
@@ -173,16 +164,13 @@ export const readAllFields = (q, result, offset, end, item, id) => {
     while (i < end) {
         const index = result[i];
         i++;
-        // index:255 id
-        if (index === 255) {
+        if (index === READ_ID) {
             handleUndefinedProps(id, q, item);
             return i - offset;
         }
-        // index:252 edge
-        if (index === 252) {
+        if (index === READ_EDGE) {
             let prop = result[i];
-            // index:254 ref
-            if (prop === 254) {
+            if (prop === READ_REFERENCE) {
                 i++;
                 const field = result[i];
                 i++;
@@ -206,9 +194,8 @@ export const readAllFields = (q, result, offset, end, item, id) => {
                     addField(ref.target.propDef, refItem, item);
                     i += size - 5;
                 }
-                // index:253 refs
             }
-            else if (prop === 253) {
+            else if (prop === READ_REFERENCES) {
                 i++;
                 const field = result[i];
                 i++;
@@ -227,7 +214,7 @@ export const readAllFields = (q, result, offset, end, item, id) => {
                 if (t === JSON) {
                     i++;
                     const size = readUint32(result, i);
-                    addField(edgeDef, global.JSON.parse(Buffer.from(result.subarray(i + 6, size + i)).toString()), item);
+                    addField(edgeDef, global.JSON.parse(readUtf8(result, i + 6, size + i)), item);
                     i += size + 4;
                 }
                 else if (t === BINARY) {
@@ -253,9 +240,8 @@ export const readAllFields = (q, result, offset, end, item, id) => {
                     i += edgeDef.len;
                 }
             }
-            // index:254 ref
         }
-        else if (index === 254) {
+        else if (index === READ_REFERENCE) {
             const field = result[i];
             i++;
             const size = readUint32(result, i);
@@ -278,9 +264,8 @@ export const readAllFields = (q, result, offset, end, item, id) => {
                 addField(ref.target.propDef, refItem, item);
                 i += size - 5;
             }
-            // index:253 refs
         }
-        else if (index === 253) {
+        else if (index === READ_REFERENCES) {
             const field = result[i];
             i++;
             const ref = q.references.get(field);
@@ -296,7 +281,13 @@ export const readAllFields = (q, result, offset, end, item, id) => {
         }
         else {
             const prop = q.schema.reverseProps[index];
-            if (prop.typeIndex === JSON) {
+            if (prop.typeIndex === CARDINALITY) {
+                q.include.propsRead[index] = id;
+                const size = readUint32(result, i);
+                addField(prop, readUint32(result, i + 4), item);
+                i += size + 4;
+            }
+            else if (prop.typeIndex === JSON) {
                 q.include.propsRead[index] = id;
                 const size = readUint32(result, i);
                 addField(prop, global.JSON.parse(Buffer.from(result.subarray(i + 6, i + size)).toString()), item);
@@ -329,7 +320,6 @@ export const readAllFields = (q, result, offset, end, item, id) => {
                         addField(prop, read(result, i + 4, size), item);
                     }
                     else {
-                        // q.include.propsRead[index] = id
                         addField(prop, read(result, i + 4, size), item, false, result[i + 4]);
                     }
                 }
@@ -360,7 +350,6 @@ export const readAllFields = (q, result, offset, end, item, id) => {
             }
         }
     }
-    // to add defaults - may not optimal for performance
     handleUndefinedProps(id, q, item);
     return i - offset;
 };

package/dist/src/client/query/read/types.d.ts

@@ -0,0 +1,4 @@
+export declare const READ_ID = 255;
+export declare const READ_EDGE = 252;
+export declare const READ_REFERENCES = 253;
+export declare const READ_REFERENCE = 254;

package/dist/src/client/query/read/types.js

@@ -0,0 +1,5 @@
+export const READ_ID = 255;
+export const READ_EDGE = 252;
+export const READ_REFERENCES = 253;
+export const READ_REFERENCE = 254;
+//# sourceMappingURL=types.js.map

package/dist/src/client/query/validation.d.ts

@@ -1,5 +1,6 @@
-import { PropDef, SchemaTypeDef } from '../../server/schema/types.js';
+import { PropDef, PropDefEdge, SchemaTypeDef } from '../../server/schema/types.js';
 import { DbClient } from '../index.js';
+import { Filter } from './query.js';
 import { QueryByAliasObj, QueryDef } from './types.js';
 export type QueryError = {
     code: number;
@@ -29,8 +30,13 @@ declare const messages: {
     7: (p: any) => string;
     12: (p: any) => string;
     9: (p: any) => string;
+    8: (p: Filter) => string;
+    11: (p: any) => string;
+    10: (p: any) => string;
 };
 export type ErrorCode = keyof typeof messages;
+export declare const isValidId: (id: number) => boolean;
+export declare const validateFilter: (def: QueryDef, prop: PropDef | PropDefEdge, f: Filter) => boolean;
 export declare const validateType: (db: DbClient, def: QueryDef, type: string) => SchemaTypeDef;
 export declare const filterOperatorDoesNotExist: (def: QueryDef, field: string) => void;
 export declare const filterInvalidLang: (def: QueryDef, field: string) => void;

package/dist/src/client/query/validation.js

@@ -1,5 +1,7 @@
-import { ALIAS } from '../../server/schema/types.js';
-import { MAX_ID, MAX_IDS_PER_QUERY } from './thresholds.js';
+import { ALIAS, BOOLEAN, REFERENCE, REFERENCES, STRING, TEXT, VECTOR, } from '../../server/schema/types.js';
+import { propIsNumerical } from '../../server/schema/utils.js';
+import { EQUAL, HAS, isNumerical, LIKE, operatorReverseMap, VECTOR_FNS, } from './filter/types.js';
+import { MAX_ID, MAX_ID_VALUE, MAX_IDS_PER_QUERY, MIN_ID_VALUE, } from './thresholds.js';
 export const ERR_TARGET_INVAL_TYPE = 1;
 export const ERR_TARGET_INVAL_ALIAS = 2;
 export const ERR_TARGET_EXCEED_MAX_IDS = 3;
@@ -13,17 +15,20 @@ export const ERR_FILTER_INVALID_VAL = 10;
 export const ERR_FILTER_INVALID_OPTS = 11;
 export const ERR_FILTER_INVALID_LANG = 12;
 export const ERR_INCLUDE_INVALID_LANG = 13;
+const safeStringify = (p) => {
+    var v;
+    try {
+        v = JSON.stringify(p).replace(/"/g, '');
+    }
+    catch (err) {
+        v = '';
+    }
+    return v;
+};
 const messages = {
     [ERR_TARGET_INVAL_TYPE]: (p) => `Type "${p}" does not exist`,
     [ERR_TARGET_INVAL_ALIAS]: (p) => {
-        var v;
-        try {
-            v = JSON.stringify(p).replace(/"/g, '');
-        }
-        catch (err) {
-            v = '';
-        }
-        return `Invalid alias prodived to query "${v}"`;
+        return `Invalid alias prodived to query "${safeStringify(p)}"`;
     },
     [ERR_TARGET_EXCEED_MAX_IDS]: (p) => `Exceeds max ids ${~~(p.length / 1e3)}k (max ${MAX_IDS_PER_QUERY / 1e3}k)`,
     [ERR_TARGET_INVAL_IDS]: (p) => `Ids should be of type array or Uint32Array with valid ids`,
@@ -33,6 +38,118 @@ const messages = {
     [ERR_FILTER_ENOENT]: (p) => `Filter: field does not exist "${p}"`,
     [ERR_FILTER_INVALID_LANG]: (p) => `Filter: invalid lang "${p}"`,
     [ERR_FILTER_OP_ENOENT]: (p) => `Filter: invalid operator "${p}"`,
+    [ERR_FILTER_OP_FIELD]: (p) => `Cannot use operator "${operatorReverseMap[p[1].operation]}" on field "${p[0]}"`,
+    [ERR_FILTER_INVALID_OPTS]: (p) => {
+        return `Filter: Invalid opts "${safeStringify(p)}"`;
+    },
+    [ERR_FILTER_INVALID_VAL]: (p) => {
+        return `Filter: Invalid value ${p[0]} ${operatorReverseMap[p[1].operation]} "${safeStringify(p[2])}"`;
+    },
+};
+export const isValidId = (id) => {
+    if (typeof id != 'number') {
+        return false;
+    }
+    else if (id < MIN_ID_VALUE || id > MAX_ID_VALUE) {
+        return false;
+    }
+    return true;
+};
+export const validateFilter = (def, prop, f) => {
+    const t = prop.typeIndex;
+    const op = f[1].operation;
+    const value = f[2];
+    if (t === REFERENCES) {
+        if (op == LIKE) {
+            def.errors.push({
+                code: ERR_FILTER_OP_FIELD,
+                payload: f,
+            });
+            return true;
+        }
+    }
+    else if (t === REFERENCE) {
+        if (op != EQUAL) {
+            def.errors.push({
+                code: ERR_FILTER_OP_FIELD,
+                payload: f,
+            });
+            return true;
+        }
+        if (Array.isArray(value)) {
+            for (const v of value) {
+                if (!isValidId(v)) {
+                    def.errors.push({
+                        code: ERR_FILTER_INVALID_VAL,
+                        payload: f,
+                    });
+                    return true;
+                }
+            }
+        }
+        else if (!isValidId(value)) {
+            def.errors.push({
+                code: ERR_FILTER_INVALID_VAL,
+                payload: f,
+            });
+            return true;
+        }
+    }
+    else if (t === VECTOR) {
+        if (isNumerical(op) || op === HAS) {
+            def.errors.push({
+                code: ERR_FILTER_OP_FIELD,
+                payload: f,
+            });
+            return true;
+        }
+        if (op === LIKE) {
+            const opts = f[1].opts;
+            if ((opts.fn && !VECTOR_FNS.includes(opts.fn)) ||
+                (opts.score != undefined && typeof opts.score !== 'number')) {
+                def.errors.push({
+                    code: ERR_FILTER_INVALID_OPTS,
+                    payload: f,
+                });
+                return true;
+            }
+        }
+    }
+    else if (t === TEXT || t === STRING) {
+        if (isNumerical(op)) {
+            def.errors.push({
+                code: ERR_FILTER_OP_FIELD,
+                payload: f,
+            });
+            return true;
+        }
+        if (op === LIKE) {
+            const opts = f[1].opts;
+            if (opts.score &&
+                (typeof opts.score !== 'number' || opts.score < 0 || opts.score > 255)) {
+                def.errors.push({
+                    code: ERR_FILTER_INVALID_OPTS,
+                    payload: f,
+                });
+                return true;
+            }
+        }
+    }
+    else if (propIsNumerical(prop) && op !== EQUAL && !isNumerical(op)) {
+        def.errors.push({
+            code: ERR_FILTER_OP_FIELD,
+            payload: f,
+        });
+        return true;
+    }
+    else if (t === BOOLEAN && op !== EQUAL) {
+        def.errors.push({
+            code: ERR_FILTER_OP_FIELD,
+            payload: f,
+        });
+        return true;
+    }
+    return false;
 };
 export const validateType = (db, def, type) => {
     const r = db.schemaTypesParsed[type];
@@ -155,8 +272,8 @@ export const handleErrors = (def) => {
         for (const err of def.errors) {
             name += `  ${messages[err.code](err.payload)}\n`;
         }
-        const err = new Error(name);
-        err.stack = '';
+        const err = new Error(`Query\n`);
+        err.stack = name;
         throw err;
     }
 };
@@ -193,40 +310,6 @@ export const EMPTY_SCHEMA_DEF = {
     stringProps: Buffer.from([]),
     stringPropsLoop: [],
 };
-// import { ALIAS, PropDef, PropDefEdge } from '../../server/schema/types.js'
-// import {
-//   MAX_IDS_PER_QUERY,
-//   MIN_ID_VALUE,
-//   MAX_ID_VALUE,
-//   MAX_BUFFER_SIZE,
-// } from './thresholds.js'
-// import { QueryByAliasObj, QueryDef } from './types.js'
-// import { DbClient } from '../index.js'
-// export const isValidId = (id: number): void => {
-//   if (typeof id != 'number') {
-//     throw new Error('Id has to be a number')
-//   } else if (id < MIN_ID_VALUE || id > MAX_ID_VALUE) {
-//     throw new Error(
-//       `Invalid Id: The Id should range between ${MIN_ID_VALUE} and ${MAX_ID_VALUE}.)`,
-//     )
-//   }
-// }
-// export const isValidType = (
-//   type: string,
-//   schema: DbClient['schemaTypesParsed'],
-// ): void => {
-//   if (!schema[type]) {
-//     throw new Error(`Incorrect type provided to query "${type}"`)
-//   }
-// }
-// export const isValidAlias = (def: QueryDef, id: QueryByAliasObj) => {
-//   for (const key in id) {
-//     const prop = def.schema.props[key]
-//     if (!prop || prop.typeIndex !== ALIAS) {
-//       throw new Error(`Incorrect alias provided to query "${key}"`)
-//     }
-//   }
-// }
 // export const checkMaxIdsPerQuery = (
 //   ids: (number | QueryByAliasObj)[],
 // ): void => {

package/dist/src/client/xxHash64.d.ts

@@ -1 +1 @@
-export declare const xxHash64: (buf: Buffer) => any;
+export declare const xxHash64: (buf: Buffer, target?: Buffer, index?: number) => Buffer;

package/dist/src/client/xxHash64.js

@@ -1,5 +1,10 @@
 import native from '../native.js';
-export const xxHash64 = (buf) => {
-    return native.xxHash64(buf);
+export const xxHash64 = (buf, target, index) => {
+    if (!target) {
+        target = Buffer.allocUnsafe(8);
+        index = 0;
+    }
+    native.xxHash64(buf, target, index);
+    return target;
 };
 //# sourceMappingURL=xxHash64.js.map

package/dist/src/index.d.ts

@@ -7,12 +7,14 @@ export * from './client/modify/modify.js';
 export { compress, decompress };
 export { ModifyCtx };
 export { DbClient, DbServer };
+export { xxHash64 } from './client/xxHash64.js';
 export declare class BasedDb {
+    #private;
     client: DbClient;
     server: DbServer;
     fileSystemPath: string;
     maxModifySize: number;
-    constructor({ path, maxModifySize, }: {
+    constructor(opt: {
         path: string;
         maxModifySize?: number;
     });
@@ -31,4 +33,5 @@ export declare class BasedDb {
     save: DbServer['save'];
     migrateSchema: DbServer['migrateSchema'];
     destroy(): Promise<void>;
+    wipe(): Promise<void>;
 }

package/dist/src/index.js

@@ -7,12 +7,18 @@ export * from './client/modify/modify.js';
 export { compress, decompress };
 export { ModifyCtx }; // TODO move this somewhere
 export { DbClient, DbServer };
+export { xxHash64 } from './client/xxHash64.js';
 export class BasedDb {
     client;
     server;
     fileSystemPath;
     maxModifySize;
-    constructor({ path, maxModifySize, }) {
+    constructor(opt) {
+        this.#init(opt);
+    }
+    #init({ path, maxModifySize }) {
+        this.fileSystemPath = path;
+        this.maxModifySize = maxModifySize;
         const server = new DbServer({
             path,
             maxModifySize,
@@ -91,5 +97,14 @@ export class BasedDb {
         this.client.destroy();
         return this.server.destroy();
     }
+    async wipe() {
+        const opts = {
+            maxModifySize: this.maxModifySize,
+            path: this.fileSystemPath,
+        };
+        await this.destroy();
+        this.#init(opts);
+        await this.start({ clean: true });
+    }
 }
 //# sourceMappingURL=index.js.map

package/dist/src/native.d.ts

@@ -24,6 +24,6 @@ declare const _default: {
     crc32: (buf: Buffer) => any;
     createSortIndex: (buf: Buffer, dbCtx: any) => any;
     destroySortIndex: (buf: Buffer, dbCtx: any) => any;
-    xxHash64: (buf: Buffer) => any;
+    xxHash64: (buf: Buffer, target: Buffer, index: number) => any;
 };
 export default _default;

package/dist/src/native.js

@@ -93,8 +93,8 @@ export default {
     destroySortIndex: (buf, dbCtx) => {
         return db.destroySortIndex(dbCtx, buf);
     },
-    xxHash64: (buf) => {
-        return db.xxHash64(buf);
+    xxHash64: (buf, target, index) => {
+        return db.xxHash64(buf, target, index);
     },
 };
 //# sourceMappingURL=native.js.map

package/dist/src/server/schema/utils.d.ts

@@ -2,5 +2,6 @@ import { DbClient } from '../../client/index.js';
 import { DbServer } from '../index.js';
 import { PropDef, PropDefEdge } from './types.js';
 export declare const propIsSigned: (prop: PropDef | PropDefEdge) => boolean;
+export declare const propIsNumerical: (prop: PropDef | PropDefEdge) => boolean;
 export declare const genRootId: () => number;
 export declare const genId: (db: DbServer | DbClient) => number;

package/dist/src/server/schema/utils.js

@@ -1,9 +1,22 @@
-import { INT16, INT32, INT64, INT8 } from './types.js';
+import { INT16, INT32, INT64, INT8, UINT16, UINT32, UINT8, NUMBER, TIMESTAMP, } from './types.js';
 export const propIsSigned = (prop) => {
-    if (prop.typeIndex === INT16 ||
-        prop.typeIndex === INT32 ||
-        prop.typeIndex === INT64 ||
-        prop.typeIndex === INT8) {
+    const t = prop.typeIndex;
+    if (t === INT16 || t === INT32 || t === INT64 || t === INT8) {
+        return true;
+    }
+    return false;
+};
+export const propIsNumerical = (prop) => {
+    const t = prop.typeIndex;
+    if (t === INT16 ||
+        t === INT32 ||
+        t === INT64 ||
+        t === INT8 ||
+        t === UINT8 ||
+        t === UINT16 ||
+        t === UINT32 ||
+        t === NUMBER ||
+        t === TIMESTAMP) {
         return true;
     }
     return false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.15",
+  "version": "0.0.16",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",