@based/db 0.0.16 → 0.0.17

package/dist/src/client/query/validation.js

@@ -1,7 +1,10 @@
-import { ALIAS, BOOLEAN, REFERENCE, REFERENCES, STRING, TEXT, VECTOR, } from '../../server/schema/types.js';
+import picocolors from 'picocolors';
+import { ALIAS, BINARY, BOOLEAN, REFERENCE, REFERENCES, REVERSE_TYPE_INDEX_MAP, STRING, TEXT, TIMESTAMP, VECTOR, } from '../../server/schema/types.js';
 import { propIsNumerical } from '../../server/schema/utils.js';
 import { EQUAL, HAS, isNumerical, LIKE, operatorReverseMap, VECTOR_FNS, } from './filter/types.js';
 import { MAX_ID, MAX_ID_VALUE, MAX_IDS_PER_QUERY, MIN_ID_VALUE, } from './thresholds.js';
+import { displayTarget, safeStringify } from './display.js';
+import { langCodesMap } from '@based/schema';
 export const ERR_TARGET_INVAL_TYPE = 1;
 export const ERR_TARGET_INVAL_ALIAS = 2;
 export const ERR_TARGET_EXCEED_MAX_IDS = 3;
@@ -15,23 +18,24 @@ export const ERR_FILTER_INVALID_VAL = 10;
 export const ERR_FILTER_INVALID_OPTS = 11;
 export const ERR_FILTER_INVALID_LANG = 12;
 export const ERR_INCLUDE_INVALID_LANG = 13;
-const safeStringify = (p) => {
-    var v;
-    try {
-        v = JSON.stringify(p).replace(/"/g, '');
-    }
-    catch (err) {
-        v = '';
-    }
-    return v;
-};
+export const ERR_SORT_ENOENT = 14;
+export const ERR_SORT_TYPE = 15;
+export const ERR_SORT_ORDER = 16;
+export const ERR_SORT_WRONG_TARGET = 17;
+export const ERR_RANGE_INVALID_OFFSET = 18;
+export const ERR_RANGE_INVALID_LIMIT = 19;
+export const ERR_INVALID_LANG = 20;
+export const ERR_SEARCH_ENOENT = 21;
+export const ERR_SEARCH_TYPE = 22;
+export const ERR_SEARCH_INCORRECT_VALUE = 23;
+export const ERR_SORT_LANG = 24;
 const messages = {
     [ERR_TARGET_INVAL_TYPE]: (p) => `Type "${p}" does not exist`,
     [ERR_TARGET_INVAL_ALIAS]: (p) => {
-        return `Invalid alias prodived to query "${safeStringify(p)}"`;
+        return `Invalid alias prodived to query\n  ${picocolors.italic(safeStringify(p, 100))}`;
     },
     [ERR_TARGET_EXCEED_MAX_IDS]: (p) => `Exceeds max ids ${~~(p.length / 1e3)}k (max ${MAX_IDS_PER_QUERY / 1e3}k)`,
-    [ERR_TARGET_INVAL_IDS]: (p) => `Ids should be of type array or Uint32Array with valid ids`,
+    [ERR_TARGET_INVAL_IDS]: (p) => `Ids should be of type array or Uint32Array with valid ids \n  ${picocolors.italic(safeStringify(p, 100))}`,
     [ERR_TARGET_INVAL_ID]: (p) => `Invalid id should be a number larger then 0 "${p}"`,
     [ERR_INCLUDE_ENOENT]: (p) => `Include: field does not exist "${p}"`,
     [ERR_INCLUDE_INVALID_LANG]: (p) => `Include: invalid lang "${p}"`,
@@ -45,108 +49,144 @@ const messages = {
     [ERR_FILTER_INVALID_VAL]: (p) => {
         return `Filter: Invalid value ${p[0]} ${operatorReverseMap[p[1].operation]} "${safeStringify(p[2])}"`;
     },
+    [ERR_SORT_ENOENT]: (p) => `Sort: field does not exist "${p}"`,
+    [ERR_SORT_WRONG_TARGET]: (p) => `Sort: incorrect qeury target "${displayTarget(p)}"`,
+    [ERR_SORT_ORDER]: (p) => `Sort: incorrect order option "${safeStringify(p.order)}" passed to sort "${p.field}"`,
+    [ERR_SORT_TYPE]: (p) => `Sort: cannot sort on type "${REVERSE_TYPE_INDEX_MAP[p.typeIndex]}" on field "${p.path.join('.')}"`,
+    [ERR_RANGE_INVALID_OFFSET]: (p) => `Range: incorrect offset "${safeStringify(p)}"`,
+    [ERR_RANGE_INVALID_LIMIT]: (p) => `Range: incorrect limit "${safeStringify(p)}"`,
+    [ERR_INVALID_LANG]: (p) => `Invalid locale "${p}"`,
+    [ERR_SEARCH_ENOENT]: (p) => `Search: field does not exist "${p}"`,
+    [ERR_SEARCH_TYPE]: (p) => `Search: incorrect type "${p.path.join('.')}"`,
+    [ERR_SEARCH_INCORRECT_VALUE]: (p) => `Search: incorrect query on field "${safeStringify(p)}"`,
+    [ERR_SORT_LANG]: (p) => `Sort: invalid lang`,
+};
+export const searchIncorrecQueryValue = (def, payload) => {
+    def.errors.push({ code: ERR_SEARCH_INCORRECT_VALUE, payload });
+};
+export const searchIncorrectType = (def, payload) => {
+    def.errors.push({ code: ERR_SEARCH_TYPE, payload });
+};
+export const searchDoesNotExist = (def, field, isVector) => {
+    def.errors.push({ code: ERR_SEARCH_ENOENT, payload: field });
+    if (isVector) {
+        return ERROR_VECTOR;
+    }
+    return ERROR_STRING;
+};
+export const validateRange = (def, offset, limit) => {
+    var r = false;
+    if (typeof offset !== 'number' || offset > MAX_ID || offset < 0) {
+        def.errors.push({ code: ERR_RANGE_INVALID_OFFSET, payload: offset });
+        r = true;
+    }
+    if (typeof limit !== 'number' || limit > MAX_ID || limit < 1) {
+        def.errors.push({ code: ERR_RANGE_INVALID_LIMIT, payload: limit });
+        r = true;
+    }
+    return r;
 };
 export const isValidId = (id) => {
-    if (typeof id != 'number') {
+    if (typeof id != 'number' || id < MIN_ID_VALUE || id > MAX_ID_VALUE) {
         return false;
     }
-    else if (id < MIN_ID_VALUE || id > MAX_ID_VALUE) {
+    return true;
+};
+export const isValidString = (v) => {
+    const isVal = typeof v === 'string' ||
+        v instanceof Buffer ||
+        ArrayBuffer.isView(v);
+    return isVal;
+};
+export const validateVal = (def, f, validate) => {
+    if (def.skipValidation) {
         return false;
     }
-    return true;
+    const value = f[2];
+    if (Array.isArray(value)) {
+        for (const v of value) {
+            if (!validate(v)) {
+                def.errors.push({ code: ERR_FILTER_INVALID_VAL, payload: f });
+                return true;
+            }
+        }
+    }
+    else if (!validate(value)) {
+        def.errors.push({
+            code: ERR_FILTER_INVALID_VAL,
+            payload: f,
+        });
+        return true;
+    }
+    return false;
 };
 export const validateFilter = (def, prop, f) => {
+    if (def.skipValidation) {
+        return false;
+    }
     const t = prop.typeIndex;
     const op = f[1].operation;
-    const value = f[2];
-    if (t === REFERENCES) {
+    if (t === REFERENCES || t === REFERENCE) {
         if (op == LIKE) {
-            def.errors.push({
-                code: ERR_FILTER_OP_FIELD,
-                payload: f,
-            });
+            def.errors.push({ code: ERR_FILTER_OP_FIELD, payload: f });
             return true;
         }
-    }
-    else if (t === REFERENCE) {
-        if (op != EQUAL) {
+        if (t === REFERENCE && op != EQUAL) {
             def.errors.push({
                 code: ERR_FILTER_OP_FIELD,
                 payload: f,
             });
             return true;
         }
-        if (Array.isArray(value)) {
-            for (const v of value) {
-                if (!isValidId(v)) {
-                    def.errors.push({
-                        code: ERR_FILTER_INVALID_VAL,
-                        payload: f,
-                    });
-                    return true;
-                }
-            }
-        }
-        else if (!isValidId(value)) {
-            def.errors.push({
-                code: ERR_FILTER_INVALID_VAL,
-                payload: f,
-            });
+        if (validateVal(def, f, isValidId)) {
             return true;
         }
     }
     else if (t === VECTOR) {
         if (isNumerical(op) || op === HAS) {
-            def.errors.push({
-                code: ERR_FILTER_OP_FIELD,
-                payload: f,
-            });
+            def.errors.push({ code: ERR_FILTER_OP_FIELD, payload: f });
             return true;
         }
         if (op === LIKE) {
             const opts = f[1].opts;
             if ((opts.fn && !VECTOR_FNS.includes(opts.fn)) ||
                 (opts.score != undefined && typeof opts.score !== 'number')) {
-                def.errors.push({
-                    code: ERR_FILTER_INVALID_OPTS,
-                    payload: f,
-                });
+                def.errors.push({ code: ERR_FILTER_INVALID_OPTS, payload: f });
                 return true;
             }
         }
+        if (validateVal(def, f, (v) => ArrayBuffer.isView(v) || v instanceof ArrayBuffer)) {
+            return true;
+        }
     }
-    else if (t === TEXT || t === STRING) {
+    else if (t === TEXT || t === STRING || t === BINARY) {
         if (isNumerical(op)) {
-            def.errors.push({
-                code: ERR_FILTER_OP_FIELD,
-                payload: f,
-            });
+            def.errors.push({ code: ERR_FILTER_OP_FIELD, payload: f });
             return true;
         }
         if (op === LIKE) {
             const opts = f[1].opts;
             if (opts.score &&
                 (typeof opts.score !== 'number' || opts.score < 0 || opts.score > 255)) {
-                def.errors.push({
-                    code: ERR_FILTER_INVALID_OPTS,
-                    payload: f,
-                });
+                def.errors.push({ code: ERR_FILTER_INVALID_OPTS, payload: f });
                 return true;
             }
         }
+        if (validateVal(def, f, isValidString)) {
+            return true;
+        }
     }
-    else if (propIsNumerical(prop) && op !== EQUAL && !isNumerical(op)) {
-        def.errors.push({
-            code: ERR_FILTER_OP_FIELD,
-            payload: f,
-        });
-        return true;
+    else if (propIsNumerical(prop)) {
+        if (op !== EQUAL && !isNumerical(op)) {
+            def.errors.push({ code: ERR_FILTER_OP_FIELD, payload: f });
+            return true;
+        }
+        if (validateVal(def, f, (v) => t == TIMESTAMP || typeof v === 'number')) {
+            return true;
+        }
     }
     else if (t === BOOLEAN && op !== EQUAL) {
-        def.errors.push({
-            code: ERR_FILTER_OP_FIELD,
-            payload: f,
-        });
+        def.errors.push({ code: ERR_FILTER_OP_FIELD, payload: f });
         return true;
     }
     return false;
@@ -193,6 +233,81 @@ export const includeLangDoesNotExist = (def, field) => {
         payload: field,
     });
 };
+export const validateLocale = (def, lang) => {
+    const schema = def.schema;
+    if (!(lang in schema.locales)) {
+        def.errors.push({
+            code: ERR_INVALID_LANG,
+            payload: lang,
+        });
+    }
+};
+export const validateSort = (def, field, orderInput) => {
+    let propDef = def.props[field];
+    if (orderInput && orderInput !== 'asc' && orderInput !== 'desc') {
+        def.errors.push({
+            code: ERR_SORT_ORDER,
+            payload: { order: orderInput, field },
+        });
+    }
+    const order = orderInput === 'asc' || orderInput === undefined ? 0 : 1;
+    // IF ! FIX
+    let lang = 0;
+    if (!propDef) {
+        let isText = false;
+        if (field.includes('.')) {
+            const path = field.split('.');
+            const x = path.slice(0, -1).join('.');
+            propDef = def.props[x];
+            if (propDef && propDef.typeIndex === TEXT) {
+                const k = path[path.length - 1];
+                lang = langCodesMap.get(k);
+                isText = true;
+            }
+        }
+        if (!isText) {
+            def.errors.push({
+                code: ERR_SORT_ENOENT,
+                payload: field,
+            });
+            return {
+                prop: EMPTY_ALIAS_PROP_DEF,
+                order,
+                lang: def.lang,
+            };
+        }
+    }
+    const type = propDef.typeIndex;
+    if (type === REFERENCES || type === REFERENCE) {
+        def.errors.push({
+            code: ERR_SORT_TYPE,
+            payload: propDef,
+        });
+    }
+    else if (type === TEXT) {
+        if (lang === 0) {
+            lang = def.lang ?? 0;
+            if (lang === 0) {
+                def.errors.push({
+                    code: ERR_SORT_LANG,
+                    payload: propDef,
+                });
+            }
+        }
+    }
+    // add propDef LANG
+    if ('id' in def.target || 'alias' in def.target) {
+        def.errors.push({
+            code: ERR_SORT_WRONG_TARGET,
+            payload: def,
+        });
+    }
+    return {
+        prop: propDef,
+        order,
+        lang,
+    };
+};
 export const validateAlias = (alias, path, def) => {
     const schema = def.schema;
     for (const k in alias) {
@@ -217,7 +332,10 @@ export const validateAlias = (alias, path, def) => {
     return { value: '', def: EMPTY_ALIAS_PROP_DEF };
 };
 export const validateId = (def, id) => {
-    if (typeof id != 'number' || id == 0 || id > MAX_ID) {
+    if (def.skipValidation) {
+        return id;
+    }
+    if (!isValidId(id)) {
         def.errors.push({
             code: ERR_TARGET_INVAL_ID,
             payload: id,
@@ -227,17 +345,18 @@ export const validateId = (def, id) => {
     return id;
 };
 export const validateIds = (def, ids) => {
+    const origIds = ids;
     if (!Array.isArray(ids) && !(ids instanceof Uint32Array)) {
         def.errors.push({
             code: ERR_TARGET_INVAL_IDS,
-            payload: ids,
+            payload: origIds,
         });
         return new Uint32Array([]);
     }
     if (ids.length > MAX_IDS_PER_QUERY) {
         def.errors.push({
             code: ERR_TARGET_EXCEED_MAX_IDS,
-            payload: ids,
+            payload: origIds,
         });
         return new Uint32Array([]);
     }
@@ -249,17 +368,20 @@ export const validateIds = (def, ids) => {
         catch (err) {
             def.errors.push({
                 code: ERR_TARGET_INVAL_IDS,
-                payload: ids,
+                payload: origIds,
             });
             return new Uint32Array([]);
         }
     }
-    // pretty heavy if it are a lot...
-    for (const id of ids) {
-        if (typeof id != 'number' || id == 0 || id > MAX_ID) {
+    if (def.skipValidation) {
+        return ids;
+    }
+    for (let i = 0; i < ids.length; i++) {
+        const id = ids[i];
+        if (typeof id !== 'number' || id == 0 || id > MAX_ID) {
             def.errors.push({
                 code: ERR_TARGET_INVAL_IDS,
-                payload: ids,
+                payload: origIds,
             });
             return new Uint32Array([]);
         }
@@ -268,9 +390,14 @@ export const validateIds = (def, ids) => {
 };
 export const handleErrors = (def) => {
     if (def.errors.length) {
-        let name = `Query\n`;
+        let name = picocolors.red(`QueryError[${displayTarget(def)}]\n`);
         for (const err of def.errors) {
-            name += `  ${messages[err.code](err.payload)}\n`;
+            try {
+                name += `  ${messages[err.code](err.payload)}\n`;
+            }
+            catch (e) {
+                name += `  Cannot parse error:${err.code}\n`;
+            }
         }
         const err = new Error(`Query\n`);
         err.stack = name;
@@ -286,6 +413,24 @@ export const EMPTY_ALIAS_PROP_DEF = {
     start: 0,
     path: ['ERROR_ALIAS'],
 };
+export const ERROR_STRING = {
+    prop: 1,
+    typeIndex: STRING,
+    __isPropDef: true,
+    separate: true,
+    len: 0,
+    start: 0,
+    path: ['ERROR_STRING'],
+};
+export const ERROR_VECTOR = {
+    prop: 1,
+    typeIndex: VECTOR,
+    __isPropDef: true,
+    separate: true,
+    len: 0,
+    start: 0,
+    path: ['ERROR_VECTOR'],
+};
 export const EMPTY_SCHEMA_DEF = {
     type: '_error',
     cnt: 0,
@@ -304,41 +449,19 @@ export const EMPTY_SCHEMA_DEF = {
     main: {},
     separate: [],
     tree: {},
-    hasStringProp: false,
-    stringPropsSize: 0,
-    stringPropsCurrent: Buffer.from([]),
-    stringProps: Buffer.from([]),
-    stringPropsLoop: [],
+    hasSeperateSort: false,
+    seperateSort: {
+        size: 0,
+        buffer: Buffer.from([]),
+        bufferTmp: Buffer.from([]),
+        props: [],
+    },
+    hasSeperateTextSort: false,
+    seperateTextSort: {
+        size: 0,
+        buffer: Buffer.from([]),
+        bufferTmp: Buffer.from([]),
+        props: [],
+    },
 };
-// export const checkMaxIdsPerQuery = (
-//   ids: (number | QueryByAliasObj)[],
-// ): void => {
-//   if (ids.length > MAX_IDS_PER_QUERY) {
-//     throw new Error(`The number of IDs cannot exceed ${MAX_IDS_PER_QUERY}.`)
-//   }
-// }
-// export const checkMaxBufferSize = (buf: Buffer): void => {
-//   if (buf.byteLength > MAX_BUFFER_SIZE) {
-//     throw new Error(
-//       `The buffer size exceeds the maximum threshold of ${MAX_BUFFER_SIZE} bytes.` +
-//         `Crrent size is ${buf.byteLength} bytes.`,
-//     )
-//   }
-// }
-// export const checkTotalBufferSize = (bufers: Buffer[]): void => {
-//   let totalSize = 0
-//   for (const buffer of bufers) {
-//     totalSize += buffer.byteLength
-//     if (totalSize > MAX_BUFFER_SIZE) {
-//       throw new Error(
-//         `The total buffer size exceeds the maximum threshold of ${MAX_BUFFER_SIZE} bytes.` +
-//           `Crrent size is ${totalSize} bytes.`,
-//       )
-//     }
-//   }
-// }
-// // ------------------------------
-// export const includeDoesNotExist = (def: QueryDef, field: string) => {
-//   throw new Error(`Incorrect include field provided to query "${field}")`)
-// }
 //# sourceMappingURL=validation.js.map

package/dist/src/index.d.ts

@@ -8,15 +8,17 @@ export { compress, decompress };
 export { ModifyCtx };
 export { DbClient, DbServer };
 export { xxHash64 } from './client/xxHash64.js';
+export { crc32 } from './client/crc32.js';
 export declare class BasedDb {
     #private;
     client: DbClient;
     server: DbServer;
     fileSystemPath: string;
     maxModifySize: number;
-    constructor(opt: {
+    constructor(opts: {
         path: string;
         maxModifySize?: number;
+        debug?: boolean;
     });
     create: DbClient['create'];
     copy: DbClient['copy'];

package/dist/src/index.js

@@ -2,19 +2,34 @@ import { compress, decompress } from './client/string.js';
 import { ModifyCtx } from './client/operations.js';
 import { DbServer } from './server/index.js';
 import { DbClient } from './client/index.js';
+import picocolors from 'picocolors';
 export * from './server/schema/typeDef.js';
 export * from './client/modify/modify.js';
 export { compress, decompress };
 export { ModifyCtx }; // TODO move this somewhere
 export { DbClient, DbServer };
 export { xxHash64 } from './client/xxHash64.js';
+export { crc32 } from './client/crc32.js';
 export class BasedDb {
     client;
     server;
     fileSystemPath;
     maxModifySize;
-    constructor(opt) {
-        this.#init(opt);
+    constructor(opts) {
+        this.#init(opts);
+        if (opts.debug) {
+            for (const key in this) {
+                const fn = this[key];
+                if (typeof fn === 'function') {
+                    // @ts-ignore
+                    this[key] = function () {
+                        const str = [`[${key}]`, ...arguments].join(' ');
+                        console.info(picocolors.dim(str));
+                        return fn.apply(this, arguments);
+                    };
+                }
+            }
+        }
     }
     #init({ path, maxModifySize }) {
         this.fileSystemPath = path;

package/dist/src/native.d.ts

@@ -1,4 +1,6 @@
 declare const _default: {
+    historyAppend(history: any, typeId: number, nodeId: number): any;
+    historyCreate(pathname: string, mainLen: number): any;
     workerCtxInit: () => void;
     markMerkleBlock: (buf: Buffer) => any;
     externalFromInt(address: BigInt): any;

package/dist/src/native.js

@@ -3,6 +3,12 @@ import db from '../../basedDbNative.cjs';
 var compressor = null;
 var decompressor = null;
 export default {
+    historyAppend(history, typeId, nodeId) {
+        return db.historyCreate(history, typeId, nodeId);
+    },
+    historyCreate(pathname, mainLen) {
+        return db.historyCreate(Buffer.from(pathname), mainLen + 16 - (mainLen % 16));
+    },
     workerCtxInit: () => {
         return db.workerCtxInit();
     },

package/dist/src/server/index.d.ts

@@ -1,4 +1,4 @@
-import { StrictSchema } from '@based/schema';
+import { LangName, StrictSchema } from '@based/schema';
 import { SchemaTypeDef } from './schema/types.js';
 import { createTree } from './csmt/index.js';
 import { Worker, MessagePort } from 'node:worker_threads';
@@ -67,19 +67,21 @@ export declare class DbServer {
     sortIndexes: {
         [type: number]: {
             [field: number]: {
-                [start: number]: SortIndex;
+                [start: number]: {
+                    [lang: number]: SortIndex;
+                };
             };
         };
     };
     cleanupTimer: NodeJS.Timeout;
     cleanup(): void;
-    createSortIndex(type: string, field: string): SortIndex;
-    destroySortIndex(type: string, field: string): any;
-    getSortIndex(typeId: number, field: number, start: number): SortIndex;
+    createSortIndex(type: string, field: string, lang?: LangName): SortIndex;
+    destroySortIndex(type: string, field: string, lang?: LangName): any;
+    getSortIndex(typeId: number, field: number, start: number, lang: number): SortIndex;
     migrateSchema(schema: StrictSchema, transform?: Record<string, (node: Record<string, any>) => Record<string, any> | [string, Record<string, any>]>): Promise<StrictSchema & {
         lastId: number;
     }>;
-    createSortIndexBuffer(typeId: number, field: number, start: number): SortIndex;
+    createSortIndexBuffer(typeId: number, field: number, start: number, lang: number): SortIndex;
     updateMerkleTree(): void;
     putSchema(strictSchema: StrictSchema, fromStart?: boolean, transformFns?: TransformFns): (StrictSchema & {
         lastId: number;