@based/db 0.0.66 → 0.0.67

package/dist/src/client/query/BasedDbQuery.js

@@ -11,7 +11,7 @@ import { langCodesMap } from '@based/schema';
 import { convertFilter } from './filter/convertFilter.js';
 import { validateLocale, validateRange } from './validation.js';
 import { DEF_RANGE_PROP_LIMIT } from './thresholds.js';
-import { wait } from '@saulx/utils';
+import { AggregateType } from './aggregates/types.js';
 import { displayTarget } from './display.js';
 import picocolors from 'picocolors';
 export class QueryBranch {
@@ -152,7 +152,7 @@ export class QueryBranch {
         }
         else {
             const p = field.split('.');
-            addAggregate(2 /* AggregateType.COUNT */, this.def, p);
+            addAggregate(AggregateType.COUNT, this.def, p);
         }
         // @ts-ignore
         return this;
@@ -168,7 +168,7 @@ export class QueryBranch {
             });
         }
         else {
-            addAggregate(1 /* AggregateType.SUM */, this.def, fields);
+            addAggregate(AggregateType.SUM, this.def, fields);
         }
         // @ts-ignore
         return this;
@@ -184,7 +184,87 @@ export class QueryBranch {
             });
         }
         else {
-            addAggregate(3 /* AggregateType.CARDINALITY */, this.def, fields);
+            addAggregate(AggregateType.CARDINALITY, this.def, fields);
+        }
+        // @ts-ignore
+        return this;
+    }
+    stddev(...fields) {
+        if (fields.length === 0) {
+            throw new Error('Empty standard deviation function called');
+        }
+        if (this.queryCommands) {
+            this.queryCommands.push({
+                method: 'stddev',
+                args: fields,
+            });
+        }
+        else {
+            addAggregate(AggregateType.STDDEV, this.def, fields);
+        }
+        // @ts-ignore
+        return this;
+    }
+    var(...fields) {
+        if (fields.length === 0) {
+            throw new Error('Empty variance function called');
+        }
+        if (this.queryCommands) {
+            this.queryCommands.push({
+                method: 'var',
+                args: fields,
+            });
+        }
+        else {
+            addAggregate(AggregateType.VARIANCE, this.def, fields);
+        }
+        // @ts-ignore
+        return this;
+    }
+    avg(...fields) {
+        if (fields.length === 0) {
+            throw new Error('Empty average function called');
+        }
+        if (this.queryCommands) {
+            this.queryCommands.push({
+                method: 'avg',
+                args: fields,
+            });
+        }
+        else {
+            addAggregate(AggregateType.AVERAGE, this.def, fields);
+        }
+        // @ts-ignore
+        return this;
+    }
+    max(...fields) {
+        if (fields.length === 0) {
+            throw new Error('Empty maximum function called');
+        }
+        if (this.queryCommands) {
+            this.queryCommands.push({
+                method: 'max',
+                args: fields,
+            });
+        }
+        else {
+            addAggregate(AggregateType.MAX, this.def, fields);
+        }
+        // @ts-ignore
+        return this;
+    }
+    min(...fields) {
+        if (fields.length === 0) {
+            throw new Error('Empty minimum function called');
+        }
+        if (this.queryCommands) {
+            this.queryCommands.push({
+                method: 'min',
+                args: fields,
+            });
+        }
+        else {
+            addAggregate(AggregateType.MIN, this.def, fields);
         }
         // @ts-ignore
         return this;
@@ -354,13 +434,19 @@ export class BasedDbQuery extends QueryBranch {
         const res = await this.db.hooks.getQueryBuf(buf);
         if (res.byteLength === 1) {
             if (res[0] === 0) {
-                this.reset();
-                this.db.emit('info', 'query get schema mismatch - awaiting new schema (max 15s)');
-                const ok = await Promise.race([wait(15e3), this.db.once('schema')]);
-                if (!ok) {
-                    reject(new Error('schema mismath'));
+                if (this.def && this.def.schemaChecksum === this.db.schema?.hash) {
+                    // my schema did not change since last time, wait for the schema to change
+                    this.reset();
+                    this.db.emit('info', 'query get schema mismatch - awaiting new schema');
+                    await this.db.once('schema');
+                    return this.#getInternal(resolve, reject);
+                }
+                else {
+                    // its changed so lets send again
+                    this.db.emit('info', 'query get schema mismatch - got the same already');
+                    this.reset();
+                    return this.#getInternal(resolve, reject);
                 }
-                return this.#getInternal(resolve, reject);
             }
             else {
                 reject(new Error('unexpected error'));
@@ -382,16 +468,28 @@ export class BasedDbQuery extends QueryBranch {
     register() {
         registerQuery(this);
     }
-    locale(locale) {
+    locale(locale, fallBack) {
         if (this.queryCommands) {
-            this.queryCommands.push({
+            this.queryCommands.unshift({
                 method: 'locale',
                 args: [locale],
             });
         }
         else {
+            if (fallBack === undefined) {
+                // Uses fallback from schema if available
+                const localeDescriptor = this.def.schema.locales[locale];
+                fallBack =
+                    typeof localeDescriptor === 'object'
+                        ? localeDescriptor.fallback || false
+                        : false;
+            }
             validateLocale(this.def, locale);
-            this.def.lang = langCodesMap.get(locale) ?? 0;
+            const fallBackCode = fallBack === false ? [] : [langCodesMap.get(fallBack)];
+            this.def.lang = {
+                lang: langCodesMap.get(locale) ?? 0,
+                fallback: fallBackCode,
+            };
         }
         return this;
     }
@@ -401,7 +499,6 @@ export class BasedDbQuery extends QueryBranch {
                 onData(res);
             }
             catch (err) {
-                // const t = displayTarget(this.def)
                 const def = this.def;
                 let name = picocolors.red(`QueryError[${displayTarget(def)}]\n`);
                 name += `  Error executing onData handler in subscription\n`;

package/dist/src/client/query/aggregates/aggregation.js

@@ -1,7 +1,9 @@
 import { writeUint16 } from '@saulx/utils';
 import { QueryDefType } from '../types.js';
+import { AggregateType } from './types.js';
 import { UINT32 } from '@based/schema/def';
 import { aggregationFieldDoesNotExist } from '../validation.js';
+import { aggregateTypeMap } from '../aggregates/types.js';
 export const aggregateToBuffer = (aggregates) => {
     const aggBuffer = new Uint8Array(aggregates.size);
     let i = 0;
@@ -21,7 +23,9 @@ export const aggregateToBuffer = (aggregates) => {
         aggBuffer[i] = 0 /* GroupBy.NONE */;
         i += 1;
     }
-    writeUint16(aggBuffer, aggregates.totalResultsPos, i);
+    writeUint16(aggBuffer, aggregates.totalResultsSize, i);
+    i += 2;
+    writeUint16(aggBuffer, aggregates.totalAccumulatorSize, i);
     i += 2;
     for (const [prop, aggregatesArray] of aggregates.aggregates.entries()) {
         aggBuffer[i] = prop;
@@ -39,6 +43,8 @@ export const aggregateToBuffer = (aggregates) => {
             i += 2;
             writeUint16(aggBuffer, agg.resultPos, i);
             i += 2;
+            writeUint16(aggBuffer, agg.accumulatorPos, i);
+            i += 2;
             size += i - startI;
         }
         writeUint16(aggBuffer, size, sizeIndex);
@@ -48,13 +54,13 @@ export const aggregateToBuffer = (aggregates) => {
 const ensureAggregate = (def) => {
     if (!def.aggregate) {
         def.aggregate = {
-            size: 3,
+            size: 5,
             aggregates: new Map(),
-            totalResultsPos: 0,
+            totalResultsSize: 0,
+            totalAccumulatorSize: 0,
         };
     }
 };
-// Group by is great for normal stuff as well (do later)
 export const groupBy = (def, field) => {
     const fieldDef = def.schema.props[field];
     if (!fieldDef) {
@@ -74,7 +80,7 @@ export const addAggregate = (type, def, fields) => {
             addAggregate(type, def, field);
         }
         else {
-            const fieldDef = type === 2 /* AggregateType.COUNT */
+            const fieldDef = type === AggregateType.COUNT
                 ? {
                     prop: 255,
                     path: [field],
@@ -98,13 +104,20 @@ export const addAggregate = (type, def, fields) => {
             aggregateField.push({
                 propDef: fieldDef,
                 type,
-                resultPos: def.aggregate.totalResultsPos,
+                resultPos: def.aggregate.totalResultsSize,
+                accumulatorPos: def.aggregate.totalAccumulatorSize,
             });
-            // IF FLOAT // NUMBER ETC USE 8!
-            // do this better
-            def.aggregate.totalResultsPos += 4;
+            const specificSizes = aggregateTypeMap.get(type);
+            if (specificSizes) {
+                def.aggregate.totalResultsSize += specificSizes.resultsSize;
+                def.aggregate.totalAccumulatorSize += specificSizes.accumulatorSize;
+            }
+            else {
+                def.aggregate.totalResultsSize += 8;
+                def.aggregate.totalAccumulatorSize += 8;
+            }
             // needs to add an extra field WRITE TO
-            def.aggregate.size += 6;
+            def.aggregate.size += 8;
         }
     }
 };
@@ -128,7 +141,7 @@ export const isRootCountOnly = (def, filterSize) => {
     if (aggs.length !== 1) {
         return false;
     }
-    if (aggs[0].type !== 2 /* AggregateType.COUNT */) {
+    if (aggs[0].type !== AggregateType.COUNT) {
         return false;
     }
     if (def.filter && def.filter.size > 0) {

package/dist/src/client/query/aggregates/types.d.ts

@@ -1,8 +1,27 @@
-export declare const enum AggregateType {
+export declare enum AggregateType {
     SUM = 1,
     COUNT = 2,
-    CARDINALITY = 3
+    CARDINALITY = 3,
+    STDDEV = 4,
+    AVERAGE = 5,
+    VARIANCE = 6,
+    MAX = 7,
+    MIN = 8
 }
+export declare const enum AccumulatorSize {
+    SUM = 8,
+    COUNT = 4,
+    CARDINALITY = 254,// TODO: accordinly to sparse or dense modes
+    STDDEV = 24,// count (u64) + sum (f64) + sum_sq (f64) = 8 + 8 + 8 = 24
+    AVERAGE = 16,// count (u64) + sum (f64) = 16
+    VARIANCE = 24,// count (u64) + sum (f64) + sum_sq (f64) = 8 + 8 + 8 = 24
+    MAX = 8,
+    MIN = 8
+}
+export declare const aggregateTypeMap: Map<AggregateType, {
+    resultsSize: number;
+    accumulatorSize: number;
+}>;
 export declare const enum GroupBy {
     NONE = 0,
     HAS_GROUP = 255

package/dist/src/client/query/aggregates/types.js

@@ -1,2 +1,35 @@
-export {};
+export var AggregateType;
+(function (AggregateType) {
+    AggregateType[AggregateType["SUM"] = 1] = "SUM";
+    AggregateType[AggregateType["COUNT"] = 2] = "COUNT";
+    AggregateType[AggregateType["CARDINALITY"] = 3] = "CARDINALITY";
+    AggregateType[AggregateType["STDDEV"] = 4] = "STDDEV";
+    AggregateType[AggregateType["AVERAGE"] = 5] = "AVERAGE";
+    AggregateType[AggregateType["VARIANCE"] = 6] = "VARIANCE";
+    AggregateType[AggregateType["MAX"] = 7] = "MAX";
+    AggregateType[AggregateType["MIN"] = 8] = "MIN";
+})(AggregateType || (AggregateType = {}));
+export const aggregateTypeMap = new Map([
+    [
+        AggregateType.CARDINALITY,
+        { resultsSize: 4, accumulatorSize: 254 /* AccumulatorSize.CARDINALITY */ },
+    ],
+    [
+        AggregateType.COUNT,
+        { resultsSize: 4, accumulatorSize: 4 /* AccumulatorSize.COUNT */ },
+    ],
+    [
+        AggregateType.STDDEV,
+        { resultsSize: 8, accumulatorSize: 24 /* AccumulatorSize.STDDEV */ },
+    ],
+    [
+        AggregateType.AVERAGE,
+        { resultsSize: 8, accumulatorSize: 16 /* AccumulatorSize.AVERAGE */ },
+    ],
+    [
+        AggregateType.VARIANCE,
+        { resultsSize: 8, accumulatorSize: 24 /* AccumulatorSize.VARIANCE */ },
+    ],
+    // Othe types like MAX, MIN, SUM fall in the else case in aggregation.ts 8/8
+]);
 //# sourceMappingURL=types.js.map

package/dist/src/client/query/display.js

@@ -1,6 +1,7 @@
 import picocolors from 'picocolors';
 import { BINARY, CARDINALITY, NUMBER, REFERENCE, REFERENCES, STRING, TEXT, TIMESTAMP, } from '@based/schema/def';
 import { ENCODER } from '@saulx/utils';
+import { AggregateType } from './aggregates/types.js';
 const decimals = (v) => ~~(v * 100) / 100;
 const sizeCalc = (size) => {
     if (size > 1e6) {
@@ -158,7 +159,10 @@ const inspectObject = (object, q, path, level, isLast, isFirst, isObject, depth)
             if (typeof v === 'number') {
                 if (q.aggregate) {
                     str += printNumber(v);
-                    str += picocolors.italic(picocolors.dim(` ${key.indexOf('count') >= 0 ? ' count' : ' sum'}`));
+                    // TBD: replace comptime const enum and reverse map it
+                    const [[__, akv], _] = q.aggregate.aggregates;
+                    const aggType = akv[0].type;
+                    str += picocolors.italic(picocolors.dim(` ${AggregateType[aggType].toLowerCase()}`));
                     str += ',\n';
                 }
                 else {
@@ -219,7 +223,9 @@ const inspectObject = (object, q, path, level, isLast, isFirst, isObject, depth)
                 if (typeof v === 'number') {
                     if (q.aggregate) {
                         str += printNumber(v);
-                        str += picocolors.italic(picocolors.dim(` ${key.indexOf('count') >= 0 ? ' count' : ' sum'}`));
+                        const [[__, akv], _] = q.aggregate.aggregates;
+                        const aggType = akv[0].type;
+                        str += picocolors.italic(picocolors.dim(` ${AggregateType[aggType].toLowerCase()}`));
                     }
                     else {
                         str += printNumber(v);

package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts

@@ -1,5 +1,4 @@
 import { PropDef, PropDefEdge } from '@based/schema/def';
 import { FilterCtx } from './types.js';
-import { LangCode } from '@based/schema';
-import { FilterCondition } from '../types.js';
-export declare const createVariableFilterBuffer: (value: any, prop: PropDef | PropDefEdge, ctx: FilterCtx, lang: LangCode) => FilterCondition;
+import { FilterCondition, QueryDef } from '../types.js';
+export declare const createVariableFilterBuffer: (value: any, prop: PropDef | PropDefEdge, ctx: FilterCtx, lang: QueryDef["lang"]) => FilterCondition;

package/dist/src/client/query/filter/createVariableFilterBuffer.js

@@ -33,9 +33,15 @@ const parseValue = (value, prop, ctx, lang) => {
             val = ENCODER.encode(val.normalize('NFKD'));
         }
         if (prop.typeIndex === TEXT) {
-            const tmp = new Uint8Array(val.byteLength + 1);
+            // 1 + size
+            const fallbacksSize = lang.lang === 0 ? 0 : lang.fallback.length;
+            const tmp = new Uint8Array(val.byteLength + 2 + fallbacksSize);
             tmp.set(val);
-            tmp[tmp.byteLength - 1] = lang;
+            tmp[tmp.byteLength - 1] = fallbacksSize;
+            tmp[tmp.byteLength - 2] = lang.lang;
+            for (let i = 0; i < fallbacksSize; i++) {
+                tmp[tmp.byteLength - 2 - fallbacksSize + i] = lang.fallback[i];
+            }
             val = tmp;
         }
     }
@@ -90,9 +96,10 @@ export const createVariableFilterBuffer = (value, prop, ctx, lang) => {
                 prop.typeIndex !== ALIAS &&
                 prop.typeIndex !== VECTOR) {
                 if (prop.typeIndex === TEXT) {
-                    const crc = crc32(val.slice(0, -1));
-                    const len = val.byteLength - 1;
-                    const v = new Uint8Array(9);
+                    const fbLen = 2 + val[val.byteLength - 1];
+                    const crc = crc32(val.slice(0, -fbLen));
+                    const len = val.byteLength - fbLen;
+                    const v = new Uint8Array(8 + fbLen);
                     v[0] = crc;
                     v[1] = crc >>> 8;
                     v[2] = crc >>> 16;
@@ -101,11 +108,17 @@ export const createVariableFilterBuffer = (value, prop, ctx, lang) => {
                     v[5] = len >>> 8;
                     v[6] = len >>> 16;
                     v[7] = len >>> 24;
-                    v[8] = val[val.length - 1];
+                    for (let i = 0; i < fbLen; i++) {
+                        v[v.byteLength - (i + 1)] = val[val.byteLength - (i + 1)];
+                    }
                     parsedCondition = writeVarFilter(mode, v, ctx, prop, 0, 0);
                 }
                 else {
-                    parsedCondition = createFixedFilterBuffer(prop, 8, { operation: EQUAL_CRC32, type: ctx.type, opts: ctx.opts }, val, false);
+                    parsedCondition = createFixedFilterBuffer(prop, 8, {
+                        operation: EQUAL_CRC32,
+                        type: ctx.type,
+                        opts: ctx.opts,
+                    }, val, false);
                 }
             }
             else {

package/dist/src/client/query/filter/filter.js

@@ -22,7 +22,11 @@ const referencesFilter = (db, filter, schema, conditions, def) => {
                     if (edgeDef) {
                         conditions.edges ??= new Map();
                         size +=
-                            3 + primitiveFilter(def, edgeDef, filter, conditions, def.lang);
+                            3 +
+                                primitiveFilter(def, edgeDef, filter, conditions, {
+                                    lang: def.lang.lang,
+                                    fallback: [],
+                                });
                     }
                 }
             }
@@ -71,12 +75,18 @@ export const filterRaw = (db, filter, schema, conditions, def) => {
                     filterInvalidLang(def, field);
                     return 0;
                 }
-                return primitiveFilter(def, fieldDef, filter, conditions, code);
+                return primitiveFilter(def, fieldDef, filter, conditions, {
+                    lang: code,
+                    fallback: [],
+                });
             }
         }
         if (field === 'id') {
             fieldDef = ID_FIELD_DEF;
-            return primitiveFilter(def, fieldDef, filter, conditions, def.lang);
+            return primitiveFilter(def, fieldDef, filter, conditions, {
+                lang: def.lang.lang,
+                fallback: [], // only fallbacks for this
+            });
         }
         return referencesFilter(db, filter, schema, conditions, def);
     }

package/dist/src/client/query/filter/primitiveFilter.d.ts

@@ -1,5 +1,4 @@
 import { PropDef, PropDefEdge } from '@based/schema/def';
 import { QueryDef, QueryDefFilter } from '../types.js';
 import { Filter } from './types.js';
-import { LangCode } from '@based/schema';
-export declare const primitiveFilter: (def: QueryDef, prop: PropDef | PropDefEdge, filter: Filter, conditions: QueryDefFilter, lang: LangCode) => number;
+export declare const primitiveFilter: (def: QueryDef, prop: PropDef | PropDefEdge, filter: Filter, conditions: QueryDefFilter, lang: QueryDef["lang"]) => number;

package/dist/src/client/query/include/props.js

@@ -69,9 +69,25 @@ export const includeProp = (def, prop) => {
     }
     if (prop.typeIndex === TEXT) {
         if (!def.include.langTextFields.has(prop.prop)) {
-            def.include.langTextFields.set(prop.prop, { def: prop, codes: new Set() });
+            def.include.langTextFields.set(prop.prop, {
+                def: prop,
+                codes: new Set(),
+                fallBacks: [],
+            });
+        }
+        const langs = def.include.langTextFields.get(prop.prop);
+        if (def.lang.fallback.length > 0) {
+            for (const fallback of def.lang.fallback) {
+                if (!langs.fallBacks.includes(fallback)) {
+                    langs.fallBacks.push(fallback);
+                }
+            }
+        }
+        const langCode = def.lang.lang ?? 0;
+        langs.codes.add(langCode);
+        if (langCode === 0 || langs.codes.size > 1) {
+            langs.fallBacks = [];
         }
-        def.include.langTextFields.get(prop.prop).codes.add(def.lang ?? 0);
     }
     else {
         if (prop.separate) {

package/dist/src/client/query/include/toBuffer.js

@@ -56,21 +56,29 @@ export const includeToBuffer = (db, def) => {
         }
     }
     if (def.include.langTextFields.size) {
-        for (const [prop, { codes, def: propDef },] of def.include.langTextFields.entries()) {
+        for (const [prop, { codes, def: propDef, fallBacks },] of def.include.langTextFields.entries()) {
             def.include.propsRead[prop] = 0;
             if (codes.has(0)) {
-                const b = new Uint8Array(3);
+                const b = new Uint8Array(4);
                 b[0] = prop;
                 b[1] = propDef.typeIndex;
                 b[2] = 0;
+                b[3] = 0;
                 result.push(b);
             }
             else {
                 for (const code of codes) {
-                    const b = new Uint8Array(3);
+                    const fallBackSize = fallBacks.length;
+                    const b = new Uint8Array(4 + fallBackSize);
                     b[0] = prop;
                     b[1] = propDef.typeIndex;
                     b[2] = code;
+                    b[3] = fallBackSize;
+                    let i = 0;
+                    for (const fallback of fallBacks) {
+                        b[i + 4] = fallback;
+                        i++;
+                    }
                     result.push(b);
                 }
             }

package/dist/src/client/query/include/walk.js

@@ -55,7 +55,11 @@ export const walkDefs = (db, def, f) => {
                     return;
                 }
                 if (!def.include.langTextFields.has(t.prop)) {
-                    def.include.langTextFields.set(t.prop, { def: t, codes: new Set() });
+                    def.include.langTextFields.set(t.prop, {
+                        def: t,
+                        codes: new Set(),
+                        fallBacks: [],
+                    });
                 }
                 def.include.langTextFields.get(t.prop).codes.add(langCode);
                 return;

package/dist/src/client/query/queryDef.js

@@ -8,7 +8,10 @@ const createEmptySharedDef = (skipValidation) => {
         skipValidation,
         filter: { conditions: new Map(), size: 0 },
         range: { offset: 0, limit: 0 },
-        lang: langCodesMap.get('none'),
+        lang: {
+            lang: langCodesMap.get('none'),
+            fallback: [],
+        },
         include: {
             langTextFields: new Map(),
             stringFields: new Set(),