@based/db 0.0.35 → 0.0.37

package/dist/lib/darwin_aarch64/include/selva/fields.h

@@ -4,6 +4,7 @@
  */
 #pragma once
 
+#include <stdint.h>
 #include <sys/types.h>
 #include "selva/_export.h"
 #ifdef __zig
@@ -81,6 +82,16 @@ struct SelvaFieldsPointer {
     size_t len;
 };
 
+/**
+ * Precalculated empty strings for text translations.
+ * The size of the string is 8 for better alignment but they are only
+ * filled upto 6 bytes.
+ */
+SELVA_EXPORT
+extern const uint8_t selva_fields_text_tl_empty[_selva_lang_last][8];
+
+#define SELVA_FIELDS_TEXT_TL_EMPTY_LEN 6
+
 #if __has_c_attribute(unsequenced)
 [[unsequenced]]
 #else

package/dist/lib/darwin_aarch64/include/selva/types.h

@@ -45,6 +45,8 @@ struct EdgeFieldConstraint {
          * Skip saving this field while dumping.
          * If the field is of type SELVA_FIELD_TYPE_REFERENCES it's saved
          * regardless of this flag to preserve the original order of references.
+         * However, the meta is only save from one side, i.e. the side that's
+         * not skipped.
          */
         EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80,
     } __packed flags;

package/dist/lib/darwin_aarch64/include/selva_lang_code.h

@@ -152,6 +152,7 @@ enum selva_lang_code {
     selva_lang_yi,
     selva_lang_yo,
     selva_lang_zu,
+    _selva_lang_last,
 } __packed;
 
 #endif /* SELVA_LANG_CODE_H */

package/dist/lib/linux_aarch64/include/selva/fields.h

@@ -4,6 +4,7 @@
  */
 #pragma once
 
+#include <stdint.h>
 #include <sys/types.h>
 #include "selva/_export.h"
 #ifdef __zig
@@ -81,6 +82,16 @@ struct SelvaFieldsPointer {
     size_t len;
 };
 
+/**
+ * Precalculated empty strings for text translations.
+ * The size of the string is 8 for better alignment but they are only
+ * filled upto 6 bytes.
+ */
+SELVA_EXPORT
+extern const uint8_t selva_fields_text_tl_empty[_selva_lang_last][8];
+
+#define SELVA_FIELDS_TEXT_TL_EMPTY_LEN 6
+
 #if __has_c_attribute(unsequenced)
 [[unsequenced]]
 #else

package/dist/lib/linux_aarch64/include/selva/types.h

@@ -45,6 +45,8 @@ struct EdgeFieldConstraint {
          * Skip saving this field while dumping.
          * If the field is of type SELVA_FIELD_TYPE_REFERENCES it's saved
          * regardless of this flag to preserve the original order of references.
+         * However, the meta is only save from one side, i.e. the side that's
+         * not skipped.
          */
         EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80,
     } __packed flags;

package/dist/lib/linux_aarch64/include/selva_lang_code.h

@@ -152,6 +152,7 @@ enum selva_lang_code {
     selva_lang_yi,
     selva_lang_yo,
     selva_lang_zu,
+    _selva_lang_last,
 } __packed;
 
 #endif /* SELVA_LANG_CODE_H */

package/dist/lib/linux_x86_64/include/selva/fields.h

@@ -4,6 +4,7 @@
  */
 #pragma once
 
+#include <stdint.h>
 #include <sys/types.h>
 #include "selva/_export.h"
 #ifdef __zig
@@ -81,6 +82,16 @@ struct SelvaFieldsPointer {
     size_t len;
 };
 
+/**
+ * Precalculated empty strings for text translations.
+ * The size of the string is 8 for better alignment but they are only
+ * filled upto 6 bytes.
+ */
+SELVA_EXPORT
+extern const uint8_t selva_fields_text_tl_empty[_selva_lang_last][8];
+
+#define SELVA_FIELDS_TEXT_TL_EMPTY_LEN 6
+
 #if __has_c_attribute(unsequenced)
 [[unsequenced]]
 #else

package/dist/lib/linux_x86_64/include/selva/types.h

@@ -45,6 +45,8 @@ struct EdgeFieldConstraint {
          * Skip saving this field while dumping.
          * If the field is of type SELVA_FIELD_TYPE_REFERENCES it's saved
          * regardless of this flag to preserve the original order of references.
+         * However, the meta is only save from one side, i.e. the side that's
+         * not skipped.
          */
         EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80,
     } __packed flags;

package/dist/lib/linux_x86_64/include/selva_lang_code.h

@@ -152,6 +152,7 @@ enum selva_lang_code {
     selva_lang_yi,
     selva_lang_yo,
     selva_lang_zu,
+    _selva_lang_last,
 } __packed;
 
 #endif /* SELVA_LANG_CODE_H */

package/dist/src/client/index.d.ts

@@ -33,7 +33,6 @@ export declare class DbClient {
     flushIsReady: Promise<void>;
     hooks: DbClientHooks;
     schema: DbClientSchema;
-    schemaIsSetValue: boolean;
     schemaTypesParsed: Record<string, SchemaTypeDef>;
     schemaTypesParsedById: Record<number, SchemaTypeDef>;
     writeTime: number;
@@ -45,6 +44,8 @@ export declare class DbClient {
         p: Promise<number | ModifyRes>;
     }>;
     schemaChecksum: number;
+    schemaProcessing: number;
+    schemaPromise: Promise<DbServer['schema']>;
     setSchema(schema: Schema, fromStart?: boolean, transformFns?: TransformFns): Promise<StrictSchema>;
     putLocalSchema(schema: any): DbClientSchema;
     create(type: string, obj?: CreateObj, opts?: ModifyOpts): ModifyRes;

package/dist/src/client/index.js

@@ -7,10 +7,10 @@ import { ModifyState } from './modify/ModifyRes.js';
 import { upsert } from './modify/upsert.js';
 import { update } from './modify/update.js';
 import { deleteFn } from './modify/delete.js';
-import { deepEqual } from '@saulx/utils';
+import { wait } from '@saulx/utils';
 import { hash } from '@saulx/hash';
 import { expire } from './modify/expire.js';
-import { debugMode } from '../utils.js';
+import { debugMode, schemaLooseEqual } from '../utils.js';
 const makeFlushIsReady = (dbClient) => {
     dbClient.flushIsReady = new Promise((resolve) => {
         dbClient.flushReady = () => {
@@ -34,11 +34,7 @@ export class DbClient {
     flushReady;
     flushIsReady;
     hooks;
-    schema = {
-        lastId: 1, // we reserve one for root props
-        types: {},
-    };
-    schemaIsSetValue = false;
+    schema;
     schemaTypesParsed = {};
     schemaTypesParsedById = {};
     // modify
@@ -48,29 +44,31 @@ export class DbClient {
     maxModifySize;
     upserting = new Map();
     schemaChecksum;
+    schemaProcessing;
+    schemaPromise;
     async setSchema(schema, fromStart, transformFns) {
-        this.schemaIsSetValue = true;
-        const checksum = hash(schema);
-        if (checksum === this.schemaChecksum) {
+        const strictSchema = fromStart ? schema : parse(schema).schema;
+        // this one excludes all the ids
+        if (schemaLooseEqual(strictSchema, this.schema)) {
             return this.schema;
         }
-        const strictSchema = fromStart ? schema : parse(schema).schema;
-        const remoteSchema = await this.hooks.setSchema(strictSchema, fromStart, transformFns);
-        this.schemaChecksum = checksum;
+        const checksum = hash(strictSchema);
+        if (checksum !== this.schemaProcessing) {
+            this.schemaProcessing = checksum;
+            this.schemaPromise = this.hooks.setSchema(strictSchema, fromStart, transformFns);
+        }
+        const remoteSchema = await this.schemaPromise;
+        this.schemaProcessing = null;
+        this.schemaPromise = null;
         return this.putLocalSchema(remoteSchema);
     }
     putLocalSchema(schema) {
-        this.schemaIsSetValue = true;
-        if (deepEqual(this.schema, schema)) {
+        const checksum = hash(schema);
+        if (this.schemaChecksum === checksum) {
             return this.schema;
         }
+        this.schemaChecksum = checksum;
         this.schema = schema;
-        for (const field in this.schema.types) {
-            if (!('id' in this.schema.types[field])) {
-                this.schema.lastId++;
-                this.schema.types[field].id = this.schema.lastId;
-            }
-        }
         updateTypeDefs(this.schema, this.schemaTypesParsed, this.schemaTypesParsedById);
         // Adds bidrectional refs on defs
         schemaToSelvaBuffer(this.schemaTypesParsed);
@@ -199,18 +197,17 @@ export class DbClient {
         await this.flushIsReady;
         return;
     }
-    schemaIsSet() {
-        return new Promise((resolve) => {
-            if (this.schemaIsSetValue) {
-                resolve(true);
-            }
-            else {
-                setTimeout(() => {
-                    // TODO use subscription when its done
-                    resolve(this.schemaIsSet());
-                }, 12);
-            }
-        });
+    async schemaIsSet() {
+        if (this.schema) {
+            return true;
+        }
+        if (this.schemaPromise) {
+            await this.schemaPromise;
+        }
+        else {
+            await wait(12);
+        }
+        return this.schemaIsSet();
     }
     listeners;
     on(event, cb) {

package/dist/src/client/modify/fixed.js

@@ -96,6 +96,7 @@ map[TIMESTAMP] = (ctx, val, def) => {
     }
     const view = new DataView(ctx.buf.buffer, ctx.buf.byteOffset + ctx.len, 8);
     ctx.len += 8;
+    // Todo use new utils and store as uint64
     view.setFloat64(0, parsedValue, true);
     const ts = view.getFloat64(0);
 };

package/dist/src/client/query/BasedDbQuery.d.ts

@@ -17,7 +17,8 @@ export declare class QueryBranch<T> {
     filterBatch(f: FilterAst): this;
     search(query: string, ...fields: Search[]): T;
     search(query: ArrayBufferView, field: string, opts?: Omit<FilterOpts, 'lowerCase'>): T;
-    count(): T;
+    groupBy(field: string): T;
+    sum(...fields: (string | string[])[]): T;
     or(fn: FilterBranchFn): T;
     or(field: string, operator?: Operator | boolean, value?: any, opts?: FilterOpts): T;
     range(start: number, end?: number): T;

package/dist/src/client/query/BasedDbQuery.js

@@ -1,4 +1,4 @@
-import { createQueryDef, QueryDefType, filter, sort, defToBuffer, filterOr, isAlias, includeField, includeFields, count, } from './query.js';
+import { createQueryDef, QueryDefType, filter, sort, defToBuffer, filterOr, isAlias, includeField, includeFields, addAggregate, groupBy, } from './query.js';
 import { BasedQueryResponse } from './BasedIterable.js';
 import { createOrGetEdgeRefQueryDef, createOrGetRefQueryDef, } from './include/utils.js';
 import { FilterBranch } from './filter/FilterBranch.js';
@@ -94,8 +94,15 @@ export class QueryBranch {
         // @ts-ignore
         return this;
     }
-    count() {
-        count(this.def);
+    groupBy(field) {
+        groupBy(this.def, field);
+        // only works with aggregates for now
+        // @ts-ignore
+        return this;
+    }
+    // x
+    sum(...fields) {
+        addAggregate(1 /* AggregateType.SUM */, this.def, fields);
         // @ts-ignore
         return this;
     }
@@ -213,7 +220,7 @@ export class BasedDbQuery extends QueryBranch {
                 }
             }
         }
-        if (!db.schemaIsSetValue) {
+        if (!db.schema) {
             throw new Error('Query: No schema yet - use await db.schemaIsSet()');
         }
         const def = createQueryDef(db, QueryDefType.Root, target, skipValidation);

package/dist/src/client/query/BasedIterable.d.ts

@@ -1,5 +1,6 @@
 import { inspect } from 'node:util';
 import { QueryDef } from './types.js';
+import { Item } from './query.js';
 import { size, time, inspectData } from './display.js';
 export { time, size, inspectData };
 export declare class BasedQueryResponse {
@@ -14,7 +15,7 @@ export declare class BasedQueryResponse {
     [inspect.custom](depth: number): string;
     debug(): this;
     node(index?: number): any;
-    [Symbol.iterator](): Generator<Partial<import("./query.js").Item>, void, unknown>;
+    [Symbol.iterator](): Generator<Item, void, unknown>;
     inspect(depth?: number, raw?: boolean): this;
     forEach(fn: (item: any, key: number) => void): void;
     map(fn: (item: any, key: number) => any): any[];

package/dist/src/client/query/BasedIterable.js

@@ -65,16 +65,9 @@ export class BasedQueryResponse {
         while (i < result.byteLength - 4) {
             let id = readUint32(result, i);
             i += 4;
-            let item;
-            if (this.def.aggregation == 255 /* AggFlag.TEMP */) {
-                item = {};
-                this.def.aggregation = 4 /* AggFlag.COUNT */;
-            }
-            else {
-                item = {
-                    id,
-                };
-            }
+            let item = {
+                id,
+            };
             if (this.def.search) {
                 item.$searchScore = readFloatLE(result, i);
                 i += 4;
@@ -114,12 +107,7 @@ export class BasedQueryResponse {
     }
     get length() {
         const l = readUint32(this.result, 0);
-        if (this.def.aggregation != 0 /* AggFlag.NONE */ && this.def.aggregation != null) {
-            return l + 1;
-        }
-        else {
-            return l;
-        }
+        return l;
     }
     toObject() {
         return resultToObject(this.def, this.result, this.end - 4, 0);

package/dist/src/client/query/aggregates/aggregation.d.ts

@@ -0,0 +1,5 @@
+import { QueryDef, QueryDefAggregation } from '../types.js';
+import { AggregateType } from './types.js';
+export declare const aggregateToBuffer: (aggregates: QueryDefAggregation) => Uint8Array;
+export declare const groupBy: (def: QueryDef, field: string) => void;
+export declare const addAggregate: (type: AggregateType, def: QueryDef, fields: (string | string[])[]) => void;

package/dist/src/client/query/aggregates/aggregation.js

@@ -0,0 +1,96 @@
+import { writeUint16 } from '@saulx/utils';
+export const aggregateToBuffer = (aggregates) => {
+    const aggBuffer = new Uint8Array(aggregates.size);
+    let i = 0;
+    if (aggregates.groupBy) {
+        aggBuffer[i] = 255 /* GroupBy.HAS_GROUP */;
+        i += 1;
+        aggBuffer[i] = aggregates.groupBy.prop;
+        i += 1;
+        aggBuffer[i] = aggregates.groupBy.typeIndex;
+        i += 1;
+        writeUint16(aggBuffer, aggregates.groupBy.start, i);
+        i += 2;
+        writeUint16(aggBuffer, aggregates.groupBy.len, i);
+        i += 2;
+    }
+    else {
+        aggBuffer[i] = 1 /* GroupBy.NONE */;
+        i += 1;
+    }
+    writeUint16(aggBuffer, aggregates.totalResultsPos, i);
+    i += 2;
+    for (const [prop, aggregatesArray] of aggregates.aggregates.entries()) {
+        aggBuffer[i] = prop;
+        i += 1;
+        let sizeIndex = i;
+        let size = 0;
+        i += 2;
+        for (const agg of aggregatesArray) {
+            let startI = i;
+            aggBuffer[i] = agg.type;
+            i += 1;
+            aggBuffer[i] = agg.propDef.typeIndex;
+            i += 1;
+            writeUint16(aggBuffer, agg.propDef.start, i);
+            i += 2;
+            writeUint16(aggBuffer, agg.resultPos, i);
+            i += 2;
+            size += i - startI;
+        }
+        writeUint16(aggBuffer, size, sizeIndex);
+    }
+    return aggBuffer;
+};
+const ensureAggregate = (def) => {
+    if (!def.aggregate) {
+        def.aggregate = {
+            size: 3,
+            aggregates: new Map(),
+            totalResultsPos: 0,
+        };
+    }
+};
+// Group by is great for normal stuff as well (do later)
+export const groupBy = (def, field) => {
+    const fieldDef = def.schema.props[field];
+    if (!fieldDef) {
+        throw new Error(`Field for agg:groupBy does not exists "${field}" make better error later...`);
+    }
+    ensureAggregate(def);
+    if (!def.aggregate.groupBy) {
+        def.aggregate.size += 6;
+    }
+    def.aggregate.groupBy = fieldDef;
+};
+export const addAggregate = (type, def, fields) => {
+    ensureAggregate(def);
+    const aggregates = def.aggregate.aggregates;
+    for (const field of fields) {
+        if (Array.isArray(field)) {
+            addAggregate(type, def, field);
+        }
+        else {
+            const fieldDef = def.schema.props[field];
+            if (!fieldDef) {
+                throw new Error(`Field for agg does not exists ${field} make better error later...`);
+            }
+            if (!aggregates.get(fieldDef.prop)) {
+                aggregates.set(fieldDef.prop, []);
+                def.aggregate.size += 3;
+            }
+            const aggregateField = aggregates.get(fieldDef.prop);
+            aggregateField.push({
+                propDef: fieldDef,
+                type,
+                resultPos: def.aggregate.totalResultsPos,
+            });
+            // IF FLOAT // NUMBER ETC USE 8!
+            // do this better
+            def.aggregate.totalResultsPos += 4;
+            // needs to add an extra field WRITE TO
+            def.aggregate.size += 6;
+        }
+    }
+};
+//# sourceMappingURL=aggregation.js.map

package/dist/src/client/query/aggregates/types.d.ts

@@ -0,0 +1,7 @@
+export declare const enum AggregateType {
+    SUM = 1
+}
+export declare const enum GroupBy {
+    NONE = 1,
+    HAS_GROUP = 255
+}

package/dist/src/client/query/aggregates/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/src/client/query/display.js

@@ -145,16 +145,7 @@ const inspectObject = (object, q, path, level, isLast, isFirst, isObject, depth)
             str += ',\n';
         }
         else if (!def) {
-            if (Object.keys(object)[0] == 'count') {
-                // TODO: to flag the agg someway. This is ugly as hell!!!
-                str += picocolors.blue(v);
-                str += picocolors.italic(picocolors.dim(' count'));
-                str += ',\n';
-            }
-            else {
-                str +=
-                    inspectObject(v, q, key, level + 2, false, false, true, depth) + '';
-            }
+            str += inspectObject(v, q, key, level + 2, false, false, true, depth) + '';
         }
         else if ('__isPropDef' in def) {
             if (def.typeIndex === REFERENCES) {

package/dist/src/client/query/query.d.ts

@@ -8,4 +8,4 @@ export * from './filter/toBuffer.js';
 export * from './sort.js';
 export * from './debug.js';
 export * from './read/read.js';
-export * from './aggregation.js';
+export * from './aggregates/aggregation.js';

package/dist/src/client/query/query.js

@@ -8,5 +8,5 @@ export * from './filter/toBuffer.js';
 export * from './sort.js';
 export * from './debug.js';
 export * from './read/read.js';
-export * from './aggregation.js';
+export * from './aggregates/aggregation.js';
 //# sourceMappingURL=query.js.map

package/dist/src/client/query/queryDef.js

@@ -19,9 +19,9 @@ const createEmptySharedDef = (skipValidation) => {
                 include: {},
             },
         },
+        aggregate: null,
         sort: null,
         references: new Map(),
-        aggregation: 0 /* AggFlag.NONE */,
     };
     return q;
 };

package/dist/src/client/query/read/read.d.ts

@@ -5,5 +5,5 @@ export type Item = {
     [key: string]: any;
 };
 export type AggItem = Partial<Item>;
-export declare const readAllFields: (q: QueryDef, result: Uint8Array, offset: number, end: number, item: Item | AggItem, id: number) => number;
+export declare const readAllFields: (q: QueryDef, result: Uint8Array, offset: number, end: number, item: Item, id: number) => number;
 export declare const resultToObject: (q: QueryDef, result: Uint8Array, end: number, offset?: number) => any;

package/dist/src/client/query/read/read.js

@@ -1,9 +1,11 @@
 import { ALIAS, ALIASES, BINARY, BOOLEAN, ENUM, INT16, INT32, INT8, NUMBER, STRING, TEXT, TIMESTAMP, UINT16, UINT32, UINT8, VECTOR, JSON, CARDINALITY, } from '@based/schema/def';
 import { QueryDefType } from '../types.js';
 import { read, readUtf8 } from '../../string.js';
-import { readDoubleLE, readFloatLE, readInt16, readInt32, readUint16, readUint32, } from '@saulx/utils';
+import { DECODER, readDoubleLE, readFloatLE, readInt16, readInt32, readUint16, readUint32, setByPath, } from '@saulx/utils';
 import { inverseLangMap } from '@based/schema';
-import { READ_EDGE, READ_ID, READ_REFERENCE, READ_REFERENCES, READ_AGGREGATION, CREATE_AGGREGATION, } from '../types.js';
+import { READ_EDGE, READ_ID, READ_REFERENCE, READ_REFERENCES,
+// AggFlag,
+ } from '../types.js';
 const addField = (p, value, item, defaultOnly = false, lang = 0) => {
     let i = p.__isEdge === true ? 1 : 0;
     // TODO OPTMIZE
@@ -125,45 +127,43 @@ const readMain = (q, result, offset, item) => {
     return i - offset;
 };
 const handleUndefinedProps = (id, q, item) => {
-    if (q.aggregation === 0 /* AggFlag.NONE */ || q.aggregation === null) {
-        for (const k in q.include.propsRead) {
-            if (q.include.propsRead[k] !== id) {
-                // Only relevant for seperate props
-                const prop = q.schema.reverseProps[k];
-                if (prop.typeIndex === CARDINALITY) {
-                    addField(prop, 0, item);
-                }
-                else if (prop.typeIndex === TEXT && q.lang == 0) {
-                    const lan = getEmptyField(prop, item);
-                    const lang = q.include.langTextFields.get(prop.prop).codes;
-                    if (lang.has(0)) {
-                        for (const locale in q.schema.locales) {
-                            if (lan[locale] == undefined) {
-                                lan[locale] = prop.default[locale] || '';
-                            }
+    for (const k in q.include.propsRead) {
+        if (q.include.propsRead[k] !== id) {
+            // Only relevant for seperate props
+            const prop = q.schema.reverseProps[k];
+            if (prop.typeIndex === CARDINALITY) {
+                addField(prop, 0, item);
+            }
+            else if (prop.typeIndex === TEXT && q.lang == 0) {
+                const lan = getEmptyField(prop, item);
+                const lang = q.include.langTextFields.get(prop.prop).codes;
+                if (lang.has(0)) {
+                    for (const locale in q.schema.locales) {
+                        if (lan[locale] == undefined) {
+                            lan[locale] = prop.default[locale] || '';
                         }
                     }
-                    else {
-                        for (const code of lang) {
-                            const locale = inverseLangMap.get(code);
-                            if (!lan[locale]) {
-                                lan[locale] = prop.default[locale] || '';
-                            }
+                }
+                else {
+                    for (const code of lang) {
+                        const locale = inverseLangMap.get(code);
+                        if (!lan[locale]) {
+                            lan[locale] = prop.default[locale] || '';
                         }
                     }
                 }
-                else if (prop.typeIndex === BINARY) {
+            }
+            else if (prop.typeIndex === BINARY) {
+                addField(prop, prop.default, item);
+            }
+            else if (prop.typeIndex === TEXT) {
+                addField(prop, '', item);
+            }
+            else {
+                1;
+                if (prop.default !== undefined) {
                     addField(prop, prop.default, item);
                 }
-                else if (prop.typeIndex === TEXT) {
-                    addField(prop, '', item);
-                }
-                else {
-                    1;
-                    if (prop.default !== undefined) {
-                        addField(prop, prop.default, item);
-                    }
-                }
             }
         }
     }
@@ -290,27 +290,6 @@ export const readAllFields = (q, result, offset, end, item, id) => {
             addField(ref.target.propDef, refs, item);
             i += size + 4;
         }
-        else if (index === CREATE_AGGREGATION) {
-            i--;
-            result[i] = READ_AGGREGATION;
-            result[0] = result[0] + 1;
-            q.aggregation = 255 /* AggFlag.TEMP */;
-            return i - offset - 4 - (q.search ? 4 : 0);
-        }
-        else if (index === READ_AGGREGATION) {
-            // TODO: Change to a map and also to get the aggregate field name from a query function parameter
-            const propAgg = {
-                name: 'count',
-                path: ['count'],
-                typeIndex: UINT32,
-            };
-            const size = readUint32(result, i);
-            addField(propAgg, readUint32(result, i + 4), item);
-            result[0] = result[0] - 1;
-            i--;
-            result[i] = CREATE_AGGREGATION;
-            i += 4 + size + 4;
-        }
         else if (index === 0) {
             i += readMain(q, result, i, item);
         }
@@ -391,6 +370,32 @@ export const readAllFields = (q, result, offset, end, item, id) => {
 };
 let cnt = 0;
 export const resultToObject = (q, result, end, offset = 0) => {
+    if (q.aggregate) {
+        const results = {};
+        if (q.aggregate.groupBy) {
+            // key size = 2 for now... not perfect...
+            let i = 0;
+            while (i < result.byteLength - 4) {
+                const key = DECODER.decode(result.subarray(i, i + 2));
+                i += 2;
+                const resultKey = (results[key] = {});
+                for (const aggregatesArray of q.aggregate.aggregates.values()) {
+                    for (const agg of aggregatesArray) {
+                        setByPath(resultKey, agg.propDef.path, readUint32(result, agg.resultPos + i));
+                    }
+                }
+                i += q.aggregate.totalResultsPos;
+            }
+        }
+        else {
+            for (const aggregatesArray of q.aggregate.aggregates.values()) {
+                for (const agg of aggregatesArray) {
+                    setByPath(results, agg.propDef.path, readUint32(result, agg.resultPos));
+                }
+            }
+        }
+        return results;
+    }
     const len = readUint32(result, offset);
     if (len === 0) {
         if ('id' in q.target || 'alias' in q.target) {
@@ -403,16 +408,9 @@ export const resultToObject = (q, result, end, offset = 0) => {
     while (i < end) {
         const id = readUint32(result, i);
         i += 4;
-        let item;
-        if (q.aggregation == 255 /* AggFlag.TEMP */) {
-            item = {};
-            q.aggregation = 4 /* AggFlag.COUNT */;
-        }
-        else {
-            item = {
-                id,
-            };
-        }
+        let item = {
+            id,
+        };
         if (q.search) {
             item.$searchScore = readFloatLE(result, i);
             i += 4;

package/dist/src/client/query/toBuffer.js

@@ -3,8 +3,8 @@ import { QueryDefType, QueryType } from './types.js';
 import { includeToBuffer } from './include/toBuffer.js';
 import { filterToBuffer } from './query.js';
 import { searchToBuffer } from './search/index.js';
-import { createAggFlagBuffer } from './aggregation.js';
 import { ENCODER } from '@saulx/utils';
+import { aggregateToBuffer } from './aggregates/aggregation.js';
 const byteSize = (arr) => {
     return arr.reduce((a, b) => {
         return a + b.byteLength;
@@ -20,7 +20,6 @@ export function defToBuffer(db, def) {
             def.errors.push(...ref.errors);
         }
     });
-    const aggregation = createAggFlagBuffer(def.aggregation || 0 /* AggFlag.NONE */);
     let edges;
     let edgesSize = 0;
     if (def.edges) {
@@ -37,6 +36,40 @@ export function defToBuffer(db, def) {
     // ---------------------------------------
     // move down and will handle size after store the size Var
     // only for references | edges
+    if (def.aggregate) {
+        const aggregateSize = def.aggregate.size || 0;
+        if (aggregateSize === 0) {
+            throw new Error('Wrong aggregate size (0)');
+        }
+        const filterSize = def.filter.size || 0;
+        const buf = new Uint8Array(16 + filterSize + aggregateSize);
+        buf[0] = QueryType.aggregates;
+        buf[1] = def.schema.idUint8[0];
+        buf[2] = def.schema.idUint8[1];
+        buf[3] = def.range.offset;
+        buf[4] = def.range.offset >>> 8;
+        buf[5] = def.range.offset >>> 16;
+        buf[6] = def.range.offset >>> 24;
+        buf[7] = def.range.limit;
+        buf[8] = def.range.limit >>> 8;
+        buf[9] = def.range.limit >>> 16;
+        buf[10] = def.range.limit >>> 24;
+        buf[11] = filterSize;
+        buf[12] = filterSize >>> 8;
+        if (filterSize) {
+            buf.set(filterToBuffer(def.filter), 13);
+        }
+        const aggregateBuffer = aggregateToBuffer(def.aggregate);
+        buf[14 + filterSize] = aggregateSize;
+        buf[15 + filterSize] = aggregateSize >>> 8;
+        buf.set(aggregateBuffer, 16 + filterSize);
+        result.push(buf);
+        // ignore this for now...
+        // result.push(...include)
+        // console.log('toBuffer > result: ', result)
+        // later this has to be a branch
+        return result;
+    }
     if (def.type === QueryDefType.Root) {
         let search;
         let searchSize = 0;
@@ -130,7 +163,7 @@ export function defToBuffer(db, def) {
                 result.push(buf);
             }
             else {
-                const buf = new Uint8Array(18 + filterSize + sortSize + searchSize);
+                const buf = new Uint8Array(17 + filterSize + sortSize + searchSize);
                 buf[0] = QueryType.default;
                 buf[1] = def.schema.idUint8[0];
                 buf[2] = def.schema.idUint8[1];
@@ -157,7 +190,6 @@ export function defToBuffer(db, def) {
                 if (searchSize) {
                     buf.set(search, 17 + filterSize + sortSize);
                 }
-                buf.set(aggregation, 17 + filterSize + sortSize + searchSize);
                 result.push(buf);
             }
         }
@@ -170,8 +202,8 @@ export function defToBuffer(db, def) {
         }
         const sortSize = sort?.byteLength ?? 0;
         const modsSize = filterSize + sortSize;
-        const meta = new Uint8Array(modsSize + 10 + 8 + 1);
-        const sz = size + 7 + modsSize + 8 + 1;
+        const meta = new Uint8Array(modsSize + 10 + 8);
+        const sz = size + 7 + modsSize + 8;
         meta[0] = 254;
         meta[1] = sz;
         meta[2] = sz >>> 8;
@@ -196,7 +228,6 @@ export function defToBuffer(db, def) {
         meta[15 + modsSize] = def.schema.idUint8[0];
         meta[15 + 1 + modsSize] = def.schema.idUint8[1];
         meta[15 + 2 + modsSize] = def.target.propDef.prop;
-        meta[15 + 3 + modsSize] = aggregation[0];
         result.push(meta);
     }
     else if (def.type === QueryDefType.Reference) {

package/dist/src/client/query/types.d.ts

@@ -2,6 +2,7 @@ import { LangCode } from '@based/schema';
 import { PropDef, PropDefEdge, SchemaTypeDef } from '@based/schema/def';
 import { FilterOpts } from './filter/types.js';
 import { QueryError } from './validation.js';
+import { AggregateType } from './aggregates/types.js';
 export type MainIncludes = {
     [start: string]: [number, PropDef];
 };
@@ -72,10 +73,22 @@ export type QueryDefSort = {
     order: 0 | 1;
     lang: LangCode;
 };
+export type Aggregation = {
+    type: AggregateType;
+    propDef: PropDef;
+    resultPos: number;
+};
+export type QueryDefAggregation = {
+    size: number;
+    groupBy?: PropDef;
+    aggregates: Map<number, Aggregation[]>;
+    totalResultsPos: number;
+};
 export type QueryDefShared = {
     errors: QueryError[];
     lang: LangCode;
     filter: QueryDefFilter;
+    aggregate: null | QueryDefAggregation;
     search: null | QueryDefSearch;
     sort: null | QueryDefSort;
     skipValidation: boolean;
@@ -100,7 +113,6 @@ export type QueryDefShared = {
     };
     references: Map<number, QueryDef>;
     edges?: QueryDef;
-    aggregation: AggFlag;
 };
 export type QueryDefEdges = {
     type: QueryDefType.Edge;
@@ -126,21 +138,3 @@ export declare const READ_ID = 255;
 export declare const READ_EDGE = 252;
 export declare const READ_REFERENCES = 253;
 export declare const READ_REFERENCE = 254;
-export declare const CREATE_AGGREGATION = 250;
-export declare const READ_AGGREGATION = 251;
-export declare const enum AggFlag {
-    NONE = 0,
-    AVG = 1,
-    CARDINALITY = 2,
-    CONCAT = 3,// string aggregation, delimiter should be an argument
-    COUNT = 4,
-    MAX = 5,
-    MIN = 6,
-    MODE = 7,// ordered-set
-    PERCENTILE = 8,// continuous or discrete should be optional parameters, default = discrete
-    RANK = 9,// hypothetical-set, dense should be optional parameter
-    STDDEV = 10,// population or sample should be optional parameters, default = sample
-    SUM = 11,
-    VARIANCE = 12,
-    TEMP = 255
-}

package/dist/src/client/query/types.js

@@ -30,6 +30,4 @@ export const READ_ID = 255;
 export const READ_EDGE = 252;
 export const READ_REFERENCES = 253;
 export const READ_REFERENCE = 254;
-export const CREATE_AGGREGATION = 250;
-export const READ_AGGREGATION = 251;
 //# sourceMappingURL=types.js.map

package/dist/src/index.d.ts

@@ -22,7 +22,7 @@ export declare class BasedDb {
     constructor(opts: {
         path: string;
         maxModifySize?: number;
-        debug?: boolean | 'server';
+        debug?: boolean | 'server' | 'client';
         saveIntervalInSeconds?: number;
     });
     create: DbClient['create'];

package/dist/src/index.js

@@ -24,7 +24,10 @@ export class BasedDb {
     constructor(opts) {
         this.#init(opts);
         if (opts.debug) {
-            if (opts.debug === 'server') {
+            if (opts.debug === 'client') {
+                debugServer(this.server);
+            }
+            else if (opts.debug === 'server') {
                 debugServer(this.server);
             }
             else {

package/dist/src/server/index.js

@@ -11,7 +11,7 @@ import { Worker, MessageChannel } from 'node:worker_threads';
 import { fileURLToPath } from 'node:url';
 import { setTimeout } from 'node:timers/promises';
 import { migrate } from './migrate/index.js';
-import { debugServer } from '../utils.js';
+import { debugServer, schemaLooseEqual } from '../utils.js';
 export const SCHEMA_FILE = 'schema.json';
 export const WRITELOG_FILE = 'writelog.json';
 const __filename = fileURLToPath(import.meta.url);
@@ -87,7 +87,7 @@ export class DbServer {
     processingQueries = 0;
     modifyQueue = [];
     queryQueue = new Map();
-    stopped;
+    stopped; // = true does not work
     onSchemaChange;
     unlistenExit;
     saveIntervalInSeconds;
@@ -119,6 +119,7 @@ export class DbServer {
         }
     }
     start(opts) {
+        this.stopped = false;
         return start(this, opts);
     }
     save(opts) {
@@ -283,6 +284,9 @@ export class DbServer {
     }
     setSchema(strictSchema, fromStart = false, transformFns) {
         if (!fromStart && Object.keys(this.schema.types).length > 0) {
+            if (schemaLooseEqual(strictSchema, this.schema)) {
+                return this.schema;
+            }
             return this.migrateSchema(strictSchema, transformFns);
         }
         const { lastId } = this.schema;
@@ -404,6 +408,10 @@ export class DbServer {
         return offsets;
     }
     #modify(buf) {
+        if (this.stopped) {
+            console.error('Db is stopped - trying to modify');
+            return;
+        }
         const end = buf.length - 4;
         const dataLen = readUint32LE(buf, end);
         let typesSize = readUint16LE(buf, dataLen);
@@ -452,6 +460,10 @@ export class DbServer {
         this.queryQueue.set(resolve, buf);
     }
     getQueryBuf(buf, fromQueue = false) {
+        if (this.stopped) {
+            console.error('Db is stopped - trying to query', buf.byteLength);
+            return Promise.resolve(new Uint8Array(8));
+        }
         if (this.modifyQueue.length) {
             return new Promise((resolve) => {
                 this.addToQueryQueue(resolve, buf);
@@ -533,7 +545,7 @@ export class DbServer {
             await Promise.all(this.workers.map(({ worker }) => worker.terminate()));
             this.workers = [];
             native.stop(this.dbCtxExternal);
-            await setTimeout(20);
+            await setTimeout(100);
         }
         catch (e) {
             this.stopped = false;

package/dist/src/utils.d.ts

@@ -3,3 +3,4 @@ export declare const DECODER: TextDecoder;
 export declare const ENCODER: TextEncoder;
 export declare const debugMode: (target: any, getInfo?: any) => void;
 export declare const debugServer: (server: DbServer) => void;
+export declare const schemaLooseEqual: (a: any, b: any, key?: string) => boolean;

package/dist/src/utils.js

@@ -43,4 +43,51 @@ export const debugMode = (target, getInfo = null) => {
     }
 };
 export const debugServer = (server) => debugMode(server, () => `p: ${server.processingQueries} m: ${server.modifyQueue.length} q: ${server.queryQueue.size}`);
+const exclude = new Set(['id', 'lastId']);
+export const schemaLooseEqual = (a, b, key) => {
+    if (a === b) {
+        return true;
+    }
+    const typeofA = typeof a;
+    if (typeofA !== 'object') {
+        return exclude.has(key);
+    }
+    const typeofB = typeof b;
+    if (typeofA !== typeofB) {
+        return exclude.has(key);
+    }
+    if (a === null || b === null) {
+        return false;
+    }
+    if (a.constructor !== b.constructor) {
+        return false;
+    }
+    if (Array.isArray(a)) {
+        let i = a.length;
+        if (i !== b.length) {
+            return false;
+        }
+        while (i--) {
+            if (!schemaLooseEqual(a[i], b[i])) {
+                return false;
+            }
+        }
+    }
+    else {
+        for (const k in a) {
+            if (!schemaLooseEqual(a[k], b[k], k)) {
+                return false;
+            }
+        }
+        for (const k in b) {
+            if (k in a) {
+                continue;
+            }
+            if (!schemaLooseEqual(a[k], b[k], k)) {
+                return false;
+            }
+        }
+    }
+    return true;
+};
 //# sourceMappingURL=utils.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.35",
+  "version": "0.0.37",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",
@@ -38,9 +38,9 @@
     "basedDbNative.cjs"
   ],
   "dependencies": {
-    "@based/schema": "5.0.0-alpha.12",
+    "@based/schema": "5.0.0-alpha.13",
     "@saulx/hash": "^3.0.0",
-    "@saulx/utils": "^6.5.0",
+    "@saulx/utils": "^6.6.0",
     "exit-hook": "^4.0.0",
     "picocolors": "^1.1.0",
     "@based/crc32c": "^1.0.0"

package/dist/src/client/query/aggregation.d.ts

@@ -1,3 +0,0 @@
-import { QueryDef, AggFlag } from './types.js';
-export declare const createAggFlagBuffer: (aggregation: AggFlag) => Uint8Array;
-export declare const count: (def: QueryDef) => void;

package/dist/src/client/query/aggregation.js

@@ -1,9 +0,0 @@
-export const createAggFlagBuffer = (aggregation) => {
-    const buf = new Uint8Array(1);
-    buf[0] = aggregation;
-    return buf;
-};
-export const count = (def) => {
-    def.aggregation = 4 /* AggFlag.COUNT */;
-};
-//# sourceMappingURL=aggregation.js.map