@based/db 0.0.45 → 0.0.47

package/dist/src/client/index.d.ts

@@ -8,6 +8,7 @@ import { DbServer } from '../server/index.js';
 import { TransformFns } from '../server/migrate/index.js';
 import { ModifyOpts } from './modify/types.js';
 import { OnClose, OnData, OnError } from './query/subscription/types.js';
+import { SubStore } from './query/subscription/index.js';
 export type DbClientHooks = {
     setSchema(schema: StrictSchema, fromStart?: boolean, transformFns?: TransformFns): Promise<DbServer['schema']>;
     flushModify(buf: Uint8Array): Promise<{
@@ -15,7 +16,7 @@ export type DbClientHooks = {
         dbWriteTime?: number;
     }>;
     getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
-    subscribe(q: BasedDbQuery, onData: OnData, onError?: OnError): OnClose;
+    subscribe(q: BasedDbQuery, onData: (buf: Uint8Array) => ReturnType<OnData>, onError?: OnError): OnClose;
 };
 type DbClientOpts = {
     hooks: DbClientHooks;
@@ -26,6 +27,7 @@ type DbClientOpts = {
 type DbClientSchema = DbServer['schema'];
 export declare class DbClient {
     constructor({ hooks, maxModifySize, flushTime, debug, }: DbClientOpts);
+    subs: Map<BasedDbQuery, SubStore>;
     flushTime: number;
     flushReady: () => void;
     flushIsReady: Promise<void>;

package/dist/src/client/index.js

@@ -30,6 +30,7 @@ export class DbClient {
             debugMode(this);
         }
     }
+    subs = new Map();
     flushTime;
     flushReady;
     flushIsReady;
@@ -75,9 +76,14 @@ export class DbClient {
         if (this.modifyCtx.len > 8) {
             console.info('Modify cancelled - schema updated');
         }
+        // cancel modify queue
         const resCtx = this.modifyCtx.ctx;
         this.modifyCtx.reset();
         execCtxQueue(resCtx, true);
+        // resubscribe
+        for (const [q, store] of this.subs) {
+            store.resubscribe(q);
+        }
         if (this.listeners?.schema) {
             for (const cb of this.listeners.schema) {
                 cb(this.schema);
@@ -180,10 +186,14 @@ export class DbClient {
         return expire(this, type, typeof id === 'number' ? id : id.tmpId, seconds);
     }
     destroy() {
-        this.modifyCtx.len = 0;
+        this.stop();
         this.modifyCtx.db = null; // Make sure we don't have a circular ref and leak mem
     }
     stop() {
+        for (const [, { onClose }] of this.subs) {
+            onClose();
+        }
+        this.subs.clear();
         this.modifyCtx.len = 0;
     }
     // For more advanced / internal usage - use isModified instead for most cases

package/dist/src/client/query/BasedDbQuery.d.ts

@@ -1,7 +1,7 @@
 import { QueryDef, QueryTarget, Operator, QueryByAliasObj } from './query.js';
 import { BasedQueryResponse } from './BasedIterable.js';
 import { Search } from './search/index.js';
-import { OnData, OnError } from './subscription/index.js';
+import { OnData, OnError, OnClose } from './subscription/index.js';
 import { DbClient } from '../index.js';
 import { LangName } from '@based/schema';
 import { FilterAst, FilterBranchFn, FilterOpts } from './filter/types.js';
@@ -48,7 +48,7 @@ export declare class BasedDbQuery extends QueryBranch<BasedDbQuery> {
     buffer: Uint8Array;
     register(): void;
     locale(locale: LangName): this;
-    subscribe(onData: OnData, onError?: OnError): import("./subscription/types.js").OnClose;
+    subscribe(onData: OnData, onError?: OnError): OnClose;
     _getSync(dbCtxExternal: any): BasedQueryResponse;
     toBuffer(): Uint8Array;
 }

package/dist/src/client/query/debug.js

@@ -101,6 +101,9 @@ export const debug = (x, start = 0, end = 0, label) => {
                 if (a[j + i * w] === 252) {
                     return picocolors.red(v);
                 }
+                if (a[j + i * w] === 250) {
+                    return picocolors.redBright(v);
+                }
                 return v;
             })
                 .join(' '));

package/dist/src/client/query/display.js

@@ -281,7 +281,7 @@ export const inspectData = (q, def, level, top, depth, hasId = false) => {
         str = prefix + '[';
     }
     if (def.aggregate) {
-        str += inspectObject(q.toObject(), def, '', level + 1, i === max - 1, i === 0, false, depth);
+        str += inspectObject('toObject' in q ? q.toObject() : q, def, '', level + 1, i === max - 1, i === 0, false, depth);
         return str;
     }
     for (const x of q) {

package/dist/src/client/query/include/toBuffer.js

@@ -18,7 +18,6 @@ export const includeToBuffer = (db, def) => {
             walkDefs(db, def, f);
         }
     }
-    // main
     if (def.include.main.len > 0) {
         // if (def.target.)
         const len = def.type === QueryDefType.Edge

package/dist/src/client/query/read/read.js

@@ -3,7 +3,43 @@ import { QueryDefType } from '../types.js';
 import { read, readUtf8 } from '../../string.js';
 import { DECODER, readDoubleLE, readFloatLE, readInt16, readInt32, readUint16, readUint32, setByPath, } from '@saulx/utils';
 import { inverseLangMap } from '@based/schema';
-import { READ_EDGE, READ_ID, READ_REFERENCE, READ_REFERENCES, } from '../types.js';
+import { READ_EDGE, READ_ID, READ_REFERENCE, READ_REFERENCES, READ_AGGREGATION, } from '../types.js';
+const readAggregate = (q, result, offset, len) => {
+    const results = {};
+    if (q.aggregate.groupBy) {
+        let i = offset;
+        while (i < len) {
+            let key = '';
+            if (result[i] == 0) {
+                if (q.aggregate.groupBy.default) {
+                    key = q.aggregate.groupBy.default;
+                }
+                else {
+                    key = `$undefined`;
+                }
+            }
+            else {
+                key = DECODER.decode(result.subarray(i, i + 2));
+            }
+            i += 2;
+            const resultKey = (results[key] = {});
+            for (const aggregatesArray of q.aggregate.aggregates.values()) {
+                for (const agg of aggregatesArray) {
+                    setByPath(resultKey, agg.propDef.path, readUint32(result, agg.resultPos + i));
+                }
+            }
+            i += q.aggregate.totalResultsPos;
+        }
+    }
+    else {
+        for (const aggregatesArray of q.aggregate.aggregates.values()) {
+            for (const agg of aggregatesArray) {
+                setByPath(results, agg.propDef.path, readUint32(result, agg.resultPos + offset));
+            }
+        }
+    }
+    return results;
+};
 const addField = (p, value, item, defaultOnly = false, lang = 0) => {
     let i = p.__isEdge === true ? 1 : 0;
     // TODO OPTMIZE
@@ -175,7 +211,19 @@ export const readAllFields = (q, result, offset, end, item, id) => {
             handleUndefinedProps(id, q, item);
             return i - offset;
         }
-        if (index === READ_EDGE) {
+        if (index === READ_AGGREGATION) {
+            // also for edges at some point!
+            let field = result[i];
+            i++;
+            const size = readUint32(result, i);
+            i += 4;
+            const ref = q.references.get(field);
+            addField(
+            // @ts-ignore
+            ref.target.propDef, readAggregate(ref, result, i, i + size), item);
+            i += size;
+        }
+        else if (index === READ_EDGE) {
             let prop = result[i];
             if (prop === READ_REFERENCE) {
                 i++;
@@ -369,44 +417,7 @@ export const readAllFields = (q, result, offset, end, item, id) => {
 let cnt = 0;
 export const resultToObject = (q, result, end, offset = 0) => {
     if (q.aggregate) {
-        const results = {};
-        // range for numbers
-        if (q.aggregate.groupBy) {
-            // key size = 2 for now... not perfect...
-            let i = 0;
-            while (i < result.byteLength - 4) {
-                // if group = 0
-                // add extra thing for the keys maybe?
-                let key = '';
-                if (result[i] == 0) {
-                    if (q.aggregate.groupBy.default) {
-                        key = q.aggregate.groupBy.default;
-                    }
-                    else {
-                        key = `$undefined`;
-                    }
-                }
-                else {
-                    key = DECODER.decode(result.subarray(i, i + 2));
-                }
-                i += 2;
-                const resultKey = (results[key] = {});
-                for (const aggregatesArray of q.aggregate.aggregates.values()) {
-                    for (const agg of aggregatesArray) {
-                        setByPath(resultKey, agg.propDef.path, readUint32(result, agg.resultPos + i));
-                    }
-                }
-                i += q.aggregate.totalResultsPos;
-            }
-        }
-        else {
-            for (const aggregatesArray of q.aggregate.aggregates.values()) {
-                for (const agg of aggregatesArray) {
-                    setByPath(results, agg.propDef.path, readUint32(result, agg.resultPos));
-                }
-            }
-        }
-        return results;
+        return readAggregate(q, result, 0, result.byteLength - 4);
     }
     const len = readUint32(result, offset);
     if (len === 0) {

package/dist/src/client/query/subscription/index.d.ts

@@ -1,4 +1,14 @@
 import { BasedDbQuery } from '../BasedDbQuery.js';
+import { BasedQueryResponse } from '../BasedIterable.js';
 import { OnData, OnError, OnClose } from './types.js';
+export declare class SubStore {
+    listeners: Map<OnData, OnError>;
+    onClose: OnClose;
+    response?: BasedQueryResponse;
+    checksum?: number;
+    len?: number;
+    subscribe(q: BasedDbQuery): void;
+    resubscribe(q: BasedDbQuery): void;
+}
 export declare const subscribe: (q: BasedDbQuery, onData: OnData, onError?: OnError) => OnClose;
 export * from './types.js';

package/dist/src/client/query/subscription/index.js

@@ -1,11 +1,66 @@
+import { BasedQueryResponse } from '../BasedIterable.js';
 import { includeField } from '../query.js';
 import { registerQuery } from '../registerQuery.js';
+export class SubStore {
+    listeners;
+    onClose;
+    response;
+    checksum;
+    len;
+    subscribe(q) {
+        if (!q.def.include.stringFields.size && !q.def.references.size) {
+            includeField(q.def, '*');
+        }
+        registerQuery(q);
+        this.onClose = q.db.hooks.subscribe(q, (res) => {
+            if (!this.response) {
+                this.response = new BasedQueryResponse(q.id, q.def, res, 0);
+            }
+            else {
+                this.response.result = res;
+                this.response.end = res.byteLength;
+            }
+            const checksum = this.response.checksum;
+            const len = res.byteLength;
+            if (this.len !== len || this.checksum !== checksum) {
+                for (const [onData] of this.listeners) {
+                    onData(this.response);
+                }
+                this.len = len;
+                this.checksum = checksum;
+            }
+        }, (err) => {
+            for (const [, onError] of this.listeners) {
+                onError(err);
+            }
+        });
+    }
+    resubscribe(q) {
+        this.onClose();
+        q.reBuildQuery();
+        this.response = null;
+        this.subscribe(q);
+    }
+}
 export const subscribe = (q, onData, onError) => {
-    if (!q.def.include.stringFields.size && !q.def.references.size) {
-        includeField(q.def, '*');
+    if (!q.db.subs.has(q)) {
+        const store = new SubStore();
+        store.listeners = new Map([[onData, onError]]);
+        store.subscribe(q);
+        q.db.subs.set(q, store);
+    }
+    else {
+        const store = q.db.subs.get(q);
+        store.listeners.set(onData, onError);
     }
-    registerQuery(q);
-    return q.db.hooks.subscribe(q, onData, onError);
+    return () => {
+        const store = q.db.subs.get(q);
+        store.listeners.delete(onData);
+        if (!store.listeners.size) {
+            q.db.subs.delete(q);
+            store.onClose();
+        }
+    };
 };
 export * from './types.js';
 //# sourceMappingURL=index.js.map

package/dist/src/client/query/toBuffer.js

@@ -39,30 +39,58 @@ export function defToBuffer(db, def) {
             throw new Error('Wrong aggregate size (0)');
         }
         const filterSize = def.filter.size || 0;
-        const buf = new Uint8Array(16 + filterSize + aggregateSize);
-        buf[0] = isRootCountOnly(def, filterSize)
-            ? QueryType.aggregatesCountType
-            : QueryType.aggregates;
-        buf[1] = def.schema.idUint8[0];
-        buf[2] = def.schema.idUint8[1];
-        buf[3] = def.range.offset;
-        buf[4] = def.range.offset >>> 8;
-        buf[5] = def.range.offset >>> 16;
-        buf[6] = def.range.offset >>> 24;
-        buf[7] = def.range.limit;
-        buf[8] = def.range.limit >>> 8;
-        buf[9] = def.range.limit >>> 16;
-        buf[10] = def.range.limit >>> 24;
-        buf[11] = filterSize;
-        buf[12] = filterSize >>> 8;
-        if (filterSize) {
-            buf.set(filterToBuffer(def.filter), 13);
+        if (def.type === QueryDefType.References) {
+            const buf = new Uint8Array(13 + filterSize + aggregateSize);
+            const sz = 10 + filterSize + aggregateSize;
+            buf[0] = 251 /* includeOp.REFERENCES_AGGREGATION */;
+            buf[1] = sz;
+            buf[2] = sz >>> 8;
+            // ---
+            buf[3] = filterSize;
+            buf[4] = filterSize >>> 8;
+            buf[5] = def.range.offset;
+            buf[6] = def.range.offset >>> 8;
+            buf[7] = def.range.offset >>> 16;
+            buf[8] = def.range.offset >>> 24;
+            if (filterSize) {
+                buf.set(filterToBuffer(def.filter), 9);
+            }
+            // required to get typeEntry and fieldSchema
+            buf[9 + filterSize] = def.schema.idUint8[0]; // typeId
+            buf[9 + 1 + filterSize] = def.schema.idUint8[1]; // typeId
+            buf[9 + 2 + filterSize] = def.target.propDef.prop; // refField
+            const aggregateBuffer = aggregateToBuffer(def.aggregate);
+            buf.set(aggregateBuffer, 9 + 3 + filterSize);
+            // buf[12 + filterSize] = aggregateSize
+            // buf[12 + 1 + filterSize] = aggregateSize >>> 8
+            result.push(buf);
+        }
+        else {
+            const buf = new Uint8Array(16 + filterSize + aggregateSize);
+            buf[0] = isRootCountOnly(def, filterSize)
+                ? QueryType.aggregatesCountType
+                : QueryType.aggregates;
+            buf[1] = def.schema.idUint8[0];
+            buf[2] = def.schema.idUint8[1];
+            buf[3] = def.range.offset;
+            buf[4] = def.range.offset >>> 8;
+            buf[5] = def.range.offset >>> 16;
+            buf[6] = def.range.offset >>> 24;
+            buf[7] = def.range.limit;
+            buf[8] = def.range.limit >>> 8;
+            buf[9] = def.range.limit >>> 16;
+            buf[10] = def.range.limit >>> 24;
+            buf[11] = filterSize;
+            buf[12] = filterSize >>> 8;
+            if (filterSize) {
+                buf.set(filterToBuffer(def.filter), 13);
+            }
+            const aggregateBuffer = aggregateToBuffer(def.aggregate);
+            buf[14 + filterSize] = aggregateSize;
+            buf[15 + filterSize] = aggregateSize >>> 8;
+            buf.set(aggregateBuffer, 16 + filterSize);
+            result.push(buf);
         }
-        const aggregateBuffer = aggregateToBuffer(def.aggregate);
-        buf[14 + filterSize] = aggregateSize;
-        buf[15 + filterSize] = aggregateSize >>> 8;
-        buf.set(aggregateBuffer, 16 + filterSize);
-        result.push(buf);
         // ignore this for now...
         // result.push(...include)
         if (def.type === QueryDefType.Root) {
@@ -206,7 +234,7 @@ export function defToBuffer(db, def) {
         const modsSize = filterSize + sortSize;
         const meta = new Uint8Array(modsSize + 10 + 8);
         const sz = size + 7 + modsSize + 8;
-        meta[0] = 254;
+        meta[0] = 254 /* includeOp.REFERENCES */;
         meta[1] = sz;
         meta[2] = sz >>> 8;
         meta[3] = filterSize;
@@ -235,7 +263,7 @@ export function defToBuffer(db, def) {
     else if (def.type === QueryDefType.Reference) {
         const meta = new Uint8Array(6);
         const sz = size + 3;
-        meta[0] = 255;
+        meta[0] = 255 /* includeOp.REFERENCE */;
         meta[1] = sz;
         meta[2] = sz >>> 8;
         meta[3] = def.schema.idUint8[0];
@@ -246,7 +274,7 @@ export function defToBuffer(db, def) {
     result.push(...include);
     if (edges) {
         const metaEdgeBuffer = new Uint8Array(3);
-        metaEdgeBuffer[0] = 252;
+        metaEdgeBuffer[0] = 252 /* includeOp.EDGE */;
         metaEdgeBuffer[1] = edgesSize;
         metaEdgeBuffer[2] = edgesSize >>> 8;
         result.push(metaEdgeBuffer, ...edges);

package/dist/src/client/query/types.d.ts

@@ -140,3 +140,10 @@ export declare const READ_ID = 255;
 export declare const READ_EDGE = 252;
 export declare const READ_REFERENCES = 253;
 export declare const READ_REFERENCE = 254;
+export declare const READ_AGGREGATION = 250;
+export declare const enum includeOp {
+    REFERENCES_AGGREGATION = 251,
+    EDGE = 252,
+    REFERENCES = 254,
+    REFERENCE = 255
+}

package/dist/src/client/query/types.js

@@ -31,4 +31,5 @@ export const READ_ID = 255;
 export const READ_EDGE = 252;
 export const READ_REFERENCES = 253;
 export const READ_REFERENCE = 254;
+export const READ_AGGREGATION = 250;
 //# sourceMappingURL=types.js.map

package/dist/src/index.js

@@ -3,9 +3,7 @@ import { ModifyCtx } from './client/flushModify.js';
 import { DbServer } from './server/index.js';
 import { DbClient } from './client/index.js';
 import { wait } from '@saulx/utils';
-import { debugMode, debugServer } from './utils.js';
-import { BasedQueryResponse } from './client/query/BasedIterable.js';
-import { registerQuery } from './client/query/registerQuery.js';
+import { debugMode, debugServer, getDefaultHooks } from './utils.js';
 export * from './client/modify/modify.js';
 export { compress, decompress };
 export { ModifyCtx }; // TODO move this somewhere
@@ -51,66 +49,7 @@ export class BasedDb {
         });
         const client = new DbClient({
             maxModifySize,
-            hooks: {
-                subscribe(q, onData, onError) {
-                    let killed = false;
-                    let timer;
-                    let prevChecksum;
-                    let lastLen = 0;
-                    let response;
-                    const get = async () => {
-                        const schemaVersion = q.def.schemaChecksum;
-                        if (schemaVersion && schemaVersion !== server.schema.hash) {
-                            q.reBuildQuery();
-                            registerQuery(q);
-                            response = undefined;
-                            timer = setTimeout(get, 0);
-                            return;
-                        }
-                        if (killed) {
-                            return;
-                        }
-                        const res = await server.getQueryBuf(q.buffer);
-                        if (killed) {
-                            return;
-                        }
-                        if (!response) {
-                            response = new BasedQueryResponse(q.id, q.def, res, 0);
-                        }
-                        else {
-                            response.result = res;
-                            response.end = res.byteLength;
-                        }
-                        const checksum = response.checksum;
-                        if (lastLen != res.byteLength || checksum != prevChecksum) {
-                            onData(response);
-                            lastLen = res.byteLength;
-                            prevChecksum = checksum;
-                        }
-                        timer = setTimeout(get, 200);
-                    };
-                    get();
-                    return () => {
-                        killed = true;
-                        clearTimeout(timer);
-                    };
-                },
-                setSchema(schema, fromStart) {
-                    return Promise.resolve(server.setSchema(schema, fromStart));
-                },
-                flushModify(buf) {
-                    const d = performance.now();
-                    const offsets = server.modify(buf);
-                    const dbWriteTime = performance.now() - d;
-                    return Promise.resolve({
-                        offsets,
-                        dbWriteTime,
-                    });
-                },
-                getQueryBuf(buf) {
-                    return Promise.resolve(server.getQueryBuf(buf));
-                },
-            },
+            hooks: getDefaultHooks(server),
         });
         this.server = server;
         this.client = client;

package/dist/src/utils.d.ts

@@ -1,6 +1,21 @@
 import { DbServer } from './server/index.js';
+import { BasedDbQuery } from './client/query/BasedDbQuery.js';
+import { OnError } from './client/query/subscription/types.js';
+import { StrictSchema } from '@based/schema';
 export declare const DECODER: TextDecoder;
 export declare const ENCODER: TextEncoder;
 export declare const debugMode: (target: any, getInfo?: any) => void;
 export declare const debugServer: (server: DbServer) => void;
 export declare const schemaLooseEqual: (a: any, b: any, key?: string) => boolean;
+export declare const getDefaultHooks: (server: DbServer, subInterval?: number) => {
+    subscribe(q: BasedDbQuery, onData: (res: Uint8Array) => void, onError: OnError): () => void;
+    setSchema(schema: StrictSchema, fromStart: boolean): Promise<StrictSchema & {
+        lastId: number;
+        hash?: number;
+    }>;
+    flushModify(buf: Uint8Array): Promise<{
+        offsets: Record<number, number>;
+        dbWriteTime: number;
+    }>;
+    getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
+};

package/dist/src/utils.js

@@ -90,4 +90,50 @@ export const schemaLooseEqual = (a, b, key) => {
     }
     return true;
 };
+export const getDefaultHooks = (server, subInterval = 200) => {
+    return {
+        subscribe(q, onData, onError) {
+            let timer;
+            let killed = false;
+            const poll = async () => {
+                const res = await server.getQueryBuf(q.buffer);
+                if (killed) {
+                    return;
+                }
+                if (res.byteLength >= 8) {
+                    onData(res);
+                }
+                else if (res.byteLength === 1 && res[0] === 0) {
+                    console.info('schema mismatch, should resolve after update');
+                    // ignore update and stop polling
+                    return;
+                }
+                else {
+                    onError(new Error('unexpected error'));
+                }
+                timer = setTimeout(poll, subInterval);
+            };
+            poll();
+            return () => {
+                clearTimeout(timer);
+                killed = true;
+            };
+        },
+        setSchema(schema, fromStart) {
+            return Promise.resolve(server.setSchema(schema, fromStart));
+        },
+        flushModify(buf) {
+            const d = performance.now();
+            const offsets = server.modify(buf);
+            const dbWriteTime = performance.now() - d;
+            return Promise.resolve({
+                offsets,
+                dbWriteTime,
+            });
+        },
+        getQueryBuf(buf) {
+            return Promise.resolve(server.getQueryBuf(buf));
+        },
+    };
+};
 //# sourceMappingURL=utils.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.45",
+  "version": "0.0.47",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",