@based/db 0.0.1

package/dist/src/client/query/subscription/index.js

@@ -0,0 +1,52 @@
+import { includeFields } from '../query.js';
+import { registerQuery } from '../registerQuery.js';
+import { runSubscription } from './run.js';
+import { addSubscriptionMarkers, removeSubscriptionMarkers } from './markers.js';
+export * from './types.js';
+export * from './markers.js';
+export const subscribe = (q, onData, onError) => {
+    let closed = false;
+    if (!q.def.include.stringFields.size && !q.def.references.size) {
+        includeFields(q.def, ['*']);
+    }
+    registerQuery(q);
+    if (!q.db.subscriptions.has(q.id)) {
+        const subscription = {
+            query: q,
+            subs: new Set(),
+            inProgress: false,
+            closed: false,
+        };
+        q.db.subscriptions.set(q.id, subscription);
+        addSubscriptionMarkers(q, subscription);
+    }
+    const fn = (res, err) => {
+        if (!closed) {
+            if (err) {
+                onError(err);
+            }
+            else {
+                onData(res);
+            }
+        }
+    };
+    const sub = q.db.subscriptions.get(q.id);
+    const close = () => {
+        sub.subs.delete(fn);
+        if (sub.subs.size === 0) {
+            q.db.subscriptions.delete(q.id);
+            removeSubscriptionMarkers(q);
+        }
+        closed = true;
+        return q;
+    };
+    sub.subs.add(fn);
+    if (!sub.inProgress) {
+        runSubscription(sub);
+    }
+    else if (sub.res) {
+        onData(sub.res);
+    }
+    return close;
+};
+//# sourceMappingURL=index.js.map

package/dist/src/client/query/subscription/markers.d.ts

@@ -0,0 +1,10 @@
+import { PropDef, PropDefEdge } from '../../../server/schema/types.js';
+import { Subscription, SubscriptionMarkerMap, SubscriptionMarkersCheck } from './types.js';
+import { BasedDbQuery } from '../BasedDbQuery.js';
+import { DbClient } from '../../index.js';
+export declare const getSubscriptionMarkers: (db: DbClient, typeId: number, id: number, isCreate: boolean) => SubscriptionMarkersCheck | false;
+export declare const checkSubscriptionMarkers: (db: DbClient, m: SubscriptionMarkersCheck, prop: PropDef | PropDefEdge) => void;
+export declare const createSubscriptionMarkerMap: () => SubscriptionMarkerMap;
+export declare const addSubscriptionMarkers: (q: BasedDbQuery, subscription: Subscription) => void;
+export declare const resetSubscriptionMarkers: (db: DbClient) => void;
+export declare const removeSubscriptionMarkers: (q: BasedDbQuery) => void;

package/dist/src/client/query/subscription/markers.js

@@ -0,0 +1,213 @@
+import { startSubscription } from './run.js';
+export const getSubscriptionMarkers = (db, typeId, id, isCreate) => {
+    if (!(typeId in db.subscriptionMarkers)) {
+        return false;
+    }
+    const t = db.subscriptionMarkers[typeId];
+    let subMarkersCheck = false;
+    if (!isCreate) {
+        if (t.ids.has(id)) {
+            const idMarkers = t.ids.get(id);
+            subMarkersCheck = { ids: idMarkers, collection: false };
+        }
+    }
+    if (t.collection) {
+        if (!subMarkersCheck) {
+            subMarkersCheck = { ids: false, collection: t.collection };
+        }
+        else {
+            subMarkersCheck.collection = t.collection;
+        }
+    }
+    return subMarkersCheck;
+};
+export const checkSubscriptionMarkers = (db, m, prop) => {
+    let newSub = false;
+    if (m.ids) {
+        const markers = m.ids;
+        if (prop.separate) {
+            const propSubs = markers.props[prop.prop];
+            if (propSubs) {
+                for (const s of propSubs) {
+                    if (!s.inProgress) {
+                        newSub = true;
+                        db.subscriptionsToRun.push(s);
+                    }
+                }
+            }
+        }
+        else {
+            const propSubs = markers.main[prop.start];
+            if (propSubs) {
+                for (const s of propSubs) {
+                    if (!s.inProgress) {
+                        newSub = true;
+                        db.subscriptionsToRun.push(s);
+                    }
+                }
+            }
+        }
+    }
+    if (m.collection) {
+        const markers = m.collection;
+        if (prop.separate) {
+            const propSubs = markers.props[prop.prop];
+            // @ts-ignore
+            if (propSubs && !propSubs.__handled) {
+                // @ts-ignore
+                propSubs.__handled = true;
+                for (const s of propSubs) {
+                    if (!s.inProgress) {
+                        newSub = true;
+                        db.subscriptionsToRun.push(s);
+                    }
+                }
+            }
+        }
+        else {
+            const propSubs = markers.main[prop.start];
+            // @ts-ignore
+            if (propSubs && !propSubs.__handled) {
+                // @ts-ignore
+                propSubs.__handled = true;
+                for (const s of propSubs) {
+                    if (!s.inProgress) {
+                        newSub = true;
+                        db.subscriptionsToRun.push(s);
+                    }
+                }
+            }
+        }
+    }
+    if (newSub && !db.subscriptionsInProgress) {
+        startSubscription(db);
+    }
+};
+export const createSubscriptionMarkerMap = () => {
+    return {};
+};
+const getFilterFields = (filter, results = {
+    main: new Set(),
+    props: new Set(),
+}) => {
+    const conditions = filter.conditions;
+    if (conditions) {
+        conditions.forEach((v, k) => {
+            if (k === 0) {
+                for (const buf of v) {
+                    results.main.add(buf.readUint16LE(4));
+                }
+            }
+            else {
+                results.props.add(k);
+            }
+        });
+    }
+    if (filter.or) {
+        getFilterFields(filter.or, results);
+    }
+    return results;
+};
+// resetting subs is a copy for now
+export const addSubscriptionMarkers = (q, subscription) => {
+    const typeId = q.def.schema.id;
+    if (!q.db.subscriptionMarkers[typeId]) {
+        q.db.subscriptionMarkers[typeId] = {
+            ids: new Map(),
+            collection: {
+                main: {},
+                props: {},
+            },
+        };
+    }
+    const markerType = q.db.subscriptionMarkers[typeId];
+    if ('id' in q.def.target) {
+        const id = q.def.target.id;
+        if (!markerType.ids.has(id)) {
+            markerType.ids.set(id, {
+                main: {},
+                props: {},
+            });
+        }
+        const marker = markerType.ids.get(id);
+        const props = q.def.include.props;
+        const main = q.def.include.main;
+        for (const p of props) {
+            if (!(p in marker.props)) {
+                marker.props[p] = [];
+            }
+            const markerProps = marker.props[p];
+            markerProps.push(subscription);
+        }
+        for (const p in main.include) {
+            if (!(p in marker.main)) {
+                marker.main[p] = [];
+            }
+            const markerProps = marker.main[p];
+            markerProps.push(subscription);
+        }
+    }
+    else if ('alias' in q.def.target) {
+        // later
+    }
+    else {
+        const props = q.def.include.props;
+        const main = q.def.include.main;
+        const marker = markerType.collection;
+        for (const p of props) {
+            if (!(p in marker.props)) {
+                marker.props[p] = [];
+            }
+            marker.props[p].push(subscription);
+        }
+        for (const p in main.include) {
+            if (!(p in marker.main)) {
+                marker.main[p] = [];
+            }
+            marker.main[p].push(subscription);
+        }
+        if (q.def.filter) {
+            const r = getFilterFields(q.def.filter);
+            r.main.forEach((k) => {
+                const p = String(k);
+                if (!(p in marker.main)) {
+                    marker.main[p] = [];
+                    // @ts-ignore
+                    marker.main[p].__handled = false;
+                }
+                if (!main.include[p]) {
+                    marker.main[p].push(subscription);
+                }
+            });
+            r.props.forEach((k) => {
+                const p = String(k);
+                if (!(p in marker.props)) {
+                    marker.props[p] = [];
+                    // @ts-ignore
+                    marker.props[p].__handled = false;
+                }
+                if (!props.has(k)) {
+                    marker.props[p].push(subscription);
+                }
+            });
+        }
+    }
+};
+export const resetSubscriptionMarkers = (db) => {
+    for (const typeId in db.subscriptionMarkers) {
+        const t = db.subscriptionMarkers[typeId];
+        for (const k in t.collection.main) {
+            // @ts-ignore
+            t.collection.main[k].__handled = false;
+        }
+        for (const k in t.collection.props) {
+            // @ts-ignore
+            t.collection.props[k].__handled = false;
+        }
+    }
+};
+export const removeSubscriptionMarkers = (q) => {
+    // derp
+    console.log('remove markers!');
+};
+//# sourceMappingURL=markers.js.map

package/dist/src/client/query/subscription/run.d.ts

@@ -0,0 +1,5 @@
+import { Subscription } from './types.js';
+import { DbClient } from '../../index.js';
+export declare const resultsAreEqual: (a: Buffer, b: Buffer) => boolean;
+export declare const runSubscription: (subscription: Subscription) => void;
+export declare const startSubscription: (db: DbClient) => void;

package/dist/src/client/query/subscription/run.js

@@ -0,0 +1,69 @@
+import { BasedQueryResponse } from '../BasedIterable.js';
+import { resetSubscriptionMarkers } from './markers.js';
+export const resultsAreEqual = (a, b) => {
+    const aLen = a.byteLength;
+    const bLen = b.byteLength;
+    if (aLen != bLen) {
+        return false;
+    }
+    if (a[aLen - 4] != b[bLen - 4])
+        return false;
+    if (a[aLen - 3] != b[bLen - 3])
+        return false;
+    if (a[aLen - 2] != b[bLen - 2])
+        return false;
+    if (a[aLen - 1] != b[bLen - 1])
+        return false;
+    return true;
+};
+export const runSubscription = (subscription) => {
+    if (!subscription.inProgress) {
+        subscription.inProgress = true;
+        const q = subscription.query;
+        const buf = q.buffer;
+        const d = performance.now();
+        q.db.hooks
+            .getQueryBuf(buf)
+            .then((res) => {
+            if (subscription.closed) {
+                return;
+            }
+            subscription.inProgress = false;
+            const buf = Buffer.from(res);
+            if (subscription.res) {
+                if (resultsAreEqual(subscription.res.result, buf)) {
+                    return;
+                }
+                subscription.res.execTime = performance.now() - d;
+                subscription.res.result = buf;
+                subscription.res.end = buf.byteLength;
+            }
+            else {
+                subscription.res = new BasedQueryResponse(q.id, q.def, buf, performance.now() - d);
+            }
+            subscription.subs.forEach((fn) => {
+                fn(subscription.res);
+            });
+        })
+            .catch((err) => {
+            subscription.inProgress = false;
+            console.error('Subscription getQuery errors', err);
+        });
+    }
+};
+export const startSubscription = (db) => {
+    console.log('start subscription');
+    if (!db.subscriptionsInProgress) {
+        db.subscriptionsInProgress = true;
+        setTimeout(() => {
+            console.log('run it');
+            db.subscriptionsToRun.forEach((s) => {
+                runSubscription(s);
+            });
+            db.subscriptionsToRun = [];
+            resetSubscriptionMarkers(db);
+            db.subscriptionsInProgress = false;
+        }, db.subscriptonThrottleMs);
+    }
+};
+//# sourceMappingURL=run.js.map

package/dist/src/client/query/subscription/types.d.ts

@@ -0,0 +1,33 @@
+import { BasedQueryResponse } from '../BasedIterable.js';
+import { BasedDbQuery } from '../BasedDbQuery.js';
+export type OnData = (res: BasedQueryResponse) => any;
+export type OnError = (err: Error) => any;
+export type OnClose = () => BasedDbQuery;
+export type OnSubscription = (res: any, err?: Error) => void;
+export type Subscription = {
+    query: BasedDbQuery;
+    subs: Set<OnSubscription>;
+    res?: BasedQueryResponse;
+    closed: boolean;
+    inProgress: boolean;
+};
+export type SubscriptionsMap = Map<number, Subscription>;
+export type SubscriptionsToRun = Subscription[];
+export type SubscriptionMarkers = {
+    main: {
+        [start: string]: Subscription[];
+    };
+    props: {
+        [prop: string]: Subscription[];
+    };
+};
+export type SubscriptionMarkerMap = {
+    [typeID: string]: {
+        ids: Map<number, SubscriptionMarkers>;
+        collection: SubscriptionMarkers;
+    };
+};
+export type SubscriptionMarkersCheck = {
+    collection: SubscriptionMarkers | false;
+    ids: SubscriptionMarkers | false;
+};

package/dist/src/client/query/subscription/types.js

@@ -0,0 +1,3 @@
+export {};
+// has sub
+//# sourceMappingURL=types.js.map

package/dist/src/client/query/thresholds.d.ts

@@ -0,0 +1,6 @@
+export declare const MAX_IDS_PER_QUERY = 1000000;
+export declare const MAX_BUFFER_SIZE: number;
+export declare const MAX_RANGE_PROP_LIMIT = 1000;
+export declare const MAX_RANGE_REF_LIMIT = 10000;
+export declare const MIN_ID_VALUE = 1;
+export declare const MAX_ID_VALUE: number;

package/dist/src/client/query/thresholds.js

@@ -0,0 +1,16 @@
+// Query thresholds
+export const MAX_IDS_PER_QUERY = 1e6; // Max number of IDs that can be queried at once
+export const MAX_BUFFER_SIZE = 5 * 1024 * 1024; // 5MB (max buffer size)
+export const MAX_RANGE_PROP_LIMIT = 1e3;
+export const MAX_RANGE_REF_LIMIT = 1e4;
+// Data validation thresholds
+export const MIN_ID_VALUE = 1; // Minimum allowed value for an ID (id array stared)
+export const MAX_ID_VALUE = 2 ** 32 - 1; /// Maximum allowed value for an ID (Uint32)
+// Performance-related thresholds
+// <Just making room to further threshoulds e.g. MAX_MEMORY_BLABLABLA>
+// Network-related thresholds
+// <Just making room to further threshoulds e.g. MAX_NETWORK_RETRIES, NETWORK_TIMEOUT, etc.>
+// Security-related thresholds
+// <Just making room to further threshoulds that could make sense such. e.g.
+// MAX_INPUT_LENGTH (strings/json to prevent DDoS attacks) or MAX_API_DUMP_LENGTH to avoid undesired dumps>
+//# sourceMappingURL=thresholds.js.map

package/dist/src/client/query/toBuffer.d.ts

@@ -0,0 +1,3 @@
+import { QueryDef } from './types.js';
+import { DbClient } from '../index.js';
+export declare function defToBuffer(db: DbClient, def: QueryDef): Buffer[];

package/dist/src/client/query/toBuffer.js

@@ -0,0 +1,229 @@
+import { createSortBuffer } from './sort.js';
+import { QueryDefType } from './types.js';
+import { includeToBuffer } from './include/toBuffer.js';
+import { filterToBuffer } from './query.js';
+import { searchToBuffer } from './search/index.js';
+import { ALIAS } from '../../server/schema/types.js';
+const byteSize = (arr) => {
+    return arr.reduce((a, b) => {
+        return a + b.byteLength;
+    }, 0);
+};
+const getAliasPropdef = (alias, path, def) => {
+    const schema = def.schema;
+    for (const k in alias) {
+        if (typeof alias[k] === 'string') {
+            const p = path.join('.') + k;
+            const prop = schema.props[p];
+            if (prop.typeIndex === ALIAS) {
+                return { def: prop, value: alias[k] };
+            }
+        }
+        else if (typeof alias[k] === 'object') {
+            const propDef = getAliasPropdef(alias[k], [...path, k], def);
+            if (propDef) {
+                return propDef;
+            }
+        }
+    }
+};
+export function defToBuffer(db, def) {
+    const result = [];
+    const include = includeToBuffer(db, def);
+    def.references.forEach((ref) => {
+        include.push(...defToBuffer(db, ref));
+    });
+    let edges;
+    let edgesSize = 0;
+    if (def.edges) {
+        edges = includeToBuffer(db, def.edges);
+        def.edges.references.forEach((ref) => {
+            edges.push(...defToBuffer(db, ref));
+        });
+        edgesSize = byteSize(edges);
+    }
+    const size = (edges ? edgesSize + 3 : 0) + byteSize(include);
+    if (def.type === QueryDefType.Root) {
+        let filter;
+        let filterSize = 0;
+        let search;
+        let searchSize = 0;
+        if (def.search) {
+            search = searchToBuffer(def.search);
+            searchSize = def.search.size;
+        }
+        if (def.filter.size) {
+            filter = filterToBuffer(def.filter);
+            filterSize = filter.byteLength;
+        }
+        // [type,type]
+        // [q type] 0 == id, 1 === ids, 2 === type only
+        if (def.target.alias) {
+            // put this somehwere else at some point
+            try {
+                const alias = getAliasPropdef(def.target.alias, [], def);
+                if (!alias) {
+                    throw new Error(`Cannot find valid alias type in "${JSON.stringify(def.target.alias)}"`);
+                }
+                else {
+                    const s = Buffer.byteLength(alias.value);
+                    // filter is nice for things like access
+                    const buf = Buffer.allocUnsafe(8 + filterSize + s);
+                    buf[0] = 3;
+                    buf[1] = def.schema.idUint8[0];
+                    buf[2] = def.schema.idUint8[1];
+                    buf[3] = alias.def.prop;
+                    buf.writeUint16LE(s, 4);
+                    buf.write(alias.value, 6);
+                    buf.writeUint16LE(filterSize, s + 6);
+                    if (filterSize) {
+                        buf.set(filter, 8 + s);
+                    }
+                    result.push(buf);
+                }
+            }
+            catch (err) {
+                throw new Error(`Cannot parse alias object query for type ${def.schema.type}`);
+            }
+        }
+        else if (def.target.id) {
+            // type 0
+            // 0: 4 [id]
+            // 0: 2 [filterSize]
+            const buf = Buffer.allocUnsafe(9 + filterSize);
+            buf[0] = 0;
+            buf[1] = def.schema.idUint8[0];
+            buf[2] = def.schema.idUint8[1];
+            buf.writeUInt32LE(def.target.id, 3);
+            buf.writeUint16LE(filterSize, 7);
+            if (filterSize) {
+                buf.set(filter, 9);
+            }
+            result.push(buf);
+        }
+        else {
+            let sort;
+            let sortSize = 0;
+            if (def.sort) {
+                sort = createSortBuffer(def.sort);
+                sortSize = sort.byteLength;
+            }
+            if (def.target.ids) {
+                // type 1
+                // 1: 4 + ids * 4 [ids len] [id,id,id]
+                // 1: 8 [offset, limit]
+                // 1: 2 [filter size]
+                // ?filter
+                // 1: 2 [sort size]
+                // ?sort
+                if (!sortSize &&
+                    (def.range.offset || def.range.limit < def.target.ids.length)) {
+                    def.target.ids = def.target.ids.slice(def.range.offset, def.range.offset + def.range.limit);
+                }
+                const idsSize = def.target.ids.length * 4;
+                const buf = Buffer.allocUnsafe(21 + idsSize + filterSize + sortSize + searchSize);
+                buf[0] = 1;
+                buf[1] = def.schema.idUint8[0];
+                buf[2] = def.schema.idUint8[1];
+                buf.writeUint32LE(idsSize, 3);
+                buf.set(new Uint8Array(def.target.ids.buffer), 7);
+                buf.writeUint32LE(def.range.offset, idsSize + 7);
+                buf.writeUint32LE(def.range.limit, idsSize + 11);
+                buf.writeUint16LE(filterSize, idsSize + 15);
+                if (filterSize) {
+                    buf.set(filter, idsSize + 17);
+                }
+                buf.writeUint16LE(sortSize, 17 + filterSize + idsSize);
+                if (sortSize) {
+                    buf.set(sort, 19 + filterSize + idsSize);
+                }
+                buf.writeUint16LE(searchSize, 19 + filterSize + idsSize + sortSize);
+                if (searchSize) {
+                    buf.set(search, 21 + filterSize + idsSize + sortSize);
+                }
+                // ----------
+                result.push(buf);
+            }
+            else {
+                // type 2
+                // 2: 0
+                // 2: 8 [offset, limit]
+                // 2: 2 [filter size]
+                // ?filter
+                // 2: 2 [sort size]
+                // ?sort
+                const buf = Buffer.allocUnsafe(17 + filterSize + sortSize + searchSize);
+                buf[0] = 2;
+                buf[1] = def.schema.idUint8[0];
+                buf[2] = def.schema.idUint8[1];
+                buf.writeUint32LE(def.range.offset, 3);
+                buf.writeUint32LE(def.range.limit, 7);
+                buf.writeUint16LE(filterSize, 11);
+                if (filterSize) {
+                    buf.set(filter, 13);
+                }
+                buf.writeUint16LE(sortSize, 13 + filterSize);
+                if (sortSize) {
+                    buf.set(sort, 15 + filterSize);
+                }
+                buf.writeUint16LE(searchSize, 15 + filterSize + sortSize);
+                if (searchSize) {
+                    buf.set(search, 17 + filterSize + sortSize);
+                }
+                result.push(buf);
+            }
+        }
+    }
+    else if (def.type === QueryDefType.References) {
+        // TODO filter edge
+        let filter;
+        if (def.filter.size) {
+            filter = filterToBuffer(def.filter);
+        }
+        let sort;
+        if (def.sort) {
+            sort = createSortBuffer(def.sort);
+        }
+        // ADD IT
+        // TODO: ADD RANGE [offset,limit] (8 bytes)
+        const sortSize = sort?.byteLength ?? 0;
+        const filterSize = filter?.byteLength ?? 0;
+        const modsSize = filterSize + sortSize;
+        const meta = Buffer.allocUnsafe(modsSize + 10 + 8);
+        meta[0] = 254;
+        meta.writeUint16LE(size + 7 + modsSize + 8, 1);
+        meta.writeUint16LE(filterSize, 3);
+        meta.writeUint16LE(sortSize, 5);
+        meta.writeUint32LE(def.range.offset, 7);
+        meta.writeUint32LE(def.range.limit, 7 + 4);
+        if (filter) {
+            meta.set(filter, 15);
+        }
+        if (sort) {
+            meta.set(sort, 15 + filterSize);
+        }
+        meta[15 + modsSize] = def.schema.idUint8[0];
+        meta[15 + 1 + modsSize] = def.schema.idUint8[1];
+        meta[15 + 2 + modsSize] = def.target.propDef.prop;
+        result.push(meta);
+    }
+    else if (def.type === QueryDefType.Reference) {
+        const meta = Buffer.allocUnsafe(6);
+        meta[0] = 255;
+        meta.writeUint16LE(size + 3, 1);
+        meta[3] = def.schema.idUint8[0];
+        meta[4] = def.schema.idUint8[1];
+        meta[5] = def.target.propDef.prop;
+        result.push(meta);
+    }
+    result.push(...include);
+    if (edges) {
+        const metaEdgeBuffer = Buffer.allocUnsafe(3);
+        metaEdgeBuffer[0] = 252;
+        metaEdgeBuffer.writeUint16LE(edgesSize, 1);
+        result.push(metaEdgeBuffer, ...edges);
+    }
+    // 8 byte crc 32
+    return result;
+}
+//# sourceMappingURL=toBuffer.js.map