@based/db 0.2.1 → 0.2.3

package/dist/src/client/query/filter/convertFilter.js

@@ -42,7 +42,12 @@ export const convertFilter = (query, field, operator, value, opts) => {
     }
     if (!(operator === 'exists' || operator === '!exists') &&
         (value === '' || value === undefined)) {
-        // not great...
+        if (value === '' && operator === '=') {
+            return [[field, toFilterCtx(def, '!exists', opts), undefined]];
+        }
+        else if (value === '' && operator === '!=') {
+            return [[field, toFilterCtx(def, 'exists', opts), undefined]];
+        }
         return;
     }
     if (operator === '!..') {

package/dist/src/client/query/filter/createFixedFilterBuffer.js

@@ -59,7 +59,7 @@ export const createFixedFilterBuffer = (prop, size, ctx, value, sort) => {
         const len = value.length;
         // Add 8 extra bytes for alignment
         const buffer = new Uint8Array(18 + len * size);
-        const result = { buffer };
+        const result = { buffer, propDef: prop };
         buffer[0] = ctx.type;
         buffer[1] =
             prop.typeIndex === REFERENCES && ctx.operation === EQUAL
@@ -108,13 +108,14 @@ export const createFixedFilterBuffer = (prop, size, ctx, value, sort) => {
         if (isNowQuery(prop, value, ctx)) {
             return {
                 buffer,
+                propDef: prop,
                 subscriptionMeta: {
                     now: [createNowMeta(prop, parsedValue, ctx)],
                 },
             };
         }
         else {
-            return { buffer };
+            return { buffer, propDef: prop };
         }
     }
 };

package/dist/src/client/query/filter/createReferenceFilter.js

@@ -21,6 +21,6 @@ export const createReferenceFilter = (prop, ctx, value) => {
     else {
         writeUint32(buffer, value, 11);
     }
-    return { buffer };
+    return { buffer, propDef: prop };
 };
 //# sourceMappingURL=createReferenceFilter.js.map

package/dist/src/client/query/filter/createVariableFilterBuffer.js

@@ -103,7 +103,10 @@ export const createVariableFilterBuffer = (value, prop, ctx, lang) => {
                     for (let i = 0; i < fbLen; i++) {
                         v[v.byteLength - (i + 1)] = val[val.byteLength - (i + 1)];
                     }
-                    parsedCondition = { buffer: writeVarFilter(mode, v, ctx, prop, 0, 0) };
+                    parsedCondition = {
+                        buffer: writeVarFilter(mode, v, ctx, prop, 0, 0),
+                        propDef: prop,
+                    };
                 }
                 else {
                     parsedCondition = createFixedFilterBuffer(prop, 8, {
@@ -118,12 +121,16 @@ export const createVariableFilterBuffer = (value, prop, ctx, lang) => {
                 if (val instanceof ArrayBuffer) {
                     val = new Uint8Array(val);
                 }
-                parsedCondition = { buffer: writeVarFilter(mode, val, ctx, prop, 0, 0) };
+                parsedCondition = {
+                    buffer: writeVarFilter(mode, val, ctx, prop, 0, 0),
+                    propDef: prop,
+                };
             }
         }
         else {
             parsedCondition = {
                 buffer: writeVarFilter(mode, val, ctx, prop, prop.start, prop.len),
+                propDef: prop,
             };
         }
     }

package/dist/src/client/query/filter/filter.js

@@ -1,4 +1,4 @@
-import { isPropDef, ID_FIELD_DEF, TEXT, REFERENCE, } from '@based/schema/def';
+import { isPropDef, ID_FIELD_DEF, TEXT, REFERENCE, REFERENCES, } from '@based/schema/def';
 import { primitiveFilter } from './primitiveFilter.js';
 import { IsFilter } from './types.js';
 import { langCodesMap } from '@based/schema';
@@ -36,7 +36,12 @@ const referencesFilter = (db, filter, schema, conditions, def) => {
             }
             return size;
         }
-        if (isPropDef(t) && t.typeIndex === REFERENCE) {
+        if (isPropDef(t) && t.typeIndex === REFERENCES) {
+            console.info('REFERENCES NESTED NOT IMPLEMENTED');
+            conditions.references ??= new Map();
+            return 0;
+        }
+        else if (isPropDef(t) && t.typeIndex === REFERENCE) {
             conditions.references ??= new Map();
             let refConditions = conditions.references.get(t.prop);
             if (!refConditions) {

package/dist/src/client/query/filter/primitiveFilter.js

@@ -1,5 +1,5 @@
-import { CARDINALITY, REFERENCE, REFERENCES, REVERSE_SIZE_MAP, } from '@based/schema/def';
-import { EQUAL, EXISTS, isNumerical, TYPE_NEGATE } from './types.js';
+import { CARDINALITY, REFERENCE, REFERENCES, REVERSE_SIZE_MAP, REVERSE_TYPE_INDEX_MAP, STRING, } from '@based/schema/def';
+import { EQUAL, EXISTS, isNumerical, TYPE_NEGATE, TYPE_DEFAULT, } from './types.js';
 import { createVariableFilterBuffer } from './createVariableFilterBuffer.js';
 import { createFixedFilterBuffer } from './createFixedFilterBuffer.js';
 import { createReferenceFilter } from './createReferenceFilter.js';
@@ -13,14 +13,26 @@ export const primitiveFilter = (def, prop, filter, conditions, lang) => {
     const fieldIndexChar = prop.prop;
     const bufferMap = prop.__isEdge ? conditions.edges : conditions.conditions;
     if (ctx.operation === EXISTS) {
-        if (!conditions.exists) {
-            conditions.exists = [];
+        if (!prop.separate) {
+            if (prop.typeIndex === STRING) {
+                ctx.operation = EQUAL;
+                ctx.type = ctx.type === TYPE_NEGATE ? TYPE_DEFAULT : TYPE_NEGATE;
+                value = '';
+            }
+            else {
+                console.error('MISSING EXIST / !EXIST FILTER FOR', prop.path, REVERSE_TYPE_INDEX_MAP[prop.typeIndex]);
+            }
+        }
+        else {
+            if (!conditions.exists) {
+                conditions.exists = [];
+            }
+            conditions.exists.push({
+                prop: prop,
+                negate: filter[1].type === TYPE_NEGATE,
+            });
+            return 4;
         }
-        conditions.exists.push({
-            prop: prop,
-            negate: filter[1].type === TYPE_NEGATE,
-        });
-        return 4;
     }
     let size = 0;
     const isArray = Array.isArray(value);

package/dist/src/client/query/filter/types.d.ts

@@ -44,7 +44,8 @@ export declare const META_EXISTS = 251;
 export declare const META_EDGE = 252;
 export declare const META_OR_BRANCH = 253;
 export declare const META_REFERENCE = 254;
-export type FILTER_META = typeof META_EDGE | typeof META_OR_BRANCH | typeof META_REFERENCE | typeof META_EXISTS;
+export declare const META_REFERENCES = 250;
+export type FILTER_META = typeof META_EDGE | typeof META_OR_BRANCH | typeof META_REFERENCE | typeof META_EXISTS | typeof META_REFERENCES;
 export type FilterCtx = {
     operation: OPERATOR;
     type: FILTER_TYPE;
@@ -55,7 +56,7 @@ export declare const VECTOR_DOT_PRODUCT = 0;
 export declare const VECTOR_MANHATTAN_DIST = 1;
 export declare const VECTOR_COSTINE_SIMILARITY = 2;
 export declare const VECTOR_EUCLIDEAN_DIST = 3;
-export declare const getVectorFn: (optsFn?: FilterOpts["fn"]) => 1 | 0 | 2 | 3;
+export declare const getVectorFn: (optsFn?: FilterOpts["fn"]) => 1 | 0 | 3 | 2;
 export declare const toFilterCtx: (def: QueryDef, op: Operator, opts?: FilterOpts) => FilterCtx;
 export declare const operatorReverseMap: Record<OPERATOR, string>;
 export declare const ALIGNMENT_NOT_SET = 255;

package/dist/src/client/query/filter/types.js

@@ -66,6 +66,7 @@ export const META_EXISTS = 251;
 export const META_EDGE = 252;
 export const META_OR_BRANCH = 253;
 export const META_REFERENCE = 254;
+export const META_REFERENCES = 250;
 export const VECTOR_DOT_PRODUCT = 0;
 export const VECTOR_MANHATTAN_DIST = 1;
 export const VECTOR_COSTINE_SIMILARITY = 2;

package/dist/src/client/query/include/props.js

@@ -100,7 +100,7 @@ export const includeProp = (def, prop, opts) => {
         }
         else {
             def.include.main.len += prop.len;
-            def.include.main.include[prop.start] = [0, prop, opts];
+            def.include.main.include.set(prop.start, [0, prop, opts]);
             return true;
         }
     }

package/dist/src/client/query/include/toByteCode.js

@@ -2,7 +2,7 @@ import { MICRO_BUFFER, STRING, TEXT, JSON, BINARY } from '@based/schema/def';
 import { QueryDefType, } from '../types.js';
 import { walkDefs } from './walk.js';
 import { langCodesMap } from '@based/schema';
-import { writeUint32 } from '@based/utils';
+import { writeUint16, writeUint32 } from '@based/utils';
 import { getEnd } from './utils.js';
 const EMPTY_BUFFER = new Uint8Array(0);
 export const includeToBuffer = (db, def) => {
@@ -24,33 +24,27 @@ export const includeToBuffer = (db, def) => {
             ? def.target.ref.edgeMainLen
             : def.schema.mainLen;
         if (def.include.main.len === len) {
-            // GET ALL MAIN FIELDS
+            // Get all main fields
             mainBuffer = EMPTY_BUFFER;
             let i = 2;
-            for (const key in def.include.main.include) {
-                const v = def.include.main.include[key];
-                v[0] = v[1].start;
+            for (const value of def.include.main.include.values()) {
+                value[0] = value[1].start;
                 i += 4;
             }
         }
         else {
-            // GET SOME MAIN FIELDS
-            const size = Object.keys(def.include.main.include).length;
+            const size = def.include.main.include.size;
             mainBuffer = new Uint8Array(size * 4 + 2);
-            mainBuffer[0] = def.include.main.len;
-            mainBuffer[1] = def.include.main.len >>> 8;
+            writeUint16(mainBuffer, def.include.main.len, 0);
             let i = 2;
             let m = 0;
-            for (const key in def.include.main.include) {
-                const v = def.include.main.include[key];
-                mainBuffer[i] = v[1].start;
-                mainBuffer[i + 1] = v[1].start >>> 8;
-                const len = v[1].len;
-                v[0] = m;
-                mainBuffer[i + 2] = len;
-                mainBuffer[i + 3] = len >>> 8;
+            for (const value of def.include.main.include.values()) {
+                const propDef = value[1];
+                writeUint16(mainBuffer, propDef.start, i);
+                writeUint16(mainBuffer, propDef.len, i + 2);
+                value[0] = m;
                 i += 4;
-                m += len;
+                m += propDef.len;
             }
         }
     }
@@ -61,8 +55,7 @@ export const includeToBuffer = (db, def) => {
             buf[0] = 7 /* includeOp.PARTIAL */;
             buf[1] = 0; // field name 0
             buf[2] = MICRO_BUFFER;
-            buf[3] = mainBuffer.byteLength;
-            buf[4] = mainBuffer.byteLength >>> 8;
+            writeUint16(buf, mainBuffer.byteLength, 3);
             result.push(buf, mainBuffer);
         }
         else {

package/dist/src/client/query/queryDef.js

@@ -17,7 +17,7 @@ const createEmptySharedDef = (skipValidation) => {
             props: new Map(),
             main: {
                 len: 0,
-                include: {},
+                include: new Map(),
             },
         },
         aggregate: null,

package/dist/src/client/query/queryDefToReadSchema.js

@@ -129,8 +129,7 @@ export const convertToReaderSchema = (q, locales) => {
             readerSchema.props[k] = createReaderPropDef(v.def, locales, v.opts);
         }
         readerSchema.main.len = q.include.main.len;
-        for (const k in q.include.main.include) {
-            const [start, p, opts] = q.include.main.include[k];
+        for (const [start, p, opts] of q.include.main.include.values()) {
             readerSchema.main.props[start] = createReaderPropDef(p, locales, opts);
         }
         for (const [k, v] of q.references.entries()) {

package/dist/src/client/query/subscription/toByteCode.d.ts

@@ -1,6 +1,11 @@
 import { BasedDbQuery } from '../BasedDbQuery.js';
 import { FilterMetaNow, QueryDef, QueryDefFilter } from '../types.js';
-export declare const collectFilters: (filter: QueryDefFilter, fields?: Set<number>, nowQueries?: FilterMetaNow[]) => FilterMetaNow[];
-export declare const collectFields: (def: QueryDef) => Set<number>;
+type Fields = {
+    separate: Set<number>;
+    main: Set<number>;
+};
+export declare const collectFilters: (filter: QueryDefFilter, fields?: Fields, nowQueries?: FilterMetaNow[]) => FilterMetaNow[];
+export declare const collectFields: (def: QueryDef) => Fields;
 export declare const collectTypes: (def: QueryDef | QueryDefFilter, types?: Set<number>) => Set<number>;
 export declare const registerSubscription: (query: BasedDbQuery) => void;
+export {};

package/dist/src/client/query/subscription/toByteCode.js

@@ -5,10 +5,7 @@ import { QueryType } from '../types.js';
 import { SubscriptionType } from './types.js';
 export const collectFilters = (filter, fields, nowQueries = []) => {
     if (filter.hasSubMeta) {
-        for (const [prop, conditions] of filter.conditions) {
-            if (fields) {
-                fields.add(prop);
-            }
+        for (const [, conditions] of filter.conditions) {
             for (const condition of conditions) {
                 if (condition.subscriptionMeta) {
                     if (condition.subscriptionMeta.now) {
@@ -18,10 +15,13 @@ export const collectFilters = (filter, fields, nowQueries = []) => {
             }
         }
     }
-    else {
-        for (const prop of filter.conditions.keys()) {
-            if (fields) {
-                fields.add(prop);
+    if (fields) {
+        for (const [prop, conditions] of filter.conditions) {
+            fields.separate.add(prop);
+            if (prop === 0) {
+                for (const condition of conditions) {
+                    fields.main.add(condition.propDef.start);
+                }
             }
         }
     }
@@ -32,9 +32,9 @@ export const collectFilters = (filter, fields, nowQueries = []) => {
         collectFilters(filter.and, fields, nowQueries);
     }
     if (filter.references) {
-        for (const [prop, ref] of filter.references) {
-            if (fields) {
-                fields.add(prop);
+        for (const ref of filter.references.values()) {
+            for (const prop of filter.conditions.keys()) {
+                fields.separate.add(prop);
             }
             collectFilters(ref, undefined, nowQueries);
         }
@@ -42,15 +42,22 @@ export const collectFilters = (filter, fields, nowQueries = []) => {
     return nowQueries;
 };
 export const collectFields = (def) => {
-    const fields = new Set();
+    const fields = {
+        separate: new Set(),
+        main: new Set(),
+    };
     if (def.include.main.len > 0) {
-        fields.add(0);
+        for (const [, propDef] of def.include.main.include.values()) {
+            fields.main.add(propDef.start);
+        }
+        // Add 0
+        fields.separate.add(0);
     }
     for (const prop of def.include.props.values()) {
-        fields.add(prop.def.prop);
+        fields.separate.add(prop.def.prop);
     }
     for (const prop of def.references.keys()) {
-        fields.add(prop);
+        fields.separate.add(prop);
     }
     return fields;
 };
@@ -77,22 +84,31 @@ export const registerSubscription = (query) => {
         const fields = collectFields(query.def);
         const typeId = query.def.schema.id;
         const subId = native.crc32(query.buffer.subarray(ID.id + 4, query.buffer.byteLength - 4));
-        const headerLen = 16;
+        const headerLen = 18;
         const types = collectTypes(query.def);
         const nowQueries = collectFilters(query.def.filter, fields);
-        const buffer = new Uint8Array(headerLen + fields.size + types.size * 2 + nowQueries.length * 16);
+        const buffer = new Uint8Array(headerLen +
+            fields.separate.size +
+            types.size * 2 +
+            nowQueries.length * 16 +
+            fields.main.size * 2);
         buffer[0] = SubscriptionType.singleId;
         writeUint32(buffer, subId, 1);
         writeUint16(buffer, typeId, 5);
         writeUint32(buffer, id, 7);
-        buffer[11] = fields.size;
-        writeUint16(buffer, types.size, 12);
-        writeUint16(buffer, nowQueries.length, 14);
+        buffer[11] = fields.separate.size;
+        writeUint16(buffer, fields.main.size, 12);
+        writeUint16(buffer, types.size, 14);
+        writeUint16(buffer, nowQueries.length, 16);
         let i = headerLen;
-        for (const field of fields) {
+        for (const field of fields.separate) {
             buffer[i] = field;
             i++;
         }
+        for (const offset of fields.main) {
+            writeUint16(buffer, offset, i);
+            i += 2;
+        }
         for (const type of types) {
             writeUint16(buffer, type, i);
             i += 2;
@@ -110,7 +126,10 @@ export const registerSubscription = (query) => {
     else {
         const typeId = query.def.schema.id;
         const types = collectTypes(query.def);
-        const nowQueries = collectFilters(query.def.filter, new Set());
+        const nowQueries = collectFilters(query.def.filter, {
+            separate: new Set(),
+            main: new Set(),
+        });
         const typeLen = types.has(typeId) ? types.size - 1 : types.size;
         const headerLen = 8;
         const buffer = new Uint8Array(headerLen + typeLen * 2 + nowQueries.length * 16);

package/dist/src/client/query/types.d.ts

@@ -19,9 +19,7 @@ export type IncludeField = {
     field: string;
     opts?: IncludeOpts;
 };
-export type MainIncludes = {
-    [start: string]: [number, PropDef, IncludeOpts];
-};
+export type MainIncludes = Map<number, [number, PropDef, IncludeOpts]>;
 export type IncludeTreeArr = (string | PropDef | IncludeTreeArr)[];
 export declare enum QueryType {
     id = 0,
@@ -61,6 +59,7 @@ export type FilterMetaNow = {
 };
 export type FilterCondition = {
     buffer: Uint8Array;
+    propDef: PropDef | PropDefEdge;
     subscriptionMeta?: {
         now?: FilterMetaNow[];
     };

package/dist/src/server/subscription.js

@@ -92,21 +92,21 @@ const replaceNowValues = (query, now) => {
         writeInt64(query, dateNow + offset, byteIndex);
     }
 };
-let total = 0;
-let exectime = 0;
-let int;
+// let total = 0
+// let exectime = 0
+// let int
 export const registerSubscription = (server, query, sub, onData, onError, subInterval) => {
     // this can change dynamicly
     if (subInterval) {
         server.subscriptions.subInterval = subInterval;
     }
-    if (!int)
-        int = setInterval(() => {
-            console.log('EXECED', total, exectime / total, 'ms exec time');
-            if (server.stopped) {
-                clearInterval(int);
-            }
-        }, 1e3);
+    // if (!int)
+    //   int = setInterval(() => {
+    //     console.log('EXECED', total, exectime / total, 'ms exec time')
+    //     if (server.stopped) {
+    //       clearInterval(int)
+    //     }
+    //   }, 1e3)
     let killed = false;
     // now maybe just once per second? (for now)
     if (server.subscriptions.active === 0) {
@@ -125,8 +125,8 @@ export const registerSubscription = (server, query, sub, onData, onError, subInt
                 if (killed) {
                     return;
                 }
-                total++;
-                exectime += Date.now() - d;
+                // total++
+                // exectime += Date.now() - d
                 if (res.byteLength >= 4) {
                     onData(res);
                 }
@@ -149,23 +149,24 @@ export const registerSubscription = (server, query, sub, onData, onError, subInt
     if (sub[0] === SubscriptionType.singleId) {
         const subId = readUint32(sub, 1);
         const id = readUint32(sub, 7);
-        const headerLen = 16;
+        const headerLen = 18;
         let subContainer;
         let listeners;
         if (!server.subscriptions.ids.get(subId)) {
             subContainer = { ids: new Map() };
             server.subscriptions.ids.set(subId, subContainer);
-            const typesLen = readUint16(sub, 12);
-            const nowLen = readUint16(sub, 14);
             const fLen = sub[11];
+            const mainLen = readUint16(sub, 12) * 2;
+            const typesLen = readUint16(sub, 14);
+            const nowLen = readUint16(sub, 16);
             if (typesLen != 0) {
                 // double check if this is alignment correct with the byteOffset else copy
-                const byteOffset = sub.byteOffset + headerLen + fLen;
+                const byteOffset = sub.byteOffset + headerLen + fLen + mainLen;
                 if (byteOffset % 2 === 0) {
                     subContainer.types = new Uint16Array(sub.buffer, byteOffset, typesLen);
                 }
                 else {
-                    subContainer.types = new Uint16Array(sub.slice(headerLen + fLen, headerLen + fLen + typesLen * 2));
+                    subContainer.types = new Uint16Array(sub.slice(headerLen + fLen + mainLen, headerLen + fLen + mainLen + typesLen * 2));
                 }
                 subContainer.typesListener = () => {
                     for (const set of subContainer.ids.values()) {
@@ -184,7 +185,7 @@ export const registerSubscription = (server, query, sub, onData, onError, subInt
                 // and get the date allways (as a seperate query)
                 // when getting the date mark next in line
                 // have to make a copy (subArray is weak)
-                now = sub.slice(headerLen + fLen + typesLen * 2);
+                now = sub.slice(headerLen + fLen + mainLen + typesLen * 2);
                 subContainer.nowListener = () => {
                     // per id want to have a last eval and needs next eval
                     for (const set of subContainer.ids.values()) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.2.1",
+  "version": "0.2.3",
   "type": "module",
   "main": "./dist/src/index.js",
   "scripts": {
@@ -42,9 +42,9 @@
   ],
   "dependencies": {
     "@based/hash": "1.1.0",
-    "@based/schema": "5.1.1",
+    "@based/schema": "5.1.2",
     "@based/utils": "1.2.0",
-    "@based/protocol": "0.1.1",
+    "@based/protocol": "0.1.2",
     "exit-hook": "^4.0.0"
   },
   "optionalDependencies": {