@based/db 0.0.8 → 0.0.9

package/dist/src/client/query/search/index.js

@@ -1,4 +1,6 @@
-import { STRING } from '../../../server/schema/types.js';
+import { langCodesMap } from '@based/schema';
+import { STRING, TEXT, VECTOR } from '../../../server/schema/types.js';
+import { getVectorFn } from '../filter/types.js';
 const makeSize = (nr, u8 = false) => {
     if (u8) {
         const size = Buffer.allocUnsafe(1);
@@ -9,10 +11,31 @@ const makeSize = (nr, u8 = false) => {
     size.writeUint16LE(nr);
     return size;
 };
+// vector
+export const vectorSearch = (def, q, field, opts) => {
+    const prop = def.props[field];
+    if (!prop) {
+        throw new Error(`Cannot find prop ${field}`);
+    }
+    if (prop.typeIndex !== VECTOR) {
+        throw new Error('Can only search trough strings / text');
+    }
+    // [isVec] [q len] [q len] [field] [fn] [score] [score] [score] [score] [q..]
+    let size = 9;
+    const vec = Buffer.from(q.buffer);
+    size += vec.byteLength;
+    def.search = {
+        size: size,
+        prop: prop.prop,
+        query: vec,
+        isVector: true,
+        opts,
+    };
+};
 export const search = (def, q, s) => {
+    const bufs = [];
     let blocks = 0;
     const x = q.toLowerCase().trim().split(' ');
-    const bufs = [];
     for (const s of x) {
         if (s) {
             const b = Buffer.from(s);
@@ -23,9 +46,10 @@ export const search = (def, q, s) => {
     bufs.unshift(makeSize(blocks, true));
     const query = Buffer.concat(bufs);
     def.search = {
-        size: query.byteLength + 2,
+        size: query.byteLength + 3,
         query,
         fields: [],
+        isVector: false,
     };
     if (typeof s === 'string') {
         s = [s];
@@ -35,7 +59,7 @@ export const search = (def, q, s) => {
         for (const k in def.props) {
             const prop = def.props[k];
             // if title / name / headline add ROLE:
-            if (prop.typeIndex === STRING) {
+            if (prop.typeIndex === STRING || prop.typeIndex === TEXT) {
                 s[k] = k === 'title' || k === 'name' || k === 'headline' ? 0 : 2;
             }
         }
@@ -48,35 +72,67 @@ export const search = (def, q, s) => {
         s = x;
     }
     for (const key in s) {
-        const prop = def.props[key];
+        let prop = def.props[key];
+        let lang = def.lang;
         if (!prop) {
-            throw new Error('field ' + key + ' does not exist on type');
+            if (key.includes('.')) {
+                const k = key.split('.');
+                prop = def.props[k.slice(0, -1).join('.')];
+                if (prop && prop.typeIndex === TEXT) {
+                    lang = langCodesMap.get(k[k.length - 1]);
+                }
+                else {
+                    throw new Error('field ' + key + ' does not exist on type');
+                }
+            }
+            else {
+                throw new Error('field ' + key + ' does not exist on type');
+            }
         }
-        if (prop.typeIndex !== STRING) {
-            throw new Error('Can only search trough strings');
+        if (prop.typeIndex !== STRING && prop.typeIndex !== TEXT) {
+            throw new Error('Can only search trough strings / text');
         }
-        def.search.size += 4;
+        def.search.size += 5;
         def.search.fields.push({
             weight: s[key],
+            lang,
             field: prop.prop,
-            start: prop.start ?? 0,
+            start: prop.start ?? 0, // also need lang ofc if you have start
         });
     }
 };
 export const searchToBuffer = (search) => {
-    const result = Buffer.allocUnsafe(search.size);
-    result.writeUint16LE(search.query.byteLength, 0);
-    result.set(search.query, 2);
-    const offset = search.query.byteLength + 2;
-    search.fields.sort((a, b) => {
-        return a.weight - b.weight;
-    });
-    for (let i = 0; i < search.fields.length * 4; i += 4) {
-        const f = search.fields[i / 4];
-        result[i + offset] = f.field;
-        result[i + 1 + offset] = f.weight;
-        result.writeUInt16LE(f.start, i + 2 + offset);
+    if (search.isVector) {
+        // [isVec] [q len] [q len] [field] [fn] [score] [score] [score] [score] [q..]
+        const result = Buffer.allocUnsafe(search.size);
+        result[0] = 1; // search.isVector 1
+        result.writeUint16LE(search.query.byteLength, 1);
+        result[3] = search.prop;
+        result[4] = getVectorFn(search.opts.fn);
+        result.set(Buffer.from(new Float32Array([search.opts.score ?? 0.5]).buffer), 5);
+        result.set(search.query, 9);
+        return result;
+    }
+    else {
+        const result = Buffer.allocUnsafe(search.size);
+        result[0] = 0; // search.isVector 0
+        result.writeUint16LE(search.query.byteLength, 1);
+        result.set(search.query, 3);
+        const offset = search.query.byteLength + 3;
+        // @ts-ignore
+        search.fields.sort((a, b) => {
+            return a.weight - b.weight;
+        });
+        // @ts-ignore
+        for (let i = 0; i < search.fields.length * 5; i += 5) {
+            // @ts-ignore
+            const f = search.fields[i / 5];
+            result[i + offset] = f.field;
+            result[i + 1 + offset] = f.weight;
+            result.writeUInt16LE(f.start, i + 2 + offset);
+            result[i + 4 + offset] = f.lang;
+        }
+        return result;
     }
-    return result;
 };
 //# sourceMappingURL=index.js.map

package/dist/src/client/query/subscription/index.js

@@ -1,7 +1,7 @@
 import { includeFields } from '../query.js';
 import { registerQuery } from '../registerQuery.js';
 import { runSubscription } from './run.js';
-import { addSubscriptionMarkers, removeSubscriptionMarkers } from './markers.js';
+import { addSubscriptionMarkers, deleteSubscriptionMarkers } from './markers.js';
 export * from './types.js';
 export * from './markers.js';
 export const subscribe = (q, onData, onError) => {
@@ -35,7 +35,7 @@ export const subscribe = (q, onData, onError) => {
         sub.subs.delete(fn);
         if (sub.subs.size === 0) {
             q.db.subscriptions.delete(q.id);
-            removeSubscriptionMarkers(q);
+            deleteSubscriptionMarkers(q);
         }
         closed = true;
         return q;

package/dist/src/client/query/subscription/markers.d.ts

@@ -7,4 +7,4 @@ export declare const checkSubscriptionMarkers: (db: DbClient, m: SubscriptionMar
 export declare const createSubscriptionMarkerMap: () => SubscriptionMarkerMap;
 export declare const addSubscriptionMarkers: (q: BasedDbQuery, subscription: Subscription) => void;
 export declare const resetSubscriptionMarkers: (db: DbClient) => void;
-export declare const removeSubscriptionMarkers: (q: BasedDbQuery) => void;
+export declare const deleteSubscriptionMarkers: (q: BasedDbQuery) => void;

package/dist/src/client/query/subscription/markers.js

@@ -206,8 +206,8 @@ export const resetSubscriptionMarkers = (db) => {
         }
     }
 };
-export const removeSubscriptionMarkers = (q) => {
+export const deleteSubscriptionMarkers = (q) => {
     // derp
-    console.log('remove markers!');
+    console.log('delete markers!');
 };
 //# sourceMappingURL=markers.js.map

package/dist/src/client/query/toBuffer.js

@@ -223,10 +223,6 @@ export function defToBuffer(db, def) {
         metaEdgeBuffer.writeUint16LE(edgesSize, 1);
         result.push(metaEdgeBuffer, ...edges);
     }
-    // 8 byte crc 32
-    if (def.type === QueryDefType.Root) {
-        result.push(Buffer.from([def.lang]));
-    }
     return result;
 }
 //# sourceMappingURL=toBuffer.js.map

package/dist/src/client/query/types.d.ts

@@ -1,5 +1,6 @@
 import { LangCode } from '@based/schema';
 import { PropDef, PropDefEdge, SchemaTypeDef } from '../../server/schema/schema.js';
+import { FilterOpts } from './filter/types.js';
 export type MainIncludes = {
     [start: string]: [number, PropDef];
 };
@@ -34,11 +35,19 @@ export type QueryDefFilter = {
 export type QueryDefSearch = {
     size: number;
     query: Buffer;
+    isVector: false;
     fields: {
         weight: number;
         field: number;
         start: number;
+        lang: LangCode;
     }[];
+} | {
+    size: number;
+    query: Buffer;
+    prop: number;
+    isVector: true;
+    opts: FilterOpts;
 };
 export type QueryDefSort = {
     prop: PropDefEdge | PropDef;
@@ -55,6 +64,7 @@ export type QueryDefShared = {
         limit: number;
     };
     include: {
+        langTextFields: Map<number, Set<LangCode>>;
         stringFields: Set<string>;
         props: Set<number>;
         propsRead: {

package/dist/src/client/query/validation.d.ts

@@ -1,6 +1,6 @@
 import { PropDef, PropDefEdge } from '../../server/schema/types.js';
-import { Operator } from './filter/operators.js';
-import { QueryByAliasObj } from './types.js';
+import { Operator } from './filter/types.js';
+import { QueryByAliasObj, QueryDef } from './types.js';
 export declare const isValidId: (id: number) => void;
 export declare const checkMaxIdsPerQuery: (ids: (number | QueryByAliasObj)[]) => void;
 export declare const checkMaxBufferSize: (buf: Buffer) => void;
@@ -10,6 +10,7 @@ export declare const hasFields: (fields: {
 } | {
     [path: string]: PropDef;
 }) => void;
+export declare const isValidAlias: (def: QueryDef, id: QueryByAliasObj) => void;
 export declare const hasField: (field: string) => void;
 export declare const checkOperator: (operator: Operator | boolean) => void;
 export declare const checkValue: (value: any, operator: Operator) => void;

package/dist/src/client/query/validation.js

@@ -1,5 +1,6 @@
+import { ALIAS } from '../../server/schema/types.js';
 import { MAX_IDS_PER_QUERY, MIN_ID_VALUE, MAX_ID_VALUE, MAX_BUFFER_SIZE, } from './thresholds.js';
-import { validOperators } from './filter/operators.js';
+import { validOperators } from './filter/types.js';
 export const isValidId = (id) => {
     if (typeof id != 'number') {
         throw new Error('Id has to be a number');
@@ -29,12 +30,24 @@ export const checkTotalBufferSize = (bufers) => {
         }
     }
 };
-export const hasFields = (fields) => {
+export const hasFields = (
+// get schema see if it actualy has the value (can find it on def)
+fields) => {
     if (Object.keys(fields).length === 0) {
         throw new Error('No fields available to include');
     }
 };
+export const isValidAlias = (def, id) => {
+    for (const key in id) {
+        const prop = def.schema.props[key];
+        if (!prop || prop.typeIndex !== ALIAS) {
+            throw new Error(`Incorrect alias provided for query "${key}"`);
+        }
+    }
+};
+// also wrong
 export const hasField = (field) => {
+    // get schema see if it actualy has the value (can find it on def)
     if (!field) {
         throw new Error(`Invalid field: ${field}`);
     }
@@ -43,6 +56,7 @@ export const hasField = (field) => {
     }
 };
 export const checkOperator = (operator) => {
+    // pass schema value (def) and complete for each prop + operator combination
     if (operator !== undefined &&
         typeof operator !== 'boolean' &&
         !validOperators.includes(operator)) {
@@ -50,6 +64,7 @@ export const checkOperator = (operator) => {
     }
 };
 export const checkValue = (value, operator) => {
+    // pass schema value (def) and complete for each prop + operator combination
     if (operator === '..' || operator === '!..') {
         if (!Array.isArray(value) || value.length !== 2) {
             throw new Error(`Invalid value for operator ${operator}: expected an array with two elements`);

package/dist/src/client/timestamp.d.ts

@@ -0,0 +1 @@
+export declare const convertToTimestamp: (value: string | Date | number) => number;

package/dist/src/client/timestamp.js

@@ -0,0 +1,68 @@
+const timeToNumber = (ex) => {
+    if (ex === 's') {
+        return 1000;
+    }
+    if (ex === 'm') {
+        return 1000 * 60;
+    }
+    if (ex === 'h') {
+        return 1000 * 60 * 60;
+    }
+    if (ex === 'd') {
+        return 1000 * 60 * 60 * 24;
+    }
+    if (ex === 'y') {
+        return 31556952000;
+    }
+    return 1;
+};
+export const convertToTimestamp = (value) => {
+    if (value instanceof Date) {
+        return value.valueOf();
+    }
+    if (typeof value === 'string') {
+        if (value === 'now') {
+            return Date.now();
+        }
+        // MAKE THIS INTO A UTIL
+        const y = value.replace(/([+-])/g, ' $1 ');
+        const arr = y.split(/ +/);
+        let newValue = 0;
+        let now;
+        let op = 1;
+        for (const seg of arr) {
+            if (seg === '-') {
+                op = -1;
+            }
+            else if (seg === '+') {
+                op = 1;
+            }
+            else {
+                var v = 0;
+                if (seg === 'now') {
+                    if (!now) {
+                        now = Date.now();
+                    }
+                    v = now;
+                }
+                else if (/[smhdy]$/.test(seg)) {
+                    const ex = seg[seg.length - 1];
+                    const number = parseInt(seg, 10);
+                    v = number * timeToNumber(ex);
+                }
+                else if (seg) {
+                    v = new Date(seg).valueOf();
+                }
+                if (op === -1) {
+                    newValue -= v;
+                }
+                else {
+                    newValue += v;
+                }
+            }
+        }
+        return newValue;
+    }
+    return value;
+};
+//# sourceMappingURL=timestamp.js.map

package/dist/src/client/xxHash64.d.ts

@@ -0,0 +1 @@
+export declare const xxHash64: (buf: Buffer) => any;

package/dist/src/client/xxHash64.js

@@ -0,0 +1,5 @@
+import native from '../native.js';
+export const xxHash64 = (buf) => {
+    return native.xxHash64(buf);
+};
+//# sourceMappingURL=xxHash64.js.map

package/dist/src/index.d.ts

@@ -17,9 +17,12 @@ export declare class BasedDb {
         maxModifySize?: number;
     });
     create: DbClient['create'];
+    copy: DbClient['copy'];
     update: DbClient['update'];
     upsert: DbClient['upsert'];
-    remove: DbClient['remove'];
+    delete: DbClient['delete'];
+    expire: DbClient['expire'];
+    remove: DbClient['delete'];
     query: DbClient['query'];
     putSchema: DbClient['putSchema'];
     drain: DbClient['drain'];

package/dist/src/index.js

@@ -27,9 +27,9 @@ export class BasedDb {
                     return Promise.resolve(server.putSchema(schema, fromStart));
                 },
                 flushModify(buf) {
-                    server.modify(buf);
+                    const offsets = server.modify(buf);
                     return Promise.resolve({
-                        offsets: {},
+                        offsets,
                     });
                 },
                 getQueryBuf(buf) {
@@ -44,14 +44,24 @@ export class BasedDb {
     create = function () {
         return this.client.create.apply(this.client, arguments);
     };
+    copy = function () {
+        return this.client.copy.apply(this.client, arguments);
+    };
     update = function () {
         return this.client.update.apply(this.client, arguments);
     };
     upsert = function () {
         return this.client.upsert.apply(this.client, arguments);
     };
+    delete = function () {
+        return this.client.delete.apply(this.client, arguments);
+    };
+    expire = function () {
+        return this.client.expire.apply(this.client, arguments);
+    };
     remove = function () {
-        return this.client.remove.apply(this.client, arguments);
+        console.warn('WARNING: db.remove() is deprecated. Use db.delete() instead');
+        return this.client.delete.apply(this.client, arguments);
     };
     query = function () {
         return this.client.query.apply(this.client, arguments);

package/dist/src/native.d.ts

@@ -24,5 +24,6 @@ declare const _default: {
     crc32: (buf: Buffer) => any;
     createSortIndex: (buf: Buffer, dbCtx: any) => any;
     destroySortIndex: (buf: Buffer, dbCtx: any) => any;
+    xxHash64: (buf: Buffer) => any;
 };
 export default _default;

package/dist/src/native.js

@@ -92,6 +92,9 @@ export default {
     },
     destroySortIndex: (buf, dbCtx) => {
         return db.destroySortIndex(dbCtx, buf);
-    }
+    },
+    xxHash64: (buf) => {
+        return db.xxHash64(buf);
+    },
 };
 //# sourceMappingURL=native.js.map

package/dist/src/server/csmt/tree.js

@@ -3,6 +3,7 @@ import { distance, min, max } from './tree-utils.js';
 import membershipProof from './memebership-proof.js';
 export function createTree(createHash) {
     let root = null;
+    const emptyHash = genHash(Buffer.from(''));
     function genHash(s) {
         return createHash().update(s).digest();
     }
@@ -12,7 +13,7 @@ export function createTree(createHash) {
     function createNode(left, right) {
         const hash = left && right
             ? genNodeHash(left.hash, right.hash)
-            : genHash(Buffer.from(''));
+            : emptyHash;
         return {
             hash,
             key: max(left, right),
@@ -80,7 +81,16 @@ export function createTree(createHash) {
         const left = node.left;
         const right = node.right;
         if (!left || !right) {
-            throw new Error('The tree is broken');
+            console.log(node, k);
+            if (node.data) {
+                if (node.key === k) {
+                    return null;
+                }
+                throw new Error(`k=${k} does not exist`);
+            }
+            else {
+                throw new Error('The tree is broken');
+            }
         }
         if (checkForLeaf(left, k) || checkForLeaf(right, k)) {
             if (left.key === k) {

package/dist/src/server/index.d.ts

@@ -3,6 +3,12 @@ import { SchemaTypeDef } from './schema/types.js';
 import { createTree } from './csmt/index.js';
 import { Worker, MessagePort } from 'node:worker_threads';
 import { TransformFns } from './migrate/index.js';
+declare class SortIndex {
+    constructor(buf: Buffer, dbCtxExternal: any);
+    buf: Buffer;
+    idx: any;
+    cnt: number;
+}
 export declare class DbWorker {
     constructor(address: BigInt, db: DbServer);
     db: DbServer;
@@ -59,17 +65,19 @@ export declare class DbServer {
     sortIndexes: {
         [type: number]: {
             [field: number]: {
-                [start: number]: any;
+                [start: number]: SortIndex;
             };
         };
     };
-    createSortIndex(type: string, field: string): any;
+    cleanupTimer: NodeJS.Timeout;
+    cleanup(): void;
+    createSortIndex(type: string, field: string): SortIndex;
     destroySortIndex(type: string, field: string): any;
-    hasSortIndex(typeId: number, field: number, start: number): boolean;
+    getSortIndex(typeId: number, field: number, start: number): SortIndex;
     migrateSchema(schema: StrictSchema, transform?: Record<string, (node: Record<string, any>) => Record<string, any> | [string, Record<string, any>]>): Promise<StrictSchema & {
         lastId: number;
     }>;
-    createSortIndexBuffer(typeId: number, field: number, start: number): any;
+    createSortIndexBuffer(typeId: number, field: number, start: number): SortIndex;
     updateMerkleTree(): void;
     putSchema(strictSchema: StrictSchema, fromStart?: boolean, transformFns?: TransformFns): (StrictSchema & {
         lastId: number;

package/dist/src/server/index.js

@@ -16,6 +16,15 @@ const SCHEMA_FILE = 'schema.json';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const workerPath = join(__dirname, 'worker.js');
+class SortIndex {
+    constructor(buf, dbCtxExternal) {
+        this.buf = buf;
+        this.idx = native.createSortIndex(buf, dbCtxExternal);
+    }
+    buf;
+    idx;
+    cnt = 0;
+}
 export class DbWorker {
     constructor(address, db) {
         const { port1, port2 } = new MessageChannel();
@@ -91,6 +100,35 @@ export class DbServer {
         return this.csmtHashFun;
     };
     sortIndexes;
+    cleanupTimer;
+    cleanup() {
+        if (!this.cleanupTimer) {
+            // amount accessed
+            // current mem available
+            this.cleanupTimer = global.setTimeout(() => {
+                this.cleanupTimer = null;
+                let remaining;
+                for (const type in this.sortIndexes) {
+                    for (const field in this.sortIndexes[type]) {
+                        for (const start in this.sortIndexes[type][field]) {
+                            const sortIndex = this.sortIndexes[type][field][start];
+                            sortIndex.cnt /= 2;
+                            if (sortIndex.cnt < 1) {
+                                native.destroySortIndex(sortIndex.buf, this.dbCtxExternal);
+                                delete this.sortIndexes[type][field][start];
+                            }
+                            else {
+                                remaining = true;
+                            }
+                        }
+                    }
+                }
+                if (remaining) {
+                    this.cleanup();
+                }
+            }, 60e3);
+        }
+    }
     createSortIndex(type, field) {
         const t = this.schemaTypesParsed[type];
         const prop = t.props[field];
@@ -113,7 +151,7 @@ export class DbServer {
         buf.writeUint16LE(prop.start, 3);
         buf.writeUint16LE(prop.len, 5);
         buf[7] = prop.typeIndex;
-        sortIndex = native.createSortIndex(buf, this.dbCtxExternal);
+        sortIndex = new SortIndex(buf, this.dbCtxExternal);
         fields[prop.start] = sortIndex;
         return sortIndex;
     }
@@ -138,7 +176,7 @@ export class DbServer {
             delete fields[prop.start];
         }
     }
-    hasSortIndex(typeId, field, start) {
+    getSortIndex(typeId, field, start) {
         let types = this.sortIndexes[typeId];
         if (!types) {
             types = this.sortIndexes[typeId] = {};
@@ -147,11 +185,7 @@ export class DbServer {
         if (!fields) {
             fields = types[field] = {};
         }
-        let sortIndex = fields[start];
-        if (sortIndex) {
-            return true;
-        }
-        return false;
+        return fields[start];
     }
     migrateSchema(schema, transform) {
         return migrate(this, schema, transform);
@@ -185,7 +219,7 @@ export class DbServer {
         buf.writeUint16LE(prop.len, 5);
         buf[7] = prop.typeIndex;
         // put in modify stuff
-        const sortIndex = native.createSortIndex(buf, this.dbCtxExternal);
+        const sortIndex = new SortIndex(buf, this.dbCtxExternal);
         const types = this.sortIndexes[typeId];
         const fields = types[field];
         fields[start] = sortIndex;
@@ -279,7 +313,7 @@ export class DbServer {
             }
             if (strictSchema.props) {
                 // insert a root node
-                const data = [2, 1, 0, 0, 0, 9, 1, 0, 0, 0, 7, 1, 0, 1];
+                const data = [2, 1, 0, 0, 0, 17, 9, 1, 0, 0, 0, 7, 1, 0, 1];
                 const blockKey = makeCsmtKey(1, 1);
                 const buf = Buffer.alloc(data.length + 2 + 8 + 4);
                 // add content
@@ -353,14 +387,18 @@ export class DbServer {
                     const sort = buf.slice(s + 2, s + 2 + sortLen);
                     const field = sort[1];
                     const start = sort.readUint16LE(2 + 1);
-                    if (!this.hasSortIndex(typeId, field, start)) {
+                    let sortIndex = this.getSortIndex(typeId, field, start);
+                    if (!sortIndex) {
                         if (this.processingQueries) {
                             return new Promise((resolve) => {
                                 this.queryQueue.set(resolve, buf);
                             });
                         }
-                        this.createSortIndexBuffer(typeId, field, start);
+                        sortIndex = this.createSortIndexBuffer(typeId, field, start);
                     }
+                    // increment
+                    sortIndex.cnt++;
+                    this.cleanup();
                 }
             }
             else if (queryType == 1) {
@@ -392,13 +430,21 @@ export class DbServer {
         if (this.stopped) {
             return;
         }
-        if (!noSave) {
-            await this.save();
+        this.stopped = true;
+        clearTimeout(this.cleanupTimer);
+        try {
+            if (!noSave) {
+                await this.save();
+            }
+            await Promise.all(this.workers.map(({ worker }) => worker.terminate()));
+            this.workers = [];
+            native.stop(this.dbCtxExternal);
+            await setTimeout();
+        }
+        catch (e) {
+            this.stopped = false;
+            throw e;
         }
-        await Promise.all(this.workers.map(({ worker }) => worker.terminate()));
-        this.workers = [];
-        native.stop(this.dbCtxExternal);
-        await setTimeout();
     }
     async destroy() {
         await this.stop(true);