@based/db 0.0.24 → 0.0.26

package/dist/src/client/modify/references/edge.js

@@ -75,7 +75,8 @@ export function writeEdges(t, ref, ctx) {
                     if (typeof value !== 'string') {
                         return new ModifyError(edge, value);
                     }
-                    if (ctx.len + EDGE_HEADER_SIZE + ENCODER.encode(value).byteLength > ctx.max) {
+                    if (ctx.len + EDGE_HEADER_SIZE + ENCODER.encode(value).byteLength >
+                        ctx.max) {
                         return RANGE_ERR;
                     }
                     ctx.buf[ctx.len++] = STRING;
@@ -271,6 +272,7 @@ export function writeEdges(t, ref, ctx) {
             // Index of start of fields
             const sIndex = ctx.len;
             ctx.len += mainFieldsStartSize;
+            // this has to be replaced
             // Add zeroes
             ctx.buf.fill(0, ctx.len, ctx.len + t.edgeMainLen);
             // Keep track of written bytes from append fixed

package/dist/src/client/modify/upsert.js

@@ -2,7 +2,7 @@ import { ALIAS } from '@based/schema/def';
 export async function upsert(db, type, obj, opts) {
     const tree = db.schemaTypesParsed[type].tree;
     let q;
-    let id = '';
+    let id = type;
     for (const key in obj) {
         if (tree[key].typeIndex === ALIAS) {
             id += `${key}:${obj[key]};`;

package/dist/src/client/operations.d.ts

@@ -0,0 +1,32 @@
+import { PropDef, SchemaTypeDef } from '@based/schema/def';
+import { DbClient } from './index.js';
+import { ModifyState } from './modify/ModifyRes.js';
+export declare class ModifyCtx {
+    constructor(db: DbClient);
+    len: number;
+    id: number;
+    hasSortField: number;
+    hasSortText: number;
+    queue: Map<(payload: any) => void, ModifyState>;
+    ctx: {
+        offsets?: Record<number, number>;
+    };
+    payload: Uint8Array;
+    max: number;
+    buf: Uint8Array;
+    field: number;
+    prefix0: number;
+    prefix1: number;
+    lastMain: number;
+    mergeMain: (PropDef | any)[] | null;
+    mergeMainSize: number;
+    db: DbClient;
+    dirtyRanges: Set<number>;
+    dirtyTypes: Map<number, number>;
+    markNodeDirty(schema: SchemaTypeDef, nodeId: number): void;
+    markTypeDirty(schema: SchemaTypeDef): void;
+    updateMax(): void;
+    getData(lastIds: Record<number, number>): Uint8Array;
+}
+export declare const flushBuffer: (db: DbClient) => Promise<void>;
+export declare const startDrain: (db: DbClient) => void;

package/dist/src/client/operations.js

@@ -0,0 +1,137 @@
+import { makeCsmtKeyFromNodeId } from './tree.js';
+export class ModifyCtx {
+    constructor(db) {
+        this.max = db.maxModifySize;
+        this.db = db;
+        this.buf = new Uint8Array(db.maxModifySize);
+    }
+    // default values
+    len = 0;
+    id = -1;
+    hasSortField = -1;
+    hasSortText = -1;
+    queue = new Map();
+    ctx = {}; // maybe make this different?
+    payload;
+    max;
+    buf;
+    field;
+    prefix0 = -1;
+    prefix1 = -1;
+    lastMain = -1;
+    mergeMain;
+    mergeMainSize;
+    db;
+    dirtyRanges = new Set();
+    dirtyTypes = new Map();
+    markNodeDirty(schema, nodeId) {
+        const key = makeCsmtKeyFromNodeId(schema.id, schema.blockCapacity, nodeId);
+        if (this.dirtyRanges.has(key)) {
+            return;
+        }
+        this.dirtyRanges.add(key);
+        this.updateMax();
+    }
+    markTypeDirty(schema) {
+        if (this.dirtyTypes.has(schema.id)) {
+            return;
+        }
+        this.dirtyTypes.set(schema.id, schema.lastId);
+        this.updateMax();
+    }
+    updateMax() {
+        // reserve space in the end of the buf [...data, type (16), lastId (32), typesSize (16), ...ranges (64)[], dataLen (32)]
+        this.max =
+            this.db.maxModifySize -
+                4 -
+                2 -
+                this.dirtyTypes.size * 10 -
+                this.dirtyRanges.size * 8;
+    }
+    getData(lastIds) {
+        const rangesSize = this.dirtyRanges.size;
+        const typesSize = this.dirtyTypes.size;
+        const data = this.buf.subarray(0, this.len + 4 + 2 + typesSize * 10 + rangesSize * 8);
+        let i = this.len;
+        data[i] = typesSize;
+        data[i + 1] = typesSize >>> 8;
+        i += 2;
+        for (const [id, startId] of this.dirtyTypes) {
+            const lastId = this.db.schemaTypesParsedById[id].lastId;
+            lastIds[id] = lastId;
+            data[i] = id;
+            data[i + 1] = id >>> 8;
+            i += 2;
+            data[i++] = startId;
+            data[i++] = startId >>> 8;
+            data[i++] = startId >>> 16;
+            data[i++] = startId >>> 24;
+            data[i++] = lastId;
+            data[i++] = lastId >>> 8;
+            data[i++] = lastId >>> 16;
+            data[i++] = lastId >>> 24;
+        }
+        const view = new DataView(data.buffer, data.byteOffset);
+        for (let key of this.dirtyRanges) {
+            view.setFloat64(i, key, true);
+            i += 8;
+        }
+        data[i++] = this.len;
+        data[i++] = this.len >>> 8;
+        data[i++] = this.len >>> 16;
+        data[i++] = this.len >>> 24;
+        return data;
+    }
+}
+export const flushBuffer = (db) => {
+    const ctx = db.modifyCtx;
+    let flushPromise;
+    if (ctx.len) {
+        const d = Date.now();
+        const lastIds = {};
+        const data = ctx.getData(lastIds);
+        const resCtx = ctx.ctx;
+        const queue = ctx.queue;
+        flushPromise = db.hooks.flushModify(data).then(({ offsets }) => {
+            resCtx.offsets = offsets;
+            for (const typeId in lastIds) {
+                if (typeId in offsets) {
+                    const lastId = lastIds[typeId] + offsets[typeId];
+                    const def = db.schemaTypesParsedById[typeId];
+                    const delta = lastId - def.lastId;
+                    if (delta > 0) {
+                        def.lastId += delta;
+                        def.total += delta;
+                    }
+                }
+                else {
+                    console.warn('no offset returned, very wrong');
+                }
+            }
+            db.writeTime += Date.now() - d;
+            if (queue.size) {
+                flushPromise.then(() => {
+                    for (const [resolve, res] of queue) {
+                        resolve(res.getId(offsets));
+                    }
+                });
+            }
+        });
+        ctx.dirtyTypes.clear();
+        ctx.dirtyRanges.clear();
+        ctx.len = 0;
+        ctx.prefix0 = -1;
+        ctx.prefix1 = -1;
+        ctx.max = db.maxModifySize;
+        ctx.ctx = {};
+    }
+    db.isDraining = false;
+    return flushPromise;
+};
+export const startDrain = (db) => {
+    db.isDraining = true;
+    process.nextTick(() => {
+        flushBuffer(db);
+    });
+};
+//# sourceMappingURL=operations.js.map

package/dist/src/client/query/BasedDbQuery.js

@@ -208,6 +208,9 @@ export class BasedDbQuery extends QueryBranch {
                 }
             }
         }
+        if (!db.schemaIsSetValue) {
+            throw new Error('Query: No schema yet - use await db.schemaIsSet()');
+        }
         const def = createQueryDef(db, QueryDefType.Root, target, skipValidation);
         super(db, def);
     }

package/dist/src/client/query/BasedIterable.d.ts

@@ -14,7 +14,7 @@ export declare class BasedQueryResponse {
     [inspect.custom](depth: number): string;
     debug(): this;
     node(index?: number): any;
-    [Symbol.iterator](): Generator<Item, void, unknown>;
+    [Symbol.iterator](): Generator<Partial<Item>, void, unknown>;
     inspect(depth?: number): this;
     forEach(fn: (item: any, key: number) => void): void;
     map(fn: (item: any, key: number) => any): any[];

package/dist/src/client/query/BasedIterable.js

@@ -2,7 +2,7 @@ import { inspect } from 'node:util';
 import picocolors from 'picocolors';
 import { debug, resultToObject, readAllFields } from './query.js';
 import { size, time, inspectData, defHasId, displayTarget } from './display.js';
-import { readFloatLE, readUint32 } from '../bitWise.js';
+import { readFloatLE, readUint32 } from '../../utils.js';
 export { time, size, inspectData };
 export class BasedQueryResponse {
     result;
@@ -59,9 +59,16 @@ export class BasedQueryResponse {
         while (i < result.byteLength - 4) {
             let id = readUint32(result, i);
             i += 4;
-            const item = {
-                id,
-            };
+            let item;
+            if (this.def.aggregation == 255 /* AggFlag.TEMP */) {
+                item = {};
+                this.def.aggregation = 4 /* AggFlag.COUNT */;
+            }
+            else {
+                item = {
+                    id,
+                };
+            }
             if (this.def.search) {
                 item.$searchScore = readFloatLE(result, i);
                 i += 4;
@@ -95,7 +102,13 @@ export class BasedQueryResponse {
         return readUint32(result, offset);
     }
     get length() {
-        return readUint32(this.result, 0);
+        const l = readUint32(this.result, 0);
+        if (this.def.aggregation != 0 /* AggFlag.NONE */ && this.def.aggregation != null) {
+            return l + 1;
+        }
+        else {
+            return l;
+        }
     }
     toObject() {
         return resultToObject(this.def, this.result, this.end - 4, 0);

package/dist/src/client/query/aggregation.d.ts

@@ -0,0 +1,3 @@
+import { QueryDef, AggFlag } from './types.js';
+export declare const createAggFlagBuffer: (aggregation: AggFlag) => Uint8Array;
+export declare const count: (def: QueryDef) => void;

package/dist/src/client/query/aggregation.js

@@ -0,0 +1,9 @@
+export const createAggFlagBuffer = (aggregation) => {
+    const buf = new Uint8Array(1);
+    buf[0] = aggregation;
+    return buf;
+};
+export const count = (def) => {
+    def.aggregation = 4 /* AggFlag.COUNT */;
+};
+//# sourceMappingURL=aggregation.js.map

package/dist/src/client/query/display.js

@@ -119,7 +119,8 @@ const inspectObject = (object, q, path, level, isLast, isFirst, isObject, depth)
     let edges = [];
     for (const k in object) {
         const key = path ? path + '.' + k : k;
-        let def = q.props[key];
+        let def;
+        def = q.props[key];
         let v = object[k];
         const isEdge = k[0] === '$';
         if (k === '$searchScore') {
@@ -144,7 +145,16 @@ const inspectObject = (object, q, path, level, isLast, isFirst, isObject, depth)
             str += ',\n';
         }
         else if (!def) {
-            str += inspectObject(v, q, key, level + 2, false, false, true, depth) + '';
+            if (Object.keys(object)[0] == 'count') {
+                // TODO: to flag the agg someway. This is ugly as hell!!!
+                str += picocolors.blue(v);
+                str += picocolors.italic(picocolors.dim(' count'));
+                str += ',\n';
+            }
+            else {
+                str +=
+                    inspectObject(v, q, key, level + 2, false, false, true, depth) + '';
+            }
         }
         else if ('__isPropDef' in def) {
             if (def.typeIndex === REFERENCES) {

package/dist/src/client/query/filter/toBuffer.js

@@ -33,7 +33,7 @@ const writeConditions = (result, k, offset, conditions) => {
 export const fillConditionsBuffer = (result, conditions, offset) => {
     let lastWritten = offset;
     let orJumpIndex = 0;
-    if (conditions.or) {
+    if (conditions.or && conditions.or.size != 0) {
         result[lastWritten] = META_OR_BRANCH;
         lastWritten++;
         orJumpIndex = lastWritten;
@@ -72,7 +72,7 @@ export const fillConditionsBuffer = (result, conditions, offset) => {
             result[sizeIndex + 1] = size >>> 8;
         });
     }
-    if (conditions.or) {
+    if (conditions.or && conditions.or.size != 0) {
         const size = fillConditionsBuffer(result, conditions.or, lastWritten);
         result[orJumpIndex] = size;
         result[orJumpIndex + 1] = size >>> 8;

package/dist/src/client/query/query.d.ts

@@ -8,4 +8,4 @@ export * from './filter/toBuffer.js';
 export * from './sort.js';
 export * from './debug.js';
 export * from './read/read.js';
-export * from './aggregationFn.js';
+export * from './aggregation.js';

package/dist/src/client/query/query.js

@@ -8,5 +8,5 @@ export * from './filter/toBuffer.js';
 export * from './sort.js';
 export * from './debug.js';
 export * from './read/read.js';
-export * from './aggregationFn.js';
+export * from './aggregation.js';
 //# sourceMappingURL=query.js.map

package/dist/src/client/query/queryDef.js

@@ -31,7 +31,6 @@ export const createQueryDef = (db, type, target, skipValidation) => {
         const t = target;
         const q = queryDef;
         q.props = t.ref.edges;
-        // q.reverseProps = t.ref.reverseEdges
         q.type = type;
         q.target = t;
         return q;

package/dist/src/client/query/read/read.js

@@ -1,7 +1,7 @@
 import { ALIAS, ALIASES, BINARY, BOOLEAN, ENUM, INT16, INT32, INT8, NUMBER, STRING, TEXT, TIMESTAMP, UINT16, UINT32, UINT8, VECTOR, JSON, CARDINALITY, } from '@based/schema/def';
 import { QueryDefType } from '../types.js';
 import { read, readUtf8 } from '../../string.js';
-import { readDoubleLE, readFloatLE, readInt16, readInt32, readUint16, readUint32, } from '../../bitWise.js';
+import { readDoubleLE, readFloatLE, readInt16, readInt32, readUint16, readUint32, } from '../../../utils.js';
 import { inverseLangMap } from '@based/schema';
 import { READ_EDGE, READ_ID, READ_REFERENCE, READ_REFERENCES, READ_AGGREGATION, CREATE_AGGREGATION, } from '../types.js';
 const addField = (p, value, item, defaultOnly = false, lang = 0) => {
@@ -132,7 +132,7 @@ const readMain = (q, result, offset, item) => {
     return i - offset;
 };
 const handleUndefinedProps = (id, q, item) => {
-    if (q.aggregation != -999 /* AggFn.NONE */) {
+    if (q.aggregation == 0 /* AggFlag.NONE */ || q.aggregation == null) {
         for (const k in q.include.propsRead) {
             if (q.include.propsRead[k] !== id) {
                 // Only relvant for seperate props
@@ -291,7 +291,8 @@ export const readAllFields = (q, result, offset, end, item, id) => {
         else if (index === CREATE_AGGREGATION) {
             i--;
             result[i] = READ_AGGREGATION;
-            q.aggregation = -999 /* AggFn.NONE */;
+            result[0] = result[0] + 1;
+            q.aggregation = 255 /* AggFlag.TEMP */;
             return i - offset - 4 - (q.search ? 4 : 0);
         }
         else if (index === READ_AGGREGATION) {
@@ -303,6 +304,9 @@ export const readAllFields = (q, result, offset, end, item, id) => {
             };
             const size = readUint32(result, i);
             addField(propAgg, readUint32(result, i + 4), item);
+            result[0] = result[0] - 1;
+            i--;
+            result[i] = CREATE_AGGREGATION;
             i += 4 + size + 4;
         }
         else if (index === 0) {
@@ -396,9 +400,10 @@ export const resultToObject = (q, result, end, offset = 0) => {
     while (i < end) {
         const id = readUint32(result, i);
         i += 4;
-        var item;
-        if (q.aggregation == -999 /* AggFn.NONE */) {
+        let item;
+        if (q.aggregation == 255 /* AggFlag.TEMP */) {
             item = {};
+            q.aggregation = 4 /* AggFlag.COUNT */;
         }
         else {
             item = {

package/dist/src/client/query/toBuffer.js

@@ -3,7 +3,7 @@ import { QueryDefType } from './types.js';
 import { includeToBuffer } from './include/toBuffer.js';
 import { filterToBuffer } from './query.js';
 import { searchToBuffer } from './search/index.js';
-import { createAggFnBuffer } from './aggregationFn.js';
+import { createAggFlagBuffer } from './aggregation.js';
 import { ENCODER } from '../../utils.js';
 const byteSize = (arr) => {
     return arr.reduce((a, b) => {
@@ -95,7 +95,7 @@ export function defToBuffer(db, def) {
                 sortSize = sort.byteLength;
             }
             let aggregation;
-            aggregation = createAggFnBuffer(def.aggregation);
+            aggregation = createAggFlagBuffer(def.aggregation);
             if (def.target.ids) {
                 // type 1
                 // 1: 4 + ids * 4 [ids len] [id,id,id]

package/dist/src/client/query/types.d.ts

@@ -92,7 +92,7 @@ export type QueryDefShared = {
     };
     references: Map<number, QueryDef>;
     edges?: QueryDef;
-    aggregation: AggFn;
+    aggregation: AggFlag;
 };
 export type QueryDefEdges = {
     type: QueryDefType.Edge;
@@ -120,7 +120,8 @@ export declare const READ_REFERENCES = 253;
 export declare const READ_REFERENCE = 254;
 export declare const CREATE_AGGREGATION = 250;
 export declare const READ_AGGREGATION = 251;
-export declare const enum AggFn {
+export declare const enum AggFlag {
+    NONE = 0,
     AVG = 1,
     CARDINALITY = 2,
     CONCAT = 3,// string aggregation, delimiter should be an argument
@@ -133,5 +134,5 @@ export declare const enum AggFn {
     STDDEV = 10,// population or sample should be optional parameters, default = sample
     SUM = 11,
     VARIANCE = 12,
-    NONE = -999
+    TEMP = 255
 }

package/dist/src/client/query/validation.js

@@ -1,5 +1,5 @@
 import picocolors from 'picocolors';
-import { ALIAS, BINARY, BOOLEAN, REFERENCE, REFERENCES, REVERSE_TYPE_INDEX_MAP, STRING, TEXT, TIMESTAMP, VECTOR, propIsNumerical, createEmptyDef, } from '@based/schema/def';
+import { ALIAS, BINARY, BOOLEAN, REFERENCE, REFERENCES, REVERSE_TYPE_INDEX_MAP, STRING, TEXT, TIMESTAMP, VECTOR, propIsNumerical, createEmptyDef, DEFAULT_MAP, } from '@based/schema/def';
 import { EQUAL, EXISTS, HAS, isNumerical, LIKE, operatorReverseMap, VECTOR_FNS, } from './filter/types.js';
 import { MAX_ID, MAX_ID_VALUE, MAX_IDS_PER_QUERY, MIN_ID_VALUE, } from './thresholds.js';
 import { displayTarget, safeStringify } from './display.js';
@@ -420,6 +420,7 @@ export const EMPTY_ALIAS_PROP_DEF = {
     separate: true,
     len: 0,
     start: 0,
+    default: DEFAULT_MAP[ALIAS],
     path: ['ERROR_ALIAS'],
 };
 export const ERROR_STRING = {
@@ -429,6 +430,7 @@ export const ERROR_STRING = {
     separate: true,
     len: 0,
     start: 0,
+    default: DEFAULT_MAP[STRING],
     path: ['ERROR_STRING'],
 };
 export const ERROR_VECTOR = {
@@ -438,6 +440,7 @@ export const ERROR_VECTOR = {
     separate: true,
     len: 0,
     start: 0,
+    default: DEFAULT_MAP[VECTOR],
     path: ['ERROR_VECTOR'],
 };
 export const EMPTY_SCHEMA_DEF = {
@@ -446,5 +449,6 @@ export const EMPTY_SCHEMA_DEF = {
     propNames: new Uint8Array([]),
     packed: new Uint8Array([]),
     idUint8: new Uint8Array([0, 0]),
+    mainEmptyAllZeroes: true,
 };
 //# sourceMappingURL=validation.js.map

package/dist/src/client/string.js

@@ -1,5 +1,5 @@
 import native from '../native.js';
-import { readUint32 } from './bitWise.js';
+import { readUint32 } from './../utils.js';
 import makeTmpBuffer from './tmpBuffer.js';
 import { DECODER, ENCODER } from '../utils.js';
 const { getUint8Array: getTmpBuffer } = makeTmpBuffer(4096); // the usual page size?

package/dist/src/index.d.ts

@@ -35,7 +35,10 @@ export declare class BasedDb {
     stop: DbServer['stop'];
     save: DbServer['save'];
     migrateSchema: DbServer['migrateSchema'];
-    isReady: DbClient['isModified'];
+    isModified: DbClient['isModified'];
+    schemaIsSet: DbClient['schemaIsSet'];
     destroy(): Promise<void>;
     wipe(): Promise<void>;
+    on: DbClient['on'];
+    off: DbClient['off'];
 }

package/dist/src/index.js

@@ -112,8 +112,11 @@ export class BasedDb {
     migrateSchema = function () {
         return this.server.migrateSchema.apply(this.server, arguments);
     };
-    isReady = function () {
-        return this.client.isReady.apply(this.client, arguments);
+    isModified = function () {
+        return this.client.isModified.apply(this.client, arguments);
+    };
+    schemaIsSet = function () {
+        return this.client.schemaIsSet.apply(this.client, arguments);
     };
     async destroy() {
         // Tmp fix: Gives node time to GC existing buffers else it can incorrectly re-asign to mem
@@ -131,5 +134,11 @@ export class BasedDb {
         this.#init(opts);
         await this.start({ clean: true });
     }
+    on = function () {
+        return this.client.on.apply(this.client, arguments);
+    };
+    off = function () {
+        return this.client.on.apply(this.client, arguments);
+    };
 }
 //# sourceMappingURL=index.js.map

package/dist/src/native.d.ts

@@ -4,7 +4,7 @@ declare const _default: {
     workerCtxInit: () => void;
     externalFromInt(address: BigInt): any;
     intFromExternal(external: any): BigInt;
-    modify: (data: Uint8Array, types: Uint8Array, dbCtx: any) => any;
+    modify: (data: Uint8Array, types: Uint8Array, dbCtx: any, dirtyBlocksOut: Float64Array) => any;
     getQueryBuf: (q: Uint8Array, dbCtx: any) => ArrayBuffer | null;
     start: (id: number) => any;
     stop: (dbCtx: any) => any;

package/dist/src/native.js

@@ -28,8 +28,8 @@ export default {
     intFromExternal(external) {
         return db.intFromExternal(external);
     },
-    modify: (data, types, dbCtx) => {
-        db.modify(data, types, dbCtx);
+    modify: (data, types, dbCtx, dirtyBlocksOut) => {
+        db.modify(data, types, dbCtx, dirtyBlocksOut);
     },
     getQueryBuf: (q, dbCtx) => {
         const x = db.getQueryBuf(dbCtx, q);

package/dist/src/server/csmt/tree.js

@@ -199,8 +199,8 @@ export function createTree(createHash) {
     return {
         getRoot: () => root,
         insert: (k, h, data = null) => {
-            if (!(h instanceof Uint8Array)) { // TODO can we extract the name somehow from Hash?
-                throw new TypeError('`h` must be a Uint8Array'); // TODO can we extract the name somehow from Hash?
+            if (!(h instanceof Uint8Array)) {
+                throw new TypeError('`h` must be a Uint8Array');
             }
             const newLeaf = createLeaf(k, h, data);
             root = root ? insert(root, newLeaf) : newLeaf;

package/dist/src/server/index.d.ts

@@ -4,6 +4,8 @@ import { createTree } from './csmt/index.js';
 import { Worker, MessagePort } from 'node:worker_threads';
 import { TransformFns } from './migrate/index.js';
 import exitHook from 'exit-hook';
+export declare const SCHEMA_FILE = "schema.json";
+export declare const WRITELOG_FILE = "writelog.json";
 declare class SortIndex {
     constructor(buf: Uint8Array, dbCtxExternal: any);
     buf: Uint8Array;
@@ -23,7 +25,7 @@ export declare class DbWorker {
 type OnSchemaChange = (schema: StrictSchema) => void;
 export declare class DbServer {
     #private;
-    modifyBuf: SharedArrayBuffer;
+    modifyDirtyRanges: Float64Array;
     dbCtxExternal: any;
     schema: StrictSchema & {
         lastId: number;
@@ -57,7 +59,9 @@ export declare class DbServer {
         clean?: boolean;
         hosted?: boolean;
     }): Promise<void>;
-    save(): void | Promise<void>;
+    save(opts?: {
+        forceFullDump?: boolean;
+    }): Promise<void>;
     createCsmtHashFun: () => {
         update: (buf: Uint8Array) => any;
         digest: (encoding?: "hex") => Uint8Array | string;