@based/db 0.0.69 → 0.0.71

package/dist/lib/darwin_aarch64/include/selva/hll.h

@@ -19,7 +19,7 @@ uint8_t *hll_count(struct selva_string *hllss);
 SELVA_EXPORT
 void hll_array_union(struct selva_string *res, struct selva_string *hll_array, size_t count);
 SELVA_EXPORT
-void hll_union(char*, size_t, const char*, size_t);
+void hll_union(struct selva_string *, struct selva_string *);
 
 // size_t actual_cols[13] = { 79, 159, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200 };
 // float raw_estimate_data[13][200] =

package/dist/lib/linux_aarch64/include/selva/hll.h

@@ -19,7 +19,7 @@ uint8_t *hll_count(struct selva_string *hllss);
 SELVA_EXPORT
 void hll_array_union(struct selva_string *res, struct selva_string *hll_array, size_t count);
 SELVA_EXPORT
-void hll_union(char*, size_t, const char*, size_t);
+void hll_union(struct selva_string *, struct selva_string *);
 
 // size_t actual_cols[13] = { 79, 159, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200 };
 // float raw_estimate_data[13][200] =

package/dist/lib/linux_x86_64/include/selva/hll.h

@@ -19,7 +19,7 @@ uint8_t *hll_count(struct selva_string *hllss);
 SELVA_EXPORT
 void hll_array_union(struct selva_string *res, struct selva_string *hll_array, size_t count);
 SELVA_EXPORT
-void hll_union(char*, size_t, const char*, size_t);
+void hll_union(struct selva_string *, struct selva_string *);
 
 // size_t actual_cols[13] = { 79, 159, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200 };
 // float raw_estimate_data[13][200] =

package/dist/src/client/modify/create.js

@@ -11,6 +11,7 @@ import { writeString } from './string.js';
 import { writeText } from './text.js';
 import { writeJson } from './json.js';
 import { writeAlias } from './alias.js';
+import { getByPath } from '@saulx/utils';
 const appendCreate = (ctx, schema, obj, res, unsafe) => {
     const len = ctx.len;
     let err = modify(ctx, res, obj, schema, CREATE, schema.tree, true, unsafe);
@@ -26,6 +27,9 @@ const appendCreate = (ctx, schema, obj, res, unsafe) => {
     if (schema.createTs) {
         const createTs = Date.now();
         for (const prop of schema.createTs) {
+            if (getByPath(obj, prop.path) !== undefined) {
+                continue;
+            }
             if (ctx.lastMain === -1) {
                 let mainLenU32 = schema.mainLen;
                 setCursor(ctx, schema, prop.prop, MICRO_BUFFER, res.tmpId, CREATE);

package/dist/src/client/modify/references/edge.js

@@ -7,6 +7,7 @@ import { ModifyError, ModifyState } from '../ModifyRes.js';
 import { DECREMENT, INCREMENT, RANGE_ERR, UPDATE, UPDATE_PARTIAL, } from '../types.js';
 import { appendFixedValue } from '../fixed.js';
 import { appendEdgeRefs } from './appendEdgeRefs.js';
+import { writeUint32 } from '@saulx/utils';
 function valueOperation(value) {
     if (typeof value === 'object' && value !== null) {
         if (value.increment > 0) {
@@ -24,6 +25,15 @@ export function writeEdges(t, ref, ctx) {
     let mainFields;
     let mainSize = 0;
     let hasIncr = false;
+    if (t.hasDefaultEdges) {
+        for (const key in t.edges) {
+            if (t.edges[key].separate === true &&
+                (!(key in ref) || ref[key] === undefined)) {
+                const edge = t.edges[key];
+                ref[key] = edge.default;
+            }
+        }
+    }
     for (const key in ref) {
         if (key === 'id' || key === '$index') {
             continue;
@@ -224,7 +234,9 @@ export function writeEdges(t, ref, ctx) {
             }
         }
     }
-    if (mainFields) {
+    // double check if has default edges has mainfields - add extra thing
+    if (mainFields || t.hasDefaultEdges) {
+        // add all
         // Single field in main buffer can immediately setup the main buffer
         if (!hasIncr && mainSize === t.edgeMainLen) {
             /*
@@ -245,10 +257,8 @@ export function writeEdges(t, ref, ctx) {
             ctx.buf[ctx.len++] = 0;
             ctx.buf[ctx.len++] = MICRO_BUFFER;
             let sizeU32 = mainSize;
-            ctx.buf[ctx.len++] = sizeU32;
-            ctx.buf[ctx.len++] = sizeU32 >>>= 8;
-            ctx.buf[ctx.len++] = sizeU32 >>>= 8;
-            ctx.buf[ctx.len++] = sizeU32 >>>= 8;
+            writeUint32(ctx.buf, sizeU32, ctx.len);
+            ctx.len += 4;
             for (let i = 0; i < mainFields.length; i += 3) {
                 const edge = mainFields[i];
                 const err = appendFixedValue(ctx, mainFields[i + 1], edge, UPDATE);
@@ -258,6 +268,9 @@ export function writeEdges(t, ref, ctx) {
             }
         }
         else {
+            if (!mainFields) {
+                mainFields = [];
+            }
             /*
             Partial main update:
       
@@ -293,7 +306,8 @@ export function writeEdges(t, ref, ctx) {
             const sIndex = ctx.len;
             ctx.len += mainFieldsStartSize;
             // Add zeroes
-            ctx.buf.fill(0, ctx.len, ctx.len + t.edgeMainLen);
+            ctx.buf.set(t.edgeMainEmpty, ctx.len);
+            // ctx.buf.fill(0, ctx.len, ctx.len + t.edgeMainLen)
             // Keep track of written bytes from append fixed
             let startMain = ctx.len;
             for (let i = 0; i < mainFields.length; i += 3) {

package/dist/src/client/modify/references/reference.js

@@ -24,7 +24,7 @@ function writeRef(id, ctx, schema, def, parentId, modifyOp, hasEdges, isTmpId) {
     ctx.buf[ctx.len++] = id >>>= 8;
     ctx.buf[ctx.len++] = id >>>= 8;
 }
-function singleReferenceEdges(ref, ctx, schema, def, parentId, modifyOp) {
+function singleReferenceEdges(ref, ctx, schema, def, parentId, modifyOp, processDefaultEdges) {
     let id = ref.id;
     let isTmpId;
     if (typeof id !== 'number') {
@@ -77,23 +77,35 @@ export function writeReference(value, ctx, schema, def, res, modifyOp) {
         }
         setCursor(ctx, schema, def.prop, def.typeIndex, res.tmpId, modifyOp);
         ctx.buf[ctx.len++] = DELETE;
+        return;
     }
-    else if (typeof value === 'number') {
-        return writeRef(value, ctx, schema, def, res.tmpId, modifyOp, false, false);
+    const processDefaultEdges = def.hasDefaultEdges;
+    if (typeof value === 'number') {
+        if (processDefaultEdges) {
+            value = { id: value };
+        }
+        else {
+            return writeRef(value, ctx, schema, def, res.tmpId, modifyOp, false, false);
+        }
     }
-    else if (value instanceof ModifyState) {
+    if (value instanceof ModifyState) {
         if (value.error) {
             return value.error;
         }
         const id = value.getId();
-        if (id) {
-            return writeRef(id, ctx, schema, def, res.tmpId, modifyOp, false, false);
+        if (processDefaultEdges) {
+            value = { id: id || value.tmpId };
+        }
+        else {
+            if (id) {
+                return writeRef(id, ctx, schema, def, res.tmpId, modifyOp, false, false);
+            }
+            return writeRef(value.tmpId, ctx, schema, def, res.tmpId, modifyOp, false, true);
         }
-        return writeRef(value.tmpId, ctx, schema, def, res.tmpId, modifyOp, false, true);
     }
-    else if (typeof value === 'object' && value !== null) {
+    if (typeof value === 'object') {
         if (def.edges) {
-            return singleReferenceEdges(value, ctx, schema, def, res.tmpId, modifyOp);
+            return singleReferenceEdges(value, ctx, schema, def, res.tmpId, modifyOp, processDefaultEdges);
         }
         else if (typeof value.id === 'number') {
             return writeRef(value.id, ctx, schema, def, res.tmpId, modifyOp, false, false);

package/dist/src/client/modify/references/references.js

@@ -130,10 +130,18 @@ function appendRefs(def, ctx, modifyOp, refs, op, remaining) {
     ctx.buf[ctx.len++] = remaining >>>= 8;
     ctx.buf[ctx.len++] = remaining >>>= 8;
     for (; i < refs.length; i++) {
-        const ref = refs[i];
+        let ref = refs[i];
         let id;
         let index;
         let isTmpId;
+        if (def.hasDefaultEdges) {
+            if (typeof ref !== 'object') {
+                ref = { id: ref };
+            }
+            else if (ref instanceof ModifyState) {
+                ref = { id: ref };
+            }
+        }
         if (typeof ref === 'object') {
             if (ref instanceof ModifyState) {
                 if (ref.error) {

package/dist/src/client/modify/update.js

@@ -5,6 +5,7 @@ import { modify } from './modify.js';
 import { ModifyState } from './ModifyRes.js';
 import { RANGE_ERR, UPDATE } from './types.js';
 import { appendFixedValue } from './fixed.js';
+import { getByPath } from '@saulx/utils';
 const appendUpdate = (ctx, def, obj, res, overwrite) => {
     const err = modify(ctx, res, obj, def, UPDATE, def.tree, overwrite);
     if (err) {
@@ -13,6 +14,9 @@ const appendUpdate = (ctx, def, obj, res, overwrite) => {
     if (def.updateTs) {
         const updateTs = Date.now();
         for (const prop of def.updateTs) {
+            if (getByPath(obj, prop.path) !== undefined) {
+                continue;
+            }
             if (ctx.mergeMain) {
                 ctx.mergeMain.push(prop, updateTs);
                 ctx.mergeMainSize += prop.len + 4;

package/dist/src/client/query/BasedDbQuery.d.ts

@@ -25,7 +25,7 @@ export declare class QueryBranch<T> {
     groupBy(field: string): T;
     count(field?: string): T;
     sum(...fields: (string | string[])[]): T;
-    cardinality(...fields: (string | string[])[]): T;
+    cardinality(field: string): T;
     stddev(...fields: (string | string[])[]): T;
     var(...fields: (string | string[])[]): T;
     avg(...fields: (string | string[])[]): T;

package/dist/src/client/query/BasedDbQuery.js

@@ -173,18 +173,18 @@ export class QueryBranch {
         // @ts-ignore
         return this;
     }
-    cardinality(...fields) {
-        if (fields.length === 0) {
+    cardinality(field) {
+        if (field.length === 0) {
             throw new Error('Empty cardinality() called');
         }
         if (this.queryCommands) {
             this.queryCommands.push({
                 method: 'cardinality',
-                args: fields,
+                args: [field],
             });
         }
         else {
-            addAggregate(AggregateType.CARDINALITY, this.def, fields);
+            addAggregate(AggregateType.CARDINALITY, this.def, [field]);
         }
         // @ts-ignore
         return this;

package/dist/src/client/query/aggregates/types.d.ts

@@ -11,7 +11,7 @@ export declare enum AggregateType {
 export declare const enum AccumulatorSize {
     SUM = 8,
     COUNT = 4,
-    CARDINALITY = 254,// TODO: accordinly to sparse or dense modes
+    CARDINALITY = 4,
     STDDEV = 24,// count (u64) + sum (f64) + sum_sq (f64) = 8 + 8 + 8 = 24
     AVERAGE = 16,// count (u64) + sum (f64) = 16
     VARIANCE = 24,// count (u64) + sum (f64) + sum_sq (f64) = 8 + 8 + 8 = 24

package/dist/src/client/query/aggregates/types.js

@@ -12,7 +12,7 @@ export var AggregateType;
 export const aggregateTypeMap = new Map([
     [
         AggregateType.CARDINALITY,
-        { resultsSize: 4, accumulatorSize: 254 /* AccumulatorSize.CARDINALITY */ },
+        { resultsSize: 4, accumulatorSize: 4 /* AccumulatorSize.CARDINALITY */ },
     ],
     [
         AggregateType.COUNT,

package/dist/src/client/query/validation.js

@@ -424,7 +424,7 @@ export const handleErrors = (def) => {
         }
         const err = new Error(`Query\n`);
         err.message = name;
-        err.stack = '';
+        err.stack = name;
         throw err;
     }
 };

package/dist/src/server/migrate/index.js

@@ -7,6 +7,7 @@ import { foreachDirtyBlock } from '../blocks.js';
 import { fileURLToPath } from 'url';
 import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from '../schema.js';
 import { setToAwake, waitUntilSleeping } from './utils.js';
+import { serialize } from '@based/schema';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const workerPath = join(__dirname, 'worker.js');
@@ -74,8 +75,14 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
             isDbMigrateWorker: true,
             from: fromAddress,
             to: toAddress,
-            fromSchema,
-            toSchema,
+            fromSchema: serialize(fromSchema, {
+                stripMetaInformation: true,
+                stripTransform: true,
+            }),
+            toSchema: serialize(toSchema, {
+                stripMetaInformation: true,
+                stripTransform: true,
+            }),
             channel: port2,
             workerState,
             transformFns,
@@ -83,9 +90,12 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
         transferList: [port2],
     });
     // handle?
-    worker.on('error', (err) => {
-        killed = true;
-        console.error(`Error in migration ${err.message}`);
+    const errorPromise = new Promise((resolve) => {
+        worker.once('error', (err) => {
+            killed = true;
+            console.error(`Error in migration ${err.message}, aborting migration`);
+            resolve();
+        });
     });
     // Block handling
     let i = 0;
@@ -107,7 +117,7 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
         const leafData = rangesToMigrate[i++];
         port1.postMessage(leafData);
         setToAwake(workerState, true);
-        await waitUntilSleeping(workerState);
+        await Promise.race([errorPromise, waitUntilSleeping(workerState)]);
         // exec queued modifies
         server.activeReaders--;
         server.onQueryEnd();
@@ -156,7 +166,11 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
         return;
     }
     native.membarSyncRead();
-    await save(server, { forceFullDump: true, skipDirtyCheck: true, skipMigrationCheck: true });
+    await save(server, {
+        forceFullDump: true,
+        skipDirtyCheck: true,
+        skipMigrationCheck: true,
+    });
     await writeSchemaFile(server, toSchema);
     server.migrating = 0;
     process.nextTick(() => server.emit('schema', server.schema));

package/dist/src/server/migrate/worker.js

@@ -6,6 +6,7 @@ import { isTypedArray } from 'node:util/types';
 import { setSchemaOnServer } from '../schema.js';
 import { setToSleep } from './utils.js';
 import { setLocalClientSchema } from '../../client/setLocalClientSchema.js';
+import { deSerialize } from '@based/schema';
 if (isMainThread) {
     console.warn('running worker.ts in mainthread');
 }
@@ -41,80 +42,87 @@ else if (workerData?.isDbMigrateWorker) {
     };
     fromDb.server.dbCtxExternal = fromCtx;
     toDb.server.dbCtxExternal = toCtx;
-    setSchemaOnServer(fromDb.server, fromSchema);
-    setSchemaOnServer(toDb.server, toSchema);
+    setSchemaOnServer(fromDb.server, deSerialize(fromSchema));
+    setSchemaOnServer(toDb.server, deSerialize(toSchema));
     setLocalClientSchema(fromDb.client, fromDb.server.schema);
     setLocalClientSchema(toDb.client, toDb.server.schema);
-    const map = {};
-    for (const type in fromDb.server.schemaTypesParsed) {
-        const { id, props } = fromDb.server.schemaTypesParsed[type];
-        const include = Object.keys(props);
-        let i = include.length;
-        while (i--) {
-            const path = include[i];
-            if (props[path].typeIndex === REFERENCE ||
-                props[path].typeIndex === REFERENCES) {
-                include[i] = `${path}.id`;
-                if (props[path].edges) {
-                    for (const key in props[path].edges) {
-                        const prop = props[path].edges[key];
-                        if (prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) {
-                            include.push(`${path}.${key}.id`);
-                        }
-                        else {
-                            include.push(`${path}.${key}`);
+    try {
+        const map = {};
+        for (const type in fromDb.server.schemaTypesParsed) {
+            const { id, props } = fromDb.server.schemaTypesParsed[type];
+            const include = Object.keys(props);
+            let i = include.length;
+            while (i--) {
+                const path = include[i];
+                if (props[path].typeIndex === REFERENCE ||
+                    props[path].typeIndex === REFERENCES) {
+                    include[i] = `${path}.id`;
+                    if (props[path].edges) {
+                        for (const key in props[path].edges) {
+                            const prop = props[path].edges[key];
+                            if (prop.typeIndex === REFERENCE ||
+                                prop.typeIndex === REFERENCES) {
+                                include.push(`${path}.${key}.id`);
+                            }
+                            else {
+                                include.push(`${path}.${key}`);
+                            }
                         }
                     }
                 }
             }
+            map[id] = { type, include };
         }
-        map[id] = { type, include };
-    }
-    for (const type in transformFns) {
-        const fnOrNull = transformFns[type];
-        transformFns[type] = eval(`(${fnOrNull})`);
-    }
-    while (true) {
-        let msg;
-        while ((msg = receiveMessageOnPort(channel))) {
-            const leafData = msg.message;
-            const { type, include } = map[leafData.typeId];
-            const typeTransformFn = transformFns[type];
-            if (typeTransformFn) {
-                const nodes = fromDb
-                    .query(type)
-                    .include(include)
-                    .range(leafData.start - 1, leafData.end)
-                    ._getSync(fromCtx);
-                for (const node of nodes) {
-                    const res = typeTransformFn(node);
-                    if (res === null) {
-                        continue;
-                    }
-                    if (Array.isArray(res)) {
-                        toDb.create(res[0], res[1] || node, { unsafe: true });
-                    }
-                    else {
-                        toDb.create(type, res || node, { unsafe: true });
+        for (const type in transformFns) {
+            const fnOrNull = transformFns[type];
+            transformFns[type] = eval(`(${fnOrNull})`);
+        }
+        while (true) {
+            let msg;
+            while ((msg = receiveMessageOnPort(channel))) {
+                const leafData = msg.message;
+                const { type, include } = map[leafData.typeId];
+                const typeTransformFn = transformFns[type];
+                if (typeTransformFn) {
+                    const nodes = fromDb
+                        .query(type)
+                        .include(include)
+                        .range(leafData.start - 1, leafData.end)
+                        ._getSync(fromCtx);
+                    for (const node of nodes) {
+                        const res = typeTransformFn(node);
+                        if (res === null) {
+                            continue;
+                        }
+                        if (Array.isArray(res)) {
+                            toDb.create(res[0], res[1] || node, { unsafe: true });
+                        }
+                        else {
+                            toDb.create(type, res || node, { unsafe: true });
+                        }
                     }
                 }
-            }
-            else if (type in toDb.server.schemaTypesParsed) {
-                const nodes = fromDb
-                    .query(type)
-                    .include(include)
-                    .range(leafData.start - 1, leafData.end)
-                    ._getSync(fromCtx);
-                for (const node of nodes) {
-                    toDb.create(type, node, { unsafe: true });
+                else if (type in toDb.server.schemaTypesParsed) {
+                    const nodes = fromDb
+                        .query(type)
+                        .include(include)
+                        .range(leafData.start - 1, leafData.end)
+                        ._getSync(fromCtx);
+                    for (const node of nodes) {
+                        toDb.create(type, node, { unsafe: true });
+                    }
                 }
             }
+            await toDb.drain();
+            native.membarSyncWrite();
+            // WE ARE ONLY GOING TO SEND { type: lastNodeId }
+            channel.postMessage(cp(toDb.server.schemaTypesParsed));
+            setToSleep(workerState);
         }
-        await toDb.drain();
-        native.membarSyncWrite();
-        // WE ARE ONLY GOING TO SEND { type: lastNodeId }
-        channel.postMessage(cp(toDb.server.schemaTypesParsed));
-        setToSleep(workerState);
+    }
+    catch (e) {
+        console.error(e);
+        throw e;
     }
 }
 else {

package/dist/src/server/save.js

@@ -2,17 +2,18 @@ import native from '../native.js';
 import { isMainThread } from 'node:worker_threads';
 import { writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { VerifTree, destructureTreeKey, } from './tree.js';
+import { VerifTree, destructureTreeKey } from './tree.js';
 import { saveBlock, foreachBlock, foreachDirtyBlock, saveBlocks, } from './blocks.js';
 import { writeFileSync } from 'node:fs';
 import { bufToHex } from '@saulx/utils';
 import { COMMON_SDB_FILE, WRITELOG_FILE } from '../types.js';
 function hasPartialTypes(db) {
-    let res = false;
-    for (let id in db.schemaTypesParsedById) {
-        res = res || db.schemaTypesParsedById[id].partial;
+    for (const id in db.schemaTypesParsedById) {
+        if (db.schemaTypesParsedById[id].partial) {
+            return true;
+        }
     }
-    return res;
+    return false;
 }
 function inhibitSave(db, { skipDirtyCheck, forceFullDump, skipMigrationCheck }) {
     // RFE isMainThread needed??
@@ -138,7 +139,7 @@ export async function save(db, opts = {}) {
                 blocks.push({
                     filepath,
                     typeId,
-                    start
+                    start,
                 });
             });
         }

package/dist/src/server/schema.js

@@ -15,12 +15,14 @@ export const setSchemaOnServer = (server, schema) => {
     server.schemaTypesParsedById = schemaTypesParsedById;
 };
 export const writeSchemaFile = async (server, schema) => {
-    const schemaFilePath = join(server.fileSystemPath, SCHEMA_FILE);
-    try {
-        await writeFile(schemaFilePath, serialize(schema));
-    }
-    catch (err) {
-        throw new Error(`Error writing schema to a file path ${schemaFilePath}}`);
+    if (server.fileSystemPath) {
+        const schemaFilePath = join(server.fileSystemPath, SCHEMA_FILE);
+        try {
+            await writeFile(schemaFilePath, serialize(schema));
+        }
+        catch (err) {
+            throw new Error(`Error writing schema to a file path ${schemaFilePath}}`);
+        }
     }
 };
 export const setNativeSchema = (server, schema) => {

package/dist/src/types.d.ts

@@ -3,7 +3,7 @@ export declare const SCHEMA_FILE = "schema.bin";
 export declare const WRITELOG_FILE = "writelog.json";
 export declare const COMMON_SDB_FILE = "common.sdb";
 export type BasedDbOpts = {
-    path: string;
+    path: string | null;
     maxModifySize?: number;
     debug?: boolean | 'server' | 'client';
     saveIntervalInSeconds?: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.69",
+  "version": "0.0.71",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",
@@ -39,7 +39,7 @@
     "basedDbNative.cjs"
   ],
   "dependencies": {
-    "@based/schema": "5.0.0-alpha.25",
+    "@based/schema": "5.0.0-alpha.27",
     "@saulx/hash": "^3.0.0",
     "@saulx/utils": "^6.7.2",
     "exit-hook": "^4.0.0",
@@ -56,6 +56,7 @@
     "axios": "^1.7.9",
     "rimraf": "^3.0.2",
     "tar": "^7.4.3",
-    "typescript": "^5.6.3"
+    "typescript": "^5.6.3",
+    "fs-extra": "^11.1.1"
   }
 }