@based/db 0.1.1 → 0.1.3-alpha.1

package/dist/lib/darwin_aarch64/include/selva/hll.h

@@ -13,6 +13,8 @@
 SELVA_EXPORT
 void hll_init(struct selva_string *hllss, uint8_t precision, bool is_sparse);
 SELVA_EXPORT
+void hll_init_like(struct selva_string *hlla, struct selva_string *hllb);
+SELVA_EXPORT
 void hll_add(struct selva_string *hllss, uint64_t element);
 SELVA_EXPORT
 uint8_t *hll_count(struct selva_string *hllss);

package/dist/lib/darwin_aarch64/include/selva/membar.h

@@ -4,6 +4,15 @@
  */
 #include "selva/_export.h"
 
+/*
+ * When you run threads on different cores on Graviton it doesn't guarantee that
+ * a write in one thread is seen in the other one immediately.
+ * https://github.com/aws/aws-graviton-getting-started/blob/main/optimizing.md#ordering-issues
+ *
+ * This is also true on the Apple M arch:
+ * https://developer.apple.com/documentation/apple-silicon/addressing-architectural-differences-in-your-macos-code
+ */
+
 /**
  * Read memory barrier.
  * Call this function before read in case another thread has potentially

package/dist/lib/linux_aarch64/include/selva/hll.h

@@ -13,6 +13,8 @@
 SELVA_EXPORT
 void hll_init(struct selva_string *hllss, uint8_t precision, bool is_sparse);
 SELVA_EXPORT
+void hll_init_like(struct selva_string *hlla, struct selva_string *hllb);
+SELVA_EXPORT
 void hll_add(struct selva_string *hllss, uint64_t element);
 SELVA_EXPORT
 uint8_t *hll_count(struct selva_string *hllss);

package/dist/lib/linux_aarch64/include/selva/membar.h

@@ -4,6 +4,15 @@
  */
 #include "selva/_export.h"
 
+/*
+ * When you run threads on different cores on Graviton it doesn't guarantee that
+ * a write in one thread is seen in the other one immediately.
+ * https://github.com/aws/aws-graviton-getting-started/blob/main/optimizing.md#ordering-issues
+ *
+ * This is also true on the Apple M arch:
+ * https://developer.apple.com/documentation/apple-silicon/addressing-architectural-differences-in-your-macos-code
+ */
+
 /**
  * Read memory barrier.
  * Call this function before read in case another thread has potentially

package/dist/lib/linux_x86_64/include/selva/hll.h

@@ -13,6 +13,8 @@
 SELVA_EXPORT
 void hll_init(struct selva_string *hllss, uint8_t precision, bool is_sparse);
 SELVA_EXPORT
+void hll_init_like(struct selva_string *hlla, struct selva_string *hllb);
+SELVA_EXPORT
 void hll_add(struct selva_string *hllss, uint64_t element);
 SELVA_EXPORT
 uint8_t *hll_count(struct selva_string *hllss);

package/dist/lib/linux_x86_64/include/selva/membar.h

@@ -4,6 +4,15 @@
  */
 #include "selva/_export.h"
 
+/*
+ * When you run threads on different cores on Graviton it doesn't guarantee that
+ * a write in one thread is seen in the other one immediately.
+ * https://github.com/aws/aws-graviton-getting-started/blob/main/optimizing.md#ordering-issues
+ *
+ * This is also true on the Apple M arch:
+ * https://developer.apple.com/documentation/apple-silicon/addressing-architectural-differences-in-your-macos-code
+ */
+
 /**
  * Read memory barrier.
  * Call this function before read in case another thread has potentially

package/dist/src/client/index.d.ts

@@ -1,9 +1,8 @@
-import { MigrateFns, Schema } from '@based/schema';
+import { MigrateFns, Schema, SchemaChecksum } from '@based/schema';
 import { BasedDbQuery, QueryByAliasObj } from './query/BasedDbQuery.js';
 import { SubStore } from './query/subscription/index.js';
 import { DbShared } from '../shared/DbBase.js';
 import { DbClientHooks } from '../hooks.js';
-import { SchemaChecksum } from '../schema.js';
 import { ModifyOpts } from './modify/types.js';
 import { Ctx } from './modify/Ctx.js';
 type DbClientOpts = {

package/dist/src/client/modify/Ctx.js

@@ -3,7 +3,7 @@ export class Ctx {
     constructor(schemaChecksum, array) {
         this.array = array;
         this.max = array.buffer.maxByteLength - 4; // dataLen
-        this.size = array.buffer.byteLength - this.max;
+        this.size = array.buffer.byteLength - 4;
         writeUint64(array, schemaChecksum, 0);
     }
     id;

package/dist/src/client/modify/cursor.d.ts

@@ -5,6 +5,6 @@ export declare const PROP_CURSOR_SIZE = 3;
 export declare const NODE_CURSOR_SIZE = 5;
 export declare const FULL_CURSOR_SIZE: number;
 export declare const writeTypeCursor: (ctx: Ctx) => void;
-export declare const writePropCursor: (ctx: Ctx, def: PropDef) => void;
+export declare const writePropCursor: (ctx: Ctx, def: PropDef, typeIndex?: import("@based/schema/prop-types").TypeIndex) => void;
 export declare const writeMainCursor: (ctx: Ctx) => void;
 export declare const writeNodeCursor: (ctx: Ctx) => void;

package/dist/src/client/modify/cursor.js

@@ -16,11 +16,11 @@ export const writeTypeCursor = (ctx) => {
         ctx.cursor.id = null;
     }
 };
-export const writePropCursor = (ctx, def) => {
+export const writePropCursor = (ctx, def, typeIndex = def.typeIndex) => {
     if (def.prop !== ctx.cursor.prop) {
         ctx.array[ctx.index] = SWITCH_FIELD;
         ctx.array[ctx.index + 1] = def.prop;
-        ctx.array[ctx.index + 2] = def.typeIndex;
+        ctx.array[ctx.index + 2] = typeIndex;
         ctx.index += 3;
         ctx.cursor.prop = def.prop;
     }

package/dist/src/client/modify/error.js

@@ -47,8 +47,6 @@ export const handleError = (db, ctx, fn, args, e) => {
         return fn.apply(null, args);
     }
     if (typeof e.then === 'function') {
-        if (e.id)
-            process.exit();
         return e.then((id) => {
             if (!(e instanceof Tmp)) {
                 e.id = id;

package/dist/src/client/modify/props/binary.js

@@ -1,6 +1,7 @@
 import { writeU16, writeU32, writeU8, writeU8Array } from '../uint.js';
 import { PROP_CURSOR_SIZE, writePropCursor } from '../cursor.js';
 import { validate } from '../validate.js';
+import { BINARY } from '@based/schema/def';
 import native from '../../../native.js';
 import { reserve } from '../resize.js';
 import { deleteProp } from './delete.js';
@@ -45,7 +46,7 @@ export const writeBinary = (ctx, def, val, validated) => {
     }
     const size = buf.byteLength + 6;
     reserve(ctx, PROP_CURSOR_SIZE + size + 11);
-    writePropCursor(ctx, def);
+    writePropCursor(ctx, def, BINARY);
     writeU8(ctx, ctx.operation);
     writeBinaryRaw(ctx, buf);
 };

package/dist/src/client/modify/props/cardinality.js

@@ -6,6 +6,7 @@ import { ENCODER } from '@based/utils';
 import { reserve } from '../resize.js';
 import { PROP_CURSOR_SIZE, writePropCursor } from '../cursor.js';
 import { CREATE } from '../types.js';
+import { writeBinary } from './binary.js';
 export const writeCardinalityRaw = (ctx, def, val, sizeFixBecauseEdgeIsDifferent = val.length) => {
     writeU32(ctx, sizeFixBecauseEdgeIsDifferent);
     for (const item of val) {
@@ -27,6 +28,10 @@ export const writeCardinality = (ctx, def, val) => {
         deleteProp(ctx, def);
         return;
     }
+    if (val instanceof Uint8Array && val.byteLength !== 8) {
+        writeBinary(ctx, def, val, true);
+        return;
+    }
     if (!Array.isArray(val)) {
         val = [val];
     }
@@ -37,6 +42,8 @@ export const writeCardinality = (ctx, def, val) => {
     reserve(ctx, PROP_CURSOR_SIZE + size + 1);
     writePropCursor(ctx, def);
     writeU8(ctx, ctx.operation);
+    writeU8(ctx, def.cardinalityMode);
+    writeU8(ctx, def.cardinalityPrecision);
     writeCardinalityRaw(ctx, def, val);
     if (ctx.operation === CREATE) {
         ctx.schema.separateSort.bufferTmp[def.prop] = 2;

package/dist/src/client/modify/props/object.d.ts

@@ -1,3 +1,5 @@
 import { SchemaTypeDef } from '@based/schema/def';
 import { Ctx } from '../Ctx.js';
+export declare const writeObjectSafe: (ctx: Ctx, tree: SchemaTypeDef["tree"], obj: Record<string, any>) => void;
+export declare const writeObjectUnsafe: (ctx: Ctx, tree: SchemaTypeDef["tree"], obj: Record<string, any>) => void;
 export declare const writeObject: (ctx: Ctx, tree: SchemaTypeDef["tree"], obj: Record<string, any>) => void;

package/dist/src/client/modify/props/object.js

@@ -2,7 +2,21 @@ import { isPropDef } from '@based/schema/def';
 import { writeSeparate } from './separate.js';
 import { writeMainValue } from './main.js';
 import { writeIncrement } from './increment.js';
-export const writeObject = (ctx, tree, obj) => {
+const writeProp = (ctx, def, val) => {
+    if (def.separate) {
+        writeSeparate(ctx, def, val);
+    }
+    else if (ctx.overwrite) {
+        writeMainValue(ctx, def, val);
+    }
+    else if (typeof val === 'object' && val !== null) {
+        writeIncrement(ctx, def, val);
+    }
+    else {
+        ctx.main.set(def, val);
+    }
+};
+export const writeObjectSafe = (ctx, tree, obj) => {
     for (const key in obj) {
         const val = obj[key];
         if (val === undefined) {
@@ -10,28 +24,47 @@ export const writeObject = (ctx, tree, obj) => {
         }
         const def = tree[key];
         if (def === undefined) {
-            if (ctx.unsafe) {
-                continue;
-            }
-            throw [tree, key];
+            throw [def, val];
         }
-        if (!isPropDef(def)) {
-            writeObject(ctx, def, val);
-            continue;
+        if (isPropDef(def)) {
+            writeProp(ctx, def, val);
         }
-        if (def.separate) {
-            writeSeparate(ctx, def, val);
-            continue;
+        else {
+            writeObjectSafe(ctx, def, val);
         }
-        if (ctx.overwrite) {
-            writeMainValue(ctx, def, val);
+    }
+};
+export const writeObjectUnsafe = (ctx, tree, obj) => {
+    for (const key in obj) {
+        const def = tree[key];
+        const val = obj[key];
+        if (def === undefined || val === undefined) {
             continue;
         }
-        if (typeof val === 'object' && val !== null) {
-            writeIncrement(ctx, def, val);
-            continue;
+        if (isPropDef(def)) {
+            const index = ctx.index;
+            try {
+                writeProp(ctx, def, val);
+            }
+            catch (e) {
+                if (Array.isArray(e)) {
+                    ctx.index = index;
+                    continue;
+                }
+                throw e;
+            }
         }
-        ctx.main.set(def, val);
+        else {
+            writeObjectUnsafe(ctx, def, val);
+        }
+    }
+};
+export const writeObject = (ctx, tree, obj) => {
+    if (ctx.unsafe) {
+        writeObjectUnsafe(ctx, tree, obj);
+    }
+    else {
+        writeObjectSafe(ctx, tree, obj);
     }
 };
 //# sourceMappingURL=object.js.map

package/dist/src/client/query/BasedQueryResponse.js

@@ -3,11 +3,8 @@ import picocolors from 'picocolors';
 import { debug } from './query.js';
 import { size, time, inspectData, defHasId, displayTarget } from './display.js';
 import { readFloatLE, readUint32 } from '@based/utils';
-import { resultToObject, readProps, readId, } from '@based/protocol/db-read';
+import { resultToObject, readProps, readId, readChecksum, readVersion, } from '@based/protocol/db-read';
 export { time, size, inspectData };
-const BITS_FOR_BYTE_LEN = 21;
-const FACTOR = 2 ** BITS_FOR_BYTE_LEN;
-const MASK_B = FACTOR - 1;
 export class BasedQueryResponse {
     result;
     def;
@@ -26,7 +23,7 @@ export class BasedQueryResponse {
         return this.def.queryId;
     }
     get version() {
-        return (this.checksum >>> 0) * FACTOR + (this.result.byteLength & MASK_B);
+        return readVersion(this.result);
     }
     get size() {
         return this.result.length;
@@ -106,9 +103,7 @@ export class BasedQueryResponse {
         return arr;
     }
     get checksum() {
-        const result = this.result;
-        const offset = result.byteLength - 4;
-        return readUint32(result, offset);
+        return readChecksum(this.result);
     }
     get length() {
         const l = readUint32(this.result, 0);

package/dist/src/client/query/display.js

@@ -1,7 +1,6 @@
 import picocolors from 'picocolors';
 import { BINARY, CARDINALITY, NUMBER, REFERENCE, REFERENCES, STRING, TEXT, TIMESTAMP, } from '@based/schema/def';
 import { ENCODER } from '@based/utils';
-import { AggregateType } from '@based/protocol/db-read';
 const decimals = (v) => ~~(v * 100) / 100;
 const sizeCalc = (size) => {
     if (size > 1e6) {
@@ -160,10 +159,7 @@ const inspectObject = (object, q, path, level, isLast, isFirst, isObject, depth)
             if (typeof v === 'number') {
                 if (q.aggregate) {
                     str += printNumber(v);
-                    // TBD: replace comptime const enum and reverse map it
-                    const [[__, akv], _] = q.aggregate.aggregates;
-                    const aggType = akv[0].type;
-                    str += picocolors.italic(picocolors.dim(` ${AggregateType[aggType].toLowerCase()}`));
+                    str += picocolors.italic(picocolors.dim(` ${k.toLowerCase()}`));
                     str += ',\n';
                 }
                 else {
@@ -179,9 +175,7 @@ const inspectObject = (object, q, path, level, isLast, isFirst, isObject, depth)
             if (def.typeIndex === REFERENCES) {
                 if (q.aggregate) {
                     str += printNumber(v);
-                    const [[__, akv], _] = q.aggregate.aggregates;
-                    const aggType = akv[0].type;
-                    str += picocolors.italic(picocolors.dim(` ${AggregateType[aggType].toLowerCase()}`));
+                    str += picocolors.italic(picocolors.dim(` ${k.toLowerCase()}`));
                 }
                 else {
                     str += inspectData(v, q.references.get(def.prop), level + 2, false, depth);
@@ -194,9 +188,7 @@ const inspectObject = (object, q, path, level, isLast, isFirst, isObject, depth)
                 else {
                     if (q.aggregate) {
                         str += printNumber(v);
-                        const [[__, akv], _] = q.aggregate.aggregates;
-                        const aggType = akv[0].type;
-                        str += picocolors.italic(picocolors.dim(` ${AggregateType[aggType].toLowerCase()}`));
+                        str += picocolors.italic(picocolors.dim(` ${k.toLowerCase()}`));
                     }
                     else {
                         str += inspectObject(v, q.references.get(def.prop), '', level + 2, false, false, true, depth);
@@ -240,22 +232,23 @@ const inspectObject = (object, q, path, level, isLast, isFirst, isObject, depth)
                 if (typeof v === 'number') {
                     if (q.aggregate) {
                         str += printNumber(v);
-                        const [[__, akv], _] = q.aggregate.aggregates;
-                        const aggType = akv[0].type;
-                        str += picocolors.italic(picocolors.dim(` ${AggregateType[aggType].toLowerCase()}`));
+                        str += picocolors.italic(picocolors.dim(` ${k.toLowerCase()}`));
                     }
                     else {
                         str += printNumber(v);
                     }
                 }
                 else if (typeof v === 'object' && v) {
-                    inspectObject(v, q, key, level + 2, false, false, true, depth) + '';
+                    str +=
+                        inspectObject(v, q, key, level + 2, false, false, true, depth) + '';
                 }
                 else {
                     str += v;
                 }
             }
-            if (def?.typeIndex !== REFERENCE && def?.typeIndex !== REFERENCES) {
+            if (def?.typeIndex !== REFERENCE &&
+                def?.typeIndex !== REFERENCES &&
+                typeof v !== 'object') {
                 str += ',\n';
             }
         }

package/dist/src/client/query/include/toByteCode.js

@@ -1,4 +1,4 @@
-import { MICRO_BUFFER, STRING, TEXT, JSON } from '@based/schema/def';
+import { MICRO_BUFFER, STRING, TEXT, JSON, BINARY } from '@based/schema/def';
 import { QueryDefType } from '../types.js';
 import { walkDefs } from './walk.js';
 import { langCodesMap } from '@based/schema';
@@ -76,6 +76,7 @@ export const includeToBuffer = (db, def) => {
     }
     if (propSize) {
         for (const [prop, propDef] of def.include.props.entries()) {
+            const typeIndex = propDef.opts?.raw ? BINARY : propDef.def.typeIndex;
             if (propDef.opts?.meta) {
                 if (propDef.opts.codes) {
                     if (propDef.opts.codes.has(0)) {
@@ -84,7 +85,7 @@ export const includeToBuffer = (db, def) => {
                             const buf = new Uint8Array(4);
                             buf[0] = 6 /* includeOp.META */;
                             buf[1] = prop;
-                            buf[2] = propDef.def.typeIndex;
+                            buf[2] = typeIndex;
                             buf[3] = langCodesMap.get(code);
                             result.push(buf);
                         }
@@ -94,7 +95,7 @@ export const includeToBuffer = (db, def) => {
                             const buf = new Uint8Array(4);
                             buf[0] = 6 /* includeOp.META */;
                             buf[1] = prop;
-                            buf[2] = propDef.def.typeIndex;
+                            buf[2] = typeIndex;
                             buf[3] = code;
                             result.push(buf);
                         }
@@ -104,21 +105,20 @@ export const includeToBuffer = (db, def) => {
                     const buf = new Uint8Array(4);
                     buf[0] = 6 /* includeOp.META */;
                     buf[1] = prop;
-                    buf[2] = propDef.def.typeIndex;
+                    buf[2] = typeIndex;
                     buf[3] = 0;
                     result.push(buf);
                 }
             }
             if (propDef.opts?.meta !== 'only') {
                 const hasEnd = propDef.opts?.end;
-                const t = propDef.def.typeIndex;
-                if (t === TEXT) {
+                if (typeIndex === TEXT) {
                     const codes = propDef.opts.codes;
                     if (codes.has(0)) {
                         const b = new Uint8Array(hasEnd ? 12 : 4);
                         b[0] = 1 /* includeOp.DEFAULT */;
                         b[1] = prop;
-                        b[2] = propDef.def.typeIndex;
+                        b[2] = typeIndex;
                         if (hasEnd) {
                             b[3] = 8; // opts len
                             b[4] = 0; // lang code
@@ -139,7 +139,7 @@ export const includeToBuffer = (db, def) => {
                             const b = new Uint8Array(7 + (endCode ? 5 : 0) + fallBackSize);
                             b[0] = 1 /* includeOp.DEFAULT */;
                             b[1] = prop;
-                            b[2] = propDef.def.typeIndex;
+                            b[2] = typeIndex;
                             let i = 0;
                             if (endCode) {
                                 b[3] = fallBackSize + 8; // opts
@@ -169,10 +169,14 @@ export const includeToBuffer = (db, def) => {
                     const buf = new Uint8Array(hasEnd ? 9 : 4);
                     buf[0] = 1 /* includeOp.DEFAULT */;
                     buf[1] = prop;
-                    buf[2] = propDef.def.typeIndex;
+                    buf[2] = typeIndex;
                     if (hasEnd) {
                         buf[3] = 5; // opts len
-                        buf[4] = propDef.opts?.bytes || (t !== JSON && t !== STRING) ? 0 : 1;
+                        buf[4] =
+                            propDef.opts?.bytes ||
+                                (typeIndex !== JSON && typeIndex !== STRING)
+                                ? 0
+                                : 1;
                         writeUint32(buf, getEnd(propDef.opts), 5);
                     }
                     else {

package/dist/src/client/query/queryDefToReadSchema.js

@@ -1,11 +1,11 @@
 // import type { IncludeOpts, QueryDef, Target } from '@based/db'
 import { inverseLangMap, langCodesMap } from '@based/schema';
-import { COLVEC, ENUM, TEXT, VECTOR, } from '@based/schema/def';
+import { COLVEC, ENUM, TEXT, VECTOR, BINARY, CARDINALITY, } from '@based/schema/def';
 import { ReaderMeta, ReaderSchemaEnum, } from '@based/protocol/db-read';
 const createReaderPropDef = (p, locales, opts) => {
     const readerPropDef = {
         path: p.__isEdge ? p.path.slice(1) : p.path,
-        typeIndex: p.typeIndex,
+        typeIndex: opts?.raw ? BINARY : p.typeIndex,
         readBy: 0,
     };
     if (opts?.meta) {
@@ -19,6 +19,10 @@ const createReaderPropDef = (p, locales, opts) => {
         readerPropDef.vectorBaseType = p.vectorBaseType;
         readerPropDef.len = p.len;
     }
+    if (p.typeIndex === CARDINALITY) {
+        readerPropDef.cardinalityMode = p.cardinalityMode;
+        readerPropDef.cardinalityPrecision = p.cardinalityPrecision;
+    }
     if (p.typeIndex === TEXT) {
         if (opts.codes.has(0)) {
             readerPropDef.locales = locales;
@@ -51,6 +55,7 @@ export const convertToReaderSchema = (q, locales) => {
     const readerSchema = {
         readId: 0,
         props: {},
+        search: false,
         main: { len: 0, props: {} },
         refs: {},
         type: isEdge

package/dist/src/client/query/registerQuery.js

@@ -15,11 +15,9 @@ export const registerQuery = (q) => {
         q.target, q.skipValidation);
         def.schemaChecksum = q.db.schema?.hash || 0;
         q.def = def;
-        // proposal:
         for (const command of commands) {
             q[command.method](...command.args);
         }
-        // locale first...
         if (!q.def.include.stringFields.size && !q.def.references.size) {
             includeField(q.def, { field: '*' });
         }
@@ -32,10 +30,13 @@ export const registerQuery = (q) => {
         q.buffer = buf;
         // console.log('--------------------------------------------------')
         // console.dir(convertToReaderSchema(q.def), { depth: 100 })
-        // console.log(deSerializeSchema(serialize(convertToReaderSchema(q.def))))
+        // const c = convertToReaderSchema(q.def)
+        // const s = serialize(c)
+        // console.log(deSerializeSchema(s))
         // q.def.readSchema = deSerializeSchema(
         //   serialize(convertToReaderSchema(q.def)),
         // )
+        // console.log('--------------------------------------------------')
         q.def.readSchema = convertToReaderSchema(q.def);
         handleErrors(q.def);
         return buf;

package/dist/src/client/query/types.d.ts

@@ -13,6 +13,7 @@ export type IncludeOpts = {
     codes?: Set<LangCode>;
     fallBacks?: LangCode[];
     localeFromDef?: LangCode;
+    raw?: true;
 };
 export type IncludeField = {
     field: string;

package/dist/src/client/query/validation.js

@@ -352,7 +352,10 @@ export const validateAlias = (def, alias, path) => {
         if (typeof alias[k] === 'string') {
             const p = path ? `${path}.${k}` : k;
             const prop = schema.props[p];
-            if (prop.typeIndex === ALIAS) {
+            if (!prop) {
+                // def.errors.push({ code: ERR_TARGET_INVAL_ALIAS, payload: def })
+            }
+            else if (prop.typeIndex === ALIAS) {
                 return { def: prop, value: alias[k] };
             }
         }

package/dist/src/client/setLocalClientSchema.d.ts

@@ -1,3 +1,3 @@
+import { DbSchema } from '@based/schema';
 import { DbClient } from '../index.js';
-import { DbSchema } from '../schema.js';
 export declare const setLocalClientSchema: (client: DbClient, schema: DbSchema) => DbSchema;

package/dist/src/hooks.d.ts

@@ -1,8 +1,7 @@
-import { StrictSchema, MigrateFns } from '@based/schema';
+import { StrictSchema, MigrateFns, DbSchema, SchemaChecksum } from '@based/schema';
 import { BasedDbQuery } from './client/query/BasedDbQuery.js';
 import { OnClose, OnData, OnError } from './client/query/subscription/types.js';
 import { DbServer } from './server/index.js';
-import { DbSchema, SchemaChecksum } from './schema.js';
 export type DbClientHooks = {
     setSchema(schema: StrictSchema, transformFns?: MigrateFns): Promise<SchemaChecksum>;
     flushModify(buf: Uint8Array): Promise<{

package/dist/src/native.d.ts

@@ -16,6 +16,8 @@ declare const native: {
     updateSchemaType: (prefix: number, buf: Uint8Array, dbCtx: any) => any;
     getTypeInfo: (typeId: number, dbCtx: any) => any;
     getNodeRangeHash: (typeId: number, start: number, end: number, bufOut: Uint8Array, dbCtx: any) => any;
+    createCompressor(): any;
+    compressRaw: (compressor: any, buf: Uint8Array, offset: number, stringSize: number) => any;
     compress: (buf: Uint8Array, offset: number, stringSize: number) => any;
     decompress: (input: Uint8Array, output: Uint8Array, offset: number, len: number) => any;
     crc32: (buf: Uint8Array) => any;

package/dist/src/native.js

@@ -74,6 +74,12 @@ const native = {
     getNodeRangeHash: (typeId, start, end, bufOut, dbCtx) => {
         return db.getNodeRangeHash(typeId, start, end, bufOut, dbCtx);
     },
+    createCompressor() {
+        return db.createCompressor();
+    },
+    compressRaw: (compressor, buf, offset, stringSize) => {
+        return db.compress(compressor, buf, offset, stringSize);
+    },
     compress: (buf, offset, stringSize) => {
         return db.compress(compressor, buf, offset, stringSize);
     },
@@ -106,7 +112,7 @@ const native = {
     },
     colvecTest: (dbCtx, typeId, field, nodeId, len) => {
         return db.colvecTest(dbCtx, typeId, field, nodeId, len);
-    }
+    },
 };
 global.__basedDb__native__ = native;
 export default native;

package/dist/src/server/index.d.ts

@@ -1,8 +1,7 @@
-import { LangName, MigrateFns, StrictSchema } from '@based/schema';
+import { StrictSchema, LangName, MigrateFns, SchemaChecksum } from '@based/schema';
 import { StartOpts } from './start.js';
 import { VerifTree } from './tree.js';
 import exitHook from 'exit-hook';
-import { SchemaChecksum } from '../schema.js';
 import { IoWorker } from './IoWorker.js';
 import { QueryWorker } from './QueryWorker.js';
 import { DbShared } from '../shared/DbBase.js';

package/dist/src/server/index.js

@@ -1,6 +1,6 @@
 import native from '../native.js';
 import { rm } from 'node:fs/promises';
-import { langCodesMap } from '@based/schema';
+import { langCodesMap, strictSchemaToDbSchema } from '@based/schema';
 import { ID_FIELD_DEF } from '@based/schema/def';
 import { start } from './start.js';
 import { destructureTreeKey, makeTreeKeyFromNodeId } from './tree.js';
@@ -10,7 +10,6 @@ import { migrate } from './migrate/index.js';
 import { debugServer } from '../utils.js';
 import { readUint16, readUint32, readUint64, writeUint32 } from '@based/utils';
 import { QueryType } from '../client/query/types.js';
-import { strictSchemaToDbSchema } from './schema.js';
 import { DbShared } from '../shared/DbBase.js';
 import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from './schema.js';
 import { resizeModifyDirtyRanges } from './resizeModifyDirtyRanges.js';
@@ -263,8 +262,8 @@ export class DbServer extends DbShared {
             return this.schema.hash;
         }
         setSchemaOnServer(this, schema);
+        setNativeSchema(this, schema);
         await writeSchemaFile(this, schema);
-        await setNativeSchema(this, schema);
         process.nextTick(() => {
             this.emit('schema', this.schema);
         });

package/dist/src/server/migrate/index.d.ts

@@ -1,6 +1,5 @@
 import { DbServer } from '../index.js';
-import { DbSchema } from '../../schema.js';
-import { MigrateFns } from '@based/schema';
+import { DbSchema, MigrateFns } from '@based/schema';
 export type MigrateRange = {
     typeId: number;
     start: number;

package/dist/src/server/migrate/worker.js

@@ -1,7 +1,7 @@
 import { isMainThread, receiveMessageOnPort, workerData, } from 'node:worker_threads';
 import native from '../../native.js';
 import { BasedDb } from '../../index.js';
-import { REFERENCE, REFERENCES } from '@based/schema/def';
+import { CARDINALITY, REFERENCE, REFERENCES } from '@based/schema/def';
 import { setSchemaOnServer } from '../schema.js';
 import { setToSleep } from './utils.js';
 import { setLocalClientSchema } from '../../client/setLocalClientSchema.js';
@@ -27,28 +27,36 @@ else if (workerData?.isDbMigrateWorker) {
         const map = {};
         for (const type in fromDb.server.schemaTypesParsed) {
             const { id, props } = fromDb.server.schemaTypesParsed[type];
-            const include = Object.keys(props);
-            let i = include.length;
-            while (i--) {
-                const path = include[i];
-                if (props[path].typeIndex === REFERENCE ||
-                    props[path].typeIndex === REFERENCES) {
-                    include[i] = `${path}.id`;
-                    if (props[path].edges) {
-                        for (const key in props[path].edges) {
-                            const prop = props[path].edges[key];
-                            if (prop.typeIndex === REFERENCE ||
-                                prop.typeIndex === REFERENCES) {
+            const include = [];
+            const includeRaw = [];
+            for (const path in props) {
+                const prop = props[path];
+                if (prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) {
+                    include.push(`${path}.id`);
+                    if (prop.edges) {
+                        for (const key in prop.edges) {
+                            const edge = prop.edges[key];
+                            if (edge.typeIndex === REFERENCE ||
+                                edge.typeIndex === REFERENCES) {
                                 include.push(`${path}.${key}.id`);
                             }
+                            else if (edge.typeIndex === CARDINALITY) {
+                                includeRaw.push(`${path}.${key}`);
+                            }
                             else {
                                 include.push(`${path}.${key}`);
                             }
                         }
                     }
                 }
+                else if (prop.typeIndex === CARDINALITY) {
+                    includeRaw.push(path);
+                }
+                else {
+                    include.push(path);
+                }
             }
-            map[id] = { type, include };
+            map[id] = { type, include, includeRaw };
         }
         for (const type in transformFns) {
             const fnOrNull = transformFns[type];
@@ -58,14 +66,17 @@ else if (workerData?.isDbMigrateWorker) {
             let msg;
             while ((msg = receiveMessageOnPort(channel))) {
                 const leafData = msg.message;
-                const { type, include } = map[leafData.typeId];
+                const { type, include, includeRaw } = map[leafData.typeId];
                 const typeTransformFn = transformFns[type];
+                const query = fromDb
+                    .query(type)
+                    .range(leafData.start - 1, leafData.end)
+                    .include(include);
+                for (const rawProp of includeRaw) {
+                    query.include(rawProp, { raw: true });
+                }
+                const nodes = query._getSync(fromCtx);
                 if (typeTransformFn) {
-                    const nodes = fromDb
-                        .query(type)
-                        .include(include)
-                        .range(leafData.start - 1, leafData.end)
-                        ._getSync(fromCtx);
                     for (const node of nodes) {
                         const res = typeTransformFn(node);
                         if (res === null) {
@@ -80,11 +91,6 @@ else if (workerData?.isDbMigrateWorker) {
                     }
                 }
                 else if (type in toDb.server.schemaTypesParsed) {
-                    const nodes = fromDb
-                        .query(type)
-                        .include(include)
-                        .range(leafData.start - 1, leafData.end)
-                        ._getSync(fromCtx);
                     for (const node of nodes) {
                         toDb.create(type, node, { unsafe: true });
                     }

package/dist/src/server/schema.d.ts

@@ -1,7 +1,11 @@
-import { StrictSchema } from '@based/schema';
+import { DbSchema } from '@based/schema';
 import { DbServer } from './index.js';
-import { DbSchema } from '../schema.js';
 export declare const setSchemaOnServer: (server: DbServer, schema: DbSchema) => void;
 export declare const writeSchemaFile: (server: DbServer, schema: DbSchema) => Promise<void>;
+/**
+ * Set schema used in native code.
+ * This function should be only called when a new schema is set to an empty DB
+ * instance. If a `common.sdb` file is loaded then calling this function isn't
+ * necessary because `common.sdb` already contains the required schema.
+ */
 export declare const setNativeSchema: (server: DbServer, schema: DbSchema) => void;
-export declare const strictSchemaToDbSchema: (schema: StrictSchema) => DbSchema;

package/dist/src/server/schema.js

@@ -1,12 +1,10 @@
 import { schemaToSelvaBuffer, updateTypeDefs } from '@based/schema/def';
-import { deepCopy } from '@based/utils';
-import { getPropType, serialize } from '@based/schema';
+import { serialize } from '@based/schema';
 import { join } from 'node:path';
 import { writeFile } from 'node:fs/promises';
 import native from '../native.js';
 import { SCHEMA_FILE } from '../types.js';
 import { saveSync } from './save.js';
-import { hash } from '@based/hash';
 import { writeCreate } from '../client/modify/create/index.js';
 import { Ctx } from '../client/modify/Ctx.js';
 import { consume } from '../client/modify/drain.js';
@@ -27,6 +25,12 @@ export const writeSchemaFile = async (server, schema) => {
         }
     }
 };
+/**
+ * Set schema used in native code.
+ * This function should be only called when a new schema is set to an empty DB
+ * instance. If a `common.sdb` file is loaded then calling this function isn't
+ * necessary because `common.sdb` already contains the required schema.
+ */
 export const setNativeSchema = (server, schema) => {
     const types = Object.keys(server.schemaTypesParsed);
     const s = schemaToSelvaBuffer(server.schemaTypesParsed);
@@ -52,57 +56,4 @@ export const setNativeSchema = (server, schema) => {
         saveSync(server, { skipDirtyCheck: true });
     }
 };
-export const strictSchemaToDbSchema = (schema) => {
-    // @ts-ignore
-    let dbSchema = deepCopy(schema);
-    // reserve 1 for root (even if you dont have it)
-    dbSchema.lastId = 1;
-    if (dbSchema.props) {
-        for (const key in dbSchema.props) {
-            const prop = dbSchema.props[key];
-            const propType = getPropType(prop);
-            let refProp;
-            if (propType === 'reference') {
-                refProp = prop;
-            }
-            else if (propType === 'references') {
-                refProp = prop.items;
-                prop.items = refProp;
-            }
-            if (refProp) {
-                const type = dbSchema.types[refProp.ref];
-                const inverseKey = '_' + key;
-                dbSchema.types[refProp.ref] = {
-                    ...type,
-                    props: {
-                        ...type.props,
-                        [inverseKey]: {
-                            items: {
-                                ref: '_root',
-                                prop: key,
-                            },
-                        },
-                    },
-                };
-                refProp.prop = inverseKey;
-            }
-        }
-        dbSchema.types ??= {};
-        // @ts-ignore This creates an internal type to use for root props
-        dbSchema.types._root = {
-            id: 1,
-            props: dbSchema.props,
-        };
-        delete dbSchema.props;
-    }
-    for (const field in dbSchema.types) {
-        if (!('id' in dbSchema.types[field])) {
-            dbSchema.lastId++;
-            dbSchema.types[field].id = dbSchema.lastId;
-        }
-    }
-    const { hash: _, ...rest } = dbSchema;
-    dbSchema.hash = hash(rest);
-    return dbSchema;
-};
 //# sourceMappingURL=schema.js.map

package/dist/src/server/workers/DbWorker.js

@@ -18,14 +18,19 @@ export class DbWorker {
             },
             transferList: [port2],
         });
-        this.readyPromise = new Promise((resolve) => {
+        this.readyPromise = new Promise((resolve, reject) => {
             const onReady = (msg) => {
                 if (msg?.status === 'READY') {
                     // TODO Also call native.destroyThreadCtx() somewhere
                     this.#threadId = msg.threadId;
-                    native.createThreadCtx(db.dbCtxExternal, this.#threadId);
-                    this.worker.off('message', onReady);
-                    resolve(true);
+                    try {
+                        native.createThreadCtx(db.dbCtxExternal, this.#threadId);
+                        this.worker.off('message', onReady);
+                        resolve(true);
+                    }
+                    catch (e) {
+                        reject(e);
+                    }
                 }
             };
             this.worker.on('message', onReady);
@@ -58,8 +63,17 @@ export class DbWorker {
     resolvers = [];
     readyPromise;
     async terminate() {
-        // TODO do we want to force this.worker.terminate() after a timeout?
+        // worker.terminate() is known to cause crashes but this is the best we can
+        // do if the thread won't exit voluntarily.
+        const tim = setTimeout(() => this.worker.terminate(), 5e3);
+        const p = new Promise((resolve) => {
+            this.worker.on('exit', () => {
+                clearTimeout(tim);
+                resolve();
+            });
+        });
         await this.call(0n);
+        return p;
     }
     callback = (resolve) => {
         this.resolvers.push(resolve);

package/dist/src/shared/DbBase.d.ts

@@ -1,5 +1,5 @@
 import { SchemaTypeDef } from '@based/schema/def';
-import { DbSchema } from '../schema.js';
+import { DbSchema } from '@based/schema';
 import { Emitter } from './Emitter.js';
 export type EventMap = {
     schema: DbSchema;

package/dist/src/shared/Emitter.d.ts

@@ -1,4 +1,4 @@
-import { DbSchema } from '../schema.js';
+import { DbSchema } from "@based/schema";
 export type EventMap = {
     schema: DbSchema;
     info: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.1.1",
+  "version": "0.1.3-alpha.1",
   "type": "module",
   "main": "./dist/src/index.js",
   "scripts": {
@@ -22,7 +22,7 @@
     "test-fast-linux_aarch64-schema": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; npm run test-fast schemadebug'",
     "test-fast-linux_aarch64": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; npm run test-fast'",
     "test-fast-linux_aarch64-gdb": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; LOCPATH=../locale/locale-x86_64-gnu/locale gdb -ex run --args node ./scripts/test.js'",
-    "test-fast-linux_aarch64-valgrind": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; LOCPATH=../locale/locale-aarch64-gnu/locale valgrind --leak-check=full node ./scripts/test.js references:update2'",
+    "test-fast-linux_aarch64-valgrind": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; LOCPATH=../locale/locale-aarch64-gnu/locale valgrind --leak-check=full node ./scripts/test.js'",
     "test-vector": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; npm run test-fast -- mem.js'",
     "test-zig": "npm run build-zig && tsc && npm run test-fast",
     "test-zig-debug": "npm run build-zig -- debug && tsc && LOCPATH=../locale/locale-x86_64-gnu/locale ./scripts/lldb-node ./scripts/test.js",
@@ -43,8 +43,8 @@
   "dependencies": {
     "@based/hash": "1.0.0",
     "@based/schema": "^5.0.2",
-    "@based/utils": "^1.0.0",
-    "@based/protocol": "^0.0.3",
+    "@based/utils": "^1.1.0",
+    "@based/protocol": "^0.0.4",
     "exit-hook": "^4.0.0",
     "picocolors": "^1.1.0"
   },

package/dist/src/schema.d.ts

@@ -1,6 +0,0 @@
-import { StrictSchema } from '@based/schema';
-export type DbSchema = StrictSchema & {
-    lastId: number;
-    hash: number;
-};
-export type SchemaChecksum = number;

package/dist/src/schema.js

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=schema.js.map