@based/db 0.0.66 → 0.0.68

package/README.md

@@ -17,9 +17,9 @@ BasedDb is a powerful database solution that supports various data types, refere
 **Prerequisites:**
 
 - recent GNU make
-- gcc with recent enough C23 support
+- gcc 14.2 on Linux or clang 17.0.0 on macOS
 - zig 0.14.0
-- npm & node.js, v20.11.1 or newer
+- npm & node.js v22.14.0 or newer
 
 ```bash
 npm i

package/dist/lib/darwin_aarch64/include/selva/colvec.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2025 SAULX
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "selva/types.h"
+#include "selva/selva_hash128.h"
+#include "selva/_export.h"
+
+struct SelvaColvec {
+    field_t field;
+    uint16_t vec_size;
+    size_t slab_size; /* Size of each slab in v. */
+
+    /**
+     * Array of pointers to vector slabs.
+     * te->blocks->len number of pointers to vector blocks each containing
+     * te->blocks->block_capacity vectors.
+     */
+    void **v;
+};
+
+/**
+ * Initialize colvec record keeping under te->col_fields.
+ */
+void colvec_init_te(struct SelvaTypeEntry *te);
+
+/**
+ * Deinitialize colvec record keeping and col slabs under te->col_fields.
+ */
+void colvec_deinit_te(struct SelvaTypeEntry *te);
+
+/**
+ * Initialize a slab at block_i if not already Initialized.
+ */
+void *colvec_init_slab(struct SelvaColvec *colvec, block_id_t block_i);
+
+/**
+ * Initialize all columnar fields of node.
+ * This function will also initialize the slab(s) if it's not allocated.
+ */
+void colvec_init_node(struct SelvaTypeEntry *te, struct SelvaNode *node);
+
+/**
+ * Compute the hash for a node_id in colvec.
+ */
+void colvec_hash_update(struct SelvaTypeEntry *te, node_id_t node_id, struct SelvaColvec *colvec, selva_hash_state_t *hash_state);
+
+/**
+ * Get the whole column for fs.
+ */
+SELVA_EXPORT
+struct SelvaColvec *colvec_get(struct SelvaTypeEntry *te, const struct SelvaFieldSchema *fs);
+
+/**
+ * Get a single vector in a colvec field by node_id.
+ */
+SELVA_EXPORT
+void *colvec_get_vec(struct SelvaTypeEntry *te, node_id_t node_id, const struct SelvaFieldSchema *fs);
+
+/**
+ * Set a single vector in a colvec field.
+ */
+SELVA_EXPORT
+void colvec_set_vec(struct SelvaTypeEntry *te, node_id_t node_id, const struct SelvaFieldSchema *fs, const void *vec);
+
+SELVA_EXPORT
+int colvec_foreach(struct SelvaTypeEntry *te, const struct SelvaFieldSchema *fs, node_id_t start, uint32_t len, void (*cb)(node_id_t node_id, void *vec, void *arg), void *arg);

package/dist/lib/darwin_aarch64/include/selva/db.h

@@ -241,6 +241,9 @@ void selva_db_expire_tick(struct SelvaDb *db, selva_dirty_node_cb_t dirty_cb, vo
 SELVA_EXPORT
 void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx) __attribute__((nonnull(1, 2, 3)));
 
+SELVA_EXPORT
+void selva_del_block(struct SelvaDb *db, struct SelvaTypeEntry *te, node_id_t start);
+
 /**
  * Get a node by id.
  */

package/dist/lib/darwin_aarch64/include/selva/fields.h

@@ -239,7 +239,7 @@ int selva_fields_get_reference_meta_mutable_string(
         struct SelvaNode *node,
         struct SelvaNodeReference *ref,
         const struct EdgeFieldConstraint *efc,
-        field_t field,
+        const struct SelvaFieldSchema *efs,
         size_t len,
         struct selva_string **s);
 

package/dist/lib/darwin_aarch64/include/selva/types.h

@@ -32,6 +32,7 @@ enum SelvaFieldType {
     SELVA_FIELD_TYPE_WEAK_REFERENCES = 7,
     SELVA_FIELD_TYPE_ALIAS = 8,
     SELVA_FIELD_TYPE_ALIASES = 9,
+    SELVA_FIELD_TYPE_COLVEC = 10,
 } __packed;
 
 struct EdgeFieldConstraint {
@@ -71,6 +72,11 @@ struct SelvaFieldSchema {
             uint16_t len;
         } smb; /*!< SELVA_FIELD_TYPE_MICRO_BUFFER */
         size_t alias_index; /*!< Index in aliases for SELVA_FIELD_TYPE_ALIAS and SELVA_FIELD_TYPE_ALIASES. */
+        struct {
+            uint16_t vec_len; /*!< Length of a single vector. */
+            uint16_t comp_size; /*!< Component size in the vector. */
+            field_t index; /*!< Index in te->col_fields.colvec.v. */
+        } colvec;
     };
 } __designated_init;
 
@@ -87,6 +93,7 @@ struct SelvaFieldsSchema {
 
 struct SelvaNodeSchema {
     size_t nr_aliases; /*!< Number of alias fields in this type. */
+    size_t nr_colvecs; /*!< Number of columnar vector fields. */
     struct SelvaFieldsSchema fields_schema;
     /* Nothing must be put after this line. */
 };

package/dist/lib/linux_aarch64/include/selva/colvec.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2025 SAULX
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "selva/types.h"
+#include "selva/selva_hash128.h"
+#include "selva/_export.h"
+
+struct SelvaColvec {
+    field_t field;
+    uint16_t vec_size;
+    size_t slab_size; /* Size of each slab in v. */
+
+    /**
+     * Array of pointers to vector slabs.
+     * te->blocks->len number of pointers to vector blocks each containing
+     * te->blocks->block_capacity vectors.
+     */
+    void **v;
+};
+
+/**
+ * Initialize colvec record keeping under te->col_fields.
+ */
+void colvec_init_te(struct SelvaTypeEntry *te);
+
+/**
+ * Deinitialize colvec record keeping and col slabs under te->col_fields.
+ */
+void colvec_deinit_te(struct SelvaTypeEntry *te);
+
+/**
+ * Initialize a slab at block_i if not already Initialized.
+ */
+void *colvec_init_slab(struct SelvaColvec *colvec, block_id_t block_i);
+
+/**
+ * Initialize all columnar fields of node.
+ * This function will also initialize the slab(s) if it's not allocated.
+ */
+void colvec_init_node(struct SelvaTypeEntry *te, struct SelvaNode *node);
+
+/**
+ * Compute the hash for a node_id in colvec.
+ */
+void colvec_hash_update(struct SelvaTypeEntry *te, node_id_t node_id, struct SelvaColvec *colvec, selva_hash_state_t *hash_state);
+
+/**
+ * Get the whole column for fs.
+ */
+SELVA_EXPORT
+struct SelvaColvec *colvec_get(struct SelvaTypeEntry *te, const struct SelvaFieldSchema *fs);
+
+/**
+ * Get a single vector in a colvec field by node_id.
+ */
+SELVA_EXPORT
+void *colvec_get_vec(struct SelvaTypeEntry *te, node_id_t node_id, const struct SelvaFieldSchema *fs);
+
+/**
+ * Set a single vector in a colvec field.
+ */
+SELVA_EXPORT
+void colvec_set_vec(struct SelvaTypeEntry *te, node_id_t node_id, const struct SelvaFieldSchema *fs, const void *vec);
+
+SELVA_EXPORT
+int colvec_foreach(struct SelvaTypeEntry *te, const struct SelvaFieldSchema *fs, node_id_t start, uint32_t len, void (*cb)(node_id_t node_id, void *vec, void *arg), void *arg);

package/dist/lib/linux_aarch64/include/selva/db.h

@@ -241,6 +241,9 @@ void selva_db_expire_tick(struct SelvaDb *db, selva_dirty_node_cb_t dirty_cb, vo
 SELVA_EXPORT
 void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx) __attribute__((nonnull(1, 2, 3)));
 
+SELVA_EXPORT
+void selva_del_block(struct SelvaDb *db, struct SelvaTypeEntry *te, node_id_t start);
+
 /**
  * Get a node by id.
  */

package/dist/lib/linux_aarch64/include/selva/fields.h

@@ -239,7 +239,7 @@ int selva_fields_get_reference_meta_mutable_string(
         struct SelvaNode *node,
         struct SelvaNodeReference *ref,
         const struct EdgeFieldConstraint *efc,
-        field_t field,
+        const struct SelvaFieldSchema *efs,
         size_t len,
         struct selva_string **s);
 

package/dist/lib/linux_aarch64/include/selva/types.h

@@ -32,6 +32,7 @@ enum SelvaFieldType {
     SELVA_FIELD_TYPE_WEAK_REFERENCES = 7,
     SELVA_FIELD_TYPE_ALIAS = 8,
     SELVA_FIELD_TYPE_ALIASES = 9,
+    SELVA_FIELD_TYPE_COLVEC = 10,
 } __packed;
 
 struct EdgeFieldConstraint {
@@ -71,6 +72,11 @@ struct SelvaFieldSchema {
             uint16_t len;
         } smb; /*!< SELVA_FIELD_TYPE_MICRO_BUFFER */
         size_t alias_index; /*!< Index in aliases for SELVA_FIELD_TYPE_ALIAS and SELVA_FIELD_TYPE_ALIASES. */
+        struct {
+            uint16_t vec_len; /*!< Length of a single vector. */
+            uint16_t comp_size; /*!< Component size in the vector. */
+            field_t index; /*!< Index in te->col_fields.colvec.v. */
+        } colvec;
     };
 } __designated_init;
 
@@ -87,6 +93,7 @@ struct SelvaFieldsSchema {
 
 struct SelvaNodeSchema {
     size_t nr_aliases; /*!< Number of alias fields in this type. */
+    size_t nr_colvecs; /*!< Number of columnar vector fields. */
     struct SelvaFieldsSchema fields_schema;
     /* Nothing must be put after this line. */
 };

package/dist/lib/linux_x86_64/include/selva/colvec.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2025 SAULX
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "selva/types.h"
+#include "selva/selva_hash128.h"
+#include "selva/_export.h"
+
+struct SelvaColvec {
+    field_t field;
+    uint16_t vec_size;
+    size_t slab_size; /* Size of each slab in v. */
+
+    /**
+     * Array of pointers to vector slabs.
+     * te->blocks->len number of pointers to vector blocks each containing
+     * te->blocks->block_capacity vectors.
+     */
+    void **v;
+};
+
+/**
+ * Initialize colvec record keeping under te->col_fields.
+ */
+void colvec_init_te(struct SelvaTypeEntry *te);
+
+/**
+ * Deinitialize colvec record keeping and col slabs under te->col_fields.
+ */
+void colvec_deinit_te(struct SelvaTypeEntry *te);
+
+/**
+ * Initialize a slab at block_i if not already Initialized.
+ */
+void *colvec_init_slab(struct SelvaColvec *colvec, block_id_t block_i);
+
+/**
+ * Initialize all columnar fields of node.
+ * This function will also initialize the slab(s) if it's not allocated.
+ */
+void colvec_init_node(struct SelvaTypeEntry *te, struct SelvaNode *node);
+
+/**
+ * Compute the hash for a node_id in colvec.
+ */
+void colvec_hash_update(struct SelvaTypeEntry *te, node_id_t node_id, struct SelvaColvec *colvec, selva_hash_state_t *hash_state);
+
+/**
+ * Get the whole column for fs.
+ */
+SELVA_EXPORT
+struct SelvaColvec *colvec_get(struct SelvaTypeEntry *te, const struct SelvaFieldSchema *fs);
+
+/**
+ * Get a single vector in a colvec field by node_id.
+ */
+SELVA_EXPORT
+void *colvec_get_vec(struct SelvaTypeEntry *te, node_id_t node_id, const struct SelvaFieldSchema *fs);
+
+/**
+ * Set a single vector in a colvec field.
+ */
+SELVA_EXPORT
+void colvec_set_vec(struct SelvaTypeEntry *te, node_id_t node_id, const struct SelvaFieldSchema *fs, const void *vec);
+
+SELVA_EXPORT
+int colvec_foreach(struct SelvaTypeEntry *te, const struct SelvaFieldSchema *fs, node_id_t start, uint32_t len, void (*cb)(node_id_t node_id, void *vec, void *arg), void *arg);

package/dist/lib/linux_x86_64/include/selva/db.h

@@ -241,6 +241,9 @@ void selva_db_expire_tick(struct SelvaDb *db, selva_dirty_node_cb_t dirty_cb, vo
 SELVA_EXPORT
 void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx) __attribute__((nonnull(1, 2, 3)));
 
+SELVA_EXPORT
+void selva_del_block(struct SelvaDb *db, struct SelvaTypeEntry *te, node_id_t start);
+
 /**
  * Get a node by id.
  */

package/dist/lib/linux_x86_64/include/selva/fields.h

@@ -239,7 +239,7 @@ int selva_fields_get_reference_meta_mutable_string(
         struct SelvaNode *node,
         struct SelvaNodeReference *ref,
         const struct EdgeFieldConstraint *efc,
-        field_t field,
+        const struct SelvaFieldSchema *efs,
         size_t len,
         struct selva_string **s);
 

package/dist/lib/linux_x86_64/include/selva/types.h

@@ -32,6 +32,7 @@ enum SelvaFieldType {
     SELVA_FIELD_TYPE_WEAK_REFERENCES = 7,
     SELVA_FIELD_TYPE_ALIAS = 8,
     SELVA_FIELD_TYPE_ALIASES = 9,
+    SELVA_FIELD_TYPE_COLVEC = 10,
 } __packed;
 
 struct EdgeFieldConstraint {
@@ -71,6 +72,11 @@ struct SelvaFieldSchema {
             uint16_t len;
         } smb; /*!< SELVA_FIELD_TYPE_MICRO_BUFFER */
         size_t alias_index; /*!< Index in aliases for SELVA_FIELD_TYPE_ALIAS and SELVA_FIELD_TYPE_ALIASES. */
+        struct {
+            uint16_t vec_len; /*!< Length of a single vector. */
+            uint16_t comp_size; /*!< Component size in the vector. */
+            field_t index; /*!< Index in te->col_fields.colvec.v. */
+        } colvec;
     };
 } __designated_init;
 
@@ -87,6 +93,7 @@ struct SelvaFieldsSchema {
 
 struct SelvaNodeSchema {
     size_t nr_aliases; /*!< Number of alias fields in this type. */
+    size_t nr_colvecs; /*!< Number of columnar vector fields. */
     struct SelvaFieldsSchema fields_schema;
     /* Nothing must be put after this line. */
 };

package/dist/src/client/flushModify.d.ts

@@ -7,6 +7,7 @@ export declare class ModifyCtx {
     id: number;
     hasSortField: number;
     hasSortText: number;
+    hasDefaults: number;
     ctx: {
         queue?: Map<(payload: any) => void, ModifyState>;
         offsets?: Record<number, number>;
@@ -26,7 +27,7 @@ export declare class ModifyCtx {
     markNodeDirty(schema: SchemaTypeDef, nodeId: number): void;
     markTypeDirty(schema: SchemaTypeDef): void;
     updateMax(): void;
-    getData(lastIds: Record<number, number>): Uint8Array;
+    getData(lastIds: Record<number, number>): Uint8Array<ArrayBufferLike>;
     reset(): void;
 }
 export declare const execCtxQueue: (resCtx: ModifyCtx["ctx"], error?: boolean) => void;

package/dist/src/client/flushModify.js

@@ -1,6 +1,6 @@
 import { writeUint64 } from '@saulx/utils';
 // TODO This definitely shouldn't be copy-pasted here from server/tree.ts
-const makeCsmtKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
+const makeTreeKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
     const tmp = nodeId - +!(nodeId % blockCapacity);
     return typeId * 4294967296 + ((tmp / blockCapacity) | 0) * blockCapacity + 1;
 };
@@ -16,6 +16,7 @@ export class ModifyCtx {
     id = -1;
     hasSortField = -1;
     hasSortText = -1;
+    hasDefaults = -1;
     ctx = {};
     payload;
     max;
@@ -30,7 +31,7 @@ export class ModifyCtx {
     dirtyRanges = new Set();
     dirtyTypes = new Map();
     markNodeDirty(schema, nodeId) {
-        const key = makeCsmtKeyFromNodeId(schema.id, schema.blockCapacity, nodeId);
+        const key = makeTreeKeyFromNodeId(schema.id, schema.blockCapacity, nodeId);
         if (this.dirtyRanges.has(key)) {
             return;
         }
@@ -77,9 +78,16 @@ export class ModifyCtx {
             data[i++] = lastId >>> 24;
         }
         const view = new DataView(data.buffer, data.byteOffset);
+        // this is a problem need to remove these dirtyRanges in general...
         for (let key of this.dirtyRanges) {
-            view.setFloat64(i, key, true);
-            i += 8;
+            if (i < view.byteLength - 8) {
+                view.setFloat64(i, key, true);
+                i += 8;
+            }
+            else {
+                console.warn('Dirty range does not fit - will remove this', this.dirtyRanges.size, i);
+                break;
+            }
         }
         const lenMinusSchemaHash = this.len - 8;
         data[i++] = lenMinusSchemaHash;

package/dist/src/client/modify/create.js

@@ -52,6 +52,7 @@ const appendCreate = (ctx, def, obj, res, unsafe) => {
             ctx.buf.set(def.mainEmpty, ctx.len);
             ctx.len += def.mainLen;
         }
+        // add text & string here
     }
     if (def.hasSeperateSort) {
         if (ctx.hasSortField !== def.seperateSort.size - 1) {
@@ -79,6 +80,16 @@ const appendCreate = (ctx, def, obj, res, unsafe) => {
         // add test for this
         ctx.hasSortField = -1;
     }
+    // if (def.hasSeperateDefaults) {
+    //   const buf = def.seperateDefaults.bufferTmp
+    //   if (ctx.hasDefaults !== def.seperateDefaults.props.size - 1) {
+    //     //
+    //   }
+    //   if (ctx.hasDefaults !== -1) {
+    //     buf.set(def.seperateTextSort.buffer, 0)
+    //   }
+    //   ctx.hasDefaults = -1
+    // }
     if (def.hasSeperateTextSort) {
         const buf = def.seperateTextSort.bufferTmp;
         if (ctx.hasSortText !== def.seperateTextSort.size - 1) {

package/dist/src/client/modify/delete.js

@@ -8,6 +8,9 @@ export const deleteFn = (db, type, id, opts) => {
     if (!def) {
         throw new Error(`Unknown type: ${type}. Did you mean on of: ${Object.keys(db.schemaTypesParsed).join(', ')}`);
     }
+    if (def.insertOnly) {
+        throw new Error(`This type is insertOnly`);
+    }
     const ctx = db.modifyCtx;
     const res = new ModifyState(def.id, id, db, opts);
     const schema = db.schemaTypesParsed[type];

package/dist/src/client/modify/modify.js

@@ -1,4 +1,4 @@
-import { isPropDef, REFERENCE, REFERENCES, STRING, TEXT, ALIAS, BINARY, CARDINALITY, VECTOR, MICRO_BUFFER, JSON, } from '@based/schema/def';
+import { isPropDef, REFERENCE, REFERENCES, STRING, TEXT, ALIAS, BINARY, CARDINALITY, VECTOR, MICRO_BUFFER, JSON, COLVEC, } from '@based/schema/def';
 import { ModifyError } from './ModifyRes.js';
 import { writeReference } from './references/reference.js';
 import { writeReferences } from './references/references.js';
@@ -50,7 +50,7 @@ function _modify(ctx, res, obj, schema, mod, tree, overwrite, unsafe) {
                 else if (type === CARDINALITY) {
                     err = writeHll(val, ctx, schema, def, res.tmpId, mod);
                 }
-                else if (type === VECTOR) {
+                else if (type === VECTOR || type === COLVEC) {
                     err = writeVector(val, ctx, schema, def, res.tmpId, mod);
                 }
                 else if (type === JSON) {

package/dist/src/client/modify/setCursor.d.ts

@@ -1,4 +1,5 @@
 import { ModifyCtx } from '../../index.js';
 import { SchemaTypeDef } from '@based/schema/def';
 import { ModifyOp } from './types.js';
-export declare const setCursor: (ctx: ModifyCtx, schema: SchemaTypeDef, field: number, typeIndex: number, id: number, modifyOp: ModifyOp, ignoreField?: boolean) => void;
+export declare const setCursor: (ctx: ModifyCtx, schema: SchemaTypeDef, field: number, // TODO pass propdef better
+typeIndex: number, id: number, modifyOp: ModifyOp, ignoreField?: boolean) => void;

package/dist/src/client/query/BasedDbQuery.d.ts

@@ -1,4 +1,4 @@
-import { QueryDef, QueryTarget, Operator, QueryByAliasObj } from './query.js';
+import { QueryDef, QueryTarget, Operator, QueryByAliasObj, LangFallback } from './query.js';
 import { BasedQueryResponse } from './BasedIterable.js';
 import { Search } from './search/index.js';
 import { OnData, OnError } from './subscription/index.js';
@@ -26,6 +26,11 @@ export declare class QueryBranch<T> {
     count(field?: string): T;
     sum(...fields: (string | string[])[]): T;
     cardinality(...fields: (string | string[])[]): T;
+    stddev(...fields: (string | string[])[]): T;
+    var(...fields: (string | string[])[]): T;
+    avg(...fields: (string | string[])[]): T;
+    max(...fields: (string | string[])[]): T;
+    min(...fields: (string | string[])[]): T;
     or(fn: FilterBranchFn): T;
     or(field: string, operator?: Operator | boolean, value?: any, opts?: FilterOpts): T;
     range(start: number, end?: number): T;
@@ -48,7 +53,7 @@ export declare class BasedDbQuery extends QueryBranch<BasedDbQuery> {
     get(): GetPromise;
     buffer: Uint8Array;
     register(): void;
-    locale(locale: LangName): this;
+    locale(locale: LangName, fallBack?: LangFallback): this;
     subscribe(onData: OnData, onError?: OnError): import("./subscription/types.js").OnClose;
     _getSync(dbCtxExternal: any): BasedQueryResponse;
 }