@based/db 0.0.8 → 0.0.9

package/README.md

@@ -1,4 +1,16 @@
-#Based-db
+# BasedDb
+
+BasedDb is a powerful database solution that supports various data types, references, edges, and operations. It also offers concurrency handling, client-server architecture support, and more.
+
+## Features
+
+- Schema definition and management
+- Data creation, querying, updating, and deletion
+- Support for strings, numbers, booleans, binaries, aliases, enums, and cardinality
+- Edges and references for advanced data modeling
+- Concurrency support for high-load scenarios
+- Client-server design for distributed systems
+- Checksum, analytics, and expiration features
 
 ## Install
 
@@ -9,7 +21,7 @@
 - zig 0.13.0
 - npm & node.js, v20.11.1 or newer
 
-```
+```bash
 npm i
 npm run get-napi // only need this the first time
 npm run build
@@ -18,26 +30,431 @@ npm run build
 ## Running tests
 
 Run all tests + ldb + build c, zig and js
-`npm run test`
+
+```bash
+npm run test
+```
 
 Run specific test file - does substring matching
-`npm run test -- range.js`
+
+```bash
+npm run test -- range.js
+```
 
 Run specific test file & run specific test
-`npm run test -- range.js:range`
+
+```bash
+npm run test -- range.js:range
+```
 
 Different flavours of test
 
 Only builds zig
-`npm run test-zig`
+
+```bash
+npm run test-zig
+```
 
 Builds nothing only runs tests
-`npm run test-fast`
 
-## Getting started
+```bash
+npm run test-fast
+```
+
+## Getting Started
+
+To get started with BasedDb, follow these steps:
+
+1. Install the package
+2. Define your schema
+3. Start the database
+4. Perform operations like create, query, update, and delete
 
 ```ts
 const db = new BasedDb({
   path: '/persistent-file-path',
 })
 ```
+
+## Schema Definition
+
+Define your schema using the `putSchema` method. Here is an example:
+
+```typescript
+await db.putSchema({
+  types: {
+    user: {
+      name: 'string',
+      age: 'number',
+      email: 'alias',
+      isNice: 'boolean',
+      roles: ['admin', 'editor', 'viewer'],
+      file: 'binary',
+      bestFriend: {
+        ref: 'user',
+        prop: 'bestFriend',
+      },
+      friends: {
+        items: {
+          ref: 'user',
+          prop: 'friends',
+        },
+      },
+    },
+    article: {
+      body: 'string',
+      myUniqueValuesCount: 'cardinality',
+      contributors: {
+        type: 'references',
+        items: {
+          ref: 'user',
+          prop: 'articles',
+          $role: ['writer', 'editor'],
+        },
+      },
+    },
+    page: {
+      name: 'string',
+      clients: {
+        items: {
+          ref: '_client',
+          prop: 'pages',
+          $viewers: 'uint8',
+        },
+      },
+      activeViewers: {
+        type: 'uint32',
+        path: 'clients.$viewers.#sum',
+        history: {
+          interval: 'second',
+        },
+      },
+    },
+    _client: {
+      name: 'string',
+    },
+  },
+})
+```
+
+## Data Operations
+
+### Create
+
+Create new records using the `create` method:
+
+```typescript
+const userId = await db.create('user', {
+  name: 'youzi',
+  email: 'youzi@example.com',
+  isNice: true,
+  roles: 'admin',
+})
+```
+
+### Query & Filters
+
+Query records using the `query` method:
+
+```typescript
+const results = await db
+  .query('user')
+  .filter('isNice', '=', true)
+  .filter('name', '=', 'youzi')
+  .get()
+  .toObject()
+```
+
+### Update
+
+Update records using the `update` method:
+
+```typescript
+await db.update('user', userId, {
+  roles: 'editor',
+})
+```
+
+### Delete
+
+Delete records using the `delete` method:
+
+```typescript
+await db.delete('user', userId)
+```
+
+## Advanced Features
+
+### Copy
+
+Copy records with transformations:
+
+```typescript
+await db.copy('edition', editionId, {
+  copiedByYouzi: true,
+  versionOf({ id }) {
+    return id
+  },
+  name({ name }) {
+    return name + ' (edition copy)'
+  },
+  sequences({ sequences }) {
+    return sequences.map(({ id }) => {
+      return db.copy('sequence', id, {
+        copiedByYouzi: true,
+        name({ name }) {
+          return name + ' (sequence copy)'
+        },
+        pages({ pages }) {
+          return pages.map(({ id }) =>
+            db.copy('page', id, {
+              copiedByYouzi: true,
+              name({ name }) {
+                return name + ' (page copy)'
+              },
+            }),
+          )
+        },
+      })
+    })
+  },
+})
+```
+
+### Concurrency
+
+Concurrent write and read operations are supported. The example below shows multiple concurrent queries and creates. Handle concurrency carefully to avoid conflicts:
+
+```typescript
+let id = 0
+let queries = 0
+let refs = []
+let timer = setTimeout(() => {
+  timer = null
+}, 5e3)
+
+const query = async () => {
+  queries++
+  try {
+    await db.query('user').include('friends').range(0, 1000_000).get()
+  } catch (e) {
+    console.error('err:', e)
+  }
+  queries--
+}
+
+while (timer) {
+  let i = 100
+  while (i--) {
+    query()
+  }
+  while (timer && queries) {
+    db.create('user', {
+      friends: refs,
+    })
+    refs.push(++id)
+    await db.drain()
+    await setTimeoutAsync()
+  }
+}
+
+clearTimeout(timer)
+```
+
+### Client-Server
+
+Set up a client-server architecture:
+
+```typescript
+const server = new DbServer({
+  path: t.tmp,
+  onSchemaChange(schema) {
+    client1.putLocalSchema(schema)
+    client2.putLocalSchema(schema)
+  },
+})
+
+await server.start({ clean: true })
+
+const hooks: DbClientHooks = {
+  async putSchema(schema, fromStart, transformFns) {
+    return server.putSchema(schema, fromStart, transformFns)
+  },
+  async flushModify(buf) {
+    const offsets = server.modify(buf)
+    return { offsets }
+  },
+  async getQueryBuf(buf) {
+    return server.getQueryBuf(buf)
+  },
+}
+
+const client1 = new DbClient({ hooks })
+const client2 = new DbClient({ hooks })
+
+await client1.putSchema({
+  types: {
+    user: {
+      name: 'string',
+    },
+  },
+})
+
+const youzi = await client1.create('user', { name: 'youzi' })
+const jamez = await client1.create('user', { name: 'jamez' })
+
+deepEqual(await client1.query('user').get().toObject(), [
+  { id: 1, name: 'youzi' },
+  { id: 2, name: 'jamez' },
+])
+```
+
+### Checksum
+
+Include checksum in queries:
+
+```typescript
+await db.query('article').include('*', '_checksum')
+```
+
+### Cardinality
+
+Use cardinality for unique value counts:
+
+```typescript
+await db.putSchema({
+  types: {
+    article: {
+      myUniqueValuesCount: 'cardinality',
+    },
+  },
+})
+
+const myArticle = await db.create('article', {
+  myUniqueValuesCount: 'myCoolValue',
+})
+```
+
+### Boolean
+
+Handle boolean properties:
+
+```typescript
+await db.putSchema({
+  types: {
+    user: {
+      props: {
+        isNice: 'boolean',
+      },
+    },
+  },
+})
+
+db.create('user', {})
+db.create('user', { isNice: true })
+db.create('user', { isNice: false })
+
+await db.drain()
+
+deepEqual((await db.query('user').get()).toObject(), [
+  { id: 1, isNice: false },
+  { id: 2, isNice: true },
+  { id: 3, isNice: false },
+])
+```
+
+### Binary
+
+Handle binary data:
+
+```typescript
+await db.putSchema({
+  types: {
+    user: {
+      props: {
+        file: { type: 'binary' },
+      },
+    },
+  },
+})
+
+db.create('user', {
+  file: new Uint32Array([1, 2, 3, 4]),
+})
+
+await db.drain()
+
+deepEqual((await db.query('user').get()).toObject(), [
+  {
+    id: 1,
+    file: new Uint8Array(new Uint32Array([1, 2, 3, 4]).buffer),
+  },
+])
+```
+
+### Analytics
+
+Perform analytics on data:
+
+```typescript
+await db.putSchema({
+  types: {
+    page: {
+      name: 'string',
+      clients: {
+        items: {
+          ref: '_client',
+          prop: 'pages',
+          $viewers: 'uint8',
+        },
+      },
+      activeViewers: {
+        type: 'uint32',
+        path: 'clients.$viewers.#sum',
+        history: {
+          interval: 'second',
+        },
+      },
+    },
+  },
+})
+
+const client = await db.create('_client', {})
+const page = await db.create('page', {
+  clients: [
+    {
+      id: client,
+      $viewers: { increment: 1 },
+    },
+  ],
+})
+```
+
+### Alias
+
+Use aliases for properties:
+
+```typescript
+await db.putSchema({
+  types: {
+    user: {
+      props: {
+        externalId: 'alias',
+        potato: 'string',
+      },
+    },
+  },
+})
+
+const user1 = db.create('user', {
+  externalId: 'cool',
+})
+
+await db.drain()
+
+deepEqual((await db.query('user', user1).get()).toObject(), {
+  id: 1,
+  externalId: 'cool',
+  potato: '',
+})
+```

package/dist/lib/darwin_aarch64/include/selva/db.h

@@ -107,6 +107,15 @@ enum SelvaFieldType selva_get_fs_type(const struct SelvaFieldSchema *fs);
 SELVA_EXPORT
 const struct EdgeFieldConstraint *selva_get_edge_field_constraint(const struct SelvaFieldSchema *fs) __attribute__((nonnull));
 
+SELVA_EXPORT
+const struct SelvaFieldsSchema *selva_get_edge_field_fields_schema(struct SelvaDb *db, const struct EdgeFieldConstraint *efc);
+
+SELVA_EXPORT
+void selva_expire_node(struct SelvaDb *db, node_type_t type, node_id_t node_id, int64_t ts);
+
+SELVA_EXPORT
+void selva_db_expire_tick(struct SelvaDb *db, int64_t now);
+
 /**
  * Delete a node.
  */
@@ -216,13 +225,13 @@ node_id_t selva_get_node_id(const struct SelvaNode *node) __attribute__((nonnull
  * @param tmp_hash_state is only used for computation and it's reset before use.
  */
 SELVA_EXPORT
-selva_hash128_t selva_node_hash_update(struct SelvaTypeEntry *type, struct SelvaNode *node, struct XXH3_state_s *tmp_hash_state);
+selva_hash128_t selva_node_hash_update(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node, struct XXH3_state_s *tmp_hash_state);
 
 SELVA_EXPORT
-selva_hash128_t selva_node_hash_update2(struct SelvaTypeEntry *type, struct SelvaNode *node);
+selva_hash128_t selva_node_hash_update2(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node);
 
 SELVA_EXPORT
-selva_hash128_t selva_node_hash_range(struct SelvaTypeEntry *type, node_id_t start, node_id_t end) __attribute__((nonnull));
+selva_hash128_t selva_node_hash_range(struct SelvaDb *db, struct SelvaTypeEntry *type, node_id_t start, node_id_t end) __attribute__((nonnull));
 
 /**
  * @}

package/dist/lib/darwin_aarch64/include/selva/fields.h

@@ -87,6 +87,7 @@ struct SelvaFieldsAny {
         struct SelvaNodeWeakReferences weak_references; /*!< SELVA_FIELD_TYPE_WEAK_REFERENCES */
         struct SelvaMicroBuffer *smb; /*!< SELVA_FIELD_TYPE_MICRO_BUFFER */
 #endif
+        // HyperLogLogPlusPlus cardinality; /*!< SELVA_FIELD_TYPE_HLL */
     };
 };
 
@@ -137,6 +138,17 @@ int selva_fields_get_mutable_string(
         struct selva_string **s)
     __attribute__((access(write_only, 4)));
 
+/*
+ * TODO prefix with selva_
+ * TODO Document diff to get_mutable_string
+ */
+SELVA_EXPORT
+struct selva_string *fields_ensure_string(
+        struct SelvaDb *db,
+        struct SelvaNode *node,
+        const struct SelvaFieldSchema *fs,
+        size_t initial_len);
+
 SELVA_EXPORT
 int selva_fields_reference_set(
         struct SelvaDb *db,
@@ -204,6 +216,7 @@ int selva_fields_references_swap(
 
 SELVA_EXPORT
 int selva_fields_set_reference_meta(
+        struct SelvaDb *db,
         struct SelvaNode *node,
         struct SelvaNodeReference *ref,
         const struct EdgeFieldConstraint *efc,
@@ -212,6 +225,7 @@ int selva_fields_set_reference_meta(
 
 SELVA_EXPORT
 int selva_fields_get_reference_meta_mutable_string(
+        struct SelvaDb *db,
         struct SelvaNode *node,
         struct SelvaNodeReference *ref,
         const struct EdgeFieldConstraint *efc,
@@ -283,6 +297,12 @@ struct SelvaNode *selva_fields_resolve_weak_reference(
         const struct SelvaFieldSchema *fs,
         const struct SelvaNodeWeakReference *weak_ref);
 
+SELVA_EXPORT
+struct selva_string *selva_fields_get_selva_string2(struct SelvaFields *fields, const struct SelvaFieldSchema *fs);
+
+SELVA_EXPORT
+struct selva_string *selva_fields_get_selva_string(struct SelvaNode *node, const struct SelvaFieldSchema *fs);
+
 SELVA_EXPORT
 struct SelvaFieldsPointer selva_fields_get_raw2(struct SelvaFields *fields, const struct SelvaFieldSchema *fs)
     __attribute__((nonnull));
@@ -319,20 +339,11 @@ void selva_fields_init(const struct SelvaFieldsSchema *schema, struct SelvaField
 SELVA_EXPORT
 void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node);
 
-SELVA_EXPORT
-int selva_fields_get_string_crc(const struct SelvaNode *node, const struct SelvaFieldSchema *fs, uint32_t *crc);
-
-SELVA_EXPORT
-int selva_fields_get_string_crc2(const struct SelvaFields *fields, const struct SelvaFieldSchema *fs, uint32_t *crc);
-
 SELVA_EXPORT
 int selva_fields_get_text_crc(const struct SelvaNode *node, const struct SelvaFieldSchema *fs, enum selva_lang_code lang, uint32_t *crc);
 
 SELVA_EXPORT
-int selva_fields_get_text_crc2(const struct SelvaFields *fields, const struct SelvaFieldSchema *fs, enum selva_lang_code lang, uint32_t *crc);
-
-SELVA_EXPORT
-void selva_fields_hash_update(struct XXH3_state_s *hash_state, const struct SelvaFieldsSchema *schema, const struct SelvaFields *fields);
+void selva_fields_hash_update(struct XXH3_state_s *hash_state, struct SelvaDb *db, const struct SelvaFieldsSchema *schema, const struct SelvaFields *fields);
 
 SELVA_EXPORT
-selva_hash128_t selva_fields_hash(const struct SelvaFieldsSchema *schema, const struct SelvaFields *fields);
+selva_hash128_t selva_fields_hash(struct SelvaDb *db, const struct SelvaFieldsSchema *schema, const struct SelvaFields *fields);

package/dist/lib/darwin_aarch64/include/selva/history.h

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2025 SAULX
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "selva/types.h"
+#include "selva/_export.h"
+
+struct selva_history;
+
+struct selva_history_event {
+    int64_t ts;
+    node_id_t node_id;
+    uint32_t crc;
+} __packed __attribute__((aligned(4)));
+
+static_assert(alignof(struct selva_history_event) == alignof(uint32_t));
+
+SELVA_EXPORT
+int selva_history_init(const char *pathname, size_t bsize, struct selva_history **hist_out);
+
+SELVA_EXPORT
+void selva_history_destroy(struct selva_history *hist);
+
+SELVA_EXPORT
+void selva_history_append(struct selva_history *hist, int64_t ts, node_id_t node_id, void *buf);
+
+SELVA_EXPORT
+void selva_history_fsync(struct selva_history *hist);
+
+/**
+ * Find a range.
+ * The returned buffer must be freed with selva_history_free_range().
+ */
+SELVA_EXPORT
+uint32_t *selva_history_find_range(struct selva_history *hist, int64_t from, int64_t to, size_t *size_out);
+
+/**
+ * Find a range.
+ * The returned buffer must be freed with selva_history_free_range().
+ */
+SELVA_EXPORT
+uint32_t *selva_history_find_range_node(struct selva_history *hist, int64_t from, int64_t to, node_id_t node_id, size_t *size_out);
+
+SELVA_EXPORT
+void selva_history_free_range(uint32_t *range);

package/dist/lib/darwin_aarch64/include/selva/hll.h

@@ -0,0 +1,21 @@
+#ifndef UTIL_HLL_H
+#define UTIL_HLL_H
+
+#include <stdlib.h>
+#include <stdint.h>
+#include "selva/_export.h"
+#include "selva_string.h"
+#include <stdbool.h>
+#include "cdefs.h"
+
+
+SELVA_EXPORT
+void hll_init(struct selva_string *hllss, uint8_t precision, bool is_sparse);
+SELVA_EXPORT
+void hll_add(struct selva_string *hllss, uint64_t element);
+SELVA_EXPORT
+double hll_count(struct selva_string *hllss);
+SELVA_EXPORT
+struct selva_string hll_array_union(struct selva_string *hll_array, size_t count);
+
+#endif

package/dist/lib/darwin_aarch64/include/selva/sort.h

@@ -30,6 +30,8 @@ enum SelvaSortOrder {
     SELVA_SORT_ORDER_NONE = 0,
     SELVA_SORT_ORDER_I64_ASC,
     SELVA_SORT_ORDER_I64_DESC,
+    SELVA_SORT_ORDER_FLOAT_ASC,
+    SELVA_SORT_ORDER_FLOAT_DESC,
     SELVA_SORT_ORDER_DOUBLE_ASC,
     SELVA_SORT_ORDER_DOUBLE_DESC,
     SELVA_SORT_ORDER_BUFFER_ASC,
@@ -58,6 +60,9 @@ void selva_sort_insert(struct SelvaSortCtx *ctx, const void *p);
 SELVA_EXPORT
 void selva_sort_insert_i64(struct SelvaSortCtx *ctx, int64_t v, const void *p);
 
+SELVA_EXPORT
+void selva_sort_insert_float(struct SelvaSortCtx *ctx, float f, const void *p);
+
 SELVA_EXPORT
 void selva_sort_insert_double(struct SelvaSortCtx *ctx, double d, const void *p);
 
@@ -73,6 +78,9 @@ void selva_sort_remove(struct SelvaSortCtx *ctx, const void *p);
 SELVA_EXPORT
 void selva_sort_remove_i64(struct SelvaSortCtx *ctx, int64_t v, const void *p);
 
+SELVA_EXPORT
+void selva_sort_remove_float(struct SelvaSortCtx *ctx, float f, const void *p);
+
 SELVA_EXPORT
 void selva_sort_remove_double(struct SelvaSortCtx *ctx, double d, const void *p);
 
@@ -100,6 +108,12 @@ void *selva_sort_foreach_i64(struct SelvaSortCtx *ctx, int64_t *v);
 SELVA_EXPORT
 void *selva_sort_foreach_i64_reverse(struct SelvaSortCtx *ctx, int64_t *v);
 
+SELVA_EXPORT
+void *selva_sort_foreach_float(struct SelvaSortCtx *ctx, float *f);
+
+SELVA_EXPORT
+void *selva_sort_foreach_float_reverse(struct SelvaSortCtx *ctx, float *f);
+
 SELVA_EXPORT
 void *selva_sort_foreach_double(struct SelvaSortCtx *ctx, double *d);
 

package/dist/lib/darwin_aarch64/include/selva/types.h

@@ -55,14 +55,19 @@ enum SelvaFieldType {
 
 struct EdgeFieldConstraint {
     enum EdgeFieldConstraintFlag {
+        EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT = 0x01,
         /**
          * Skip saving this field while dumping.
          */
-        EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP        = 0x80,
+        EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80,
     } __packed flags;
     field_t inverse_field;
     node_type_t dst_node_type;
-    struct SelvaFieldsSchema *fields_schema;
+    /**
+     * Don't use directly!
+     * Use: `selva_get_edge_field_fields_schema()`
+     */
+    struct SelvaFieldsSchema *_fields_schema;
 };
 
 struct SelvaFieldSchema {
@@ -92,7 +97,9 @@ struct SelvaFieldsSchema {
 };
 
 struct SelvaNodeSchema {
+    size_t nr_aliases; /*!< Number of alias fields in this type. */
     struct SelvaFieldsSchema fields_schema;
+    /* Nothing must be put after this line. */
 };
 
 struct SelvaAlias;