@based/db 0.0.30 → 0.0.31

package/dist/lib/darwin_aarch64/include/selva/db.h

@@ -209,9 +209,10 @@ void selva_db_expire_tick(struct SelvaDb *db, int64_t now);
 
 /**
  * Delete a node.
+ * @param dirty_cb is called for any newly dirty nodes in addition to the node being deleted.
  */
 SELVA_EXPORT
-void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node) __attribute__((nonnull));
+void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx) __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Get a node by id.

package/dist/lib/darwin_aarch64/include/selva/fields.h

@@ -341,10 +341,11 @@ struct SelvaFieldsPointer selva_fields_get_raw(struct SelvaNode *node, const str
 
 /**
  * Delete field.
+ * @param dirty_cb will be called with the deleted node_id in case of a reference(s) field.
  */
 SELVA_EXPORT
-int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Delete an edge from a references field.
@@ -352,9 +353,12 @@ int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct Se
 SELVA_EXPORT
 int selva_fields_del_ref(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, node_id_t dst_node_id);
 
+/**
+ * Clear a references field but don't free it.
+ */
 SELVA_EXPORT
-void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Init the fields struct of a node or edge.
@@ -369,7 +373,8 @@ void selva_fields_init(const struct SelvaFieldsSchema *schema, struct SelvaField
  * regardless wether the schema defines fields for this node.
  */
 SELVA_EXPORT
-void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node);
+void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2)));
 
 SELVA_EXPORT
 void selva_fields_hash_update(struct XXH3_state_s *hash_state, struct SelvaDb *db, const struct SelvaFieldsSchema *schema, const struct SelvaFields *fields);

package/dist/lib/darwin_aarch64/include/selva/types.h

@@ -43,7 +43,7 @@ struct EdgeFieldConstraint {
         EDGE_FIELD_CONSTRAINT_FLAG_SCHEMA_REF_CACHED = 0x40,
         /**
          * Skip saving this field while dumping.
-         * If the field is of type SELVA_FIELD_TYPE_WEAK_REFERENCES it's saved
+         * If the field is of type SELVA_FIELD_TYPE_REFERENCES it's saved
          * regardless of this flag to preserve the original order of references.
          */
         EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80,
@@ -97,6 +97,8 @@ struct SelvaFields;
 struct SelvaNode;
 struct SelvaTypeEntry;
 
+typedef void (*selva_dirty_node_cb_t)(void *ctx, node_type_t type, node_id_t node_id);
+
 SELVA_EXPORT
 bool selva_is_valid_field_type(enum SelvaFieldType ftype);
 

package/dist/lib/linux_aarch64/include/selva/db.h

@@ -209,9 +209,10 @@ void selva_db_expire_tick(struct SelvaDb *db, int64_t now);
 
 /**
  * Delete a node.
+ * @param dirty_cb is called for any newly dirty nodes in addition to the node being deleted.
  */
 SELVA_EXPORT
-void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node) __attribute__((nonnull));
+void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx) __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Get a node by id.

package/dist/lib/linux_aarch64/include/selva/fields.h

@@ -341,10 +341,11 @@ struct SelvaFieldsPointer selva_fields_get_raw(struct SelvaNode *node, const str
 
 /**
  * Delete field.
+ * @param dirty_cb will be called with the deleted node_id in case of a reference(s) field.
  */
 SELVA_EXPORT
-int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Delete an edge from a references field.
@@ -352,9 +353,12 @@ int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct Se
 SELVA_EXPORT
 int selva_fields_del_ref(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, node_id_t dst_node_id);
 
+/**
+ * Clear a references field but don't free it.
+ */
 SELVA_EXPORT
-void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Init the fields struct of a node or edge.
@@ -369,7 +373,8 @@ void selva_fields_init(const struct SelvaFieldsSchema *schema, struct SelvaField
  * regardless wether the schema defines fields for this node.
  */
 SELVA_EXPORT
-void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node);
+void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2)));
 
 SELVA_EXPORT
 void selva_fields_hash_update(struct XXH3_state_s *hash_state, struct SelvaDb *db, const struct SelvaFieldsSchema *schema, const struct SelvaFields *fields);

package/dist/lib/linux_aarch64/include/selva/types.h

@@ -43,7 +43,7 @@ struct EdgeFieldConstraint {
         EDGE_FIELD_CONSTRAINT_FLAG_SCHEMA_REF_CACHED = 0x40,
         /**
          * Skip saving this field while dumping.
-         * If the field is of type SELVA_FIELD_TYPE_WEAK_REFERENCES it's saved
+         * If the field is of type SELVA_FIELD_TYPE_REFERENCES it's saved
          * regardless of this flag to preserve the original order of references.
          */
         EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80,
@@ -97,6 +97,8 @@ struct SelvaFields;
 struct SelvaNode;
 struct SelvaTypeEntry;
 
+typedef void (*selva_dirty_node_cb_t)(void *ctx, node_type_t type, node_id_t node_id);
+
 SELVA_EXPORT
 bool selva_is_valid_field_type(enum SelvaFieldType ftype);
 

package/dist/lib/linux_x86_64/include/selva/db.h

@@ -209,9 +209,10 @@ void selva_db_expire_tick(struct SelvaDb *db, int64_t now);
 
 /**
  * Delete a node.
+ * @param dirty_cb is called for any newly dirty nodes in addition to the node being deleted.
  */
 SELVA_EXPORT
-void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node) __attribute__((nonnull));
+void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx) __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Get a node by id.

package/dist/lib/linux_x86_64/include/selva/fields.h

@@ -341,10 +341,11 @@ struct SelvaFieldsPointer selva_fields_get_raw(struct SelvaNode *node, const str
 
 /**
  * Delete field.
+ * @param dirty_cb will be called with the deleted node_id in case of a reference(s) field.
  */
 SELVA_EXPORT
-int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Delete an edge from a references field.
@@ -352,9 +353,12 @@ int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct Se
 SELVA_EXPORT
 int selva_fields_del_ref(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, node_id_t dst_node_id);
 
+/**
+ * Clear a references field but don't free it.
+ */
 SELVA_EXPORT
-void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Init the fields struct of a node or edge.
@@ -369,7 +373,8 @@ void selva_fields_init(const struct SelvaFieldsSchema *schema, struct SelvaField
  * regardless wether the schema defines fields for this node.
  */
 SELVA_EXPORT
-void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node);
+void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2)));
 
 SELVA_EXPORT
 void selva_fields_hash_update(struct XXH3_state_s *hash_state, struct SelvaDb *db, const struct SelvaFieldsSchema *schema, const struct SelvaFields *fields);

package/dist/lib/linux_x86_64/include/selva/types.h

@@ -43,7 +43,7 @@ struct EdgeFieldConstraint {
         EDGE_FIELD_CONSTRAINT_FLAG_SCHEMA_REF_CACHED = 0x40,
         /**
          * Skip saving this field while dumping.
-         * If the field is of type SELVA_FIELD_TYPE_WEAK_REFERENCES it's saved
+         * If the field is of type SELVA_FIELD_TYPE_REFERENCES it's saved
          * regardless of this flag to preserve the original order of references.
          */
         EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80,
@@ -97,6 +97,8 @@ struct SelvaFields;
 struct SelvaNode;
 struct SelvaTypeEntry;
 
+typedef void (*selva_dirty_node_cb_t)(void *ctx, node_type_t type, node_id_t node_id);
+
 SELVA_EXPORT
 bool selva_is_valid_field_type(enum SelvaFieldType ftype);
 

package/dist/src/index.js

@@ -4,6 +4,7 @@ import { DbServer } from './server/index.js';
 import { DbClient } from './client/index.js';
 import { wait } from '@saulx/utils';
 import { debugMode, debugServer } from './utils.js';
+import { BasedQueryResponse } from './client/query/BasedIterable.js';
 export * from './client/modify/modify.js';
 export { compress, decompress };
 export { ModifyCtx }; // TODO move this somewhere
@@ -45,8 +46,31 @@ export class BasedDb {
             maxModifySize,
             hooks: {
                 subscribe(q, onData, onError) {
-                    console.warn('Subscription not supported without based-server!');
-                    return () => { };
+                    let timer;
+                    let prevChecksum;
+                    let lastLen = 0;
+                    let response;
+                    const get = async () => {
+                        const res = await server.getQueryBuf(q.buffer);
+                        if (!response) {
+                            response = new BasedQueryResponse(q.id, q.def, res, 0);
+                        }
+                        else {
+                            response.result = res;
+                            response.end = res.byteLength;
+                        }
+                        const checksum = response.checksum;
+                        if (lastLen != res.byteLength || checksum != prevChecksum) {
+                            onData(response);
+                            lastLen = res.byteLength;
+                            prevChecksum = checksum;
+                        }
+                        setTimeout(get, 200);
+                    };
+                    get();
+                    return () => {
+                        clearTimeout(timer);
+                    };
                 },
                 setSchema(schema, fromStart) {
                     return Promise.resolve(server.setSchema(schema, fromStart));

package/dist/src/server/save.js

@@ -12,9 +12,8 @@ function saveRange(db, typeId, start, end, hashOut) {
     const file = block_sdb_file(typeId, start, end);
     const path = join(db.fileSystemPath, file);
     const err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hashOut);
-    if (err == -8) {
-        // TODO ENOENT
-        return '';
+    if (err == -8) { // TODO ENOENT
+        return ''; // empty range
     }
     else if (err) {
         // TODO print the error string
@@ -61,18 +60,16 @@ export function save(db, sync = false, forceFullDump = false) {
     else {
         foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
             const hash = new Uint8Array(16);
-            // TODO Don't save if empty
             const file = saveRange(db, typeId, start, end, hash);
             if (file === null) {
                 // The previous state should remain in the merkle tree for
                 // load and sync purposes.
                 return;
             }
-            else if (file.length === 0) {
-                // The range is empty
-            }
             else {
                 const oldLeaf = db.merkleTree.search(mtKey);
+                // If (file.length === 0) then the range is empty but that's fine,
+                // we'll keep them around for now to maintain the order of the tree.
                 if (oldLeaf) {
                     oldLeaf.data.file = file;
                     db.merkleTree.update(mtKey, hash);

package/dist/src/server/start.js

@@ -36,11 +36,13 @@ export async function start(db, opts) {
             const dumps = writelog.rangeDumps[typeId];
             for (const dump of dumps) {
                 const fname = dump.file;
-                try {
-                    native.loadRange(join(path, fname), db.dbCtxExternal);
-                }
-                catch (e) {
-                    console.error(e.message);
+                if (fname?.length > 0) {
+                    try {
+                        native.loadRange(join(path, fname), db.dbCtxExternal);
+                    }
+                    catch (e) {
+                        console.error(e.message);
+                    }
                 }
             }
         }
@@ -71,7 +73,7 @@ export async function start(db, opts) {
                 end,
             };
             db.merkleTree.insert(mtKey, hash, data);
-        });
+        }, true);
     }
     if (writelog?.hash) {
         const oldHash = hexToBuf(writelog.hash);

package/dist/src/server/tree.d.ts

@@ -11,5 +11,5 @@ export declare const destructureCsmtKey: (key: number) => number[];
 export declare const makeCsmtKey: (typeId: number, start: number) => number;
 export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
 export declare function initCsmt(db: DbServer): {};
-export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): void;
+export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void, includeEmptyBlocks?: boolean): void;
 export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;

package/dist/src/server/tree.js

@@ -38,13 +38,13 @@ export function initCsmt(db) {
     }
     return types;
 }
-export function foreachBlock(db, def, cb) {
+export function foreachBlock(db, def, cb, includeEmptyBlocks = false) {
     const step = def.blockCapacity;
     for (let start = 1; start <= def.lastId; start += step) {
         const end = start + step - 1;
         const hash = new Uint8Array(16);
         const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
-        if (res) {
+        if (res || includeEmptyBlocks) {
             cb(start, end, hash);
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.30",
+  "version": "0.0.31",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",