@based/db 0.0.29 → 0.0.31

package/dist/lib/darwin_aarch64/include/selva/db.h

@@ -209,9 +209,10 @@ void selva_db_expire_tick(struct SelvaDb *db, int64_t now);
 
 /**
  * Delete a node.
+ * @param dirty_cb is called for any newly dirty nodes in addition to the node being deleted.
  */
 SELVA_EXPORT
-void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node) __attribute__((nonnull));
+void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx) __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Get a node by id.

package/dist/lib/darwin_aarch64/include/selva/fields.h

@@ -152,14 +152,22 @@ struct selva_string *selva_fields_ensure_string2(
         const struct SelvaFieldSchema *fs,
         size_t initial_len);
 
+/**
+ * Set reference to fields.
+ * @param dirty_nodes returns the nodes that were changed, apart from src and dst.
+ *                    [n].id = 0 = nil;
+ *                    [0] = the node src was pointing to previously (same type as dst);
+ *                    [1] = the node dst was pointing to previously (same type as src).
+ */
 SELVA_EXPORT
 int selva_fields_reference_set(
         struct SelvaDb *db,
         struct SelvaNode * restrict src,
         const struct SelvaFieldSchema *fs_src,
         struct SelvaNode * restrict dst,
-        struct SelvaNodeReference **ref_out)
-    __attribute__((access(write_only, 5)));
+        struct SelvaNodeReference **ref_out,
+        node_id_t dirty_nodes[static 2])
+    __attribute__((access(write_only, 5), access(write_only, 6)));
 
 /**
  * @param index 0 = first; -1 = last.
@@ -333,10 +341,11 @@ struct SelvaFieldsPointer selva_fields_get_raw(struct SelvaNode *node, const str
 
 /**
  * Delete field.
+ * @param dirty_cb will be called with the deleted node_id in case of a reference(s) field.
  */
 SELVA_EXPORT
-int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Delete an edge from a references field.
@@ -344,9 +353,12 @@ int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct Se
 SELVA_EXPORT
 int selva_fields_del_ref(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, node_id_t dst_node_id);
 
+/**
+ * Clear a references field but don't free it.
+ */
 SELVA_EXPORT
-void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Init the fields struct of a node or edge.
@@ -361,7 +373,8 @@ void selva_fields_init(const struct SelvaFieldsSchema *schema, struct SelvaField
  * regardless wether the schema defines fields for this node.
  */
 SELVA_EXPORT
-void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node);
+void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2)));
 
 SELVA_EXPORT
 void selva_fields_hash_update(struct XXH3_state_s *hash_state, struct SelvaDb *db, const struct SelvaFieldsSchema *schema, const struct SelvaFields *fields);

package/dist/lib/darwin_aarch64/include/selva/types.h

@@ -43,7 +43,7 @@ struct EdgeFieldConstraint {
         EDGE_FIELD_CONSTRAINT_FLAG_SCHEMA_REF_CACHED = 0x40,
         /**
          * Skip saving this field while dumping.
-         * If the field is of type SELVA_FIELD_TYPE_WEAK_REFERENCES it's saved
+         * If the field is of type SELVA_FIELD_TYPE_REFERENCES it's saved
          * regardless of this flag to preserve the original order of references.
          */
         EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80,
@@ -97,6 +97,8 @@ struct SelvaFields;
 struct SelvaNode;
 struct SelvaTypeEntry;
 
+typedef void (*selva_dirty_node_cb_t)(void *ctx, node_type_t type, node_id_t node_id);
+
 SELVA_EXPORT
 bool selva_is_valid_field_type(enum SelvaFieldType ftype);
 

package/dist/lib/linux_aarch64/include/selva/db.h

@@ -209,9 +209,10 @@ void selva_db_expire_tick(struct SelvaDb *db, int64_t now);
 
 /**
  * Delete a node.
+ * @param dirty_cb is called for any newly dirty nodes in addition to the node being deleted.
  */
 SELVA_EXPORT
-void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node) __attribute__((nonnull));
+void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx) __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Get a node by id.

package/dist/lib/linux_aarch64/include/selva/fields.h

@@ -152,14 +152,22 @@ struct selva_string *selva_fields_ensure_string2(
         const struct SelvaFieldSchema *fs,
         size_t initial_len);
 
+/**
+ * Set reference to fields.
+ * @param dirty_nodes returns the nodes that were changed, apart from src and dst.
+ *                    [n].id = 0 = nil;
+ *                    [0] = the node src was pointing to previously (same type as dst);
+ *                    [1] = the node dst was pointing to previously (same type as src).
+ */
 SELVA_EXPORT
 int selva_fields_reference_set(
         struct SelvaDb *db,
         struct SelvaNode * restrict src,
         const struct SelvaFieldSchema *fs_src,
         struct SelvaNode * restrict dst,
-        struct SelvaNodeReference **ref_out)
-    __attribute__((access(write_only, 5)));
+        struct SelvaNodeReference **ref_out,
+        node_id_t dirty_nodes[static 2])
+    __attribute__((access(write_only, 5), access(write_only, 6)));
 
 /**
  * @param index 0 = first; -1 = last.
@@ -333,10 +341,11 @@ struct SelvaFieldsPointer selva_fields_get_raw(struct SelvaNode *node, const str
 
 /**
  * Delete field.
+ * @param dirty_cb will be called with the deleted node_id in case of a reference(s) field.
  */
 SELVA_EXPORT
-int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Delete an edge from a references field.
@@ -344,9 +353,12 @@ int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct Se
 SELVA_EXPORT
 int selva_fields_del_ref(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, node_id_t dst_node_id);
 
+/**
+ * Clear a references field but don't free it.
+ */
 SELVA_EXPORT
-void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Init the fields struct of a node or edge.
@@ -361,7 +373,8 @@ void selva_fields_init(const struct SelvaFieldsSchema *schema, struct SelvaField
  * regardless wether the schema defines fields for this node.
  */
 SELVA_EXPORT
-void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node);
+void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2)));
 
 SELVA_EXPORT
 void selva_fields_hash_update(struct XXH3_state_s *hash_state, struct SelvaDb *db, const struct SelvaFieldsSchema *schema, const struct SelvaFields *fields);

package/dist/lib/linux_aarch64/include/selva/types.h

@@ -43,7 +43,7 @@ struct EdgeFieldConstraint {
         EDGE_FIELD_CONSTRAINT_FLAG_SCHEMA_REF_CACHED = 0x40,
         /**
          * Skip saving this field while dumping.
-         * If the field is of type SELVA_FIELD_TYPE_WEAK_REFERENCES it's saved
+         * If the field is of type SELVA_FIELD_TYPE_REFERENCES it's saved
          * regardless of this flag to preserve the original order of references.
          */
         EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80,
@@ -97,6 +97,8 @@ struct SelvaFields;
 struct SelvaNode;
 struct SelvaTypeEntry;
 
+typedef void (*selva_dirty_node_cb_t)(void *ctx, node_type_t type, node_id_t node_id);
+
 SELVA_EXPORT
 bool selva_is_valid_field_type(enum SelvaFieldType ftype);
 

package/dist/lib/linux_x86_64/include/selva/db.h

@@ -209,9 +209,10 @@ void selva_db_expire_tick(struct SelvaDb *db, int64_t now);
 
 /**
  * Delete a node.
+ * @param dirty_cb is called for any newly dirty nodes in addition to the node being deleted.
  */
 SELVA_EXPORT
-void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node) __attribute__((nonnull));
+void selva_del_node(struct SelvaDb *db, struct SelvaTypeEntry *type, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx) __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Get a node by id.

package/dist/lib/linux_x86_64/include/selva/fields.h

@@ -152,14 +152,22 @@ struct selva_string *selva_fields_ensure_string2(
         const struct SelvaFieldSchema *fs,
         size_t initial_len);
 
+/**
+ * Set reference to fields.
+ * @param dirty_nodes returns the nodes that were changed, apart from src and dst.
+ *                    [n].id = 0 = nil;
+ *                    [0] = the node src was pointing to previously (same type as dst);
+ *                    [1] = the node dst was pointing to previously (same type as src).
+ */
 SELVA_EXPORT
 int selva_fields_reference_set(
         struct SelvaDb *db,
         struct SelvaNode * restrict src,
         const struct SelvaFieldSchema *fs_src,
         struct SelvaNode * restrict dst,
-        struct SelvaNodeReference **ref_out)
-    __attribute__((access(write_only, 5)));
+        struct SelvaNodeReference **ref_out,
+        node_id_t dirty_nodes[static 2])
+    __attribute__((access(write_only, 5), access(write_only, 6)));
 
 /**
  * @param index 0 = first; -1 = last.
@@ -333,10 +341,11 @@ struct SelvaFieldsPointer selva_fields_get_raw(struct SelvaNode *node, const str
 
 /**
  * Delete field.
+ * @param dirty_cb will be called with the deleted node_id in case of a reference(s) field.
  */
 SELVA_EXPORT
-int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Delete an edge from a references field.
@@ -344,9 +353,12 @@ int selva_fields_del(struct SelvaDb *db, struct SelvaNode *node, const struct Se
 SELVA_EXPORT
 int selva_fields_del_ref(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, node_id_t dst_node_id);
 
+/**
+ * Clear a references field but don't free it.
+ */
 SELVA_EXPORT
-void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs)
-    __attribute__((nonnull));
+void selva_fields_clear_references(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFieldSchema *fs, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2, 3)));
 
 /**
  * Init the fields struct of a node or edge.
@@ -361,7 +373,8 @@ void selva_fields_init(const struct SelvaFieldsSchema *schema, struct SelvaField
  * regardless wether the schema defines fields for this node.
  */
 SELVA_EXPORT
-void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node);
+void selva_fields_destroy(struct SelvaDb *db, struct SelvaNode *node, selva_dirty_node_cb_t dirty_cb, void *dirty_ctx)
+    __attribute__((nonnull(1, 2)));
 
 SELVA_EXPORT
 void selva_fields_hash_update(struct XXH3_state_s *hash_state, struct SelvaDb *db, const struct SelvaFieldsSchema *schema, const struct SelvaFields *fields);

package/dist/lib/linux_x86_64/include/selva/types.h

@@ -43,7 +43,7 @@ struct EdgeFieldConstraint {
         EDGE_FIELD_CONSTRAINT_FLAG_SCHEMA_REF_CACHED = 0x40,
         /**
          * Skip saving this field while dumping.
-         * If the field is of type SELVA_FIELD_TYPE_WEAK_REFERENCES it's saved
+         * If the field is of type SELVA_FIELD_TYPE_REFERENCES it's saved
          * regardless of this flag to preserve the original order of references.
          */
         EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80,
@@ -97,6 +97,8 @@ struct SelvaFields;
 struct SelvaNode;
 struct SelvaTypeEntry;
 
+typedef void (*selva_dirty_node_cb_t)(void *ctx, node_type_t type, node_id_t node_id);
+
 SELVA_EXPORT
 bool selva_is_valid_field_type(enum SelvaFieldType ftype);
 

package/dist/src/client/flushModify.js

@@ -90,11 +90,13 @@ export const flushBuffer = (db) => {
     const ctx = db.modifyCtx;
     let flushPromise;
     if (ctx.len) {
-        const d = Date.now();
         const lastIds = {};
         const data = ctx.getData(lastIds);
         const resCtx = ctx.ctx;
+        const d = performance.now();
         flushPromise = db.hooks.flushModify(data).then(({ offsets }) => {
+            // return time there
+            db.writeTime += performance.now() - d;
             resCtx.offsets = offsets;
             for (const typeId in lastIds) {
                 if (typeId in offsets) {
@@ -107,7 +109,7 @@ export const flushBuffer = (db) => {
                     }
                 }
                 else {
-                    console.warn('no offset returned, very wrong');
+                    console.error('Panic: No offset returned in flushModify');
                 }
             }
             if (resCtx.queue?.size) {
@@ -117,7 +119,6 @@ export const flushBuffer = (db) => {
                     resolve(res.getId());
                 }
             }
-            db.writeTime += Date.now() - d;
             db.flushReady();
         });
         ctx.dirtyTypes.clear();

package/dist/src/client/index.d.ts

@@ -2,18 +2,19 @@ import { Schema, StrictSchema } from '@based/schema';
 import { CreateObj } from './modify/create.js';
 import { SchemaTypeDef } from '@based/schema/def';
 import { ModifyCtx } from './flushModify.js';
-import { SubscriptionMarkerMap, SubscriptionsMap, SubscriptionsToRun } from './query/subscription/index.js';
 import { BasedDbQuery, QueryByAliasObj } from './query/BasedDbQuery.js';
 import { ModifyRes } from './modify/ModifyRes.js';
 import { DbServer } from '../server/index.js';
 import { TransformFns } from '../server/migrate/index.js';
 import { ModifyOpts } from './modify/types.js';
+import { OnClose, OnData, OnError } from './query/subscription/types.js';
 export type DbClientHooks = {
     setSchema(schema: StrictSchema, fromStart?: boolean, transformFns?: TransformFns): Promise<DbServer['schema']>;
     flushModify(buf: Uint8Array): Promise<{
         offsets: Record<number, number>;
     }>;
     getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
+    subscribe(q: BasedDbQuery, onData: OnData, onError?: OnError): OnClose;
 };
 type DbClientOpts = {
     hooks: DbClientHooks;
@@ -42,11 +43,6 @@ export declare class DbClient {
         o: Record<string, any>;
         p: Promise<number | ModifyRes>;
     }>;
-    subscriptionsInProgress: boolean;
-    subscriptonThrottleMs: number;
-    subscriptions: SubscriptionsMap;
-    subscriptionMarkers: SubscriptionMarkerMap;
-    subscriptionsToRun: SubscriptionsToRun;
     schemaChecksum: number;
     setSchema(schema: Schema, fromStart?: boolean, transformFns?: TransformFns): Promise<StrictSchema>;
     putLocalSchema(schema: any): DbClientSchema;

package/dist/src/client/index.js

@@ -47,12 +47,6 @@ export class DbClient {
     modifyCtx;
     maxModifySize;
     upserting = new Map();
-    // subscriptions
-    subscriptionsInProgress = false;
-    subscriptonThrottleMs = 20;
-    subscriptions = new Map();
-    subscriptionMarkers = {};
-    subscriptionsToRun = [];
     schemaChecksum;
     async setSchema(schema, fromStart, transformFns) {
         this.schemaIsSetValue = true;

package/dist/src/client/modify/ModifyRes.d.ts

@@ -1,5 +1,4 @@
 import { PropDef, PropDefEdge, SchemaPropTree } from '@based/schema/def';
-import { SubscriptionMarkersCheck } from '../query/subscription/index.js';
 import { DbClient } from '../index.js';
 import { ModifyOpts } from './types.js';
 import { LangCode } from '@based/schema';
@@ -12,8 +11,7 @@ export declare class ModifyError extends Error {
 }
 export declare class ModifyState {
     #private;
-    constructor(typeId: number, tmpId: number, db: DbClient, subMarkers: SubscriptionMarkersCheck | false, opts: ModifyOpts, update?: boolean);
-    subMarkers: SubscriptionMarkersCheck | false;
+    constructor(typeId: number, tmpId: number, db: DbClient, opts: ModifyOpts, update?: boolean);
     update: boolean;
     locale: LangCode;
     tmpId: number;

package/dist/src/client/modify/ModifyRes.js

@@ -42,17 +42,15 @@ export class ModifyError extends Error {
     }
 }
 export class ModifyState {
-    constructor(typeId, tmpId, db, subMarkers, opts, update = false) {
+    constructor(typeId, tmpId, db, opts, update = false) {
         this.tmpId = tmpId;
         this.#typeId = typeId;
         this.#ctx = db.modifyCtx.ctx;
-        this.subMarkers = subMarkers;
         this.update = update;
         if (opts?.locale) {
             this.locale = langCodesMap.get(opts.locale);
         }
     }
-    subMarkers;
     update;
     locale;
     #ctx;

package/dist/src/client/modify/create.js

@@ -5,7 +5,6 @@ import { modify } from './modify.js';
 import { ModifyState } from './ModifyRes.js';
 import { CREATE, RANGE_ERR, ADD_EMPTY_SORT, ADD_EMPTY_SORT_TEXT, } from './types.js';
 import { writeFixedValue } from './fixed.js';
-import { getSubscriptionMarkers } from '../query/subscription/index.js';
 const appendCreate = (ctx, def, obj, res, unsafe) => {
     const len = ctx.len;
     let err = modify(ctx, res, obj, def, CREATE, def.tree, true, unsafe);
@@ -136,7 +135,7 @@ export function create(db, type, obj, opts) {
         id = def.lastId + 1;
     }
     const ctx = db.modifyCtx;
-    const res = new ModifyState(def.id, id, db, getSubscriptionMarkers(db, def.id, id, true), opts);
+    const res = new ModifyState(def.id, id, db, opts);
     const pos = ctx.len;
     const err = appendCreate(ctx, def, obj, res, opts?.unsafe);
     if (err) {

package/dist/src/client/modify/delete.js

@@ -1,5 +1,4 @@
 import { flushBuffer, startDrain } from '../flushModify.js';
-import { getSubscriptionMarkers } from '../query/subscription/markers.js';
 import { ModifyState } from './ModifyRes.js';
 import { setCursor } from './setCursor.js';
 import { UPDATE, DELETE_SORT_INDEX, DELETE_NODE, } from './types.js';
@@ -10,7 +9,7 @@ export const deleteFn = (db, type, id, opts) => {
         throw new Error(`Unknown type: ${type}. Did you mean on of: ${Object.keys(db.schemaTypesParsed).join(', ')}`);
     }
     const ctx = db.modifyCtx;
-    const res = new ModifyState(def.id, id, db, getSubscriptionMarkers(db, def.id, id, true), opts);
+    const res = new ModifyState(def.id, id, db, opts);
     const schema = db.schemaTypesParsed[type];
     const separate = schema.separate;
     if (separate) {

package/dist/src/client/modify/modify.js

@@ -11,7 +11,6 @@ import { appendFixedValue, writeFixedValue } from './fixed.js';
 import { writeAlias } from './alias.js';
 import { writeHll } from './cardinality.js';
 import { writeVector } from './vector.js';
-import { checkSubscriptionMarkers } from '../query/subscription/index.js';
 import { writeJson } from './json.js';
 function _modify(ctx, res, obj, schema, mod, tree, overwrite, unsafe) {
     for (const key in obj) {
@@ -28,9 +27,6 @@ function _modify(ctx, res, obj, schema, mod, tree, overwrite, unsafe) {
             if (val === undefined) {
                 continue;
             }
-            if (res.subMarkers) {
-                checkSubscriptionMarkers(ctx.db, res.subMarkers, def);
-            }
             const type = def.typeIndex;
             if (def.separate) {
                 if (type === STRING) {

package/dist/src/client/modify/setCursor.js

@@ -1,5 +1,5 @@
 import { CREATE, SWITCH_FIELD, SWITCH_ID_CREATE, SWITCH_ID_UPDATE, SWITCH_TYPE, } from './types.js';
-export const setCursor = (ctx, schema, field, // PROPDEF
+export const setCursor = (ctx, schema, field, // TODO pass propdef better
 typeIndex, id, modifyOp, ignoreField) => {
     const prefix0 = schema.idUint8[0];
     const prefix1 = schema.idUint8[1];

package/dist/src/client/modify/update.js

@@ -5,7 +5,6 @@ import { modify } from './modify.js';
 import { ModifyState } from './ModifyRes.js';
 import { RANGE_ERR, UPDATE } from './types.js';
 import { appendFixedValue } from './fixed.js';
-import { getSubscriptionMarkers } from '../query/subscription/index.js';
 const appendUpdate = (ctx, def, obj, res, overwrite) => {
     const err = modify(ctx, res, obj, def, UPDATE, def.tree, overwrite);
     if (err) {
@@ -62,7 +61,7 @@ export const update = (db, type, id, obj, opts) => {
     }
     const ctx = db.modifyCtx;
     const pos = ctx.len;
-    const res = new ModifyState(def.id, id, db, getSubscriptionMarkers(db, def.id, id, false), opts, true);
+    const res = new ModifyState(def.id, id, db, opts, true);
     const err = appendUpdate(ctx, def, obj, res, opts?.overwrite);
     if (err) {
         ctx.prefix0 = -1; // force a new cursor

package/dist/src/client/query/read/read.js

@@ -146,7 +146,8 @@ const handleUndefinedProps = (id, q, item) => {
                     if (lang.has(0)) {
                         for (const locale in q.schema.locales) {
                             if (!lan[locale]) {
-                                lan[locale] = '';
+                                // console.log('???', locale, prop.default)
+                                lan[locale] = prop.default[locale] || '';
                             }
                         }
                     }
@@ -154,16 +155,24 @@ const handleUndefinedProps = (id, q, item) => {
                         for (const code of lang) {
                             const locale = inverseLangMap.get(code);
                             if (!lan[locale]) {
-                                lan[locale] = '';
+                                lan[locale] = prop.default[locale] || '';
                             }
                         }
                     }
                 }
                 else if (prop.typeIndex === BINARY) {
-                    addField(prop, new Uint8Array(0), item);
+                    addField(prop, prop.default, item);
+                }
+                else if (prop.typeIndex === TEXT) {
+                    // q.lang
+                    // console.log('???')
+                    addField(prop, '', item);
                 }
                 else {
-                    addField(prop, prop.typeIndex === JSON ? null : '', item);
+                    1;
+                    if (prop.default !== undefined) {
+                        addField(prop, prop.default, item);
+                    }
                 }
             }
         }

package/dist/src/client/query/subscription/index.d.ts

@@ -1,5 +1,4 @@
 import { BasedDbQuery } from '../BasedDbQuery.js';
 import { OnData, OnError, OnClose } from './types.js';
+export declare const subscribe: (q: BasedDbQuery, onData: OnData, onError?: OnError) => OnClose;
 export * from './types.js';
-export * from './markers.js';
-export declare const subscribe: (q: BasedDbQuery, onData: OnData, onError: OnError) => OnClose;

package/dist/src/client/query/subscription/index.js

@@ -1,58 +1,11 @@
 import { includeField } from '../query.js';
 import { registerQuery } from '../registerQuery.js';
-import { runSubscription } from './run.js';
-import { addSubscriptionMarkers, deleteSubscriptionMarkers } from './markers.js';
-export * from './types.js';
-export * from './markers.js';
 export const subscribe = (q, onData, onError) => {
-    let closed = false;
     if (!q.def.include.stringFields.size && !q.def.references.size) {
         includeField(q.def, '*');
     }
-    try {
-        registerQuery(q);
-    }
-    catch (err) {
-        onError(err);
-        return () => q;
-    }
-    if (!q.db.subscriptions.has(q.id)) {
-        const subscription = {
-            query: q,
-            subs: new Set(),
-            inProgress: false,
-            closed: false,
-        };
-        q.db.subscriptions.set(q.id, subscription);
-        addSubscriptionMarkers(q, subscription);
-    }
-    const fn = (res, err) => {
-        if (!closed) {
-            if (err) {
-                onError(err);
-            }
-            else {
-                onData(res);
-            }
-        }
-    };
-    const sub = q.db.subscriptions.get(q.id);
-    const close = () => {
-        sub.subs.delete(fn);
-        if (sub.subs.size === 0) {
-            q.db.subscriptions.delete(q.id);
-            deleteSubscriptionMarkers(q);
-        }
-        closed = true;
-        return q;
-    };
-    sub.subs.add(fn);
-    if (sub.res) {
-        onData(sub.res);
-    }
-    if (!sub.inProgress) {
-        runSubscription(sub);
-    }
-    return close;
+    registerQuery(q);
+    return q.db.hooks.subscribe(q, onData, onError);
 };
+export * from './types.js';
 //# sourceMappingURL=index.js.map

package/dist/src/client/query/subscription/types.d.ts

@@ -1,33 +1,5 @@
 import { BasedQueryResponse } from '../BasedIterable.js';
-import { BasedDbQuery } from '../BasedDbQuery.js';
 export type OnData = (res: BasedQueryResponse) => any;
 export type OnError = (err: Error) => any;
-export type OnClose = () => BasedDbQuery;
+export type OnClose = () => any;
 export type OnSubscription = (res: any, err?: Error) => void;
-export type Subscription = {
-    query: BasedDbQuery;
-    subs: Set<OnSubscription>;
-    res?: BasedQueryResponse;
-    closed: boolean;
-    inProgress: boolean;
-};
-export type SubscriptionsMap = Map<number, Subscription>;
-export type SubscriptionsToRun = Subscription[];
-export type SubscriptionMarkers = {
-    main: {
-        [start: string]: Subscription[];
-    };
-    props: {
-        [prop: string]: Subscription[];
-    };
-};
-export type SubscriptionMarkerMap = {
-    [typeID: string]: {
-        ids: Map<number, SubscriptionMarkers>;
-        collection: SubscriptionMarkers;
-    };
-};
-export type SubscriptionMarkersCheck = {
-    collection: SubscriptionMarkers | false;
-    ids: SubscriptionMarkers | false;
-};

package/dist/src/client/query/subscription/types.js

@@ -1,3 +1,10 @@
 export {};
-// has sub
+// export type Subscription = {
+//   query: BasedDbQuery
+//   subs: Set<OnSubscription>
+//   res?: BasedQueryResponse
+//   closed: boolean
+//   inProgress: boolean // dont need to check
+//   // filter - realy nice to add
+// }
 //# sourceMappingURL=types.js.map

package/dist/src/client/string.js

@@ -10,7 +10,7 @@ export const write = (buf, value, offset, noCompression, lang) => {
     buf[offset] = lang || 0;
     const { written: l } = ENCODER.encodeInto(value, buf.subarray(offset + 2));
     let crc = native.crc32(buf.subarray(offset + 2, offset + 2 + l));
-    // 50 maybe if lvl 1
+    // 50 len maybe if lvl 1
     if (value.length > 200 && !noCompression) {
         buf.copyWithin(offset + 6 + l, offset + 2, offset + 2 + l);
         const size = native.compress(buf, offset + 6, l);

package/dist/src/index.js

@@ -4,6 +4,7 @@ import { DbServer } from './server/index.js';
 import { DbClient } from './client/index.js';
 import { wait } from '@saulx/utils';
 import { debugMode, debugServer } from './utils.js';
+import { BasedQueryResponse } from './client/query/BasedIterable.js';
 export * from './client/modify/modify.js';
 export { compress, decompress };
 export { ModifyCtx }; // TODO move this somewhere
@@ -44,6 +45,33 @@ export class BasedDb {
         const client = new DbClient({
             maxModifySize,
             hooks: {
+                subscribe(q, onData, onError) {
+                    let timer;
+                    let prevChecksum;
+                    let lastLen = 0;
+                    let response;
+                    const get = async () => {
+                        const res = await server.getQueryBuf(q.buffer);
+                        if (!response) {
+                            response = new BasedQueryResponse(q.id, q.def, res, 0);
+                        }
+                        else {
+                            response.result = res;
+                            response.end = res.byteLength;
+                        }
+                        const checksum = response.checksum;
+                        if (lastLen != res.byteLength || checksum != prevChecksum) {
+                            onData(response);
+                            lastLen = res.byteLength;
+                            prevChecksum = checksum;
+                        }
+                        setTimeout(get, 200);
+                    };
+                    get();
+                    return () => {
+                        clearTimeout(timer);
+                    };
+                },
                 setSchema(schema, fromStart) {
                     return Promise.resolve(server.setSchema(schema, fromStart));
                 },

package/dist/src/native.d.ts

@@ -1,4 +1,4 @@
-declare const _default: {
+declare const native: {
     historyAppend(history: any, typeId: number, nodeId: number, dbCtx: any): any;
     historyCreate(pathname: string, mainLen: number): any;
     workerCtxInit: () => void;
@@ -24,4 +24,4 @@ declare const _default: {
     equals: (a: Uint8Array, b: Uint8Array) => boolean;
     expire: (dbCtx: any) => void;
 };
-export default _default;
+export default native;

package/dist/src/native.js

@@ -11,7 +11,7 @@ function SelvaIoErrlogToString(buf) {
     let len = (i = buf.indexOf(0)) >= 0 ? i : buf.byteLength;
     return DECODER.decode(selvaIoErrlog.slice(0, len));
 }
-export default {
+const native = {
     historyAppend(history, typeId, nodeId, dbCtx) {
         return db.historyAppend(history, typeId, nodeId, dbCtx);
     },
@@ -97,4 +97,6 @@ export default {
         db.expire(dbCtx);
     },
 };
+global.__basedDb__native__ = native;
+export default native;
 //# sourceMappingURL=native.js.map

package/dist/src/server/save.js

@@ -12,9 +12,8 @@ function saveRange(db, typeId, start, end, hashOut) {
     const file = block_sdb_file(typeId, start, end);
     const path = join(db.fileSystemPath, file);
     const err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hashOut);
-    if (err == -8) {
-        // TODO ENOENT
-        return '';
+    if (err == -8) { // TODO ENOENT
+        return ''; // empty range
     }
     else if (err) {
         // TODO print the error string
@@ -61,18 +60,16 @@ export function save(db, sync = false, forceFullDump = false) {
     else {
         foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
             const hash = new Uint8Array(16);
-            // TODO Don't save if empty
             const file = saveRange(db, typeId, start, end, hash);
             if (file === null) {
                 // The previous state should remain in the merkle tree for
                 // load and sync purposes.
                 return;
             }
-            else if (file.length === 0) {
-                // The range is empty
-            }
             else {
                 const oldLeaf = db.merkleTree.search(mtKey);
+                // If (file.length === 0) then the range is empty but that's fine,
+                // we'll keep them around for now to maintain the order of the tree.
                 if (oldLeaf) {
                     oldLeaf.data.file = file;
                     db.merkleTree.update(mtKey, hash);

package/dist/src/server/start.js

@@ -36,11 +36,13 @@ export async function start(db, opts) {
             const dumps = writelog.rangeDumps[typeId];
             for (const dump of dumps) {
                 const fname = dump.file;
-                try {
-                    native.loadRange(join(path, fname), db.dbCtxExternal);
-                }
-                catch (e) {
-                    console.error(e.message);
+                if (fname?.length > 0) {
+                    try {
+                        native.loadRange(join(path, fname), db.dbCtxExternal);
+                    }
+                    catch (e) {
+                        console.error(e.message);
+                    }
                 }
             }
         }
@@ -71,7 +73,7 @@ export async function start(db, opts) {
                 end,
             };
             db.merkleTree.insert(mtKey, hash, data);
-        });
+        }, true);
     }
     if (writelog?.hash) {
         const oldHash = hexToBuf(writelog.hash);

package/dist/src/server/tree.d.ts

@@ -11,5 +11,5 @@ export declare const destructureCsmtKey: (key: number) => number[];
 export declare const makeCsmtKey: (typeId: number, start: number) => number;
 export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
 export declare function initCsmt(db: DbServer): {};
-export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): void;
+export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void, includeEmptyBlocks?: boolean): void;
 export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;

package/dist/src/server/tree.js

@@ -38,13 +38,13 @@ export function initCsmt(db) {
     }
     return types;
 }
-export function foreachBlock(db, def, cb) {
+export function foreachBlock(db, def, cb, includeEmptyBlocks = false) {
     const step = def.blockCapacity;
     for (let start = 1; start <= def.lastId; start += step) {
         const end = start + step - 1;
         const hash = new Uint8Array(16);
         const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
-        if (res) {
+        if (res || includeEmptyBlocks) {
             cb(start, end, hash);
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.29",
+  "version": "0.0.31",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",