@bod.ee/db 0.10.2 → 0.12.1

package/src/server/ReplicationEngine.ts

@@ -52,6 +52,7 @@ export class ReplicationEngine {
   private _started = false;
   private _seq = 0;
   private _emitting = false;
+  private _pendingReplEvents: WriteEvent[] | null = null;
 
   get isReplica(): boolean { return this.options.role === 'replica'; }
   get isPrimary(): boolean { return this.options.role === 'primary'; }
@@ -154,11 +155,21 @@ export class ReplicationEngine {
     return this.options.excludePrefixes.some(p => path.startsWith(p));
   }
 
+  /** Buffer replication events during transactions, emit immediately otherwise */
   private emit(ev: WriteEvent): void {
-    // Guard against recursion (db.push('_repl') triggers _fireWrite → emit)
     if (this._emitting) return;
     if (this.isExcluded(ev.path)) return;
 
+    // If buffering (transaction in progress), collect events
+    if (this._pendingReplEvents) {
+      this._pendingReplEvents.push(ev);
+      return;
+    }
+
+    this._emitNow(ev);
+  }
+
+  private _emitNow(ev: WriteEvent): void {
     this._emitting = true;
     try {
       const replEvent: ReplEvent = { ...ev, ts: Date.now() };
@@ -170,6 +181,25 @@ export class ReplicationEngine {
     }
   }
 
+  /** Start buffering replication events (call before transaction) */
+  beginBatch(): void {
+    this._pendingReplEvents = [];
+  }
+
+  /** Flush buffered replication events (call after transaction commit) */
+  flushBatch(): void {
+    const events = this._pendingReplEvents;
+    this._pendingReplEvents = null;
+    if (events) {
+      for (const ev of events) this._emitNow(ev);
+    }
+  }
+
+  /** Discard buffered events (call on transaction rollback) */
+  discardBatch(): void {
+    this._pendingReplEvents = null;
+  }
+
   // --- Replica mode ---
 
   private async startReplica(): Promise<void> {

package/src/server/StorageEngine.ts

@@ -305,11 +305,97 @@ export class StorageEngine {
     path = validatePath(path);
     const prefix = path + '/';
 
-    // Get all rows under this path
+    // Try SQL push-down for push rows (single JSON values at depth 1)
+    if (this._canSqlPushDown(prefix, filters)) {
+      return this._sqlPushDownQuery(path, prefix, filters, order, limit, offset);
+    }
+
+    // Fallback: full JS-based query
+    return this._jsQuery(path, prefix, filters, order, limit, offset);
+  }
+
+  /** Check if all filters target top-level JSON fields AND data is push-row format */
+  private _canSqlPushDown(prefix: string, filters?: QueryFilter[]): boolean {
+    if (!filters?.length) return false;
+    // All filter fields must be simple identifiers (no nested paths)
+    if (!filters.every(f => /^[a-zA-Z_]\w*$/.test(f.field))) return false;
+    // Check if direct children are single JSON rows (push-row format, not flattened)
+    // Sample the first row: if its path relative to prefix has no '/', it's a push row
+    const sample = this.db.prepare('SELECT path FROM nodes WHERE path >= ? AND path < ? AND mq_status IS NULL LIMIT 1')
+      .get(prefix, prefixEnd(prefix)) as { path: string } | null;
+    if (!sample) return false;
+    const rel = sample.path.slice(prefix.length);
+    return !rel.includes('/'); // push row = no nesting
+  }
+
+  /** SQL push-down query for push rows using json_extract */
+  private _sqlPushDownQuery(
+    path: string, prefix: string,
+    filters?: QueryFilter[], order?: OrderClause,
+    limit?: number, offset?: number,
+  ): Array<{ _path: string; _key: string; [k: string]: unknown }> {
+    const params: unknown[] = [prefix, prefixEnd(prefix)];
+    const whereClauses = ['path >= ?', 'path < ?', 'mq_status IS NULL'];
+
+    // Only match direct children (no '/' in relative path)
+    // We filter this in JS since SQLite doesn't have a good way to check depth
+
+    const VALID_OPS: Record<string, string> = { '==': '=', '!=': '!=', '<': '<', '<=': '<=', '>': '>', '>=': '>=' };
+    if (filters?.length) {
+      for (const f of filters) {
+        const sqlOp = VALID_OPS[f.op];
+        if (!sqlOp) continue; // skip unknown operators
+        const jsonPath = `$.${f.field}`; // f.field already validated by _canSqlPushDown regex
+        whereClauses.push(`json_extract(value, '${jsonPath}') ${sqlOp} ?`);
+        params.push(f.value);
+      }
+    }
+
+    let sql = `SELECT path, value FROM nodes WHERE ${whereClauses.join(' AND ')}`;
+
+    if (order && /^[a-zA-Z_]\w*$/.test(order.field)) {
+      sql += ` ORDER BY json_extract(value, '$.${order.field}') ${order.dir === 'desc' ? 'DESC' : 'ASC'}`;
+    }
+
+    if (limit != null) {
+      sql += ` LIMIT ?`;
+      params.push(limit);
+      if (offset != null) {
+        sql += ` OFFSET ?`;
+        params.push(offset);
+      }
+    } else if (offset != null) {
+      sql += ` LIMIT -1 OFFSET ?`;
+      params.push(offset);
+    }
+
+    const rows = this.db.prepare(sql).all(...params) as Array<{ path: string; value: string }>;
+
+    // Filter to direct children only and reconstruct
+    const results: Array<{ _path: string; _key: string; [k: string]: unknown }> = [];
+    for (const row of rows) {
+      const rel = row.path.slice(prefix.length);
+      if (rel.includes('/')) continue; // skip nested rows
+      const key = rel;
+      const val = JSON.parse(row.value);
+      if (val && typeof val === 'object' && !Array.isArray(val)) {
+        results.push({ _path: row.path, _key: key, ...(val as Record<string, unknown>) });
+      } else {
+        results.push({ _path: row.path, _key: key, _value: val });
+      }
+    }
+    return results;
+  }
+
+  /** Full JS-based query (original implementation) */
+  private _jsQuery(
+    path: string, prefix: string,
+    filters?: QueryFilter[], order?: OrderClause,
+    limit?: number, offset?: number,
+  ): Array<{ _path: string; _key: string; [k: string]: unknown }> {
     const rows = this.stmtPrefix.all(prefix, prefixEnd(prefix)) as Array<{ path: string; value: string }>;
     if (rows.length === 0) return [];
 
-    // Group rows by direct child key
     const children = new Map<string, Array<{ path: string; value: string }>>();
     for (const row of rows) {
       const rel = row.path.slice(prefix.length);
@@ -318,7 +404,6 @@ export class StorageEngine {
       children.get(childKey)!.push(row);
     }
 
-    // Reconstruct each child
     let results: Array<{ _path: string; _key: string; [k: string]: unknown }> = [];
     for (const [key, childRows] of children) {
       const childPath = `${path}/${key}`;
@@ -330,7 +415,6 @@ export class StorageEngine {
       }
     }
 
-    // Apply filters
     if (filters?.length) {
       for (const f of filters) {
         results = results.filter(item => {
@@ -348,7 +432,6 @@ export class StorageEngine {
       }
     }
 
-    // Apply ordering
     if (order) {
       const dir = order.dir === 'desc' ? -1 : 1;
       results.sort((a, b) => {
@@ -361,7 +444,6 @@ export class StorageEngine {
       });
     }
 
-    // Apply offset + limit
     if (offset) results = results.slice(offset);
     if (limit) results = results.slice(0, limit);
 
@@ -438,6 +520,35 @@ export class StorageEngine {
     }));
   }
 
+  /** Check which paths from a list have data (exact or subtree). Returns the set of existing paths. */
+  existsMany(paths: string[]): Set<string> {
+    if (paths.length === 0) return new Set();
+    const result = new Set<string>();
+    const validated = paths.map(p => validatePath(p));
+    const params: string[] = [];
+    const conditions: string[] = [];
+    for (const vp of validated) {
+      const prefix = vp + '/';
+      conditions.push('(path = ? OR (path >= ? AND path < ?))');
+      params.push(vp, prefix, prefixEnd(prefix));
+    }
+    const sql = `SELECT DISTINCT path FROM nodes WHERE (${conditions.join(' OR ')}) AND mq_status IS NULL`;
+    const rows = this.db.prepare(sql).all(...params) as Array<{ path: string }>;
+    // Build a lookup set for O(1) matching
+    const vpSet = new Set(validated);
+    for (const row of rows) {
+      // Check exact match
+      if (vpSet.has(row.path)) { result.add(row.path); continue; }
+      // Check if row is a descendant of any requested path
+      const parts = row.path.split('/');
+      for (let i = parts.length - 1; i >= 1; i--) {
+        const ancestor = parts.slice(0, i).join('/');
+        if (vpSet.has(ancestor)) { result.add(ancestor); break; }
+      }
+    }
+    return result;
+  }
+
   close() {
     this.db.close();
   }

package/src/server/StreamEngine.ts

@@ -61,12 +61,30 @@ export class StreamEngine {
   }
 
   /** Get materialized view: snapshot merged with events on top */
-  materialize(topic: string, opts?: { keepKey?: string }): Record<string, unknown> {
+  materialize(topic: string, opts?: { keepKey?: string }): Record<string, unknown>;
+  materialize(topic: string, opts: { keepKey?: string; batchSize: number; cursor?: string }): { data: Record<string, unknown>; nextCursor?: string };
+  materialize(topic: string, opts?: { keepKey?: string; batchSize?: number; cursor?: string }): Record<string, unknown> | { data: Record<string, unknown>; nextCursor?: string } {
     const snap = this.snapshot(topic);
     const result: Record<string, unknown> = snap ? { ...snap.data } : {};
     const keepKey = opts?.keepKey;
+    const batchSize = opts?.batchSize;
+
+    if (batchSize != null && batchSize < Infinity) {
+      // Cursor-based: return one batch at a time
+      const afterKey = opts?.cursor ?? snap?.key ?? null;
+      const events = this.storage.queryAfterKey(topic, afterKey, batchSize);
+      for (const ev of events) {
+        if (keepKey && ev.value && typeof ev.value === 'object' && keepKey in (ev.value as any)) {
+          result[String((ev.value as any)[keepKey])] = ev.value;
+        } else if (ev.value && typeof ev.value === 'object' && !Array.isArray(ev.value)) {
+          Object.assign(result, ev.value);
+        }
+      }
+      const nextCursor = events.length === batchSize ? events[events.length - 1].key : undefined;
+      return { data: result, nextCursor };
+    }
 
-    // Replay all events after snapshot
+    // Full materialize (backward compatible)
     const afterKey = snap?.key ?? null;
     const events = this.storage.queryAfterKey(topic, afterKey, Number.MAX_SAFE_INTEGER);
     for (const ev of events) {

package/src/server/SubscriptionEngine.ts

@@ -1,11 +1,27 @@
 import { ancestors } from '../shared/pathUtils.ts';
 
-export type SubscriptionCallback = (snapshot: { path: string; val: () => unknown }) => void;
+export type SubscriptionCallback = (snapshot: { path: string; val: () => unknown; updatedAt?: number }) => void;
 export type ChildCallback = (event: { type: 'added' | 'changed' | 'removed'; key: string; path: string; val: () => unknown }) => void;
 
+export class SubscriptionEngineOptions {
+  /** Maximum total subscriptions (default 50000) */
+  maxSubscriptions: number = 50000;
+  /** Enable async batched notifications (default false) */
+  asyncNotify: boolean = false;
+}
+
 export class SubscriptionEngine {
+  readonly options: SubscriptionEngineOptions;
   private valueSubs = new Map<string, Set<SubscriptionCallback>>();
   private childSubs = new Map<string, Set<ChildCallback>>();
+  private _pendingNotify: Set<string> | null = null;
+  private _pendingGetFn: ((path: string) => unknown) | null = null;
+  private _pendingExisted: Set<string> | null = null;
+  private _flushScheduled = false;
+
+  constructor(options?: Partial<SubscriptionEngineOptions>) {
+    this.options = { ...new SubscriptionEngineOptions(), ...options };
+  }
 
   get hasSubscriptions(): boolean {
     return this.valueSubs.size > 0 || this.childSubs.size > 0;
@@ -31,6 +47,7 @@ export class SubscriptionEngine {
 
   /** Subscribe to value changes at a path (fires for exact + descendant writes) */
   onValue(path: string, cb: SubscriptionCallback): () => void {
+    if (this.size >= this.options.maxSubscriptions) throw new Error('Subscription limit reached');
     if (!this.valueSubs.has(path)) this.valueSubs.set(path, new Set());
     this.valueSubs.get(path)!.add(cb);
     return () => {
@@ -41,6 +58,7 @@ export class SubscriptionEngine {
 
   /** Subscribe to child events at a path */
   onChild(path: string, cb: ChildCallback): () => void {
+    if (this.size >= this.options.maxSubscriptions) throw new Error('Subscription limit reached');
     if (!this.childSubs.has(path)) this.childSubs.set(path, new Set());
     this.childSubs.get(path)!.add(cb);
     return () => {
@@ -49,6 +67,28 @@ export class SubscriptionEngine {
     };
   }
 
+  /** Push a value directly to subscribers of a specific path + its ancestors. No DB involved. */
+  pushValue(path: string, value: unknown): void {
+    const valFn = () => value;
+    const now = Date.now();
+    const subs = this.valueSubs.get(path);
+    if (subs?.size) {
+      const snapshot = { path, val: valFn, updatedAt: now };
+      for (const cb of subs) { try { cb(snapshot); } catch {} }
+    }
+    // Walk ancestors by finding '/' separators (avoids split/slice/join allocations)
+    let idx = path.lastIndexOf('/');
+    while (idx > 0) {
+      const ancestor = path.substring(0, idx);
+      const asubs = this.valueSubs.get(ancestor);
+      if (asubs?.size) {
+        const snap = { path: ancestor, val: valFn, updatedAt: now };
+        for (const cb of asubs) { try { cb(snap); } catch {} }
+      }
+      idx = path.lastIndexOf('/', idx - 1);
+    }
+  }
+
   /**
    * Notify after a write.
    * @param changedPaths - all leaf paths that were written
@@ -56,56 +96,103 @@ export class SubscriptionEngine {
    * @param existedBefore - set of paths that existed before the write (for added/changed detection)
    */
   notify(changedPaths: string[], getFn: (path: string) => unknown, existedBefore?: Set<string>) {
-    const notified = new Set<string>();
+    // Value subscriptions — walk each leaf + ancestors using indexOf (no array alloc)
+    const notified = changedPaths.length > 1 ? new Set<string>() : null;
 
     for (const leafPath of changedPaths) {
-      const pathsToNotify = [leafPath, ...ancestors(leafPath)];
-
-      for (const p of pathsToNotify) {
-        if (notified.has(p)) continue;
-        notified.add(p);
-
-        const subs = this.valueSubs.get(p);
+      // Notify exact path
+      if (!notified || !notified.has(leafPath)) {
+        notified?.add(leafPath);
+        const subs = this.valueSubs.get(leafPath);
+        if (subs?.size) {
+          const snapshot = { path: leafPath, val: () => getFn(leafPath) };
+          for (const cb of subs) { try { cb(snapshot); } catch {} }
+        }
+      }
+      // Walk ancestors via lastIndexOf (no split/slice/join)
+      let idx = leafPath.lastIndexOf('/');
+      while (idx > 0) {
+        const ancestor = leafPath.substring(0, idx);
+        if (notified && notified.has(ancestor)) { idx = leafPath.lastIndexOf('/', idx - 1); continue; }
+        notified?.add(ancestor);
+        const subs = this.valueSubs.get(ancestor);
         if (subs?.size) {
-          const snapshot = { path: p, val: () => getFn(p) };
-          for (const cb of subs) cb(snapshot);
+          const snapshot = { path: ancestor, val: () => getFn(ancestor) };
+          for (const cb of subs) { try { cb(snapshot); } catch {} }
         }
+        idx = leafPath.lastIndexOf('/', idx - 1);
       }
     }
 
-    // Child events
-    const childNotified = new Set<string>();
-    for (const leafPath of changedPaths) {
-      const parts = leafPath.split('/');
-      if (parts.length < 2) continue;
+    // Child events — skip entirely when no child subs
+    if (!this.childSubs.size) return;
 
-      for (let i = 1; i < parts.length; i++) {
-        const parentPath = parts.slice(0, i).join('/');
-        const childKey = parts[i];
+    const childNotified = changedPaths.length > 1 ? new Set<string>() : null;
+    for (const leafPath of changedPaths) {
+      // Walk parent/child pairs via indexOf
+      let start = 0;
+      let slashIdx = leafPath.indexOf('/', start);
+      while (slashIdx !== -1) {
+        const parentPath = leafPath.substring(0, slashIdx);
+        const nextSlash = leafPath.indexOf('/', slashIdx + 1);
+        const childKey = nextSlash === -1 ? leafPath.substring(slashIdx + 1) : leafPath.substring(slashIdx + 1, nextSlash);
         const dedupKey = `${parentPath}/${childKey}`;
 
-        if (childNotified.has(dedupKey)) continue;
-        childNotified.add(dedupKey);
-
-        const subs = this.childSubs.get(parentPath);
-        if (subs?.size) {
-          const childPath = `${parentPath}/${childKey}`;
-          const val = getFn(childPath);
-          let type: 'added' | 'changed' | 'removed';
-          if (val === null) {
-            type = 'removed';
-          } else if (existedBefore && !existedBefore.has(childPath)) {
-            type = 'added';
-          } else {
-            type = 'changed';
+        if (!childNotified || !childNotified.has(dedupKey)) {
+          childNotified?.add(dedupKey);
+          const subs = this.childSubs.get(parentPath);
+          if (subs?.size) {
+            const childPath = dedupKey;
+            const val = getFn(childPath);
+            let type: 'added' | 'changed' | 'removed';
+            if (val === null) {
+              type = 'removed';
+            } else if (existedBefore && !existedBefore.has(childPath)) {
+              type = 'added';
+            } else {
+              type = 'changed';
+            }
+            const event = { type, key: childKey, path: childPath, val: () => val };
+            for (const cb of subs) { try { cb(event); } catch {} }
           }
-          const event = { type, key: childKey, path: childPath, val: () => val };
-          for (const cb of subs) cb(event);
         }
+        start = slashIdx + 1;
+        slashIdx = leafPath.indexOf('/', start);
       }
     }
   }
 
+  /** Queue notifications for async batched delivery. Coalesces multiple writes in same tick. */
+  queueNotify(changedPaths: string[], getFn: (path: string) => unknown, existedBefore?: Set<string>) {
+    if (!this.options.asyncNotify) {
+      return this.notify(changedPaths, getFn, existedBefore);
+    }
+    if (!this._pendingNotify) this._pendingNotify = new Set();
+    for (const p of changedPaths) this._pendingNotify.add(p);
+    this._pendingGetFn = getFn;
+    if (existedBefore) {
+      if (!this._pendingExisted) this._pendingExisted = new Set();
+      for (const p of existedBefore) this._pendingExisted.add(p);
+    }
+    if (!this._flushScheduled) {
+      this._flushScheduled = true;
+      queueMicrotask(() => this._flushNotify());
+    }
+  }
+
+  private _flushNotify() {
+    this._flushScheduled = false;
+    const paths = this._pendingNotify;
+    const getFn = this._pendingGetFn;
+    const existed = this._pendingExisted;
+    this._pendingNotify = null;
+    this._pendingGetFn = null;
+    this._pendingExisted = null;
+    if (paths && getFn) {
+      this.notify([...paths], getFn, existed ?? undefined);
+    }
+  }
+
   get size(): number {
     let count = 0;
     for (const s of this.valueSubs.values()) count += s.size;