@bod.ee/db 0.10.1 → 0.11.1

package/src/server/BodDB.ts

@@ -11,6 +11,7 @@ import { ReplicationEngine, type ReplicationOptions, type WriteEvent } from './R
 import { VFSEngine, type VFSEngineOptions } from './VFSEngine.ts';
 import { KeyAuthEngine, type KeyAuthEngineOptions } from './KeyAuthEngine.ts';
 import { validatePath } from '../shared/pathUtils.ts';
+import { Logger, type LogConfig } from '../shared/logger.ts';
 
 export interface TransactionProxy {
   get(path: string): unknown;
@@ -46,6 +47,8 @@ export class BodDBOptions {
   port?: number;
   auth?: TransportOptions['auth'];
   transport?: Partial<TransportOptions>;
+  /** Logging config — disabled by default */
+  log?: LogConfig;
 }
 
 export class BodDB {
@@ -59,6 +62,7 @@ export class BodDB {
   readonly vfs: VFSEngine | null = null;
   readonly keyAuth: KeyAuthEngine | null = null;
   readonly options: BodDBOptions;
+  readonly log: Logger;
   replication: ReplicationEngine | null = null;
   private _replaying = false;
   private _onWriteHooks: Array<(ev: WriteEvent) => void> = [];
@@ -66,11 +70,16 @@ export class BodDB {
   get transport(): Transport | null { return this._transport; }
   private sweepTimer: ReturnType<typeof setInterval> | null = null;
   private _statsInterval: ReturnType<typeof setInterval> | null = null;
+  private _statsStmtCount: any = null;
   private _lastCpuUsage = process.cpuUsage();
   private _lastCpuTime = performance.now();
+  private _lastStats: Record<string, unknown> | null = null;
 
   constructor(options?: Partial<BodDBOptions>) {
     this.options = { ...new BodDBOptions(), ...options };
+    this.log = new Logger(this.options.log);
+    const _log = this.log.forComponent('db');
+    _log.info(`Initializing BodDB (path: ${this.options.path})`);
     this.storage = new StorageEngine({ path: this.options.path });
     this.subs = new SubscriptionEngine();
     this.stream = new StreamEngine(this.storage, this.subs, { compact: this.options.compact });
@@ -88,6 +97,7 @@ export class BodDB {
       ? BodDB.loadRulesFileSync(rulesConfig)
       : (rulesConfig ?? {});
     this.rules = new RulesEngine({ rules: resolvedRules, defaultDeny: this.options.defaultDeny });
+    _log.debug(`Rules loaded (${Object.keys(resolvedRules).length} paths, defaultDeny: ${!!this.options.defaultDeny})`);
 
     // Auto-create indexes from config
     if (this.options.indexes) {
@@ -96,21 +106,25 @@ export class BodDB {
           this.storage.createIndex(basePath, field);
         }
       }
+      _log.debug(`Indexes created: ${JSON.stringify(this.options.indexes)}`);
     }
 
     // Initialize FTS if configured
     if (this.options.fts) {
       this.fts = new FTSEngine(this.storage.db, this.options.fts);
+      _log.info('FTS5 engine enabled');
     }
 
     // Initialize vectors if configured
     if (this.options.vectors) {
       this.vectors = new VectorEngine(this.storage.db, this.options.vectors);
+      _log.info(`Vector engine enabled (dimensions: ${this.options.vectors.dimensions})`);
     }
 
     // Initialize VFS if configured
     if (this.options.vfs) {
       (this as { vfs: VFSEngine }).vfs = new VFSEngine(this, this.options.vfs);
+      _log.info(`VFS enabled (root: ${this.options.vfs.storageRoot})`);
     }
 
     // Initialize KeyAuth if configured
@@ -119,11 +133,13 @@ export class BodDB {
       if (this.options.keyAuth.rootPublicKey) {
         this.keyAuth!.initRoot(this.options.keyAuth.rootPublicKey);
       }
+      _log.info('KeyAuth engine enabled');
     }
 
     // Start TTL sweep
     if (this.options.sweepInterval > 0) {
       this.sweepTimer = setInterval(() => this.sweep(), this.options.sweepInterval);
+      _log.debug(`TTL sweep interval: ${this.options.sweepInterval}ms`);
     }
 
     // Init replication
@@ -134,7 +150,10 @@ export class BodDB {
         const compactOpts = this.options.replication.compact ?? { keepKey: 'path', maxCount: 10000 };
         this.stream.options.compact = { ...this.stream.options.compact, _repl: compactOpts };
       }
+      _log.info(`Replication enabled (role: ${this.replication.isPrimary ? 'primary' : 'replica'})`);
     }
+
+    _log.info('BodDB ready');
   }
 
   /** Load rules from a JSON or TS file synchronously */
@@ -195,6 +214,7 @@ export class BodDB {
   }
 
   get(path: string): unknown {
+    if (path === '_admin/stats' && this._lastStats) return this._lastStats;
     return this.storage.get(path);
   }
 
@@ -204,7 +224,7 @@ export class BodDB {
 
   set(path: string, value: unknown, options?: { ttl?: number }): void {
     const p = validatePath(path);
-    const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : new Set<string>();
+    const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : undefined;
     const changed = this.storage.set(path, value);
     if (options?.ttl) {
       this.storage.setExpiry(path, options.ttl);
@@ -218,12 +238,15 @@ export class BodDB {
   /** Manually trigger TTL sweep + stream auto-compact, returns expired paths */
   sweep(): string[] {
     const expired = this.storage.sweep();
-    if (expired.length > 0 && this.subs.hasSubscriptions) {
-      this.subs.notify(expired, (p) => this.storage.get(p));
-    }
-    // Fire delete events for expired paths (replication)
-    for (const p of expired) {
-      this._fireWrite({ op: 'delete', path: p });
+    if (expired.length > 0) {
+      if (this.subs.hasSubscriptions) {
+        this.subs.notify(expired, (p) => this.storage.get(p));
+      }
+      // Fire delete events for expired paths (replication)
+      for (const p of expired) {
+        this._fireWrite({ op: 'delete', path: p });
+      }
+      this.log.forComponent('storage').debug(`Sweep: ${expired.length} expired paths removed`);
     }
     this.stream.autoCompact();
     this.mq.sweep();
@@ -250,7 +273,7 @@ export class BodDB {
 
   delete(path: string): void {
     const p = validatePath(path);
-    const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : new Set<string>();
+    const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : undefined;
     this.storage.delete(path);
     if (this.subs.hasSubscriptions) {
       this.subs.notify([p], (pp) => this.storage.get(pp), existedBefore);
@@ -261,7 +284,7 @@ export class BodDB {
   /** Push a value with auto-generated time-sortable key (stored as single JSON row, not flattened) */
   push(path: string, value: unknown, opts?: { idempotencyKey?: string }): string {
     const p = validatePath(path);
-    const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : new Set<string>();
+    const existedBefore = this.subs.hasChildSubscriptions ? this.snapshotExisting(p) : undefined;
     const { key, changedPaths, duplicate } = this.storage.push(path, value, opts);
     if (!duplicate && this.subs.hasSubscriptions) {
       this.subs.notify(changedPaths, (pp) => this.storage.get(pp), existedBefore);
@@ -350,13 +373,21 @@ export class BodDB {
       },
     };
 
+    this.replication?.beginBatch();
     const sqliteTx = this.storage.db.transaction(() => fn(proxy));
-    const result = sqliteTx();
+    let result: T;
+    try {
+      result = sqliteTx();
+    } catch (e) {
+      this.replication?.discardBatch();
+      throw e;
+    }
 
     if (this.subs.hasSubscriptions && allChanged.length > 0) {
       this.subs.notify(allChanged, (p) => this.storage.get(p), allExistedBefore);
     }
     for (const ev of txWriteEvents) this._fireWrite(ev);
+    this.replication?.flushBatch();
     return result;
   }
 
@@ -366,53 +397,95 @@ export class BodDB {
     const { statSync } = require('fs');
     const { cpus, totalmem } = require('os');
     let lastOsCpus = cpus();
+    // Clean up old flattened _admin/stats/* leaf rows (migrated to single JSON row)
+    this.storage.db.prepare("DELETE FROM nodes WHERE path LIKE '_admin/stats/%'").run();
+
+    // Reuse prepared statement across start/stop cycles
+    if (!this._statsStmtCount) {
+      this._statsStmtCount = this.storage.db.prepare('SELECT COUNT(*) as n FROM nodes WHERE mq_status IS NULL');
+    }
+    const stmtCount = this._statsStmtCount;
+    const statsPath = '_admin/stats';
+
+    // Reuse a single stats object to minimize allocations
+    const statsData: Record<string, unknown> = {
+      process: {}, db: {}, system: {},
+      subs: 0, clients: 0, repl: null, ts: 0,
+    };
+    const proc = statsData.process as Record<string, number>;
+    const dbStats = statsData.db as Record<string, number>;
+    const sys = statsData.system as Record<string, number>;
+
+    // Heavy calls (cpus, statSync, stmtCount) run every 5s; lightweight calls every 1s
+    let tick = 0;
+    const MB = 1 / (1024 * 1024);
+    sys.cpuCores = cpus().length;
+    sys.totalMemMb = Math.round(totalmem() * MB);
 
     this._statsInterval = setInterval(() => {
-      if (!this.subs.subscriberCount('_admin')) return;
+      if (!this._transport) return;
+      tick++;
 
       const now = performance.now();
       const cpu = process.cpuUsage();
       const elapsedUs = (now - this._lastCpuTime) * 1000;
-      const cpuPercent = +((cpu.user - this._lastCpuUsage.user + cpu.system - this._lastCpuUsage.system) / elapsedUs * 100).toFixed(1);
+      proc.cpuPercent = Math.round((cpu.user - this._lastCpuUsage.user + cpu.system - this._lastCpuUsage.system) / elapsedUs * 1000) / 10;
       this._lastCpuUsage = cpu;
       this._lastCpuTime = now;
 
       const mem = process.memoryUsage();
-      let nodeCount = 0;
-      try { nodeCount = (this.storage.db.query('SELECT COUNT(*) as n FROM nodes WHERE mq_status IS NULL').get() as any).n; } catch {}
-      let dbSizeMb = 0;
-      try { dbSizeMb = +(statSync(this.options.path).size / 1024 / 1024).toFixed(2); } catch {}
-
-      // System CPU
-      const cur = cpus();
-      let idleDelta = 0, totalDelta = 0;
-      for (let i = 0; i < cur.length; i++) {
-        const prev = lastOsCpus[i]?.times ?? cur[i].times;
-        const c = cur[i].times;
-        idleDelta += c.idle - prev.idle;
-        totalDelta += (c.user + c.nice + c.sys + c.irq + c.idle) - (prev.user + prev.nice + prev.sys + prev.irq + prev.idle);
+      proc.heapUsedMb = Math.round(mem.heapUsed * MB * 100) / 100;
+      proc.rssMb = Math.round(mem.rss * MB * 100) / 100;
+      proc.uptimeSec = Math.floor(process.uptime());
+
+      // Expensive calls: only every 5 ticks
+      if (tick % 5 === 0) {
+        try { dbStats.nodeCount = (stmtCount.get() as any).n; } catch {}
+        try { dbStats.sizeMb = Math.round(statSync(this.options.path).size * MB * 100) / 100; } catch {}
+        const cur = cpus();
+        let idleDelta = 0, totalDelta = 0;
+        for (let i = 0; i < cur.length; i++) {
+          const prev = lastOsCpus[i]?.times ?? cur[i].times;
+          const c = cur[i].times;
+          idleDelta += c.idle - prev.idle;
+          totalDelta += (c.user + c.nice + c.sys + c.irq + c.idle) - (prev.user + prev.nice + prev.sys + prev.irq + prev.idle);
+        }
+        lastOsCpus = cur;
+        sys.cpuPercent = totalDelta > 0 ? Math.round((1 - idleDelta / totalDelta) * 1000) / 10 : 0;
       }
-      lastOsCpus = cur;
-      const sysCpu = totalDelta > 0 ? +((1 - idleDelta / totalDelta) * 100).toFixed(1) : 0;
-
-      this.set('_admin/stats', {
-        process: { cpuPercent, heapUsedMb: +(mem.heapUsed / 1024 / 1024).toFixed(2), rssMb: +(mem.rss / 1024 / 1024).toFixed(2), uptimeSec: Math.floor(process.uptime()) },
-        db: { nodeCount, sizeMb: dbSizeMb },
-        system: { cpuCores: cur.length, totalMemMb: +(totalmem() / 1024 / 1024).toFixed(0), cpuPercent: sysCpu },
-        subs: this.subs.subscriberCount(),
-        clients: this._transport?.clientCount ?? 0,
-        repl: this.replication?.stats() ?? null,
-        ts: Date.now(),
-      });
+
+      statsData.subs = this.subs.subscriberCount();
+      statsData.clients = this._transport?.clientCount ?? 0;
+      statsData.repl = this.replication?.stats() ?? null;
+      statsData.ts = Date.now();
+
+      // Push directly to WS clients — bypass SubscriptionEngine entirely
+      this._lastStats = statsData;
+      if (this._transport) this._transport.broadcastStats(statsData, Date.now());
+
     }, 1000);
+    this.log.forComponent('stats').info('Stats publisher started');
+  }
+
+  /** Stop the stats publisher. */
+  stopStatsPublisher(): void {
+    if (!this._statsInterval) return;
+    clearInterval(this._statsInterval);
+    this._statsInterval = null;
+    this._lastStats = null;
+    this.subs.pushValue('_admin/stats', null);
+    this.log.forComponent('stats').info('Stats publisher stopped');
   }
 
+  get statsEnabled(): boolean { return this._statsInterval !== null; }
+
   /** Start standalone WebSocket + REST server on its own port */
   serve(options?: Partial<TransportOptions>) {
     const port = options?.port ?? this.options.port;
     const auth = options?.auth ?? this.options.auth ?? (this.keyAuth ? this.keyAuth.authCallback() : undefined);
     this._transport = new Transport(this, this.rules, { ...this.options.transport, ...options, port, auth, keyAuth: this.keyAuth ?? undefined });
     this.startStatsPublisher();
+    this.log.forComponent('transport').info(`Server starting on port ${port}`);
     return this._transport.start();
   }
 
@@ -467,13 +540,11 @@ export class BodDB {
   }
 
   private snapshotExisting(path: string): Set<string> {
-    const existing = new Set<string>();
-    if (this.storage.exists(path)) existing.add(path);
+    const paths = [path];
     const parts = path.split('/');
     for (let i = 1; i < parts.length; i++) {
-      const childPath = parts.slice(0, i + 1).join('/');
-      if (this.storage.exists(childPath)) existing.add(childPath);
+      paths.push(parts.slice(0, i + 1).join('/'));
     }
-    return existing;
+    return this.storage.existsMany(paths);
   }
 }

package/src/server/KeyAuthEngine.ts

@@ -19,6 +19,12 @@ export class KeyAuthEngineOptions {
   clockSkew: number = 30;
   /** Allow unauthenticated device registration (default true) */
   allowOpenRegistration: boolean = true;
+  /** PBKDF2 iterations for password-based key derivation (default 100000) */
+  pbkdf2Iterations: number = 100_000;
+  /** Max failed auth attempts before lockout (0 = disabled) */
+  maxAuthFailures: number = 0;
+  /** Lockout duration in seconds after max failures */
+  authLockoutSeconds: number = 60;
 }
 
 /** Reserved prefix — _auth/ writes blocked for external clients */
@@ -80,7 +86,7 @@ export class KeyAuthEngine {
    *  If this is the first account ever created, it is auto-elevated to root. */
   createAccount(password: string, roles: string[] = [], displayName?: string, devicePublicKey?: string): { publicKey: string; fingerprint: string; isRoot: boolean; deviceFingerprint?: string } {
     const kp = generateEd25519KeyPair();
-    const enc = encryptPrivateKey(kp.privateKey, password);
+    const enc = encryptPrivateKey(kp.privateKey, password, this.options.pbkdf2Iterations);
     const fp = fingerprint(kp.publicKeyBase64);
 
     // First account becomes root automatically
@@ -145,6 +151,14 @@ export class KeyAuthEngine {
 
   /** Verify a signed nonce. Returns token + expiry or null. */
   verify(publicKeyBase64: string, signatureBase64: string, nonce: string): { token: string; expiresAt: number } | null {
+    const fp = fingerprint(publicKeyBase64);
+
+    // Rate limit check
+    if (this.options.maxAuthFailures > 0) {
+      const rlData = this.db.get(`_auth/rateLimit/${fp}`) as { count: number } | null;
+      if (rlData && rlData.count >= this.options.maxAuthFailures) return null;
+    }
+
     // Check nonce exists (one-time use)
     const nonceData = this.db.get(`_auth/nonces/${nonce}`);
     if (!nonceData) return null;
@@ -154,9 +168,10 @@ export class KeyAuthEngine {
     // Verify signature
     const pubKeyDer = Buffer.from(publicKeyBase64, 'base64');
     const sig = Buffer.from(signatureBase64, 'base64');
-    if (!verifySignature(Buffer.from(nonce), sig, new Uint8Array(pubKeyDer))) return null;
-
-    const fp = fingerprint(publicKeyBase64);
+    if (!verifySignature(Buffer.from(nonce), sig, new Uint8Array(pubKeyDer))) {
+      this._recordAuthFailure(fp);
+      return null;
+    }
 
     // Check if root
     const root = this.db.get('_auth/root') as { fingerprint: string } | null;
@@ -200,6 +215,8 @@ export class KeyAuthEngine {
       sid, fp, accountFp, roles, root: isRoot, exp: expiresAt,
     };
     const token = encodeToken(payload, this.serverPrivateKey);
+    // Clear rate limit on success
+    if (this.options.maxAuthFailures > 0) this.db.delete(`_auth/rateLimit/${fp}`);
     return { token, expiresAt };
   }
 
@@ -213,7 +230,7 @@ export class KeyAuthEngine {
     let accountPrivateKey: Uint8Array;
     try {
       accountPrivateKey = decryptPrivateKey(
-        account.encryptedPrivateKey, account.salt, account.iv, account.authTag, password,
+        account.encryptedPrivateKey, account.salt, account.iv, account.authTag, password, this.options.pbkdf2Iterations,
       );
     } catch {
       return null; // Wrong password
@@ -269,12 +286,12 @@ export class KeyAuthEngine {
 
     let privateKey: Uint8Array;
     try {
-      privateKey = decryptPrivateKey(account.encryptedPrivateKey, account.salt, account.iv, account.authTag, oldPassword);
+      privateKey = decryptPrivateKey(account.encryptedPrivateKey, account.salt, account.iv, account.authTag, oldPassword, this.options.pbkdf2Iterations);
     } catch {
       return false;
     }
 
-    const enc = encryptPrivateKey(privateKey, newPassword);
+    const enc = encryptPrivateKey(privateKey, newPassword, this.options.pbkdf2Iterations);
     privateKey.fill(0); // Wipe
 
     // Atomic: single update call writes all fields together
@@ -460,6 +477,14 @@ export class KeyAuthEngine {
     return { accountFp, roles: account.roles ?? [] };
   }
 
+  /** Record a failed auth attempt for rate limiting */
+  private _recordAuthFailure(fp: string): void {
+    if (this.options.maxAuthFailures <= 0) return;
+    const rlData = this.db.get(`_auth/rateLimit/${fp}`) as { count: number } | null;
+    const count = (rlData?.count ?? 0) + 1;
+    this.db.set(`_auth/rateLimit/${fp}`, { count }, { ttl: this.options.authLockoutSeconds });
+  }
+
   /** Invalidate all sessions for a given device fingerprint */
   private _invalidateDeviceSessions(deviceFp: string): void {
     const sessions = this.db.getShallow('_auth/sessions');

package/src/server/ReplicationEngine.ts

@@ -52,6 +52,7 @@ export class ReplicationEngine {
   private _started = false;
   private _seq = 0;
   private _emitting = false;
+  private _pendingReplEvents: WriteEvent[] | null = null;
 
   get isReplica(): boolean { return this.options.role === 'replica'; }
   get isPrimary(): boolean { return this.options.role === 'primary'; }
@@ -154,11 +155,21 @@ export class ReplicationEngine {
     return this.options.excludePrefixes.some(p => path.startsWith(p));
   }
 
+  /** Buffer replication events during transactions, emit immediately otherwise */
   private emit(ev: WriteEvent): void {
-    // Guard against recursion (db.push('_repl') triggers _fireWrite → emit)
     if (this._emitting) return;
     if (this.isExcluded(ev.path)) return;
 
+    // If buffering (transaction in progress), collect events
+    if (this._pendingReplEvents) {
+      this._pendingReplEvents.push(ev);
+      return;
+    }
+
+    this._emitNow(ev);
+  }
+
+  private _emitNow(ev: WriteEvent): void {
     this._emitting = true;
     try {
       const replEvent: ReplEvent = { ...ev, ts: Date.now() };
@@ -170,6 +181,25 @@ export class ReplicationEngine {
     }
   }
 
+  /** Start buffering replication events (call before transaction) */
+  beginBatch(): void {
+    this._pendingReplEvents = [];
+  }
+
+  /** Flush buffered replication events (call after transaction commit) */
+  flushBatch(): void {
+    const events = this._pendingReplEvents;
+    this._pendingReplEvents = null;
+    if (events) {
+      for (const ev of events) this._emitNow(ev);
+    }
+  }
+
+  /** Discard buffered events (call on transaction rollback) */
+  discardBatch(): void {
+    this._pendingReplEvents = null;
+  }
+
   // --- Replica mode ---
 
   private async startReplica(): Promise<void> {

package/src/server/StorageEngine.ts

@@ -305,11 +305,97 @@ export class StorageEngine {
     path = validatePath(path);
     const prefix = path + '/';
 
-    // Get all rows under this path
+    // Try SQL push-down for push rows (single JSON values at depth 1)
+    if (this._canSqlPushDown(prefix, filters)) {
+      return this._sqlPushDownQuery(path, prefix, filters, order, limit, offset);
+    }
+
+    // Fallback: full JS-based query
+    return this._jsQuery(path, prefix, filters, order, limit, offset);
+  }
+
+  /** Check if all filters target top-level JSON fields AND data is push-row format */
+  private _canSqlPushDown(prefix: string, filters?: QueryFilter[]): boolean {
+    if (!filters?.length) return false;
+    // All filter fields must be simple identifiers (no nested paths)
+    if (!filters.every(f => /^[a-zA-Z_]\w*$/.test(f.field))) return false;
+    // Check if direct children are single JSON rows (push-row format, not flattened)
+    // Sample the first row: if its path relative to prefix has no '/', it's a push row
+    const sample = this.db.prepare('SELECT path FROM nodes WHERE path >= ? AND path < ? AND mq_status IS NULL LIMIT 1')
+      .get(prefix, prefixEnd(prefix)) as { path: string } | null;
+    if (!sample) return false;
+    const rel = sample.path.slice(prefix.length);
+    return !rel.includes('/'); // push row = no nesting
+  }
+
+  /** SQL push-down query for push rows using json_extract */
+  private _sqlPushDownQuery(
+    path: string, prefix: string,
+    filters?: QueryFilter[], order?: OrderClause,
+    limit?: number, offset?: number,
+  ): Array<{ _path: string; _key: string; [k: string]: unknown }> {
+    const params: unknown[] = [prefix, prefixEnd(prefix)];
+    const whereClauses = ['path >= ?', 'path < ?', 'mq_status IS NULL'];
+
+    // Only match direct children (no '/' in relative path)
+    // We filter this in JS since SQLite doesn't have a good way to check depth
+
+    const VALID_OPS: Record<string, string> = { '==': '=', '!=': '!=', '<': '<', '<=': '<=', '>': '>', '>=': '>=' };
+    if (filters?.length) {
+      for (const f of filters) {
+        const sqlOp = VALID_OPS[f.op];
+        if (!sqlOp) continue; // skip unknown operators
+        const jsonPath = `$.${f.field}`; // f.field already validated by _canSqlPushDown regex
+        whereClauses.push(`json_extract(value, '${jsonPath}') ${sqlOp} ?`);
+        params.push(f.value);
+      }
+    }
+
+    let sql = `SELECT path, value FROM nodes WHERE ${whereClauses.join(' AND ')}`;
+
+    if (order && /^[a-zA-Z_]\w*$/.test(order.field)) {
+      sql += ` ORDER BY json_extract(value, '$.${order.field}') ${order.dir === 'desc' ? 'DESC' : 'ASC'}`;
+    }
+
+    if (limit != null) {
+      sql += ` LIMIT ?`;
+      params.push(limit);
+      if (offset != null) {
+        sql += ` OFFSET ?`;
+        params.push(offset);
+      }
+    } else if (offset != null) {
+      sql += ` LIMIT -1 OFFSET ?`;
+      params.push(offset);
+    }
+
+    const rows = this.db.prepare(sql).all(...params) as Array<{ path: string; value: string }>;
+
+    // Filter to direct children only and reconstruct
+    const results: Array<{ _path: string; _key: string; [k: string]: unknown }> = [];
+    for (const row of rows) {
+      const rel = row.path.slice(prefix.length);
+      if (rel.includes('/')) continue; // skip nested rows
+      const key = rel;
+      const val = JSON.parse(row.value);
+      if (val && typeof val === 'object' && !Array.isArray(val)) {
+        results.push({ _path: row.path, _key: key, ...(val as Record<string, unknown>) });
+      } else {
+        results.push({ _path: row.path, _key: key, _value: val });
+      }
+    }
+    return results;
+  }
+
+  /** Full JS-based query (original implementation) */
+  private _jsQuery(
+    path: string, prefix: string,
+    filters?: QueryFilter[], order?: OrderClause,
+    limit?: number, offset?: number,
+  ): Array<{ _path: string; _key: string; [k: string]: unknown }> {
     const rows = this.stmtPrefix.all(prefix, prefixEnd(prefix)) as Array<{ path: string; value: string }>;
     if (rows.length === 0) return [];
 
-    // Group rows by direct child key
     const children = new Map<string, Array<{ path: string; value: string }>>();
     for (const row of rows) {
       const rel = row.path.slice(prefix.length);
@@ -318,7 +404,6 @@ export class StorageEngine {
       children.get(childKey)!.push(row);
     }
 
-    // Reconstruct each child
     let results: Array<{ _path: string; _key: string; [k: string]: unknown }> = [];
     for (const [key, childRows] of children) {
       const childPath = `${path}/${key}`;
@@ -330,7 +415,6 @@ export class StorageEngine {
       }
     }
 
-    // Apply filters
     if (filters?.length) {
       for (const f of filters) {
         results = results.filter(item => {
@@ -348,7 +432,6 @@ export class StorageEngine {
       }
     }
 
-    // Apply ordering
     if (order) {
       const dir = order.dir === 'desc' ? -1 : 1;
       results.sort((a, b) => {
@@ -361,7 +444,6 @@ export class StorageEngine {
       });
     }
 
-    // Apply offset + limit
     if (offset) results = results.slice(offset);
     if (limit) results = results.slice(0, limit);
 
@@ -438,6 +520,35 @@ export class StorageEngine {
     }));
   }
 
+  /** Check which paths from a list have data (exact or subtree). Returns the set of existing paths. */
+  existsMany(paths: string[]): Set<string> {
+    if (paths.length === 0) return new Set();
+    const result = new Set<string>();
+    const validated = paths.map(p => validatePath(p));
+    const params: string[] = [];
+    const conditions: string[] = [];
+    for (const vp of validated) {
+      const prefix = vp + '/';
+      conditions.push('(path = ? OR (path >= ? AND path < ?))');
+      params.push(vp, prefix, prefixEnd(prefix));
+    }
+    const sql = `SELECT DISTINCT path FROM nodes WHERE (${conditions.join(' OR ')}) AND mq_status IS NULL`;
+    const rows = this.db.prepare(sql).all(...params) as Array<{ path: string }>;
+    // Build a lookup set for O(1) matching
+    const vpSet = new Set(validated);
+    for (const row of rows) {
+      // Check exact match
+      if (vpSet.has(row.path)) { result.add(row.path); continue; }
+      // Check if row is a descendant of any requested path
+      const parts = row.path.split('/');
+      for (let i = parts.length - 1; i >= 1; i--) {
+        const ancestor = parts.slice(0, i).join('/');
+        if (vpSet.has(ancestor)) { result.add(ancestor); break; }
+      }
+    }
+    return result;
+  }
+
   close() {
     this.db.close();
   }

package/src/server/StreamEngine.ts

@@ -61,12 +61,30 @@ export class StreamEngine {
   }
 
   /** Get materialized view: snapshot merged with events on top */
-  materialize(topic: string, opts?: { keepKey?: string }): Record<string, unknown> {
+  materialize(topic: string, opts?: { keepKey?: string }): Record<string, unknown>;
+  materialize(topic: string, opts: { keepKey?: string; batchSize: number; cursor?: string }): { data: Record<string, unknown>; nextCursor?: string };
+  materialize(topic: string, opts?: { keepKey?: string; batchSize?: number; cursor?: string }): Record<string, unknown> | { data: Record<string, unknown>; nextCursor?: string } {
     const snap = this.snapshot(topic);
     const result: Record<string, unknown> = snap ? { ...snap.data } : {};
     const keepKey = opts?.keepKey;
+    const batchSize = opts?.batchSize;
+
+    if (batchSize != null && batchSize < Infinity) {
+      // Cursor-based: return one batch at a time
+      const afterKey = opts?.cursor ?? snap?.key ?? null;
+      const events = this.storage.queryAfterKey(topic, afterKey, batchSize);
+      for (const ev of events) {
+        if (keepKey && ev.value && typeof ev.value === 'object' && keepKey in (ev.value as any)) {
+          result[String((ev.value as any)[keepKey])] = ev.value;
+        } else if (ev.value && typeof ev.value === 'object' && !Array.isArray(ev.value)) {
+          Object.assign(result, ev.value);
+        }
+      }
+      const nextCursor = events.length === batchSize ? events[events.length - 1].key : undefined;
+      return { data: result, nextCursor };
+    }
 
-    // Replay all events after snapshot
+    // Full materialize (backward compatible)
     const afterKey = snap?.key ?? null;
     const events = this.storage.queryAfterKey(topic, afterKey, Number.MAX_SAFE_INTEGER);
     for (const ev of events) {