@bod.ee/db 0.7.0

package/src/server/StreamEngine.ts

@@ -0,0 +1,320 @@
+import type { StorageEngine } from './StorageEngine.ts';
+import type { SubscriptionEngine } from './SubscriptionEngine.ts';
+import { validatePath, prefixEnd } from '../shared/pathUtils.ts';
+
+export interface StreamEvent {
+  key: string;
+  data: unknown;
+}
+
+export interface CompactOptions {
+  /** Delete events older than this many seconds */
+  maxAge?: number;
+  /** Keep only the last N events (rest folded into snapshot) */
+  maxCount?: number;
+  /** Kafka-style: snapshot keyed by this field, latest value per key wins */
+  keepKey?: string;
+}
+
+export interface CompactResult {
+  deleted: number;
+  snapshotSize: number;
+}
+
+export interface StreamSnapshot {
+  /** Push key of the last event folded into this snapshot */
+  key: string;
+  /** Merged final state. With keepKey: keyed by field value. Without: single merged object. */
+  data: Record<string, unknown>;
+}
+
+export class StreamEngineOptions {
+  defaultLimit: number = 100;
+  /** Auto-compact topics: { "events/orders": { maxAge: 86400 } } */
+  compact?: Record<string, CompactOptions>;
+}
+
+export class StreamEngine {
+  readonly options: StreamEngineOptions;
+  private activeSubs = new Map<string, () => void>(); // subKey → unsub
+
+  constructor(
+    private storage: StorageEngine,
+    private subs: SubscriptionEngine,
+    options?: Partial<StreamEngineOptions>,
+  ) {
+    this.options = { ...new StreamEngineOptions(), ...options };
+  }
+
+  /** Read events from a topic after the group's stored offset */
+  read(topic: string, groupId: string, limit?: number): StreamEvent[] {
+    const offset = this.getOffset(topic, groupId);
+    return this.storage.queryAfterKey(topic, offset, limit ?? this.options.defaultLimit)
+      .map(r => ({ key: r.key, data: r.value }));
+  }
+
+  /** Get the snapshot for a topic (base state before events) */
+  snapshot(topic: string): StreamSnapshot | null {
+    const val = this.storage.get(this.snapshotPath(topic));
+    if (val && typeof val === 'object' && 'key' in (val as any)) return val as StreamSnapshot;
+    return null;
+  }
+
+  /** Get materialized view: snapshot merged with events on top */
+  materialize(topic: string, opts?: { keepKey?: string }): Record<string, unknown> {
+    const snap = this.snapshot(topic);
+    const result: Record<string, unknown> = snap ? { ...snap.data } : {};
+    const keepKey = opts?.keepKey;
+
+    // Replay all events after snapshot
+    const afterKey = snap?.key ?? null;
+    const events = this.storage.queryAfterKey(topic, afterKey, Number.MAX_SAFE_INTEGER);
+    for (const ev of events) {
+      if (keepKey && ev.value && typeof ev.value === 'object' && keepKey in (ev.value as any)) {
+        result[String((ev.value as any)[keepKey])] = ev.value;
+      } else if (ev.value && typeof ev.value === 'object' && !Array.isArray(ev.value)) {
+        Object.assign(result, ev.value);
+      }
+    }
+    return result;
+  }
+
+  /** Acknowledge processing up to (and including) a key */
+  ack(topic: string, groupId: string, key: string): void {
+    this.setOffset(topic, groupId, key);
+  }
+
+  /**
+   * Subscribe to a topic with replay from last offset.
+   * Uses subscribe-then-backfill to prevent race conditions.
+   * Returns an unsubscribe function.
+   */
+  subscribe(topic: string, groupId: string, send: (events: StreamEvent[]) => void): () => void {
+    const subKey = `${topic}:${groupId}`;
+
+    // If already subscribed, unsub first
+    this.activeSubs.get(subKey)?.();
+
+    // 1. Attach child subscription, buffer events
+    const buffer: StreamEvent[] = [];
+    let replayed = false;
+    let lastReplayedKey: string | null = null;
+
+    const childUnsub = this.subs.onChild(topic, (event) => {
+      if (event.type !== 'added') return;
+      const streamEvent: StreamEvent = { key: event.key, data: event.val() };
+      if (!replayed) {
+        buffer.push(streamEvent);
+      } else {
+        send([streamEvent]);
+      }
+    });
+
+    // 2. Fetch replay batch
+    const offset = this.getOffset(topic, groupId);
+    const replayBatch = this.storage.queryAfterKey(topic, offset, this.options.defaultLimit);
+
+    // 3. Send replay batch
+    if (replayBatch.length > 0) {
+      lastReplayedKey = replayBatch[replayBatch.length - 1].key;
+      send(replayBatch.map(r => ({ key: r.key, data: r.value })));
+    }
+
+    // 4. Drain buffer, dedup against last replayed key
+    replayed = true;
+    if (buffer.length > 0) {
+      const deduped = lastReplayedKey
+        ? buffer.filter(e => e.key > lastReplayedKey!)
+        : buffer;
+      if (deduped.length > 0) send(deduped);
+    }
+
+    // Store unsub
+    const unsub = () => {
+      childUnsub();
+      this.activeSubs.delete(subKey);
+    };
+    this.activeSubs.set(subKey, unsub);
+    return unsub;
+  }
+
+  /**
+   * Compact a topic — fold events into snapshot, delete folded events.
+   * Snapshot is the base state; events layer on top.
+   * Safety: never deletes events beyond the minimum consumer group offset.
+   */
+  compact(topic: string, opts: CompactOptions): CompactResult {
+    topic = validatePath(topic);
+    const prefix = topic + '/';
+    const end = prefixEnd(prefix);
+
+    // Get all event rows for this topic (ordered by path = chronological)
+    const rows = this.storage.db
+      .prepare('SELECT path, value, updated_at FROM nodes WHERE path >= ? AND path < ? ORDER BY path ASC')
+      .all(prefix, end) as Array<{ path: string; value: string; updated_at: number }>;
+
+    if (rows.length === 0) return { deleted: 0, snapshotSize: 0 };
+
+    // Determine which events to fold into snapshot
+    const toFold = new Set<number>(); // indices into rows
+
+    if (opts.maxAge != null) {
+      const cutoff = Date.now() - opts.maxAge * 1000;
+      for (let i = 0; i < rows.length; i++) {
+        if (rows[i].updated_at < cutoff) toFold.add(i);
+      }
+    }
+
+    if (opts.maxCount != null && rows.length > opts.maxCount) {
+      const excess = rows.length - opts.maxCount;
+      for (let i = 0; i < excess; i++) {
+        toFold.add(i);
+      }
+    }
+
+    if (opts.keepKey != null) {
+      const field = opts.keepKey;
+      const latestIdx = new Map<string, number>(); // fieldValue → latest index
+      for (let i = 0; i < rows.length; i++) {
+        try {
+          const data = JSON.parse(rows[i].value);
+          if (data != null && typeof data === 'object' && field in data) {
+            const keyVal = String(data[field]);
+            const prev = latestIdx.get(keyVal);
+            if (prev != null) toFold.add(prev);
+            latestIdx.set(keyVal, i);
+          }
+        } catch { /* skip */ }
+      }
+    }
+
+    if (toFold.size === 0) return { deleted: 0, snapshotSize: 0 };
+
+    // Safety: never fold events beyond the minimum consumer group offset
+    const minSafeOffset = this.getMinOffset(topic);
+
+    // Load existing snapshot
+    const existingSnap = this.snapshot(topic);
+    const snapData: Record<string, unknown> = existingSnap ? { ...existingSnap.data } : {};
+    let lastFoldedKey = existingSnap?.key ?? '';
+
+    // Sort fold indices and process in order
+    const sortedIndices = [...toFold].sort((a, b) => a - b);
+
+    const stmtDel = this.storage.db.prepare('DELETE FROM nodes WHERE path = ?');
+    let deleted = 0;
+
+    const tx = this.storage.db.transaction(() => {
+      for (const idx of sortedIndices) {
+        const row = rows[idx];
+        const key = row.path.slice(prefix.length);
+
+        // Safety check: only fold if all groups have acked past this event
+        if (minSafeOffset !== null && key > minSafeOffset) continue;
+
+        // Merge into snapshot
+        try {
+          const data = JSON.parse(row.value);
+          if (opts.keepKey && data != null && typeof data === 'object' && opts.keepKey in data) {
+            // Keyed mode: group by field value, last write wins per key
+            const keyVal = String(data[opts.keepKey]);
+            snapData[keyVal] = data;
+          } else if (data != null && typeof data === 'object' && !Array.isArray(data)) {
+            // No keepKey: merge all events into a single state object
+            Object.assign(snapData, data);
+          }
+        } catch { /* skip unparseable */ }
+
+        // Delete the event row
+        stmtDel.run(row.path);
+        deleted++;
+        if (key > lastFoldedKey) lastFoldedKey = key;
+      }
+    });
+    tx();
+
+    // Write snapshot as single JSON row (like push rows — avoids leaf flattening)
+    if (deleted > 0 && lastFoldedKey) {
+      const snapshot: StreamSnapshot = { key: lastFoldedKey, data: snapData };
+      const snapshotPath = this.snapshotPath(topic);
+      this.storage.db.prepare('INSERT OR REPLACE INTO nodes (path, value, updated_at) VALUES (?, ?, ?)').run(
+        snapshotPath, JSON.stringify(snapshot), Date.now()
+      );
+    }
+
+    const snapshotSize = Object.keys(snapData).length;
+    return { deleted, snapshotSize };
+  }
+
+  /** Run auto-compact for all configured topics. Called from BodDB sweep. */
+  autoCompact(): CompactResult {
+    if (!this.options.compact) return { deleted: 0, snapshotSize: 0 };
+    let totalDeleted = 0;
+    let totalSnap = 0;
+    for (const [topic, opts] of Object.entries(this.options.compact)) {
+      const r = this.compact(topic, opts);
+      totalDeleted += r.deleted;
+      totalSnap += r.snapshotSize;
+    }
+    return { deleted: totalDeleted, snapshotSize: totalSnap };
+  }
+
+  /** Reset a topic — delete all events, snapshot, and consumer offsets. Notifies subscribers. */
+  reset(topic: string): void {
+    topic = validatePath(topic);
+    const prefix = topic + '/';
+    const end = prefixEnd(prefix);
+    // Collect event paths before deleting (for subscriber notification)
+    const eventPaths = (this.storage.db
+      .prepare('SELECT path FROM nodes WHERE path >= ? AND path < ?')
+      .all(prefix, end) as Array<{ path: string }>).map(r => r.path);
+    // Delete all event rows
+    this.storage.db.prepare('DELETE FROM nodes WHERE path >= ? AND path < ?').run(prefix, end);
+    // Delete snapshot + all group offsets
+    const metaPrefix = `_streams/${topic}/`;
+    const metaEnd = prefixEnd(metaPrefix);
+    this.storage.db.prepare('DELETE FROM nodes WHERE path >= ? AND path < ?').run(metaPrefix, metaEnd);
+    // Notify subscribers of removed events
+    if (eventPaths.length > 0 && this.subs.hasSubscriptions) {
+      this.subs.notify(eventPaths, (p) => this.storage.get(p));
+    }
+  }
+
+  /** Unsubscribe all active stream subscriptions (e.g. on disconnect) */
+  unsubscribeAll(): void {
+    for (const unsub of this.activeSubs.values()) unsub();
+    this.activeSubs.clear();
+  }
+
+  /** Get the minimum (earliest) offset across all consumer groups for a topic */
+  private getMinOffset(topic: string): string | null {
+    const groupsPath = `_streams/${topic}/groups`;
+    const groups = this.storage.getShallow(groupsPath);
+    if (groups.length === 0) return null;
+    let min: string | null = null;
+    for (const g of groups) {
+      const offset = this.getOffset(topic, g.key);
+      if (offset !== null && (min === null || offset < min)) {
+        min = offset;
+      }
+    }
+    return min;
+  }
+
+  private snapshotPath(topic: string): string {
+    return `_streams/${topic}/snapshot`;
+  }
+
+  private offsetPath(topic: string, groupId: string): string {
+    return `_streams/${topic}/groups/${groupId}/offset`;
+  }
+
+  private getOffset(topic: string, groupId: string): string | null {
+    const val = this.storage.get(this.offsetPath(topic, groupId));
+    return typeof val === 'string' ? val : null;
+  }
+
+  private setOffset(topic: string, groupId: string, key: string): void {
+    this.storage.set(this.offsetPath(topic, groupId), key);
+  }
+}

package/src/server/SubscriptionEngine.ts

@@ -0,0 +1,120 @@
+import { ancestors } from '../shared/pathUtils.ts';
+
+export type SubscriptionCallback = (snapshot: { path: string; val: () => unknown }) => void;
+export type ChildCallback = (event: { type: 'added' | 'changed' | 'removed'; key: string; path: string; val: () => unknown }) => void;
+
+export class SubscriptionEngine {
+  private valueSubs = new Map<string, Set<SubscriptionCallback>>();
+  private childSubs = new Map<string, Set<ChildCallback>>();
+
+  get hasSubscriptions(): boolean {
+    return this.valueSubs.size > 0 || this.childSubs.size > 0;
+  }
+
+  get hasChildSubscriptions(): boolean {
+    return this.childSubs.size > 0;
+  }
+
+  /** Count subscribers, optionally filtered by path prefix */
+  subscriberCount(prefix?: string): number {
+    let count = 0;
+    if (prefix) {
+      const p = prefix + '/';
+      for (const [key, subs] of this.valueSubs) if (key === prefix || key.startsWith(p)) count += subs.size;
+      for (const [key, subs] of this.childSubs) if (key === prefix || key.startsWith(p)) count += subs.size;
+    } else {
+      for (const subs of this.valueSubs.values()) count += subs.size;
+      for (const subs of this.childSubs.values()) count += subs.size;
+    }
+    return count;
+  }
+
+  /** Subscribe to value changes at a path (fires for exact + descendant writes) */
+  onValue(path: string, cb: SubscriptionCallback): () => void {
+    if (!this.valueSubs.has(path)) this.valueSubs.set(path, new Set());
+    this.valueSubs.get(path)!.add(cb);
+    return () => {
+      this.valueSubs.get(path)?.delete(cb);
+      if (this.valueSubs.get(path)?.size === 0) this.valueSubs.delete(path);
+    };
+  }
+
+  /** Subscribe to child events at a path */
+  onChild(path: string, cb: ChildCallback): () => void {
+    if (!this.childSubs.has(path)) this.childSubs.set(path, new Set());
+    this.childSubs.get(path)!.add(cb);
+    return () => {
+      this.childSubs.get(path)?.delete(cb);
+      if (this.childSubs.get(path)?.size === 0) this.childSubs.delete(path);
+    };
+  }
+
+  /**
+   * Notify after a write.
+   * @param changedPaths - all leaf paths that were written
+   * @param getFn - reads fresh from storage
+   * @param existedBefore - set of paths that existed before the write (for added/changed detection)
+   */
+  notify(changedPaths: string[], getFn: (path: string) => unknown, existedBefore?: Set<string>) {
+    const notified = new Set<string>();
+
+    for (const leafPath of changedPaths) {
+      const pathsToNotify = [leafPath, ...ancestors(leafPath)];
+
+      for (const p of pathsToNotify) {
+        if (notified.has(p)) continue;
+        notified.add(p);
+
+        const subs = this.valueSubs.get(p);
+        if (subs?.size) {
+          const snapshot = { path: p, val: () => getFn(p) };
+          for (const cb of subs) cb(snapshot);
+        }
+      }
+    }
+
+    // Child events
+    const childNotified = new Set<string>();
+    for (const leafPath of changedPaths) {
+      const parts = leafPath.split('/');
+      if (parts.length < 2) continue;
+
+      for (let i = 1; i < parts.length; i++) {
+        const parentPath = parts.slice(0, i).join('/');
+        const childKey = parts[i];
+        const dedupKey = `${parentPath}/${childKey}`;
+
+        if (childNotified.has(dedupKey)) continue;
+        childNotified.add(dedupKey);
+
+        const subs = this.childSubs.get(parentPath);
+        if (subs?.size) {
+          const childPath = `${parentPath}/${childKey}`;
+          const val = getFn(childPath);
+          let type: 'added' | 'changed' | 'removed';
+          if (val === null) {
+            type = 'removed';
+          } else if (existedBefore && !existedBefore.has(childPath)) {
+            type = 'added';
+          } else {
+            type = 'changed';
+          }
+          const event = { type, key: childKey, path: childPath, val: () => val };
+          for (const cb of subs) cb(event);
+        }
+      }
+    }
+  }
+
+  get size(): number {
+    let count = 0;
+    for (const s of this.valueSubs.values()) count += s.size;
+    for (const s of this.childSubs.values()) count += s.size;
+    return count;
+  }
+
+  clear() {
+    this.valueSubs.clear();
+    this.childSubs.clear();
+  }
+}