@bod.ee/db 0.12.8 → 0.13.1

package/src/server/ReplicationEngine.ts

@@ -116,6 +116,8 @@ export class ReplicationSource {
   paths: string[] = [];
   localPrefix?: string;
   excludePrefixes?: string[];
+  /** Subscribe to `_repl_scoped/<prefix>` instead of filtering `_repl`. Requires primary to have matching `scopedPrefixes`. */
+  scopedStream?: boolean;
 }
 
 export class ReplicationOptions {
@@ -124,7 +126,7 @@ export class ReplicationOptions {
   primaryAuth?: () => string | Promise<string>;
   replicaId?: string;
   /** Paths excluded from replication. Note: when `paths` explicitly configures a prefix (e.g. `_auth`), it overrides this exclusion. */
-  excludePrefixes: string[] = ['_repl', '_streams', '_mq', '_admin', '_auth'];
+  excludePrefixes: string[] = ['_repl', '_repl_scoped', '_streams', '_mq', '_admin', '_auth'];
   /** Bootstrap replica from primary's full state before applying _repl stream */
   fullBootstrap: boolean = true;
   compact?: CompactOptions;
@@ -133,6 +135,8 @@ export class ReplicationOptions {
   sources?: ReplicationSource[];
   /** Per-path topology: strings default to 'sync', objects specify mode. When absent, role governs all paths. */
   paths?: Array<string | PathTopology>;
+  /** Path prefixes that get their own `_repl_scoped/<prefix>` stream. Auto-derived from paths/sources if not set. */
+  scopedPrefixes?: string[];
 }
 
 const BOOTSTRAP_BATCH_SIZE = 200;
@@ -142,6 +146,8 @@ type ProxyableMessage = Extract<ClientMessage, { op: 'set' | 'delete' | 'update'
 export class ReplicationEngine {
   readonly options: ReplicationOptions;
   readonly router: PathTopologyRouter | null = null;
+  /** Resolved scoped prefixes (sorted longest-first for matching) */
+  private _scopedPrefixes: string[] = [];
   private client: BodClient | null = null;
   private unsubWrite: (() => void) | null = null;
   private unsubStream: (() => void) | null = null;
@@ -193,6 +199,8 @@ export class ReplicationEngine {
     if (this.options.paths?.length) {
       this.router = new PathTopologyRouter(this.options.paths, this.options.role);
     }
+    // Resolve scoped prefixes
+    this._scopedPrefixes = this._resolveScopedPrefixes();
     if (!this.options.replicaId) {
       // Derive stable ID from config when possible, fall back to random
       if (this.options.primaryUrl) {
@@ -293,6 +301,29 @@ export class ReplicationEngine {
     }
   }
 
+  /** Derive scoped prefixes for primary emit. Only auto-derives from topology paths when role is 'primary'.
+   *  Replicas must use explicit `scopedPrefixes` config since they don't control what the primary emits. */
+  private _resolveScopedPrefixes(): string[] {
+    if (this.options.scopedPrefixes) return [...this.options.scopedPrefixes].sort((a, b) => b.length - a.length);
+    // Only auto-derive for primary role — replica/sync nodes don't control primary's emit topics
+    if (this.options.role !== 'primary') return [];
+    const set = new Set<string>();
+    if (this.router) {
+      for (const e of this.router.getEntries()) {
+        if (e.path) set.add(e.path);
+      }
+    }
+    return [...set].sort((a, b) => b.length - a.length);
+  }
+
+  /** Longest-prefix match against scoped prefixes */
+  private _matchScopedPrefix(path: string): string | null {
+    for (const p of this._scopedPrefixes) {
+      if (path === p || path.startsWith(p + '/')) return p;
+    }
+    return null;
+  }
+
   // --- Primary mode ---
 
   private startPrimary(): void {
@@ -307,6 +338,9 @@ export class ReplicationEngine {
     const compact = this.options.compact ?? { maxCount: 500, keepKey: 'path' };
     if (compact.maxCount || compact.maxAge) {
       try { this.db.stream.compact('_repl', compact); } catch {}
+      for (const prefix of this._scopedPrefixes) {
+        try { this.db.stream.compact(`_repl_scoped/${prefix}`, compact); } catch {}
+      }
     }
   }
 
@@ -324,7 +358,7 @@ export class ReplicationEngine {
 
   /** Buffer replication events during transactions, emit immediately otherwise */
   private emit(ev: WriteEvent): void {
-    // _repl writes must never emit to _repl (infinite loop), regardless of config
+    // _repl/_repl_scoped writes must never emit (infinite loop)
     if (ev.path.startsWith('_repl')) return;
     if (this._emitting) {
       this.log.debug('emit: skipped (re-entrant)', { path: ev.path });
@@ -363,6 +397,19 @@ export class ReplicationEngine {
       const idempotencyKey = `${replEvent.ts}:${seq}:${ev.path}`;
       this.db.push('_repl', replEvent, { idempotencyKey });
       this.log.info('_repl emit', { seq, op: ev.op, path: ev.path });
+
+      // Dual emit to scoped stream
+      const scopedPrefix = this._matchScopedPrefix(ev.path);
+      if (scopedPrefix) {
+        const scopedTopic = `_repl_scoped/${scopedPrefix}`;
+        const scopedKey = `${replEvent.ts}:${seq}:scoped:${ev.path}`;
+        try {
+          this.db.push(scopedTopic, replEvent, { idempotencyKey: scopedKey });
+          this.log.debug('scoped emit', { topic: scopedTopic, path: ev.path });
+        } catch (e: any) {
+          this.log.error('scoped emit failed', { topic: scopedTopic, error: e.message });
+        }
+      }
     } catch (e: any) {
       this.log.error('_repl emit failed', { path: ev.path, error: e.message });
     } finally {
@@ -375,6 +422,10 @@ export class ReplicationEngine {
       this._emitCount = 0;
       const compact = this.options.compact ?? { maxCount: 500, keepKey: 'path' };
       try { this.db.stream.compact('_repl', compact); } catch {}
+      // Compact scoped streams too
+      for (const prefix of this._scopedPrefixes) {
+        try { this.db.stream.compact(`_repl_scoped/${prefix}`, compact); } catch {}
+      }
     }
   }
 
@@ -416,30 +467,67 @@ export class ReplicationEngine {
       await this.bootstrapFullState(bootstrapPaths);
     }
 
-    // Stream bootstrap filtered (cursor-based to avoid huge single response)
-    const applied = await this.bootstrapFromStream(this.client!, { filter: ev => this.matchesPathPrefixes(ev.path, pathPrefixes) });
-    this.log.info(`Stream bootstrap (paths): ${applied} events applied`);
-
-    // Subscribe to ongoing events, filter by paths
+    // Use scoped streams for paths that have them, fall back to filtered _repl
     const groupId = this.options.replicaId!;
-    this.log.info('Subscribing to _repl stream', { groupId, pathPrefixes });
-    this.unsubStream = this.client.stream('_repl', groupId).on((events) => {
-      this.log.info('_repl events received', { count: events.length });
-      this.db.setReplaying(true);
-      try {
-        for (const e of events) {
-          const ev = e.val() as ReplEvent;
-          const matched = this.matchesPathPrefixes(ev.path, pathPrefixes);
-          this.log.info('_repl event', { op: ev.op, path: ev.path, matched });
-          if (matched) {
-            this.applyEvent(ev);
+    const scopedSubs: Array<() => void> = [];
+    const unscopedPaths: string[] = [];
+
+    // Also bootstrap scoped paths from _repl (catches historical events from before scoped streams existed)
+    const scopedPathPrefixes = pathPrefixes.filter(p => this._scopedPrefixes.includes(p));
+    if (scopedPathPrefixes.length > 0) {
+      const fallbackApplied = await this.bootstrapFromStream(this.client!, { filter: ev => this.matchesPathPrefixes(ev.path, scopedPathPrefixes), groupId });
+      if (fallbackApplied > 0) this.log.info(`Scoped _repl fallback bootstrap: ${fallbackApplied} events`);
+    }
+
+    for (const prefix of pathPrefixes) {
+      if (this._scopedPrefixes.includes(prefix)) {
+        const scopedTopic = `_repl_scoped/${prefix}`;
+        // Bootstrap from scoped stream (may apply duplicates from _repl fallback — applyEvent is idempotent via set/delete)
+        const applied = await this.bootstrapFromStream(this.client!, { streamTopic: scopedTopic, groupId });
+        this.log.info(`Scoped stream bootstrap: ${scopedTopic} → ${applied} events`);
+
+        // Subscribe to scoped stream (no filter needed)
+        const unsub = this.client!.stream(scopedTopic, groupId).on((events) => {
+          this.db.setReplaying(true);
+          try {
+            for (const e of events) {
+              this.applyEvent(e.val() as ReplEvent);
+              this.client!.stream(scopedTopic, groupId).ack(e.key).catch(() => {});
+            }
+          } finally {
+            this.db.setReplaying(false);
           }
-          this.client!.stream('_repl', groupId).ack(e.key).catch(() => {});
-        }
-      } finally {
-        this.db.setReplaying(false);
+        });
+        scopedSubs.push(unsub);
+      } else {
+        unscopedPaths.push(prefix);
       }
-    });
+    }
+
+    // Fall back to _repl with filter for paths without scoped streams
+    if (unscopedPaths.length > 0) {
+      const applied = await this.bootstrapFromStream(this.client!, { filter: ev => this.matchesPathPrefixes(ev.path, unscopedPaths), groupId });
+      this.log.info(`Stream bootstrap (filtered _repl): ${applied} events for [${unscopedPaths.join(', ')}]`);
+
+      const unsub = this.client!.stream('_repl', groupId).on((events) => {
+        this.db.setReplaying(true);
+        try {
+          for (const e of events) {
+            const ev = e.val() as ReplEvent;
+            if (this.matchesPathPrefixes(ev.path, unscopedPaths)) {
+              this.applyEvent(ev);
+            }
+            this.client!.stream('_repl', groupId).ack(e.key).catch(() => {});
+          }
+        } finally {
+          this.db.setReplaying(false);
+        }
+      });
+      scopedSubs.push(unsub);
+    }
+
+    // Combine all unsub functions
+    this.unsubStream = () => { for (const u of scopedSubs) u(); };
   }
 
   /** Check if path matches any of the given prefixes */
@@ -464,11 +552,11 @@ export class ReplicationEngine {
     }
 
     // Stream bootstrap: cursor-based to avoid huge single response
-    const applied = await this.bootstrapFromStream(this.client!);
+    const groupId = this.options.replicaId!;
+    const applied = await this.bootstrapFromStream(this.client!, { groupId });
     this.log.info(`Stream bootstrap: ${applied} events applied`);
 
     // Subscribe to ongoing events
-    const groupId = this.options.replicaId!;
     this.log.info(`Subscribing to stream as '${groupId}'`);
     this.unsubStream = this.client.stream('_repl', groupId).on((events) => {
       this.log.debug(`Received ${events.length} events`);
@@ -529,28 +617,39 @@ export class ReplicationEngine {
     const client = new BodClient({ url: source.url, auth: source.auth });
     await client.connect();
 
-    // Bootstrap: cursor-based materialize _repl, filter by source paths
+    const groupId = source.id || `source_${source.url}_${source.paths.sort().join('+')}`;
+    const connState = { client, unsub: () => {}, lastEventTs: Date.now(), eventsApplied: 0, pending: 0 };
+
+    // Use scoped stream if explicitly enabled and source has a single path
+    if (source.scopedStream && source.paths.length > 1) {
+      this.log.warn('scopedStream only supported for single-path sources, falling back to filtered _repl', { paths: source.paths });
+    }
+    const useScoped = source.scopedStream && source.paths.length === 1;
+    const topic = useScoped ? `_repl_scoped/${source.paths[0]}` : '_repl';
+    const needsFilter = !useScoped;
+
+    // Bootstrap
     await this.bootstrapFromStream(client, {
-      filter: ev => this.matchesSourcePaths(ev.path, source),
+      filter: needsFilter ? (ev => this.matchesSourcePaths(ev.path, source)) : undefined,
       source,
+      streamTopic: topic,
+      groupId,
     });
 
     // Subscribe to ongoing events
-    const groupId = source.id || `source_${source.url}_${source.paths.sort().join('+')}`;
-    const connState = { client, unsub: () => {}, lastEventTs: Date.now(), eventsApplied: 0, pending: 0 };
-    const unsub = client.stream('_repl', groupId).on((events) => {
+    const unsub = client.stream(topic, groupId).on((events) => {
       connState.pending += events.length;
       this.db.setReplaying(true);
       try {
         for (const e of events) {
           const ev = e.val() as ReplEvent;
           connState.lastEventTs = ev.ts || Date.now();
-          if (this.matchesSourcePaths(ev.path, source)) {
+          if (!needsFilter || this.matchesSourcePaths(ev.path, source)) {
             this.applyEvent(ev, source);
             connState.eventsApplied++;
           }
           connState.pending--;
-          client.stream('_repl', groupId).ack(e.key).catch(() => {});
+          client.stream(topic, groupId).ack(e.key).catch(() => {});
         }
       } finally {
         this.db.setReplaying(false);
@@ -570,8 +669,22 @@ export class ReplicationEngine {
     return source.localPrefix ? `${source.localPrefix}/${path}` : path;
   }
 
-  /** Cursor-based stream bootstrap: pages through _repl materialize to avoid huge single responses */
-  private async bootstrapFromStream(client: BodClient, opts?: { filter?: (ev: ReplEvent) => boolean; source?: ReplicationSource }): Promise<number> {
+  /** Cursor-based stream bootstrap: pages through _repl (or scoped stream) materialize to avoid huge single responses.
+   *  Skips bootstrap entirely when the consumer group already has a stored offset (subscribe handles incremental replay). */
+  private async bootstrapFromStream(client: BodClient, opts?: { filter?: (ev: ReplEvent) => boolean; source?: ReplicationSource; streamTopic?: string; groupId?: string }): Promise<number> {
+    const topic = opts?.streamTopic ?? '_repl';
+    const groupId = opts?.groupId;
+
+    // If consumer group has a stored offset, skip full bootstrap — subscribe() will replay incrementally from that offset
+    if (groupId) {
+      const offsetPath = `_streams/${topic}/groups/${groupId}/offset`;
+      const existingOffset = await client.get(offsetPath);
+      if (existingOffset != null) {
+        this.log.info(`Skipping bootstrap for ${topic} (group '${groupId}' has offset, subscribe will resume)`);
+        return 0;
+      }
+    }
+
     let cursor: string | undefined;
     let applied = 0;
     const filter = opts?.filter;
@@ -579,7 +692,7 @@ export class ReplicationEngine {
     this.db.setReplaying(true);
     try {
       do {
-        const page = await client.streamMaterialize('_repl', { keepKey: 'path', batchSize: BOOTSTRAP_BATCH_SIZE, cursor });
+        const page = await client.streamMaterialize(topic, { keepKey: 'path', batchSize: BOOTSTRAP_BATCH_SIZE, cursor });
         if (page.data) {
           for (const [, event] of Object.entries(page.data)) {
             const ev = event as ReplEvent;

package/src/server/StreamEngine.ts

@@ -128,9 +128,9 @@ export class StreamEngine {
       }
     });
 
-    // 2. Fetch replay batch
+    // 2. Fetch replay batch (use large limit to ensure all missed events are replayed for catch-up)
     const offset = this.getOffset(topic, groupId);
-    const replayBatch = this.storage.queryAfterKey(topic, offset, this.options.defaultLimit);
+    const replayBatch = this.storage.queryAfterKey(topic, offset, Number.MAX_SAFE_INTEGER);
 
     // 3. Send replay batch
     if (replayBatch.length > 0) {

package/src/server/Transport.ts

@@ -30,6 +30,8 @@ export class TransportOptions {
   keyAuth?: KeyAuthEngine;
   /** Map URL paths to local file paths, e.g. { '/admin': './admin/ui.html' } */
   staticRoutes?: Record<string, string>;
+  /** Map URL path prefixes to local directories, e.g. { '/assets': './admin/dist/assets' } */
+  staticDirs?: Record<string, string>;
   /** Custom REST routes checked before 404. Return null to fall through. */
   extraRoutes?: Record<string, (req: Request, url: URL) => Response | Promise<Response> | null>;
   /** Maximum concurrent WebSocket connections (default 10000) */
@@ -52,6 +54,11 @@ export class Transport {
   private _log: ComponentLogger;
   get clientCount(): number { return this._clients.size; }
 
+  /** Returns true if any WS client is subscribed to _admin/stats or _admin. */
+  get hasStatsSubscribers(): boolean {
+    return !!(this._valueGroups.get('_admin/stats')?.size || this._valueGroups.get('_admin')?.size);
+  }
+
   /** Send pre-serialized stats directly to subscribed WS clients, bypassing SubscriptionEngine. */
   broadcastStats(data: unknown, updatedAt: number): void {
     // Check both exact path and ancestor subscribers
@@ -378,6 +385,16 @@ export class Transport {
       if (filePath) return new Response(Bun.file(filePath));
     }
 
+    // Static dirs (prefix-based, for SPA assets)
+    if (this.options.staticDirs) {
+      for (const [prefix, dir] of Object.entries(this.options.staticDirs)) {
+        if (url.pathname.startsWith(prefix)) {
+          const rel = url.pathname.slice(prefix.length);
+          return new Response(Bun.file(`${dir}${rel}`));
+        }
+      }
+    }
+
     // Extra routes (custom REST handlers)
     if (this.options.extraRoutes) {
       for (const [pattern, handler] of Object.entries(this.options.extraRoutes)) {

package/tests/replication.test.ts

@@ -276,7 +276,7 @@ describe('ReplicationEngine', () => {
 
   // --- Source feed subscription tests ---
 
-  function createSourceLocal(sources: Array<{ url: string; paths: string[]; localPrefix?: string; excludePrefixes?: string[] }>) {
+  function createSourceLocal(sources: Array<{ url: string; paths: string[]; localPrefix?: string; excludePrefixes?: string[]; scopedStream?: boolean }>) {
     const port = getPort();
     const db = new BodDB({
       path: ':memory:',
@@ -393,6 +393,167 @@ describe('ReplicationEngine', () => {
     expect(local.get('src/users/u1')).toBeNull();
   });
 
+  // --- Scoped replication stream tests ---
+
+  function createScopedPrimary(scopedPrefixes: string[]) {
+    const port = getPort();
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary', scopedPrefixes },
+    });
+    db.replication!.start();
+    db.serve({ port });
+    instances.push(db);
+    return { db, port };
+  }
+
+  it('scoped primary dual-emits to _repl and _repl_scoped/<prefix>', async () => {
+    const { db } = createScopedPrimary(['users']);
+    db.set('users/u1', { name: 'Eli' });
+    db.set('orders/o1', { total: 100 });
+    await tick();
+
+    // Both events in _repl
+    const replData = db.get('_repl') as Record<string, any>;
+    const replPaths = Object.values(replData).map((e: any) => e.path);
+    expect(replPaths).toContain('users/u1');
+    expect(replPaths).toContain('orders/o1');
+
+    // Only users event in scoped stream
+    const scopedData = db.get('_repl_scoped/users') as Record<string, any>;
+    const scopedPaths = Object.values(scopedData).map((e: any) => e.path);
+    expect(scopedPaths).toContain('users/u1');
+    expect(scopedPaths).not.toContain('orders/o1');
+  });
+
+  it('source with scopedStream uses _repl_scoped instead of filtering _repl', async () => {
+    const { db: remote, port: rPort } = createScopedPrimary(['users']);
+    remote.set('users/u1', { name: 'Eli' });
+    remote.set('orders/o1', { total: 100 });
+
+    const { db: local } = createSourceLocal([
+      { url: `ws://localhost:${rPort}`, paths: ['users'], scopedStream: true },
+    ]);
+    await local.replication!.start();
+    await wait(500);
+
+    expect(local.get('users/u1')).toEqual({ name: 'Eli' });
+    expect(local.get('orders/o1')).toBeNull();
+  });
+
+  it('source with scopedStream receives ongoing events', async () => {
+    const { db: remote, port: rPort } = createScopedPrimary(['users']);
+    remote.set('users/u1', { name: 'Eli' });
+
+    const { db: local } = createSourceLocal([
+      { url: `ws://localhost:${rPort}`, paths: ['users'], scopedStream: true },
+    ]);
+    await local.replication!.start();
+    await wait(500);
+
+    expect(local.get('users/u1')).toEqual({ name: 'Eli' });
+
+    remote.set('users/u2', { name: 'Dan' });
+    await wait(500);
+    expect(local.get('users/u2')).toEqual({ name: 'Dan' });
+
+    // Non-matching writes should not appear
+    remote.set('orders/o1', { total: 50 });
+    await wait(500);
+    expect(local.get('orders/o1')).toBeNull();
+  });
+
+  it('whole-DB replica still works unchanged on _repl', async () => {
+    const { db: primary, port: pPort } = createScopedPrimary(['users']);
+    primary.set('users/u1', { name: 'Eli' });
+    primary.set('orders/o1', { total: 100 });
+
+    const { db: replica } = createReplica(pPort);
+    await replica.replication!.start();
+    await wait(500);
+
+    expect(replica.get('users/u1')).toEqual({ name: 'Eli' });
+    expect(replica.get('orders/o1')).toEqual({ total: 100 });
+  });
+
+  it('scoped compaction runs on _repl_scoped streams', async () => {
+    const { db } = createScopedPrimary(['users']);
+    // Write enough to trigger auto-compact threshold
+    for (let i = 0; i < 5; i++) db.set(`users/u${i}`, { n: i });
+    await tick();
+
+    const scopedData = db.get('_repl_scoped/users') as Record<string, any>;
+    expect(Object.keys(scopedData).length).toBeGreaterThan(0);
+  });
+
+  it('per-path topology replica uses scoped streams', async () => {
+    const pPort = getPort();
+    const primary = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary', scopedPrefixes: ['users', 'orders'] },
+    });
+    primary.replication!.start();
+    primary.serve({ port: pPort });
+    instances.push(primary);
+
+    primary.set('users/u1', { name: 'Eli' });
+    primary.set('orders/o1', { total: 100 });
+    primary.set('config/theme', 'dark');
+
+    const rPort = getPort();
+    const replica = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: {
+        role: 'replica',
+        primaryUrl: `ws://localhost:${pPort}`,
+        replicaId: `test-scoped-topo-${rPort}`,
+        paths: [{ path: 'users', mode: 'replica' as const }],
+        scopedPrefixes: ['users'],
+      },
+    });
+    replica.serve({ port: rPort });
+    instances.push(replica);
+    await replica.replication!.start();
+    await wait(500);
+
+    expect(replica.get('users/u1')).toEqual({ name: 'Eli' });
+    // orders not in topology — should not be replicated
+    expect(replica.get('orders/o1')).toBeNull();
+  });
+
+  it('multi-path source with scopedStream falls back to filtered _repl', async () => {
+    const { db: remote, port: rPort } = createScopedPrimary(['users', 'orders']);
+    remote.set('users/u1', { name: 'Eli' });
+    remote.set('orders/o1', { total: 100 });
+
+    const { db: local } = createSourceLocal([
+      { url: `ws://localhost:${rPort}`, paths: ['users', 'orders'], scopedStream: true },
+    ]);
+    await local.replication!.start();
+    await wait(500);
+
+    // Should still work via _repl fallback
+    expect(local.get('users/u1')).toEqual({ name: 'Eli' });
+    expect(local.get('orders/o1')).toEqual({ total: 100 });
+  });
+
+  it('source with scopedStream and localPrefix remaps correctly', async () => {
+    const { db: remote, port: rPort } = createScopedPrimary(['users']);
+    remote.set('users/u1', { name: 'Eli' });
+
+    const { db: local } = createSourceLocal([
+      { url: `ws://localhost:${rPort}`, paths: ['users'], scopedStream: true, localPrefix: 'ext' },
+    ]);
+    await local.replication!.start();
+    await wait(500);
+
+    expect(local.get('ext/users/u1')).toEqual({ name: 'Eli' });
+    expect(local.get('users/u1')).toBeNull();
+  });
+
   it('stop() disconnects source clients', async () => {
     const { db: remote, port: rPort } = createPrimary();
     remote.set('users/u1', { name: 'Eli' });