@bod.ee/db 0.12.1 → 0.12.4

package/src/server/VFSEngine.ts

@@ -133,9 +133,14 @@ export class VFSEngine {
 
   async write(virtualPath: string, data: Uint8Array, mime?: string): Promise<FileStat> {
     const vp = normalizePath(virtualPath);
-    const existing = this.db.get(this.metaPath(vp)) as Record<string, unknown> | null;
+    const existing = this.db.get(this.metaPath(vp)) as FileStat | null;
     const fileId = this.options.pathAsFileId ? vp : ((existing?.fileId as string) || generatePushId());
 
+    const hash = await computeHash(data);
+
+    // Skip write entirely if content hasn't changed (prevents unnecessary replication events)
+    if (existing?.hash === hash) return existing;
+
     await this.backend.write(fileId, data);
 
     const name = vp.split('/').pop()!;
@@ -147,6 +152,7 @@ export class VFSEngine {
       mtime: Date.now(),
       fileId,
       isDir: false,
+      hash,
     };
     this.db.set(this.metaPath(vp), stat);
     return stat;
@@ -251,13 +257,18 @@ export class VFSEngine {
 
     const newName = dstPath.split('/').pop()!;
     const fileId = this.options.pathAsFileId ? dstPath : meta.fileId;
-    const updated: FileStat = { ...meta, name: newName, path: dstPath, fileId, mtime: Date.now() };
+    const updated: FileStat = { ...meta, name: newName, path: dstPath, fileId, mtime: Date.now(), hash: meta.hash };
     this.db.set(this.metaPath(dstPath), updated);
     this.db.delete(this.containerPath(srcPath));
     return updated;
   }
 }
 
+async function computeHash(data: Uint8Array): Promise<string> {
+  const digest = await crypto.subtle.digest('SHA-256', data);
+  return Array.from(new Uint8Array(digest)).map(b => b.toString(16).padStart(2, '0')).join('');
+}
+
 function guessMime(name: string): string {
   const ext = name.split('.').pop()?.toLowerCase();
   const mimes: Record<string, string> = {

package/src/shared/protocol.ts

@@ -26,7 +26,7 @@ export type ClientMessage =
   | { id: string; op: 'vector-search'; query: number[]; path?: string; limit?: number; threshold?: number }
   | { id: string; op: 'vector-store'; path: string; embedding: number[] }
   | { id: string; op: 'stream-snapshot'; path: string }
-  | { id: string; op: 'stream-materialize'; path: string; keepKey?: string }
+  | { id: string; op: 'stream-materialize'; path: string; keepKey?: string; batchSize?: number; cursor?: string }
   | { id: string; op: 'stream-compact'; path: string; maxAge?: number; maxCount?: number; keepKey?: string }
   | { id: string; op: 'stream-reset'; path: string }
   // VFS ops
@@ -101,6 +101,7 @@ export interface FileStat {
   mtime: number;
   fileId?: string;
   isDir: boolean;
+  hash?: string;
 }
 
 export type BatchOp =

package/tests/repl-load.test.ts

@@ -0,0 +1,370 @@
+import { describe, it, expect, afterEach } from 'bun:test';
+import { BodDB } from '../src/server/BodDB.ts';
+import { BodClient } from '../src/client/BodClient.ts';
+
+const wait = (ms: number) => new Promise(r => setTimeout(r, ms));
+let nextPort = 27400 + Math.floor(Math.random() * 1000);
+
+/**
+ * Massive load tests — battle-test cursor-based bootstrap + threshold compact
+ * under realistic and extreme conditions.
+ */
+describe('repl load test', () => {
+  const instances: BodDB[] = [];
+  const clients: BodClient[] = [];
+
+  afterEach(() => {
+    for (const c of clients) c.disconnect();
+    clients.length = 0;
+    for (const db of [...instances].reverse()) db.close();
+    instances.length = 0;
+  });
+
+  function port() { return nextPort++; }
+
+  function primary(opts?: { compact?: any; autoCompactThreshold?: number }) {
+    const p = port();
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary', compact: opts?.compact ?? {}, autoCompactThreshold: opts?.autoCompactThreshold ?? 0 },
+    });
+    db.replication!.start();
+    db.serve({ port: p });
+    instances.push(db);
+    return { db, port: p };
+  }
+
+  function replica(primaryPort: number, opts?: Partial<{ replicaId: string; fullBootstrap: boolean }>) {
+    const p = port();
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: {
+        role: 'replica',
+        primaryUrl: `ws://localhost:${primaryPort}`,
+        replicaId: opts?.replicaId ?? `load-replica-${p}`,
+        fullBootstrap: opts?.fullBootstrap ?? true,
+      },
+    });
+    db.serve({ port: p });
+    instances.push(db);
+    return { db, port: p };
+  }
+
+  function connect(p: number, opts?: any) {
+    const c = new BodClient({ url: `ws://localhost:${p}`, ...opts });
+    clients.push(c);
+    return c;
+  }
+
+  // ─── 1. 20k entries: cursor pagination end-to-end ───
+
+  it('20k entries: cursor pagination collects every key', async () => {
+    const { db, port: p } = primary();
+    for (let i = 0; i < 20_000; i++) {
+      db.set(`items/i${i}`, { v: i, ts: Date.now() });
+    }
+
+    const client = connect(p);
+    await client.connect();
+
+    const keys = new Set<string>();
+    let cursor: string | undefined;
+    let pages = 0;
+    const t0 = Date.now();
+    do {
+      const page = await client.streamMaterialize('_repl', { keepKey: 'path', batchSize: 500, cursor });
+      for (const k of Object.keys(page.data)) keys.add(k);
+      cursor = page.nextCursor;
+      pages++;
+    } while (cursor);
+    const elapsed = Date.now() - t0;
+
+    console.log(`  20k entries: ${pages} pages, ${keys.size} unique keys, ${elapsed}ms`);
+    expect(keys.size).toBe(20_000);
+    expect(pages).toBeGreaterThanOrEqual(40); // 20k / 500
+  }, 30_000);
+
+  // ─── 2. Cursor vs monolithic: per-page response stays small ───
+
+  it('cursor pages stay under 1MB each while monolithic is huge', async () => {
+    const { db, port: p } = primary();
+    const payload = 'z'.repeat(500);
+    for (let i = 0; i < 5000; i++) {
+      db.set(`big/p${i}`, { data: payload, i });
+    }
+
+    const client = connect(p);
+    await client.connect();
+
+    // Monolithic
+    const mono = await client.streamMaterialize('_repl', { keepKey: 'path' });
+    const monoSize = JSON.stringify(mono).length;
+
+    // Cursor-based
+    let maxPageSize = 0;
+    let cursor: string | undefined;
+    do {
+      const page = await client.streamMaterialize('_repl', { keepKey: 'path', batchSize: 200, cursor });
+      const sz = JSON.stringify(page.data).length;
+      if (sz > maxPageSize) maxPageSize = sz;
+      cursor = page.nextCursor;
+    } while (cursor);
+
+    console.log(`  monolithic: ${(monoSize / 1024 / 1024).toFixed(2)}MB, max page: ${(maxPageSize / 1024).toFixed(0)}KB`);
+    expect(monoSize).toBeGreaterThan(2 * 1024 * 1024); // >2MB total
+    expect(maxPageSize).toBeLessThan(1024 * 1024);       // each page <1MB
+  }, 15_000);
+
+  // ─── 3. Auto-compact under sustained write load ───
+
+  it('auto-compact keeps _repl bounded under sustained 10k writes', () => {
+    const { db } = primary({ compact: { maxCount: 200, keepKey: 'path' }, autoCompactThreshold: 500 });
+
+    for (let i = 0; i < 10_000; i++) {
+      db.set(`stream/key${i % 300}`, { round: Math.floor(i / 300), i });
+    }
+
+    const repl = db.get('_repl') as Record<string, any>;
+    const count = repl ? Object.keys(repl).length : 0;
+    console.log(`  10k writes (300 unique paths), threshold=500, maxCount=200: ${count} _repl entries`);
+    // Without compact: 10k. With compact every 500 writes keeping 200: should be way under 1000.
+    expect(count).toBeLessThan(1000);
+  }, 15_000);
+
+  // ─── 4. Replica bootstrap with 10k entries via cursor ───
+
+  it('replica bootstraps 10k entries via cursor without timeout', async () => {
+    const { db: p, port: pp } = primary();
+    for (let i = 0; i < 10_000; i++) {
+      p.set(`data/node${i}`, { value: i, name: `node-${i}`, tags: ['a', 'b'] });
+    }
+
+    const { db: r } = replica(pp);
+    const t0 = Date.now();
+    await r.replication!.start();
+    const elapsed = Date.now() - t0;
+
+    console.log(`  10k entry replica bootstrap: ${elapsed}ms`);
+
+    await wait(500);
+    // Spot-check
+    for (const idx of [0, 999, 5000, 9999]) {
+      const val = r.get(`data/node${idx}`) as any;
+      expect(val?.value).toBe(idx);
+    }
+  }, 30_000);
+
+  // ─── 5. Multiple replicas bootstrap concurrently ───
+
+  it('3 replicas bootstrap concurrently from same primary (10k entries)', async () => {
+    const { db: p, port: pp } = primary();
+    for (let i = 0; i < 10_000; i++) {
+      p.set(`shared/item${i}`, { v: i });
+    }
+
+    const replicas = [replica(pp), replica(pp), replica(pp)];
+    const t0 = Date.now();
+    await Promise.all(replicas.map(r => r.db.replication!.start()));
+    const elapsed = Date.now() - t0;
+
+    console.log(`  3 concurrent replica bootstraps (10k): ${elapsed}ms`);
+
+    await wait(500);
+    for (const r of replicas) {
+      const v0 = r.db.get('shared/item0') as any;
+      const v9999 = r.db.get('shared/item9999') as any;
+      expect(v0?.v).toBe(0);
+      expect(v9999?.v).toBe(9999);
+    }
+  }, 45_000);
+
+  // ─── 6. Writes during bootstrap: replica catches up via stream sub ───
+
+  it('writes during bootstrap are caught via ongoing stream subscription', async () => {
+    const { db: p, port: pp } = primary();
+    // Pre-fill
+    for (let i = 0; i < 5000; i++) {
+      p.set(`pre/item${i}`, { v: i });
+    }
+
+    const { db: r } = replica(pp);
+    // Start replica (bootstrap starts)
+    const startPromise = r.replication!.start();
+
+    // Write more to primary while bootstrap is in progress
+    for (let i = 0; i < 500; i++) {
+      p.set(`live/item${i}`, { v: i + 100_000 });
+    }
+
+    await startPromise;
+    // Give stream sub time to deliver live writes
+    await wait(1000);
+
+    // Pre-fill data should be there
+    const pre0 = r.get('pre/item0') as any;
+    expect(pre0?.v).toBe(0);
+    const pre4999 = r.get('pre/item4999') as any;
+    expect(pre4999?.v).toBe(4999);
+
+    // Live writes should eventually arrive
+    const live499 = r.get('live/item499') as any;
+    expect(live499?.v).toBe(100_499);
+  }, 30_000);
+
+  // ─── 7. Heavy overwrite scenario: same 50 paths written 1000× each ───
+
+  it('50 paths × 1000 overwrites: compact deduplicates correctly', () => {
+    const { db } = primary({ compact: { maxCount: 100, keepKey: 'path' }, autoCompactThreshold: 1000 });
+
+    for (let round = 0; round < 1000; round++) {
+      for (let i = 0; i < 50; i++) {
+        db.set(`hot/key${i}`, { round, value: round * 50 + i });
+      }
+    }
+
+    const repl = db.get('_repl') as Record<string, any>;
+    const count = repl ? Object.keys(repl).length : 0;
+    console.log(`  50×1000 overwrites: ${count} _repl entries (expect ≤ ~1100)`);
+    // 50k total writes. Compact every 1k writes keeping 100. Should be bounded.
+    expect(count).toBeLessThan(1500);
+
+    // Verify latest values survived compaction
+    const materialized = db.stream.materialize('_repl', { keepKey: 'path' });
+    const paths = Object.keys(materialized);
+    expect(paths.length).toBe(50);
+  }, 30_000);
+
+  // ─── 8. Cursor pagination with snapshot (post-compact) ───
+
+  it('cursor pagination works correctly after compaction (snapshot + live events)', async () => {
+    const { db, port: p } = primary();
+    // Write 2000, compact to 200, write 800 more
+    for (let i = 0; i < 2000; i++) db.set(`a/item${i}`, { v: i });
+    db.stream.compact('_repl', { maxCount: 200, keepKey: 'path' });
+    for (let i = 2000; i < 2800; i++) db.set(`a/item${i}`, { v: i });
+
+    const client = connect(p);
+    await client.connect();
+
+    const keys = new Set<string>();
+    let cursor: string | undefined;
+    let pages = 0;
+    do {
+      const page = await client.streamMaterialize('_repl', { keepKey: 'path', batchSize: 100, cursor });
+      for (const k of Object.keys(page.data)) keys.add(k);
+      cursor = page.nextCursor;
+      pages++;
+    } while (cursor);
+
+    console.log(`  2000 + compact + 800 more: ${keys.size} keys in ${pages} pages`);
+    // All 2800 unique paths should be present (snapshot has older ones, events have newer)
+    expect(keys.size).toBe(2800);
+  }, 15_000);
+
+  // ─── 9. Replica with auto-compact primary: data integrity ───
+
+  it('replica gets correct data when primary auto-compacts during heavy writes', async () => {
+    const { db: p, port: pp } = primary({ compact: { maxCount: 300, keepKey: 'path' }, autoCompactThreshold: 500 });
+
+    // 5000 writes — auto-compact fires multiple times
+    for (let i = 0; i < 5000; i++) {
+      p.set(`verified/item${i}`, { value: i * 7, tag: 'check' });
+    }
+
+    const { db: r } = replica(pp);
+    await r.replication!.start();
+    await wait(500);
+
+    // Exhaustive integrity check on a sample
+    const sample = [0, 100, 999, 2500, 4000, 4999];
+    for (const idx of sample) {
+      const val = r.get(`verified/item${idx}`) as any;
+      expect(val?.value).toBe(idx * 7);
+      expect(val?.tag).toBe('check');
+    }
+  }, 30_000);
+
+  // ─── 10. Mixed deletes + sets under load ───
+
+  it('deletes replicate correctly through cursor-based bootstrap (no fullBootstrap)', async () => {
+    const { db: p, port: pp } = primary();
+
+    // Create 1000, then delete half
+    for (let i = 0; i < 1000; i++) {
+      p.set(`mix/item${i}`, { v: i });
+    }
+    for (let i = 0; i < 1000; i += 2) {
+      p.delete(`mix/item${i}`);
+    }
+
+    // Disable fullBootstrap so only _repl stream materialize is used
+    const { db: r } = replica(pp, { fullBootstrap: false });
+    await r.replication!.start();
+    await wait(500);
+
+    // Even indices: _repl has set then delete — materialize with keepKey=path keeps last op (delete)
+    // But materialize folds by keepKey, and delete events have op:'delete' — they apply as db.delete()
+    // Odd indices should exist from set events
+    for (const i of [1, 3, 99, 999]) {
+      const val = r.get(`mix/item${i}`) as any;
+      expect(val?.v).toBe(i);
+    }
+    // Verify primary has them deleted
+    for (const i of [0, 2, 100, 998]) {
+      expect(p.get(`mix/item${i}`)).toBeNull();
+    }
+  }, 15_000);
+
+  // ─── 11. Rapid batchSize=1 pagination (worst case) ───
+
+  it('batchSize=1 pagination still completes for 500 entries', async () => {
+    const { db, port: p } = primary();
+    for (let i = 0; i < 500; i++) db.set(`tiny/k${i}`, { i });
+
+    const client = connect(p);
+    await client.connect();
+
+    const keys = new Set<string>();
+    let cursor: string | undefined;
+    let pages = 0;
+    const t0 = Date.now();
+    do {
+      const page = await client.streamMaterialize('_repl', { keepKey: 'path', batchSize: 1, cursor });
+      for (const k of Object.keys(page.data)) keys.add(k);
+      cursor = page.nextCursor;
+      pages++;
+    } while (cursor);
+    const elapsed = Date.now() - t0;
+
+    console.log(`  batchSize=1 over 500 entries: ${pages} pages, ${elapsed}ms`);
+    expect(keys.size).toBe(500);
+    expect(pages).toBeGreaterThanOrEqual(500);
+  }, 30_000);
+
+  // ─── 12. Throughput benchmark: writes/sec with auto-compact ───
+
+  it('write throughput with auto-compact enabled', () => {
+    const { db: dbCompact } = primary({ compact: { maxCount: 200, keepKey: 'path' }, autoCompactThreshold: 500 });
+    const { db: dbPlain } = primary();
+
+    const N = 10_000;
+
+    const t0 = Date.now();
+    for (let i = 0; i < N; i++) dbPlain.set(`bench/k${i}`, { i });
+    const plainMs = Date.now() - t0;
+
+    const t1 = Date.now();
+    for (let i = 0; i < N; i++) dbCompact.set(`bench/k${i}`, { i });
+    const compactMs = Date.now() - t1;
+
+    const plainWps = Math.round(N / (plainMs / 1000));
+    const compactWps = Math.round(N / (compactMs / 1000));
+    const overhead = ((compactMs - plainMs) / plainMs * 100).toFixed(1);
+
+    console.log(`  ${N} writes — plain: ${plainMs}ms (${plainWps} w/s), compact: ${compactMs}ms (${compactWps} w/s), overhead: ${overhead}%`);
+    // Auto-compact overhead should be < 100% (compact is cheap relative to N writes)
+    expect(compactMs).toBeLessThan(plainMs * 3);
+  }, 30_000);
+});