@bod.ee/db 0.12.1 → 0.12.4

package/tests/repl-stream-bloat.test.ts

@@ -0,0 +1,295 @@
+import { describe, it, expect, afterEach } from 'bun:test';
+import { BodDB } from '../src/server/BodDB.ts';
+import { BodClient } from '../src/client/BodClient.ts';
+
+const wait = (ms: number) => new Promise(r => setTimeout(r, ms));
+let nextPort = 26400 + Math.floor(Math.random() * 1000);
+
+describe('_repl stream bloat', () => {
+  const instances: BodDB[] = [];
+  const clients: BodClient[] = [];
+
+  afterEach(() => {
+    for (const c of clients) c.disconnect();
+    clients.length = 0;
+    for (const db of [...instances].reverse()) db.close();
+    instances.length = 0;
+  });
+
+  function getPort() { return nextPort++; }
+
+  /** Primary with auto-compact DISABLED so _repl grows unbounded */
+  function createPrimary(opts?: { maxMessageSize?: number; compact?: any; autoCompactThreshold?: number }) {
+    const port = getPort();
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: { role: 'primary', compact: opts?.compact ?? {}, autoCompactThreshold: opts?.autoCompactThreshold ?? 0 },
+    });
+    db.replication!.start();
+    db.serve({ port, maxMessageSize: opts?.maxMessageSize });
+    instances.push(db);
+    return { db, port };
+  }
+
+  function createReplica(primaryPort: number) {
+    const port = getPort();
+    const db = new BodDB({
+      path: ':memory:',
+      sweepInterval: 0,
+      replication: {
+        role: 'replica',
+        primaryUrl: `ws://localhost:${primaryPort}`,
+        replicaId: `bloat-replica-${port}`,
+      },
+    });
+    db.serve({ port });
+    instances.push(db);
+    return { db, port };
+  }
+
+  // Helper: fill _repl with N entries, each ~payloadBytes
+  function fillRepl(db: BodDB, count: number, payloadBytes: number) {
+    const padding = 'x'.repeat(Math.max(0, payloadBytes - 100));
+    for (let i = 0; i < count; i++) {
+      db.set(`vfs/files/project/src/deep/nested/path/module${i}/component.tsx`, {
+        size: 1024 + i,
+        mime: 'application/typescript',
+        mtime: Date.now(),
+        fileId: `fid_${i}_${padding}`,
+        hash: `sha256_${i.toString(16).padStart(64, '0')}`,
+        isDir: false,
+        metadata: { author: 'test', tags: ['a', 'b', 'c'], version: i },
+      });
+    }
+  }
+
+  // --- Accumulation ---
+
+  it('_repl grows unbounded without compaction', () => {
+    const { db } = createPrimary();
+    fillRepl(db, 5000, 300);
+    const repl = db.get('_repl') as Record<string, any>;
+    expect(Object.keys(repl).length).toBe(5000);
+  });
+
+  // --- Scaling: measure materialize time at increasing sizes ---
+
+  it('materialize time scales linearly (O(n) proof)', async () => {
+    const sizes = [500, 2000, 5000];
+    const timings: { count: number; ms: number; bytes: number }[] = [];
+
+    for (const count of sizes) {
+      const { db, port } = createPrimary();
+      fillRepl(db, count, 500);
+
+      const client = new BodClient({ url: `ws://localhost:${port}` });
+      clients.push(client);
+      await client.connect();
+
+      const start = Date.now();
+      const result = await client.streamMaterialize('_repl', { keepKey: 'path' });
+      const elapsed = Date.now() - start;
+
+      const json = JSON.stringify(result);
+      timings.push({ count, ms: elapsed, bytes: json.length });
+    }
+
+    for (const t of timings) {
+      console.log(`  ${t.count} entries → ${t.ms}ms, ${(t.bytes / 1024 / 1024).toFixed(2)}MB response`);
+    }
+
+    // Response size should scale roughly linearly
+    const ratio = timings[2].bytes / timings[0].bytes;
+    console.log(`  Size ratio (${sizes[2]}/${sizes[0]}): ${ratio.toFixed(1)}x (expect ~${(sizes[2] / sizes[0]).toFixed(1)}x)`);
+    expect(ratio).toBeGreaterThan(5); // 5000/500 = 10x, allow some dedup
+  });
+
+  // --- Cursor-based materialize ---
+
+  it('cursor-based materialize pages correctly over large _repl', async () => {
+    const { db, port } = createPrimary();
+    fillRepl(db, 1000, 200);
+
+    const client = new BodClient({ url: `ws://localhost:${port}` });
+    clients.push(client);
+    await client.connect();
+
+    // Page through with batchSize=200
+    const allKeys = new Set<string>();
+    let cursor: string | undefined;
+    let pages = 0;
+    do {
+      const page = await client.streamMaterialize('_repl', { keepKey: 'path', batchSize: 200, cursor });
+      if (page.data) {
+        for (const key of Object.keys(page.data)) allKeys.add(key);
+      }
+      cursor = page.nextCursor;
+      pages++;
+    } while (cursor);
+
+    console.log(`  1000 entries paged in ${pages} pages, got ${allKeys.size} unique keys`);
+    expect(allKeys.size).toBe(1000);
+    expect(pages).toBeGreaterThanOrEqual(5); // 1000/200 = 5 pages minimum
+  });
+
+  // --- Auto-compact on write threshold ---
+
+  it('auto-compact triggers after N writes', () => {
+    const { db } = createPrimary({ compact: { maxCount: 50, keepKey: 'path' }, autoCompactThreshold: 100 });
+
+    // Write 250 entries — compact triggers at 100, 200; maxCount=50 keeps only 50 each time
+    for (let i = 0; i < 250; i++) {
+      db.set(`data/item${i}`, { value: i });
+    }
+
+    const repl = db.get('_repl') as Record<string, any>;
+    const count = repl ? Object.keys(repl).length : 0;
+    console.log(`  250 writes with threshold=100, maxCount=50: ${count} _repl entries`);
+    // After compact at 200 (keeps 50), then 50 more → ~100. Way less than 250.
+    expect(count).toBeLessThan(150);
+  });
+
+  // --- Timeout reproduction: short requestTimeout simulates real-world failure ---
+
+  it('short requestTimeout causes streamMaterialize to fail on bloated _repl', async () => {
+    const { db: primary, port: primaryPort } = createPrimary();
+    fillRepl(primary, 10000, 1000); // 10k entries × 1KB = ~10MB response
+
+    const replCount = Object.keys(primary.get('_repl') as Record<string, any>).length;
+    expect(replCount).toBe(10000);
+
+    const client = new BodClient({
+      url: `ws://localhost:${primaryPort}`,
+      requestTimeout: 1,
+    });
+    clients.push(client);
+    await client.connect();
+
+    let error: Error | null = null;
+    try {
+      await client.streamMaterialize('_repl', { keepKey: 'path' });
+    } catch (e) {
+      error = e as Error;
+    }
+
+    expect(error).not.toBeNull();
+    console.log(`  REPRODUCED: streamMaterialize failed: ${error!.message}`);
+    expect(error!.message).toContain('timeout');
+  });
+
+  it('streamMaterialize response exceeds typical WS comfort zone', async () => {
+    const { db, port } = createPrimary();
+    fillRepl(db, 10000, 1000);
+
+    const client = new BodClient({ url: `ws://localhost:${port}` });
+    clients.push(client);
+    await client.connect();
+
+    const start = Date.now();
+    const result = await client.streamMaterialize('_repl', { keepKey: 'path' });
+    const elapsed = Date.now() - start;
+    const responseSize = JSON.stringify(result).length;
+
+    console.log(`  10k entries × 1KB: ${(responseSize / 1024 / 1024).toFixed(1)}MB response in ${elapsed}ms`);
+    expect(responseSize).toBeGreaterThan(5 * 1024 * 1024); // >5MB
+  });
+
+  // --- Payload size bomb ---
+
+  it('large payloads per entry amplify the problem', async () => {
+    const { db, port } = createPrimary();
+
+    const bigPadding = 'y'.repeat(2000);
+    for (let i = 0; i < 1000; i++) {
+      db.set(`data/big${i}`, {
+        content: bigPadding,
+        nested: { deep: { value: bigPadding.slice(0, 500) } },
+        index: i,
+      });
+    }
+
+    const client = new BodClient({ url: `ws://localhost:${port}` });
+    clients.push(client);
+    await client.connect();
+
+    const start = Date.now();
+    const result = await client.streamMaterialize('_repl', { keepKey: 'path' });
+    const elapsed = Date.now() - start;
+    const responseSize = JSON.stringify(result).length;
+
+    console.log(`  1000 entries × 2KB = ${(responseSize / 1024 / 1024).toFixed(2)}MB, ${elapsed}ms`);
+    expect(responseSize).toBeGreaterThan(2 * 1024 * 1024);
+  });
+
+  // --- Compaction fixes it ---
+
+  it('compaction reduces _repl and speeds up bootstrap', async () => {
+    const { db: primary, port: primaryPort } = createPrimary();
+    fillRepl(primary, 5000, 500);
+
+    const beforeCount = Object.keys(primary.get('_repl') as Record<string, any>).length;
+    expect(beforeCount).toBe(5000);
+
+    primary.stream.compact('_repl', { maxCount: 500, keepKey: 'path' });
+    const afterRepl = primary.get('_repl') as Record<string, any>;
+    const afterCount = afterRepl ? Object.keys(afterRepl).length : 0;
+    console.log(`  Compacted: ${beforeCount} → ${afterCount} entries`);
+    expect(afterCount).toBeLessThanOrEqual(500);
+
+    const { db: replica } = createReplica(primaryPort);
+    const start = Date.now();
+    await replica.replication!.start();
+    const elapsed = Date.now() - start;
+
+    console.log(`  Compacted bootstrap: ${elapsed}ms`);
+    expect(elapsed).toBeLessThan(3000);
+
+    await wait(300);
+    const val = replica.get('vfs/files/project/src/deep/nested/path/module4999/component.tsx') as any;
+    expect(val?.size).toBe(1024 + 4999);
+  });
+
+  // --- Repeated writes ---
+
+  it('repeated writes to same paths bloat _repl with duplicates', () => {
+    const { db } = createPrimary();
+
+    for (let round = 0; round < 50; round++) {
+      for (let i = 0; i < 100; i++) {
+        db.set(`config/setting${i}`, { value: round, updated: Date.now() });
+      }
+    }
+
+    const repl = db.get('_repl') as Record<string, any>;
+    const totalEntries = Object.keys(repl).length;
+    console.log(`  100 paths × 50 writes = ${totalEntries} _repl entries`);
+    expect(totalEntries).toBe(5000);
+
+    db.stream.compact('_repl', { keepKey: 'path' });
+    const after = db.get('_repl') as Record<string, any>;
+    const afterCount = after ? Object.keys(after).length : 0;
+    console.log(`  After compact: ${afterCount} entries (expect ~100 unique paths)`);
+    expect(afterCount).toBeLessThanOrEqual(150);
+  });
+
+  // --- Cursor-based bootstrap works for replica ---
+
+  it('replica bootstrap uses cursor-based pagination (no timeout on large _repl)', async () => {
+    const { db: primary, port: primaryPort } = createPrimary();
+    fillRepl(primary, 3000, 300);
+
+    const { db: replica } = createReplica(primaryPort);
+
+    const start = Date.now();
+    await replica.replication!.start();
+    const elapsed = Date.now() - start;
+
+    console.log(`  Cursor-based replica bootstrap: ${elapsed}ms for 3000 entries`);
+
+    await wait(300);
+    // Verify data arrived
+    const val = replica.get('vfs/files/project/src/deep/nested/path/module2999/component.tsx') as any;
+    expect(val?.size).toBe(1024 + 2999);
+  });
+});