@vuer-ai/vuer-rtc-server 0.1.1

package/tests/e2e/sync-perf.test.ts

@@ -0,0 +1,365 @@
+/**
+ * Sync Performance Tests
+ *
+ * Stress tests for bloom filter sync reconciliation under heavy load.
+ * Measures: digest build time, convergence time with packet loss,
+ * compaction impact, and scaling with multiple clients.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from '@jest/globals';
+import { buildSyncDigest } from '@vuer-ai/vuer-rtc';
+import type { GraphStore } from '@vuer-ai/vuer-rtc';
+import { createTestServer, type TestServer } from './helpers/createTestServer.js';
+import { waitFor, waitForConvergence, assertGraphsEqual } from './helpers/assertions.js';
+
+function seedGraph(store: GraphStore): void {
+  store.edit({
+    otype: 'node.insert',
+    key: '',
+    path: 'children',
+    value: { key: 'scene', tag: 'Scene', name: 'Scene' },
+  });
+  store.edit({
+    otype: 'node.insert',
+    key: 'scene',
+    path: 'children',
+    value: {
+      key: 'cube-1',
+      tag: 'Mesh',
+      name: 'Cube',
+      position: [0, 0, 0],
+      opacity: 1,
+    },
+  });
+  store.commit('seed scene');
+}
+
+/** Generate N position edits on cube-1, one commit each. */
+function generateEdits(store: GraphStore, n: number): void {
+  for (let i = 0; i < n; i++) {
+    store.edit({
+      otype: 'vector3.set',
+      key: 'cube-1',
+      path: 'position',
+      value: [i, i * 2, i * 3],
+    });
+    store.commit(`edit ${i}`);
+  }
+}
+
+function hrMs(start: [number, number]): number {
+  const diff = process.hrtime(start);
+  return diff[0] * 1000 + diff[1] / 1e6;
+}
+
+// ── Tests ─────────────────────────────────────────────────────
+
+describe('sync-perf', () => {
+  let ts: TestServer;
+
+  beforeEach(() => {
+    ts = createTestServer();
+  });
+
+  afterEach(() => {
+    ts.close();
+  });
+
+  // ── 1. Digest build time at scale ───────────────────────────
+
+  it('digest build time scales linearly (1K, 5K, 10K messages)', async () => {
+    const clientA = ts.connectClient('room-1', 'session-a');
+    seedGraph(clientA.store);
+
+    const sizes = [1000, 5000, 10000];
+    const results: Array<{ n: number; buildMs: number; filterBytes: number }> = [];
+
+    for (const n of sizes) {
+      const currentJournal = clientA.store.getState().journal.length;
+      const needed = n - currentJournal + 1;
+      if (needed > 0) generateEdits(clientA.store, needed);
+
+      // Measure digest build time (average of 10 runs)
+      const runs = 10;
+      const t0 = process.hrtime();
+      for (let i = 0; i < runs; i++) {
+        buildSyncDigest(clientA.store.getState());
+      }
+      const elapsed = hrMs(t0);
+      const avgMs = elapsed / runs;
+
+      const digest = buildSyncDigest(clientA.store.getState());
+      const filterBytes = digest.filter.length;
+
+      results.push({ n: digest.count, buildMs: avgMs, filterBytes });
+    }
+
+    console.log('\n  Digest build time:');
+    console.log('  ┌───────────┬───────────────┬──────────────┬────────────────┐');
+    console.log('  │  Messages │  Build (ms)   │ Filter (KB)  │  msgs/ms       │');
+    console.log('  ├───────────┼───────────────┼──────────────┼────────────────┤');
+    for (const r of results) {
+      const kb = (r.filterBytes / 1024).toFixed(1);
+      const throughput = (r.n / r.buildMs).toFixed(0);
+      console.log(
+        `  │  ${String(r.n).padStart(7)} │  ${r.buildMs.toFixed(3).padStart(11)} │  ${kb.padStart(10)} │  ${throughput.padStart(12)} │`,
+      );
+    }
+    console.log('  └───────────┴───────────────┴──────────────┴────────────────┘');
+
+    // Sanity: 10K messages should build in < 50ms
+    const last = results[results.length - 1];
+    expect(last.buildMs).toBeLessThan(50);
+    expect(last.filterBytes).toBeLessThan(last.n * 3);
+  });
+
+  // ── 2. Compaction shrinks digest dramatically ───────────────
+
+  it('compaction reduces digest size after ack', async () => {
+    const clientA = ts.connectClient('room-1', 'session-a');
+    const clientB = ts.connectClient('room-1', 'session-b');
+    seedGraph(clientA.store);
+    await waitForConvergence([clientA.store, clientB.store], 3000);
+
+    // B makes 2000 edits
+    generateEdits(clientB.store, 2000);
+    await waitForConvergence([clientA.store, clientB.store], 5000);
+
+    const beforeDigest = buildSyncDigest(clientA.store.getState());
+    const beforeCount = beforeDigest.count;
+    const beforeFilterKB = beforeDigest.filter.length / 1024;
+
+    clientA.store.compact();
+
+    const afterState = clientA.store.getState();
+    const afterDigest = buildSyncDigest(afterState);
+    const afterCount = afterDigest.count;
+    const afterFilterKB = afterDigest.filter.length / 1024;
+
+    console.log('\n  Compaction impact (client A, 2000 remote edits):');
+    console.log(`    Before: journal=${beforeCount}, filter=${beforeFilterKB.toFixed(1)} KB`);
+    console.log(`    After:  journal=${afterCount}, filter=${afterFilterKB.toFixed(1)} KB`);
+    console.log(`    Reduction: ${((1 - afterCount / beforeCount) * 100).toFixed(0)}%`);
+
+    expect(afterCount).toBeLessThan(beforeCount);
+    expect(Object.keys(afterState.snapshot.vectorClock).length).toBeGreaterThan(0);
+  });
+
+  // ── 3. Heavy load convergence with 50% drops + sync ────────
+
+  it('converges 1000 edits with 50% drop + sync recovery', async () => {
+    const clientA = ts.connectClient('room-1', 'session-a');
+    const clientB = ts.connectClient('room-1', 'session-b', { dropRate: 0.5 });
+    seedGraph(clientA.store);
+
+    for (const entry of clientA.store.getState().journal) {
+      clientB.store.receive(entry.msg);
+    }
+
+    const editStart = process.hrtime();
+    generateEdits(clientA.store, 1000);
+    const editMs = hrMs(editStart);
+
+    await new Promise((r) => setTimeout(r, 500));
+
+    const bJournal = clientB.store.getState().journal.length;
+    const aJournal = clientA.store.getState().journal.length;
+    const missing = aJournal - bJournal;
+
+    clientB.disableDrop();
+    const syncStart = process.hrtime();
+
+    for (let round = 0; round < 5; round++) {
+      clientB.retryUnacked();
+      clientB.sendSync();
+      await new Promise((r) => setTimeout(r, 200));
+    }
+
+    await waitForConvergence([clientA.store, clientB.store], 10000);
+    const syncMs = hrMs(syncStart);
+
+    assertGraphsEqual(
+      clientA.store.getState().graph,
+      clientB.store.getState().graph,
+    );
+
+    console.log('\n  Heavy load convergence (1000 edits, 50% drop):');
+    console.log(`    Edit generation: ${editMs.toFixed(1)} ms`);
+    console.log(`    Messages dropped: ~${missing} of ${aJournal}`);
+    console.log(`    Sync recovery: ${syncMs.toFixed(1)} ms`);
+    console.log(`    Final position: ${JSON.stringify(clientB.store.getState().graph.nodes['cube-1'].position)}`);
+
+    expect(clientB.store.getState().graph.nodes['cube-1'].position).toEqual([999, 1998, 2997]);
+  });
+
+  // ── 4. Multi-client scaling: 5 clients, 100 edits each ─────
+
+  it('5 clients × 100 edits converge via sync', async () => {
+    const N_CLIENTS = 5;
+    const EDITS_PER_CLIENT = 100;
+    const clients = [];
+
+    for (let i = 0; i < N_CLIENTS; i++) {
+      const c = ts.connectClient('room-1', `session-${i}`, { dropRate: 0.3 });
+      clients.push(c);
+    }
+
+    // First client seeds reliably
+    clients[0].disableDrop();
+    seedGraph(clients[0].store);
+    await new Promise((r) => setTimeout(r, 100));
+
+    // Replay seed to all clients
+    const seed = clients[0].store.getState().journal;
+    for (let i = 1; i < N_CLIENTS; i++) {
+      for (const entry of seed) {
+        clients[i].store.receive(entry.msg);
+      }
+    }
+
+    // All clients fire edits
+    const editStart = process.hrtime();
+    for (let i = 0; i < N_CLIENTS; i++) {
+      for (let j = 0; j < EDITS_PER_CLIENT; j++) {
+        clients[i].store.edit({
+          otype: 'number.set',
+          key: 'cube-1',
+          path: `prop_${i}_${j}`,
+          value: i * 1000 + j,
+        });
+        clients[i].store.commit(`client-${i} edit ${j}`);
+      }
+    }
+    const editMs = hrMs(editStart);
+
+    // Let some broadcasts arrive
+    await new Promise((r) => setTimeout(r, 300));
+
+    // Disable drops and sync repeatedly
+    const syncStart = process.hrtime();
+    for (const c of clients) c.disableDrop();
+
+    for (let round = 0; round < 15; round++) {
+      for (const c of clients) {
+        c.retryUnacked();
+        c.sendSync();
+      }
+      await new Promise((r) => setTimeout(r, 200));
+    }
+
+    await waitForConvergence(
+      clients.map((c) => c.store),
+      20000,
+    );
+    const syncMs = hrMs(syncStart);
+
+    for (let i = 1; i < N_CLIENTS; i++) {
+      assertGraphsEqual(
+        clients[0].store.getState().graph,
+        clients[i].store.getState().graph,
+      );
+    }
+
+    const totalEdits = N_CLIENTS * EDITS_PER_CLIENT;
+    const journalSizes = clients.map((c) => c.store.getState().journal.length);
+
+    console.log(`\n  Multi-client convergence (${N_CLIENTS} clients × ${EDITS_PER_CLIENT} edits):`);
+    console.log(`    Total edits: ${totalEdits}`);
+    console.log(`    Edit generation: ${editMs.toFixed(1)} ms (${(totalEdits / editMs * 1000).toFixed(0)} edits/sec)`);
+    console.log(`    Sync recovery: ${syncMs.toFixed(1)} ms`);
+    console.log(`    Journal sizes: [${journalSizes.join(', ')}]`);
+
+    const node = clients[0].store.getState().graph.nodes['cube-1'];
+    const propCount = Object.keys(node).filter((k) => k.startsWith('prop_')).length;
+    expect(propCount).toBe(totalEdits);
+  }, 60_000);
+
+  // ── 5. Long session: compaction keeps journal bounded ───────
+
+  it('long session: 2000 edits with periodic compaction', async () => {
+    // Two clients — A edits, B receives. A compacts periodically.
+    // Key insight: compaction works on ACKED entries, so we need to
+    // let acks arrive (async) before calling compact.
+    const clientA = ts.connectClient('room-1', 'session-a');
+    const clientB = ts.connectClient('room-1', 'session-b', { dropRate: 0.2 });
+    seedGraph(clientA.store);
+
+    for (const entry of clientA.store.getState().journal) {
+      clientB.store.receive(entry.msg);
+    }
+
+    const TOTAL = 2000;
+    const BATCH_SIZE = 500;
+    const compactionStats: Array<{ at: number; journalBefore: number; journalAfter: number }> = [];
+
+    const start = process.hrtime();
+
+    for (let batch = 0; batch < TOTAL / BATCH_SIZE; batch++) {
+      // Fire a batch of edits
+      for (let i = 0; i < BATCH_SIZE; i++) {
+        const idx = batch * BATCH_SIZE + i;
+        clientA.store.edit({
+          otype: 'number.set',
+          key: 'cube-1',
+          path: `val_${idx % 100}`,
+          value: idx,
+        });
+        clientA.store.commit(`edit ${idx}`);
+      }
+
+      // Wait for acks to arrive (they come back via setTimeout(0))
+      await waitFor(
+        () => clientA.store.getState().journal.every((e) => e.ack),
+        3000,
+      );
+
+      // Now compact — all entries are acked
+      const jBefore = clientA.store.getState().journal.length;
+      clientA.store.compact();
+      const jAfter = clientA.store.getState().journal.length;
+      compactionStats.push({
+        at: (batch + 1) * BATCH_SIZE,
+        journalBefore: jBefore,
+        journalAfter: jAfter,
+      });
+    }
+
+    const editMs = hrMs(start);
+
+    // Final sync for B
+    clientB.disableDrop();
+    for (let round = 0; round < 10; round++) {
+      clientB.retryUnacked();
+      clientB.sendSync();
+      await new Promise((r) => setTimeout(r, 200));
+    }
+
+    await waitForConvergence([clientA.store, clientB.store], 15000);
+    const totalMs = hrMs(start);
+
+    assertGraphsEqual(
+      clientA.store.getState().graph,
+      clientB.store.getState().graph,
+    );
+
+    const digestA = buildSyncDigest(clientA.store.getState());
+    const digestB = buildSyncDigest(clientB.store.getState());
+    const snapshotClock = clientA.store.getState().snapshot.vectorClock;
+
+    console.log(`\n  Long session (${TOTAL} edits, compact every ${BATCH_SIZE}):`);
+    console.log(`    Edit + compact: ${editMs.toFixed(1)} ms (${(TOTAL / editMs * 1000).toFixed(0)} edits/sec)`);
+    console.log(`    Total time: ${totalMs.toFixed(1)} ms`);
+    console.log(`    Final journal: A=${digestA.count}, B=${digestB.count}`);
+    console.log(`    Final filter: A=${(digestA.filter.length / 1024).toFixed(1)} KB, B=${(digestB.filter.length / 1024).toFixed(1)} KB`);
+    console.log(`    Snapshot vector clock: ${JSON.stringify(snapshotClock)}`);
+    console.log('    Compaction rounds:');
+    for (const s of compactionStats) {
+      console.log(`      @${s.at}: journal ${s.journalBefore} → ${s.journalAfter}`);
+    }
+
+    // After periodic compaction, A's journal should be near-empty
+    expect(digestA.count).toBeLessThan(BATCH_SIZE);
+    // Vector clock should cover A's compacted entries
+    expect(snapshotClock['session-a']).toBeGreaterThan(0);
+  }, 60_000);
+});

package/tests/e2e/sync-reconciliation.test.ts

@@ -0,0 +1,237 @@
+/**
+ * E2E: Bloom filter-based sync reconciliation.
+ *
+ * Verifies that clients can recover from dropped server→client broadcasts
+ * by sending a bloom filter digest and receiving missing messages.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from '@jest/globals';
+import { createTestServer, type TestServer } from './helpers/createTestServer.js';
+import { waitForConvergence, assertGraphsEqual } from './helpers/assertions.js';
+import type { GraphStore } from '@vuer-ai/vuer-rtc';
+
+// ── Helpers ───────────────────────────────────────────────────
+
+function seedGraph(store: GraphStore): void {
+  store.edit({
+    otype: 'node.insert',
+    key: '',
+    path: 'children',
+    value: { key: 'scene', tag: 'Scene', name: 'Scene' },
+  });
+  store.edit({
+    otype: 'node.insert',
+    key: 'scene',
+    path: 'children',
+    value: {
+      key: 'cube-1',
+      tag: 'Mesh',
+      name: 'Cube',
+      position: [0, 0, 0],
+      opacity: 1,
+    },
+  });
+  store.commit('seed scene');
+}
+
+// ── Tests ─────────────────────────────────────────────────────
+
+describe('sync-reconciliation', () => {
+  let ts: TestServer;
+
+  beforeEach(() => {
+    ts = createTestServer();
+  });
+
+  afterEach(() => {
+    ts.close();
+  });
+
+  // ── 1. Full recovery via sync after 100% drop ─────────────
+
+  it('recovers all messages via sync after 100% broadcast drop', async () => {
+    // A sends reliably to the server, but B drops all incoming (100% drop)
+    const clientA = ts.connectClient('room-1', 'session-a');
+    const clientB = ts.connectClient('room-1', 'session-b', { dropRate: 1.0 });
+
+    seedGraph(clientA.store);
+
+    // Wait for A's messages to be processed by the server
+    await new Promise((r) => setTimeout(r, 100));
+
+    // B should have nothing since all broadcasts were dropped
+    expect(Object.keys(clientB.store.getState().graph.nodes).length).toBe(0);
+
+    // Disable drop and send sync — B tells server what it has (nothing),
+    // server retransmits everything
+    clientB.disableDrop();
+    clientB.sendSync();
+
+    await waitForConvergence([clientA.store, clientB.store], 3000);
+
+    const graphA = clientA.store.getState().graph;
+    const graphB = clientB.store.getState().graph;
+    assertGraphsEqual(graphA, graphB);
+    expect(Object.keys(graphB.nodes).length).toBeGreaterThan(0);
+  });
+
+  // ── 2. Partial recovery via sync after 50% drop ───────────
+
+  it('recovers from partial drop via sync', async () => {
+    // Seed reliably first so both clients have the base graph
+    const clientA = ts.connectClient('room-1', 'session-a');
+    const clientB = ts.connectClient('room-1', 'session-b');
+
+    seedGraph(clientA.store);
+    await waitForConvergence([clientA.store, clientB.store], 3000);
+
+    // Connect a third client with 50% drops — it starts behind
+    const clientC = ts.connectClient('room-1', 'session-c', { dropRate: 0.5 });
+
+    // Replay journal so C has the base graph
+    for (const entry of clientA.store.getState().journal) {
+      clientC.store.receive(entry.msg);
+    }
+
+    // A makes property edits (commutative, safe for out-of-order replay)
+    clientA.store.edit({
+      otype: 'vector3.set',
+      key: 'cube-1',
+      path: 'position',
+      value: [10, 20, 30],
+    });
+    clientA.store.commit('move cube');
+
+    clientA.store.edit({
+      otype: 'number.set',
+      key: 'cube-1',
+      path: 'opacity',
+      value: 0.5,
+    });
+    clientA.store.commit('set opacity');
+
+    // Wait for processing — C may have received some broadcasts, missed others
+    await new Promise((r) => setTimeout(r, 200));
+
+    // Disable drop and sync to recover missing messages
+    clientC.disableDrop();
+    clientC.sendSync();
+
+    await waitForConvergence([clientA.store, clientC.store], 3000);
+
+    const graphA = clientA.store.getState().graph;
+    const graphC = clientC.store.getState().graph;
+    assertGraphsEqual(graphA, graphC);
+  });
+
+  // ── 3. Sync is idempotent ─────────────────────────────────
+
+  it('sync is idempotent when nothing is missing', async () => {
+    const clientA = ts.connectClient('room-1', 'session-a');
+    const clientB = ts.connectClient('room-1', 'session-b');
+
+    seedGraph(clientA.store);
+    await waitForConvergence([clientA.store, clientB.store], 3000);
+
+    // Both converged, now send sync — should be a no-op
+    const graphBefore = JSON.stringify(clientB.store.getState().graph);
+    clientB.sendSync();
+
+    await new Promise((r) => setTimeout(r, 200));
+
+    const graphAfter = JSON.stringify(clientB.store.getState().graph);
+    expect(graphAfter).toBe(graphBefore);
+
+    assertGraphsEqual(clientA.store.getState().graph, clientB.store.getState().graph);
+  });
+
+  // ── 4. Bidirectional sync recovers both sides ─────────────
+
+  it('bidirectional sync recovers both clients', async () => {
+    // Both clients drop each other's broadcasts
+    const clientA = ts.connectClient('room-1', 'session-a', { dropRate: 1.0 });
+    const clientB = ts.connectClient('room-1', 'session-b', { dropRate: 1.0 });
+
+    // Seed from A — server gets it, but broadcast to B is dropped
+    seedGraph(clientA.store);
+
+    // B makes its own edit — server gets it, but broadcast to A is dropped
+    await new Promise((r) => setTimeout(r, 100));
+
+    // Disable drop for sync
+    clientA.disableDrop();
+    clientB.disableDrop();
+
+    // B also needs to retry its unacked messages (since A→server messages were also
+    // dropped when clientA had 100% dropRate)
+    clientA.retryUnacked();
+    clientB.retryUnacked();
+
+    await new Promise((r) => setTimeout(r, 200));
+
+    // Now both send sync
+    clientA.sendSync();
+    clientB.sendSync();
+
+    await waitForConvergence([clientA.store, clientB.store], 3000);
+
+    const graphA = clientA.store.getState().graph;
+    const graphB = clientB.store.getState().graph;
+    assertGraphsEqual(graphA, graphB);
+  });
+
+  // ── 5. Sync + retry together ──────────────────────────────
+
+  it('sync + retry together achieve full convergence', async () => {
+    const clientA = ts.connectClient('room-1', 'session-a', { dropRate: 0.5 });
+    const clientB = ts.connectClient('room-1', 'session-b', { dropRate: 0.5 });
+
+    // Seed reliably
+    clientA.disableDrop();
+    clientB.disableDrop();
+    seedGraph(clientA.store);
+    await waitForConvergence([clientA.store, clientB.store], 3000);
+
+    // Re-enable drops for edits
+    clientA.setLatency(0);
+    clientB.setLatency(0);
+
+    // Both make edits (some may be dropped)
+    clientA.store.edit({
+      otype: 'vector3.set',
+      key: 'cube-1',
+      path: 'position',
+      value: [99, 88, 77],
+    });
+    clientA.store.commit('move');
+
+    clientB.store.edit({
+      otype: 'number.set',
+      key: 'cube-1',
+      path: 'opacity',
+      value: 0.3,
+    });
+    clientB.store.commit('opacity');
+
+    await new Promise((r) => setTimeout(r, 200));
+
+    // Full recovery: disable drop, retry unacked, sync
+    clientA.disableDrop();
+    clientB.disableDrop();
+    clientA.retryUnacked();
+    clientB.retryUnacked();
+
+    await new Promise((r) => setTimeout(r, 200));
+
+    clientA.sendSync();
+    clientB.sendSync();
+
+    await waitForConvergence([clientA.store, clientB.store], 3000);
+
+    const graphA = clientA.store.getState().graph;
+    const graphB = clientB.store.getState().graph;
+    assertGraphsEqual(graphA, graphB);
+    expect(graphB.nodes['cube-1'].position).toEqual([99, 88, 77]);
+    expect(graphB.nodes['cube-1'].opacity).toBe(0.3);
+  });
+});