@vuer-ai/vuer-rtc-server 0.2.0 → 0.2.2

package/tests/journal/text-coalescing.test.ts

@@ -0,0 +1,210 @@
+/**
+ * Text CRDT Coalescing Test
+ *
+ * Verifies that compaction merges single-char TextRope items into
+ * multi-char spans, preventing B-tree depth explosion.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from '@jest/globals';
+import type { CRDTMessage, Operation, TextRope } from '@vuer-ai/vuer-rtc';
+import { JournalService } from '../../src/journal/JournalService.js';
+import { PrismaClient } from '@prisma/client';
+import { getItems } from '@vuer-ai/vuer-rtc';
+
+describe('TextRope Coalescing in Compaction', () => {
+  let service: JournalService;
+  let prisma: PrismaClient;
+  let documentId: string;
+
+  beforeEach(async () => {
+    prisma = new PrismaClient();
+    await prisma.$connect();
+
+    // Clean up test data
+    await prisma.journalBatch.deleteMany({});
+    await prisma.document.deleteMany({});
+
+    service = new JournalService(prisma);
+    documentId = await service.createDocument('test-doc', 'test-user');
+  });
+
+  afterEach(async () => {
+    await prisma.$disconnect();
+  });
+
+  it('should coalesce single-char text inserts into multi-char spans after compaction', async () => {
+    // Create text node
+    const initMsg: CRDTMessage = {
+      id: 'msg-init',
+      sessionId: 'alice',
+      clock: { alice: 1 },
+      lamportTime: 1,
+      timestamp: Date.now() / 1000,
+      ops: [
+        {
+          key: 'default-scene',
+          otype: 'node.insert',
+          path: 'children',
+          value: {
+            key: 'text-doc',
+            tag: 'Text',
+            name: 'Text Doc',
+          },
+        } as Operation,
+        {
+          key: 'text-doc',
+          otype: 'text.init',
+          path: 'content',
+          value: '',
+        } as Operation,
+      ],
+    };
+
+    await service.processMessage(documentId, initMsg);
+
+    // Type "hello" character by character (simulating rapid typing)
+    const chars = ['h', 'e', 'l', 'l', 'o'];
+    for (let i = 0; i < chars.length; i++) {
+      const msg: CRDTMessage = {
+        id: `msg-char-${i}`,
+        sessionId: 'alice',
+        clock: { alice: 2 + i },
+        lamportTime: 2 + i,
+        timestamp: Date.now() / 1000,
+        ops: [
+          {
+            key: 'text-doc',
+            otype: 'text.insert',
+            path: 'content',
+            position: i,
+            value: chars[i],
+          } as Operation,
+        ],
+      };
+      await service.processMessage(documentId, msg);
+    }
+
+    // Before compaction: get state and check TextRope structure
+    const stateBefore = await service.getStateForClient(documentId);
+    expect(stateBefore).not.toBeNull();
+
+    const graphBefore = service.computeGraph((stateBefore as any).snapshot);
+    const nodeBefore = graphBefore.nodes['text-doc'];
+    const ropeBefore = (nodeBefore as any)['_textRope.content'] as TextRope;
+
+    expect(ropeBefore).toBeDefined();
+    const itemsBefore = getItems(ropeBefore);
+    console.log(`Before compaction: ${itemsBefore.length} items`);
+
+    // Each character should be a separate item
+    expect(itemsBefore.length).toBeGreaterThanOrEqual(5);
+
+    // Compact
+    await service.compact(documentId);
+
+    // After compaction: check TextRope structure
+    const stateAfter = await service.getStateForClient(documentId);
+    expect(stateAfter).not.toBeNull();
+
+    const graphAfter = service.computeGraph((stateAfter as any).snapshot);
+    const nodeAfter = graphAfter.nodes['text-doc'];
+    const ropeAfter = (nodeAfter as any)['_textRope.content'] as TextRope;
+
+    expect(ropeAfter).toBeDefined();
+    const itemsAfter = getItems(ropeAfter);
+    console.log(`After compaction: ${itemsAfter.length} items`);
+
+    // Should be coalesced into 1 or fewer items
+    expect(itemsAfter.length).toBeLessThan(itemsBefore.length);
+    expect(itemsAfter.length).toBeLessThanOrEqual(1);
+
+    // Content should still be correct
+    const textAfter = nodeAfter.content;
+    expect(textAfter).toBe('hello');
+  });
+
+  it('should handle large text documents efficiently', async () => {
+    // Create text node
+    const initMsg: CRDTMessage = {
+      id: 'msg-init',
+      sessionId: 'alice',
+      clock: { alice: 1 },
+      lamportTime: 1,
+      timestamp: Date.now() / 1000,
+      ops: [
+        {
+          key: 'default-scene',
+          otype: 'node.insert',
+          path: 'children',
+          value: {
+            key: 'text-doc',
+            tag: 'Text',
+            name: 'Text Doc',
+          },
+        } as Operation,
+        {
+          key: 'text-doc',
+          otype: 'text.init',
+          path: 'content',
+          value: '',
+        } as Operation,
+      ],
+    };
+
+    await service.processMessage(documentId, initMsg);
+
+    // Type 100 characters
+    const text = 'a'.repeat(100);
+    for (let i = 0; i < 100; i++) {
+      const msg: CRDTMessage = {
+        id: `msg-char-${i}`,
+        sessionId: 'alice',
+        clock: { alice: 2 + i },
+        lamportTime: 2 + i,
+        timestamp: Date.now() / 1000,
+        ops: [
+          {
+            key: 'text-doc',
+            otype: 'text.insert',
+            path: 'content',
+            position: i,
+            value: text[i],
+          } as Operation,
+        ],
+      };
+      await service.processMessage(documentId, msg);
+    }
+
+    // Before compaction
+    const stateBefore = await service.getStateForClient(documentId);
+    const graphBefore = service.computeGraph((stateBefore as any).snapshot);
+    const nodeBefore = graphBefore.nodes['text-doc'];
+    const ropeBefore = (nodeBefore as any)['_textRope.content'] as TextRope;
+    const itemsBefore = getItems(ropeBefore);
+
+    console.log(`Before compaction: ${itemsBefore.length} items for 100 chars`);
+    expect(itemsBefore.length).toBeGreaterThanOrEqual(100);
+
+    // Compact
+    await service.compact(documentId);
+
+    // After compaction
+    const stateAfter = await service.getStateForClient(documentId);
+    const graphAfter = service.computeGraph((stateAfter as any).snapshot);
+    const nodeAfter = graphAfter.nodes['text-doc'];
+    const ropeAfter = (nodeAfter as any)['_textRope.content'] as TextRope;
+    const itemsAfter = getItems(ropeAfter);
+
+    const reductionPercent = Math.round((1 - itemsAfter.length / itemsBefore.length) * 100);
+    console.log(`After compaction: ${itemsAfter.length} items for 100 chars`);
+    console.log(`Reduction: ${itemsBefore.length}x → ${itemsAfter.length}x (${reductionPercent}% smaller)`);
+
+    // Should be significantly reduced
+    expect(itemsAfter.length).toBeLessThan(itemsBefore.length * 0.2); // At least 80% reduction
+    expect(itemsAfter.length).toBeLessThanOrEqual(1);
+
+    // Content should still be correct
+    const textAfter = nodeAfter.content;
+    expect(textAfter).toBe(text);
+  });
+});

package/tests/unit/s3-compression.test.ts

@@ -0,0 +1,257 @@
+/**
+ * S3 Cold Storage Compression Tests
+ *
+ * Tests:
+ * - Unit test: compress snapshot, verify decompression matches original
+ * - Benchmark test: compression ratio on sample data
+ */
+
+import { describe, it, expect } from '@jest/globals';
+import {
+  compressSnapshot,
+  decompressSnapshot,
+  getCompressionStats,
+} from '../../src/compression/CompressionUtils';
+import type { Snapshot } from '@vuer-ai/vuer-rtc';
+
+/**
+ * Helper to format bytes as human-readable string
+ */
+function formatBytes(bytes: number): string {
+  const units = ['B', 'KB', 'MB', 'GB'];
+  let size = bytes;
+  let unitIndex = 0;
+
+  while (size >= 1024 && unitIndex < units.length - 1) {
+    size /= 1024;
+    unitIndex += 1;
+  }
+
+  return `${size.toFixed(2)} ${units[unitIndex]}`;
+}
+
+/**
+ * Calculate compression ratio
+ */
+function calculateCompressionRatio(
+  originalSize: number,
+  compressedSize: number
+): { ratio: number; savings: string } {
+  const ratio = compressedSize / originalSize;
+  const savingsPercent = ((1 - ratio) * 100).toFixed(1);
+  return {
+    ratio,
+    savings: `${savingsPercent}%`,
+  };
+}
+
+/**
+ * Create a test snapshot with variable complexity
+ */
+function createTestSnapshot(nodeCount: number = 100, depth: number = 3): Snapshot {
+  const nodes: Record<string, any> = {};
+
+  for (let i = 0; i < nodeCount; i++) {
+    nodes[`node_${i}`] = {
+      id: `node_${i}`,
+      name: `Test Node ${i}`,
+      type: 'mesh',
+      position: [Math.random() * 100, Math.random() * 100, Math.random() * 100],
+      rotation: [0, 0, 0, 1],
+      scale: [1, 1, 1],
+      visible: true,
+      metadata: {
+        createdAt: Date.now(),
+        tags: ['test', 'benchmark'],
+        description: `This is a test node with index ${i}`.repeat(5), // Repeat for more data
+      },
+      children: Array.from({ length: depth }, (_, j) => `node_${i}_child_${j}`),
+      properties: {
+        color: [Math.random(), Math.random(), Math.random()],
+        emissive: [0, 0, 0],
+        roughness: 0.5,
+        metalness: 0.5,
+      },
+    };
+  }
+
+  return {
+    graph: {
+      rootKey: 'scene_root',
+      nodes,
+    },
+    vectorClock: {
+      session1: 100,
+      session2: 50,
+      session3: 75,
+    },
+    lamportTime: 150,
+    journalIndex: 200,
+  };
+}
+
+describe('S3 Cold Storage - Compression', () => {
+  describe('Unit Tests', () => {
+    it('should compress and decompress a snapshot correctly', () => {
+      // Create test snapshot
+      const originalSnapshot = createTestSnapshot(50, 2);
+
+      // Compress
+      const compressed = compressSnapshot(originalSnapshot);
+
+      // Verify compression produced a buffer
+      expect(compressed.compressed).toBeInstanceOf(Buffer);
+      expect(compressed.originalSize).toBeGreaterThan(0);
+      expect(compressed.compressedSize).toBeGreaterThan(0);
+      expect(compressed.ratio).toBeGreaterThan(0); // Ratio is percentage
+
+      // Decompress
+      const decompressed = decompressSnapshot(compressed.compressed);
+
+      // Verify decompressed matches original
+      const decompressedSnapshot = JSON.parse(decompressed.decompressed.toString('utf-8'));
+      expect(decompressedSnapshot).toEqual(originalSnapshot);
+      expect(decompressed.verified).toBe(true);
+    });
+
+    it('should handle large snapshots', () => {
+      // Create a larger snapshot
+      const largeSnapshot = createTestSnapshot(1000, 5);
+
+      // Compress
+      const compressed = compressSnapshot(largeSnapshot);
+
+      // Should achieve good compression (expect ~75-80% savings)
+      const savingsPercent = 100 - compressed.ratio;
+      expect(savingsPercent).toBeGreaterThan(50); // At least 50% savings
+
+      // Decompress and verify
+      const decompressed = decompressSnapshot(compressed.compressed);
+      const decompressedSnapshot = JSON.parse(decompressed.decompressed.toString('utf-8'));
+      expect(decompressedSnapshot).toEqual(largeSnapshot);
+    });
+
+    it('should handle empty snapshots', () => {
+      const emptySnapshot: Snapshot = {
+        graph: { rootKey: '', nodes: {} },
+        vectorClock: {},
+        lamportTime: 0,
+        journalIndex: 0,
+      };
+
+      // Compress
+      const compressed = compressSnapshot(emptySnapshot);
+      expect(compressed.compressed).toBeInstanceOf(Buffer);
+
+      // Decompress and verify
+      const decompressed = decompressSnapshot(compressed.compressed);
+      const decompressedSnapshot = JSON.parse(decompressed.decompressed.toString('utf-8'));
+      expect(decompressedSnapshot).toEqual(emptySnapshot);
+    });
+
+    it('should provide accurate compression metrics', () => {
+      const snapshot = createTestSnapshot(100, 3);
+      const compressed = compressSnapshot(snapshot);
+
+      // Verify metrics
+      expect(compressed.originalSize).toBeGreaterThan(0);
+      expect(compressed.compressedSize).toBeGreaterThan(0);
+      expect(compressed.ratio).toBe((compressed.compressedSize / compressed.originalSize) * 100);
+    });
+
+    it('should reject corrupted compressed data', () => {
+      const corruptedData = Buffer.from('not valid gzip data');
+
+      // Should throw error on decompression
+      expect(() => decompressSnapshot(corruptedData)).toThrow();
+    });
+  });
+
+  describe('Benchmark Tests', () => {
+    it('should compress typical snapshots with 75-80% savings', () => {
+      // Create a realistic snapshot
+      const snapshot = createTestSnapshot(500, 4);
+
+      const compressed = compressSnapshot(snapshot);
+      const savingsPercent = 100 - compressed.ratio;
+
+      console.log('\nCompression Benchmark Results:');
+      console.log(`  Original size:    ${formatBytes(compressed.originalSize)}`);
+      console.log(`  Compressed size:  ${formatBytes(compressed.compressedSize)}`);
+      console.log(`  Savings:          ${savingsPercent.toFixed(1)}%`);
+      console.log(`  Compression ratio: ${(compressed.ratio / 100).toFixed(4)}`);
+
+      // Verify benchmark expectations
+      expect(savingsPercent).toBeGreaterThan(70);
+    });
+
+    it('should handle compression ratio scaling with snapshot size', () => {
+      const results: {
+        size: string;
+        original: number;
+        compressed: number;
+        ratio: number;
+        savings: string;
+      }[] = [];
+
+      // Test different snapshot sizes
+      for (const nodeCount of [100, 500, 1000]) {
+        const snapshot = createTestSnapshot(nodeCount, 4);
+        const compressed = compressSnapshot(snapshot);
+        const { ratio, savings } = calculateCompressionRatio(
+          compressed.originalSize,
+          compressed.compressedSize
+        );
+
+        results.push({
+          size: `${nodeCount} nodes`,
+          original: compressed.originalSize,
+          compressed: compressed.compressedSize,
+          ratio,
+          savings,
+        });
+      }
+
+      console.log('\nCompression Ratio vs Snapshot Size:');
+      for (const r of results) {
+        console.log(
+          `  ${r.size}: ${formatBytes(r.original)} to ${formatBytes(r.compressed)} (${r.savings})`
+        );
+      }
+
+      // All should have good compression
+      for (const r of results) {
+        expect(parseFloat(r.savings)).toBeGreaterThan(60);
+      }
+    });
+  });
+
+  describe('Utility Functions', () => {
+    it('should calculate compression ratio correctly', () => {
+      const original = 1000;
+      const compressed = 250;
+
+      const result = calculateCompressionRatio(original, compressed);
+
+      expect(result.ratio).toBe(0.25);
+      expect(result.savings).toBe('75.0%');
+    });
+
+    it('should format bytes correctly', () => {
+      expect(formatBytes(0)).toBe('0.00 B');
+      expect(formatBytes(1024)).toBe('1.00 KB');
+      expect(formatBytes(1024 * 1024)).toBe('1.00 MB');
+      expect(formatBytes(1024 * 1024 * 1024)).toBe('1.00 GB');
+    });
+
+    it('should get compression stats correctly', () => {
+      const snapshot = createTestSnapshot(100, 3);
+      const stats = getCompressionStats(snapshot);
+
+      expect(stats.originalSize).toBeGreaterThan(0);
+      expect(stats.compressedSize).toBeGreaterThan(0);
+      expect(stats.ratio).toBeGreaterThan(0);
+      expect(stats.savedBytes).toBeGreaterThan(0);
+    });
+  });
+});

package/PHASE1_SUMMARY.md

@@ -1,94 +0,0 @@
-# Phase 1: Foundation - Completed ✅
-
-## Summary
-Successfully implemented the foundation for Vuer RTC with comprehensive test coverage using Test-Driven Development (TDD).
-
-## What Was Built
-
-### 1. Testing Infrastructure
-- **Jest** with TypeScript and ESM support
-- Parallel test execution (50% of available CPUs)
-- Async test support for integration tests
-- Test directory structure: `unit/`, `integration/`, `e2e/`
-- **Test Coverage**: 45 passing unit tests
-
-### 2. Vector Clock Implementation (CRDT)
-- `VectorClockManager` class with full CRDT support
-- Operations: create, increment, merge, compare
-- Causal ordering detection
-- Concurrent operation detection
-- **23 tests** covering all edge cases
-
-### 3. Operation Types & Validation
-- **4 operation types**: CREATE, UPDATE, DELETE, TRANSFORM
-- Complete TypeScript type definitions
-- `OperationValidator` with schema validation
-- Transform3D with quaternion rotation support
-- **20 tests** for type guards and validation
-
-### 4. Prisma Schema (MongoDB)
-- **5 data models**: Document, SceneObject, Operation, JournalBatch, Session
-- CRDT metadata for conflict resolution
-- Soft deletes with tombstone pattern
-- Optimized indexes for queries
-- Write-ahead log structure (33ms batching)
-
-### 5. Database Repositories
-- `PrismaClient` singleton for connection management
-- `DocumentRepository` with CRUD operations
-- `SessionRepository` with presence tracking
-- Version incrementing and clock value management
-- Integration tests ready (require MongoDB)
-
-### 6. Code Quality
-- Modular architecture with clean exports
-- High-level abstractions
-- Comprehensive documentation
-- Type-safe implementations
-
-## Git Commits
-
-1. `19da4ca` - Initialize Jest testing framework
-2. `4cfcd90` - Implement vector clock with tests (23 tests)
-3. `6ab7c9e` - Add operation types and validation (20 tests)
-4. `ce90de3` - Add Prisma schema for all models
-5. `35920c6` - Add database repositories
-6. `b2a81a8` - Refactor with modular exports
-
-## Metrics
-
-- **Total Tests**: 45 passing
-- **Code Coverage**: Ready for >80% threshold
-- **TypeScript**: 100% type-safe
-- **MongoDB Models**: 5 complete schemas
-- **Repository Methods**: 15+ database operations
-
-## Ready for Next Phase
-
-Phase 1 provides the complete foundation for:
-- Phase 2: State Management (SceneState, StateManager, ConflictResolver)
-- Phase 3: WebSocket Server (real-time communication)
-- Phase 4: Write-Ahead Log (journal with batching)
-- Phase 5: History/Undo system
-- Phase 6: Client Library
-- Phase 7: Documentation site
-
-## To Create PR
-
-```bash
-# Add GitHub remote (if not already done)
-git remote add origin https://github.com/USERNAME/vuer-rtc.git
-
-# Push to GitHub
-git push -u origin main
-
-# Create PR using GitHub CLI
-gh pr create \
-  --title "Phase 1: Foundation - Jest, Vector Clocks, Operations, Prisma" \
-  --body "$(cat PHASE1_SUMMARY.md)" \
-  --reviewer marvinluo1
-```
-
-## Next Steps
-
-Continue with Phase 2: State Management implementation.