@vuer-ai/vuer-rtc-server 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/.env +1 -0
  2. package/S3_COMPRESSION_GUIDE.md +233 -0
  3. package/dist/archive/ArchivalService.d.ts +117 -0
  4. package/dist/archive/ArchivalService.d.ts.map +1 -0
  5. package/dist/archive/ArchivalService.js +181 -0
  6. package/dist/archive/ArchivalService.js.map +1 -0
  7. package/dist/broker/InMemoryBroker.d.ts +2 -0
  8. package/dist/broker/InMemoryBroker.d.ts.map +1 -1
  9. package/dist/broker/InMemoryBroker.js +4 -0
  10. package/dist/broker/InMemoryBroker.js.map +1 -1
  11. package/dist/compression/CompressionUtils.d.ts +57 -0
  12. package/dist/compression/CompressionUtils.d.ts.map +1 -0
  13. package/dist/compression/CompressionUtils.js +90 -0
  14. package/dist/compression/CompressionUtils.js.map +1 -0
  15. package/dist/compression/index.d.ts +7 -0
  16. package/dist/compression/index.d.ts.map +1 -0
  17. package/dist/compression/index.js +7 -0
  18. package/dist/compression/index.js.map +1 -0
  19. package/dist/journal/CoalescingService.d.ts +63 -0
  20. package/dist/journal/CoalescingService.d.ts.map +1 -0
  21. package/dist/journal/CoalescingService.js +507 -0
  22. package/dist/journal/CoalescingService.js.map +1 -0
  23. package/dist/journal/JournalRLE.d.ts +81 -0
  24. package/dist/journal/JournalRLE.d.ts.map +1 -0
  25. package/dist/journal/JournalRLE.js +199 -0
  26. package/dist/journal/JournalRLE.js.map +1 -0
  27. package/dist/journal/JournalService.d.ts +7 -3
  28. package/dist/journal/JournalService.d.ts.map +1 -1
  29. package/dist/journal/JournalService.js +152 -12
  30. package/dist/journal/JournalService.js.map +1 -1
  31. package/dist/journal/RLECompression.d.ts +73 -0
  32. package/dist/journal/RLECompression.d.ts.map +1 -0
  33. package/dist/journal/RLECompression.js +152 -0
  34. package/dist/journal/RLECompression.js.map +1 -0
  35. package/dist/journal/rle-demo.d.ts +8 -0
  36. package/dist/journal/rle-demo.d.ts.map +1 -0
  37. package/dist/journal/rle-demo.js +159 -0
  38. package/dist/journal/rle-demo.js.map +1 -0
  39. package/dist/persistence/S3ColdStorage.d.ts +62 -0
  40. package/dist/persistence/S3ColdStorage.d.ts.map +1 -0
  41. package/dist/persistence/S3ColdStorage.js +88 -0
  42. package/dist/persistence/S3ColdStorage.js.map +1 -0
  43. package/dist/persistence/S3ColdStorageIntegration.d.ts +78 -0
  44. package/dist/persistence/S3ColdStorageIntegration.d.ts.map +1 -0
  45. package/dist/persistence/S3ColdStorageIntegration.js +93 -0
  46. package/dist/persistence/S3ColdStorageIntegration.js.map +1 -0
  47. package/dist/serve.d.ts +2 -0
  48. package/dist/serve.d.ts.map +1 -1
  49. package/dist/serve.js +623 -15
  50. package/dist/serve.js.map +1 -1
  51. package/docs/RLE_COMPRESSION.md +397 -0
  52. package/examples/compression-example.ts +259 -0
  53. package/package.json +14 -14
  54. package/src/archive/ArchivalService.ts +250 -0
  55. package/src/broker/InMemoryBroker.ts +5 -0
  56. package/src/compression/CompressionUtils.ts +113 -0
  57. package/src/compression/index.ts +14 -0
  58. package/src/journal/COALESCING.md +267 -0
  59. package/src/journal/CoalescingService.ts +626 -0
  60. package/src/journal/JournalRLE.ts +265 -0
  61. package/src/journal/JournalService.ts +163 -11
  62. package/src/journal/RLECompression.ts +210 -0
  63. package/src/journal/rle-demo.ts +193 -0
  64. package/src/serve.ts +702 -15
  65. package/tests/benchmark/journal-optimization-benchmark.test.ts +482 -0
  66. package/tests/compression/compression.test.ts +343 -0
  67. package/tests/integration/repositories.test.ts +89 -0
  68. package/tests/journal/compaction-load-bug.test.ts +409 -0
  69. package/tests/journal/compaction.test.ts +42 -2
  70. package/tests/journal/journal-rle.test.ts +511 -0
  71. package/tests/journal/lww-ordering-bug.test.ts +248 -0
  72. package/tests/journal/multi-session-coalescing.test.ts +871 -0
  73. package/tests/journal/rle-compression.test.ts +526 -0
  74. package/tests/journal/text-coalescing.test.ts +210 -0
  75. package/tests/unit/s3-compression.test.ts +257 -0
  76. package/PHASE1_SUMMARY.md +0 -94
@@ -0,0 +1,343 @@
1
+ /**
2
+ * Unit Tests for CompressionUtils
3
+ *
4
+ * Tests verify:
5
+ * A. Compression reduces data size
6
+ * B. Decompression roundtrip preserves data integrity
7
+ * C. Compression ratio is within expected bounds
8
+ * D. Edge cases (empty, large, nested objects)
9
+ */
10
+
11
+ import { describe, it, expect } from '@jest/globals';
12
+ import {
13
+ compressSnapshot,
14
+ decompressSnapshot,
15
+ verifyCompressionRoundtrip,
16
+ getCompressionStats,
17
+ } from '../../src/compression/CompressionUtils';
18
+
19
+ describe('CompressionUtils', () => {
20
+ describe('compressSnapshot', () => {
21
+ it('should compress a simple snapshot', () => {
22
+ const snapshot = {
23
+ graph: {
24
+ nodes: {
25
+ root: { key: 'root', tag: 'Group', name: 'Root' },
26
+ },
27
+ rootKey: 'root',
28
+ },
29
+ vectorClock: { session1: 10 },
30
+ lamportTime: 10,
31
+ journalIndex: 10,
32
+ };
33
+
34
+ const result = compressSnapshot(snapshot);
35
+
36
+ expect(result.compressed).toBeDefined();
37
+ expect(result.compressedSize).toBeLessThan(result.originalSize);
38
+ expect(result.ratio).toBeGreaterThan(0);
39
+ expect(result.ratio).toBeLessThan(100);
40
+ });
41
+
42
+ it('should compress a complex nested snapshot', () => {
43
+ const nodeMap: Record<string, any> = {
44
+ root: { key: 'root', tag: 'Group', name: 'Root Node', children: [] },
45
+ };
46
+
47
+ // Create 50 child nodes
48
+ for (let i = 0; i < 50; i++) {
49
+ nodeMap[`child-${i}`] = {
50
+ key: `child-${i}`,
51
+ tag: 'Mesh',
52
+ name: `Child ${i}`,
53
+ position: [Math.random(), Math.random(), Math.random()],
54
+ rotation: [Math.random(), Math.random(), Math.random()],
55
+ scale: [1, 1, 1],
56
+ children: [],
57
+ };
58
+ }
59
+
60
+ const snapshot = {
61
+ graph: {
62
+ nodes: nodeMap,
63
+ rootKey: 'root',
64
+ },
65
+ vectorClock: {
66
+ session1: 100,
67
+ session2: 85,
68
+ session3: 92,
69
+ },
70
+ lamportTime: 100,
71
+ journalIndex: 100,
72
+ };
73
+
74
+ const result = compressSnapshot(snapshot);
75
+
76
+ // Complex nested object should compress well
77
+ expect(result.compressedSize).toBeLessThan(result.originalSize);
78
+ // With repetitive structure, expect decent compression (30-60% of original)
79
+ expect(result.ratio).toBeLessThan(70);
80
+ });
81
+
82
+ it('should return compression metrics', () => {
83
+ const snapshot = { data: 'test' };
84
+ const result = compressSnapshot(snapshot);
85
+
86
+ expect(result).toHaveProperty('compressed');
87
+ expect(result).toHaveProperty('original');
88
+ expect(result).toHaveProperty('ratio');
89
+ expect(result).toHaveProperty('originalSize');
90
+ expect(result).toHaveProperty('compressedSize');
91
+
92
+ expect(typeof result.originalSize).toBe('number');
93
+ expect(typeof result.compressedSize).toBe('number');
94
+ expect(typeof result.ratio).toBe('number');
95
+ });
96
+ });
97
+
98
+ describe('decompressSnapshot', () => {
99
+ it('should decompress a compressed snapshot', () => {
100
+ const original = {
101
+ graph: {
102
+ nodes: {
103
+ root: { key: 'root', tag: 'Group', name: 'Root', children: [] },
104
+ },
105
+ rootKey: 'root',
106
+ },
107
+ vectorClock: { session1: 5 },
108
+ lamportTime: 5,
109
+ journalIndex: 5,
110
+ };
111
+
112
+ const compressed = compressSnapshot(original);
113
+ const result = decompressSnapshot(compressed.compressed);
114
+
115
+ expect(result.decompressed).toBeDefined();
116
+ expect(result.verified).toBe(true);
117
+
118
+ const decompressedObj = JSON.parse(result.decompressed.toString('utf-8'));
119
+ expect(decompressedObj).toEqual(original);
120
+ });
121
+
122
+ it('should handle decompression of large data', () => {
123
+ const nodeMap: Record<string, any> = {
124
+ root: { key: 'root', tag: 'Group', name: 'Root', children: [] },
125
+ };
126
+
127
+ // Create 200 nodes with repetitive data
128
+ for (let i = 0; i < 200; i++) {
129
+ nodeMap[`node-${i}`] = {
130
+ key: `node-${i}`,
131
+ tag: 'Mesh',
132
+ position: [Math.random(), Math.random(), Math.random()],
133
+ data: `Large data field ${i}`.repeat(10),
134
+ children: [],
135
+ };
136
+ }
137
+
138
+ const largeSnapshot = {
139
+ graph: {
140
+ nodes: nodeMap,
141
+ rootKey: 'root',
142
+ },
143
+ vectorClock: Object.fromEntries(
144
+ Array.from({ length: 10 }, (_, i) => [`session${i}`, i * 10])
145
+ ),
146
+ lamportTime: 100,
147
+ journalIndex: 100,
148
+ };
149
+
150
+ const compressed = compressSnapshot(largeSnapshot);
151
+ const result = decompressSnapshot(compressed.compressed);
152
+
153
+ expect(result.verified).toBe(true);
154
+ const decompressed = JSON.parse(result.decompressed.toString('utf-8'));
155
+ expect(decompressed).toEqual(largeSnapshot);
156
+ });
157
+
158
+ it('should throw on corrupted data', () => {
159
+ const corruptedBuffer = Buffer.from('not-gzip-data');
160
+ expect(() => decompressSnapshot(corruptedBuffer)).toThrow();
161
+ });
162
+
163
+ it('should throw on empty buffer', () => {
164
+ const emptyBuffer = Buffer.alloc(0);
165
+ expect(() => decompressSnapshot(emptyBuffer)).toThrow();
166
+ });
167
+ });
168
+
169
+ describe('verifyCompressionRoundtrip', () => {
170
+ it('should verify roundtrip for simple snapshot', () => {
171
+ const snapshot = {
172
+ graph: {
173
+ nodes: {
174
+ root: { key: 'root', tag: 'Group', name: 'Root', children: [] },
175
+ },
176
+ rootKey: 'root',
177
+ },
178
+ vectorClock: {},
179
+ lamportTime: 0,
180
+ journalIndex: 0,
181
+ };
182
+
183
+ const verified = verifyCompressionRoundtrip(snapshot);
184
+ expect(verified).toBe(true);
185
+ });
186
+
187
+ it('should verify roundtrip for complex snapshot', () => {
188
+ const nodeMap: Record<string, any> = {
189
+ root: { key: 'root', tag: 'Group', name: 'Root', children: [] },
190
+ };
191
+
192
+ // Create 100 child nodes
193
+ for (let i = 0; i < 100; i++) {
194
+ nodeMap[`child-${i}`] = {
195
+ key: `child-${i}`,
196
+ tag: 'Mesh',
197
+ name: `Child ${i}`,
198
+ position: [1, 2, 3],
199
+ properties: { a: 1, b: 'test', c: [1, 2, 3] },
200
+ children: [],
201
+ };
202
+ }
203
+
204
+ const snapshot = {
205
+ graph: {
206
+ nodes: nodeMap,
207
+ rootKey: 'root',
208
+ },
209
+ vectorClock: { s1: 50, s2: 50, s3: 50 },
210
+ lamportTime: 50,
211
+ journalIndex: 50,
212
+ };
213
+
214
+ const verified = verifyCompressionRoundtrip(snapshot);
215
+ expect(verified).toBe(true);
216
+ });
217
+
218
+ it('should handle null and undefined', () => {
219
+ // null is valid JSON
220
+ expect(verifyCompressionRoundtrip(null)).toBe(true);
221
+
222
+ // undefined becomes null in JSON
223
+ const result = verifyCompressionRoundtrip(undefined);
224
+ expect(typeof result).toBe('boolean');
225
+ });
226
+ });
227
+
228
+ describe('getCompressionStats', () => {
229
+ it('should return statistics without storing data', () => {
230
+ const snapshot = {
231
+ data: 'test data'.repeat(100),
232
+ nested: { deep: { structure: { value: 123 } } },
233
+ };
234
+
235
+ const stats = getCompressionStats(snapshot);
236
+
237
+ expect(stats).toHaveProperty('originalSize');
238
+ expect(stats).toHaveProperty('compressedSize');
239
+ expect(stats).toHaveProperty('ratio');
240
+ expect(stats).toHaveProperty('savedBytes');
241
+
242
+ expect(stats.originalSize).toBeGreaterThan(0);
243
+ expect(stats.compressedSize).toBeGreaterThan(0);
244
+ expect(stats.savedBytes).toBe(stats.originalSize - stats.compressedSize);
245
+ expect(stats.ratio).toBeLessThan(100);
246
+ });
247
+
248
+ it('should show significant savings for repetitive data', () => {
249
+ const repetitiveSnapshot = {
250
+ items: Array.from({ length: 1000 }, (_, i) => ({
251
+ id: i,
252
+ name: `Item ${i}`,
253
+ type: 'standard',
254
+ status: 'active',
255
+ tags: ['tag1', 'tag2', 'tag3'],
256
+ })),
257
+ };
258
+
259
+ const stats = getCompressionStats(repetitiveSnapshot);
260
+
261
+ // Repetitive data should compress well (expect <50% of original)
262
+ expect(stats.ratio).toBeLessThan(50);
263
+ expect(stats.savedBytes).toBeGreaterThan(stats.originalSize * 0.5);
264
+ });
265
+ });
266
+
267
+ describe('Integration: Real-world snapshots', () => {
268
+ it('should achieve 75-80% compression on realistic journal snapshot', () => {
269
+ // Simulate a realistic journal snapshot with multiple rooms/scenes
270
+ const nodes: Record<string, any> = {
271
+ root: { key: 'root', tag: 'Group', name: 'World Root', children: [] },
272
+ };
273
+
274
+ // Create 20 rooms with 50 meshes each
275
+ for (let roomIdx = 0; roomIdx < 20; roomIdx++) {
276
+ nodes[`room-${roomIdx}`] = {
277
+ key: `room-${roomIdx}`,
278
+ tag: 'Group',
279
+ name: `Room ${roomIdx}`,
280
+ children: [],
281
+ };
282
+
283
+ for (let nodeIdx = 0; nodeIdx < 50; nodeIdx++) {
284
+ nodes[`node-${roomIdx}-${nodeIdx}`] = {
285
+ key: `node-${roomIdx}-${nodeIdx}`,
286
+ tag: 'Mesh',
287
+ name: `Mesh ${nodeIdx}`,
288
+ position: [Math.random() * 100, Math.random() * 100, Math.random() * 100],
289
+ rotation: [0, 0, 0],
290
+ scale: [1, 1, 1],
291
+ material: {
292
+ color: `#${Math.random().toString(16).slice(2, 8)}`,
293
+ roughness: 0.5,
294
+ metallic: 0.5,
295
+ },
296
+ children: [],
297
+ };
298
+ }
299
+ }
300
+
301
+ const realisticSnapshot = {
302
+ graph: {
303
+ nodes,
304
+ rootKey: 'root',
305
+ },
306
+ vectorClock: Object.fromEntries(
307
+ Array.from({ length: 50 }, (_, i) => [`session-${i}`, 1000 + i * 100])
308
+ ),
309
+ lamportTime: 6000,
310
+ journalIndex: 6000,
311
+ };
312
+
313
+ const stats = getCompressionStats(realisticSnapshot);
314
+
315
+ // Should achieve our target compression ratio
316
+ expect(stats.ratio).toBeLessThan(25); // Less than 25% is 75%+ compression
317
+ console.log(
318
+ `✓ Compression: ${stats.originalSize} → ${stats.compressedSize} bytes (${stats.ratio.toFixed(1)}% of original, ${(100 - stats.ratio).toFixed(1)}% saved)`
319
+ );
320
+ });
321
+
322
+ it('should handle empty snapshot', () => {
323
+ const emptySnapshot = {
324
+ graph: {
325
+ nodes: {
326
+ root: { key: 'root', tag: 'Group', name: 'Root', children: [] },
327
+ },
328
+ rootKey: 'root',
329
+ },
330
+ vectorClock: {},
331
+ lamportTime: 0,
332
+ journalIndex: 0,
333
+ };
334
+
335
+ const result = compressSnapshot(emptySnapshot);
336
+ expect(result.compressedSize).toBeGreaterThan(0);
337
+
338
+ // Even empty snapshots benefit from compression slightly
339
+ const decompressed = decompressSnapshot(result.compressed);
340
+ expect(decompressed.verified).toBe(true);
341
+ });
342
+ });
343
+ });
@@ -19,6 +19,7 @@ interface ISessionRepository {
19
19
  findByDocument(documentId: string): Promise<any[]>;
20
20
  updatePresence(id: string, presence: any): Promise<any>;
21
21
  disconnect(id: string): Promise<any>;
22
+ deleteDisconnected(olderThan: Date): Promise<number>;
22
23
  }
23
24
 
24
25
  // Try to import - will fail until implemented
@@ -69,6 +70,9 @@ try {
69
70
  disconnect() {
70
71
  throw new Error('Not implemented');
71
72
  }
73
+ deleteDisconnected() {
74
+ throw new Error('Not implemented');
75
+ }
72
76
  } as any;
73
77
  }
74
78
 
@@ -316,5 +320,90 @@ describe('Database Repositories', () => {
316
320
  // Cleanup
317
321
  await sessionRepo.disconnect(session1.id);
318
322
  });
323
+
324
+ it('should delete old disconnected sessions', async () => {
325
+ // Create and disconnect a session
326
+ const oldSession = await sessionRepo.create({
327
+ documentId: testDocId,
328
+ userId: 'user-old',
329
+ clientId: 'client-old',
330
+ });
331
+ await sessionRepo.disconnect(oldSession.id);
332
+
333
+ // Create a recent disconnected session
334
+ const recentSession = await sessionRepo.create({
335
+ documentId: testDocId,
336
+ userId: 'user-recent',
337
+ clientId: 'client-recent',
338
+ });
339
+ await sessionRepo.disconnect(recentSession.id);
340
+
341
+ // Create an active session
342
+ const activeSession = await sessionRepo.create({
343
+ documentId: testDocId,
344
+ userId: 'user-active',
345
+ clientId: 'client-active',
346
+ });
347
+
348
+ // Delete sessions disconnected before now (should catch the old one but not recent)
349
+ // We'll use a cutoff that's slightly in the past
350
+ const cutoffDate = new Date(Date.now() - 1000); // 1 second ago
351
+
352
+ // Wait a bit to ensure timestamps are different
353
+ await new Promise(resolve => setTimeout(resolve, 1100));
354
+
355
+ const deletedCount = await sessionRepo.deleteDisconnected(cutoffDate);
356
+
357
+ // Should delete both disconnected sessions (old and recent)
358
+ expect(deletedCount).toBeGreaterThanOrEqual(2);
359
+
360
+ // Active session should still exist
361
+ const activeFound = await sessionRepo.findById(activeSession.id);
362
+ expect(activeFound).toBeDefined();
363
+ expect(activeFound!.connected).toBe(true);
364
+
365
+ // Old session should be gone
366
+ const oldFound = await sessionRepo.findById(oldSession.id);
367
+ expect(oldFound).toBeNull();
368
+
369
+ // Recent session should be gone
370
+ const recentFound = await sessionRepo.findById(recentSession.id);
371
+ expect(recentFound).toBeNull();
372
+
373
+ // Cleanup
374
+ await sessionRepo.disconnect(activeSession.id);
375
+ });
376
+
377
+ it('should not delete active sessions', async () => {
378
+ // Create active sessions
379
+ const session1 = await sessionRepo.create({
380
+ documentId: testDocId,
381
+ userId: 'user-active-1',
382
+ clientId: 'client-active-1',
383
+ });
384
+
385
+ const session2 = await sessionRepo.create({
386
+ documentId: testDocId,
387
+ userId: 'user-active-2',
388
+ clientId: 'client-active-2',
389
+ });
390
+
391
+ // Try to delete all sessions (even future ones)
392
+ const futureDate = new Date(Date.now() + 24 * 60 * 60 * 1000);
393
+ const deletedCount = await sessionRepo.deleteDisconnected(futureDate);
394
+
395
+ // Should not delete any active sessions
396
+ const session1Found = await sessionRepo.findById(session1.id);
397
+ const session2Found = await sessionRepo.findById(session2.id);
398
+
399
+ expect(session1Found).toBeDefined();
400
+ expect(session1Found!.connected).toBe(true);
401
+ expect(session2Found).toBeDefined();
402
+ expect(session2Found!.connected).toBe(true);
403
+
404
+ // Cleanup
405
+ await sessionRepo.disconnect(session1.id);
406
+ await sessionRepo.disconnect(session2.id);
407
+ });
319
408
  });
320
409
  });