@vuer-ai/vuer-rtc-server 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/.env +1 -0
  2. package/S3_COMPRESSION_GUIDE.md +233 -0
  3. package/dist/archive/ArchivalService.d.ts +117 -0
  4. package/dist/archive/ArchivalService.d.ts.map +1 -0
  5. package/dist/archive/ArchivalService.js +181 -0
  6. package/dist/archive/ArchivalService.js.map +1 -0
  7. package/dist/broker/InMemoryBroker.d.ts +2 -0
  8. package/dist/broker/InMemoryBroker.d.ts.map +1 -1
  9. package/dist/broker/InMemoryBroker.js +4 -0
  10. package/dist/broker/InMemoryBroker.js.map +1 -1
  11. package/dist/compression/CompressionUtils.d.ts +57 -0
  12. package/dist/compression/CompressionUtils.d.ts.map +1 -0
  13. package/dist/compression/CompressionUtils.js +90 -0
  14. package/dist/compression/CompressionUtils.js.map +1 -0
  15. package/dist/compression/index.d.ts +7 -0
  16. package/dist/compression/index.d.ts.map +1 -0
  17. package/dist/compression/index.js +7 -0
  18. package/dist/compression/index.js.map +1 -0
  19. package/dist/journal/CoalescingService.d.ts +63 -0
  20. package/dist/journal/CoalescingService.d.ts.map +1 -0
  21. package/dist/journal/CoalescingService.js +507 -0
  22. package/dist/journal/CoalescingService.js.map +1 -0
  23. package/dist/journal/JournalRLE.d.ts +81 -0
  24. package/dist/journal/JournalRLE.d.ts.map +1 -0
  25. package/dist/journal/JournalRLE.js +199 -0
  26. package/dist/journal/JournalRLE.js.map +1 -0
  27. package/dist/journal/JournalService.d.ts +7 -3
  28. package/dist/journal/JournalService.d.ts.map +1 -1
  29. package/dist/journal/JournalService.js +152 -12
  30. package/dist/journal/JournalService.js.map +1 -1
  31. package/dist/journal/RLECompression.d.ts +73 -0
  32. package/dist/journal/RLECompression.d.ts.map +1 -0
  33. package/dist/journal/RLECompression.js +152 -0
  34. package/dist/journal/RLECompression.js.map +1 -0
  35. package/dist/journal/rle-demo.d.ts +8 -0
  36. package/dist/journal/rle-demo.d.ts.map +1 -0
  37. package/dist/journal/rle-demo.js +159 -0
  38. package/dist/journal/rle-demo.js.map +1 -0
  39. package/dist/persistence/S3ColdStorage.d.ts +62 -0
  40. package/dist/persistence/S3ColdStorage.d.ts.map +1 -0
  41. package/dist/persistence/S3ColdStorage.js +88 -0
  42. package/dist/persistence/S3ColdStorage.js.map +1 -0
  43. package/dist/persistence/S3ColdStorageIntegration.d.ts +78 -0
  44. package/dist/persistence/S3ColdStorageIntegration.d.ts.map +1 -0
  45. package/dist/persistence/S3ColdStorageIntegration.js +93 -0
  46. package/dist/persistence/S3ColdStorageIntegration.js.map +1 -0
  47. package/dist/serve.d.ts +2 -0
  48. package/dist/serve.d.ts.map +1 -1
  49. package/dist/serve.js +623 -15
  50. package/dist/serve.js.map +1 -1
  51. package/docs/RLE_COMPRESSION.md +397 -0
  52. package/examples/compression-example.ts +259 -0
  53. package/package.json +14 -14
  54. package/src/archive/ArchivalService.ts +250 -0
  55. package/src/broker/InMemoryBroker.ts +5 -0
  56. package/src/compression/CompressionUtils.ts +113 -0
  57. package/src/compression/index.ts +14 -0
  58. package/src/journal/COALESCING.md +267 -0
  59. package/src/journal/CoalescingService.ts +626 -0
  60. package/src/journal/JournalRLE.ts +265 -0
  61. package/src/journal/JournalService.ts +163 -11
  62. package/src/journal/RLECompression.ts +210 -0
  63. package/src/journal/rle-demo.ts +193 -0
  64. package/src/serve.ts +702 -15
  65. package/tests/benchmark/journal-optimization-benchmark.test.ts +482 -0
  66. package/tests/compression/compression.test.ts +343 -0
  67. package/tests/integration/repositories.test.ts +89 -0
  68. package/tests/journal/compaction-load-bug.test.ts +409 -0
  69. package/tests/journal/compaction.test.ts +42 -2
  70. package/tests/journal/journal-rle.test.ts +511 -0
  71. package/tests/journal/lww-ordering-bug.test.ts +248 -0
  72. package/tests/journal/multi-session-coalescing.test.ts +871 -0
  73. package/tests/journal/rle-compression.test.ts +526 -0
  74. package/tests/journal/text-coalescing.test.ts +210 -0
  75. package/tests/unit/s3-compression.test.ts +257 -0
  76. package/PHASE1_SUMMARY.md +0 -94
@@ -0,0 +1,248 @@
1
+ /**
2
+ * Test demonstrating LWW bug caused by session grouping in coalescing
3
+ *
4
+ * The bug: When coalescing groups all operations by session, it uses the
5
+ * last message's lamportTime for ALL operations from that session.
6
+ * This breaks LWW conflict resolution when operations are interleaved.
7
+ */
8
+
9
+ import { describe, it, expect } from '@jest/globals';
10
+ import type { CRDTMessage, Operation, SceneGraph } from '@vuer-ai/vuer-rtc';
11
+ import { createEmptyGraph, applyMessage } from '@vuer-ai/vuer-rtc';
12
+
13
+ /**
14
+ * Helper to coalesce messages using current (buggy) implementation
15
+ */
16
+ function coalesceBuggy(messages: CRDTMessage[]): CRDTMessage[] {
17
+ if (messages.length === 0) return [];
18
+
19
+ // Current implementation: Group by session
20
+ const sessionGroups = new Map<string, CRDTMessage[]>();
21
+ for (const msg of messages) {
22
+ if (!sessionGroups.has(msg.sessionId)) {
23
+ sessionGroups.set(msg.sessionId, []);
24
+ }
25
+ sessionGroups.get(msg.sessionId)!.push(msg);
26
+ }
27
+
28
+ const coalesced: CRDTMessage[] = [];
29
+ for (const [sessionId, sessionMessages] of sessionGroups) {
30
+ // Flatten all ops from this session
31
+ const allOps: Operation[] = [];
32
+ for (const msg of sessionMessages) {
33
+ allOps.push(...msg.ops);
34
+ }
35
+
36
+ // Use last message's metadata (THIS IS THE BUG)
37
+ const lastMsg = sessionMessages[sessionMessages.length - 1];
38
+ coalesced.push({
39
+ ...lastMsg,
40
+ ops: allOps,
41
+ });
42
+ }
43
+
44
+ return coalesced;
45
+ }
46
+
47
+ /**
48
+ * Helper to coalesce messages preserving interleaved order
49
+ */
50
+ function coalesceCorrect(messages: CRDTMessage[]): CRDTMessage[] {
51
+ if (messages.length === 0) return [];
52
+
53
+ // Correct implementation: Group into consecutive runs
54
+ const runs: CRDTMessage[][] = [];
55
+ let currentRun: CRDTMessage[] = [];
56
+ let lastSessionId: string | null = null;
57
+
58
+ for (const msg of messages) {
59
+ if (msg.sessionId !== lastSessionId && currentRun.length > 0) {
60
+ runs.push(currentRun);
61
+ currentRun = [];
62
+ }
63
+ currentRun.push(msg);
64
+ lastSessionId = msg.sessionId;
65
+ }
66
+ if (currentRun.length > 0) runs.push(currentRun);
67
+
68
+ // Coalesce each run
69
+ return runs.map(run => {
70
+ const allOps: Operation[] = [];
71
+ for (const msg of run) {
72
+ allOps.push(...msg.ops);
73
+ }
74
+ const lastMsg = run[run.length - 1];
75
+ return {
76
+ ...lastMsg,
77
+ ops: allOps,
78
+ };
79
+ });
80
+ }
81
+
82
+ describe('LWW Ordering Bug', () => {
83
+ /**
84
+ * Scenario:
85
+ * 1. Alice sets cube.color = "red" (lamport=10)
86
+ * 2. Bob sets cube.color = "blue" (lamport=15)
87
+ * 3. Alice sets cube.color = "green" (lamport=20)
88
+ *
89
+ * Expected: Final color should be "green" (lamport=20 wins)
90
+ *
91
+ * Bug: When coalescing groups all Alice's ops together and uses
92
+ * lamport=20 for both, BOTH red and green get lamport=20.
93
+ * This breaks LWW resolution.
94
+ */
95
+ it('should preserve LWW ordering when operations are interleaved', () => {
96
+ // Create the node first
97
+ let graph = createEmptyGraph();
98
+ graph = applyMessage(graph, {
99
+ id: 'init',
100
+ sessionId: 'init',
101
+ clock: { init: 1 },
102
+ lamportTime: 1,
103
+ timestamp: 0,
104
+ ops: [{ otype: 'node.insert', key: '__root', path: 'children', value: { key: 'cube', tag: 'Mesh', name: 'Cube' } }],
105
+ });
106
+
107
+ // Original messages in interleaved order
108
+ const messages: CRDTMessage[] = [
109
+ {
110
+ id: 'msg1',
111
+ sessionId: 'alice',
112
+ clock: { alice: 10 },
113
+ lamportTime: 10,
114
+ timestamp: 1000,
115
+ ops: [
116
+ {
117
+ otype: 'string.set',
118
+ key: 'cube',
119
+ path: 'color',
120
+ value: 'red',
121
+ },
122
+ ],
123
+ },
124
+ {
125
+ id: 'msg2',
126
+ sessionId: 'bob',
127
+ clock: { alice: 10, bob: 15 },
128
+ lamportTime: 15,
129
+ timestamp: 1500,
130
+ ops: [
131
+ {
132
+ otype: 'string.set',
133
+ key: 'cube',
134
+ path: 'color',
135
+ value: 'blue',
136
+ },
137
+ ],
138
+ },
139
+ {
140
+ id: 'msg3',
141
+ sessionId: 'alice',
142
+ clock: { alice: 20, bob: 15 },
143
+ lamportTime: 20,
144
+ timestamp: 2000,
145
+ ops: [
146
+ {
147
+ otype: 'string.set',
148
+ key: 'cube',
149
+ path: 'color',
150
+ value: 'green',
151
+ },
152
+ ],
153
+ },
154
+ ];
155
+
156
+ // Apply original messages - should result in green
157
+ let graphOriginal = graph;
158
+ for (const msg of messages) {
159
+ graphOriginal = applyMessage(graphOriginal, msg);
160
+ }
161
+ expect(graphOriginal.nodes['cube']?.color).toBe('green');
162
+
163
+ // Apply buggy coalescing
164
+ const coalescedBuggy = coalesceBuggy(messages);
165
+ console.log('Buggy coalescing:');
166
+ coalescedBuggy.forEach(msg => {
167
+ console.log(` session=${msg.sessionId}, lamport=${msg.lamportTime}, ops=${msg.ops.length}`);
168
+ msg.ops.forEach(op => console.log(` ${JSON.stringify(op)}`));
169
+ });
170
+
171
+ let graphBuggy = graph;
172
+ for (const msg of coalescedBuggy) {
173
+ graphBuggy = applyMessage(graphBuggy, msg);
174
+ }
175
+
176
+ // THIS WILL FAIL - buggy version might produce wrong result
177
+ console.log(`Buggy result: ${graphBuggy.nodes['cube']?.color}`);
178
+
179
+ // Apply correct coalescing
180
+ const coalescedCorrect = coalesceCorrect(messages);
181
+ console.log('\nCorrect coalescing:');
182
+ coalescedCorrect.forEach(msg => {
183
+ console.log(` session=${msg.sessionId}, lamport=${msg.lamportTime}, ops=${msg.ops.length}`);
184
+ msg.ops.forEach(op => console.log(` ${JSON.stringify(op)}`));
185
+ });
186
+
187
+ let graphCorrect = graph;
188
+ for (const msg of coalescedCorrect) {
189
+ graphCorrect = applyMessage(graphCorrect, msg);
190
+ }
191
+ console.log(`Correct result: ${graphCorrect.nodes['cube']?.color}`);
192
+
193
+ // Both should produce "green"
194
+ expect(graphCorrect.nodes['cube']?.color).toBe('green');
195
+
196
+ // This expectation will FAIL with buggy coalescing
197
+ expect(graphBuggy.nodes['cube']?.color).toBe('green');
198
+ });
199
+
200
+ it('demonstrates the metadata inheritance problem', () => {
201
+ const messages: CRDTMessage[] = [
202
+ {
203
+ id: 'msg1',
204
+ sessionId: 'alice',
205
+ clock: { alice: 10 },
206
+ lamportTime: 10,
207
+ timestamp: 1000,
208
+ ops: [{ otype: 'number.set', key: 'x', path: 'value', value: 1 }],
209
+ },
210
+ {
211
+ id: 'msg2',
212
+ sessionId: 'bob',
213
+ clock: { alice: 10, bob: 15 },
214
+ lamportTime: 15,
215
+ timestamp: 1500,
216
+ ops: [{ otype: 'number.set', key: 'x', path: 'value', value: 2 }],
217
+ },
218
+ {
219
+ id: 'msg3',
220
+ sessionId: 'alice',
221
+ clock: { alice: 20, bob: 15 },
222
+ lamportTime: 20,
223
+ timestamp: 2000,
224
+ ops: [{ otype: 'number.set', key: 'x', path: 'value', value: 3 }],
225
+ },
226
+ ];
227
+
228
+ const coalesced = coalesceBuggy(messages);
229
+
230
+ // Find alice's coalesced message
231
+ const aliceMsg = coalesced.find(m => m.sessionId === 'alice')!;
232
+
233
+ // Bug: ALL of Alice's ops inherit lamportTime=20 from msg3
234
+ expect(aliceMsg.lamportTime).toBe(20);
235
+ expect(aliceMsg.ops).toHaveLength(2); // Both set operations
236
+
237
+ // When these ops are applied, they BOTH get lamportTime=20
238
+ // This means the first operation (value=1) incorrectly gets lamport=20
239
+ // instead of its original lamport=10
240
+
241
+ console.log('Alice coalesced message:', {
242
+ lamportTime: aliceMsg.lamportTime,
243
+ ops: aliceMsg.ops,
244
+ });
245
+
246
+ // The problem: Both ops get lamport=20, so LWW resolution is broken
247
+ });
248
+ });