@vuer-ai/vuer-rtc-server 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env +1 -0
- package/S3_COMPRESSION_GUIDE.md +233 -0
- package/dist/archive/ArchivalService.d.ts +117 -0
- package/dist/archive/ArchivalService.d.ts.map +1 -0
- package/dist/archive/ArchivalService.js +181 -0
- package/dist/archive/ArchivalService.js.map +1 -0
- package/dist/broker/InMemoryBroker.d.ts +2 -0
- package/dist/broker/InMemoryBroker.d.ts.map +1 -1
- package/dist/broker/InMemoryBroker.js +4 -0
- package/dist/broker/InMemoryBroker.js.map +1 -1
- package/dist/compression/CompressionUtils.d.ts +57 -0
- package/dist/compression/CompressionUtils.d.ts.map +1 -0
- package/dist/compression/CompressionUtils.js +90 -0
- package/dist/compression/CompressionUtils.js.map +1 -0
- package/dist/compression/index.d.ts +7 -0
- package/dist/compression/index.d.ts.map +1 -0
- package/dist/compression/index.js +7 -0
- package/dist/compression/index.js.map +1 -0
- package/dist/journal/CoalescingService.d.ts +63 -0
- package/dist/journal/CoalescingService.d.ts.map +1 -0
- package/dist/journal/CoalescingService.js +507 -0
- package/dist/journal/CoalescingService.js.map +1 -0
- package/dist/journal/JournalRLE.d.ts +81 -0
- package/dist/journal/JournalRLE.d.ts.map +1 -0
- package/dist/journal/JournalRLE.js +199 -0
- package/dist/journal/JournalRLE.js.map +1 -0
- package/dist/journal/JournalService.d.ts +7 -3
- package/dist/journal/JournalService.d.ts.map +1 -1
- package/dist/journal/JournalService.js +152 -12
- package/dist/journal/JournalService.js.map +1 -1
- package/dist/journal/RLECompression.d.ts +73 -0
- package/dist/journal/RLECompression.d.ts.map +1 -0
- package/dist/journal/RLECompression.js +152 -0
- package/dist/journal/RLECompression.js.map +1 -0
- package/dist/journal/rle-demo.d.ts +8 -0
- package/dist/journal/rle-demo.d.ts.map +1 -0
- package/dist/journal/rle-demo.js +159 -0
- package/dist/journal/rle-demo.js.map +1 -0
- package/dist/persistence/S3ColdStorage.d.ts +62 -0
- package/dist/persistence/S3ColdStorage.d.ts.map +1 -0
- package/dist/persistence/S3ColdStorage.js +88 -0
- package/dist/persistence/S3ColdStorage.js.map +1 -0
- package/dist/persistence/S3ColdStorageIntegration.d.ts +78 -0
- package/dist/persistence/S3ColdStorageIntegration.d.ts.map +1 -0
- package/dist/persistence/S3ColdStorageIntegration.js +93 -0
- package/dist/persistence/S3ColdStorageIntegration.js.map +1 -0
- package/dist/serve.d.ts +2 -0
- package/dist/serve.d.ts.map +1 -1
- package/dist/serve.js +623 -15
- package/dist/serve.js.map +1 -1
- package/docs/RLE_COMPRESSION.md +397 -0
- package/examples/compression-example.ts +259 -0
- package/package.json +14 -14
- package/src/archive/ArchivalService.ts +250 -0
- package/src/broker/InMemoryBroker.ts +5 -0
- package/src/compression/CompressionUtils.ts +113 -0
- package/src/compression/index.ts +14 -0
- package/src/journal/COALESCING.md +267 -0
- package/src/journal/CoalescingService.ts +626 -0
- package/src/journal/JournalRLE.ts +265 -0
- package/src/journal/JournalService.ts +163 -11
- package/src/journal/RLECompression.ts +210 -0
- package/src/journal/rle-demo.ts +193 -0
- package/src/serve.ts +702 -15
- package/tests/benchmark/journal-optimization-benchmark.test.ts +482 -0
- package/tests/compression/compression.test.ts +343 -0
- package/tests/integration/repositories.test.ts +89 -0
- package/tests/journal/compaction-load-bug.test.ts +409 -0
- package/tests/journal/compaction.test.ts +42 -2
- package/tests/journal/journal-rle.test.ts +511 -0
- package/tests/journal/lww-ordering-bug.test.ts +248 -0
- package/tests/journal/multi-session-coalescing.test.ts +871 -0
- package/tests/journal/rle-compression.test.ts +526 -0
- package/tests/journal/text-coalescing.test.ts +210 -0
- package/tests/unit/s3-compression.test.ts +257 -0
- package/PHASE1_SUMMARY.md +0 -94
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test demonstrating LWW bug caused by session grouping in coalescing
|
|
3
|
+
*
|
|
4
|
+
* The bug: When coalescing groups all operations by session, it uses the
|
|
5
|
+
* last message's lamportTime for ALL operations from that session.
|
|
6
|
+
* This breaks LWW conflict resolution when operations are interleaved.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { describe, it, expect } from '@jest/globals';
|
|
10
|
+
import type { CRDTMessage, Operation, SceneGraph } from '@vuer-ai/vuer-rtc';
|
|
11
|
+
import { createEmptyGraph, applyMessage } from '@vuer-ai/vuer-rtc';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Helper to coalesce messages using current (buggy) implementation
|
|
15
|
+
*/
|
|
16
|
+
function coalesceBuggy(messages: CRDTMessage[]): CRDTMessage[] {
|
|
17
|
+
if (messages.length === 0) return [];
|
|
18
|
+
|
|
19
|
+
// Current implementation: Group by session
|
|
20
|
+
const sessionGroups = new Map<string, CRDTMessage[]>();
|
|
21
|
+
for (const msg of messages) {
|
|
22
|
+
if (!sessionGroups.has(msg.sessionId)) {
|
|
23
|
+
sessionGroups.set(msg.sessionId, []);
|
|
24
|
+
}
|
|
25
|
+
sessionGroups.get(msg.sessionId)!.push(msg);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const coalesced: CRDTMessage[] = [];
|
|
29
|
+
for (const [sessionId, sessionMessages] of sessionGroups) {
|
|
30
|
+
// Flatten all ops from this session
|
|
31
|
+
const allOps: Operation[] = [];
|
|
32
|
+
for (const msg of sessionMessages) {
|
|
33
|
+
allOps.push(...msg.ops);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// Use last message's metadata (THIS IS THE BUG)
|
|
37
|
+
const lastMsg = sessionMessages[sessionMessages.length - 1];
|
|
38
|
+
coalesced.push({
|
|
39
|
+
...lastMsg,
|
|
40
|
+
ops: allOps,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
return coalesced;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Helper to coalesce messages preserving interleaved order
|
|
49
|
+
*/
|
|
50
|
+
function coalesceCorrect(messages: CRDTMessage[]): CRDTMessage[] {
|
|
51
|
+
if (messages.length === 0) return [];
|
|
52
|
+
|
|
53
|
+
// Correct implementation: Group into consecutive runs
|
|
54
|
+
const runs: CRDTMessage[][] = [];
|
|
55
|
+
let currentRun: CRDTMessage[] = [];
|
|
56
|
+
let lastSessionId: string | null = null;
|
|
57
|
+
|
|
58
|
+
for (const msg of messages) {
|
|
59
|
+
if (msg.sessionId !== lastSessionId && currentRun.length > 0) {
|
|
60
|
+
runs.push(currentRun);
|
|
61
|
+
currentRun = [];
|
|
62
|
+
}
|
|
63
|
+
currentRun.push(msg);
|
|
64
|
+
lastSessionId = msg.sessionId;
|
|
65
|
+
}
|
|
66
|
+
if (currentRun.length > 0) runs.push(currentRun);
|
|
67
|
+
|
|
68
|
+
// Coalesce each run
|
|
69
|
+
return runs.map(run => {
|
|
70
|
+
const allOps: Operation[] = [];
|
|
71
|
+
for (const msg of run) {
|
|
72
|
+
allOps.push(...msg.ops);
|
|
73
|
+
}
|
|
74
|
+
const lastMsg = run[run.length - 1];
|
|
75
|
+
return {
|
|
76
|
+
...lastMsg,
|
|
77
|
+
ops: allOps,
|
|
78
|
+
};
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
describe('LWW Ordering Bug', () => {
|
|
83
|
+
/**
|
|
84
|
+
* Scenario:
|
|
85
|
+
* 1. Alice sets cube.color = "red" (lamport=10)
|
|
86
|
+
* 2. Bob sets cube.color = "blue" (lamport=15)
|
|
87
|
+
* 3. Alice sets cube.color = "green" (lamport=20)
|
|
88
|
+
*
|
|
89
|
+
* Expected: Final color should be "green" (lamport=20 wins)
|
|
90
|
+
*
|
|
91
|
+
* Bug: When coalescing groups all Alice's ops together and uses
|
|
92
|
+
* lamport=20 for both, BOTH red and green get lamport=20.
|
|
93
|
+
* This breaks LWW resolution.
|
|
94
|
+
*/
|
|
95
|
+
it('should preserve LWW ordering when operations are interleaved', () => {
|
|
96
|
+
// Create the node first
|
|
97
|
+
let graph = createEmptyGraph();
|
|
98
|
+
graph = applyMessage(graph, {
|
|
99
|
+
id: 'init',
|
|
100
|
+
sessionId: 'init',
|
|
101
|
+
clock: { init: 1 },
|
|
102
|
+
lamportTime: 1,
|
|
103
|
+
timestamp: 0,
|
|
104
|
+
ops: [{ otype: 'node.insert', key: '__root', path: 'children', value: { key: 'cube', tag: 'Mesh', name: 'Cube' } }],
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
// Original messages in interleaved order
|
|
108
|
+
const messages: CRDTMessage[] = [
|
|
109
|
+
{
|
|
110
|
+
id: 'msg1',
|
|
111
|
+
sessionId: 'alice',
|
|
112
|
+
clock: { alice: 10 },
|
|
113
|
+
lamportTime: 10,
|
|
114
|
+
timestamp: 1000,
|
|
115
|
+
ops: [
|
|
116
|
+
{
|
|
117
|
+
otype: 'string.set',
|
|
118
|
+
key: 'cube',
|
|
119
|
+
path: 'color',
|
|
120
|
+
value: 'red',
|
|
121
|
+
},
|
|
122
|
+
],
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
id: 'msg2',
|
|
126
|
+
sessionId: 'bob',
|
|
127
|
+
clock: { alice: 10, bob: 15 },
|
|
128
|
+
lamportTime: 15,
|
|
129
|
+
timestamp: 1500,
|
|
130
|
+
ops: [
|
|
131
|
+
{
|
|
132
|
+
otype: 'string.set',
|
|
133
|
+
key: 'cube',
|
|
134
|
+
path: 'color',
|
|
135
|
+
value: 'blue',
|
|
136
|
+
},
|
|
137
|
+
],
|
|
138
|
+
},
|
|
139
|
+
{
|
|
140
|
+
id: 'msg3',
|
|
141
|
+
sessionId: 'alice',
|
|
142
|
+
clock: { alice: 20, bob: 15 },
|
|
143
|
+
lamportTime: 20,
|
|
144
|
+
timestamp: 2000,
|
|
145
|
+
ops: [
|
|
146
|
+
{
|
|
147
|
+
otype: 'string.set',
|
|
148
|
+
key: 'cube',
|
|
149
|
+
path: 'color',
|
|
150
|
+
value: 'green',
|
|
151
|
+
},
|
|
152
|
+
],
|
|
153
|
+
},
|
|
154
|
+
];
|
|
155
|
+
|
|
156
|
+
// Apply original messages - should result in green
|
|
157
|
+
let graphOriginal = graph;
|
|
158
|
+
for (const msg of messages) {
|
|
159
|
+
graphOriginal = applyMessage(graphOriginal, msg);
|
|
160
|
+
}
|
|
161
|
+
expect(graphOriginal.nodes['cube']?.color).toBe('green');
|
|
162
|
+
|
|
163
|
+
// Apply buggy coalescing
|
|
164
|
+
const coalescedBuggy = coalesceBuggy(messages);
|
|
165
|
+
console.log('Buggy coalescing:');
|
|
166
|
+
coalescedBuggy.forEach(msg => {
|
|
167
|
+
console.log(` session=${msg.sessionId}, lamport=${msg.lamportTime}, ops=${msg.ops.length}`);
|
|
168
|
+
msg.ops.forEach(op => console.log(` ${JSON.stringify(op)}`));
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
let graphBuggy = graph;
|
|
172
|
+
for (const msg of coalescedBuggy) {
|
|
173
|
+
graphBuggy = applyMessage(graphBuggy, msg);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// THIS WILL FAIL - buggy version might produce wrong result
|
|
177
|
+
console.log(`Buggy result: ${graphBuggy.nodes['cube']?.color}`);
|
|
178
|
+
|
|
179
|
+
// Apply correct coalescing
|
|
180
|
+
const coalescedCorrect = coalesceCorrect(messages);
|
|
181
|
+
console.log('\nCorrect coalescing:');
|
|
182
|
+
coalescedCorrect.forEach(msg => {
|
|
183
|
+
console.log(` session=${msg.sessionId}, lamport=${msg.lamportTime}, ops=${msg.ops.length}`);
|
|
184
|
+
msg.ops.forEach(op => console.log(` ${JSON.stringify(op)}`));
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
let graphCorrect = graph;
|
|
188
|
+
for (const msg of coalescedCorrect) {
|
|
189
|
+
graphCorrect = applyMessage(graphCorrect, msg);
|
|
190
|
+
}
|
|
191
|
+
console.log(`Correct result: ${graphCorrect.nodes['cube']?.color}`);
|
|
192
|
+
|
|
193
|
+
// Both should produce "green"
|
|
194
|
+
expect(graphCorrect.nodes['cube']?.color).toBe('green');
|
|
195
|
+
|
|
196
|
+
// This expectation will FAIL with buggy coalescing
|
|
197
|
+
expect(graphBuggy.nodes['cube']?.color).toBe('green');
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
it('demonstrates the metadata inheritance problem', () => {
|
|
201
|
+
const messages: CRDTMessage[] = [
|
|
202
|
+
{
|
|
203
|
+
id: 'msg1',
|
|
204
|
+
sessionId: 'alice',
|
|
205
|
+
clock: { alice: 10 },
|
|
206
|
+
lamportTime: 10,
|
|
207
|
+
timestamp: 1000,
|
|
208
|
+
ops: [{ otype: 'number.set', key: 'x', path: 'value', value: 1 }],
|
|
209
|
+
},
|
|
210
|
+
{
|
|
211
|
+
id: 'msg2',
|
|
212
|
+
sessionId: 'bob',
|
|
213
|
+
clock: { alice: 10, bob: 15 },
|
|
214
|
+
lamportTime: 15,
|
|
215
|
+
timestamp: 1500,
|
|
216
|
+
ops: [{ otype: 'number.set', key: 'x', path: 'value', value: 2 }],
|
|
217
|
+
},
|
|
218
|
+
{
|
|
219
|
+
id: 'msg3',
|
|
220
|
+
sessionId: 'alice',
|
|
221
|
+
clock: { alice: 20, bob: 15 },
|
|
222
|
+
lamportTime: 20,
|
|
223
|
+
timestamp: 2000,
|
|
224
|
+
ops: [{ otype: 'number.set', key: 'x', path: 'value', value: 3 }],
|
|
225
|
+
},
|
|
226
|
+
];
|
|
227
|
+
|
|
228
|
+
const coalesced = coalesceBuggy(messages);
|
|
229
|
+
|
|
230
|
+
// Find alice's coalesced message
|
|
231
|
+
const aliceMsg = coalesced.find(m => m.sessionId === 'alice')!;
|
|
232
|
+
|
|
233
|
+
// Bug: ALL of Alice's ops inherit lamportTime=20 from msg3
|
|
234
|
+
expect(aliceMsg.lamportTime).toBe(20);
|
|
235
|
+
expect(aliceMsg.ops).toHaveLength(2); // Both set operations
|
|
236
|
+
|
|
237
|
+
// When these ops are applied, they BOTH get lamportTime=20
|
|
238
|
+
// This means the first operation (value=1) incorrectly gets lamport=20
|
|
239
|
+
// instead of its original lamport=10
|
|
240
|
+
|
|
241
|
+
console.log('Alice coalesced message:', {
|
|
242
|
+
lamportTime: aliceMsg.lamportTime,
|
|
243
|
+
ops: aliceMsg.ops,
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
// The problem: Both ops get lamport=20, so LWW resolution is broken
|
|
247
|
+
});
|
|
248
|
+
});
|