@vuer-ai/vuer-rtc-server 0.2.3 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env +1 -1
- package/README.md +56 -0
- package/dist/archive/ArchivalService.js +1 -1
- package/dist/archive/ArchivalService.js.map +1 -1
- package/dist/broker/InMemoryBroker.d.ts +2 -2
- package/dist/broker/InMemoryBroker.d.ts.map +1 -1
- package/dist/broker/InMemoryBroker.js +4 -4
- package/dist/broker/InMemoryBroker.js.map +1 -1
- package/dist/broker/types.d.ts +3 -3
- package/dist/broker/types.d.ts.map +1 -1
- package/dist/journal/CoalescingService.d.ts.map +1 -1
- package/dist/journal/CoalescingService.js +18 -208
- package/dist/journal/CoalescingService.js.map +1 -1
- package/dist/journal/GraphJournalService.d.ts +127 -0
- package/dist/journal/GraphJournalService.d.ts.map +1 -0
- package/dist/journal/GraphJournalService.js +491 -0
- package/dist/journal/GraphJournalService.js.map +1 -0
- package/dist/journal/JournalRLE.d.ts +2 -2
- package/dist/journal/JournalRLE.js +14 -14
- package/dist/journal/JournalRLE.js.map +1 -1
- package/dist/journal/JournalRepository.js +7 -7
- package/dist/journal/JournalRepository.js.map +1 -1
- package/dist/journal/JournalService.d.ts.map +1 -1
- package/dist/journal/JournalService.js +6 -40
- package/dist/journal/JournalService.js.map +1 -1
- package/dist/journal/RLECompression.d.ts +9 -9
- package/dist/journal/RLECompression.d.ts.map +1 -1
- package/dist/journal/RLECompression.js +22 -22
- package/dist/journal/RLECompression.js.map +1 -1
- package/dist/journal/TextJournalService.d.ts +98 -0
- package/dist/journal/TextJournalService.d.ts.map +1 -0
- package/dist/journal/TextJournalService.js +401 -0
- package/dist/journal/TextJournalService.js.map +1 -0
- package/dist/journal/index.d.ts +3 -1
- package/dist/journal/index.d.ts.map +1 -1
- package/dist/journal/index.js +4 -1
- package/dist/journal/index.js.map +1 -1
- package/dist/journal/rle-demo.js +11 -11
- package/dist/journal/rle-demo.js.map +1 -1
- package/dist/serve.d.ts +29 -11
- package/dist/serve.d.ts.map +1 -1
- package/dist/serve.js +558 -93
- package/dist/serve.js.map +1 -1
- package/dist/transport/RTCServer.d.ts +2 -2
- package/dist/transport/RTCServer.d.ts.map +1 -1
- package/dist/transport/RTCServer.js +22 -22
- package/dist/transport/RTCServer.js.map +1 -1
- package/docs/API.md +642 -0
- package/examples/compression-example.ts +3 -3
- package/package.json +2 -2
- package/prisma/schema.prisma +124 -6
- package/src/archive/ArchivalService.ts +1 -1
- package/src/broker/InMemoryBroker.ts +4 -4
- package/src/broker/types.ts +3 -3
- package/src/journal/CoalescingService.ts +18 -235
- package/src/journal/{JournalService.ts → GraphJournalService.ts} +34 -74
- package/src/journal/JournalRLE.ts +15 -15
- package/src/journal/JournalRepository.ts +7 -7
- package/src/journal/RLECompression.ts +24 -24
- package/src/journal/TextJournalService.ts +483 -0
- package/src/journal/index.ts +10 -2
- package/src/journal/rle-demo.ts +11 -11
- package/src/serve.ts +598 -94
- package/src/transport/RTCServer.ts +23 -23
- package/tests/benchmark/journal-optimization-benchmark.test.ts +14 -14
- package/tests/compression/compression.test.ts +8 -8
- package/tests/demo.ts +88 -88
- package/tests/e2e/convergence.test.ts +9 -9
- package/tests/e2e/helpers/assertions.ts +22 -0
- package/tests/e2e/helpers/createTestServer.ts +4 -4
- package/tests/e2e/latency.test.ts +47 -41
- package/tests/e2e/packet-loss.test.ts +6 -6
- package/tests/e2e/relay.test.ts +9 -9
- package/tests/e2e/sync-perf.test.ts +5 -5
- package/tests/e2e/sync-reconciliation.test.ts +6 -6
- package/tests/e2e/text-sync.test.ts +14 -14
- package/tests/e2e/tombstone-convergence.test.ts +22 -22
- package/tests/fixtures/array-ops.jsonl +6 -6
- package/tests/fixtures/boolean-ops.jsonl +6 -6
- package/tests/fixtures/color-ops.jsonl +4 -4
- package/tests/fixtures/edit-buffer.jsonl +3 -3
- package/tests/fixtures/messages.jsonl +4 -4
- package/tests/fixtures/node-ops.jsonl +6 -6
- package/tests/fixtures/number-ops.jsonl +7 -7
- package/tests/fixtures/object-ops.jsonl +4 -4
- package/tests/fixtures/operations.jsonl +7 -7
- package/tests/fixtures/string-ops.jsonl +4 -4
- package/tests/fixtures/undo-redo.jsonl +3 -3
- package/tests/fixtures/vector-ops.jsonl +9 -9
- package/tests/integration/repositories.test.ts +8 -9
- package/tests/journal/compaction-load-bug.test.ts +31 -31
- package/tests/journal/compaction.test.ts +26 -26
- package/tests/journal/journal-rle.test.ts +38 -38
- package/tests/journal/journal-service.test.ts +13 -13
- package/tests/journal/lww-ordering-bug.test.ts +39 -39
- package/tests/journal/rle-compression.test.ts +71 -71
- package/tests/journal/text-coalescing.test.ts +34 -34
- package/tests/test-data/datatypes.ts +85 -85
- package/tests/test-data/operations-example.ts +62 -62
- package/tests/test-data/scene-example.ts +11 -11
- package/tests/unit/operations.test.ts +7 -7
- package/tests/unit/s3-compression.test.ts +5 -3
- package/tests/unit/vectorClock.test.ts +2 -2
- package/tests/journal/multi-session-coalescing.test.ts +0 -871
|
@@ -41,7 +41,7 @@ function toUint8Array(data: BinaryData | string): Uint8Array | null {
|
|
|
41
41
|
// function summarizeOps(msg: CRDTMessage): string {
|
|
42
42
|
// if (!msg.ops || msg.ops.length === 0) return 'empty';
|
|
43
43
|
// const first = msg.ops[0];
|
|
44
|
-
// const summary = `${first.
|
|
44
|
+
// const summary = `${first.ot} ${first.key}.${first.path}`;
|
|
45
45
|
// return msg.ops.length > 1 ? `${summary} (+${msg.ops.length - 1} more)` : summary;
|
|
46
46
|
// }
|
|
47
47
|
|
|
@@ -65,8 +65,8 @@ export class RTCServer {
|
|
|
65
65
|
*
|
|
66
66
|
* Call this once per connection after the WS upgrade completes.
|
|
67
67
|
*/
|
|
68
|
-
handleConnection(ws: MinimalWebSocket, roomId: string,
|
|
69
|
-
// console.log(`[connect] room=${roomId}
|
|
68
|
+
handleConnection(ws: MinimalWebSocket, roomId: string, client: string): void {
|
|
69
|
+
// console.log(`[connect] room=${roomId} client=${client}`);
|
|
70
70
|
|
|
71
71
|
// Track this socket for control broadcasts (e.g. room-reset)
|
|
72
72
|
if (!this.roomSockets.has(roomId)) {
|
|
@@ -75,8 +75,8 @@ export class RTCServer {
|
|
|
75
75
|
this.roomSockets.get(roomId)!.add(ws);
|
|
76
76
|
|
|
77
77
|
// Register member as connected
|
|
78
|
-
this.broker.setMember(roomId,
|
|
79
|
-
|
|
78
|
+
this.broker.setMember(roomId, client, {
|
|
79
|
+
client,
|
|
80
80
|
vectorClock: {},
|
|
81
81
|
lastSeen: Date.now(),
|
|
82
82
|
connected: true,
|
|
@@ -85,7 +85,7 @@ export class RTCServer {
|
|
|
85
85
|
// Subscribe to room broadcasts — forward to this client
|
|
86
86
|
const unsubscribe = this.broker.subscribe(roomId, (msg: SequencedMessage) => {
|
|
87
87
|
// Don't echo back to the sender; they get an ack instead
|
|
88
|
-
if (msg.
|
|
88
|
+
if (msg.client === client) return;
|
|
89
89
|
ws.send(serialize({ mtype: 'broadcast', msg }));
|
|
90
90
|
});
|
|
91
91
|
|
|
@@ -106,12 +106,12 @@ export class RTCServer {
|
|
|
106
106
|
}
|
|
107
107
|
|
|
108
108
|
if (data.mtype === 'heartbeat') {
|
|
109
|
-
this.handleHeartbeat(roomId,
|
|
109
|
+
this.handleHeartbeat(roomId, client, data.vectorClock ?? {});
|
|
110
110
|
return;
|
|
111
111
|
}
|
|
112
112
|
|
|
113
113
|
if (data.mtype === 'sync') {
|
|
114
|
-
this.handleSyncRequest(ws, roomId,
|
|
114
|
+
this.handleSyncRequest(ws, roomId, client, data);
|
|
115
115
|
return;
|
|
116
116
|
}
|
|
117
117
|
|
|
@@ -124,11 +124,11 @@ export class RTCServer {
|
|
|
124
124
|
|
|
125
125
|
// Cleanup on close
|
|
126
126
|
ws.on('close', () => {
|
|
127
|
-
// console.log(`[disconnect] room=${roomId}
|
|
127
|
+
// console.log(`[disconnect] room=${roomId} client=${client}`);
|
|
128
128
|
unsubscribe();
|
|
129
129
|
this.roomSockets.get(roomId)?.delete(ws);
|
|
130
|
-
this.broker.setMember(roomId,
|
|
131
|
-
|
|
130
|
+
this.broker.setMember(roomId, client, {
|
|
131
|
+
client,
|
|
132
132
|
vectorClock: {},
|
|
133
133
|
lastSeen: Date.now(),
|
|
134
134
|
connected: false,
|
|
@@ -209,14 +209,14 @@ export class RTCServer {
|
|
|
209
209
|
*/
|
|
210
210
|
async getRoomState(roomId: string): Promise<{
|
|
211
211
|
historyCount: number;
|
|
212
|
-
members: Array<{
|
|
212
|
+
members: Array<{ client: string; connected: boolean; lastSeen: number }>;
|
|
213
213
|
}> {
|
|
214
214
|
const history = this.messageHistory.get(roomId);
|
|
215
215
|
const members = await this.broker.getMembers(roomId);
|
|
216
216
|
return {
|
|
217
217
|
historyCount: history?.length ?? 0,
|
|
218
218
|
members: Array.from(members.values()).map(m => ({
|
|
219
|
-
|
|
219
|
+
client: m.client,
|
|
220
220
|
connected: m.connected,
|
|
221
221
|
lastSeen: m.lastSeen,
|
|
222
222
|
})),
|
|
@@ -229,7 +229,7 @@ export class RTCServer {
|
|
|
229
229
|
roomId: string,
|
|
230
230
|
msg: CRDTMessage,
|
|
231
231
|
): Promise<void> {
|
|
232
|
-
// console.log(`[msg] room=${roomId}
|
|
232
|
+
// console.log(`[msg] room=${roomId} client=${msg.client} lamport=${msg.lt} ${summarizeOps(msg)}`);
|
|
233
233
|
|
|
234
234
|
// Persist / dedup if journal is available
|
|
235
235
|
if (this.journal) {
|
|
@@ -280,9 +280,9 @@ export class RTCServer {
|
|
|
280
280
|
* (either in its journal or compacted into the snapshot).
|
|
281
281
|
*/
|
|
282
282
|
private isKnownByVectorClock(msg: CRDTMessage, clientClock: VectorClock): boolean {
|
|
283
|
-
const authorSeq = msg.clock[msg.
|
|
283
|
+
const authorSeq = msg.clock[msg.client];
|
|
284
284
|
if (authorSeq === undefined) return false;
|
|
285
|
-
return (clientClock[msg.
|
|
285
|
+
return (clientClock[msg.client] ?? 0) >= authorSeq;
|
|
286
286
|
}
|
|
287
287
|
|
|
288
288
|
/**
|
|
@@ -292,12 +292,12 @@ export class RTCServer {
|
|
|
292
292
|
private handleSyncRequest(
|
|
293
293
|
ws: MinimalWebSocket,
|
|
294
294
|
roomId: string,
|
|
295
|
-
|
|
295
|
+
client: string,
|
|
296
296
|
data: { mtype: 'sync'; vectorClock?: VectorClock; filter: Uint8Array | string; count: number },
|
|
297
297
|
): void {
|
|
298
298
|
const history = this.messageHistory.get(roomId);
|
|
299
299
|
if (!history || history.length === 0) {
|
|
300
|
-
// console.log(`[sync] room=${roomId}
|
|
300
|
+
// console.log(`[sync] room=${roomId} client=${client} no history`);
|
|
301
301
|
return;
|
|
302
302
|
}
|
|
303
303
|
|
|
@@ -310,7 +310,7 @@ export class RTCServer {
|
|
|
310
310
|
let retransmitted = 0;
|
|
311
311
|
for (const msg of history) {
|
|
312
312
|
// Skip the client's own messages (they already have them)
|
|
313
|
-
if (msg.
|
|
313
|
+
if (msg.client === client) continue;
|
|
314
314
|
// Skip messages covered by the client's snapshot (compacted)
|
|
315
315
|
if (this.isKnownByVectorClock(msg, clientClock)) continue;
|
|
316
316
|
// Check the bloom filter for remaining (uncompacted) messages
|
|
@@ -320,16 +320,16 @@ export class RTCServer {
|
|
|
320
320
|
}
|
|
321
321
|
}
|
|
322
322
|
|
|
323
|
-
// console.log(`[sync] room=${roomId}
|
|
323
|
+
// console.log(`[sync] room=${roomId} client=${client} clientMsgs=${data.count} history=${history.length} retransmitted=${retransmitted}`);
|
|
324
324
|
}
|
|
325
325
|
|
|
326
326
|
private handleHeartbeat(
|
|
327
327
|
roomId: string,
|
|
328
|
-
|
|
328
|
+
client: string,
|
|
329
329
|
vectorClock: Record<string, number>,
|
|
330
330
|
): void {
|
|
331
|
-
this.broker.setMember(roomId,
|
|
332
|
-
|
|
331
|
+
this.broker.setMember(roomId, client, {
|
|
332
|
+
client,
|
|
333
333
|
vectorClock,
|
|
334
334
|
lastSeen: Date.now(),
|
|
335
335
|
connected: true,
|
|
@@ -32,21 +32,21 @@ import {
|
|
|
32
32
|
|
|
33
33
|
function createMessage(
|
|
34
34
|
id: string,
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
client: string,
|
|
36
|
+
lt: number,
|
|
37
37
|
ops: any[] = []
|
|
38
38
|
): CRDTMessage {
|
|
39
39
|
return {
|
|
40
40
|
id,
|
|
41
|
-
|
|
42
|
-
clock: { [
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
client,
|
|
42
|
+
clock: { [client]: lt },
|
|
43
|
+
lt,
|
|
44
|
+
ts: Date.now() / 1000,
|
|
45
45
|
ops: ops.length > 0 ? ops : [{
|
|
46
46
|
key: 'cube-1',
|
|
47
|
-
|
|
47
|
+
ot: 'vector3.set',
|
|
48
48
|
path: 'position',
|
|
49
|
-
value: [
|
|
49
|
+
value: [lt, 0, 0],
|
|
50
50
|
}],
|
|
51
51
|
};
|
|
52
52
|
}
|
|
@@ -72,7 +72,7 @@ function generateScenario(config: ScenarioConfig): CRDTMessage[] {
|
|
|
72
72
|
for (let i = 0; i < messageCount; i++) {
|
|
73
73
|
messages.push(
|
|
74
74
|
createMessage(`msg-${i}`, 'session-1', i, [
|
|
75
|
-
{ key: 'doc',
|
|
75
|
+
{ key: 'doc', ot: 'text.insert', path: 'content', value: 'a' },
|
|
76
76
|
])
|
|
77
77
|
);
|
|
78
78
|
}
|
|
@@ -90,7 +90,7 @@ function generateScenario(config: ScenarioConfig): CRDTMessage[] {
|
|
|
90
90
|
`msg-${lamport}`,
|
|
91
91
|
`session-${agent}`,
|
|
92
92
|
lamport,
|
|
93
|
-
[{ key: 'doc',
|
|
93
|
+
[{ key: 'doc', ot: 'text.insert', path: 'content', value: 'x' }]
|
|
94
94
|
)
|
|
95
95
|
);
|
|
96
96
|
lamport++;
|
|
@@ -102,10 +102,10 @@ function generateScenario(config: ScenarioConfig): CRDTMessage[] {
|
|
|
102
102
|
case 'alternating': {
|
|
103
103
|
// Worst case: different agent each message
|
|
104
104
|
for (let i = 0; i < messageCount; i++) {
|
|
105
|
-
const
|
|
105
|
+
const client = `session-${i % agentCount}`;
|
|
106
106
|
messages.push(
|
|
107
|
-
createMessage(`msg-${i}`,
|
|
108
|
-
{ key: 'doc',
|
|
107
|
+
createMessage(`msg-${i}`, client, i, [
|
|
108
|
+
{ key: 'doc', ot: 'text.insert', path: 'content', value: 'y' },
|
|
109
109
|
])
|
|
110
110
|
);
|
|
111
111
|
}
|
|
@@ -124,7 +124,7 @@ function generateScenario(config: ScenarioConfig): CRDTMessage[] {
|
|
|
124
124
|
`msg-${lamport}`,
|
|
125
125
|
`session-${agent}`,
|
|
126
126
|
lamport,
|
|
127
|
-
[{ key: 'doc',
|
|
127
|
+
[{ key: 'doc', ot: 'text.insert', path: 'content', value: 'z' }]
|
|
128
128
|
)
|
|
129
129
|
);
|
|
130
130
|
lamport++;
|
|
@@ -27,7 +27,7 @@ describe('CompressionUtils', () => {
|
|
|
27
27
|
rootKey: 'root',
|
|
28
28
|
},
|
|
29
29
|
vectorClock: { session1: 10 },
|
|
30
|
-
|
|
30
|
+
lt: 10,
|
|
31
31
|
journalIndex: 10,
|
|
32
32
|
};
|
|
33
33
|
|
|
@@ -67,7 +67,7 @@ describe('CompressionUtils', () => {
|
|
|
67
67
|
session2: 85,
|
|
68
68
|
session3: 92,
|
|
69
69
|
},
|
|
70
|
-
|
|
70
|
+
lt: 100,
|
|
71
71
|
journalIndex: 100,
|
|
72
72
|
};
|
|
73
73
|
|
|
@@ -105,7 +105,7 @@ describe('CompressionUtils', () => {
|
|
|
105
105
|
rootKey: 'root',
|
|
106
106
|
},
|
|
107
107
|
vectorClock: { session1: 5 },
|
|
108
|
-
|
|
108
|
+
lt: 5,
|
|
109
109
|
journalIndex: 5,
|
|
110
110
|
};
|
|
111
111
|
|
|
@@ -143,7 +143,7 @@ describe('CompressionUtils', () => {
|
|
|
143
143
|
vectorClock: Object.fromEntries(
|
|
144
144
|
Array.from({ length: 10 }, (_, i) => [`session${i}`, i * 10])
|
|
145
145
|
),
|
|
146
|
-
|
|
146
|
+
lt: 100,
|
|
147
147
|
journalIndex: 100,
|
|
148
148
|
};
|
|
149
149
|
|
|
@@ -176,7 +176,7 @@ describe('CompressionUtils', () => {
|
|
|
176
176
|
rootKey: 'root',
|
|
177
177
|
},
|
|
178
178
|
vectorClock: {},
|
|
179
|
-
|
|
179
|
+
lt: 0,
|
|
180
180
|
journalIndex: 0,
|
|
181
181
|
};
|
|
182
182
|
|
|
@@ -207,7 +207,7 @@ describe('CompressionUtils', () => {
|
|
|
207
207
|
rootKey: 'root',
|
|
208
208
|
},
|
|
209
209
|
vectorClock: { s1: 50, s2: 50, s3: 50 },
|
|
210
|
-
|
|
210
|
+
lt: 50,
|
|
211
211
|
journalIndex: 50,
|
|
212
212
|
};
|
|
213
213
|
|
|
@@ -306,7 +306,7 @@ describe('CompressionUtils', () => {
|
|
|
306
306
|
vectorClock: Object.fromEntries(
|
|
307
307
|
Array.from({ length: 50 }, (_, i) => [`session-${i}`, 1000 + i * 100])
|
|
308
308
|
),
|
|
309
|
-
|
|
309
|
+
lt: 6000,
|
|
310
310
|
journalIndex: 6000,
|
|
311
311
|
};
|
|
312
312
|
|
|
@@ -328,7 +328,7 @@ describe('CompressionUtils', () => {
|
|
|
328
328
|
rootKey: 'root',
|
|
329
329
|
},
|
|
330
330
|
vectorClock: {},
|
|
331
|
-
|
|
331
|
+
lt: 0,
|
|
332
332
|
journalIndex: 0,
|
|
333
333
|
};
|
|
334
334
|
|