cojson 0.19.19 → 0.19.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +10 -0
- package/dist/config.js.map +1 -1
- package/dist/exports.d.ts +7 -1
- package/dist/exports.d.ts.map +1 -1
- package/dist/exports.js +4 -1
- package/dist/exports.js.map +1 -1
- package/dist/queue/IncomingMessagesQueue.d.ts +6 -7
- package/dist/queue/IncomingMessagesQueue.d.ts.map +1 -1
- package/dist/queue/IncomingMessagesQueue.js +7 -30
- package/dist/queue/IncomingMessagesQueue.js.map +1 -1
- package/dist/queue/LinkedList.d.ts +1 -1
- package/dist/queue/LinkedList.d.ts.map +1 -1
- package/dist/queue/LinkedList.js.map +1 -1
- package/dist/queue/StorageStreamingQueue.d.ts +43 -0
- package/dist/queue/StorageStreamingQueue.d.ts.map +1 -0
- package/dist/queue/StorageStreamingQueue.js +70 -0
- package/dist/queue/StorageStreamingQueue.js.map +1 -0
- package/dist/storage/storageSync.d.ts +8 -2
- package/dist/storage/storageSync.d.ts.map +1 -1
- package/dist/storage/storageSync.js +56 -44
- package/dist/storage/storageSync.js.map +1 -1
- package/dist/storage/types.d.ts +2 -0
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/sync.d.ts +17 -0
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +74 -5
- package/dist/sync.js.map +1 -1
- package/dist/tests/IncomingMessagesQueue.test.js +4 -150
- package/dist/tests/IncomingMessagesQueue.test.js.map +1 -1
- package/dist/tests/StorageStreamingQueue.test.d.ts +2 -0
- package/dist/tests/StorageStreamingQueue.test.d.ts.map +1 -0
- package/dist/tests/StorageStreamingQueue.test.js +213 -0
- package/dist/tests/StorageStreamingQueue.test.js.map +1 -0
- package/dist/tests/SyncManager.processQueues.test.d.ts +2 -0
- package/dist/tests/SyncManager.processQueues.test.d.ts.map +1 -0
- package/dist/tests/SyncManager.processQueues.test.js +208 -0
- package/dist/tests/SyncManager.processQueues.test.js.map +1 -0
- package/dist/tests/setup.d.ts +2 -0
- package/dist/tests/setup.d.ts.map +1 -0
- package/dist/tests/setup.js +4 -0
- package/dist/tests/setup.js.map +1 -0
- package/dist/tests/sync.garbageCollection.test.js.map +1 -1
- package/dist/tests/sync.mesh.test.js +19 -19
- package/dist/tests/sync.storage.test.js +176 -20
- package/dist/tests/sync.storage.test.js.map +1 -1
- package/dist/tests/sync.test.js +1 -1
- package/dist/tests/sync.test.js.map +1 -1
- package/dist/tests/testUtils.d.ts +2 -2
- package/dist/tests/testUtils.js +2 -2
- package/dist/tests/testUtils.js.map +1 -1
- package/package.json +4 -4
- package/src/config.ts +13 -0
- package/src/exports.ts +6 -0
- package/src/queue/IncomingMessagesQueue.ts +7 -39
- package/src/queue/LinkedList.ts +1 -1
- package/src/queue/StorageStreamingQueue.ts +96 -0
- package/src/storage/storageSync.ts +99 -55
- package/src/storage/types.ts +3 -0
- package/src/sync.ts +84 -5
- package/src/tests/IncomingMessagesQueue.test.ts +4 -206
- package/src/tests/StorageStreamingQueue.test.ts +276 -0
- package/src/tests/SyncManager.processQueues.test.ts +287 -0
- package/src/tests/setup.ts +4 -0
- package/src/tests/sync.garbageCollection.test.ts +1 -3
- package/src/tests/sync.mesh.test.ts +19 -19
- package/src/tests/sync.storage.test.ts +224 -32
- package/src/tests/sync.test.ts +1 -9
- package/src/tests/testUtils.ts +2 -2
- package/vitest.config.ts +1 -0
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
|
|
2
|
+
import { CO_VALUE_PRIORITY } from "../priority.js";
|
|
3
|
+
import { StorageStreamingQueue } from "../queue/StorageStreamingQueue.js";
|
|
4
|
+
import {
|
|
5
|
+
SyncMessagesLog,
|
|
6
|
+
loadCoValueOrFail,
|
|
7
|
+
setupTestNode,
|
|
8
|
+
waitFor,
|
|
9
|
+
} from "./testUtils.js";
|
|
10
|
+
|
|
11
|
+
describe("SyncManager.processQueues", () => {
|
|
12
|
+
let jazzCloud: ReturnType<typeof setupTestNode>;
|
|
13
|
+
|
|
14
|
+
beforeEach(async () => {
|
|
15
|
+
SyncMessagesLog.clear();
|
|
16
|
+
jazzCloud = setupTestNode({
|
|
17
|
+
isSyncServer: true,
|
|
18
|
+
});
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
describe("incoming messages processing", () => {
|
|
22
|
+
test("should process incoming messages from peers", async () => {
|
|
23
|
+
const client = setupTestNode();
|
|
24
|
+
client.connectToSyncServer();
|
|
25
|
+
|
|
26
|
+
const group = jazzCloud.node.createGroup();
|
|
27
|
+
const map = group.createMap();
|
|
28
|
+
map.set("hello", "world", "trusting");
|
|
29
|
+
|
|
30
|
+
const mapOnClient = await loadCoValueOrFail(client.node, map.id);
|
|
31
|
+
expect(mapOnClient.get("hello")).toEqual("world");
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("should process multiple messages in sequence", async () => {
|
|
35
|
+
const client = setupTestNode();
|
|
36
|
+
client.connectToSyncServer();
|
|
37
|
+
|
|
38
|
+
const group = jazzCloud.node.createGroup();
|
|
39
|
+
const map1 = group.createMap();
|
|
40
|
+
const map2 = group.createMap();
|
|
41
|
+
const map3 = group.createMap();
|
|
42
|
+
|
|
43
|
+
map1.set("key", "value1", "trusting");
|
|
44
|
+
map2.set("key", "value2", "trusting");
|
|
45
|
+
map3.set("key", "value3", "trusting");
|
|
46
|
+
|
|
47
|
+
const [loadedMap1, loadedMap2, loadedMap3] = await Promise.all([
|
|
48
|
+
loadCoValueOrFail(client.node, map1.id),
|
|
49
|
+
loadCoValueOrFail(client.node, map2.id),
|
|
50
|
+
loadCoValueOrFail(client.node, map3.id),
|
|
51
|
+
]);
|
|
52
|
+
|
|
53
|
+
expect(loadedMap1.get("key")).toEqual("value1");
|
|
54
|
+
expect(loadedMap2.get("key")).toEqual("value2");
|
|
55
|
+
expect(loadedMap3.get("key")).toEqual("value3");
|
|
56
|
+
});
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
describe("storage streaming processing", () => {
|
|
60
|
+
test("should process storage streaming callbacks", async () => {
|
|
61
|
+
const client = setupTestNode();
|
|
62
|
+
client.connectToSyncServer();
|
|
63
|
+
const { storage } = client.addStorage();
|
|
64
|
+
|
|
65
|
+
const group = jazzCloud.node.createGroup();
|
|
66
|
+
const map = group.createMap();
|
|
67
|
+
map.set("hello", "world", "trusting");
|
|
68
|
+
|
|
69
|
+
// First load to populate storage
|
|
70
|
+
await loadCoValueOrFail(client.node, map.id);
|
|
71
|
+
|
|
72
|
+
// Restart and load from storage
|
|
73
|
+
client.restart();
|
|
74
|
+
client.connectToSyncServer();
|
|
75
|
+
client.addStorage({ storage });
|
|
76
|
+
|
|
77
|
+
SyncMessagesLog.clear();
|
|
78
|
+
|
|
79
|
+
const mapOnClient = await loadCoValueOrFail(client.node, map.id);
|
|
80
|
+
expect(mapOnClient.get("hello")).toEqual("world");
|
|
81
|
+
|
|
82
|
+
// Verify content came from storage
|
|
83
|
+
const storageMessages = SyncMessagesLog.messages.filter(
|
|
84
|
+
(msg) => msg.from === "storage" || msg.to === "storage",
|
|
85
|
+
);
|
|
86
|
+
expect(storageMessages.length).toBeGreaterThan(0);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("should invoke streaming callbacks when pulled", async () => {
|
|
90
|
+
const client = setupTestNode();
|
|
91
|
+
const { storage } = client.addStorage();
|
|
92
|
+
|
|
93
|
+
const callback = vi.fn();
|
|
94
|
+
storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
|
|
95
|
+
storage.streamingQueue?.emit();
|
|
96
|
+
|
|
97
|
+
// Wait for processQueues to run
|
|
98
|
+
await waitFor(() => callback.mock.calls.length > 0);
|
|
99
|
+
|
|
100
|
+
expect(callback).toHaveBeenCalledTimes(1);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("should process MEDIUM priority before LOW priority", async () => {
|
|
104
|
+
const client = setupTestNode();
|
|
105
|
+
const { storage } = client.addStorage();
|
|
106
|
+
|
|
107
|
+
const order: string[] = [];
|
|
108
|
+
const lowCallback = () => order.push("low");
|
|
109
|
+
const mediumCallback = () => order.push("medium");
|
|
110
|
+
const highCallback = () => order.push("high");
|
|
111
|
+
|
|
112
|
+
// Push LOW first, then MEDIUM
|
|
113
|
+
storage.streamingQueue?.push(lowCallback, CO_VALUE_PRIORITY.LOW);
|
|
114
|
+
storage.streamingQueue?.push(mediumCallback, CO_VALUE_PRIORITY.MEDIUM);
|
|
115
|
+
storage.streamingQueue?.push(highCallback, CO_VALUE_PRIORITY.HIGH);
|
|
116
|
+
storage.streamingQueue?.emit();
|
|
117
|
+
|
|
118
|
+
// Wait for both to be processed
|
|
119
|
+
await waitFor(() => order.length === 3);
|
|
120
|
+
|
|
121
|
+
// MEDIUM should be processed first
|
|
122
|
+
expect(order).toEqual(["high", "medium", "low"]);
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
describe("unified scheduling", () => {
|
|
127
|
+
test("should process both incoming messages and storage streaming", async () => {
|
|
128
|
+
const client = setupTestNode();
|
|
129
|
+
client.connectToSyncServer();
|
|
130
|
+
const { storage } = client.addStorage();
|
|
131
|
+
|
|
132
|
+
const group = jazzCloud.node.createGroup();
|
|
133
|
+
const map = group.createMap();
|
|
134
|
+
map.set("hello", "world", "trusting");
|
|
135
|
+
|
|
136
|
+
// Queue a storage streaming callback
|
|
137
|
+
const streamingCallback = vi.fn();
|
|
138
|
+
storage.streamingQueue?.push(streamingCallback, CO_VALUE_PRIORITY.MEDIUM);
|
|
139
|
+
storage.streamingQueue?.emit();
|
|
140
|
+
|
|
141
|
+
// Load from server (incoming messages)
|
|
142
|
+
const mapOnClient = await loadCoValueOrFail(client.node, map.id);
|
|
143
|
+
|
|
144
|
+
expect(mapOnClient.get("hello")).toEqual("world");
|
|
145
|
+
expect(streamingCallback).toHaveBeenCalled();
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("should alternate between message queue and storage queue", async () => {
|
|
149
|
+
const client = setupTestNode();
|
|
150
|
+
const { storage } = client.addStorage();
|
|
151
|
+
|
|
152
|
+
const order: string[] = [];
|
|
153
|
+
|
|
154
|
+
// Push multiple storage callbacks
|
|
155
|
+
storage.streamingQueue?.push(
|
|
156
|
+
() => order.push("storage1"),
|
|
157
|
+
CO_VALUE_PRIORITY.MEDIUM,
|
|
158
|
+
);
|
|
159
|
+
storage.streamingQueue?.push(
|
|
160
|
+
() => order.push("storage2"),
|
|
161
|
+
CO_VALUE_PRIORITY.MEDIUM,
|
|
162
|
+
);
|
|
163
|
+
storage.streamingQueue?.emit();
|
|
164
|
+
|
|
165
|
+
// Wait for processing
|
|
166
|
+
await waitFor(() => order.length === 2);
|
|
167
|
+
|
|
168
|
+
expect(order).toContain("storage1");
|
|
169
|
+
expect(order).toContain("storage2");
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
describe("processing flag", () => {
|
|
174
|
+
test("should prevent concurrent processQueues calls", async () => {
|
|
175
|
+
const client = setupTestNode();
|
|
176
|
+
const { storage } = client.addStorage();
|
|
177
|
+
|
|
178
|
+
let concurrentCalls = 0;
|
|
179
|
+
let maxConcurrentCalls = 0;
|
|
180
|
+
|
|
181
|
+
const callback = () => {
|
|
182
|
+
concurrentCalls++;
|
|
183
|
+
maxConcurrentCalls = Math.max(maxConcurrentCalls, concurrentCalls);
|
|
184
|
+
// Simulate some work
|
|
185
|
+
for (let i = 0; i < 1000; i++) {
|
|
186
|
+
Math.random();
|
|
187
|
+
}
|
|
188
|
+
concurrentCalls--;
|
|
189
|
+
};
|
|
190
|
+
|
|
191
|
+
// Push multiple callbacks
|
|
192
|
+
for (let i = 0; i < 10; i++) {
|
|
193
|
+
storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Emit multiple times to trigger multiple processQueues calls
|
|
197
|
+
storage.streamingQueue?.emit();
|
|
198
|
+
storage.streamingQueue?.emit();
|
|
199
|
+
storage.streamingQueue?.emit();
|
|
200
|
+
|
|
201
|
+
// Wait for all to complete
|
|
202
|
+
await waitFor(() => storage.streamingQueue?.isEmpty());
|
|
203
|
+
|
|
204
|
+
// Should never have more than 1 concurrent call
|
|
205
|
+
expect(maxConcurrentCalls).toBe(1);
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
describe("error handling", () => {
|
|
210
|
+
test("should continue processing after storage callback error", async () => {
|
|
211
|
+
const client = setupTestNode();
|
|
212
|
+
const { storage } = client.addStorage();
|
|
213
|
+
|
|
214
|
+
const processed: string[] = [];
|
|
215
|
+
|
|
216
|
+
storage.streamingQueue?.push(() => {
|
|
217
|
+
processed.push("before");
|
|
218
|
+
}, CO_VALUE_PRIORITY.MEDIUM);
|
|
219
|
+
|
|
220
|
+
storage.streamingQueue?.push(() => {
|
|
221
|
+
throw new Error("Test error");
|
|
222
|
+
}, CO_VALUE_PRIORITY.MEDIUM);
|
|
223
|
+
|
|
224
|
+
storage.streamingQueue?.push(() => {
|
|
225
|
+
processed.push("after");
|
|
226
|
+
}, CO_VALUE_PRIORITY.MEDIUM);
|
|
227
|
+
|
|
228
|
+
storage.streamingQueue?.emit();
|
|
229
|
+
|
|
230
|
+
// Wait for processing to complete
|
|
231
|
+
await waitFor(() => storage.streamingQueue?.isEmpty());
|
|
232
|
+
|
|
233
|
+
// Both before and after should be processed despite error
|
|
234
|
+
expect(processed).toContain("before");
|
|
235
|
+
expect(processed).toContain("after");
|
|
236
|
+
});
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
describe("queue triggers", () => {
|
|
240
|
+
test("IncomingMessagesQueue.push should trigger processQueues", async () => {
|
|
241
|
+
const client = setupTestNode();
|
|
242
|
+
client.connectToSyncServer();
|
|
243
|
+
|
|
244
|
+
const group = jazzCloud.node.createGroup();
|
|
245
|
+
const map = group.createMap();
|
|
246
|
+
map.set("hello", "world", "trusting");
|
|
247
|
+
|
|
248
|
+
// Loading should trigger message processing automatically
|
|
249
|
+
const mapOnClient = await loadCoValueOrFail(client.node, map.id);
|
|
250
|
+
expect(mapOnClient.get("hello")).toEqual("world");
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
test("StorageStreamingQueue.emit should trigger processQueues", async () => {
|
|
254
|
+
const client = setupTestNode();
|
|
255
|
+
const { storage } = client.addStorage();
|
|
256
|
+
|
|
257
|
+
const callback = vi.fn();
|
|
258
|
+
storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
|
|
259
|
+
|
|
260
|
+
// Before emit, callback should not be called
|
|
261
|
+
expect(callback).not.toHaveBeenCalled();
|
|
262
|
+
|
|
263
|
+
// After emit, processQueues should be triggered
|
|
264
|
+
storage.streamingQueue?.emit();
|
|
265
|
+
|
|
266
|
+
await waitFor(() => callback.mock.calls.length > 0);
|
|
267
|
+
expect(callback).toHaveBeenCalled();
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
test("setStorage should connect queue listener", async () => {
|
|
271
|
+
const client = setupTestNode();
|
|
272
|
+
|
|
273
|
+
// Before adding storage, there's no queue
|
|
274
|
+
const queueBefore = (
|
|
275
|
+
client.node.syncManager as any
|
|
276
|
+
).getStorageStreamingQueue?.();
|
|
277
|
+
expect(queueBefore).toBeUndefined();
|
|
278
|
+
|
|
279
|
+
// After adding storage, queue should be available
|
|
280
|
+
const { storage } = client.addStorage();
|
|
281
|
+
const queueAfter = (
|
|
282
|
+
client.node.syncManager as any
|
|
283
|
+
).getStorageStreamingQueue?.();
|
|
284
|
+
expect(queueAfter).toBe(storage.streamingQueue);
|
|
285
|
+
});
|
|
286
|
+
});
|
|
287
|
+
});
|
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { beforeEach, describe, expect, test } from "vitest";
|
|
2
2
|
|
|
3
3
|
import { setGarbageCollectorMaxAge } from "../config";
|
|
4
|
-
import { emptyKnownState } from "../exports";
|
|
5
4
|
import {
|
|
6
5
|
SyncMessagesLog,
|
|
7
6
|
TEST_NODE_CONFIG,
|
|
8
7
|
loadCoValueOrFail,
|
|
9
8
|
setupTestNode,
|
|
10
|
-
waitFor,
|
|
11
9
|
} from "./testUtils";
|
|
12
10
|
|
|
13
11
|
// We want to simulate a real world communication that happens asynchronously
|
|
@@ -553,6 +553,14 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
|
|
|
553
553
|
"edge -> core | LOAD Map sessions: header/100",
|
|
554
554
|
"edge -> client | CONTENT Group header: true new: After: 0 New: 5",
|
|
555
555
|
"edge -> client | CONTENT Map header: true new: After: 0 New: 21 expectContentUntil: header/100",
|
|
556
|
+
"storage -> edge | CONTENT Map header: true new: After: 21 New: 21",
|
|
557
|
+
"edge -> client | CONTENT Map header: false new: After: 21 New: 21 expectContentUntil: header/100",
|
|
558
|
+
"storage -> edge | CONTENT Map header: true new: After: 42 New: 21",
|
|
559
|
+
"edge -> client | CONTENT Map header: false new: After: 42 New: 21 expectContentUntil: header/100",
|
|
560
|
+
"storage -> edge | CONTENT Map header: true new: After: 63 New: 21",
|
|
561
|
+
"edge -> client | CONTENT Map header: false new: After: 63 New: 21 expectContentUntil: header/100",
|
|
562
|
+
"storage -> edge | CONTENT Map header: true new: After: 84 New: 16",
|
|
563
|
+
"edge -> client | CONTENT Map header: false new: After: 84 New: 16",
|
|
556
564
|
"core -> storage | LOAD Group sessions: empty",
|
|
557
565
|
"storage -> core | KNOWN Group sessions: empty",
|
|
558
566
|
"core -> edge | KNOWN Group sessions: empty",
|
|
@@ -563,16 +571,20 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
|
|
|
563
571
|
"client -> storage | CONTENT Group header: true new: After: 0 New: 5",
|
|
564
572
|
"client -> edge | KNOWN Map sessions: header/21",
|
|
565
573
|
"client -> storage | CONTENT Map header: true new: After: 0 New: 21",
|
|
566
|
-
"
|
|
567
|
-
"
|
|
574
|
+
"client -> edge | KNOWN Map sessions: header/42",
|
|
575
|
+
"client -> storage | CONTENT Map header: false new: After: 21 New: 21",
|
|
576
|
+
"client -> edge | KNOWN Map sessions: header/63",
|
|
577
|
+
"client -> storage | CONTENT Map header: false new: After: 42 New: 21",
|
|
578
|
+
"client -> edge | KNOWN Map sessions: header/84",
|
|
579
|
+
"client -> storage | CONTENT Map header: false new: After: 63 New: 21",
|
|
580
|
+
"client -> edge | KNOWN Map sessions: header/100",
|
|
581
|
+
"client -> storage | CONTENT Map header: false new: After: 84 New: 16",
|
|
568
582
|
"edge -> core | CONTENT Group header: true new: After: 0 New: 5",
|
|
569
583
|
"edge -> core | CONTENT Map header: true new: After: 0 New: 21 expectContentUntil: header/100",
|
|
570
584
|
"edge -> core | CONTENT Map header: false new: After: 21 New: 21",
|
|
571
|
-
"
|
|
572
|
-
"
|
|
573
|
-
"
|
|
574
|
-
"edge -> core | CONTENT Map header: false new: After: 42 New: 21 expectContentUntil: header/100",
|
|
575
|
-
"edge -> client | CONTENT Map header: false new: After: 42 New: 21 expectContentUntil: header/100",
|
|
585
|
+
"edge -> core | CONTENT Map header: false new: After: 42 New: 21",
|
|
586
|
+
"edge -> core | CONTENT Map header: false new: After: 63 New: 21",
|
|
587
|
+
"edge -> core | CONTENT Map header: false new: After: 84 New: 16",
|
|
576
588
|
"core -> edge | KNOWN Group sessions: header/5",
|
|
577
589
|
"core -> storage | CONTENT Group header: true new: After: 0 New: 5",
|
|
578
590
|
"core -> edge | KNOWN Map sessions: header/21",
|
|
@@ -581,22 +593,10 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
|
|
|
581
593
|
"core -> storage | CONTENT Map header: false new: After: 21 New: 21",
|
|
582
594
|
"core -> edge | KNOWN Map sessions: header/63",
|
|
583
595
|
"core -> storage | CONTENT Map header: false new: After: 42 New: 21",
|
|
584
|
-
"client -> edge | KNOWN Map sessions: header/63",
|
|
585
|
-
"client -> storage | CONTENT Map header: false new: After: 42 New: 21",
|
|
586
|
-
"storage -> edge | CONTENT Map header: true new: After: 63 New: 21",
|
|
587
|
-
"edge -> core | CONTENT Map header: false new: After: 63 New: 21 expectContentUntil: header/100",
|
|
588
|
-
"edge -> client | CONTENT Map header: false new: After: 63 New: 21 expectContentUntil: header/100",
|
|
589
596
|
"core -> edge | KNOWN Map sessions: header/84",
|
|
590
597
|
"core -> storage | CONTENT Map header: false new: After: 63 New: 21",
|
|
591
|
-
"client -> edge | KNOWN Map sessions: header/84",
|
|
592
|
-
"client -> storage | CONTENT Map header: false new: After: 63 New: 21",
|
|
593
|
-
"storage -> edge | CONTENT Map header: true new: After: 84 New: 16",
|
|
594
|
-
"edge -> core | CONTENT Map header: false new: After: 84 New: 16",
|
|
595
|
-
"edge -> client | CONTENT Map header: false new: After: 84 New: 16",
|
|
596
598
|
"core -> edge | KNOWN Map sessions: header/100",
|
|
597
599
|
"core -> storage | CONTENT Map header: false new: After: 84 New: 16",
|
|
598
|
-
"client -> edge | KNOWN Map sessions: header/100",
|
|
599
|
-
"client -> storage | CONTENT Map header: false new: After: 84 New: 16",
|
|
600
600
|
]
|
|
601
601
|
`);
|
|
602
602
|
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
vi,
|
|
9
9
|
} from "vitest";
|
|
10
10
|
|
|
11
|
-
import { emptyKnownState } from "../exports";
|
|
11
|
+
import { cojsonInternals, emptyKnownState } from "../exports";
|
|
12
12
|
import {
|
|
13
13
|
SyncMessagesLog,
|
|
14
14
|
TEST_NODE_CONFIG,
|
|
@@ -24,6 +24,10 @@ import { stableStringify } from "../jsonStringify";
|
|
|
24
24
|
// We want to simulate a real world communication that happens asynchronously
|
|
25
25
|
TEST_NODE_CONFIG.withAsyncPeers = true;
|
|
26
26
|
|
|
27
|
+
beforeEach(() => {
|
|
28
|
+
cojsonInternals.setIncomingMessagesTimeBudget(10_000);
|
|
29
|
+
});
|
|
30
|
+
|
|
27
31
|
describe("client with storage syncs with server", () => {
|
|
28
32
|
let jazzCloud: ReturnType<typeof setupTestNode>;
|
|
29
33
|
|
|
@@ -356,13 +360,6 @@ describe("client syncs with a server with storage", () => {
|
|
|
356
360
|
|
|
357
361
|
await largeMap.core.waitForSync();
|
|
358
362
|
|
|
359
|
-
// Test streaming counter during initial sync
|
|
360
|
-
// The streaming counter should be 0 after the sync is complete
|
|
361
|
-
const streamingCounterAfterSync = await metricReader.getMetricValue(
|
|
362
|
-
"jazz.storage.streaming",
|
|
363
|
-
);
|
|
364
|
-
expect(streamingCounterAfterSync).toBe(0);
|
|
365
|
-
|
|
366
363
|
expect(
|
|
367
364
|
SyncMessagesLog.getMessages({
|
|
368
365
|
Group: group.core,
|
|
@@ -403,31 +400,11 @@ describe("client syncs with a server with storage", () => {
|
|
|
403
400
|
storage,
|
|
404
401
|
});
|
|
405
402
|
|
|
406
|
-
// Test streaming counter before loading the large coValue
|
|
407
|
-
const streamingCounterBeforeLoad = await metricReader.getMetricValue(
|
|
408
|
-
"jazz.storage.streaming",
|
|
409
|
-
);
|
|
410
|
-
expect(streamingCounterBeforeLoad).toBe(0);
|
|
411
|
-
|
|
412
403
|
const promise = loadCoValueOrFail(client.node, largeMap.id);
|
|
413
404
|
|
|
414
|
-
// Test streaming counter during loading (should be 1 during streaming)
|
|
415
|
-
const streamingCounterDuringLoad = await metricReader.getMetricValue(
|
|
416
|
-
"jazz.storage.streaming",
|
|
417
|
-
);
|
|
418
|
-
expect(streamingCounterDuringLoad).toBe(1);
|
|
419
|
-
|
|
420
405
|
const mapOnClient2 = await promise;
|
|
421
406
|
await mapOnClient2.core.waitForFullStreaming();
|
|
422
407
|
|
|
423
|
-
// Test streaming counter after loading is complete (should be 0)
|
|
424
|
-
await waitFor(async () => {
|
|
425
|
-
const streamingCounterAfterLoad = await metricReader.getMetricValue(
|
|
426
|
-
"jazz.storage.streaming",
|
|
427
|
-
);
|
|
428
|
-
expect(streamingCounterAfterLoad).toBe(0);
|
|
429
|
-
});
|
|
430
|
-
|
|
431
408
|
expect(
|
|
432
409
|
SyncMessagesLog.getMessages({
|
|
433
410
|
Group: group.core,
|
|
@@ -440,8 +417,6 @@ describe("client syncs with a server with storage", () => {
|
|
|
440
417
|
"client -> server | LOAD Group sessions: header/5",
|
|
441
418
|
"storage -> client | CONTENT Map header: true new: After: 0 New: 73 expectContentUntil: header/200",
|
|
442
419
|
"client -> server | LOAD Map sessions: header/200",
|
|
443
|
-
"server -> client | KNOWN Group sessions: header/5",
|
|
444
|
-
"server -> client | KNOWN Map sessions: header/200",
|
|
445
420
|
"storage -> client | CONTENT Map header: true new: After: 73 New: 73",
|
|
446
421
|
"storage -> client | CONTENT Map header: true new: After: 146 New: 54",
|
|
447
422
|
]
|
|
@@ -830,16 +805,233 @@ describe("client syncs with a server with storage", () => {
|
|
|
830
805
|
"server -> bob | CONTENT Map header: true new: After: 0 New: 1",
|
|
831
806
|
"storage -> syncServer | CONTENT ParentGroup header: true new: After: 76 New: 73",
|
|
832
807
|
"server -> bob | CONTENT ParentGroup header: false new: After: 76 New: 73 expectContentUntil: header/205",
|
|
808
|
+
"storage -> syncServer | CONTENT ParentGroup header: true new: After: 149 New: 56",
|
|
809
|
+
"server -> bob | CONTENT ParentGroup header: false new: After: 149 New: 56",
|
|
833
810
|
"bob -> server | KNOWN ParentGroup sessions: header/76",
|
|
834
811
|
"bob -> server | KNOWN Group sessions: header/5",
|
|
835
812
|
"bob -> server | KNOWN Map sessions: header/1",
|
|
836
813
|
"bob -> server | KNOWN ParentGroup sessions: header/149",
|
|
837
|
-
"storage -> syncServer | CONTENT ParentGroup header: true new: After: 149 New: 56",
|
|
838
|
-
"server -> bob | CONTENT ParentGroup header: false new: After: 149 New: 56",
|
|
839
814
|
"bob -> server | KNOWN ParentGroup sessions: header/205",
|
|
840
815
|
]
|
|
841
816
|
`);
|
|
842
817
|
|
|
843
818
|
expect(mapOnBob.get("hello")).toEqual("world");
|
|
844
819
|
});
|
|
820
|
+
|
|
821
|
+
describe("storage content streaming queue", () => {
|
|
822
|
+
test("multiple CoValues can stream concurrently", async () => {
|
|
823
|
+
cojsonInternals.setIncomingMessagesTimeBudget(0); // Should force the queue processing to be async
|
|
824
|
+
|
|
825
|
+
const client = setupTestNode();
|
|
826
|
+
client.connectToSyncServer();
|
|
827
|
+
const { storage } = client.addStorage();
|
|
828
|
+
|
|
829
|
+
const group = jazzCloud.node.createGroup();
|
|
830
|
+
const map1 = group.createMap();
|
|
831
|
+
const map2 = group.createMap();
|
|
832
|
+
const map3 = group.createMap();
|
|
833
|
+
|
|
834
|
+
fillCoMapWithLargeData(map1);
|
|
835
|
+
fillCoMapWithLargeData(map2);
|
|
836
|
+
fillCoMapWithLargeData(map3);
|
|
837
|
+
|
|
838
|
+
// Load all and sync to storage
|
|
839
|
+
await Promise.all([
|
|
840
|
+
loadCoValueOrFail(client.node, map1.id),
|
|
841
|
+
loadCoValueOrFail(client.node, map2.id),
|
|
842
|
+
loadCoValueOrFail(client.node, map3.id),
|
|
843
|
+
]);
|
|
844
|
+
await Promise.all([
|
|
845
|
+
map1.core.waitForSync(),
|
|
846
|
+
map2.core.waitForSync(),
|
|
847
|
+
map3.core.waitForSync(),
|
|
848
|
+
]);
|
|
849
|
+
|
|
850
|
+
// Restart to load from storage
|
|
851
|
+
client.restart();
|
|
852
|
+
client.addStorage({ storage });
|
|
853
|
+
|
|
854
|
+
// Load all maps concurrently from storage
|
|
855
|
+
const [loadedMap1, loadedMap2, loadedMap3] = await Promise.all([
|
|
856
|
+
loadCoValueOrFail(client.node, map1.id),
|
|
857
|
+
loadCoValueOrFail(client.node, map2.id),
|
|
858
|
+
loadCoValueOrFail(client.node, map3.id),
|
|
859
|
+
]);
|
|
860
|
+
|
|
861
|
+
// Wait for all to complete streaming
|
|
862
|
+
await Promise.all([
|
|
863
|
+
loadedMap1.core.waitForAsync((value) => value.isCompletelyDownloaded()),
|
|
864
|
+
loadedMap2.core.waitForAsync((value) => value.isCompletelyDownloaded()),
|
|
865
|
+
loadedMap3.core.waitForAsync((value) => value.isCompletelyDownloaded()),
|
|
866
|
+
]);
|
|
867
|
+
|
|
868
|
+
// All content should be loaded
|
|
869
|
+
expect(loadedMap1.core.isCompletelyDownloaded()).toBe(true);
|
|
870
|
+
expect(loadedMap2.core.isCompletelyDownloaded()).toBe(true);
|
|
871
|
+
expect(loadedMap3.core.isCompletelyDownloaded()).toBe(true);
|
|
872
|
+
|
|
873
|
+
// Queue should be empty
|
|
874
|
+
expect(storage.streamingQueue?.isEmpty()).toBe(true);
|
|
875
|
+
});
|
|
876
|
+
|
|
877
|
+
test("large content streaming interleaved with incoming messages", async () => {
|
|
878
|
+
cojsonInternals.setIncomingMessagesTimeBudget(0); // Should force the queue processing to be async
|
|
879
|
+
|
|
880
|
+
const client = setupTestNode();
|
|
881
|
+
client.connectToSyncServer();
|
|
882
|
+
const { storage } = client.addStorage();
|
|
883
|
+
|
|
884
|
+
// Create a large map on the server and sync to client storage
|
|
885
|
+
const group = jazzCloud.node.createGroup();
|
|
886
|
+
const largeMap = group.createMap();
|
|
887
|
+
fillCoMapWithLargeData(largeMap);
|
|
888
|
+
|
|
889
|
+
await loadCoValueOrFail(client.node, largeMap.id);
|
|
890
|
+
await largeMap.core.waitForSync();
|
|
891
|
+
|
|
892
|
+
SyncMessagesLog.clear();
|
|
893
|
+
|
|
894
|
+
// Restart client with storage
|
|
895
|
+
client.restart();
|
|
896
|
+
client.connectToSyncServer();
|
|
897
|
+
client.addStorage({ storage });
|
|
898
|
+
|
|
899
|
+
// Create a new small map on the server (will come as incoming message)
|
|
900
|
+
const smallMap = group.createMap();
|
|
901
|
+
smallMap.set("hello", "world", "trusting");
|
|
902
|
+
|
|
903
|
+
// Load both simultaneously - large from storage, small from server
|
|
904
|
+
const [loadedLargeMap, loadedSmallMap] = await Promise.all([
|
|
905
|
+
loadCoValueOrFail(client.node, largeMap.id),
|
|
906
|
+
loadCoValueOrFail(client.node, smallMap.id),
|
|
907
|
+
]);
|
|
908
|
+
|
|
909
|
+
// Wait for complete download
|
|
910
|
+
await Promise.all([
|
|
911
|
+
loadedLargeMap.core.waitForAsync((value) =>
|
|
912
|
+
value.isCompletelyDownloaded(),
|
|
913
|
+
),
|
|
914
|
+
loadedSmallMap.core.waitForAsync((value) =>
|
|
915
|
+
value.isCompletelyDownloaded(),
|
|
916
|
+
),
|
|
917
|
+
]);
|
|
918
|
+
|
|
919
|
+
// Both should be loaded correctly
|
|
920
|
+
expect(loadedLargeMap.core.isCompletelyDownloaded()).toBe(true);
|
|
921
|
+
expect(loadedSmallMap.get("hello")).toEqual("world");
|
|
922
|
+
});
|
|
923
|
+
|
|
924
|
+
test("large parent group streaming from storage", async () => {
|
|
925
|
+
cojsonInternals.setIncomingMessagesTimeBudget(0);
|
|
926
|
+
|
|
927
|
+
const syncServer = setupTestNode({
|
|
928
|
+
isSyncServer: true,
|
|
929
|
+
});
|
|
930
|
+
const { storage } = syncServer.addStorage({
|
|
931
|
+
ourName: "syncServer",
|
|
932
|
+
});
|
|
933
|
+
|
|
934
|
+
const alice = setupTestNode();
|
|
935
|
+
alice.connectToSyncServer({
|
|
936
|
+
syncServer: syncServer.node,
|
|
937
|
+
});
|
|
938
|
+
|
|
939
|
+
const parentGroup = alice.node.createGroup();
|
|
940
|
+
const group = alice.node.createGroup();
|
|
941
|
+
group.extend(parentGroup);
|
|
942
|
+
|
|
943
|
+
const map = group.createMap();
|
|
944
|
+
|
|
945
|
+
fillCoMapWithLargeData(parentGroup);
|
|
946
|
+
|
|
947
|
+
parentGroup.addMember("everyone", "reader");
|
|
948
|
+
|
|
949
|
+
map.set("hello", "world");
|
|
950
|
+
|
|
951
|
+
await map.core.waitForSync();
|
|
952
|
+
await parentGroup.core.waitForSync();
|
|
953
|
+
|
|
954
|
+
expect(
|
|
955
|
+
SyncMessagesLog.getMessages({
|
|
956
|
+
Group: group.core,
|
|
957
|
+
ParentGroup: parentGroup.core,
|
|
958
|
+
Map: map.core,
|
|
959
|
+
}),
|
|
960
|
+
).toMatchInlineSnapshot(`
|
|
961
|
+
[
|
|
962
|
+
"client -> server | CONTENT ParentGroup header: true new: After: 0 New: 3",
|
|
963
|
+
"client -> server | CONTENT Group header: true new: After: 0 New: 5",
|
|
964
|
+
"client -> server | CONTENT Map header: true new: ",
|
|
965
|
+
"client -> server | CONTENT ParentGroup header: false new: After: 3 New: 73 expectContentUntil: header/205",
|
|
966
|
+
"client -> server | CONTENT ParentGroup header: false new: After: 76 New: 73",
|
|
967
|
+
"client -> server | CONTENT ParentGroup header: false new: After: 149 New: 56",
|
|
968
|
+
"client -> server | CONTENT Map header: false new: After: 0 New: 1",
|
|
969
|
+
"server -> client | KNOWN ParentGroup sessions: header/3",
|
|
970
|
+
"syncServer -> storage | CONTENT ParentGroup header: true new: After: 0 New: 3",
|
|
971
|
+
"server -> client | KNOWN Group sessions: header/5",
|
|
972
|
+
"syncServer -> storage | CONTENT Group header: true new: After: 0 New: 5",
|
|
973
|
+
"server -> client | KNOWN Map sessions: header/0",
|
|
974
|
+
"syncServer -> storage | CONTENT Map header: true new: ",
|
|
975
|
+
"server -> client | KNOWN ParentGroup sessions: header/76",
|
|
976
|
+
"syncServer -> storage | CONTENT ParentGroup header: false new: After: 3 New: 73",
|
|
977
|
+
"server -> client | KNOWN ParentGroup sessions: header/149",
|
|
978
|
+
"syncServer -> storage | CONTENT ParentGroup header: false new: After: 76 New: 73",
|
|
979
|
+
"server -> client | KNOWN ParentGroup sessions: header/205",
|
|
980
|
+
"syncServer -> storage | CONTENT ParentGroup header: false new: After: 149 New: 56",
|
|
981
|
+
"server -> client | KNOWN Map sessions: header/1",
|
|
982
|
+
"syncServer -> storage | CONTENT Map header: false new: After: 0 New: 1",
|
|
983
|
+
]
|
|
984
|
+
`);
|
|
985
|
+
|
|
986
|
+
SyncMessagesLog.clear();
|
|
987
|
+
|
|
988
|
+
syncServer.restart();
|
|
989
|
+
syncServer.addStorage({
|
|
990
|
+
ourName: "syncServer",
|
|
991
|
+
storage,
|
|
992
|
+
});
|
|
993
|
+
|
|
994
|
+
const bob = setupTestNode();
|
|
995
|
+
bob.connectToSyncServer({
|
|
996
|
+
syncServer: syncServer.node,
|
|
997
|
+
ourName: "bob",
|
|
998
|
+
});
|
|
999
|
+
|
|
1000
|
+
let mapOnBob = await loadCoValueOrFail(bob.node, map.id);
|
|
1001
|
+
|
|
1002
|
+
await mapOnBob.core.waitForAsync((value) =>
|
|
1003
|
+
value.isCompletelyDownloaded(),
|
|
1004
|
+
);
|
|
1005
|
+
|
|
1006
|
+
expect(
|
|
1007
|
+
SyncMessagesLog.getMessages({
|
|
1008
|
+
ParentGroup: parentGroup.core,
|
|
1009
|
+
Group: group.core,
|
|
1010
|
+
Map: map.core,
|
|
1011
|
+
}),
|
|
1012
|
+
).toMatchInlineSnapshot(`
|
|
1013
|
+
[
|
|
1014
|
+
"bob -> server | LOAD Map sessions: empty",
|
|
1015
|
+
"syncServer -> storage | LOAD Map sessions: empty",
|
|
1016
|
+
"storage -> syncServer | CONTENT ParentGroup header: true new: After: 0 New: 76 expectContentUntil: header/205",
|
|
1017
|
+
"storage -> syncServer | CONTENT Group header: true new: After: 0 New: 5",
|
|
1018
|
+
"storage -> syncServer | CONTENT Map header: true new: After: 0 New: 1",
|
|
1019
|
+
"server -> bob | CONTENT ParentGroup header: true new: After: 0 New: 76 expectContentUntil: header/205",
|
|
1020
|
+
"server -> bob | CONTENT Group header: true new: After: 0 New: 5",
|
|
1021
|
+
"server -> bob | CONTENT Map header: true new: After: 0 New: 1",
|
|
1022
|
+
"storage -> syncServer | CONTENT ParentGroup header: true new: After: 76 New: 73",
|
|
1023
|
+
"server -> bob | CONTENT ParentGroup header: false new: After: 76 New: 73 expectContentUntil: header/205",
|
|
1024
|
+
"bob -> server | KNOWN ParentGroup sessions: header/76",
|
|
1025
|
+
"storage -> syncServer | CONTENT ParentGroup header: true new: After: 149 New: 56",
|
|
1026
|
+
"server -> bob | CONTENT ParentGroup header: false new: After: 149 New: 56",
|
|
1027
|
+
"bob -> server | KNOWN Group sessions: header/5",
|
|
1028
|
+
"bob -> server | KNOWN Map sessions: header/1",
|
|
1029
|
+
"bob -> server | KNOWN ParentGroup sessions: header/149",
|
|
1030
|
+
"bob -> server | KNOWN ParentGroup sessions: header/205",
|
|
1031
|
+
]
|
|
1032
|
+
`);
|
|
1033
|
+
|
|
1034
|
+
expect(mapOnBob.get("hello")).toEqual("world");
|
|
1035
|
+
});
|
|
1036
|
+
});
|
|
845
1037
|
});
|