cojson 0.19.19 → 0.19.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/.turbo/turbo-build.log +1 -1
  2. package/dist/config.d.ts +6 -0
  3. package/dist/config.d.ts.map +1 -1
  4. package/dist/config.js +10 -0
  5. package/dist/config.js.map +1 -1
  6. package/dist/exports.d.ts +7 -1
  7. package/dist/exports.d.ts.map +1 -1
  8. package/dist/exports.js +4 -1
  9. package/dist/exports.js.map +1 -1
  10. package/dist/queue/IncomingMessagesQueue.d.ts +6 -7
  11. package/dist/queue/IncomingMessagesQueue.d.ts.map +1 -1
  12. package/dist/queue/IncomingMessagesQueue.js +7 -30
  13. package/dist/queue/IncomingMessagesQueue.js.map +1 -1
  14. package/dist/queue/LinkedList.d.ts +1 -1
  15. package/dist/queue/LinkedList.d.ts.map +1 -1
  16. package/dist/queue/LinkedList.js.map +1 -1
  17. package/dist/queue/StorageStreamingQueue.d.ts +43 -0
  18. package/dist/queue/StorageStreamingQueue.d.ts.map +1 -0
  19. package/dist/queue/StorageStreamingQueue.js +70 -0
  20. package/dist/queue/StorageStreamingQueue.js.map +1 -0
  21. package/dist/storage/storageSync.d.ts +8 -2
  22. package/dist/storage/storageSync.d.ts.map +1 -1
  23. package/dist/storage/storageSync.js +56 -44
  24. package/dist/storage/storageSync.js.map +1 -1
  25. package/dist/storage/types.d.ts +2 -0
  26. package/dist/storage/types.d.ts.map +1 -1
  27. package/dist/sync.d.ts +17 -0
  28. package/dist/sync.d.ts.map +1 -1
  29. package/dist/sync.js +74 -5
  30. package/dist/sync.js.map +1 -1
  31. package/dist/tests/IncomingMessagesQueue.test.js +4 -150
  32. package/dist/tests/IncomingMessagesQueue.test.js.map +1 -1
  33. package/dist/tests/StorageStreamingQueue.test.d.ts +2 -0
  34. package/dist/tests/StorageStreamingQueue.test.d.ts.map +1 -0
  35. package/dist/tests/StorageStreamingQueue.test.js +213 -0
  36. package/dist/tests/StorageStreamingQueue.test.js.map +1 -0
  37. package/dist/tests/SyncManager.processQueues.test.d.ts +2 -0
  38. package/dist/tests/SyncManager.processQueues.test.d.ts.map +1 -0
  39. package/dist/tests/SyncManager.processQueues.test.js +208 -0
  40. package/dist/tests/SyncManager.processQueues.test.js.map +1 -0
  41. package/dist/tests/setup.d.ts +2 -0
  42. package/dist/tests/setup.d.ts.map +1 -0
  43. package/dist/tests/setup.js +4 -0
  44. package/dist/tests/setup.js.map +1 -0
  45. package/dist/tests/sync.garbageCollection.test.js.map +1 -1
  46. package/dist/tests/sync.mesh.test.js +19 -19
  47. package/dist/tests/sync.storage.test.js +176 -20
  48. package/dist/tests/sync.storage.test.js.map +1 -1
  49. package/dist/tests/sync.test.js +1 -1
  50. package/dist/tests/sync.test.js.map +1 -1
  51. package/dist/tests/testUtils.d.ts +2 -2
  52. package/dist/tests/testUtils.js +2 -2
  53. package/dist/tests/testUtils.js.map +1 -1
  54. package/package.json +4 -4
  55. package/src/config.ts +13 -0
  56. package/src/exports.ts +6 -0
  57. package/src/queue/IncomingMessagesQueue.ts +7 -39
  58. package/src/queue/LinkedList.ts +1 -1
  59. package/src/queue/StorageStreamingQueue.ts +96 -0
  60. package/src/storage/storageSync.ts +99 -55
  61. package/src/storage/types.ts +3 -0
  62. package/src/sync.ts +84 -5
  63. package/src/tests/IncomingMessagesQueue.test.ts +4 -206
  64. package/src/tests/StorageStreamingQueue.test.ts +276 -0
  65. package/src/tests/SyncManager.processQueues.test.ts +287 -0
  66. package/src/tests/setup.ts +4 -0
  67. package/src/tests/sync.garbageCollection.test.ts +1 -3
  68. package/src/tests/sync.mesh.test.ts +19 -19
  69. package/src/tests/sync.storage.test.ts +224 -32
  70. package/src/tests/sync.test.ts +1 -9
  71. package/src/tests/testUtils.ts +2 -2
  72. package/vitest.config.ts +1 -0
@@ -0,0 +1,287 @@
1
+ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
2
+ import { CO_VALUE_PRIORITY } from "../priority.js";
3
+ import { StorageStreamingQueue } from "../queue/StorageStreamingQueue.js";
4
+ import {
5
+ SyncMessagesLog,
6
+ loadCoValueOrFail,
7
+ setupTestNode,
8
+ waitFor,
9
+ } from "./testUtils.js";
10
+
11
+ describe("SyncManager.processQueues", () => {
12
+ let jazzCloud: ReturnType<typeof setupTestNode>;
13
+
14
+ beforeEach(async () => {
15
+ SyncMessagesLog.clear();
16
+ jazzCloud = setupTestNode({
17
+ isSyncServer: true,
18
+ });
19
+ });
20
+
21
+ describe("incoming messages processing", () => {
22
+ test("should process incoming messages from peers", async () => {
23
+ const client = setupTestNode();
24
+ client.connectToSyncServer();
25
+
26
+ const group = jazzCloud.node.createGroup();
27
+ const map = group.createMap();
28
+ map.set("hello", "world", "trusting");
29
+
30
+ const mapOnClient = await loadCoValueOrFail(client.node, map.id);
31
+ expect(mapOnClient.get("hello")).toEqual("world");
32
+ });
33
+
34
+ test("should process multiple messages in sequence", async () => {
35
+ const client = setupTestNode();
36
+ client.connectToSyncServer();
37
+
38
+ const group = jazzCloud.node.createGroup();
39
+ const map1 = group.createMap();
40
+ const map2 = group.createMap();
41
+ const map3 = group.createMap();
42
+
43
+ map1.set("key", "value1", "trusting");
44
+ map2.set("key", "value2", "trusting");
45
+ map3.set("key", "value3", "trusting");
46
+
47
+ const [loadedMap1, loadedMap2, loadedMap3] = await Promise.all([
48
+ loadCoValueOrFail(client.node, map1.id),
49
+ loadCoValueOrFail(client.node, map2.id),
50
+ loadCoValueOrFail(client.node, map3.id),
51
+ ]);
52
+
53
+ expect(loadedMap1.get("key")).toEqual("value1");
54
+ expect(loadedMap2.get("key")).toEqual("value2");
55
+ expect(loadedMap3.get("key")).toEqual("value3");
56
+ });
57
+ });
58
+
59
+ describe("storage streaming processing", () => {
60
+ test("should process storage streaming callbacks", async () => {
61
+ const client = setupTestNode();
62
+ client.connectToSyncServer();
63
+ const { storage } = client.addStorage();
64
+
65
+ const group = jazzCloud.node.createGroup();
66
+ const map = group.createMap();
67
+ map.set("hello", "world", "trusting");
68
+
69
+ // First load to populate storage
70
+ await loadCoValueOrFail(client.node, map.id);
71
+
72
+ // Restart and load from storage
73
+ client.restart();
74
+ client.connectToSyncServer();
75
+ client.addStorage({ storage });
76
+
77
+ SyncMessagesLog.clear();
78
+
79
+ const mapOnClient = await loadCoValueOrFail(client.node, map.id);
80
+ expect(mapOnClient.get("hello")).toEqual("world");
81
+
82
+ // Verify content came from storage
83
+ const storageMessages = SyncMessagesLog.messages.filter(
84
+ (msg) => msg.from === "storage" || msg.to === "storage",
85
+ );
86
+ expect(storageMessages.length).toBeGreaterThan(0);
87
+ });
88
+
89
+ test("should invoke streaming callbacks when pulled", async () => {
90
+ const client = setupTestNode();
91
+ const { storage } = client.addStorage();
92
+
93
+ const callback = vi.fn();
94
+ storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
95
+ storage.streamingQueue?.emit();
96
+
97
+ // Wait for processQueues to run
98
+ await waitFor(() => callback.mock.calls.length > 0);
99
+
100
+ expect(callback).toHaveBeenCalledTimes(1);
101
+ });
102
+
103
+ test("should process MEDIUM priority before LOW priority", async () => {
104
+ const client = setupTestNode();
105
+ const { storage } = client.addStorage();
106
+
107
+ const order: string[] = [];
108
+ const lowCallback = () => order.push("low");
109
+ const mediumCallback = () => order.push("medium");
110
+ const highCallback = () => order.push("high");
111
+
112
+ // Push LOW first, then MEDIUM
113
+ storage.streamingQueue?.push(lowCallback, CO_VALUE_PRIORITY.LOW);
114
+ storage.streamingQueue?.push(mediumCallback, CO_VALUE_PRIORITY.MEDIUM);
115
+ storage.streamingQueue?.push(highCallback, CO_VALUE_PRIORITY.HIGH);
116
+ storage.streamingQueue?.emit();
117
+
118
+ // Wait for both to be processed
119
+ await waitFor(() => order.length === 3);
120
+
121
+ // MEDIUM should be processed first
122
+ expect(order).toEqual(["high", "medium", "low"]);
123
+ });
124
+ });
125
+
126
+ describe("unified scheduling", () => {
127
+ test("should process both incoming messages and storage streaming", async () => {
128
+ const client = setupTestNode();
129
+ client.connectToSyncServer();
130
+ const { storage } = client.addStorage();
131
+
132
+ const group = jazzCloud.node.createGroup();
133
+ const map = group.createMap();
134
+ map.set("hello", "world", "trusting");
135
+
136
+ // Queue a storage streaming callback
137
+ const streamingCallback = vi.fn();
138
+ storage.streamingQueue?.push(streamingCallback, CO_VALUE_PRIORITY.MEDIUM);
139
+ storage.streamingQueue?.emit();
140
+
141
+ // Load from server (incoming messages)
142
+ const mapOnClient = await loadCoValueOrFail(client.node, map.id);
143
+
144
+ expect(mapOnClient.get("hello")).toEqual("world");
145
+ expect(streamingCallback).toHaveBeenCalled();
146
+ });
147
+
148
+ test("should alternate between message queue and storage queue", async () => {
149
+ const client = setupTestNode();
150
+ const { storage } = client.addStorage();
151
+
152
+ const order: string[] = [];
153
+
154
+ // Push multiple storage callbacks
155
+ storage.streamingQueue?.push(
156
+ () => order.push("storage1"),
157
+ CO_VALUE_PRIORITY.MEDIUM,
158
+ );
159
+ storage.streamingQueue?.push(
160
+ () => order.push("storage2"),
161
+ CO_VALUE_PRIORITY.MEDIUM,
162
+ );
163
+ storage.streamingQueue?.emit();
164
+
165
+ // Wait for processing
166
+ await waitFor(() => order.length === 2);
167
+
168
+ expect(order).toContain("storage1");
169
+ expect(order).toContain("storage2");
170
+ });
171
+ });
172
+
173
+ describe("processing flag", () => {
174
+ test("should prevent concurrent processQueues calls", async () => {
175
+ const client = setupTestNode();
176
+ const { storage } = client.addStorage();
177
+
178
+ let concurrentCalls = 0;
179
+ let maxConcurrentCalls = 0;
180
+
181
+ const callback = () => {
182
+ concurrentCalls++;
183
+ maxConcurrentCalls = Math.max(maxConcurrentCalls, concurrentCalls);
184
+ // Simulate some work
185
+ for (let i = 0; i < 1000; i++) {
186
+ Math.random();
187
+ }
188
+ concurrentCalls--;
189
+ };
190
+
191
+ // Push multiple callbacks
192
+ for (let i = 0; i < 10; i++) {
193
+ storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
194
+ }
195
+
196
+ // Emit multiple times to trigger multiple processQueues calls
197
+ storage.streamingQueue?.emit();
198
+ storage.streamingQueue?.emit();
199
+ storage.streamingQueue?.emit();
200
+
201
+ // Wait for all to complete
202
+ await waitFor(() => storage.streamingQueue?.isEmpty());
203
+
204
+ // Should never have more than 1 concurrent call
205
+ expect(maxConcurrentCalls).toBe(1);
206
+ });
207
+ });
208
+
209
+ describe("error handling", () => {
210
+ test("should continue processing after storage callback error", async () => {
211
+ const client = setupTestNode();
212
+ const { storage } = client.addStorage();
213
+
214
+ const processed: string[] = [];
215
+
216
+ storage.streamingQueue?.push(() => {
217
+ processed.push("before");
218
+ }, CO_VALUE_PRIORITY.MEDIUM);
219
+
220
+ storage.streamingQueue?.push(() => {
221
+ throw new Error("Test error");
222
+ }, CO_VALUE_PRIORITY.MEDIUM);
223
+
224
+ storage.streamingQueue?.push(() => {
225
+ processed.push("after");
226
+ }, CO_VALUE_PRIORITY.MEDIUM);
227
+
228
+ storage.streamingQueue?.emit();
229
+
230
+ // Wait for processing to complete
231
+ await waitFor(() => storage.streamingQueue?.isEmpty());
232
+
233
+ // Both before and after should be processed despite error
234
+ expect(processed).toContain("before");
235
+ expect(processed).toContain("after");
236
+ });
237
+ });
238
+
239
+ describe("queue triggers", () => {
240
+ test("IncomingMessagesQueue.push should trigger processQueues", async () => {
241
+ const client = setupTestNode();
242
+ client.connectToSyncServer();
243
+
244
+ const group = jazzCloud.node.createGroup();
245
+ const map = group.createMap();
246
+ map.set("hello", "world", "trusting");
247
+
248
+ // Loading should trigger message processing automatically
249
+ const mapOnClient = await loadCoValueOrFail(client.node, map.id);
250
+ expect(mapOnClient.get("hello")).toEqual("world");
251
+ });
252
+
253
+ test("StorageStreamingQueue.emit should trigger processQueues", async () => {
254
+ const client = setupTestNode();
255
+ const { storage } = client.addStorage();
256
+
257
+ const callback = vi.fn();
258
+ storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
259
+
260
+ // Before emit, callback should not be called
261
+ expect(callback).not.toHaveBeenCalled();
262
+
263
+ // After emit, processQueues should be triggered
264
+ storage.streamingQueue?.emit();
265
+
266
+ await waitFor(() => callback.mock.calls.length > 0);
267
+ expect(callback).toHaveBeenCalled();
268
+ });
269
+
270
+ test("setStorage should connect queue listener", async () => {
271
+ const client = setupTestNode();
272
+
273
+ // Before adding storage, there's no queue
274
+ const queueBefore = (
275
+ client.node.syncManager as any
276
+ ).getStorageStreamingQueue?.();
277
+ expect(queueBefore).toBeUndefined();
278
+
279
+ // After adding storage, queue should be available
280
+ const { storage } = client.addStorage();
281
+ const queueAfter = (
282
+ client.node.syncManager as any
283
+ ).getStorageStreamingQueue?.();
284
+ expect(queueAfter).toBe(storage.streamingQueue);
285
+ });
286
+ });
287
+ });
@@ -0,0 +1,4 @@
1
+ import { cojsonInternals } from "../exports.js";
2
+
3
+ // Use a very high budget to avoid that slow tests fail due to the budget being exceeded.
4
+ cojsonInternals.setIncomingMessagesTimeBudget(10000); // 10 seconds
@@ -1,13 +1,11 @@
1
- import { assert, beforeEach, describe, expect, test, vi } from "vitest";
1
+ import { beforeEach, describe, expect, test } from "vitest";
2
2
 
3
3
  import { setGarbageCollectorMaxAge } from "../config";
4
- import { emptyKnownState } from "../exports";
5
4
  import {
6
5
  SyncMessagesLog,
7
6
  TEST_NODE_CONFIG,
8
7
  loadCoValueOrFail,
9
8
  setupTestNode,
10
- waitFor,
11
9
  } from "./testUtils";
12
10
 
13
11
  // We want to simulate a real world communication that happens asynchronously
@@ -553,6 +553,14 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
553
553
  "edge -> core | LOAD Map sessions: header/100",
554
554
  "edge -> client | CONTENT Group header: true new: After: 0 New: 5",
555
555
  "edge -> client | CONTENT Map header: true new: After: 0 New: 21 expectContentUntil: header/100",
556
+ "storage -> edge | CONTENT Map header: true new: After: 21 New: 21",
557
+ "edge -> client | CONTENT Map header: false new: After: 21 New: 21 expectContentUntil: header/100",
558
+ "storage -> edge | CONTENT Map header: true new: After: 42 New: 21",
559
+ "edge -> client | CONTENT Map header: false new: After: 42 New: 21 expectContentUntil: header/100",
560
+ "storage -> edge | CONTENT Map header: true new: After: 63 New: 21",
561
+ "edge -> client | CONTENT Map header: false new: After: 63 New: 21 expectContentUntil: header/100",
562
+ "storage -> edge | CONTENT Map header: true new: After: 84 New: 16",
563
+ "edge -> client | CONTENT Map header: false new: After: 84 New: 16",
556
564
  "core -> storage | LOAD Group sessions: empty",
557
565
  "storage -> core | KNOWN Group sessions: empty",
558
566
  "core -> edge | KNOWN Group sessions: empty",
@@ -563,16 +571,20 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
563
571
  "client -> storage | CONTENT Group header: true new: After: 0 New: 5",
564
572
  "client -> edge | KNOWN Map sessions: header/21",
565
573
  "client -> storage | CONTENT Map header: true new: After: 0 New: 21",
566
- "storage -> edge | CONTENT Map header: true new: After: 21 New: 21",
567
- "edge -> client | CONTENT Map header: false new: After: 21 New: 21 expectContentUntil: header/100",
574
+ "client -> edge | KNOWN Map sessions: header/42",
575
+ "client -> storage | CONTENT Map header: false new: After: 21 New: 21",
576
+ "client -> edge | KNOWN Map sessions: header/63",
577
+ "client -> storage | CONTENT Map header: false new: After: 42 New: 21",
578
+ "client -> edge | KNOWN Map sessions: header/84",
579
+ "client -> storage | CONTENT Map header: false new: After: 63 New: 21",
580
+ "client -> edge | KNOWN Map sessions: header/100",
581
+ "client -> storage | CONTENT Map header: false new: After: 84 New: 16",
568
582
  "edge -> core | CONTENT Group header: true new: After: 0 New: 5",
569
583
  "edge -> core | CONTENT Map header: true new: After: 0 New: 21 expectContentUntil: header/100",
570
584
  "edge -> core | CONTENT Map header: false new: After: 21 New: 21",
571
- "client -> edge | KNOWN Map sessions: header/42",
572
- "client -> storage | CONTENT Map header: false new: After: 21 New: 21",
573
- "storage -> edge | CONTENT Map header: true new: After: 42 New: 21",
574
- "edge -> core | CONTENT Map header: false new: After: 42 New: 21 expectContentUntil: header/100",
575
- "edge -> client | CONTENT Map header: false new: After: 42 New: 21 expectContentUntil: header/100",
585
+ "edge -> core | CONTENT Map header: false new: After: 42 New: 21",
586
+ "edge -> core | CONTENT Map header: false new: After: 63 New: 21",
587
+ "edge -> core | CONTENT Map header: false new: After: 84 New: 16",
576
588
  "core -> edge | KNOWN Group sessions: header/5",
577
589
  "core -> storage | CONTENT Group header: true new: After: 0 New: 5",
578
590
  "core -> edge | KNOWN Map sessions: header/21",
@@ -581,22 +593,10 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
581
593
  "core -> storage | CONTENT Map header: false new: After: 21 New: 21",
582
594
  "core -> edge | KNOWN Map sessions: header/63",
583
595
  "core -> storage | CONTENT Map header: false new: After: 42 New: 21",
584
- "client -> edge | KNOWN Map sessions: header/63",
585
- "client -> storage | CONTENT Map header: false new: After: 42 New: 21",
586
- "storage -> edge | CONTENT Map header: true new: After: 63 New: 21",
587
- "edge -> core | CONTENT Map header: false new: After: 63 New: 21 expectContentUntil: header/100",
588
- "edge -> client | CONTENT Map header: false new: After: 63 New: 21 expectContentUntil: header/100",
589
596
  "core -> edge | KNOWN Map sessions: header/84",
590
597
  "core -> storage | CONTENT Map header: false new: After: 63 New: 21",
591
- "client -> edge | KNOWN Map sessions: header/84",
592
- "client -> storage | CONTENT Map header: false new: After: 63 New: 21",
593
- "storage -> edge | CONTENT Map header: true new: After: 84 New: 16",
594
- "edge -> core | CONTENT Map header: false new: After: 84 New: 16",
595
- "edge -> client | CONTENT Map header: false new: After: 84 New: 16",
596
598
  "core -> edge | KNOWN Map sessions: header/100",
597
599
  "core -> storage | CONTENT Map header: false new: After: 84 New: 16",
598
- "client -> edge | KNOWN Map sessions: header/100",
599
- "client -> storage | CONTENT Map header: false new: After: 84 New: 16",
600
600
  ]
601
601
  `);
602
602
 
@@ -8,7 +8,7 @@ import {
8
8
  vi,
9
9
  } from "vitest";
10
10
 
11
- import { emptyKnownState } from "../exports";
11
+ import { cojsonInternals, emptyKnownState } from "../exports";
12
12
  import {
13
13
  SyncMessagesLog,
14
14
  TEST_NODE_CONFIG,
@@ -24,6 +24,10 @@ import { stableStringify } from "../jsonStringify";
24
24
  // We want to simulate a real world communication that happens asynchronously
25
25
  TEST_NODE_CONFIG.withAsyncPeers = true;
26
26
 
27
+ beforeEach(() => {
28
+ cojsonInternals.setIncomingMessagesTimeBudget(10_000);
29
+ });
30
+
27
31
  describe("client with storage syncs with server", () => {
28
32
  let jazzCloud: ReturnType<typeof setupTestNode>;
29
33
 
@@ -356,13 +360,6 @@ describe("client syncs with a server with storage", () => {
356
360
 
357
361
  await largeMap.core.waitForSync();
358
362
 
359
- // Test streaming counter during initial sync
360
- // The streaming counter should be 0 after the sync is complete
361
- const streamingCounterAfterSync = await metricReader.getMetricValue(
362
- "jazz.storage.streaming",
363
- );
364
- expect(streamingCounterAfterSync).toBe(0);
365
-
366
363
  expect(
367
364
  SyncMessagesLog.getMessages({
368
365
  Group: group.core,
@@ -403,31 +400,11 @@ describe("client syncs with a server with storage", () => {
403
400
  storage,
404
401
  });
405
402
 
406
- // Test streaming counter before loading the large coValue
407
- const streamingCounterBeforeLoad = await metricReader.getMetricValue(
408
- "jazz.storage.streaming",
409
- );
410
- expect(streamingCounterBeforeLoad).toBe(0);
411
-
412
403
  const promise = loadCoValueOrFail(client.node, largeMap.id);
413
404
 
414
- // Test streaming counter during loading (should be 1 during streaming)
415
- const streamingCounterDuringLoad = await metricReader.getMetricValue(
416
- "jazz.storage.streaming",
417
- );
418
- expect(streamingCounterDuringLoad).toBe(1);
419
-
420
405
  const mapOnClient2 = await promise;
421
406
  await mapOnClient2.core.waitForFullStreaming();
422
407
 
423
- // Test streaming counter after loading is complete (should be 0)
424
- await waitFor(async () => {
425
- const streamingCounterAfterLoad = await metricReader.getMetricValue(
426
- "jazz.storage.streaming",
427
- );
428
- expect(streamingCounterAfterLoad).toBe(0);
429
- });
430
-
431
408
  expect(
432
409
  SyncMessagesLog.getMessages({
433
410
  Group: group.core,
@@ -440,8 +417,6 @@ describe("client syncs with a server with storage", () => {
440
417
  "client -> server | LOAD Group sessions: header/5",
441
418
  "storage -> client | CONTENT Map header: true new: After: 0 New: 73 expectContentUntil: header/200",
442
419
  "client -> server | LOAD Map sessions: header/200",
443
- "server -> client | KNOWN Group sessions: header/5",
444
- "server -> client | KNOWN Map sessions: header/200",
445
420
  "storage -> client | CONTENT Map header: true new: After: 73 New: 73",
446
421
  "storage -> client | CONTENT Map header: true new: After: 146 New: 54",
447
422
  ]
@@ -830,16 +805,233 @@ describe("client syncs with a server with storage", () => {
830
805
  "server -> bob | CONTENT Map header: true new: After: 0 New: 1",
831
806
  "storage -> syncServer | CONTENT ParentGroup header: true new: After: 76 New: 73",
832
807
  "server -> bob | CONTENT ParentGroup header: false new: After: 76 New: 73 expectContentUntil: header/205",
808
+ "storage -> syncServer | CONTENT ParentGroup header: true new: After: 149 New: 56",
809
+ "server -> bob | CONTENT ParentGroup header: false new: After: 149 New: 56",
833
810
  "bob -> server | KNOWN ParentGroup sessions: header/76",
834
811
  "bob -> server | KNOWN Group sessions: header/5",
835
812
  "bob -> server | KNOWN Map sessions: header/1",
836
813
  "bob -> server | KNOWN ParentGroup sessions: header/149",
837
- "storage -> syncServer | CONTENT ParentGroup header: true new: After: 149 New: 56",
838
- "server -> bob | CONTENT ParentGroup header: false new: After: 149 New: 56",
839
814
  "bob -> server | KNOWN ParentGroup sessions: header/205",
840
815
  ]
841
816
  `);
842
817
 
843
818
  expect(mapOnBob.get("hello")).toEqual("world");
844
819
  });
820
+
821
+ describe("storage content streaming queue", () => {
822
+ test("multiple CoValues can stream concurrently", async () => {
823
+ cojsonInternals.setIncomingMessagesTimeBudget(0); // Should force the queue processing to be async
824
+
825
+ const client = setupTestNode();
826
+ client.connectToSyncServer();
827
+ const { storage } = client.addStorage();
828
+
829
+ const group = jazzCloud.node.createGroup();
830
+ const map1 = group.createMap();
831
+ const map2 = group.createMap();
832
+ const map3 = group.createMap();
833
+
834
+ fillCoMapWithLargeData(map1);
835
+ fillCoMapWithLargeData(map2);
836
+ fillCoMapWithLargeData(map3);
837
+
838
+ // Load all and sync to storage
839
+ await Promise.all([
840
+ loadCoValueOrFail(client.node, map1.id),
841
+ loadCoValueOrFail(client.node, map2.id),
842
+ loadCoValueOrFail(client.node, map3.id),
843
+ ]);
844
+ await Promise.all([
845
+ map1.core.waitForSync(),
846
+ map2.core.waitForSync(),
847
+ map3.core.waitForSync(),
848
+ ]);
849
+
850
+ // Restart to load from storage
851
+ client.restart();
852
+ client.addStorage({ storage });
853
+
854
+ // Load all maps concurrently from storage
855
+ const [loadedMap1, loadedMap2, loadedMap3] = await Promise.all([
856
+ loadCoValueOrFail(client.node, map1.id),
857
+ loadCoValueOrFail(client.node, map2.id),
858
+ loadCoValueOrFail(client.node, map3.id),
859
+ ]);
860
+
861
+ // Wait for all to complete streaming
862
+ await Promise.all([
863
+ loadedMap1.core.waitForAsync((value) => value.isCompletelyDownloaded()),
864
+ loadedMap2.core.waitForAsync((value) => value.isCompletelyDownloaded()),
865
+ loadedMap3.core.waitForAsync((value) => value.isCompletelyDownloaded()),
866
+ ]);
867
+
868
+ // All content should be loaded
869
+ expect(loadedMap1.core.isCompletelyDownloaded()).toBe(true);
870
+ expect(loadedMap2.core.isCompletelyDownloaded()).toBe(true);
871
+ expect(loadedMap3.core.isCompletelyDownloaded()).toBe(true);
872
+
873
+ // Queue should be empty
874
+ expect(storage.streamingQueue?.isEmpty()).toBe(true);
875
+ });
876
+
877
+ test("large content streaming interleaved with incoming messages", async () => {
878
+ cojsonInternals.setIncomingMessagesTimeBudget(0); // Should force the queue processing to be async
879
+
880
+ const client = setupTestNode();
881
+ client.connectToSyncServer();
882
+ const { storage } = client.addStorage();
883
+
884
+ // Create a large map on the server and sync to client storage
885
+ const group = jazzCloud.node.createGroup();
886
+ const largeMap = group.createMap();
887
+ fillCoMapWithLargeData(largeMap);
888
+
889
+ await loadCoValueOrFail(client.node, largeMap.id);
890
+ await largeMap.core.waitForSync();
891
+
892
+ SyncMessagesLog.clear();
893
+
894
+ // Restart client with storage
895
+ client.restart();
896
+ client.connectToSyncServer();
897
+ client.addStorage({ storage });
898
+
899
+ // Create a new small map on the server (will come as incoming message)
900
+ const smallMap = group.createMap();
901
+ smallMap.set("hello", "world", "trusting");
902
+
903
+ // Load both simultaneously - large from storage, small from server
904
+ const [loadedLargeMap, loadedSmallMap] = await Promise.all([
905
+ loadCoValueOrFail(client.node, largeMap.id),
906
+ loadCoValueOrFail(client.node, smallMap.id),
907
+ ]);
908
+
909
+ // Wait for complete download
910
+ await Promise.all([
911
+ loadedLargeMap.core.waitForAsync((value) =>
912
+ value.isCompletelyDownloaded(),
913
+ ),
914
+ loadedSmallMap.core.waitForAsync((value) =>
915
+ value.isCompletelyDownloaded(),
916
+ ),
917
+ ]);
918
+
919
+ // Both should be loaded correctly
920
+ expect(loadedLargeMap.core.isCompletelyDownloaded()).toBe(true);
921
+ expect(loadedSmallMap.get("hello")).toEqual("world");
922
+ });
923
+
924
+ test("large parent group streaming from storage", async () => {
925
+ cojsonInternals.setIncomingMessagesTimeBudget(0);
926
+
927
+ const syncServer = setupTestNode({
928
+ isSyncServer: true,
929
+ });
930
+ const { storage } = syncServer.addStorage({
931
+ ourName: "syncServer",
932
+ });
933
+
934
+ const alice = setupTestNode();
935
+ alice.connectToSyncServer({
936
+ syncServer: syncServer.node,
937
+ });
938
+
939
+ const parentGroup = alice.node.createGroup();
940
+ const group = alice.node.createGroup();
941
+ group.extend(parentGroup);
942
+
943
+ const map = group.createMap();
944
+
945
+ fillCoMapWithLargeData(parentGroup);
946
+
947
+ parentGroup.addMember("everyone", "reader");
948
+
949
+ map.set("hello", "world");
950
+
951
+ await map.core.waitForSync();
952
+ await parentGroup.core.waitForSync();
953
+
954
+ expect(
955
+ SyncMessagesLog.getMessages({
956
+ Group: group.core,
957
+ ParentGroup: parentGroup.core,
958
+ Map: map.core,
959
+ }),
960
+ ).toMatchInlineSnapshot(`
961
+ [
962
+ "client -> server | CONTENT ParentGroup header: true new: After: 0 New: 3",
963
+ "client -> server | CONTENT Group header: true new: After: 0 New: 5",
964
+ "client -> server | CONTENT Map header: true new: ",
965
+ "client -> server | CONTENT ParentGroup header: false new: After: 3 New: 73 expectContentUntil: header/205",
966
+ "client -> server | CONTENT ParentGroup header: false new: After: 76 New: 73",
967
+ "client -> server | CONTENT ParentGroup header: false new: After: 149 New: 56",
968
+ "client -> server | CONTENT Map header: false new: After: 0 New: 1",
969
+ "server -> client | KNOWN ParentGroup sessions: header/3",
970
+ "syncServer -> storage | CONTENT ParentGroup header: true new: After: 0 New: 3",
971
+ "server -> client | KNOWN Group sessions: header/5",
972
+ "syncServer -> storage | CONTENT Group header: true new: After: 0 New: 5",
973
+ "server -> client | KNOWN Map sessions: header/0",
974
+ "syncServer -> storage | CONTENT Map header: true new: ",
975
+ "server -> client | KNOWN ParentGroup sessions: header/76",
976
+ "syncServer -> storage | CONTENT ParentGroup header: false new: After: 3 New: 73",
977
+ "server -> client | KNOWN ParentGroup sessions: header/149",
978
+ "syncServer -> storage | CONTENT ParentGroup header: false new: After: 76 New: 73",
979
+ "server -> client | KNOWN ParentGroup sessions: header/205",
980
+ "syncServer -> storage | CONTENT ParentGroup header: false new: After: 149 New: 56",
981
+ "server -> client | KNOWN Map sessions: header/1",
982
+ "syncServer -> storage | CONTENT Map header: false new: After: 0 New: 1",
983
+ ]
984
+ `);
985
+
986
+ SyncMessagesLog.clear();
987
+
988
+ syncServer.restart();
989
+ syncServer.addStorage({
990
+ ourName: "syncServer",
991
+ storage,
992
+ });
993
+
994
+ const bob = setupTestNode();
995
+ bob.connectToSyncServer({
996
+ syncServer: syncServer.node,
997
+ ourName: "bob",
998
+ });
999
+
1000
+ let mapOnBob = await loadCoValueOrFail(bob.node, map.id);
1001
+
1002
+ await mapOnBob.core.waitForAsync((value) =>
1003
+ value.isCompletelyDownloaded(),
1004
+ );
1005
+
1006
+ expect(
1007
+ SyncMessagesLog.getMessages({
1008
+ ParentGroup: parentGroup.core,
1009
+ Group: group.core,
1010
+ Map: map.core,
1011
+ }),
1012
+ ).toMatchInlineSnapshot(`
1013
+ [
1014
+ "bob -> server | LOAD Map sessions: empty",
1015
+ "syncServer -> storage | LOAD Map sessions: empty",
1016
+ "storage -> syncServer | CONTENT ParentGroup header: true new: After: 0 New: 76 expectContentUntil: header/205",
1017
+ "storage -> syncServer | CONTENT Group header: true new: After: 0 New: 5",
1018
+ "storage -> syncServer | CONTENT Map header: true new: After: 0 New: 1",
1019
+ "server -> bob | CONTENT ParentGroup header: true new: After: 0 New: 76 expectContentUntil: header/205",
1020
+ "server -> bob | CONTENT Group header: true new: After: 0 New: 5",
1021
+ "server -> bob | CONTENT Map header: true new: After: 0 New: 1",
1022
+ "storage -> syncServer | CONTENT ParentGroup header: true new: After: 76 New: 73",
1023
+ "server -> bob | CONTENT ParentGroup header: false new: After: 76 New: 73 expectContentUntil: header/205",
1024
+ "bob -> server | KNOWN ParentGroup sessions: header/76",
1025
+ "storage -> syncServer | CONTENT ParentGroup header: true new: After: 149 New: 56",
1026
+ "server -> bob | CONTENT ParentGroup header: false new: After: 149 New: 56",
1027
+ "bob -> server | KNOWN Group sessions: header/5",
1028
+ "bob -> server | KNOWN Map sessions: header/1",
1029
+ "bob -> server | KNOWN ParentGroup sessions: header/149",
1030
+ "bob -> server | KNOWN ParentGroup sessions: header/205",
1031
+ ]
1032
+ `);
1033
+
1034
+ expect(mapOnBob.get("hello")).toEqual("world");
1035
+ });
1036
+ });
845
1037
  });