cojson 0.19.20 → 0.19.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/.turbo/turbo-build.log +1 -1
  2. package/CHANGELOG.md +13 -0
  3. package/dist/CojsonMessageChannel/CojsonMessageChannel.d.ts +42 -0
  4. package/dist/CojsonMessageChannel/CojsonMessageChannel.d.ts.map +1 -0
  5. package/dist/CojsonMessageChannel/CojsonMessageChannel.js +261 -0
  6. package/dist/CojsonMessageChannel/CojsonMessageChannel.js.map +1 -0
  7. package/dist/CojsonMessageChannel/MessagePortOutgoingChannel.d.ts +18 -0
  8. package/dist/CojsonMessageChannel/MessagePortOutgoingChannel.d.ts.map +1 -0
  9. package/dist/CojsonMessageChannel/MessagePortOutgoingChannel.js +37 -0
  10. package/dist/CojsonMessageChannel/MessagePortOutgoingChannel.js.map +1 -0
  11. package/dist/CojsonMessageChannel/index.d.ts +3 -0
  12. package/dist/CojsonMessageChannel/index.d.ts.map +1 -0
  13. package/dist/CojsonMessageChannel/index.js +2 -0
  14. package/dist/CojsonMessageChannel/index.js.map +1 -0
  15. package/dist/CojsonMessageChannel/types.d.ts +149 -0
  16. package/dist/CojsonMessageChannel/types.d.ts.map +1 -0
  17. package/dist/CojsonMessageChannel/types.js +36 -0
  18. package/dist/CojsonMessageChannel/types.js.map +1 -0
  19. package/dist/GarbageCollector.d.ts +4 -2
  20. package/dist/GarbageCollector.d.ts.map +1 -1
  21. package/dist/GarbageCollector.js +5 -3
  22. package/dist/GarbageCollector.js.map +1 -1
  23. package/dist/SyncStateManager.d.ts +3 -3
  24. package/dist/SyncStateManager.d.ts.map +1 -1
  25. package/dist/SyncStateManager.js +4 -4
  26. package/dist/SyncStateManager.js.map +1 -1
  27. package/dist/coValueCore/coValueCore.d.ts +28 -1
  28. package/dist/coValueCore/coValueCore.d.ts.map +1 -1
  29. package/dist/coValueCore/coValueCore.js +50 -5
  30. package/dist/coValueCore/coValueCore.js.map +1 -1
  31. package/dist/coValues/account.d.ts.map +1 -1
  32. package/dist/coValues/account.js +10 -10
  33. package/dist/coValues/account.js.map +1 -1
  34. package/dist/exports.d.ts +1 -0
  35. package/dist/exports.d.ts.map +1 -1
  36. package/dist/exports.js +1 -0
  37. package/dist/exports.js.map +1 -1
  38. package/dist/ids.d.ts +1 -1
  39. package/dist/ids.d.ts.map +1 -1
  40. package/dist/ids.js.map +1 -1
  41. package/dist/knownState.d.ts +5 -0
  42. package/dist/knownState.d.ts.map +1 -1
  43. package/dist/knownState.js +15 -0
  44. package/dist/knownState.js.map +1 -1
  45. package/dist/localNode.d.ts +1 -3
  46. package/dist/localNode.d.ts.map +1 -1
  47. package/dist/localNode.js +11 -4
  48. package/dist/localNode.js.map +1 -1
  49. package/dist/storage/knownState.d.ts +5 -0
  50. package/dist/storage/knownState.d.ts.map +1 -1
  51. package/dist/storage/knownState.js +11 -0
  52. package/dist/storage/knownState.js.map +1 -1
  53. package/dist/storage/sqlite/client.d.ts +2 -0
  54. package/dist/storage/sqlite/client.d.ts.map +1 -1
  55. package/dist/storage/sqlite/client.js +18 -0
  56. package/dist/storage/sqlite/client.js.map +1 -1
  57. package/dist/storage/sqliteAsync/client.d.ts +2 -0
  58. package/dist/storage/sqliteAsync/client.d.ts.map +1 -1
  59. package/dist/storage/sqliteAsync/client.js +20 -0
  60. package/dist/storage/sqliteAsync/client.js.map +1 -1
  61. package/dist/storage/storageAsync.d.ts +10 -3
  62. package/dist/storage/storageAsync.d.ts.map +1 -1
  63. package/dist/storage/storageAsync.js +52 -3
  64. package/dist/storage/storageAsync.js.map +1 -1
  65. package/dist/storage/storageSync.d.ts +9 -3
  66. package/dist/storage/storageSync.d.ts.map +1 -1
  67. package/dist/storage/storageSync.js +27 -3
  68. package/dist/storage/storageSync.js.map +1 -1
  69. package/dist/storage/types.d.ts +23 -0
  70. package/dist/storage/types.d.ts.map +1 -1
  71. package/dist/sync.d.ts +23 -0
  72. package/dist/sync.d.ts.map +1 -1
  73. package/dist/sync.js +136 -45
  74. package/dist/sync.js.map +1 -1
  75. package/dist/tests/CojsonMessageChannel.test.d.ts +2 -0
  76. package/dist/tests/CojsonMessageChannel.test.d.ts.map +1 -0
  77. package/dist/tests/CojsonMessageChannel.test.js +236 -0
  78. package/dist/tests/CojsonMessageChannel.test.js.map +1 -0
  79. package/dist/tests/GarbageCollector.test.js +87 -13
  80. package/dist/tests/GarbageCollector.test.js.map +1 -1
  81. package/dist/tests/StorageApiAsync.test.js +124 -1
  82. package/dist/tests/StorageApiAsync.test.js.map +1 -1
  83. package/dist/tests/StorageApiSync.test.js +123 -0
  84. package/dist/tests/StorageApiSync.test.js.map +1 -1
  85. package/dist/tests/SyncManager.processQueues.test.js +1 -1
  86. package/dist/tests/SyncManager.processQueues.test.js.map +1 -1
  87. package/dist/tests/SyncStateManager.test.js +1 -1
  88. package/dist/tests/SyncStateManager.test.js.map +1 -1
  89. package/dist/tests/coPlainText.test.js +1 -1
  90. package/dist/tests/coPlainText.test.js.map +1 -1
  91. package/dist/tests/coValueCore.loadFromStorage.test.js +2 -0
  92. package/dist/tests/coValueCore.loadFromStorage.test.js.map +1 -1
  93. package/dist/tests/knownState.lazyLoading.test.d.ts +2 -0
  94. package/dist/tests/knownState.lazyLoading.test.d.ts.map +1 -0
  95. package/dist/tests/knownState.lazyLoading.test.js +167 -0
  96. package/dist/tests/knownState.lazyLoading.test.js.map +1 -0
  97. package/dist/tests/messagesTestUtils.d.ts +5 -2
  98. package/dist/tests/messagesTestUtils.d.ts.map +1 -1
  99. package/dist/tests/messagesTestUtils.js +4 -0
  100. package/dist/tests/messagesTestUtils.js.map +1 -1
  101. package/dist/tests/sync.garbageCollection.test.js +56 -32
  102. package/dist/tests/sync.garbageCollection.test.js.map +1 -1
  103. package/dist/tests/sync.load.test.js +387 -1
  104. package/dist/tests/sync.load.test.js.map +1 -1
  105. package/dist/tests/sync.mesh.test.js +5 -5
  106. package/dist/tests/sync.mesh.test.js.map +1 -1
  107. package/dist/tests/sync.peerReconciliation.test.js +3 -3
  108. package/dist/tests/sync.peerReconciliation.test.js.map +1 -1
  109. package/dist/tests/sync.storage.test.js +9 -9
  110. package/dist/tests/sync.storage.test.js.map +1 -1
  111. package/dist/tests/sync.storageAsync.test.js +7 -7
  112. package/dist/tests/sync.storageAsync.test.js.map +1 -1
  113. package/dist/tests/sync.tracking.test.js +35 -4
  114. package/dist/tests/sync.tracking.test.js.map +1 -1
  115. package/dist/tests/testStorage.js +38 -2
  116. package/dist/tests/testStorage.js.map +1 -1
  117. package/dist/tests/testUtils.d.ts +38 -4
  118. package/dist/tests/testUtils.d.ts.map +1 -1
  119. package/dist/tests/testUtils.js +68 -7
  120. package/dist/tests/testUtils.js.map +1 -1
  121. package/package.json +4 -4
  122. package/src/CojsonMessageChannel/CojsonMessageChannel.ts +332 -0
  123. package/src/CojsonMessageChannel/MessagePortOutgoingChannel.ts +52 -0
  124. package/src/CojsonMessageChannel/index.ts +9 -0
  125. package/src/CojsonMessageChannel/types.ts +200 -0
  126. package/src/GarbageCollector.ts +5 -5
  127. package/src/SyncStateManager.ts +6 -6
  128. package/src/coValueCore/coValueCore.ts +56 -7
  129. package/src/coValues/account.ts +12 -14
  130. package/src/exports.ts +1 -0
  131. package/src/ids.ts +1 -1
  132. package/src/knownState.ts +24 -0
  133. package/src/localNode.ts +12 -7
  134. package/src/storage/knownState.ts +12 -0
  135. package/src/storage/sqlite/client.ts +31 -0
  136. package/src/storage/sqliteAsync/client.ts +35 -0
  137. package/src/storage/storageAsync.ts +66 -4
  138. package/src/storage/storageSync.ts +37 -4
  139. package/src/storage/types.ts +32 -0
  140. package/src/sync.ts +159 -46
  141. package/src/tests/CojsonMessageChannel.test.ts +306 -0
  142. package/src/tests/GarbageCollector.test.ts +114 -13
  143. package/src/tests/StorageApiAsync.test.ts +186 -1
  144. package/src/tests/StorageApiSync.test.ts +181 -0
  145. package/src/tests/SyncManager.processQueues.test.ts +1 -1
  146. package/src/tests/SyncStateManager.test.ts +1 -1
  147. package/src/tests/coPlainText.test.ts +1 -1
  148. package/src/tests/coValueCore.loadFromStorage.test.ts +5 -0
  149. package/src/tests/knownState.lazyLoading.test.ts +219 -0
  150. package/src/tests/messagesTestUtils.ts +10 -3
  151. package/src/tests/sync.garbageCollection.test.ts +69 -36
  152. package/src/tests/sync.load.test.ts +482 -2
  153. package/src/tests/sync.mesh.test.ts +5 -5
  154. package/src/tests/sync.peerReconciliation.test.ts +3 -3
  155. package/src/tests/sync.storage.test.ts +9 -9
  156. package/src/tests/sync.storageAsync.test.ts +7 -7
  157. package/src/tests/sync.tracking.test.ts +54 -4
  158. package/src/tests/testStorage.ts +40 -2
  159. package/src/tests/testUtils.ts +99 -8
@@ -4,7 +4,7 @@ import {
4
4
  CO_VALUE_LOADING_CONFIG,
5
5
  setCoValueLoadingRetryDelay,
6
6
  } from "../config";
7
- import { RawCoMap } from "../exports";
7
+ import { CojsonInternalTypes, RawCoMap, SessionID } from "../exports";
8
8
  import {
9
9
  SyncMessagesLog,
10
10
  TEST_NODE_CONFIG,
@@ -514,7 +514,7 @@ describe("loading coValues from server", () => {
514
514
  });
515
515
 
516
516
  // Makes the CoValues unavailable on the server
517
- jazzCloud.restart();
517
+ await jazzCloud.restart();
518
518
 
519
519
  const client = setupTestNode({
520
520
  connected: true,
@@ -1452,3 +1452,483 @@ describe("loading coValues from server", () => {
1452
1452
  expect(shardedCoreNode.hasCoValue(group.id)).toBe(false);
1453
1453
  });
1454
1454
  });
1455
+
1456
+ describe("lazy storage load optimization", () => {
1457
+ test("handleLoad skips full load when peer already has all content", async () => {
1458
+ // Setup server with storage
1459
+ const { storage } = jazzCloud.addStorage({ ourName: "server" });
1460
+
1461
+ // Create content on server and sync to storage
1462
+ const group = jazzCloud.node.createGroup();
1463
+ const map = group.createMap();
1464
+ map.set("hello", "world", "trusting");
1465
+ await map.core.waitForSync();
1466
+
1467
+ // Setup client and load the content (client now has everything)
1468
+ const client = setupTestNode({
1469
+ connected: true,
1470
+ });
1471
+ const mapOnClient = await loadCoValueOrFail(client.node, map.id);
1472
+ expect(mapOnClient.get("hello")).toEqual("world");
1473
+
1474
+ // Disconnect client
1475
+ client.disconnect();
1476
+
1477
+ // Restart the server to clear memory (keeping storage)
1478
+ // Now the server has no CoValues in memory, only in storage
1479
+ await jazzCloud.restart();
1480
+ jazzCloud.node.setStorage(storage);
1481
+
1482
+ SyncMessagesLog.clear();
1483
+
1484
+ // Reconnect client - it will send LOAD with its knownState
1485
+ // Server should use LAZY_LOAD to check storage and see peer already has everything
1486
+ client.connectToSyncServer();
1487
+
1488
+ await client.node.syncManager.waitForAllCoValuesSync();
1489
+
1490
+ // Verify the flow: LAZY_LOAD checks storage to get knownState,
1491
+ // sees peer already has everything, responds with KNOWN (skips full LOAD)
1492
+ expect(
1493
+ SyncMessagesLog.getMessages({
1494
+ Group: group.core,
1495
+ Map: map.core,
1496
+ }),
1497
+ ).toMatchInlineSnapshot(`
1498
+ [
1499
+ "client -> server | LOAD Group sessions: header/3",
1500
+ "client -> server | LOAD Map sessions: header/1",
1501
+ "server -> storage | GET_KNOWN_STATE Group",
1502
+ "storage -> server | GET_KNOWN_STATE_RESULT Group sessions: header/3",
1503
+ "server -> client | KNOWN Group sessions: header/3",
1504
+ "server -> storage | GET_KNOWN_STATE Map",
1505
+ "storage -> server | GET_KNOWN_STATE_RESULT Map sessions: header/1",
1506
+ "server -> client | KNOWN Map sessions: header/1",
1507
+ ]
1508
+ `);
1509
+ });
1510
+
1511
+ test("handleLoad does full load when peer needs content", async () => {
1512
+ // Setup server with storage
1513
+ const { storage } = jazzCloud.addStorage({ ourName: "server" });
1514
+
1515
+ // Create content on server and sync to storage
1516
+ const group = jazzCloud.node.createGroup();
1517
+ const map = group.createMap();
1518
+ map.set("hello", "world", "trusting");
1519
+ await map.core.waitForSync();
1520
+
1521
+ // Restart the server to clear memory (keeping storage)
1522
+ await jazzCloud.restart();
1523
+ jazzCloud.node.setStorage(storage);
1524
+
1525
+ SyncMessagesLog.clear();
1526
+
1527
+ // Setup client without any data
1528
+ const client = setupTestNode({
1529
+ connected: true,
1530
+ });
1531
+
1532
+ // Client requests a load - server needs to load from storage and send content
1533
+ const mapOnClient = await loadCoValueOrFail(client.node, map.id);
1534
+ expect(mapOnClient.get("hello")).toEqual("world");
1535
+
1536
+ // Verify the flow:
1537
+ // 1. Client sends LOAD with empty sessions (no header)
1538
+ // 2. Server skips LAZY_LOAD since peer has no content - goes directly to full LOAD
1539
+ // 3. Server sends CONTENT to client
1540
+ expect(
1541
+ SyncMessagesLog.getMessages({
1542
+ Group: group.core,
1543
+ Map: map.core,
1544
+ }),
1545
+ ).toMatchInlineSnapshot(`
1546
+ [
1547
+ "client -> server | LOAD Map sessions: empty",
1548
+ "server -> storage | LOAD Map sessions: empty",
1549
+ "storage -> server | CONTENT Group header: true new: After: 0 New: 3",
1550
+ "storage -> server | CONTENT Map header: true new: After: 0 New: 1",
1551
+ "server -> client | CONTENT Group header: true new: After: 0 New: 3",
1552
+ "server -> client | CONTENT Map header: true new: After: 0 New: 1",
1553
+ "client -> server | KNOWN Group sessions: header/3",
1554
+ "client -> server | KNOWN Map sessions: header/1",
1555
+ ]
1556
+ `);
1557
+ });
1558
+
1559
+ test("handleLoad falls back to peers when not in storage", async () => {
1560
+ // Setup server WITHOUT storage
1561
+ // Create content on server (in memory only)
1562
+ const group = jazzCloud.node.createGroup();
1563
+ const map = group.createMap();
1564
+ map.set("hello", "world", "trusting");
1565
+
1566
+ SyncMessagesLog.clear();
1567
+
1568
+ // Setup client
1569
+ const client = setupTestNode({
1570
+ connected: true,
1571
+ });
1572
+
1573
+ // Client requests a load - server should respond from memory
1574
+ const mapOnClient = await loadCoValueOrFail(client.node, map.id);
1575
+ expect(mapOnClient.get("hello")).toEqual("world");
1576
+
1577
+ // Verify the content was delivered
1578
+ expect(
1579
+ SyncMessagesLog.getMessages({
1580
+ Group: group.core,
1581
+ Map: map.core,
1582
+ }),
1583
+ ).toMatchInlineSnapshot(`
1584
+ [
1585
+ "client -> server | LOAD Map sessions: empty",
1586
+ "server -> client | CONTENT Group header: true new: After: 0 New: 3",
1587
+ "server -> client | CONTENT Map header: true new: After: 0 New: 1",
1588
+ "client -> server | KNOWN Group sessions: header/3",
1589
+ "client -> server | KNOWN Map sessions: header/1",
1590
+ ]
1591
+ `);
1592
+ });
1593
+
1594
+ test("handleNewContent loads from storage for garbage-collected CoValues", async () => {
1595
+ // Setup server with storage
1596
+ jazzCloud.addStorage({ ourName: "server" });
1597
+
1598
+ // Create content on server and sync to storage
1599
+ const group = jazzCloud.node.createGroup();
1600
+ group.addMember("everyone", "writer");
1601
+ const map = group.createMap();
1602
+ map.set("initial", "value", "trusting");
1603
+ await map.core.waitForSync();
1604
+
1605
+ // Verify storage has the data
1606
+ expect(jazzCloud.node.storage).toBeDefined();
1607
+
1608
+ // Load the content on a client first to get the knownState
1609
+ const client1 = setupTestNode({
1610
+ connected: true,
1611
+ });
1612
+ const mapOnClient1 = await loadCoValueOrFail(client1.node, map.id);
1613
+ expect(mapOnClient1.get("initial")).toEqual("value");
1614
+
1615
+ // Now simulate the CoValue being garbage collected from server memory
1616
+ // by removing it and then receiving new content from a different client
1617
+ jazzCloud.node.internalDeleteCoValue(map.id);
1618
+
1619
+ // Clear messages to track what happens next
1620
+ SyncMessagesLog.clear();
1621
+
1622
+ // Have client1 make an update - this should trigger handleNewContent
1623
+ // which should load from storage since the CoValue was "garbage collected"
1624
+ mapOnClient1.set("new", "update", "trusting");
1625
+
1626
+ await waitFor(() => {
1627
+ // The server should have reloaded from storage and processed the update
1628
+ const serverMap = jazzCloud.node.getCoValue(map.id);
1629
+ return serverMap.isAvailable();
1630
+ });
1631
+
1632
+ // Verify the server has the updated content
1633
+ const serverMap = jazzCloud.node.getCoValue(map.id);
1634
+ expect(serverMap.isAvailable()).toBe(true);
1635
+
1636
+ // Verify that the server did a full load from storage
1637
+ expect(
1638
+ SyncMessagesLog.getMessages({
1639
+ Group: group.core,
1640
+ Map: map.core,
1641
+ }),
1642
+ ).toMatchInlineSnapshot(`
1643
+ [
1644
+ "client -> server | CONTENT Map header: false new: After: 0 New: 1",
1645
+ "server -> storage | LOAD Map sessions: empty",
1646
+ "storage -> server | CONTENT Map header: true new: After: 0 New: 1",
1647
+ "server -> client | KNOWN Map sessions: header/2",
1648
+ "server -> storage | CONTENT Map header: false new: After: 0 New: 1",
1649
+ ]
1650
+ `);
1651
+ });
1652
+
1653
+ test("handleNewContent loads large CoValue from storage when garbage-collected", async () => {
1654
+ // Setup server with storage
1655
+ jazzCloud.addStorage({ ourName: "server" });
1656
+
1657
+ // Create a large map on server and sync to storage
1658
+ const group = jazzCloud.node.createGroup();
1659
+ group.addMember("everyone", "writer");
1660
+ const largeMap = group.createMap();
1661
+ fillCoMapWithLargeData(largeMap);
1662
+ await largeMap.core.waitForSync();
1663
+
1664
+ // Verify storage has the data
1665
+ expect(jazzCloud.node.storage).toBeDefined();
1666
+
1667
+ // Load the content on a client first
1668
+ const client1 = setupTestNode({
1669
+ connected: true,
1670
+ });
1671
+ const mapOnClient1 = await loadCoValueOrFail(client1.node, largeMap.id);
1672
+ await mapOnClient1.core.waitForFullStreaming();
1673
+
1674
+ // Simulate the CoValue being garbage collected from server memory
1675
+ jazzCloud.node.internalDeleteCoValue(largeMap.id);
1676
+
1677
+ // Clear messages to track what happens next
1678
+ SyncMessagesLog.clear();
1679
+
1680
+ // Have client1 make an update - this should trigger handleNewContent
1681
+ // which should load from storage (streaming) since the CoValue was "garbage collected"
1682
+ mapOnClient1.set("new", "update", "trusting");
1683
+
1684
+ await waitFor(() => {
1685
+ // The server should have reloaded from storage and processed the update
1686
+ const serverMap = jazzCloud.node.getCoValue(largeMap.id);
1687
+ return serverMap.isAvailable();
1688
+ });
1689
+
1690
+ // Verify the server has the updated content
1691
+ const serverMap = jazzCloud.node.getCoValue(largeMap.id);
1692
+ expect(serverMap.isAvailable()).toBe(true);
1693
+
1694
+ // Verify that the server did a full load from storage (with streaming for large data)
1695
+ expect(
1696
+ SyncMessagesLog.getMessages({
1697
+ Group: group.core,
1698
+ Map: largeMap.core,
1699
+ }),
1700
+ ).toMatchInlineSnapshot(`
1701
+ [
1702
+ "client -> server | CONTENT Map header: false new: After: 0 New: 1",
1703
+ "server -> storage | LOAD Map sessions: empty",
1704
+ "storage -> server | CONTENT Map header: true new: After: 0 New: 73 expectContentUntil: header/201",
1705
+ "server -> client | KNOWN Map sessions: header/74",
1706
+ "server -> storage | CONTENT Map header: false new: After: 0 New: 1",
1707
+ "storage -> server | CONTENT Map header: true new: After: 73 New: 73",
1708
+ "storage -> server | CONTENT Map header: true new: After: 146 New: 54",
1709
+ ]
1710
+ `);
1711
+ });
1712
+
1713
+ test("handleNewContent loads CoValue from storage when group is large and garbage-collected", async () => {
1714
+ // Setup server with storage
1715
+ jazzCloud.addStorage({ ourName: "server" });
1716
+
1717
+ // Create a group with large data
1718
+ const group = jazzCloud.node.createGroup();
1719
+ group.addMember("everyone", "writer");
1720
+
1721
+ // Add large data to the group itself
1722
+ for (let i = 0; i < 200; i++) {
1723
+ const value = Buffer.alloc(1024, `value${i}`).toString("base64");
1724
+ group.set(`key${i}` as any, value as never, "trusting");
1725
+ }
1726
+
1727
+ const map = group.createMap();
1728
+ map.set("initial", "value", "trusting");
1729
+
1730
+ await map.core.waitForSync();
1731
+ await group.core.waitForSync();
1732
+
1733
+ // Verify storage has the data
1734
+ expect(jazzCloud.node.storage).toBeDefined();
1735
+
1736
+ // Load the content on a client first
1737
+ const client1 = setupTestNode({
1738
+ connected: true,
1739
+ });
1740
+ const mapOnClient1 = await loadCoValueOrFail(client1.node, map.id);
1741
+ expect(mapOnClient1.get("initial")).toEqual("value");
1742
+
1743
+ // Wait for the group to finish streaming
1744
+ const groupOnClient1 = client1.node.getCoValue(group.id);
1745
+ await groupOnClient1.waitForAvailableOrUnavailable();
1746
+
1747
+ // Simulate the map being garbage collected from server memory
1748
+ // The group should also be deleted to force reload from storage
1749
+ jazzCloud.node.internalDeleteCoValue(map.id);
1750
+ jazzCloud.node.internalDeleteCoValue(group.id);
1751
+
1752
+ // Clear messages to track what happens next
1753
+ SyncMessagesLog.clear();
1754
+
1755
+ // Have client1 make an update - this should trigger handleNewContent
1756
+ // which should load from storage (with the large group streaming)
1757
+ mapOnClient1.set("new", "update", "trusting");
1758
+
1759
+ await waitFor(() => {
1760
+ // The server should have reloaded from storage and processed the update
1761
+ const serverMap = jazzCloud.node.getCoValue(map.id);
1762
+ return serverMap.isAvailable();
1763
+ });
1764
+
1765
+ // Verify the server has the updated content
1766
+ const serverMap = jazzCloud.node.getCoValue(map.id);
1767
+ expect(serverMap.isAvailable()).toBe(true);
1768
+
1769
+ // Verify that the server did a full load from storage
1770
+ // Note: No LAZY_LOAD here because the content message has no header (it's an update),
1771
+ // so the server goes directly to full LOAD
1772
+ expect(
1773
+ SyncMessagesLog.getMessages({
1774
+ Group: group.core,
1775
+ Map: map.core,
1776
+ }),
1777
+ ).toMatchInlineSnapshot(`
1778
+ [
1779
+ "client -> server | CONTENT Map header: false new: After: 0 New: 1",
1780
+ "server -> storage | LOAD Map sessions: empty",
1781
+ "storage -> server | CONTENT Group header: true new: After: 0 New: 78 expectContentUntil: header/205",
1782
+ "storage -> server | CONTENT Map header: true new: After: 0 New: 1",
1783
+ "server -> client | KNOWN Map sessions: header/2",
1784
+ "server -> storage | CONTENT Map header: false new: After: 0 New: 1",
1785
+ "storage -> server | CONTENT Group header: true new: After: 78 New: 73",
1786
+ "storage -> server | CONTENT Group header: true new: After: 151 New: 54",
1787
+ ]
1788
+ `);
1789
+ });
1790
+
1791
+ test("handleNewContent loads CoValue from storage when parent group is large and garbage-collected", async () => {
1792
+ // Setup server with storage
1793
+ jazzCloud.addStorage({ ourName: "server" });
1794
+
1795
+ // Create parent group with large data
1796
+ const parentGroup = jazzCloud.node.createGroup();
1797
+ parentGroup.addMember("everyone", "reader");
1798
+
1799
+ // Add large data to the parent group
1800
+ fillCoMapWithLargeData(parentGroup);
1801
+
1802
+ // Create child group that extends parent
1803
+ const group = jazzCloud.node.createGroup();
1804
+ group.addMember("everyone", "writer");
1805
+ group.extend(parentGroup);
1806
+
1807
+ const map = group.createMap();
1808
+ map.set("initial", "value", "trusting");
1809
+
1810
+ await map.core.waitForSync();
1811
+ await group.core.waitForSync();
1812
+ await parentGroup.core.waitForSync();
1813
+
1814
+ // Verify storage has the data
1815
+ expect(jazzCloud.node.storage).toBeDefined();
1816
+
1817
+ // Load the content on a client first
1818
+ const client1 = setupTestNode({
1819
+ connected: true,
1820
+ });
1821
+ const mapOnClient1 = await loadCoValueOrFail(client1.node, map.id);
1822
+ expect(mapOnClient1.get("initial")).toEqual("value");
1823
+
1824
+ // Wait for the parent group to finish streaming
1825
+ const parentGroupOnClient1 = client1.node.getCoValue(parentGroup.id);
1826
+ await parentGroupOnClient1.waitForAvailableOrUnavailable();
1827
+ if (parentGroupOnClient1.isAvailable()) {
1828
+ await parentGroupOnClient1.waitForFullStreaming();
1829
+ }
1830
+
1831
+ // Simulate CoValues being garbage collected from server memory
1832
+ jazzCloud.node.internalDeleteCoValue(map.id);
1833
+ jazzCloud.node.internalDeleteCoValue(group.id);
1834
+ jazzCloud.node.internalDeleteCoValue(parentGroup.id);
1835
+
1836
+ // Clear messages to track what happens next
1837
+ SyncMessagesLog.clear();
1838
+
1839
+ // Have client1 make an update - this should trigger handleNewContent
1840
+ // which should load from storage (with the large parent group streaming)
1841
+ mapOnClient1.set("new", "update", "trusting");
1842
+
1843
+ await waitFor(() => {
1844
+ // The server should have reloaded from storage and processed the update
1845
+ const serverMap = jazzCloud.node.getCoValue(map.id);
1846
+ return serverMap.isAvailable();
1847
+ });
1848
+
1849
+ // Verify the server has the updated content
1850
+ const serverMap = jazzCloud.node.getCoValue(map.id);
1851
+ expect(serverMap.isAvailable()).toBe(true);
1852
+
1853
+ // Verify that the server did a full load from storage for all CoValues
1854
+ // The snapshot shows the complete flow: loading Map triggers loading its dependencies
1855
+ expect(
1856
+ SyncMessagesLog.getMessages({
1857
+ ParentGroup: parentGroup.core,
1858
+ Group: group.core,
1859
+ Map: map.core,
1860
+ }),
1861
+ ).toMatchInlineSnapshot(`
1862
+ [
1863
+ "client -> server | CONTENT Map header: false new: After: 0 New: 1",
1864
+ "server -> storage | LOAD Map sessions: empty",
1865
+ "storage -> server | CONTENT ParentGroup header: true new: After: 0 New: 78 expectContentUntil: header/205",
1866
+ "storage -> server | CONTENT Group header: true new: After: 0 New: 7",
1867
+ "storage -> server | CONTENT Map header: true new: After: 0 New: 1",
1868
+ "server -> client | KNOWN Map sessions: header/2",
1869
+ "server -> storage | CONTENT Map header: false new: After: 0 New: 1",
1870
+ "storage -> server | CONTENT ParentGroup header: true new: After: 78 New: 73",
1871
+ "storage -> server | CONTENT ParentGroup header: true new: After: 151 New: 54",
1872
+ ]
1873
+ `);
1874
+ });
1875
+
1876
+ test("handles gracefully when CoValue is garbage collected mid-stream from storage", async () => {
1877
+ // This test verifies the edge case where:
1878
+ // 1. Storage is streaming a large CoValue in chunks
1879
+ // 2. The CoValue is garbage collected mid-stream
1880
+ // 3. Subsequent chunks from storage (without header) should be handled gracefully
1881
+
1882
+ // Setup server with storage
1883
+ jazzCloud.addStorage({ ourName: "server" });
1884
+
1885
+ // Create a large CoValue that will stream
1886
+ const group = jazzCloud.node.createGroup();
1887
+ group.addMember("everyone", "writer");
1888
+ const largeMap = group.createMap();
1889
+ fillCoMapWithLargeData(largeMap);
1890
+ await largeMap.core.waitForSync();
1891
+
1892
+ // Get the sessions from the largeMap to create a realistic streaming chunk
1893
+ const sessions = Object.entries(largeMap.core.knownState().sessions);
1894
+ const [sessionId, txCount] = sessions[0]!;
1895
+
1896
+ // Simulate receiving a streaming chunk from storage for a non-existent CoValue
1897
+ // This happens when:
1898
+ // 1. A large CoValue starts streaming from storage
1899
+ // 2. The CoValue gets garbage collected mid-stream
1900
+ // 3. Remaining chunks arrive with no header (they're continuation chunks)
1901
+
1902
+ // First, ensure the CoValue doesn't exist in server memory
1903
+ jazzCloud.node.internalDeleteCoValue(largeMap.id);
1904
+ expect(jazzCloud.node.hasCoValue(largeMap.id)).toBe(false);
1905
+
1906
+ // Now simulate a streaming chunk arriving from storage without a header
1907
+ // This is what happens when GC runs between streaming chunks
1908
+ const streamingChunk = {
1909
+ action: "content" as const,
1910
+ id: largeMap.id,
1911
+ header: undefined, // No header - it's a continuation chunk
1912
+ priority: 0 as const,
1913
+ new: {
1914
+ [sessionId as SessionID]: {
1915
+ after: Math.floor(txCount / 2), // Middle of the stream
1916
+ newTransactions: [],
1917
+ lastSignature: "test" as CojsonInternalTypes.Signature,
1918
+ },
1919
+ },
1920
+ };
1921
+
1922
+ // Call handleNewContent directly with the storage message
1923
+ // This should NOT crash, just log a warning and return early
1924
+ jazzCloud.node.syncManager.handleNewContent(streamingChunk, "storage");
1925
+
1926
+ // The CoValue entry gets created by getOrCreateCoValue, but it should
1927
+ // NOT be available (the chunk was ignored because it had no header)
1928
+ const coValue = jazzCloud.node.getCoValue(largeMap.id);
1929
+ expect(coValue).toBeDefined();
1930
+ expect(coValue?.isAvailable()).toBe(false);
1931
+
1932
+ // Test passes if we reach here without crashing
1933
+ });
1934
+ });
@@ -506,7 +506,7 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
506
506
  ]
507
507
  `);
508
508
 
509
- edge.restart();
509
+ await edge.restart();
510
510
 
511
511
  edge.connectToSyncServer({
512
512
  syncServerName: "core",
@@ -561,11 +561,11 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
561
561
  "edge -> client | CONTENT Map header: false new: After: 63 New: 21 expectContentUntil: header/100",
562
562
  "storage -> edge | CONTENT Map header: true new: After: 84 New: 16",
563
563
  "edge -> client | CONTENT Map header: false new: After: 84 New: 16",
564
- "core -> storage | LOAD Group sessions: empty",
565
- "storage -> core | KNOWN Group sessions: empty",
564
+ "core -> storage | GET_KNOWN_STATE Group",
565
+ "storage -> core | GET_KNOWN_STATE_RESULT Group sessions: empty",
566
566
  "core -> edge | KNOWN Group sessions: empty",
567
- "core -> storage | LOAD Map sessions: empty",
568
- "storage -> core | KNOWN Map sessions: empty",
567
+ "core -> storage | GET_KNOWN_STATE Map",
568
+ "storage -> core | GET_KNOWN_STATE_RESULT Map sessions: empty",
569
569
  "core -> edge | KNOWN Map sessions: empty",
570
570
  "client -> edge | KNOWN Group sessions: header/5",
571
571
  "client -> storage | CONTENT Group header: true new: After: 0 New: 5",
@@ -167,7 +167,7 @@ describe("peer reconciliation", () => {
167
167
 
168
168
  await map.core.waitForSync();
169
169
 
170
- jazzCloud.restart();
170
+ await jazzCloud.restart();
171
171
  SyncMessagesLog.clear();
172
172
  client.connectToSyncServer();
173
173
 
@@ -222,7 +222,7 @@ describe("peer reconciliation", () => {
222
222
 
223
223
  await map.core.waitForSync();
224
224
 
225
- jazzCloud.restart();
225
+ await jazzCloud.restart();
226
226
  SyncMessagesLog.clear();
227
227
  client.connectToSyncServer();
228
228
 
@@ -305,7 +305,7 @@ describe("peer reconciliation", () => {
305
305
 
306
306
  await map.core.waitForSync();
307
307
 
308
- jazzCloud.restart();
308
+ await jazzCloud.restart();
309
309
 
310
310
  SyncMessagesLog.clear();
311
311
  client.connectToSyncServer();
@@ -83,7 +83,7 @@ describe("client with storage syncs with server", () => {
83
83
 
84
84
  await loadCoValueOrFail(client.node, map.id);
85
85
 
86
- client.restart();
86
+ await client.restart();
87
87
 
88
88
  client.connectToSyncServer();
89
89
  client.addStorage({
@@ -167,7 +167,7 @@ describe("client with storage syncs with server", () => {
167
167
 
168
168
  await map.core.waitForSync();
169
169
 
170
- client.restart();
170
+ await client.restart();
171
171
 
172
172
  client.addStorage({
173
173
  storage,
@@ -217,7 +217,7 @@ describe("client with storage syncs with server", () => {
217
217
  branch.set("branchKey", "branchValue");
218
218
  await branch.core.waitForSync();
219
219
 
220
- client.restart();
220
+ await client.restart();
221
221
  client.addStorage({
222
222
  storage,
223
223
  });
@@ -388,7 +388,7 @@ describe("client syncs with a server with storage", () => {
388
388
 
389
389
  SyncMessagesLog.clear();
390
390
 
391
- client.restart();
391
+ await client.restart();
392
392
 
393
393
  client.connectToSyncServer({
394
394
  ourName: "client",
@@ -457,7 +457,7 @@ describe("client syncs with a server with storage", () => {
457
457
 
458
458
  expect(correctionSpy).not.toHaveBeenCalled();
459
459
 
460
- client.restart();
460
+ await client.restart();
461
461
 
462
462
  client.connectToSyncServer({
463
463
  ourName: "client",
@@ -771,7 +771,7 @@ describe("client syncs with a server with storage", () => {
771
771
 
772
772
  SyncMessagesLog.clear();
773
773
 
774
- syncServer.restart();
774
+ await syncServer.restart();
775
775
  syncServer.addStorage({
776
776
  ourName: "syncServer",
777
777
  storage,
@@ -848,7 +848,7 @@ describe("client syncs with a server with storage", () => {
848
848
  ]);
849
849
 
850
850
  // Restart to load from storage
851
- client.restart();
851
+ await client.restart();
852
852
  client.addStorage({ storage });
853
853
 
854
854
  // Load all maps concurrently from storage
@@ -892,7 +892,7 @@ describe("client syncs with a server with storage", () => {
892
892
  SyncMessagesLog.clear();
893
893
 
894
894
  // Restart client with storage
895
- client.restart();
895
+ await client.restart();
896
896
  client.connectToSyncServer();
897
897
  client.addStorage({ storage });
898
898
 
@@ -985,7 +985,7 @@ describe("client syncs with a server with storage", () => {
985
985
 
986
986
  SyncMessagesLog.clear();
987
987
 
988
- syncServer.restart();
988
+ await syncServer.restart();
989
989
  syncServer.addStorage({
990
990
  ourName: "syncServer",
991
991
  storage,
@@ -72,7 +72,7 @@ describe("client with storage syncs with server", () => {
72
72
  const firstLoad = await loadCoValueOrFail(client.node, map.id);
73
73
  await firstLoad.core.waitForSync(); // Need to wait for sync with storage
74
74
 
75
- client.restart();
75
+ await client.restart();
76
76
 
77
77
  client.connectToSyncServer();
78
78
  client.addStorage({
@@ -311,7 +311,7 @@ describe("client syncs with a server with storage", () => {
311
311
 
312
312
  SyncMessagesLog.clear();
313
313
 
314
- client.restart();
314
+ await client.restart();
315
315
 
316
316
  client.connectToSyncServer({
317
317
  ourName: "client",
@@ -406,7 +406,7 @@ describe("client syncs with a server with storage", () => {
406
406
  const largeMapContent =
407
407
  largeMap.core.newContentSince(undefined)?.slice(0, 4) ?? [];
408
408
 
409
- client.restart();
409
+ await client.restart();
410
410
 
411
411
  const newSyncServer = setupTestNode({
412
412
  isSyncServer: true,
@@ -518,7 +518,7 @@ describe("client syncs with a server with storage", () => {
518
518
 
519
519
  expect(correctionSpy).not.toHaveBeenCalled();
520
520
 
521
- client.restart();
521
+ await client.restart();
522
522
 
523
523
  client.connectToSyncServer({
524
524
  ourName: "client",
@@ -566,14 +566,14 @@ describe("client syncs with a server with storage", () => {
566
566
 
567
567
  await largeMap.core.waitForSync();
568
568
 
569
- server.restart();
569
+ await server.restart();
570
570
 
571
571
  server.addStorage({
572
572
  ourName: "server",
573
573
  storage: serverStorage,
574
574
  });
575
575
 
576
- client.restart();
576
+ await client.restart();
577
577
 
578
578
  client.connectToSyncServer({
579
579
  ourName: "client",
@@ -703,7 +703,7 @@ describe("client syncs with a server with storage", () => {
703
703
 
704
704
  SyncMessagesLog.clear();
705
705
 
706
- syncServer.restart();
706
+ await syncServer.restart();
707
707
  syncServer.addStorage({
708
708
  ourName: "syncServer",
709
709
  storage,