cojson 0.18.32 → 0.18.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/.turbo/turbo-build.log +1 -1
  2. package/CHANGELOG.md +9 -0
  3. package/dist/SyncStateManager.d.ts.map +1 -1
  4. package/dist/SyncStateManager.js +2 -2
  5. package/dist/SyncStateManager.js.map +1 -1
  6. package/dist/coValueCore/SessionMap.d.ts +1 -0
  7. package/dist/coValueCore/SessionMap.d.ts.map +1 -1
  8. package/dist/coValueCore/SessionMap.js +17 -1
  9. package/dist/coValueCore/SessionMap.js.map +1 -1
  10. package/dist/coValueCore/coValueCore.d.ts +14 -9
  11. package/dist/coValueCore/coValueCore.d.ts.map +1 -1
  12. package/dist/coValueCore/coValueCore.js +62 -47
  13. package/dist/coValueCore/coValueCore.js.map +1 -1
  14. package/dist/coValueCore/verifiedState.d.ts +2 -2
  15. package/dist/coValueCore/verifiedState.d.ts.map +1 -1
  16. package/dist/coValueCore/verifiedState.js +86 -75
  17. package/dist/coValueCore/verifiedState.js.map +1 -1
  18. package/dist/coValues/group.d.ts +1 -0
  19. package/dist/coValues/group.d.ts.map +1 -1
  20. package/dist/coValues/group.js +24 -4
  21. package/dist/coValues/group.js.map +1 -1
  22. package/dist/knownState.d.ts +1 -1
  23. package/dist/knownState.d.ts.map +1 -1
  24. package/dist/knownState.js +1 -1
  25. package/dist/knownState.js.map +1 -1
  26. package/dist/localNode.d.ts.map +1 -1
  27. package/dist/localNode.js +1 -2
  28. package/dist/localNode.js.map +1 -1
  29. package/dist/queue/LocalTransactionsSyncQueue.d.ts.map +1 -1
  30. package/dist/queue/LocalTransactionsSyncQueue.js +16 -1
  31. package/dist/queue/LocalTransactionsSyncQueue.js.map +1 -1
  32. package/dist/storage/knownState.js +2 -2
  33. package/dist/storage/knownState.js.map +1 -1
  34. package/dist/sync.d.ts.map +1 -1
  35. package/dist/sync.js +8 -2
  36. package/dist/sync.js.map +1 -1
  37. package/dist/tests/PureJSCrypto.test.js +1 -1
  38. package/dist/tests/PureJSCrypto.test.js.map +1 -1
  39. package/dist/tests/StorageApiAsync.test.js +11 -11
  40. package/dist/tests/StorageApiAsync.test.js.map +1 -1
  41. package/dist/tests/StorageApiSync.test.js +3 -3
  42. package/dist/tests/StorageApiSync.test.js.map +1 -1
  43. package/dist/tests/WasmCrypto.test.js +1 -1
  44. package/dist/tests/WasmCrypto.test.js.map +1 -1
  45. package/dist/tests/coPlainText.test.js +1 -1
  46. package/dist/tests/coStream.test.js +12 -12
  47. package/dist/tests/coStream.test.js.map +1 -1
  48. package/dist/tests/coValueCore.isCompletelyDownloaded.test.d.ts +2 -0
  49. package/dist/tests/coValueCore.isCompletelyDownloaded.test.d.ts.map +1 -0
  50. package/dist/tests/coValueCore.isCompletelyDownloaded.test.js +421 -0
  51. package/dist/tests/coValueCore.isCompletelyDownloaded.test.js.map +1 -0
  52. package/dist/tests/coValueCore.isStreaming.test.d.ts +2 -0
  53. package/dist/tests/coValueCore.isStreaming.test.d.ts.map +1 -0
  54. package/dist/tests/coValueCore.isStreaming.test.js +181 -0
  55. package/dist/tests/coValueCore.isStreaming.test.js.map +1 -0
  56. package/dist/tests/coValueCore.newContentSince.test.d.ts +2 -0
  57. package/dist/tests/coValueCore.newContentSince.test.d.ts.map +1 -0
  58. package/dist/tests/coValueCore.newContentSince.test.js +808 -0
  59. package/dist/tests/coValueCore.newContentSince.test.js.map +1 -0
  60. package/dist/tests/coreWasm.test.js +2 -2
  61. package/dist/tests/coreWasm.test.js.map +1 -1
  62. package/dist/tests/group.childKeyRotation.test.d.ts +2 -0
  63. package/dist/tests/group.childKeyRotation.test.d.ts.map +1 -0
  64. package/dist/tests/group.childKeyRotation.test.js +261 -0
  65. package/dist/tests/group.childKeyRotation.test.js.map +1 -0
  66. package/dist/tests/group.removeMember.test.js +1 -114
  67. package/dist/tests/group.removeMember.test.js.map +1 -1
  68. package/dist/tests/knownState.test.js +11 -11
  69. package/dist/tests/knownState.test.js.map +1 -1
  70. package/dist/tests/sync.auth.test.js +6 -6
  71. package/dist/tests/sync.load.test.js +68 -5
  72. package/dist/tests/sync.load.test.js.map +1 -1
  73. package/dist/tests/sync.mesh.test.js +11 -17
  74. package/dist/tests/sync.mesh.test.js.map +1 -1
  75. package/dist/tests/sync.peerReconciliation.test.js +1 -1
  76. package/dist/tests/sync.storage.test.js +7 -7
  77. package/dist/tests/sync.storage.test.js.map +1 -1
  78. package/dist/tests/sync.storageAsync.test.js +4 -4
  79. package/dist/tests/sync.storageAsync.test.js.map +1 -1
  80. package/dist/tests/sync.upload.test.js +96 -40
  81. package/dist/tests/sync.upload.test.js.map +1 -1
  82. package/dist/tests/testUtils.d.ts +2 -0
  83. package/dist/tests/testUtils.d.ts.map +1 -1
  84. package/dist/tests/testUtils.js +22 -1
  85. package/dist/tests/testUtils.js.map +1 -1
  86. package/package.json +3 -3
  87. package/src/SyncStateManager.ts +2 -5
  88. package/src/coValueCore/SessionMap.ts +24 -0
  89. package/src/coValueCore/coValueCore.ts +77 -55
  90. package/src/coValueCore/verifiedState.ts +123 -108
  91. package/src/coValues/group.ts +27 -4
  92. package/src/knownState.ts +1 -1
  93. package/src/localNode.ts +1 -2
  94. package/src/queue/LocalTransactionsSyncQueue.ts +25 -0
  95. package/src/storage/knownState.ts +2 -2
  96. package/src/sync.ts +7 -2
  97. package/src/tests/PureJSCrypto.test.ts +1 -2
  98. package/src/tests/StorageApiAsync.test.ts +11 -11
  99. package/src/tests/StorageApiSync.test.ts +3 -3
  100. package/src/tests/WasmCrypto.test.ts +1 -2
  101. package/src/tests/coPlainText.test.ts +1 -1
  102. package/src/tests/coStream.test.ts +12 -12
  103. package/src/tests/coValueCore.isCompletelyDownloaded.test.ts +589 -0
  104. package/src/tests/coValueCore.isStreaming.test.ts +271 -0
  105. package/src/tests/coValueCore.newContentSince.test.ts +966 -0
  106. package/src/tests/coreWasm.test.ts +2 -2
  107. package/src/tests/group.childKeyRotation.test.ts +431 -0
  108. package/src/tests/group.removeMember.test.ts +1 -184
  109. package/src/tests/knownState.test.ts +11 -11
  110. package/src/tests/sync.auth.test.ts +6 -6
  111. package/src/tests/sync.load.test.ts +80 -5
  112. package/src/tests/sync.mesh.test.ts +11 -17
  113. package/src/tests/sync.peerReconciliation.test.ts +1 -1
  114. package/src/tests/sync.storage.test.ts +7 -7
  115. package/src/tests/sync.storageAsync.test.ts +4 -4
  116. package/src/tests/sync.upload.test.ts +106 -40
  117. package/src/tests/testUtils.ts +24 -2
@@ -4,6 +4,7 @@ import {
4
4
  createContentMessage,
5
5
  exceedsRecommendedSize,
6
6
  getTransactionSize,
7
+ addTransactionToContentMessage,
7
8
  } from "../coValueContentMessage.js";
8
9
  import {
9
10
  CryptoProvider,
@@ -21,7 +22,7 @@ import { NewContentMessage } from "../sync.js";
21
22
  import { TryAddTransactionsError } from "./coValueCore.js";
22
23
  import { SessionLog, SessionMap } from "./SessionMap.js";
23
24
  import { ControlledAccountOrAgent } from "../coValues/account.js";
24
- import { cloneKnownState, CoValueKnownState } from "../knownState.js";
25
+ import { CoValueKnownState, KnownStateSessions } from "../knownState.js";
25
26
 
26
27
  export type CoValueHeader = {
27
28
  type: AnyRawCoValue["type"];
@@ -55,7 +56,6 @@ export class VerifiedState {
55
56
  readonly crypto: CryptoProvider;
56
57
  readonly header: CoValueHeader;
57
58
  readonly sessions: SessionMap;
58
- private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
59
59
  public lastAccessed: number | undefined;
60
60
  public branchSourceId?: RawCoID;
61
61
  public branchName?: string;
@@ -98,10 +98,6 @@ export class VerifiedState {
98
98
  skipVerify,
99
99
  );
100
100
 
101
- if (result.isOk()) {
102
- this._cachedNewContentSinceEmpty = undefined;
103
- }
104
-
105
101
  return result;
106
102
  }
107
103
 
@@ -120,8 +116,6 @@ export class VerifiedState {
120
116
  madeAt,
121
117
  );
122
118
 
123
- this._cachedNewContentSinceEmpty = undefined;
124
-
125
119
  return result;
126
120
  }
127
121
 
@@ -144,8 +138,6 @@ export class VerifiedState {
144
138
  madeAt,
145
139
  );
146
140
 
147
- this._cachedNewContentSinceEmpty = undefined;
148
-
149
141
  return result;
150
142
  }
151
143
 
@@ -160,119 +152,147 @@ export class VerifiedState {
160
152
  );
161
153
  }
162
154
 
155
+ setStreamingKnownState(streamingKnownState: KnownStateSessions) {
156
+ this.sessions.setStreamingKnownState(streamingKnownState);
157
+ }
158
+
163
159
  newContentSince(
164
160
  knownState: CoValueKnownState | undefined,
165
161
  ): NewContentMessage[] | undefined {
166
- const isKnownStateEmpty = !knownState?.header && !knownState?.sessions;
167
-
168
- if (isKnownStateEmpty && this._cachedNewContentSinceEmpty) {
169
- return this._cachedNewContentSinceEmpty;
170
- }
171
-
172
162
  let currentPiece: NewContentMessage = createContentMessage(
173
163
  this.id,
174
164
  this.header,
175
- !knownState?.header,
165
+ false,
176
166
  );
177
-
178
- const pieces = [currentPiece];
179
-
180
- const sentState: CoValueKnownState["sessions"] = {};
181
-
167
+ const pieces: NewContentMessage[] = [currentPiece];
182
168
  let pieceSize = 0;
183
169
 
184
- let sessionsTodoAgain: Set<SessionID> | undefined | "first" = "first";
185
-
186
- while (sessionsTodoAgain === "first" || sessionsTodoAgain?.size || 0 > 0) {
187
- if (sessionsTodoAgain === "first") {
188
- sessionsTodoAgain = undefined;
170
+ const startNewPiece = () => {
171
+ currentPiece = createContentMessage(this.id, this.header, false);
172
+ pieces.push(currentPiece);
173
+ pieceSize = 0;
174
+ };
175
+
176
+ const moveSessionContentToNewPiece = (sessionID: SessionID) => {
177
+ const sessionContent = currentPiece.new[sessionID];
178
+
179
+ if (!sessionContent) {
180
+ throw new Error("Session content not found", {
181
+ cause: {
182
+ sessionID,
183
+ currentPiece,
184
+ },
185
+ });
189
186
  }
190
- const sessionsTodo = sessionsTodoAgain ?? this.sessions.keys();
191
-
192
- for (const sessionIDKey of sessionsTodo) {
193
- const sessionID = sessionIDKey as SessionID;
194
- const log = this.sessions.get(sessionID)!;
195
- const knownStateForSessionID = knownState?.sessions[sessionID];
196
- const sentStateForSessionID = sentState[sessionID];
197
- const nextKnownSignatureIdx = getNextKnownSignatureIdx(
198
- log,
199
- knownStateForSessionID,
200
- sentStateForSessionID,
201
- );
202
-
203
- const firstNewTxIdx =
204
- sentStateForSessionID ?? knownStateForSessionID ?? 0;
205
- const afterLastNewTxIdx =
206
- nextKnownSignatureIdx === undefined
207
- ? log.transactions.length
208
- : nextKnownSignatureIdx + 1;
209
-
210
- const nNewTx = Math.max(0, afterLastNewTxIdx - firstNewTxIdx);
211
-
212
- if (nNewTx === 0) {
213
- sessionsTodoAgain?.delete(sessionID);
214
- continue;
215
- }
216
187
 
217
- if (afterLastNewTxIdx < log.transactions.length) {
218
- if (!sessionsTodoAgain) {
219
- sessionsTodoAgain = new Set();
188
+ delete currentPiece.new[sessionID];
189
+
190
+ const newPiece = createContentMessage(this.id, this.header, false);
191
+ newPiece.new[sessionID] = sessionContent;
192
+
193
+ // Insert the new piece before the current piece, to ensure that the order of the new transactions is preserved
194
+ pieces.splice(pieces.length - 1, 0, newPiece);
195
+ };
196
+
197
+ const sessionSent = knownState?.sessions;
198
+
199
+ for (const [sessionID, log] of this.sessions.sessions) {
200
+ const startFrom = sessionSent?.[sessionID] ?? 0;
201
+
202
+ let currentSessionSize = 0;
203
+
204
+ for (let txIdx = startFrom; txIdx < log.transactions.length; txIdx++) {
205
+ const isLastItem = txIdx === log.transactions.length - 1;
206
+ const tx = log.transactions[txIdx]!;
207
+
208
+ currentSessionSize += getTransactionSize(tx);
209
+
210
+ const signatureAfter = log.signatureAfter[txIdx];
211
+
212
+ if (signatureAfter) {
213
+ addTransactionToContentMessage(
214
+ currentPiece,
215
+ tx,
216
+ sessionID,
217
+ signatureAfter,
218
+ txIdx,
219
+ );
220
+ // When we meet a signatureAfter it means that the transaction log exceeds the recommended size
221
+ // so we move the session content to a dedicated piece, because it must be sent in a standalone piece
222
+ moveSessionContentToNewPiece(sessionID);
223
+ currentSessionSize = 0;
224
+ } else if (isLastItem) {
225
+ if (!log.lastSignature) {
226
+ throw new Error(
227
+ "All the SessionLogs sent must have a lastSignature",
228
+ {
229
+ cause: log,
230
+ },
231
+ );
220
232
  }
221
- sessionsTodoAgain.add(sessionID);
222
- }
223
233
 
224
- const oldPieceSize = pieceSize;
225
- for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
226
- const tx = log.transactions[txIdx]!;
227
- pieceSize += getTransactionSize(tx);
228
- }
229
-
230
- if (exceedsRecommendedSize(pieceSize)) {
231
- if (!currentPiece.expectContentUntil && pieces.length === 1) {
232
- currentPiece.expectContentUntil =
233
- this.knownStateWithStreaming().sessions;
234
+ addTransactionToContentMessage(
235
+ currentPiece,
236
+ tx,
237
+ sessionID,
238
+ log.lastSignature,
239
+ txIdx,
240
+ );
241
+
242
+ // If the current session size already exceeds the recommended size, we move the session content to a dedicated piece
243
+ if (exceedsRecommendedSize(currentSessionSize)) {
244
+ assertLastSignature(sessionID, currentPiece);
245
+ moveSessionContentToNewPiece(sessionID);
246
+ } else if (exceedsRecommendedSize(pieceSize, currentSessionSize)) {
247
+ assertLastSignature(sessionID, currentPiece);
248
+ startNewPiece();
249
+ } else {
250
+ pieceSize += currentSessionSize;
234
251
  }
235
-
236
- currentPiece = createContentMessage(this.id, this.header, false);
237
- pieces.push(currentPiece);
238
- pieceSize = pieceSize - oldPieceSize;
252
+ } else {
253
+ // Unsafely add the transaction to the content message, without a signature because we don't have one for this session
254
+ // Checks and assertions are enforced in this function to avoid that a content message gets out without a signature
255
+ const signature = undefined as Signature | undefined;
256
+ addTransactionToContentMessage(
257
+ currentPiece,
258
+ tx,
259
+ sessionID,
260
+ signature!,
261
+ txIdx,
262
+ );
239
263
  }
264
+ }
240
265
 
241
- let sessionEntry = currentPiece.new[sessionID];
242
- if (!sessionEntry) {
243
- sessionEntry = {
244
- after: sentStateForSessionID ?? knownStateForSessionID ?? 0,
245
- newTransactions: [],
246
- lastSignature: "WILL_BE_REPLACED" as Signature,
247
- };
248
- currentPiece.new[sessionID] = sessionEntry;
249
- }
266
+ assertLastSignature(sessionID, currentPiece);
267
+ }
250
268
 
251
- for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
252
- const tx = log.transactions[txIdx]!;
253
- sessionEntry.newTransactions.push(tx);
254
- }
269
+ const firstPiece = pieces[0];
255
270
 
256
- sessionEntry.lastSignature =
257
- nextKnownSignatureIdx === undefined
258
- ? log.lastSignature!
259
- : log.signatureAfter[nextKnownSignatureIdx]!;
271
+ if (!firstPiece) {
272
+ throw new Error("First piece not found", {
273
+ cause: pieces,
274
+ });
275
+ }
260
276
 
261
- sentState[sessionID] =
262
- (sentStateForSessionID ?? knownStateForSessionID ?? 0) + nNewTx;
263
- }
277
+ const includeHeader = !knownState?.header;
278
+
279
+ if (includeHeader) {
280
+ firstPiece.header = this.header;
264
281
  }
265
282
 
266
283
  const piecesWithContent = pieces.filter(
267
- (piece) => Object.keys(piece.new).length > 0 || piece.header,
284
+ (piece) => piece.header || Object.keys(piece.new).length > 0,
268
285
  );
269
286
 
270
- if (piecesWithContent.length === 0) {
271
- return undefined;
287
+ if (piecesWithContent.length > 1 || this.isStreaming()) {
288
+ // Flag that more content is coming
289
+ firstPiece.expectContentUntil = {
290
+ ...this.knownStateWithStreaming().sessions,
291
+ };
272
292
  }
273
293
 
274
- if (isKnownStateEmpty) {
275
- this._cachedNewContentSinceEmpty = piecesWithContent;
294
+ if (piecesWithContent.length === 0) {
295
+ return undefined;
276
296
  }
277
297
 
278
298
  return piecesWithContent;
@@ -307,15 +327,10 @@ export class VerifiedState {
307
327
  }
308
328
  }
309
329
 
310
- function getNextKnownSignatureIdx(
311
- log: SessionLog,
312
- knownStateForSessionID?: number,
313
- sentStateForSessionID?: number,
314
- ) {
315
- return Object.keys(log.signatureAfter)
316
- .map(Number)
317
- .sort((a, b) => a - b)
318
- .find(
319
- (idx) => idx >= (sentStateForSessionID ?? knownStateForSessionID ?? -1),
320
- );
330
+ function assertLastSignature(sessionID: SessionID, content: NewContentMessage) {
331
+ if (content.new[sessionID] && !content.new[sessionID].lastSignature) {
332
+ throw new Error("The SessionContent sent must have a lastSignature", {
333
+ cause: content.new[sessionID],
334
+ });
335
+ }
321
336
  }
@@ -123,6 +123,13 @@ function healMissingKeyForEveryone(group: RawGroup) {
123
123
  }
124
124
 
125
125
  function needsKeyRotation(group: RawGroup) {
126
+ const myRole = group.myRole();
127
+
128
+ // Checking only direct membership because inside the migrations we can't navigate the parent groups
129
+ if (myRole !== "admin" && myRole !== "manager") {
130
+ return false;
131
+ }
132
+
126
133
  const currentReadKeyId = group.get("readKey");
127
134
 
128
135
  if (!currentReadKeyId) {
@@ -190,11 +197,27 @@ export class RawGroup<
190
197
  ) {
191
198
  super(core, options);
192
199
  this.crypto = core.node.crypto;
200
+ this.migrate();
201
+ }
193
202
 
194
- // Checks if this is not an account
195
- if (core.isGroup() && !core.verified.isStreaming()) {
196
- // rotateReadKeyIfNeeded(this);
203
+ migrate() {
204
+ if (!this.core.isGroup()) {
205
+ return;
206
+ }
207
+
208
+ const runMigrations = () => {
209
+ rotateReadKeyIfNeeded(this);
197
210
  healMissingKeyForEveryone(this);
211
+ };
212
+
213
+ // We need the group and their parents to be completely downloaded to correctly handle the migrations
214
+ if (!this.core.isCompletelyDownloaded()) {
215
+ this.core.waitFor({
216
+ predicate: (core) => core.isCompletelyDownloaded(),
217
+ onSuccess: runMigrations,
218
+ });
219
+ } else {
220
+ runMigrations();
198
221
  }
199
222
  }
200
223
 
@@ -318,7 +341,7 @@ export class RawGroup<
318
341
  * @category 1. Role reading
319
342
  */
320
343
  myRole(): Role | undefined {
321
- return this.roleOfInternal(this.core.node.getCurrentAgent().id);
344
+ return this.roleOfInternal(this.core.node.getCurrentAccountOrAgentID());
322
345
  }
323
346
 
324
347
  /**
package/src/knownState.ts CHANGED
@@ -109,7 +109,7 @@ export function cloneKnownState(knownState: CoValueKnownState) {
109
109
  /**
110
110
  * Checks if all the local sessions have the same counters as in remote.
111
111
  */
112
- export function areLocalSessionsUploaded(
112
+ export function isKnownStateSubsetOf(
113
113
  local: Record<string, number>,
114
114
  remote: Record<string, number>,
115
115
  ) {
package/src/localNode.ts CHANGED
@@ -650,8 +650,7 @@ export class LocalNode {
650
650
  );
651
651
 
652
652
  const contentPieces =
653
- groupAsInvite.core.verified.newContentSince(group.core.knownState()) ??
654
- [];
653
+ groupAsInvite.core.newContentSince(group.core.knownState()) ?? [];
655
654
 
656
655
  // Import the new transactions to the current localNode
657
656
  for (const contentPiece of contentPieces) {
@@ -1,10 +1,15 @@
1
1
  import {
2
2
  addTransactionToContentMessage,
3
3
  createContentMessage,
4
+ knownStateFromContent,
4
5
  } from "../coValueContentMessage.js";
5
6
  import { Transaction, VerifiedState } from "../coValueCore/verifiedState.js";
6
7
  import { Signature } from "../crypto/crypto.js";
7
8
  import { RawCoID, SessionID } from "../ids.js";
9
+ import {
10
+ combineKnownStateSessions,
11
+ KnownStateSessions,
12
+ } from "../knownState.js";
8
13
  import { NewContentMessage } from "../sync.js";
9
14
  import { LinkedList } from "./LinkedList.js";
10
15
 
@@ -113,9 +118,29 @@ export class LocalTransactionsSyncQueue {
113
118
  this.processingSyncs = true;
114
119
 
115
120
  queueMicrotask(() => {
121
+ const firstContentPieceMap = new Map<RawCoID, NewContentMessage>();
122
+
116
123
  while (this.queue.head) {
117
124
  const content = this.queue.head.value;
118
125
 
126
+ const firstContentPiece = firstContentPieceMap.get(content.id);
127
+
128
+ if (!firstContentPiece) {
129
+ firstContentPieceMap.set(content.id, content);
130
+ } else {
131
+ // There is already a content piece for this coValue, so this means that we need to flag
132
+ // that this content is going to be streamed
133
+ if (!firstContentPiece.expectContentUntil) {
134
+ firstContentPiece.expectContentUntil =
135
+ knownStateFromContent(firstContentPiece).sessions;
136
+ }
137
+
138
+ combineKnownStateSessions(
139
+ firstContentPiece.expectContentUntil,
140
+ knownStateFromContent(content).sessions,
141
+ );
142
+ }
143
+
119
144
  this.sync(content);
120
145
 
121
146
  this.queue.shift();
@@ -3,7 +3,7 @@ import { RawCoID } from "../ids.js";
3
3
  import {
4
4
  CoValueKnownState,
5
5
  emptyKnownState,
6
- areLocalSessionsUploaded,
6
+ isKnownStateSubsetOf,
7
7
  } from "../knownState.js";
8
8
 
9
9
  /**
@@ -87,7 +87,7 @@ function isInSync(
87
87
  return false;
88
88
  }
89
89
 
90
- return areLocalSessionsUploaded(
90
+ return isKnownStateSubsetOf(
91
91
  knownState.sessions,
92
92
  knownStateFromStorage.sessions,
93
93
  );
package/src/sync.ts CHANGED
@@ -228,7 +228,7 @@ export class SyncManager {
228
228
  }
229
229
  }
230
230
 
231
- const newContentPieces = coValue.verified.newContentSince(
231
+ const newContentPieces = coValue.newContentSince(
232
232
  peer.getOptimisticKnownState(id),
233
233
  );
234
234
 
@@ -585,6 +585,8 @@ export class SyncManager {
585
585
  sessions: msg.expectContentUntil,
586
586
  });
587
587
  }
588
+ } else if (msg.expectContentUntil) {
589
+ coValue.verified.setStreamingKnownState(msg.expectContentUntil);
588
590
  }
589
591
 
590
592
  // At this point the CoValue must be in memory, if not we have a bug
@@ -647,6 +649,9 @@ export class SyncManager {
647
649
  peerRole: peer.role,
648
650
  id: msg.id,
649
651
  err: result.error,
652
+ msgKnownState: knownStateFromContent(msg).sessions,
653
+ knownState: coValue.knownState().sessions,
654
+ newContent: validNewContent.new,
650
655
  });
651
656
  // TODO Mark only the session as errored, not the whole coValue
652
657
  coValue.markErrored(peer.id, result.error);
@@ -822,7 +827,7 @@ export class SyncManager {
822
827
  return undefined;
823
828
  }
824
829
 
825
- return value.verified.newContentSince(correction);
830
+ return value.newContentSince(correction);
826
831
  });
827
832
  }
828
833
 
@@ -117,8 +117,7 @@ describe("PureJSCrypto", () => {
117
117
  true,
118
118
  );
119
119
 
120
- const content =
121
- mapInOtherSession.core.verified.newContentSince(undefined)?.[0];
120
+ const content = mapInOtherSession.core.newContentSince(undefined)?.[0];
122
121
  assert(content);
123
122
 
124
123
  client.node.syncManager.handleNewContent(content, "storage");
@@ -31,7 +31,7 @@ function getNewContentSince(
31
31
  throw new Error(`CoValue ${coValue.id} is not verified`);
32
32
  }
33
33
 
34
- const contentMessage = coValue.verified.newContentSince(knownState)?.[0];
34
+ const contentMessage = coValue.newContentSince(knownState)?.[0];
35
35
 
36
36
  if (!contentMessage) {
37
37
  throw new Error(`No new content available for coValue ${coValue.id}`);
@@ -295,7 +295,7 @@ describe("StorageApiAsync", () => {
295
295
  const contentMessage = getNewContentSince(group.core, knownState);
296
296
  const correctionCallback = vi.fn((known) => {
297
297
  expect(known).toEqual(emptyKnownState(group.id));
298
- return group.core.verified.newContentSince(known);
298
+ return group.core.newContentSince(known);
299
299
  });
300
300
 
301
301
  // Get initial known state
@@ -339,7 +339,7 @@ describe("StorageApiAsync", () => {
339
339
  const contentMessage = getNewContentSince(group.core, knownState);
340
340
  const correctionCallback = vi.fn((known) => {
341
341
  expect(known).toEqual(initialKnownState);
342
- return group.core.verified.newContentSince(known);
342
+ return group.core.newContentSince(known);
343
343
  });
344
344
 
345
345
  // Get initial storage known state
@@ -539,16 +539,16 @@ describe("StorageApiAsync", () => {
539
539
  }),
540
540
  ).toMatchInlineSnapshot(`
541
541
  [
542
- "test -> test-storage | CONTENT Core header: false new: After: 1 New: 1",
543
- "test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1",
542
+ "test -> test-storage | CONTENT Core header: false new: After: 1 New: 1 expectContentUntil: header/3",
543
+ "test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1 expectContentUntil: header/3",
544
544
  "test -> test-storage | CONTENT Core header: false new: After: 2 New: 1",
545
545
  "test -> test-storage | CONTENT Core2 header: false new: After: 2 New: 1",
546
546
  "test-storage -> test | KNOWN CORRECTION Core sessions: empty",
547
547
  "test -> test-storage | CONTENT Core header: true new: After: 0 New: 3",
548
548
  "test-storage -> test | KNOWN CORRECTION Core2 sessions: empty",
549
549
  "test -> test-storage | CONTENT Core2 header: true new: After: 0 New: 3",
550
- "test -> test-storage | CONTENT Core header: false new: After: 3 New: 1",
551
- "test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1",
550
+ "test -> test-storage | CONTENT Core header: false new: After: 3 New: 1 expectContentUntil: header/5",
551
+ "test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1 expectContentUntil: header/5",
552
552
  "test -> test-storage | CONTENT Core header: false new: After: 4 New: 1",
553
553
  "test -> test-storage | CONTENT Core2 header: false new: After: 4 New: 1",
554
554
  ]
@@ -606,14 +606,14 @@ describe("StorageApiAsync", () => {
606
606
  }),
607
607
  ).toMatchInlineSnapshot(`
608
608
  [
609
- "test -> test-storage | CONTENT Core header: false new: After: 1 New: 1",
610
- "test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1",
609
+ "test -> test-storage | CONTENT Core header: false new: After: 1 New: 1 expectContentUntil: header/3",
610
+ "test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1 expectContentUntil: header/3",
611
611
  "test -> test-storage | CONTENT Core header: false new: After: 2 New: 1",
612
612
  "test -> test-storage | CONTENT Core2 header: false new: After: 2 New: 1",
613
613
  "test-storage -> test | KNOWN CORRECTION Core sessions: empty",
614
614
  "test -> test-storage | CONTENT Core header: true new: After: 0 New: 3",
615
- "test -> test-storage | CONTENT Core header: false new: After: 3 New: 1",
616
- "test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1",
615
+ "test -> test-storage | CONTENT Core header: false new: After: 3 New: 1 expectContentUntil: header/5",
616
+ "test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1 expectContentUntil: header/5",
617
617
  "test -> test-storage | CONTENT Core header: false new: After: 4 New: 1",
618
618
  "test -> test-storage | CONTENT Core2 header: false new: After: 4 New: 1",
619
619
  ]
@@ -26,7 +26,7 @@ function getNewContentSince(
26
26
  throw new Error(`CoValue ${coValue.id} is not verified`);
27
27
  }
28
28
 
29
- const contentMessage = coValue.verified.newContentSince(knownState)?.[0];
29
+ const contentMessage = coValue.newContentSince(knownState)?.[0];
30
30
 
31
31
  if (!contentMessage) {
32
32
  throw new Error(`No new content available for coValue ${coValue.id}`);
@@ -274,7 +274,7 @@ describe("StorageApiSync", () => {
274
274
  const contentMessage = getNewContentSince(group.core, knownState);
275
275
  const correctionCallback = vi.fn((known) => {
276
276
  expect(known).toEqual(emptyKnownState(group.id));
277
- return group.core.verified.newContentSince(known);
277
+ return group.core.newContentSince(known);
278
278
  });
279
279
 
280
280
  // Get initial known state
@@ -318,7 +318,7 @@ describe("StorageApiSync", () => {
318
318
  const contentMessage = getNewContentSince(group.core, knownState);
319
319
  const correctionCallback = vi.fn((known) => {
320
320
  expect(known).toEqual(initialKnownState);
321
- return group.core.verified.newContentSince(known);
321
+ return group.core.newContentSince(known);
322
322
  });
323
323
 
324
324
  // Get initial storage known state
@@ -117,8 +117,7 @@ describe("WasmCrypto", () => {
117
117
  true,
118
118
  );
119
119
 
120
- const content =
121
- mapInOtherSession.core.verified.newContentSince(undefined)?.[0];
120
+ const content = mapInOtherSession.core.newContentSince(undefined)?.[0];
122
121
  assert(content);
123
122
 
124
123
  client.node.syncManager.handleNewContent(content, "storage");
@@ -371,7 +371,7 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
371
371
  }),
372
372
  ).toMatchInlineSnapshot(`
373
373
  [
374
- "client -> storage | CONTENT CoPlainText header: true new: After: 0 New: 1",
374
+ "client -> storage | CONTENT CoPlainText header: true new: After: 0 New: 1 expectContentUntil: header/42",
375
375
  "client -> storage | CONTENT CoPlainText header: false new: After: 1 New: 1",
376
376
  "client -> storage | CONTENT CoPlainText header: false new: After: 2 New: 1",
377
377
  "client -> storage | CONTENT CoPlainText header: false new: After: 3 New: 1",
@@ -217,24 +217,24 @@ test("When adding large transactions (small fraction of MAX_RECOMMENDED_TX_SIZE)
217
217
  expect(sessionEntry.signatureAfter[10]).not.toBeDefined();
218
218
  expect(sessionEntry.signatureAfter[11]).not.toBeDefined();
219
219
 
220
- const newContent = coValue.verified.newContentSince({
220
+ const newContent = coValue.newContentSince({
221
221
  id: coValue.id,
222
222
  header: false,
223
223
  sessions: {},
224
224
  })!;
225
225
 
226
- expect(newContent.length).toEqual(5);
226
+ expect(newContent.length).toEqual(4);
227
227
  expect(newContent[0]!.header).toBeDefined();
228
- expect(newContent[1]!.new[node.currentSessionID]!.lastSignature).toEqual(
228
+ expect(newContent[0]!.new[node.currentSessionID]!.lastSignature).toEqual(
229
229
  sessionEntry.signatureAfter[3],
230
230
  );
231
- expect(newContent[2]!.new[node.currentSessionID]!.lastSignature).toEqual(
231
+ expect(newContent[1]!.new[node.currentSessionID]!.lastSignature).toEqual(
232
232
  sessionEntry.signatureAfter[6],
233
233
  );
234
- expect(newContent[3]!.new[node.currentSessionID]!.lastSignature).toEqual(
234
+ expect(newContent[2]!.new[node.currentSessionID]!.lastSignature).toEqual(
235
235
  sessionEntry.signatureAfter[9],
236
236
  );
237
- expect(newContent[4]!.new[node.currentSessionID]!.lastSignature).toEqual(
237
+ expect(newContent[3]!.new[node.currentSessionID]!.lastSignature).toEqual(
238
238
  sessionEntry.lastSignature,
239
239
  );
240
240
  });
@@ -286,24 +286,24 @@ test("When adding large transactions (bigger than MAX_RECOMMENDED_TX_SIZE), we s
286
286
  expect(sessionEntry.signatureAfter[3]).toBeDefined();
287
287
  expect(sessionEntry.signatureAfter[4]).not.toBeDefined();
288
288
 
289
- const newContent = coValue.verified.newContentSince({
289
+ const newContent = coValue.newContentSince({
290
290
  id: coValue.id,
291
291
  header: false,
292
292
  sessions: {},
293
293
  })!;
294
294
 
295
- expect(newContent.length).toEqual(5);
295
+ expect(newContent.length).toEqual(4);
296
296
  expect(newContent[0]!.header).toBeDefined();
297
- expect(newContent[1]!.new[node.currentSessionID]!.lastSignature).toEqual(
297
+ expect(newContent[0]!.new[node.currentSessionID]!.lastSignature).toEqual(
298
298
  sessionEntry.signatureAfter[1],
299
299
  );
300
- expect(newContent[2]!.new[node.currentSessionID]!.lastSignature).toEqual(
300
+ expect(newContent[1]!.new[node.currentSessionID]!.lastSignature).toEqual(
301
301
  sessionEntry.signatureAfter[2],
302
302
  );
303
- expect(newContent[3]!.new[node.currentSessionID]!.lastSignature).toEqual(
303
+ expect(newContent[2]!.new[node.currentSessionID]!.lastSignature).toEqual(
304
304
  sessionEntry.signatureAfter[3],
305
305
  );
306
- expect(newContent[4]!.new[node.currentSessionID]!.lastSignature).toEqual(
306
+ expect(newContent[3]!.new[node.currentSessionID]!.lastSignature).toEqual(
307
307
  sessionEntry.lastSignature,
308
308
  );
309
309
  });