cojson 0.18.32 → 0.18.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/CHANGELOG.md +9 -0
- package/dist/SyncStateManager.d.ts.map +1 -1
- package/dist/SyncStateManager.js +2 -2
- package/dist/SyncStateManager.js.map +1 -1
- package/dist/coValueCore/SessionMap.d.ts +1 -0
- package/dist/coValueCore/SessionMap.d.ts.map +1 -1
- package/dist/coValueCore/SessionMap.js +17 -1
- package/dist/coValueCore/SessionMap.js.map +1 -1
- package/dist/coValueCore/coValueCore.d.ts +14 -9
- package/dist/coValueCore/coValueCore.d.ts.map +1 -1
- package/dist/coValueCore/coValueCore.js +62 -47
- package/dist/coValueCore/coValueCore.js.map +1 -1
- package/dist/coValueCore/verifiedState.d.ts +2 -2
- package/dist/coValueCore/verifiedState.d.ts.map +1 -1
- package/dist/coValueCore/verifiedState.js +86 -75
- package/dist/coValueCore/verifiedState.js.map +1 -1
- package/dist/coValues/group.d.ts +1 -0
- package/dist/coValues/group.d.ts.map +1 -1
- package/dist/coValues/group.js +24 -4
- package/dist/coValues/group.js.map +1 -1
- package/dist/knownState.d.ts +1 -1
- package/dist/knownState.d.ts.map +1 -1
- package/dist/knownState.js +1 -1
- package/dist/knownState.js.map +1 -1
- package/dist/localNode.d.ts.map +1 -1
- package/dist/localNode.js +1 -2
- package/dist/localNode.js.map +1 -1
- package/dist/queue/LocalTransactionsSyncQueue.d.ts.map +1 -1
- package/dist/queue/LocalTransactionsSyncQueue.js +16 -1
- package/dist/queue/LocalTransactionsSyncQueue.js.map +1 -1
- package/dist/storage/knownState.js +2 -2
- package/dist/storage/knownState.js.map +1 -1
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +8 -2
- package/dist/sync.js.map +1 -1
- package/dist/tests/PureJSCrypto.test.js +1 -1
- package/dist/tests/PureJSCrypto.test.js.map +1 -1
- package/dist/tests/StorageApiAsync.test.js +11 -11
- package/dist/tests/StorageApiAsync.test.js.map +1 -1
- package/dist/tests/StorageApiSync.test.js +3 -3
- package/dist/tests/StorageApiSync.test.js.map +1 -1
- package/dist/tests/WasmCrypto.test.js +1 -1
- package/dist/tests/WasmCrypto.test.js.map +1 -1
- package/dist/tests/coPlainText.test.js +1 -1
- package/dist/tests/coStream.test.js +12 -12
- package/dist/tests/coStream.test.js.map +1 -1
- package/dist/tests/coValueCore.isCompletelyDownloaded.test.d.ts +2 -0
- package/dist/tests/coValueCore.isCompletelyDownloaded.test.d.ts.map +1 -0
- package/dist/tests/coValueCore.isCompletelyDownloaded.test.js +421 -0
- package/dist/tests/coValueCore.isCompletelyDownloaded.test.js.map +1 -0
- package/dist/tests/coValueCore.isStreaming.test.d.ts +2 -0
- package/dist/tests/coValueCore.isStreaming.test.d.ts.map +1 -0
- package/dist/tests/coValueCore.isStreaming.test.js +181 -0
- package/dist/tests/coValueCore.isStreaming.test.js.map +1 -0
- package/dist/tests/coValueCore.newContentSince.test.d.ts +2 -0
- package/dist/tests/coValueCore.newContentSince.test.d.ts.map +1 -0
- package/dist/tests/coValueCore.newContentSince.test.js +808 -0
- package/dist/tests/coValueCore.newContentSince.test.js.map +1 -0
- package/dist/tests/coreWasm.test.js +2 -2
- package/dist/tests/coreWasm.test.js.map +1 -1
- package/dist/tests/group.childKeyRotation.test.d.ts +2 -0
- package/dist/tests/group.childKeyRotation.test.d.ts.map +1 -0
- package/dist/tests/group.childKeyRotation.test.js +261 -0
- package/dist/tests/group.childKeyRotation.test.js.map +1 -0
- package/dist/tests/group.removeMember.test.js +1 -114
- package/dist/tests/group.removeMember.test.js.map +1 -1
- package/dist/tests/knownState.test.js +11 -11
- package/dist/tests/knownState.test.js.map +1 -1
- package/dist/tests/sync.auth.test.js +6 -6
- package/dist/tests/sync.load.test.js +68 -5
- package/dist/tests/sync.load.test.js.map +1 -1
- package/dist/tests/sync.mesh.test.js +11 -17
- package/dist/tests/sync.mesh.test.js.map +1 -1
- package/dist/tests/sync.peerReconciliation.test.js +1 -1
- package/dist/tests/sync.storage.test.js +7 -7
- package/dist/tests/sync.storage.test.js.map +1 -1
- package/dist/tests/sync.storageAsync.test.js +4 -4
- package/dist/tests/sync.storageAsync.test.js.map +1 -1
- package/dist/tests/sync.upload.test.js +96 -40
- package/dist/tests/sync.upload.test.js.map +1 -1
- package/dist/tests/testUtils.d.ts +2 -0
- package/dist/tests/testUtils.d.ts.map +1 -1
- package/dist/tests/testUtils.js +22 -1
- package/dist/tests/testUtils.js.map +1 -1
- package/package.json +3 -3
- package/src/SyncStateManager.ts +2 -5
- package/src/coValueCore/SessionMap.ts +24 -0
- package/src/coValueCore/coValueCore.ts +77 -55
- package/src/coValueCore/verifiedState.ts +123 -108
- package/src/coValues/group.ts +27 -4
- package/src/knownState.ts +1 -1
- package/src/localNode.ts +1 -2
- package/src/queue/LocalTransactionsSyncQueue.ts +25 -0
- package/src/storage/knownState.ts +2 -2
- package/src/sync.ts +7 -2
- package/src/tests/PureJSCrypto.test.ts +1 -2
- package/src/tests/StorageApiAsync.test.ts +11 -11
- package/src/tests/StorageApiSync.test.ts +3 -3
- package/src/tests/WasmCrypto.test.ts +1 -2
- package/src/tests/coPlainText.test.ts +1 -1
- package/src/tests/coStream.test.ts +12 -12
- package/src/tests/coValueCore.isCompletelyDownloaded.test.ts +589 -0
- package/src/tests/coValueCore.isStreaming.test.ts +271 -0
- package/src/tests/coValueCore.newContentSince.test.ts +966 -0
- package/src/tests/coreWasm.test.ts +2 -2
- package/src/tests/group.childKeyRotation.test.ts +431 -0
- package/src/tests/group.removeMember.test.ts +1 -184
- package/src/tests/knownState.test.ts +11 -11
- package/src/tests/sync.auth.test.ts +6 -6
- package/src/tests/sync.load.test.ts +80 -5
- package/src/tests/sync.mesh.test.ts +11 -17
- package/src/tests/sync.peerReconciliation.test.ts +1 -1
- package/src/tests/sync.storage.test.ts +7 -7
- package/src/tests/sync.storageAsync.test.ts +4 -4
- package/src/tests/sync.upload.test.ts +106 -40
- package/src/tests/testUtils.ts +24 -2
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
createContentMessage,
|
|
5
5
|
exceedsRecommendedSize,
|
|
6
6
|
getTransactionSize,
|
|
7
|
+
addTransactionToContentMessage,
|
|
7
8
|
} from "../coValueContentMessage.js";
|
|
8
9
|
import {
|
|
9
10
|
CryptoProvider,
|
|
@@ -21,7 +22,7 @@ import { NewContentMessage } from "../sync.js";
|
|
|
21
22
|
import { TryAddTransactionsError } from "./coValueCore.js";
|
|
22
23
|
import { SessionLog, SessionMap } from "./SessionMap.js";
|
|
23
24
|
import { ControlledAccountOrAgent } from "../coValues/account.js";
|
|
24
|
-
import {
|
|
25
|
+
import { CoValueKnownState, KnownStateSessions } from "../knownState.js";
|
|
25
26
|
|
|
26
27
|
export type CoValueHeader = {
|
|
27
28
|
type: AnyRawCoValue["type"];
|
|
@@ -55,7 +56,6 @@ export class VerifiedState {
|
|
|
55
56
|
readonly crypto: CryptoProvider;
|
|
56
57
|
readonly header: CoValueHeader;
|
|
57
58
|
readonly sessions: SessionMap;
|
|
58
|
-
private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
|
|
59
59
|
public lastAccessed: number | undefined;
|
|
60
60
|
public branchSourceId?: RawCoID;
|
|
61
61
|
public branchName?: string;
|
|
@@ -98,10 +98,6 @@ export class VerifiedState {
|
|
|
98
98
|
skipVerify,
|
|
99
99
|
);
|
|
100
100
|
|
|
101
|
-
if (result.isOk()) {
|
|
102
|
-
this._cachedNewContentSinceEmpty = undefined;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
101
|
return result;
|
|
106
102
|
}
|
|
107
103
|
|
|
@@ -120,8 +116,6 @@ export class VerifiedState {
|
|
|
120
116
|
madeAt,
|
|
121
117
|
);
|
|
122
118
|
|
|
123
|
-
this._cachedNewContentSinceEmpty = undefined;
|
|
124
|
-
|
|
125
119
|
return result;
|
|
126
120
|
}
|
|
127
121
|
|
|
@@ -144,8 +138,6 @@ export class VerifiedState {
|
|
|
144
138
|
madeAt,
|
|
145
139
|
);
|
|
146
140
|
|
|
147
|
-
this._cachedNewContentSinceEmpty = undefined;
|
|
148
|
-
|
|
149
141
|
return result;
|
|
150
142
|
}
|
|
151
143
|
|
|
@@ -160,119 +152,147 @@ export class VerifiedState {
|
|
|
160
152
|
);
|
|
161
153
|
}
|
|
162
154
|
|
|
155
|
+
setStreamingKnownState(streamingKnownState: KnownStateSessions) {
|
|
156
|
+
this.sessions.setStreamingKnownState(streamingKnownState);
|
|
157
|
+
}
|
|
158
|
+
|
|
163
159
|
newContentSince(
|
|
164
160
|
knownState: CoValueKnownState | undefined,
|
|
165
161
|
): NewContentMessage[] | undefined {
|
|
166
|
-
const isKnownStateEmpty = !knownState?.header && !knownState?.sessions;
|
|
167
|
-
|
|
168
|
-
if (isKnownStateEmpty && this._cachedNewContentSinceEmpty) {
|
|
169
|
-
return this._cachedNewContentSinceEmpty;
|
|
170
|
-
}
|
|
171
|
-
|
|
172
162
|
let currentPiece: NewContentMessage = createContentMessage(
|
|
173
163
|
this.id,
|
|
174
164
|
this.header,
|
|
175
|
-
|
|
165
|
+
false,
|
|
176
166
|
);
|
|
177
|
-
|
|
178
|
-
const pieces = [currentPiece];
|
|
179
|
-
|
|
180
|
-
const sentState: CoValueKnownState["sessions"] = {};
|
|
181
|
-
|
|
167
|
+
const pieces: NewContentMessage[] = [currentPiece];
|
|
182
168
|
let pieceSize = 0;
|
|
183
169
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
170
|
+
const startNewPiece = () => {
|
|
171
|
+
currentPiece = createContentMessage(this.id, this.header, false);
|
|
172
|
+
pieces.push(currentPiece);
|
|
173
|
+
pieceSize = 0;
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
const moveSessionContentToNewPiece = (sessionID: SessionID) => {
|
|
177
|
+
const sessionContent = currentPiece.new[sessionID];
|
|
178
|
+
|
|
179
|
+
if (!sessionContent) {
|
|
180
|
+
throw new Error("Session content not found", {
|
|
181
|
+
cause: {
|
|
182
|
+
sessionID,
|
|
183
|
+
currentPiece,
|
|
184
|
+
},
|
|
185
|
+
});
|
|
189
186
|
}
|
|
190
|
-
const sessionsTodo = sessionsTodoAgain ?? this.sessions.keys();
|
|
191
|
-
|
|
192
|
-
for (const sessionIDKey of sessionsTodo) {
|
|
193
|
-
const sessionID = sessionIDKey as SessionID;
|
|
194
|
-
const log = this.sessions.get(sessionID)!;
|
|
195
|
-
const knownStateForSessionID = knownState?.sessions[sessionID];
|
|
196
|
-
const sentStateForSessionID = sentState[sessionID];
|
|
197
|
-
const nextKnownSignatureIdx = getNextKnownSignatureIdx(
|
|
198
|
-
log,
|
|
199
|
-
knownStateForSessionID,
|
|
200
|
-
sentStateForSessionID,
|
|
201
|
-
);
|
|
202
|
-
|
|
203
|
-
const firstNewTxIdx =
|
|
204
|
-
sentStateForSessionID ?? knownStateForSessionID ?? 0;
|
|
205
|
-
const afterLastNewTxIdx =
|
|
206
|
-
nextKnownSignatureIdx === undefined
|
|
207
|
-
? log.transactions.length
|
|
208
|
-
: nextKnownSignatureIdx + 1;
|
|
209
|
-
|
|
210
|
-
const nNewTx = Math.max(0, afterLastNewTxIdx - firstNewTxIdx);
|
|
211
|
-
|
|
212
|
-
if (nNewTx === 0) {
|
|
213
|
-
sessionsTodoAgain?.delete(sessionID);
|
|
214
|
-
continue;
|
|
215
|
-
}
|
|
216
187
|
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
188
|
+
delete currentPiece.new[sessionID];
|
|
189
|
+
|
|
190
|
+
const newPiece = createContentMessage(this.id, this.header, false);
|
|
191
|
+
newPiece.new[sessionID] = sessionContent;
|
|
192
|
+
|
|
193
|
+
// Insert the new piece before the current piece, to ensure that the order of the new transactions is preserved
|
|
194
|
+
pieces.splice(pieces.length - 1, 0, newPiece);
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
const sessionSent = knownState?.sessions;
|
|
198
|
+
|
|
199
|
+
for (const [sessionID, log] of this.sessions.sessions) {
|
|
200
|
+
const startFrom = sessionSent?.[sessionID] ?? 0;
|
|
201
|
+
|
|
202
|
+
let currentSessionSize = 0;
|
|
203
|
+
|
|
204
|
+
for (let txIdx = startFrom; txIdx < log.transactions.length; txIdx++) {
|
|
205
|
+
const isLastItem = txIdx === log.transactions.length - 1;
|
|
206
|
+
const tx = log.transactions[txIdx]!;
|
|
207
|
+
|
|
208
|
+
currentSessionSize += getTransactionSize(tx);
|
|
209
|
+
|
|
210
|
+
const signatureAfter = log.signatureAfter[txIdx];
|
|
211
|
+
|
|
212
|
+
if (signatureAfter) {
|
|
213
|
+
addTransactionToContentMessage(
|
|
214
|
+
currentPiece,
|
|
215
|
+
tx,
|
|
216
|
+
sessionID,
|
|
217
|
+
signatureAfter,
|
|
218
|
+
txIdx,
|
|
219
|
+
);
|
|
220
|
+
// When we meet a signatureAfter it means that the transaction log exceeds the recommended size
|
|
221
|
+
// so we move the session content to a dedicated piece, because it must be sent in a standalone piece
|
|
222
|
+
moveSessionContentToNewPiece(sessionID);
|
|
223
|
+
currentSessionSize = 0;
|
|
224
|
+
} else if (isLastItem) {
|
|
225
|
+
if (!log.lastSignature) {
|
|
226
|
+
throw new Error(
|
|
227
|
+
"All the SessionLogs sent must have a lastSignature",
|
|
228
|
+
{
|
|
229
|
+
cause: log,
|
|
230
|
+
},
|
|
231
|
+
);
|
|
220
232
|
}
|
|
221
|
-
sessionsTodoAgain.add(sessionID);
|
|
222
|
-
}
|
|
223
233
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
+
addTransactionToContentMessage(
|
|
235
|
+
currentPiece,
|
|
236
|
+
tx,
|
|
237
|
+
sessionID,
|
|
238
|
+
log.lastSignature,
|
|
239
|
+
txIdx,
|
|
240
|
+
);
|
|
241
|
+
|
|
242
|
+
// If the current session size already exceeds the recommended size, we move the session content to a dedicated piece
|
|
243
|
+
if (exceedsRecommendedSize(currentSessionSize)) {
|
|
244
|
+
assertLastSignature(sessionID, currentPiece);
|
|
245
|
+
moveSessionContentToNewPiece(sessionID);
|
|
246
|
+
} else if (exceedsRecommendedSize(pieceSize, currentSessionSize)) {
|
|
247
|
+
assertLastSignature(sessionID, currentPiece);
|
|
248
|
+
startNewPiece();
|
|
249
|
+
} else {
|
|
250
|
+
pieceSize += currentSessionSize;
|
|
234
251
|
}
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
252
|
+
} else {
|
|
253
|
+
// Unsafely add the transaction to the content message, without a signature because we don't have one for this session
|
|
254
|
+
// Checks and assertions are enforced in this function to avoid that a content message gets out without a signature
|
|
255
|
+
const signature = undefined as Signature | undefined;
|
|
256
|
+
addTransactionToContentMessage(
|
|
257
|
+
currentPiece,
|
|
258
|
+
tx,
|
|
259
|
+
sessionID,
|
|
260
|
+
signature!,
|
|
261
|
+
txIdx,
|
|
262
|
+
);
|
|
239
263
|
}
|
|
264
|
+
}
|
|
240
265
|
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
sessionEntry = {
|
|
244
|
-
after: sentStateForSessionID ?? knownStateForSessionID ?? 0,
|
|
245
|
-
newTransactions: [],
|
|
246
|
-
lastSignature: "WILL_BE_REPLACED" as Signature,
|
|
247
|
-
};
|
|
248
|
-
currentPiece.new[sessionID] = sessionEntry;
|
|
249
|
-
}
|
|
266
|
+
assertLastSignature(sessionID, currentPiece);
|
|
267
|
+
}
|
|
250
268
|
|
|
251
|
-
|
|
252
|
-
const tx = log.transactions[txIdx]!;
|
|
253
|
-
sessionEntry.newTransactions.push(tx);
|
|
254
|
-
}
|
|
269
|
+
const firstPiece = pieces[0];
|
|
255
270
|
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
271
|
+
if (!firstPiece) {
|
|
272
|
+
throw new Error("First piece not found", {
|
|
273
|
+
cause: pieces,
|
|
274
|
+
});
|
|
275
|
+
}
|
|
260
276
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
277
|
+
const includeHeader = !knownState?.header;
|
|
278
|
+
|
|
279
|
+
if (includeHeader) {
|
|
280
|
+
firstPiece.header = this.header;
|
|
264
281
|
}
|
|
265
282
|
|
|
266
283
|
const piecesWithContent = pieces.filter(
|
|
267
|
-
(piece) => Object.keys(piece.new).length > 0
|
|
284
|
+
(piece) => piece.header || Object.keys(piece.new).length > 0,
|
|
268
285
|
);
|
|
269
286
|
|
|
270
|
-
if (piecesWithContent.length
|
|
271
|
-
|
|
287
|
+
if (piecesWithContent.length > 1 || this.isStreaming()) {
|
|
288
|
+
// Flag that more content is coming
|
|
289
|
+
firstPiece.expectContentUntil = {
|
|
290
|
+
...this.knownStateWithStreaming().sessions,
|
|
291
|
+
};
|
|
272
292
|
}
|
|
273
293
|
|
|
274
|
-
if (
|
|
275
|
-
|
|
294
|
+
if (piecesWithContent.length === 0) {
|
|
295
|
+
return undefined;
|
|
276
296
|
}
|
|
277
297
|
|
|
278
298
|
return piecesWithContent;
|
|
@@ -307,15 +327,10 @@ export class VerifiedState {
|
|
|
307
327
|
}
|
|
308
328
|
}
|
|
309
329
|
|
|
310
|
-
function
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
)
|
|
315
|
-
|
|
316
|
-
.map(Number)
|
|
317
|
-
.sort((a, b) => a - b)
|
|
318
|
-
.find(
|
|
319
|
-
(idx) => idx >= (sentStateForSessionID ?? knownStateForSessionID ?? -1),
|
|
320
|
-
);
|
|
330
|
+
function assertLastSignature(sessionID: SessionID, content: NewContentMessage) {
|
|
331
|
+
if (content.new[sessionID] && !content.new[sessionID].lastSignature) {
|
|
332
|
+
throw new Error("The SessionContent sent must have a lastSignature", {
|
|
333
|
+
cause: content.new[sessionID],
|
|
334
|
+
});
|
|
335
|
+
}
|
|
321
336
|
}
|
package/src/coValues/group.ts
CHANGED
|
@@ -123,6 +123,13 @@ function healMissingKeyForEveryone(group: RawGroup) {
|
|
|
123
123
|
}
|
|
124
124
|
|
|
125
125
|
function needsKeyRotation(group: RawGroup) {
|
|
126
|
+
const myRole = group.myRole();
|
|
127
|
+
|
|
128
|
+
// Checking only direct membership because inside the migrations we can't navigate the parent groups
|
|
129
|
+
if (myRole !== "admin" && myRole !== "manager") {
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
|
|
126
133
|
const currentReadKeyId = group.get("readKey");
|
|
127
134
|
|
|
128
135
|
if (!currentReadKeyId) {
|
|
@@ -190,11 +197,27 @@ export class RawGroup<
|
|
|
190
197
|
) {
|
|
191
198
|
super(core, options);
|
|
192
199
|
this.crypto = core.node.crypto;
|
|
200
|
+
this.migrate();
|
|
201
|
+
}
|
|
193
202
|
|
|
194
|
-
|
|
195
|
-
if (core.isGroup()
|
|
196
|
-
|
|
203
|
+
migrate() {
|
|
204
|
+
if (!this.core.isGroup()) {
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const runMigrations = () => {
|
|
209
|
+
rotateReadKeyIfNeeded(this);
|
|
197
210
|
healMissingKeyForEveryone(this);
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
// We need the group and their parents to be completely downloaded to correctly handle the migrations
|
|
214
|
+
if (!this.core.isCompletelyDownloaded()) {
|
|
215
|
+
this.core.waitFor({
|
|
216
|
+
predicate: (core) => core.isCompletelyDownloaded(),
|
|
217
|
+
onSuccess: runMigrations,
|
|
218
|
+
});
|
|
219
|
+
} else {
|
|
220
|
+
runMigrations();
|
|
198
221
|
}
|
|
199
222
|
}
|
|
200
223
|
|
|
@@ -318,7 +341,7 @@ export class RawGroup<
|
|
|
318
341
|
* @category 1. Role reading
|
|
319
342
|
*/
|
|
320
343
|
myRole(): Role | undefined {
|
|
321
|
-
return this.roleOfInternal(this.core.node.
|
|
344
|
+
return this.roleOfInternal(this.core.node.getCurrentAccountOrAgentID());
|
|
322
345
|
}
|
|
323
346
|
|
|
324
347
|
/**
|
package/src/knownState.ts
CHANGED
|
@@ -109,7 +109,7 @@ export function cloneKnownState(knownState: CoValueKnownState) {
|
|
|
109
109
|
/**
|
|
110
110
|
* Checks if all the local sessions have the same counters as in remote.
|
|
111
111
|
*/
|
|
112
|
-
export function
|
|
112
|
+
export function isKnownStateSubsetOf(
|
|
113
113
|
local: Record<string, number>,
|
|
114
114
|
remote: Record<string, number>,
|
|
115
115
|
) {
|
package/src/localNode.ts
CHANGED
|
@@ -650,8 +650,7 @@ export class LocalNode {
|
|
|
650
650
|
);
|
|
651
651
|
|
|
652
652
|
const contentPieces =
|
|
653
|
-
groupAsInvite.core.
|
|
654
|
-
[];
|
|
653
|
+
groupAsInvite.core.newContentSince(group.core.knownState()) ?? [];
|
|
655
654
|
|
|
656
655
|
// Import the new transactions to the current localNode
|
|
657
656
|
for (const contentPiece of contentPieces) {
|
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
import {
|
|
2
2
|
addTransactionToContentMessage,
|
|
3
3
|
createContentMessage,
|
|
4
|
+
knownStateFromContent,
|
|
4
5
|
} from "../coValueContentMessage.js";
|
|
5
6
|
import { Transaction, VerifiedState } from "../coValueCore/verifiedState.js";
|
|
6
7
|
import { Signature } from "../crypto/crypto.js";
|
|
7
8
|
import { RawCoID, SessionID } from "../ids.js";
|
|
9
|
+
import {
|
|
10
|
+
combineKnownStateSessions,
|
|
11
|
+
KnownStateSessions,
|
|
12
|
+
} from "../knownState.js";
|
|
8
13
|
import { NewContentMessage } from "../sync.js";
|
|
9
14
|
import { LinkedList } from "./LinkedList.js";
|
|
10
15
|
|
|
@@ -113,9 +118,29 @@ export class LocalTransactionsSyncQueue {
|
|
|
113
118
|
this.processingSyncs = true;
|
|
114
119
|
|
|
115
120
|
queueMicrotask(() => {
|
|
121
|
+
const firstContentPieceMap = new Map<RawCoID, NewContentMessage>();
|
|
122
|
+
|
|
116
123
|
while (this.queue.head) {
|
|
117
124
|
const content = this.queue.head.value;
|
|
118
125
|
|
|
126
|
+
const firstContentPiece = firstContentPieceMap.get(content.id);
|
|
127
|
+
|
|
128
|
+
if (!firstContentPiece) {
|
|
129
|
+
firstContentPieceMap.set(content.id, content);
|
|
130
|
+
} else {
|
|
131
|
+
// There is already a content piece for this coValue, so this means that we need to flag
|
|
132
|
+
// that this content is going to be streamed
|
|
133
|
+
if (!firstContentPiece.expectContentUntil) {
|
|
134
|
+
firstContentPiece.expectContentUntil =
|
|
135
|
+
knownStateFromContent(firstContentPiece).sessions;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
combineKnownStateSessions(
|
|
139
|
+
firstContentPiece.expectContentUntil,
|
|
140
|
+
knownStateFromContent(content).sessions,
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
|
|
119
144
|
this.sync(content);
|
|
120
145
|
|
|
121
146
|
this.queue.shift();
|
|
@@ -3,7 +3,7 @@ import { RawCoID } from "../ids.js";
|
|
|
3
3
|
import {
|
|
4
4
|
CoValueKnownState,
|
|
5
5
|
emptyKnownState,
|
|
6
|
-
|
|
6
|
+
isKnownStateSubsetOf,
|
|
7
7
|
} from "../knownState.js";
|
|
8
8
|
|
|
9
9
|
/**
|
|
@@ -87,7 +87,7 @@ function isInSync(
|
|
|
87
87
|
return false;
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
return
|
|
90
|
+
return isKnownStateSubsetOf(
|
|
91
91
|
knownState.sessions,
|
|
92
92
|
knownStateFromStorage.sessions,
|
|
93
93
|
);
|
package/src/sync.ts
CHANGED
|
@@ -228,7 +228,7 @@ export class SyncManager {
|
|
|
228
228
|
}
|
|
229
229
|
}
|
|
230
230
|
|
|
231
|
-
const newContentPieces = coValue.
|
|
231
|
+
const newContentPieces = coValue.newContentSince(
|
|
232
232
|
peer.getOptimisticKnownState(id),
|
|
233
233
|
);
|
|
234
234
|
|
|
@@ -585,6 +585,8 @@ export class SyncManager {
|
|
|
585
585
|
sessions: msg.expectContentUntil,
|
|
586
586
|
});
|
|
587
587
|
}
|
|
588
|
+
} else if (msg.expectContentUntil) {
|
|
589
|
+
coValue.verified.setStreamingKnownState(msg.expectContentUntil);
|
|
588
590
|
}
|
|
589
591
|
|
|
590
592
|
// At this point the CoValue must be in memory, if not we have a bug
|
|
@@ -647,6 +649,9 @@ export class SyncManager {
|
|
|
647
649
|
peerRole: peer.role,
|
|
648
650
|
id: msg.id,
|
|
649
651
|
err: result.error,
|
|
652
|
+
msgKnownState: knownStateFromContent(msg).sessions,
|
|
653
|
+
knownState: coValue.knownState().sessions,
|
|
654
|
+
newContent: validNewContent.new,
|
|
650
655
|
});
|
|
651
656
|
// TODO Mark only the session as errored, not the whole coValue
|
|
652
657
|
coValue.markErrored(peer.id, result.error);
|
|
@@ -822,7 +827,7 @@ export class SyncManager {
|
|
|
822
827
|
return undefined;
|
|
823
828
|
}
|
|
824
829
|
|
|
825
|
-
return value.
|
|
830
|
+
return value.newContentSince(correction);
|
|
826
831
|
});
|
|
827
832
|
}
|
|
828
833
|
|
|
@@ -117,8 +117,7 @@ describe("PureJSCrypto", () => {
|
|
|
117
117
|
true,
|
|
118
118
|
);
|
|
119
119
|
|
|
120
|
-
const content =
|
|
121
|
-
mapInOtherSession.core.verified.newContentSince(undefined)?.[0];
|
|
120
|
+
const content = mapInOtherSession.core.newContentSince(undefined)?.[0];
|
|
122
121
|
assert(content);
|
|
123
122
|
|
|
124
123
|
client.node.syncManager.handleNewContent(content, "storage");
|
|
@@ -31,7 +31,7 @@ function getNewContentSince(
|
|
|
31
31
|
throw new Error(`CoValue ${coValue.id} is not verified`);
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
const contentMessage = coValue.
|
|
34
|
+
const contentMessage = coValue.newContentSince(knownState)?.[0];
|
|
35
35
|
|
|
36
36
|
if (!contentMessage) {
|
|
37
37
|
throw new Error(`No new content available for coValue ${coValue.id}`);
|
|
@@ -295,7 +295,7 @@ describe("StorageApiAsync", () => {
|
|
|
295
295
|
const contentMessage = getNewContentSince(group.core, knownState);
|
|
296
296
|
const correctionCallback = vi.fn((known) => {
|
|
297
297
|
expect(known).toEqual(emptyKnownState(group.id));
|
|
298
|
-
return group.core.
|
|
298
|
+
return group.core.newContentSince(known);
|
|
299
299
|
});
|
|
300
300
|
|
|
301
301
|
// Get initial known state
|
|
@@ -339,7 +339,7 @@ describe("StorageApiAsync", () => {
|
|
|
339
339
|
const contentMessage = getNewContentSince(group.core, knownState);
|
|
340
340
|
const correctionCallback = vi.fn((known) => {
|
|
341
341
|
expect(known).toEqual(initialKnownState);
|
|
342
|
-
return group.core.
|
|
342
|
+
return group.core.newContentSince(known);
|
|
343
343
|
});
|
|
344
344
|
|
|
345
345
|
// Get initial storage known state
|
|
@@ -539,16 +539,16 @@ describe("StorageApiAsync", () => {
|
|
|
539
539
|
}),
|
|
540
540
|
).toMatchInlineSnapshot(`
|
|
541
541
|
[
|
|
542
|
-
"test -> test-storage | CONTENT Core header: false new: After: 1 New: 1",
|
|
543
|
-
"test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1",
|
|
542
|
+
"test -> test-storage | CONTENT Core header: false new: After: 1 New: 1 expectContentUntil: header/3",
|
|
543
|
+
"test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1 expectContentUntil: header/3",
|
|
544
544
|
"test -> test-storage | CONTENT Core header: false new: After: 2 New: 1",
|
|
545
545
|
"test -> test-storage | CONTENT Core2 header: false new: After: 2 New: 1",
|
|
546
546
|
"test-storage -> test | KNOWN CORRECTION Core sessions: empty",
|
|
547
547
|
"test -> test-storage | CONTENT Core header: true new: After: 0 New: 3",
|
|
548
548
|
"test-storage -> test | KNOWN CORRECTION Core2 sessions: empty",
|
|
549
549
|
"test -> test-storage | CONTENT Core2 header: true new: After: 0 New: 3",
|
|
550
|
-
"test -> test-storage | CONTENT Core header: false new: After: 3 New: 1",
|
|
551
|
-
"test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1",
|
|
550
|
+
"test -> test-storage | CONTENT Core header: false new: After: 3 New: 1 expectContentUntil: header/5",
|
|
551
|
+
"test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1 expectContentUntil: header/5",
|
|
552
552
|
"test -> test-storage | CONTENT Core header: false new: After: 4 New: 1",
|
|
553
553
|
"test -> test-storage | CONTENT Core2 header: false new: After: 4 New: 1",
|
|
554
554
|
]
|
|
@@ -606,14 +606,14 @@ describe("StorageApiAsync", () => {
|
|
|
606
606
|
}),
|
|
607
607
|
).toMatchInlineSnapshot(`
|
|
608
608
|
[
|
|
609
|
-
"test -> test-storage | CONTENT Core header: false new: After: 1 New: 1",
|
|
610
|
-
"test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1",
|
|
609
|
+
"test -> test-storage | CONTENT Core header: false new: After: 1 New: 1 expectContentUntil: header/3",
|
|
610
|
+
"test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1 expectContentUntil: header/3",
|
|
611
611
|
"test -> test-storage | CONTENT Core header: false new: After: 2 New: 1",
|
|
612
612
|
"test -> test-storage | CONTENT Core2 header: false new: After: 2 New: 1",
|
|
613
613
|
"test-storage -> test | KNOWN CORRECTION Core sessions: empty",
|
|
614
614
|
"test -> test-storage | CONTENT Core header: true new: After: 0 New: 3",
|
|
615
|
-
"test -> test-storage | CONTENT Core header: false new: After: 3 New: 1",
|
|
616
|
-
"test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1",
|
|
615
|
+
"test -> test-storage | CONTENT Core header: false new: After: 3 New: 1 expectContentUntil: header/5",
|
|
616
|
+
"test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1 expectContentUntil: header/5",
|
|
617
617
|
"test -> test-storage | CONTENT Core header: false new: After: 4 New: 1",
|
|
618
618
|
"test -> test-storage | CONTENT Core2 header: false new: After: 4 New: 1",
|
|
619
619
|
]
|
|
@@ -26,7 +26,7 @@ function getNewContentSince(
|
|
|
26
26
|
throw new Error(`CoValue ${coValue.id} is not verified`);
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
const contentMessage = coValue.
|
|
29
|
+
const contentMessage = coValue.newContentSince(knownState)?.[0];
|
|
30
30
|
|
|
31
31
|
if (!contentMessage) {
|
|
32
32
|
throw new Error(`No new content available for coValue ${coValue.id}`);
|
|
@@ -274,7 +274,7 @@ describe("StorageApiSync", () => {
|
|
|
274
274
|
const contentMessage = getNewContentSince(group.core, knownState);
|
|
275
275
|
const correctionCallback = vi.fn((known) => {
|
|
276
276
|
expect(known).toEqual(emptyKnownState(group.id));
|
|
277
|
-
return group.core.
|
|
277
|
+
return group.core.newContentSince(known);
|
|
278
278
|
});
|
|
279
279
|
|
|
280
280
|
// Get initial known state
|
|
@@ -318,7 +318,7 @@ describe("StorageApiSync", () => {
|
|
|
318
318
|
const contentMessage = getNewContentSince(group.core, knownState);
|
|
319
319
|
const correctionCallback = vi.fn((known) => {
|
|
320
320
|
expect(known).toEqual(initialKnownState);
|
|
321
|
-
return group.core.
|
|
321
|
+
return group.core.newContentSince(known);
|
|
322
322
|
});
|
|
323
323
|
|
|
324
324
|
// Get initial storage known state
|
|
@@ -117,8 +117,7 @@ describe("WasmCrypto", () => {
|
|
|
117
117
|
true,
|
|
118
118
|
);
|
|
119
119
|
|
|
120
|
-
const content =
|
|
121
|
-
mapInOtherSession.core.verified.newContentSince(undefined)?.[0];
|
|
120
|
+
const content = mapInOtherSession.core.newContentSince(undefined)?.[0];
|
|
122
121
|
assert(content);
|
|
123
122
|
|
|
124
123
|
client.node.syncManager.handleNewContent(content, "storage");
|
|
@@ -371,7 +371,7 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
|
|
|
371
371
|
}),
|
|
372
372
|
).toMatchInlineSnapshot(`
|
|
373
373
|
[
|
|
374
|
-
"client -> storage | CONTENT CoPlainText header: true new: After: 0 New: 1",
|
|
374
|
+
"client -> storage | CONTENT CoPlainText header: true new: After: 0 New: 1 expectContentUntil: header/42",
|
|
375
375
|
"client -> storage | CONTENT CoPlainText header: false new: After: 1 New: 1",
|
|
376
376
|
"client -> storage | CONTENT CoPlainText header: false new: After: 2 New: 1",
|
|
377
377
|
"client -> storage | CONTENT CoPlainText header: false new: After: 3 New: 1",
|
|
@@ -217,24 +217,24 @@ test("When adding large transactions (small fraction of MAX_RECOMMENDED_TX_SIZE)
|
|
|
217
217
|
expect(sessionEntry.signatureAfter[10]).not.toBeDefined();
|
|
218
218
|
expect(sessionEntry.signatureAfter[11]).not.toBeDefined();
|
|
219
219
|
|
|
220
|
-
const newContent = coValue.
|
|
220
|
+
const newContent = coValue.newContentSince({
|
|
221
221
|
id: coValue.id,
|
|
222
222
|
header: false,
|
|
223
223
|
sessions: {},
|
|
224
224
|
})!;
|
|
225
225
|
|
|
226
|
-
expect(newContent.length).toEqual(
|
|
226
|
+
expect(newContent.length).toEqual(4);
|
|
227
227
|
expect(newContent[0]!.header).toBeDefined();
|
|
228
|
-
expect(newContent[
|
|
228
|
+
expect(newContent[0]!.new[node.currentSessionID]!.lastSignature).toEqual(
|
|
229
229
|
sessionEntry.signatureAfter[3],
|
|
230
230
|
);
|
|
231
|
-
expect(newContent[
|
|
231
|
+
expect(newContent[1]!.new[node.currentSessionID]!.lastSignature).toEqual(
|
|
232
232
|
sessionEntry.signatureAfter[6],
|
|
233
233
|
);
|
|
234
|
-
expect(newContent[
|
|
234
|
+
expect(newContent[2]!.new[node.currentSessionID]!.lastSignature).toEqual(
|
|
235
235
|
sessionEntry.signatureAfter[9],
|
|
236
236
|
);
|
|
237
|
-
expect(newContent[
|
|
237
|
+
expect(newContent[3]!.new[node.currentSessionID]!.lastSignature).toEqual(
|
|
238
238
|
sessionEntry.lastSignature,
|
|
239
239
|
);
|
|
240
240
|
});
|
|
@@ -286,24 +286,24 @@ test("When adding large transactions (bigger than MAX_RECOMMENDED_TX_SIZE), we s
|
|
|
286
286
|
expect(sessionEntry.signatureAfter[3]).toBeDefined();
|
|
287
287
|
expect(sessionEntry.signatureAfter[4]).not.toBeDefined();
|
|
288
288
|
|
|
289
|
-
const newContent = coValue.
|
|
289
|
+
const newContent = coValue.newContentSince({
|
|
290
290
|
id: coValue.id,
|
|
291
291
|
header: false,
|
|
292
292
|
sessions: {},
|
|
293
293
|
})!;
|
|
294
294
|
|
|
295
|
-
expect(newContent.length).toEqual(
|
|
295
|
+
expect(newContent.length).toEqual(4);
|
|
296
296
|
expect(newContent[0]!.header).toBeDefined();
|
|
297
|
-
expect(newContent[
|
|
297
|
+
expect(newContent[0]!.new[node.currentSessionID]!.lastSignature).toEqual(
|
|
298
298
|
sessionEntry.signatureAfter[1],
|
|
299
299
|
);
|
|
300
|
-
expect(newContent[
|
|
300
|
+
expect(newContent[1]!.new[node.currentSessionID]!.lastSignature).toEqual(
|
|
301
301
|
sessionEntry.signatureAfter[2],
|
|
302
302
|
);
|
|
303
|
-
expect(newContent[
|
|
303
|
+
expect(newContent[2]!.new[node.currentSessionID]!.lastSignature).toEqual(
|
|
304
304
|
sessionEntry.signatureAfter[3],
|
|
305
305
|
);
|
|
306
|
-
expect(newContent[
|
|
306
|
+
expect(newContent[3]!.new[node.currentSessionID]!.lastSignature).toEqual(
|
|
307
307
|
sessionEntry.lastSignature,
|
|
308
308
|
);
|
|
309
309
|
});
|