@twin.org/synchronised-storage-service 0.0.1-next.2 → 0.0.1-next.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +962 -314
- package/dist/esm/index.mjs +963 -316
- package/dist/types/entities/syncSnapshotEntry.d.ts +4 -5
- package/dist/types/helpers/blobStorageHelper.d.ts +33 -0
- package/dist/types/helpers/changeSetHelper.d.ts +20 -7
- package/dist/types/helpers/localSyncStateHelper.d.ts +13 -28
- package/dist/types/helpers/remoteSyncStateHelper.d.ts +26 -21
- package/dist/types/helpers/versions.d.ts +3 -0
- package/dist/types/index.d.ts +4 -2
- package/dist/types/models/ISyncPointerStore.d.ts +15 -0
- package/dist/types/models/ISyncSnapshot.d.ts +5 -1
- package/dist/types/models/ISyncState.d.ts +4 -0
- package/dist/types/models/ISynchronisedStorageServiceConfig.d.ts +17 -11
- package/dist/types/models/ISynchronisedStorageServiceConstructorOptions.d.ts +11 -2
- package/dist/types/synchronisedStorageRoutes.d.ts +9 -1
- package/dist/types/synchronisedStorageService.d.ts +13 -4
- package/docs/architecture.md +125 -0
- package/docs/changelog.md +29 -0
- package/docs/open-api/spec.json +244 -18
- package/docs/reference/classes/SyncSnapshotEntry.md +5 -5
- package/docs/reference/classes/SynchronisedStorageService.md +38 -5
- package/docs/reference/functions/synchronisedStorageGetDecryptionKeyRequest.md +31 -0
- package/docs/reference/index.md +4 -1
- package/docs/reference/interfaces/ISyncPointerStore.md +23 -0
- package/docs/reference/interfaces/ISyncSnapshot.md +43 -0
- package/docs/reference/interfaces/ISyncState.md +19 -0
- package/docs/reference/interfaces/ISynchronisedStorageServiceConfig.md +37 -17
- package/docs/reference/interfaces/ISynchronisedStorageServiceConstructorOptions.md +25 -3
- package/locales/en.json +62 -14
- package/package.json +4 -2
- package/dist/types/models/ISyncChange.d.ts +0 -18
- package/dist/types/models/ISyncChangeSet.d.ts +0 -36
- package/dist/types/models/ISyncPointer.d.ts +0 -9
- package/docs/reference/interfaces/ISyncChange.md +0 -33
package/dist/esm/index.mjs
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { property, entity, EntitySchemaFactory, EntitySchemaHelper, ComparisonOperator } from '@twin.org/entity';
|
|
2
|
-
import { Guards, ComponentFactory, Is,
|
|
2
|
+
import { Guards, ComponentFactory, Is, Converter, Compression, CompressionType, ObjectHelper, BaseError, GeneralError, RandomHelper, NotFoundError, UnauthorizedError } from '@twin.org/core';
|
|
3
3
|
import { HttpStatusCode } from '@twin.org/web';
|
|
4
|
+
import { BlobStorageConnectorFactory } from '@twin.org/blob-storage-models';
|
|
4
5
|
import { EntityStorageConnectorFactory } from '@twin.org/entity-storage-models';
|
|
5
6
|
import { DocumentHelper, IdentityConnectorFactory } from '@twin.org/identity-models';
|
|
6
7
|
import { LoggingConnectorFactory } from '@twin.org/logging-models';
|
|
8
|
+
import { ProofTypes } from '@twin.org/standards-w3c-did';
|
|
7
9
|
import { SyncChangeOperation, SynchronisedStorageTopics } from '@twin.org/synchronised-storage-models';
|
|
10
|
+
import { VaultEncryptionType, VaultConnectorFactory } from '@twin.org/vault-models';
|
|
8
11
|
import { VerifiableStorageConnectorFactory } from '@twin.org/verifiable-storage-models';
|
|
9
|
-
import {
|
|
10
|
-
import { ProofTypes } from '@twin.org/standards-w3c-did';
|
|
12
|
+
import { RSA } from '@twin.org/crypto';
|
|
11
13
|
|
|
12
14
|
// Copyright 2024 IOTA Stiftung.
|
|
13
15
|
// SPDX-License-Identifier: Apache-2.0.
|
|
@@ -20,9 +22,9 @@ let SyncSnapshotEntry = class SyncSnapshotEntry {
|
|
|
20
22
|
*/
|
|
21
23
|
id;
|
|
22
24
|
/**
|
|
23
|
-
* The
|
|
25
|
+
* The storage key for the snapshot i.e. which entity is being synchronized.
|
|
24
26
|
*/
|
|
25
|
-
|
|
27
|
+
storageKey;
|
|
26
28
|
/**
|
|
27
29
|
* The date the snapshot was created.
|
|
28
30
|
*/
|
|
@@ -42,7 +44,7 @@ let SyncSnapshotEntry = class SyncSnapshotEntry {
|
|
|
42
44
|
/**
|
|
43
45
|
* The changes that were made in this snapshot, if this is a local snapshot.
|
|
44
46
|
*/
|
|
45
|
-
|
|
47
|
+
changes;
|
|
46
48
|
};
|
|
47
49
|
__decorate([
|
|
48
50
|
property({ type: "string", isPrimary: true }),
|
|
@@ -51,7 +53,7 @@ __decorate([
|
|
|
51
53
|
__decorate([
|
|
52
54
|
property({ type: "string", isSecondary: true }),
|
|
53
55
|
__metadata("design:type", String)
|
|
54
|
-
], SyncSnapshotEntry.prototype, "
|
|
56
|
+
], SyncSnapshotEntry.prototype, "storageKey", void 0);
|
|
55
57
|
__decorate([
|
|
56
58
|
property({ type: "string" }),
|
|
57
59
|
__metadata("design:type", String)
|
|
@@ -71,7 +73,7 @@ __decorate([
|
|
|
71
73
|
__decorate([
|
|
72
74
|
property({ type: "array", itemType: "object", optional: true }),
|
|
73
75
|
__metadata("design:type", Array)
|
|
74
|
-
], SyncSnapshotEntry.prototype, "
|
|
76
|
+
], SyncSnapshotEntry.prototype, "changes", void 0);
|
|
75
77
|
SyncSnapshotEntry = __decorate([
|
|
76
78
|
entity()
|
|
77
79
|
], SyncSnapshotEntry);
|
|
@@ -100,8 +102,8 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
|
|
|
100
102
|
operationId: "synchronisedStorageSyncChangeSetRequest",
|
|
101
103
|
summary: "Request that the node perform a sync request for a changeset.",
|
|
102
104
|
tag: tagsSynchronisedStorage[0].name,
|
|
103
|
-
method: "
|
|
104
|
-
path: `${baseRouteName}
|
|
105
|
+
method: "POST",
|
|
106
|
+
path: `${baseRouteName}/sync-changeset`,
|
|
105
107
|
handler: async (httpRequestContext, request) => synchronisedStorageSyncChangeSetRequest(httpRequestContext, componentName, request),
|
|
106
108
|
requestType: {
|
|
107
109
|
type: "ISyncChangeSetRequest",
|
|
@@ -109,8 +111,29 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
|
|
|
109
111
|
{
|
|
110
112
|
id: "synchronisedStorageSyncChangeSetRequestExample",
|
|
111
113
|
request: {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
+
body: {
|
|
115
|
+
id: "0909090909090909090909090909090909090909090909090909090909090909",
|
|
116
|
+
dateCreated: "2025-05-29T01:00:00.000Z",
|
|
117
|
+
nodeIdentity: "did:entity-storage:0xd2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2",
|
|
118
|
+
changes: [
|
|
119
|
+
{
|
|
120
|
+
entity: {
|
|
121
|
+
dateModified: "2025-01-01T00:00:00.000Z"
|
|
122
|
+
},
|
|
123
|
+
id: "test-id-1",
|
|
124
|
+
operation: "set"
|
|
125
|
+
}
|
|
126
|
+
],
|
|
127
|
+
proof: {
|
|
128
|
+
"@context": "https://www.w3.org/ns/credentials/v2",
|
|
129
|
+
created: "2025-05-29T01:00:00.000Z",
|
|
130
|
+
cryptosuite: "eddsa-jcs-2022",
|
|
131
|
+
proofPurpose: "assertionMethod",
|
|
132
|
+
proofValue: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3",
|
|
133
|
+
type: "DataIntegrityProof",
|
|
134
|
+
verificationMethod: "did:entity-storage:0xd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0#synchronised-storage-assertion"
|
|
135
|
+
},
|
|
136
|
+
storageKey: "test-type"
|
|
114
137
|
}
|
|
115
138
|
}
|
|
116
139
|
}
|
|
@@ -120,9 +143,61 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
|
|
|
120
143
|
{
|
|
121
144
|
type: "INoContentResponse"
|
|
122
145
|
}
|
|
123
|
-
]
|
|
146
|
+
],
|
|
147
|
+
// Authentication is provided by the proof in the request body.
|
|
148
|
+
skipAuth: true
|
|
124
149
|
};
|
|
125
|
-
|
|
150
|
+
const getDecryptionKeyRoute = {
|
|
151
|
+
operationId: "synchronisedStorageGetDecryptionKeyRequest",
|
|
152
|
+
summary: "Request the decryption key.",
|
|
153
|
+
tag: tagsSynchronisedStorage[0].name,
|
|
154
|
+
method: "POST",
|
|
155
|
+
path: `${baseRouteName}/decryption-key`,
|
|
156
|
+
handler: async (httpRequestContext, request) => synchronisedStorageGetDecryptionKeyRequest(httpRequestContext, componentName, request),
|
|
157
|
+
requestType: {
|
|
158
|
+
type: "ISyncChangeSetRequest",
|
|
159
|
+
examples: [
|
|
160
|
+
{
|
|
161
|
+
id: "synchronisedStorageSyncGetDecryptionKeyRequestExample",
|
|
162
|
+
request: {
|
|
163
|
+
body: {
|
|
164
|
+
nodeIdentity: "did:entity-storage:0xd2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2",
|
|
165
|
+
proof: {
|
|
166
|
+
"@context": "https://www.w3.org/ns/credentials/v2",
|
|
167
|
+
created: "2025-05-29T01:00:00.000Z",
|
|
168
|
+
cryptosuite: "eddsa-jcs-2022",
|
|
169
|
+
proofPurpose: "assertionMethod",
|
|
170
|
+
proofValue: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3",
|
|
171
|
+
type: "DataIntegrityProof",
|
|
172
|
+
verificationMethod: "did:entity-storage:0xd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0#synchronised-storage-assertion"
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
]
|
|
178
|
+
},
|
|
179
|
+
responseType: [
|
|
180
|
+
{
|
|
181
|
+
type: "ISyncDecryptionKeyResponse",
|
|
182
|
+
examples: [
|
|
183
|
+
{
|
|
184
|
+
id: "synchronisedStorageSyncGetDecryptionKeyResponseExample",
|
|
185
|
+
response: {
|
|
186
|
+
body: {
|
|
187
|
+
decryptionKey: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3"
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
]
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
type: "IUnauthorizedResponse"
|
|
195
|
+
}
|
|
196
|
+
],
|
|
197
|
+
// Authentication is provided by the proof in the request body.
|
|
198
|
+
skipAuth: true
|
|
199
|
+
};
|
|
200
|
+
return [syncChangeSetRoute, getDecryptionKeyRoute];
|
|
126
201
|
}
|
|
127
202
|
/**
|
|
128
203
|
* Perform the sync change set operation.
|
|
@@ -133,13 +208,31 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
|
|
|
133
208
|
*/
|
|
134
209
|
async function synchronisedStorageSyncChangeSetRequest(httpRequestContext, componentName, request) {
|
|
135
210
|
Guards.object(ROUTES_SOURCE, "request", request);
|
|
136
|
-
Guards.object(ROUTES_SOURCE, "request.
|
|
211
|
+
Guards.object(ROUTES_SOURCE, "request.body", request.body);
|
|
137
212
|
const component = ComponentFactory.get(componentName);
|
|
138
|
-
await component.syncChangeSet(request.
|
|
213
|
+
await component.syncChangeSet(request.body);
|
|
139
214
|
return {
|
|
140
215
|
statusCode: HttpStatusCode.noContent
|
|
141
216
|
};
|
|
142
217
|
}
|
|
218
|
+
/**
|
|
219
|
+
* Request the decryption key.
|
|
220
|
+
* @param httpRequestContext The request context for the API.
|
|
221
|
+
* @param componentName The name of the component to use in the routes.
|
|
222
|
+
* @param request The request.
|
|
223
|
+
* @returns The response object with additional http response properties.
|
|
224
|
+
*/
|
|
225
|
+
async function synchronisedStorageGetDecryptionKeyRequest(httpRequestContext, componentName, request) {
|
|
226
|
+
Guards.object(ROUTES_SOURCE, "request", request);
|
|
227
|
+
Guards.object(ROUTES_SOURCE, "request.body", request.body);
|
|
228
|
+
const component = ComponentFactory.get(componentName);
|
|
229
|
+
const key = await component.getDecryptionKey(request.body.nodeIdentity, request.body.proof);
|
|
230
|
+
return {
|
|
231
|
+
body: {
|
|
232
|
+
decryptionKey: key
|
|
233
|
+
}
|
|
234
|
+
};
|
|
235
|
+
}
|
|
143
236
|
|
|
144
237
|
const restEntryPoints = [
|
|
145
238
|
{
|
|
@@ -159,6 +252,163 @@ function initSchema() {
|
|
|
159
252
|
EntitySchemaFactory.register("SyncSnapshotEntry", () => EntitySchemaHelper.getSchema(SyncSnapshotEntry));
|
|
160
253
|
}
|
|
161
254
|
|
|
255
|
+
var mainnet = "";
|
|
256
|
+
var testnet = "";
|
|
257
|
+
var devnet = "";
|
|
258
|
+
var verifiableStorageKeys = {
|
|
259
|
+
mainnet: mainnet,
|
|
260
|
+
testnet: testnet,
|
|
261
|
+
devnet: devnet
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Class for performing blob storage operations.
|
|
266
|
+
*/
|
|
267
|
+
class BlobStorageHelper {
|
|
268
|
+
/**
|
|
269
|
+
* Runtime name for the class.
|
|
270
|
+
*/
|
|
271
|
+
CLASS_NAME = "BlobStorageHelper";
|
|
272
|
+
/**
|
|
273
|
+
* The logging connector to use for logging.
|
|
274
|
+
* @internal
|
|
275
|
+
*/
|
|
276
|
+
_logging;
|
|
277
|
+
/**
|
|
278
|
+
* The vault connector.
|
|
279
|
+
* @internal
|
|
280
|
+
*/
|
|
281
|
+
_vaultConnector;
|
|
282
|
+
/**
|
|
283
|
+
* The blob storage connector to use.
|
|
284
|
+
* @internal
|
|
285
|
+
*/
|
|
286
|
+
_blobStorageConnector;
|
|
287
|
+
/**
|
|
288
|
+
* The id of the vault key to use for encrypting/decrypting blobs.
|
|
289
|
+
* @internal
|
|
290
|
+
*/
|
|
291
|
+
_blobStorageEncryptionKeyId;
|
|
292
|
+
/**
|
|
293
|
+
* Is this a trusted node.
|
|
294
|
+
* @internal
|
|
295
|
+
*/
|
|
296
|
+
_isTrustedNode;
|
|
297
|
+
/**
|
|
298
|
+
* Create a new instance of BlobStorageHelper.
|
|
299
|
+
* @param logging The logging connector to use for logging.
|
|
300
|
+
* @param vaultConnector The vault connector to use for for the encryption key.
|
|
301
|
+
* @param blobStorageConnector The blob storage component to use.
|
|
302
|
+
* @param blobStorageEncryptionKeyId The id of the vault key to use for encrypting/decrypting blobs.
|
|
303
|
+
* @param isTrustedNode Is this a trusted node.
|
|
304
|
+
*/
|
|
305
|
+
constructor(logging, vaultConnector, blobStorageConnector, blobStorageEncryptionKeyId, isTrustedNode) {
|
|
306
|
+
this._logging = logging;
|
|
307
|
+
this._vaultConnector = vaultConnector;
|
|
308
|
+
this._blobStorageConnector = blobStorageConnector;
|
|
309
|
+
this._blobStorageEncryptionKeyId = blobStorageEncryptionKeyId;
|
|
310
|
+
this._isTrustedNode = isTrustedNode;
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Load a blob from storage.
|
|
314
|
+
* @param blobId The id of the blob to apply.
|
|
315
|
+
* @returns The blob.
|
|
316
|
+
*/
|
|
317
|
+
async load(blobId) {
|
|
318
|
+
await this._logging?.log({
|
|
319
|
+
level: "info",
|
|
320
|
+
source: this.CLASS_NAME,
|
|
321
|
+
message: "loadBlob",
|
|
322
|
+
data: {
|
|
323
|
+
blobId
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
try {
|
|
327
|
+
const encryptedBlob = await this._blobStorageConnector.get(blobId);
|
|
328
|
+
if (Is.uint8Array(encryptedBlob)) {
|
|
329
|
+
let compressedBlob;
|
|
330
|
+
// If this is a trusted node, we can decrypt the blob using the vault
|
|
331
|
+
if (this._isTrustedNode) {
|
|
332
|
+
compressedBlob = await this._vaultConnector.decrypt(this._blobStorageEncryptionKeyId, VaultEncryptionType.Rsa2048, encryptedBlob);
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
// Otherwise we need the public key stored as a secret in the vault
|
|
336
|
+
const key = await this._vaultConnector.getSecret(this._blobStorageEncryptionKeyId);
|
|
337
|
+
const rsa = new RSA(Converter.base64ToBytes(key));
|
|
338
|
+
compressedBlob = rsa.decrypt(encryptedBlob);
|
|
339
|
+
}
|
|
340
|
+
const decompressedBlob = await Compression.decompress(compressedBlob, CompressionType.Gzip);
|
|
341
|
+
await this._logging?.log({
|
|
342
|
+
level: "info",
|
|
343
|
+
source: this.CLASS_NAME,
|
|
344
|
+
message: "loadedBlob",
|
|
345
|
+
data: {
|
|
346
|
+
blobId
|
|
347
|
+
}
|
|
348
|
+
});
|
|
349
|
+
return ObjectHelper.fromBytes(decompressedBlob);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
catch (error) {
|
|
353
|
+
await this._logging?.log({
|
|
354
|
+
level: "error",
|
|
355
|
+
source: this.CLASS_NAME,
|
|
356
|
+
message: "loadBlobFailed",
|
|
357
|
+
data: {
|
|
358
|
+
blobId
|
|
359
|
+
},
|
|
360
|
+
error: BaseError.fromError(error)
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
await this._logging?.log({
|
|
364
|
+
level: "info",
|
|
365
|
+
source: this.CLASS_NAME,
|
|
366
|
+
message: "loadBlobEmpty",
|
|
367
|
+
data: {
|
|
368
|
+
blobId
|
|
369
|
+
}
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
/**
|
|
373
|
+
* Save a blob.
|
|
374
|
+
* @param blob The blob to save.
|
|
375
|
+
* @returns The id of the blob.
|
|
376
|
+
*/
|
|
377
|
+
async saveBlob(blob) {
|
|
378
|
+
await this._logging?.log({
|
|
379
|
+
level: "info",
|
|
380
|
+
source: this.CLASS_NAME,
|
|
381
|
+
message: "saveBlob"
|
|
382
|
+
});
|
|
383
|
+
if (!this._isTrustedNode) {
|
|
384
|
+
throw new GeneralError(this.CLASS_NAME, "notTrustedNode");
|
|
385
|
+
}
|
|
386
|
+
const compressedBlob = await Compression.compress(ObjectHelper.toBytes(blob), CompressionType.Gzip);
|
|
387
|
+
const encryptedBlob = await this._vaultConnector.encrypt(this._blobStorageEncryptionKeyId, VaultEncryptionType.Rsa2048, compressedBlob);
|
|
388
|
+
try {
|
|
389
|
+
const blobId = await this._blobStorageConnector.set(encryptedBlob);
|
|
390
|
+
await this._logging?.log({
|
|
391
|
+
level: "info",
|
|
392
|
+
source: this.CLASS_NAME,
|
|
393
|
+
message: "savedBlob",
|
|
394
|
+
data: {
|
|
395
|
+
blobId
|
|
396
|
+
}
|
|
397
|
+
});
|
|
398
|
+
return blobId;
|
|
399
|
+
}
|
|
400
|
+
catch (error) {
|
|
401
|
+
await this._logging?.log({
|
|
402
|
+
level: "error",
|
|
403
|
+
source: this.CLASS_NAME,
|
|
404
|
+
message: "saveBlobFailed",
|
|
405
|
+
error: BaseError.fromError(error)
|
|
406
|
+
});
|
|
407
|
+
throw error;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
162
412
|
// Copyright 2024 IOTA Stiftung.
|
|
163
413
|
// SPDX-License-Identifier: Apache-2.0.
|
|
164
414
|
/**
|
|
@@ -180,10 +430,10 @@ class ChangeSetHelper {
|
|
|
180
430
|
*/
|
|
181
431
|
_eventBusComponent;
|
|
182
432
|
/**
|
|
183
|
-
* The blob storage
|
|
433
|
+
* The blob storage helper to use for remote sync states.
|
|
184
434
|
* @internal
|
|
185
435
|
*/
|
|
186
|
-
|
|
436
|
+
_blobStorageHelper;
|
|
187
437
|
/**
|
|
188
438
|
* The identity connector to use for signing/verifying changesets.
|
|
189
439
|
* @internal
|
|
@@ -194,50 +444,85 @@ class ChangeSetHelper {
|
|
|
194
444
|
* @internal
|
|
195
445
|
*/
|
|
196
446
|
_decentralisedStorageMethodId;
|
|
447
|
+
/**
|
|
448
|
+
* The identity of the node that is performing the update.
|
|
449
|
+
* @internal
|
|
450
|
+
*/
|
|
451
|
+
_nodeIdentity;
|
|
197
452
|
/**
|
|
198
453
|
* Create a new instance of ChangeSetHelper.
|
|
199
454
|
* @param logging The logging connector to use for logging.
|
|
200
455
|
* @param eventBusComponent The event bus component to use for events.
|
|
201
|
-
* @param blobStorageComponent The blob storage component to use for remote sync states.
|
|
202
456
|
* @param identityConnector The identity connector to use for signing/verifying changesets.
|
|
457
|
+
* @param blobStorageHelper The blob storage component to use for remote sync states.
|
|
203
458
|
* @param decentralisedStorageMethodId The id of the identity method to use when signing/verifying changesets.
|
|
204
459
|
*/
|
|
205
|
-
constructor(logging, eventBusComponent,
|
|
460
|
+
constructor(logging, eventBusComponent, identityConnector, blobStorageHelper, decentralisedStorageMethodId) {
|
|
206
461
|
this._logging = logging;
|
|
207
462
|
this._eventBusComponent = eventBusComponent;
|
|
208
463
|
this._decentralisedStorageMethodId = decentralisedStorageMethodId;
|
|
209
|
-
this.
|
|
464
|
+
this._blobStorageHelper = blobStorageHelper;
|
|
210
465
|
this._identityConnector = identityConnector;
|
|
211
466
|
}
|
|
467
|
+
/**
|
|
468
|
+
* Set the node identity to use for signing changesets.
|
|
469
|
+
* @param nodeIdentity The identity of the node that is performing the update.
|
|
470
|
+
*/
|
|
471
|
+
setNodeIdentity(nodeIdentity) {
|
|
472
|
+
this._nodeIdentity = nodeIdentity;
|
|
473
|
+
}
|
|
212
474
|
/**
|
|
213
475
|
* Get and verify a changeset.
|
|
214
476
|
* @param changeSetStorageId The id of the sync changeset to apply.
|
|
215
477
|
* @returns The changeset if it was verified.
|
|
216
478
|
*/
|
|
217
479
|
async getAndVerifyChangeset(changeSetStorageId) {
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
480
|
+
await this._logging?.log({
|
|
481
|
+
level: "info",
|
|
482
|
+
source: this.CLASS_NAME,
|
|
483
|
+
message: "getChangeSet",
|
|
484
|
+
data: {
|
|
485
|
+
changeSetStorageId
|
|
486
|
+
}
|
|
222
487
|
});
|
|
223
|
-
|
|
224
|
-
const
|
|
225
|
-
|
|
226
|
-
|
|
488
|
+
try {
|
|
489
|
+
const syncChangeSet = await this._blobStorageHelper.load(changeSetStorageId);
|
|
490
|
+
if (Is.object(syncChangeSet)) {
|
|
491
|
+
const verified = await this.verifyChangesetProof(syncChangeSet);
|
|
492
|
+
return verified ? syncChangeSet : undefined;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
catch (error) {
|
|
496
|
+
await this._logging?.log({
|
|
497
|
+
level: "warn",
|
|
498
|
+
source: this.CLASS_NAME,
|
|
499
|
+
message: "getChangeSetError",
|
|
500
|
+
data: {
|
|
501
|
+
changeSetStorageId
|
|
502
|
+
},
|
|
503
|
+
error: BaseError.fromError(error)
|
|
504
|
+
});
|
|
227
505
|
}
|
|
506
|
+
await this._logging?.log({
|
|
507
|
+
level: "info",
|
|
508
|
+
source: this.CLASS_NAME,
|
|
509
|
+
message: "getChangeSetEmpty",
|
|
510
|
+
data: {
|
|
511
|
+
changeSetStorageId
|
|
512
|
+
}
|
|
513
|
+
});
|
|
228
514
|
}
|
|
229
515
|
/**
|
|
230
516
|
* Apply a sync changeset.
|
|
231
517
|
* @param changeSetStorageId The id of the sync changeset to apply.
|
|
232
|
-
* @returns
|
|
518
|
+
* @returns The changeset if it existed.
|
|
233
519
|
*/
|
|
234
520
|
async getAndApplyChangeset(changeSetStorageId) {
|
|
235
521
|
const syncChangeset = await this.getAndVerifyChangeset(changeSetStorageId);
|
|
236
522
|
if (!Is.empty(syncChangeset)) {
|
|
237
523
|
await this.applyChangeset(syncChangeset);
|
|
238
|
-
return true;
|
|
239
524
|
}
|
|
240
|
-
return
|
|
525
|
+
return syncChangeset;
|
|
241
526
|
}
|
|
242
527
|
/**
|
|
243
528
|
* Apply a sync changeset.
|
|
@@ -259,20 +544,25 @@ class ChangeSetHelper {
|
|
|
259
544
|
switch (change.operation) {
|
|
260
545
|
case SyncChangeOperation.Set:
|
|
261
546
|
if (!Is.empty(change.entity)) {
|
|
262
|
-
// The
|
|
547
|
+
// The id was stripped from the entity as it is part of the operation
|
|
548
|
+
// we make sure we reinstate it in the publish
|
|
549
|
+
// Also the node identity was stripped when stored in the changeset
|
|
263
550
|
// as the changeset is signed with the node identity.
|
|
264
551
|
// so we need to restore it here.
|
|
265
|
-
change.entity.nodeIdentity = syncChangeset.nodeIdentity;
|
|
266
552
|
await this._eventBusComponent.publish(SynchronisedStorageTopics.RemoteItemSet, {
|
|
267
|
-
|
|
268
|
-
entity:
|
|
553
|
+
storageKey: syncChangeset.storageKey,
|
|
554
|
+
entity: {
|
|
555
|
+
...change.entity,
|
|
556
|
+
id: change.id,
|
|
557
|
+
nodeIdentity: syncChangeset.nodeIdentity
|
|
558
|
+
}
|
|
269
559
|
});
|
|
270
560
|
}
|
|
271
561
|
break;
|
|
272
562
|
case SyncChangeOperation.Delete:
|
|
273
563
|
if (!Is.empty(change.id)) {
|
|
274
564
|
await this._eventBusComponent.publish(SynchronisedStorageTopics.RemoteItemRemove, {
|
|
275
|
-
|
|
565
|
+
storageKey: syncChangeset.storageKey,
|
|
276
566
|
id: change.id
|
|
277
567
|
});
|
|
278
568
|
}
|
|
@@ -295,12 +585,7 @@ class ChangeSetHelper {
|
|
|
295
585
|
id: syncChangeSet.id
|
|
296
586
|
}
|
|
297
587
|
});
|
|
298
|
-
|
|
299
|
-
// the blob storage also needs to be publicly accessible so that other nodes can retrieve it
|
|
300
|
-
return this._blobStorageComponent.create(Converter.bytesToBase64(ObjectHelper.toBytes(syncChangeSet)), undefined, undefined, undefined, {
|
|
301
|
-
disableEncryption: true,
|
|
302
|
-
compress: BlobStorageCompressionType.Gzip
|
|
303
|
-
});
|
|
588
|
+
return this._blobStorageHelper.saveBlob(syncChangeSet);
|
|
304
589
|
}
|
|
305
590
|
/**
|
|
306
591
|
* Verify the proof of a sync changeset.
|
|
@@ -319,6 +604,32 @@ class ChangeSetHelper {
|
|
|
319
604
|
});
|
|
320
605
|
return false;
|
|
321
606
|
}
|
|
607
|
+
// If the proof or verification method is missing, the proof is invalid
|
|
608
|
+
const verificationMethod = syncChangeset.proof?.verificationMethod;
|
|
609
|
+
if (!Is.stringValue(verificationMethod)) {
|
|
610
|
+
await this._logging?.log({
|
|
611
|
+
level: "error",
|
|
612
|
+
source: this.CLASS_NAME,
|
|
613
|
+
message: "verifyChangeSetProofMissing",
|
|
614
|
+
data: {
|
|
615
|
+
id: syncChangeset.id
|
|
616
|
+
}
|
|
617
|
+
});
|
|
618
|
+
}
|
|
619
|
+
// Parse the verification method and extract the node identity
|
|
620
|
+
// this should match the node identity of the changeset
|
|
621
|
+
// otherwise you could sign a changeset for another node
|
|
622
|
+
const changeSetNodeIdentity = DocumentHelper.parseId(verificationMethod ?? "");
|
|
623
|
+
if (changeSetNodeIdentity.id !== syncChangeset.nodeIdentity) {
|
|
624
|
+
await this._logging?.log({
|
|
625
|
+
level: "error",
|
|
626
|
+
source: this.CLASS_NAME,
|
|
627
|
+
message: "verifyChangeSetProofNodeIdentityMismatch",
|
|
628
|
+
data: {
|
|
629
|
+
id: syncChangeset.id
|
|
630
|
+
}
|
|
631
|
+
});
|
|
632
|
+
}
|
|
322
633
|
const changeSetWithoutProof = ObjectHelper.clone(syncChangeset);
|
|
323
634
|
delete changeSetWithoutProof.proof;
|
|
324
635
|
const isValid = await this._identityConnector.verifyProof(changeSetWithoutProof, syncChangeset.proof);
|
|
@@ -350,11 +661,12 @@ class ChangeSetHelper {
|
|
|
350
661
|
* @returns The proof.
|
|
351
662
|
*/
|
|
352
663
|
async createChangeSetProof(syncChangeset) {
|
|
664
|
+
Guards.stringValue(this.CLASS_NAME, "nodeIdentity", this._nodeIdentity);
|
|
353
665
|
const changeSetWithoutProof = ObjectHelper.clone(syncChangeset);
|
|
354
666
|
delete changeSetWithoutProof.proof;
|
|
355
|
-
const proof = await this._identityConnector.createProof(
|
|
667
|
+
const proof = await this._identityConnector.createProof(this._nodeIdentity, DocumentHelper.joinId(this._nodeIdentity, this._decentralisedStorageMethodId), ProofTypes.DataIntegrityProof, changeSetWithoutProof);
|
|
356
668
|
await this._logging?.log({
|
|
357
|
-
level: "
|
|
669
|
+
level: "info",
|
|
358
670
|
source: this.CLASS_NAME,
|
|
359
671
|
message: "createdChangeSetProof",
|
|
360
672
|
data: {
|
|
@@ -364,6 +676,35 @@ class ChangeSetHelper {
|
|
|
364
676
|
});
|
|
365
677
|
return proof;
|
|
366
678
|
}
|
|
679
|
+
/**
|
|
680
|
+
* Copy a change set.
|
|
681
|
+
* @param syncChangeSet The sync changeset to copy.
|
|
682
|
+
* @returns The id of the updated change set.
|
|
683
|
+
*/
|
|
684
|
+
async copyChangeset(syncChangeSet) {
|
|
685
|
+
if (Is.stringValue(this._nodeIdentity)) {
|
|
686
|
+
const verified = await this.verifyChangesetProof(syncChangeSet);
|
|
687
|
+
if (verified) {
|
|
688
|
+
await this._logging?.log({
|
|
689
|
+
level: "info",
|
|
690
|
+
source: this.CLASS_NAME,
|
|
691
|
+
message: "copyChangeSet",
|
|
692
|
+
data: {
|
|
693
|
+
changeSetStorageId: syncChangeSet.id
|
|
694
|
+
}
|
|
695
|
+
});
|
|
696
|
+
// Allocate a new id to the changeset copy and re-create a proof using this nodes identity
|
|
697
|
+
const copy = ObjectHelper.clone(syncChangeSet);
|
|
698
|
+
copy.id = Converter.bytesToHex(RandomHelper.generate(32));
|
|
699
|
+
copy.proof = await this.createChangeSetProof(copy);
|
|
700
|
+
// Store the copy
|
|
701
|
+
return {
|
|
702
|
+
syncChangeSet: copy,
|
|
703
|
+
changeSetStorageId: await this.storeChangeSet(copy)
|
|
704
|
+
};
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
}
|
|
367
708
|
}
|
|
368
709
|
|
|
369
710
|
// Copyright 2024 IOTA Stiftung.
|
|
@@ -385,7 +726,7 @@ class LocalSyncStateHelper {
|
|
|
385
726
|
* The storage connector for the sync snapshot entries.
|
|
386
727
|
* @internal
|
|
387
728
|
*/
|
|
388
|
-
|
|
729
|
+
_snapshotEntryEntityStorage;
|
|
389
730
|
/**
|
|
390
731
|
* The change set helper to use for applying changesets.
|
|
391
732
|
* @internal
|
|
@@ -394,44 +735,62 @@ class LocalSyncStateHelper {
|
|
|
394
735
|
/**
|
|
395
736
|
* Create a new instance of LocalSyncStateHelper.
|
|
396
737
|
* @param logging The logging connector to use for logging.
|
|
397
|
-
* @param
|
|
738
|
+
* @param snapshotEntryEntityStorage The storage connector for the sync snapshot entries.
|
|
398
739
|
* @param changeSetHelper The change set helper to use for applying changesets.
|
|
399
740
|
*/
|
|
400
|
-
constructor(logging,
|
|
741
|
+
constructor(logging, snapshotEntryEntityStorage, changeSetHelper) {
|
|
401
742
|
this._logging = logging;
|
|
402
|
-
this.
|
|
743
|
+
this._snapshotEntryEntityStorage = snapshotEntryEntityStorage;
|
|
403
744
|
this._changeSetHelper = changeSetHelper;
|
|
404
745
|
}
|
|
405
746
|
/**
|
|
406
747
|
* Add a new change to the local snapshot.
|
|
407
|
-
* @param
|
|
748
|
+
* @param storageKey The storage key of the snapshot to add the change for.
|
|
408
749
|
* @param operation The operation to perform.
|
|
409
750
|
* @param id The id of the entity to add the change for.
|
|
410
751
|
* @returns Nothing.
|
|
411
752
|
*/
|
|
412
|
-
async addLocalChange(
|
|
413
|
-
|
|
414
|
-
|
|
753
|
+
async addLocalChange(storageKey, operation, id) {
|
|
754
|
+
await this._logging?.log({
|
|
755
|
+
level: "info",
|
|
756
|
+
source: this.CLASS_NAME,
|
|
757
|
+
message: "addLocalChange",
|
|
758
|
+
data: {
|
|
759
|
+
storageKey,
|
|
760
|
+
operation,
|
|
761
|
+
id
|
|
762
|
+
}
|
|
763
|
+
});
|
|
764
|
+
const localChangeSnapshot = await this.getLocalChangeSnapshot(storageKey);
|
|
765
|
+
localChangeSnapshot.changes ??= [];
|
|
415
766
|
// If we already have a change for this id we are
|
|
416
767
|
// about to supersede it, we remove the previous change
|
|
417
768
|
// to avoid having multiple changes for the same id
|
|
418
|
-
const previousChangeIndex = localChangeSnapshot.
|
|
769
|
+
const previousChangeIndex = localChangeSnapshot.changes.findIndex(change => change.id === id);
|
|
419
770
|
if (previousChangeIndex !== -1) {
|
|
420
|
-
localChangeSnapshot.
|
|
771
|
+
localChangeSnapshot.changes.splice(previousChangeIndex, 1);
|
|
421
772
|
}
|
|
422
|
-
if (localChangeSnapshot.
|
|
773
|
+
if (localChangeSnapshot.changes.length > 0) {
|
|
423
774
|
localChangeSnapshot.dateModified = new Date(Date.now()).toISOString();
|
|
424
775
|
}
|
|
425
|
-
localChangeSnapshot.
|
|
776
|
+
localChangeSnapshot.changes.push({ operation, id });
|
|
426
777
|
await this.setLocalChangeSnapshot(localChangeSnapshot);
|
|
427
778
|
}
|
|
428
779
|
/**
|
|
429
|
-
* Get the current local snapshot.
|
|
430
|
-
* @param
|
|
780
|
+
* Get the current local snapshot which contains just the changes for this node.
|
|
781
|
+
* @param storageKey The storage key of the snapshot to get.
|
|
431
782
|
* @returns The local snapshot entry.
|
|
432
783
|
*/
|
|
433
|
-
async getLocalChangeSnapshot(
|
|
434
|
-
|
|
784
|
+
async getLocalChangeSnapshot(storageKey) {
|
|
785
|
+
await this._logging?.log({
|
|
786
|
+
level: "info",
|
|
787
|
+
source: this.CLASS_NAME,
|
|
788
|
+
message: "getLocalChangeSnapshot",
|
|
789
|
+
data: {
|
|
790
|
+
storageKey
|
|
791
|
+
}
|
|
792
|
+
});
|
|
793
|
+
const queryResult = await this._snapshotEntryEntityStorage.query({
|
|
435
794
|
conditions: [
|
|
436
795
|
{
|
|
437
796
|
property: "isLocalSnapshot",
|
|
@@ -439,33 +798,57 @@ class LocalSyncStateHelper {
|
|
|
439
798
|
comparison: ComparisonOperator.Equals
|
|
440
799
|
},
|
|
441
800
|
{
|
|
442
|
-
property: "
|
|
443
|
-
value:
|
|
801
|
+
property: "storageKey",
|
|
802
|
+
value: storageKey,
|
|
444
803
|
comparison: ComparisonOperator.Equals
|
|
445
804
|
}
|
|
446
805
|
]
|
|
447
806
|
});
|
|
448
807
|
if (queryResult.entities.length > 0) {
|
|
808
|
+
await this._logging?.log({
|
|
809
|
+
level: "info",
|
|
810
|
+
source: this.CLASS_NAME,
|
|
811
|
+
message: "localChangeSnapshotExists",
|
|
812
|
+
data: {
|
|
813
|
+
storageKey
|
|
814
|
+
}
|
|
815
|
+
});
|
|
449
816
|
return queryResult.entities[0];
|
|
450
817
|
}
|
|
818
|
+
await this._logging?.log({
|
|
819
|
+
level: "info",
|
|
820
|
+
source: this.CLASS_NAME,
|
|
821
|
+
message: "localChangeSnapshotDoesNotExist",
|
|
822
|
+
data: {
|
|
823
|
+
storageKey
|
|
824
|
+
}
|
|
825
|
+
});
|
|
451
826
|
return {
|
|
452
827
|
id: Converter.bytesToHex(RandomHelper.generate(32)),
|
|
453
|
-
|
|
828
|
+
storageKey,
|
|
454
829
|
dateCreated: new Date(Date.now()).toISOString(),
|
|
455
830
|
changeSetStorageIds: [],
|
|
456
831
|
isLocalSnapshot: true
|
|
457
832
|
};
|
|
458
833
|
}
|
|
459
834
|
/**
|
|
460
|
-
* Set the current local snapshot.
|
|
835
|
+
* Set the current local snapshot with changes for this node.
|
|
461
836
|
* @param localChangeSnapshot The local change snapshot to set.
|
|
462
837
|
* @returns Nothing.
|
|
463
838
|
*/
|
|
464
839
|
async setLocalChangeSnapshot(localChangeSnapshot) {
|
|
465
|
-
await this.
|
|
840
|
+
await this._logging?.log({
|
|
841
|
+
level: "info",
|
|
842
|
+
source: this.CLASS_NAME,
|
|
843
|
+
message: "setLocalChangeSnapshot",
|
|
844
|
+
data: {
|
|
845
|
+
storageKey: localChangeSnapshot.storageKey
|
|
846
|
+
}
|
|
847
|
+
});
|
|
848
|
+
await this._snapshotEntryEntityStorage.set(localChangeSnapshot);
|
|
466
849
|
}
|
|
467
850
|
/**
|
|
468
|
-
* Get the current local snapshot.
|
|
851
|
+
* Get the current local snapshot with the changes for this node.
|
|
469
852
|
* @param localChangeSnapshot The local change snapshot to remove.
|
|
470
853
|
* @returns Nothing.
|
|
471
854
|
*/
|
|
@@ -478,47 +861,47 @@ class LocalSyncStateHelper {
|
|
|
478
861
|
snapshotId: localChangeSnapshot.id
|
|
479
862
|
}
|
|
480
863
|
});
|
|
481
|
-
await this.
|
|
864
|
+
await this._snapshotEntryEntityStorage.remove(localChangeSnapshot.id);
|
|
482
865
|
}
|
|
483
866
|
/**
|
|
484
|
-
*
|
|
485
|
-
* @param
|
|
486
|
-
* @param
|
|
867
|
+
* Apply a sync state to the local node.
|
|
868
|
+
* @param storageKey The storage key of the snapshot to sync with.
|
|
869
|
+
* @param syncState The sync state to sync with.
|
|
487
870
|
* @returns Nothing.
|
|
488
871
|
*/
|
|
489
|
-
async
|
|
872
|
+
async applySyncState(storageKey, syncState) {
|
|
490
873
|
await this._logging?.log({
|
|
491
874
|
level: "info",
|
|
492
875
|
source: this.CLASS_NAME,
|
|
493
|
-
message: "
|
|
876
|
+
message: "applySyncState",
|
|
494
877
|
data: {
|
|
495
|
-
snapshotCount:
|
|
878
|
+
snapshotCount: syncState.snapshots.length
|
|
496
879
|
}
|
|
497
880
|
});
|
|
498
881
|
// Sort from newest to oldest
|
|
499
|
-
const
|
|
882
|
+
const sortedSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
|
|
500
883
|
const newSnapshots = [];
|
|
501
884
|
const modifiedSnapshots = [];
|
|
502
|
-
for (const
|
|
885
|
+
for (const snapshot of sortedSnapshots) {
|
|
503
886
|
await this._logging?.log({
|
|
504
887
|
level: "info",
|
|
505
888
|
source: this.CLASS_NAME,
|
|
506
|
-
message: "
|
|
889
|
+
message: "applySnapshot",
|
|
507
890
|
data: {
|
|
508
|
-
snapshotId:
|
|
509
|
-
dateCreated: new Date(
|
|
891
|
+
snapshotId: snapshot.id,
|
|
892
|
+
dateCreated: new Date(snapshot.dateCreated).toISOString()
|
|
510
893
|
}
|
|
511
894
|
});
|
|
512
|
-
const localSnapshot = await this.
|
|
895
|
+
const localSnapshot = await this._snapshotEntryEntityStorage.get(snapshot.id);
|
|
513
896
|
const remoteSnapshotWithContext = {
|
|
514
|
-
...
|
|
515
|
-
|
|
897
|
+
...snapshot,
|
|
898
|
+
storageKey
|
|
516
899
|
};
|
|
517
900
|
if (Is.empty(localSnapshot)) {
|
|
518
901
|
// We don't have the snapshot locally, so we need to process it
|
|
519
902
|
newSnapshots.push(remoteSnapshotWithContext);
|
|
520
903
|
}
|
|
521
|
-
else if (localSnapshot.dateModified !==
|
|
904
|
+
else if (localSnapshot.dateModified !== snapshot.dateModified) {
|
|
522
905
|
// If the local snapshot has a different dateModified, we need to update it
|
|
523
906
|
modifiedSnapshots.push({
|
|
524
907
|
localSnapshot,
|
|
@@ -540,13 +923,14 @@ class LocalSyncStateHelper {
|
|
|
540
923
|
* Process the modified snapshots and store them in the local storage.
|
|
541
924
|
* @param modifiedSnapshots The modified snapshots to process.
|
|
542
925
|
* @returns Nothing.
|
|
926
|
+
* @internal
|
|
543
927
|
*/
|
|
544
928
|
async processModifiedSnapshots(modifiedSnapshots) {
|
|
545
929
|
for (const modifiedSnapshot of modifiedSnapshots) {
|
|
546
930
|
await this._logging?.log({
|
|
547
931
|
level: "info",
|
|
548
932
|
source: this.CLASS_NAME,
|
|
549
|
-
message: "
|
|
933
|
+
message: "processModifiedSnapshot",
|
|
550
934
|
data: {
|
|
551
935
|
snapshotId: modifiedSnapshot.remoteSnapshot.id,
|
|
552
936
|
localModified: new Date(modifiedSnapshot.localSnapshot.dateModified ??
|
|
@@ -565,20 +949,21 @@ class LocalSyncStateHelper {
|
|
|
565
949
|
}
|
|
566
950
|
}
|
|
567
951
|
}
|
|
568
|
-
await this.
|
|
952
|
+
await this._snapshotEntryEntityStorage.set(modifiedSnapshot.remoteSnapshot);
|
|
569
953
|
}
|
|
570
954
|
}
|
|
571
955
|
/**
|
|
572
956
|
* Process the new snapshots and store them in the local storage.
|
|
573
957
|
* @param newSnapshots The new snapshots to process.
|
|
574
958
|
* @returns Nothing.
|
|
959
|
+
* @internal
|
|
575
960
|
*/
|
|
576
961
|
async processNewSnapshots(newSnapshots) {
|
|
577
962
|
for (const newSnapshot of newSnapshots) {
|
|
578
963
|
await this._logging?.log({
|
|
579
964
|
level: "info",
|
|
580
965
|
source: this.CLASS_NAME,
|
|
581
|
-
message: "
|
|
966
|
+
message: "processNewSnapshot",
|
|
582
967
|
data: {
|
|
583
968
|
snapshotId: newSnapshot.id,
|
|
584
969
|
localModified: new Date(newSnapshot.dateCreated).toISOString()
|
|
@@ -590,11 +975,17 @@ class LocalSyncStateHelper {
|
|
|
590
975
|
await this._changeSetHelper.getAndApplyChangeset(storageId);
|
|
591
976
|
}
|
|
592
977
|
}
|
|
593
|
-
await this.
|
|
978
|
+
await this._snapshotEntryEntityStorage.set(newSnapshot);
|
|
594
979
|
}
|
|
595
980
|
}
|
|
596
981
|
}
|
|
597
982
|
|
|
983
|
+
// Copyright 2024 IOTA Stiftung.
|
|
984
|
+
// SPDX-License-Identifier: Apache-2.0.
|
|
985
|
+
const SYNC_STATE_VERSION = "1";
|
|
986
|
+
const SYNC_POINTER_STORE_VERSION = "1";
|
|
987
|
+
const SYNC_SNAPSHOT_VERSION = "1";
|
|
988
|
+
|
|
598
989
|
// Copyright 2024 IOTA Stiftung.
|
|
599
990
|
// SPDX-License-Identifier: Apache-2.0.
|
|
600
991
|
/**
|
|
@@ -616,10 +1007,10 @@ class RemoteSyncStateHelper {
|
|
|
616
1007
|
*/
|
|
617
1008
|
_eventBusComponent;
|
|
618
1009
|
/**
|
|
619
|
-
* The blob storage
|
|
1010
|
+
* The blob storage helper.
|
|
620
1011
|
* @internal
|
|
621
1012
|
*/
|
|
622
|
-
|
|
1013
|
+
_blobStorageHelper;
|
|
623
1014
|
/**
|
|
624
1015
|
* The verifiable storage connector to use for storing sync pointers.
|
|
625
1016
|
* @internal
|
|
@@ -631,12 +1022,12 @@ class RemoteSyncStateHelper {
|
|
|
631
1022
|
*/
|
|
632
1023
|
_changeSetHelper;
|
|
633
1024
|
/**
|
|
634
|
-
* The storage ids of the batch responses for each
|
|
1025
|
+
* The storage ids of the batch responses for each storage key.
|
|
635
1026
|
* @internal
|
|
636
1027
|
*/
|
|
637
1028
|
_batchResponseStorageIds;
|
|
638
1029
|
/**
|
|
639
|
-
* The full changes for each
|
|
1030
|
+
* The full changes for each storage key.
|
|
640
1031
|
* @internal
|
|
641
1032
|
*/
|
|
642
1033
|
_populateFullChanges;
|
|
@@ -650,22 +1041,27 @@ class RemoteSyncStateHelper {
|
|
|
650
1041
|
* @internal
|
|
651
1042
|
*/
|
|
652
1043
|
_nodeIdentity;
|
|
1044
|
+
/**
|
|
1045
|
+
* Whether the node is trusted or not.
|
|
1046
|
+
* @internal
|
|
1047
|
+
*/
|
|
1048
|
+
_isTrustedNode;
|
|
653
1049
|
/**
|
|
654
1050
|
* Create a new instance of DecentralisedEntityStorageConnector.
|
|
655
1051
|
* @param logging The logging connector to use for logging.
|
|
656
1052
|
* @param eventBusComponent The event bus component to use for events.
|
|
657
|
-
* @param blobStorageComponent The blob storage component to use for remote sync states.
|
|
658
1053
|
* @param verifiableSyncPointerStorageConnector The verifiable storage connector to use for storing sync pointers.
|
|
1054
|
+
* @param blobStorageHelper The blob storage helper to use for remote sync states.
|
|
659
1055
|
* @param changeSetHelper The change set helper to use for managing changesets.
|
|
660
|
-
* @param
|
|
1056
|
+
* @param isTrustedNode Whether the node is trusted or not.
|
|
661
1057
|
*/
|
|
662
|
-
constructor(logging, eventBusComponent,
|
|
1058
|
+
constructor(logging, eventBusComponent, verifiableSyncPointerStorageConnector, blobStorageHelper, changeSetHelper, isTrustedNode) {
|
|
663
1059
|
this._logging = logging;
|
|
664
1060
|
this._eventBusComponent = eventBusComponent;
|
|
665
|
-
this._blobStorageComponent = blobStorageComponent;
|
|
666
1061
|
this._verifiableSyncPointerStorageConnector = verifiableSyncPointerStorageConnector;
|
|
667
1062
|
this._changeSetHelper = changeSetHelper;
|
|
668
|
-
this.
|
|
1063
|
+
this._blobStorageHelper = blobStorageHelper;
|
|
1064
|
+
this._isTrustedNode = isTrustedNode;
|
|
669
1065
|
this._batchResponseStorageIds = {};
|
|
670
1066
|
this._populateFullChanges = {};
|
|
671
1067
|
this._eventBusComponent.subscribe(SynchronisedStorageTopics.BatchResponse, async (response) => {
|
|
@@ -683,72 +1079,119 @@ class RemoteSyncStateHelper {
|
|
|
683
1079
|
this._nodeIdentity = nodeIdentity;
|
|
684
1080
|
}
|
|
685
1081
|
/**
|
|
686
|
-
*
|
|
687
|
-
* @param
|
|
1082
|
+
* Set the synchronised storage key.
|
|
1083
|
+
* @param synchronisedStorageKey The synchronised storage key to use.
|
|
1084
|
+
*/
|
|
1085
|
+
setSynchronisedStorageKey(synchronisedStorageKey) {
|
|
1086
|
+
this._synchronisedStorageKey = synchronisedStorageKey;
|
|
1087
|
+
}
|
|
1088
|
+
/**
|
|
1089
|
+
* Build a changeset.
|
|
1090
|
+
* @param storageKey The storage key of the change set.
|
|
688
1091
|
* @param changes The changes to apply.
|
|
689
1092
|
* @param completeCallback The callback to call when the changeset is created and stored.
|
|
690
1093
|
* @returns The storage id of the change set if created.
|
|
691
1094
|
*/
|
|
692
|
-
async
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
const setChanges = changes.filter(c => c.operation === SyncChangeOperation.Set);
|
|
701
|
-
if (setChanges.length === 0) {
|
|
702
|
-
// If we don't need to request any full details, we can just call the complete callback
|
|
703
|
-
await this.finaliseFullChanges(schemaType, completeCallback);
|
|
704
|
-
}
|
|
705
|
-
else {
|
|
706
|
-
// Otherwise we need to request the full details for each change
|
|
707
|
-
this._populateFullChanges[schemaType].requestIds = setChanges.map(change => change.id);
|
|
708
|
-
// Once all the requests are handled the callback will be called
|
|
709
|
-
for (const change of setChanges) {
|
|
710
|
-
// Create a request for each change to populate the full details
|
|
711
|
-
this._eventBusComponent.publish(SynchronisedStorageTopics.LocalItemRequest, {
|
|
712
|
-
schemaType,
|
|
713
|
-
id: change.id
|
|
714
|
-
});
|
|
715
|
-
}
|
|
1095
|
+
async buildChangeSet(storageKey, changes, completeCallback) {
|
|
1096
|
+
await this._logging?.log({
|
|
1097
|
+
level: "info",
|
|
1098
|
+
source: this.CLASS_NAME,
|
|
1099
|
+
message: "buildingChangeSet",
|
|
1100
|
+
data: {
|
|
1101
|
+
storageKey,
|
|
1102
|
+
changeCount: changes.length
|
|
716
1103
|
}
|
|
1104
|
+
});
|
|
1105
|
+
this._populateFullChanges[storageKey] = {
|
|
1106
|
+
changes,
|
|
1107
|
+
entities: {},
|
|
1108
|
+
requestIds: [],
|
|
1109
|
+
completeCallback: async () => this.finaliseFullChanges(storageKey, completeCallback)
|
|
1110
|
+
};
|
|
1111
|
+
const setChanges = changes.filter(c => c.operation === SyncChangeOperation.Set);
|
|
1112
|
+
if (setChanges.length === 0) {
|
|
1113
|
+
// If we don't need to request any full details, we can just call the complete callback
|
|
1114
|
+
await this.finaliseFullChanges(storageKey, completeCallback);
|
|
717
1115
|
}
|
|
718
1116
|
else {
|
|
719
|
-
|
|
1117
|
+
// Otherwise we need to request the full details for each change
|
|
1118
|
+
this._populateFullChanges[storageKey].requestIds = setChanges.map(change => change.id);
|
|
1119
|
+
// Once all the requests are handled the callback will be called
|
|
1120
|
+
for (const change of setChanges) {
|
|
1121
|
+
// Create a request for each change to populate the full details
|
|
1122
|
+
await this._logging?.log({
|
|
1123
|
+
level: "info",
|
|
1124
|
+
source: this.CLASS_NAME,
|
|
1125
|
+
message: "createChangeSetRequestingItem",
|
|
1126
|
+
data: {
|
|
1127
|
+
storageKey,
|
|
1128
|
+
id: change.id
|
|
1129
|
+
}
|
|
1130
|
+
});
|
|
1131
|
+
this._eventBusComponent.publish(SynchronisedStorageTopics.LocalItemRequest, {
|
|
1132
|
+
storageKey,
|
|
1133
|
+
id: change.id
|
|
1134
|
+
});
|
|
1135
|
+
}
|
|
720
1136
|
}
|
|
721
1137
|
}
|
|
722
1138
|
/**
|
|
723
1139
|
* Finalise the full details for the sync change set.
|
|
724
|
-
* @param
|
|
1140
|
+
* @param storageKey The storage key of the change set.
|
|
725
1141
|
* @param completeCallback The callback to call when the changeset is populated.
|
|
726
1142
|
* @returns Nothing.
|
|
727
1143
|
*/
|
|
728
|
-
async finaliseFullChanges(
|
|
1144
|
+
async finaliseFullChanges(storageKey, completeCallback) {
|
|
1145
|
+
await this._logging?.log({
|
|
1146
|
+
level: "info",
|
|
1147
|
+
source: this.CLASS_NAME,
|
|
1148
|
+
message: "finalisingSyncChanges",
|
|
1149
|
+
data: {
|
|
1150
|
+
storageKey
|
|
1151
|
+
}
|
|
1152
|
+
});
|
|
729
1153
|
if (Is.stringValue(this._nodeIdentity)) {
|
|
730
|
-
const changes = this._populateFullChanges[
|
|
1154
|
+
const changes = this._populateFullChanges[storageKey].changes;
|
|
731
1155
|
for (const change of changes) {
|
|
732
|
-
change.entity = this._populateFullChanges[
|
|
1156
|
+
change.entity = this._populateFullChanges[storageKey].entities[change.id] ?? change.entity;
|
|
733
1157
|
if (change.operation === SyncChangeOperation.Set && Is.objectValue(change.entity)) {
|
|
1158
|
+
// Remove the id from the entity as this is stored in the operation
|
|
1159
|
+
// and will be reinstated when the changeset is reconstituted
|
|
1160
|
+
ObjectHelper.propertyDelete(change.entity, "id");
|
|
734
1161
|
// Remove the node identity as the changeset has this stored at the top level
|
|
735
1162
|
// and we do not want to store it in the change itself to reduce redundancy
|
|
736
1163
|
ObjectHelper.propertyDelete(change.entity, "nodeIdentity");
|
|
737
1164
|
}
|
|
738
1165
|
}
|
|
739
|
-
// Add the changeset to the current snapshot
|
|
740
1166
|
const syncChangeSet = {
|
|
741
1167
|
id: Converter.bytesToHex(RandomHelper.generate(32)),
|
|
742
1168
|
dateCreated: new Date(Date.now()).toISOString(),
|
|
743
|
-
|
|
1169
|
+
storageKey,
|
|
744
1170
|
changes,
|
|
745
1171
|
nodeIdentity: this._nodeIdentity
|
|
746
1172
|
};
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
1173
|
+
try {
|
|
1174
|
+
// And sign it with the node identity
|
|
1175
|
+
syncChangeSet.proof = await this._changeSetHelper.createChangeSetProof(syncChangeSet);
|
|
1176
|
+
// If this is a trusted node, we also store the changeset
|
|
1177
|
+
let changeSetStorageId;
|
|
1178
|
+
if (this._isTrustedNode) {
|
|
1179
|
+
changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
|
|
1180
|
+
}
|
|
1181
|
+
await completeCallback(syncChangeSet, changeSetStorageId);
|
|
1182
|
+
}
|
|
1183
|
+
catch (err) {
|
|
1184
|
+
await this._logging?.log({
|
|
1185
|
+
level: "error",
|
|
1186
|
+
source: this.CLASS_NAME,
|
|
1187
|
+
message: "finalisingSyncChangesFailed",
|
|
1188
|
+
data: {
|
|
1189
|
+
storageKey
|
|
1190
|
+
},
|
|
1191
|
+
error: BaseError.fromError(err)
|
|
1192
|
+
});
|
|
1193
|
+
await completeCallback();
|
|
1194
|
+
}
|
|
752
1195
|
}
|
|
753
1196
|
else {
|
|
754
1197
|
await completeCallback();
|
|
@@ -756,122 +1199,138 @@ class RemoteSyncStateHelper {
|
|
|
756
1199
|
}
|
|
757
1200
|
/**
|
|
758
1201
|
* Add a new changeset into the sync state.
|
|
1202
|
+
* @param storageKey The storage key of the change set to add.
|
|
759
1203
|
* @param changeSetStorageId The id of the change set to add the the current state
|
|
760
1204
|
* @returns Nothing.
|
|
761
1205
|
*/
|
|
762
|
-
async addChangeSetToSyncState(changeSetStorageId) {
|
|
763
|
-
|
|
764
|
-
|
|
1206
|
+
async addChangeSetToSyncState(storageKey, changeSetStorageId) {
|
|
1207
|
+
await this._logging?.log({
|
|
1208
|
+
level: "info",
|
|
1209
|
+
source: this.CLASS_NAME,
|
|
1210
|
+
message: "addChangeSetToSyncState",
|
|
1211
|
+
data: {
|
|
1212
|
+
storageKey,
|
|
1213
|
+
changeSetStorageId
|
|
1214
|
+
}
|
|
1215
|
+
});
|
|
1216
|
+
// First load the sync pointer store to get the current sync pointer for the storage key
|
|
1217
|
+
const syncPointerStore = await this.getVerifiableSyncPointerStore();
|
|
765
1218
|
let syncState;
|
|
766
|
-
if (!Is.empty(
|
|
767
|
-
syncState = await this.getRemoteSyncState(
|
|
1219
|
+
if (!Is.empty(syncPointerStore.syncPointers[storageKey])) {
|
|
1220
|
+
syncState = await this.getRemoteSyncState(syncPointerStore.syncPointers[storageKey]);
|
|
768
1221
|
}
|
|
769
1222
|
// No current sync state, so we create a new one
|
|
770
1223
|
if (Is.empty(syncState)) {
|
|
771
|
-
syncState = { snapshots: [] };
|
|
1224
|
+
syncState = { version: SYNC_STATE_VERSION, snapshots: [] };
|
|
772
1225
|
}
|
|
773
1226
|
// Sort the snapshots so the newest snapshot is last in the array
|
|
774
1227
|
const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
|
|
775
1228
|
// Get the current snapshot, if it does not exist we create a new one
|
|
776
1229
|
let currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
|
|
1230
|
+
const now = new Date(Date.now()).toISOString();
|
|
777
1231
|
if (Is.empty(currentSnapshot)) {
|
|
778
1232
|
currentSnapshot = {
|
|
1233
|
+
version: SYNC_SNAPSHOT_VERSION,
|
|
779
1234
|
id: Converter.bytesToHex(RandomHelper.generate(32)),
|
|
780
|
-
dateCreated:
|
|
1235
|
+
dateCreated: now,
|
|
1236
|
+
dateModified: now,
|
|
781
1237
|
changeSetStorageIds: []
|
|
782
1238
|
};
|
|
783
1239
|
syncState.snapshots.push(currentSnapshot);
|
|
784
1240
|
}
|
|
785
1241
|
else {
|
|
786
1242
|
// Snapshot exists, we update the dateModified
|
|
787
|
-
currentSnapshot.dateModified =
|
|
1243
|
+
currentSnapshot.dateModified = now;
|
|
788
1244
|
}
|
|
789
1245
|
// Add the changeset storage id to the current snapshot
|
|
790
1246
|
currentSnapshot.changeSetStorageIds.push(changeSetStorageId);
|
|
791
1247
|
// Store the sync state in the blob storage
|
|
792
|
-
|
|
793
|
-
// Store the verifiable sync pointer in the verifiable storage
|
|
794
|
-
await this.
|
|
1248
|
+
syncPointerStore.syncPointers[storageKey] = await this.storeRemoteSyncState(syncState);
|
|
1249
|
+
// Store the verifiable sync pointer store in the verifiable storage
|
|
1250
|
+
await this.storeVerifiableSyncPointerStore(syncPointerStore);
|
|
795
1251
|
}
|
|
796
1252
|
/**
|
|
797
1253
|
* Create a consolidated snapshot for the entire storage.
|
|
798
|
-
* @param
|
|
1254
|
+
* @param storageKey The storage key of the snapshot to create.
|
|
799
1255
|
* @param batchSize The batch size to use for consolidation.
|
|
800
1256
|
* @returns Nothing.
|
|
801
1257
|
*/
|
|
802
|
-
async
|
|
1258
|
+
async consolidationStart(storageKey, batchSize) {
|
|
803
1259
|
await this._logging?.log({
|
|
804
1260
|
level: "info",
|
|
805
1261
|
source: this.CLASS_NAME,
|
|
806
1262
|
message: "consolidationStarting"
|
|
807
1263
|
});
|
|
808
|
-
|
|
1264
|
+
// Perform a batch request to start the consolidation
|
|
1265
|
+
await this._eventBusComponent.publish(SynchronisedStorageTopics.BatchRequest, { storageKey, batchSize });
|
|
809
1266
|
}
|
|
810
1267
|
/**
|
|
811
|
-
* Get the sync pointer.
|
|
812
|
-
* @returns The sync pointer.
|
|
1268
|
+
* Get the sync pointer store.
|
|
1269
|
+
* @returns The sync pointer store.
|
|
813
1270
|
*/
|
|
814
|
-
async
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
level: "info",
|
|
818
|
-
source: this.CLASS_NAME,
|
|
819
|
-
message: "verifiableSyncPointerRetrieving",
|
|
820
|
-
data: {
|
|
821
|
-
key: this._synchronisedStorageKey
|
|
822
|
-
}
|
|
823
|
-
});
|
|
824
|
-
const syncPointerStore = await this._verifiableSyncPointerStorageConnector.get(this._synchronisedStorageKey, { includeData: true });
|
|
825
|
-
if (Is.uint8Array(syncPointerStore.data)) {
|
|
826
|
-
const syncPointer = ObjectHelper.fromBytes(syncPointerStore.data);
|
|
1271
|
+
async getVerifiableSyncPointerStore() {
|
|
1272
|
+
if (Is.stringValue(this._synchronisedStorageKey)) {
|
|
1273
|
+
try {
|
|
827
1274
|
await this._logging?.log({
|
|
828
1275
|
level: "info",
|
|
829
1276
|
source: this.CLASS_NAME,
|
|
830
|
-
message: "
|
|
1277
|
+
message: "verifiableSyncPointerStoreRetrieving",
|
|
831
1278
|
data: {
|
|
832
|
-
key: this._synchronisedStorageKey
|
|
833
|
-
syncPointerId: syncPointer.syncPointerId
|
|
1279
|
+
key: this._synchronisedStorageKey
|
|
834
1280
|
}
|
|
835
1281
|
});
|
|
836
|
-
|
|
1282
|
+
const syncPointerStore = await this._verifiableSyncPointerStorageConnector.get(this._synchronisedStorageKey, { includeData: true });
|
|
1283
|
+
if (Is.uint8Array(syncPointerStore.data)) {
|
|
1284
|
+
const syncPointer = ObjectHelper.fromBytes(syncPointerStore.data);
|
|
1285
|
+
await this._logging?.log({
|
|
1286
|
+
level: "info",
|
|
1287
|
+
source: this.CLASS_NAME,
|
|
1288
|
+
message: "verifiableSyncPointerStoreRetrieved",
|
|
1289
|
+
data: {
|
|
1290
|
+
key: this._synchronisedStorageKey
|
|
1291
|
+
}
|
|
1292
|
+
});
|
|
1293
|
+
return syncPointer;
|
|
1294
|
+
}
|
|
837
1295
|
}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
1296
|
+
catch (err) {
|
|
1297
|
+
if (!BaseError.someErrorName(err, NotFoundError.CLASS_NAME)) {
|
|
1298
|
+
throw err;
|
|
1299
|
+
}
|
|
842
1300
|
}
|
|
1301
|
+
await this._logging?.log({
|
|
1302
|
+
level: "info",
|
|
1303
|
+
source: this.CLASS_NAME,
|
|
1304
|
+
message: "verifiableSyncPointerStoreNotFound",
|
|
1305
|
+
data: {
|
|
1306
|
+
key: this._synchronisedStorageKey
|
|
1307
|
+
}
|
|
1308
|
+
});
|
|
843
1309
|
}
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
key: this._synchronisedStorageKey
|
|
850
|
-
}
|
|
851
|
-
});
|
|
1310
|
+
// If no sync pointer store exists, we return an empty one
|
|
1311
|
+
return {
|
|
1312
|
+
version: SYNC_POINTER_STORE_VERSION,
|
|
1313
|
+
syncPointers: {}
|
|
1314
|
+
};
|
|
852
1315
|
}
|
|
853
1316
|
/**
|
|
854
1317
|
* Store the verifiable sync pointer in the verifiable storage.
|
|
855
|
-
* @param
|
|
1318
|
+
* @param syncPointerStore The sync pointer store to store.
|
|
856
1319
|
* @returns Nothing.
|
|
857
1320
|
*/
|
|
858
|
-
async
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
});
|
|
872
|
-
// Store the verifiable sync pointer in the verifiable storage
|
|
873
|
-
await this._verifiableSyncPointerStorageConnector.create(this._synchronisedStorageKey, ObjectHelper.toBytes(verifiableSyncPointer));
|
|
874
|
-
return verifiableSyncPointer;
|
|
1321
|
+
async storeVerifiableSyncPointerStore(syncPointerStore) {
|
|
1322
|
+
if (Is.stringValue(this._nodeIdentity) && Is.stringValue(this._synchronisedStorageKey)) {
|
|
1323
|
+
await this._logging?.log({
|
|
1324
|
+
level: "info",
|
|
1325
|
+
source: this.CLASS_NAME,
|
|
1326
|
+
message: "verifiableSyncPointerStoreStoring",
|
|
1327
|
+
data: {
|
|
1328
|
+
key: this._synchronisedStorageKey
|
|
1329
|
+
}
|
|
1330
|
+
});
|
|
1331
|
+
// Store the verifiable sync pointer in the verifiable storage
|
|
1332
|
+
await this._verifiableSyncPointerStorageConnector.update(this._nodeIdentity, this._synchronisedStorageKey, ObjectHelper.toBytes(syncPointerStore));
|
|
1333
|
+
}
|
|
875
1334
|
}
|
|
876
1335
|
/**
|
|
877
1336
|
* Store the remote sync state.
|
|
@@ -887,9 +1346,7 @@ class RemoteSyncStateHelper {
|
|
|
887
1346
|
snapshotCount: syncState.snapshots.length
|
|
888
1347
|
}
|
|
889
1348
|
});
|
|
890
|
-
|
|
891
|
-
// the blob storage also needs to be publicly accessible so that other nodes can retrieve it
|
|
892
|
-
return this._blobStorageComponent.create(Converter.bytesToBase64(ObjectHelper.toBytes(syncState)), undefined, undefined, undefined, { disableEncryption: true, compress: BlobStorageCompressionType.Gzip });
|
|
1349
|
+
return this._blobStorageHelper.saveBlob(syncState);
|
|
893
1350
|
}
|
|
894
1351
|
/**
|
|
895
1352
|
* Get the remote sync state.
|
|
@@ -906,11 +1363,8 @@ class RemoteSyncStateHelper {
|
|
|
906
1363
|
syncPointerId
|
|
907
1364
|
}
|
|
908
1365
|
});
|
|
909
|
-
const
|
|
910
|
-
|
|
911
|
-
});
|
|
912
|
-
if (Is.stringBase64(blobEntry.blob)) {
|
|
913
|
-
const syncState = ObjectHelper.fromBytes(Converter.base64ToBytes(blobEntry.blob));
|
|
1366
|
+
const syncState = await this._blobStorageHelper.load(syncPointerId);
|
|
1367
|
+
if (Is.object(syncState)) {
|
|
914
1368
|
await this._logging?.log({
|
|
915
1369
|
level: "info",
|
|
916
1370
|
source: this.CLASS_NAME,
|
|
@@ -923,10 +1377,16 @@ class RemoteSyncStateHelper {
|
|
|
923
1377
|
return syncState;
|
|
924
1378
|
}
|
|
925
1379
|
}
|
|
926
|
-
catch (
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
1380
|
+
catch (error) {
|
|
1381
|
+
await this._logging?.log({
|
|
1382
|
+
level: "warn",
|
|
1383
|
+
source: this.CLASS_NAME,
|
|
1384
|
+
message: "getSyncStateError",
|
|
1385
|
+
data: {
|
|
1386
|
+
syncPointerId
|
|
1387
|
+
},
|
|
1388
|
+
error: BaseError.fromError(error)
|
|
1389
|
+
});
|
|
930
1390
|
}
|
|
931
1391
|
await this._logging?.log({
|
|
932
1392
|
level: "info",
|
|
@@ -938,20 +1398,22 @@ class RemoteSyncStateHelper {
|
|
|
938
1398
|
});
|
|
939
1399
|
}
|
|
940
1400
|
/**
|
|
941
|
-
* Handle the batch response.
|
|
1401
|
+
* Handle the batch response which is triggered from a consolidation request.
|
|
942
1402
|
* @param response The batch response to handle.
|
|
943
1403
|
*/
|
|
944
1404
|
async handleBatchResponse(response) {
|
|
945
1405
|
if (Is.stringValue(this._nodeIdentity)) {
|
|
1406
|
+
const now = new Date(Date.now()).toISOString();
|
|
946
1407
|
// Create a new snapshot entry for the current batch
|
|
947
1408
|
const syncChangeSet = {
|
|
948
1409
|
id: Converter.bytesToHex(RandomHelper.generate(32)),
|
|
949
|
-
dateCreated:
|
|
1410
|
+
dateCreated: now,
|
|
1411
|
+
dateModified: now,
|
|
950
1412
|
changes: response.entities.map(change => ({
|
|
951
1413
|
operation: SyncChangeOperation.Set,
|
|
952
|
-
id: change
|
|
1414
|
+
id: change.id
|
|
953
1415
|
})),
|
|
954
|
-
|
|
1416
|
+
storageKey: response.storageKey,
|
|
955
1417
|
nodeIdentity: this._nodeIdentity
|
|
956
1418
|
};
|
|
957
1419
|
// And sign it with the node identity
|
|
@@ -959,20 +1421,35 @@ class RemoteSyncStateHelper {
|
|
|
959
1421
|
// Store the changeset in the blob storage
|
|
960
1422
|
const changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
|
|
961
1423
|
// Add the changeset storage id to the snapshot ids
|
|
962
|
-
this._batchResponseStorageIds[response.
|
|
963
|
-
this._batchResponseStorageIds[response.
|
|
1424
|
+
this._batchResponseStorageIds[response.storageKey] ??= [];
|
|
1425
|
+
this._batchResponseStorageIds[response.storageKey].push(changeSetStorageId);
|
|
1426
|
+
// If this is the last entry in the batch response, we can create the consolidated snapshot
|
|
964
1427
|
if (response.lastEntry) {
|
|
965
|
-
|
|
1428
|
+
// Get the current sync pointer store
|
|
1429
|
+
const syncPointerStore = await this.getVerifiableSyncPointerStore();
|
|
1430
|
+
let syncState;
|
|
1431
|
+
if (Is.stringValue(syncPointerStore.syncPointers[response.storageKey])) {
|
|
1432
|
+
// If the sync pointer exists, we load the current sync state
|
|
1433
|
+
syncState = await this.getRemoteSyncState(syncPointerStore.syncPointers[response.storageKey]);
|
|
1434
|
+
}
|
|
1435
|
+
// If the sync state does not exist, we create a new one
|
|
1436
|
+
syncState ??= { version: SYNC_STATE_VERSION, snapshots: [] };
|
|
966
1437
|
const batchSnapshot = {
|
|
1438
|
+
version: SYNC_SNAPSHOT_VERSION,
|
|
967
1439
|
id: Converter.bytesToHex(RandomHelper.generate(32)),
|
|
968
|
-
dateCreated:
|
|
969
|
-
|
|
1440
|
+
dateCreated: now,
|
|
1441
|
+
dateModified: now,
|
|
1442
|
+
changeSetStorageIds: this._batchResponseStorageIds[response.storageKey]
|
|
970
1443
|
};
|
|
971
1444
|
syncState.snapshots.push(batchSnapshot);
|
|
972
1445
|
// Store the sync state in the blob storage
|
|
973
1446
|
const syncStateId = await this.storeRemoteSyncState(syncState);
|
|
1447
|
+
syncPointerStore.syncPointers[response.storageKey] = syncStateId;
|
|
974
1448
|
// Store the verifiable sync pointer in the verifiable storage
|
|
975
|
-
await this.
|
|
1449
|
+
await this.storeVerifiableSyncPointerStore(syncPointerStore);
|
|
1450
|
+
// Remove the batch response storage ids for the storage key
|
|
1451
|
+
// as we have consolidated the changes
|
|
1452
|
+
delete this._batchResponseStorageIds[response.storageKey];
|
|
976
1453
|
await this._logging?.log({
|
|
977
1454
|
level: "info",
|
|
978
1455
|
source: this.CLASS_NAME,
|
|
@@ -986,13 +1463,22 @@ class RemoteSyncStateHelper {
|
|
|
986
1463
|
* @param response The item response to handle.
|
|
987
1464
|
*/
|
|
988
1465
|
async handleLocalItemResponse(response) {
|
|
989
|
-
|
|
990
|
-
|
|
1466
|
+
await this._logging?.log({
|
|
1467
|
+
level: "info",
|
|
1468
|
+
source: this.CLASS_NAME,
|
|
1469
|
+
message: "createChangeSetRespondingItem",
|
|
1470
|
+
data: {
|
|
1471
|
+
storageKey: response.storageKey,
|
|
1472
|
+
id: response.id
|
|
1473
|
+
}
|
|
1474
|
+
});
|
|
1475
|
+
if (!Is.empty(this._populateFullChanges[response.storageKey])) {
|
|
1476
|
+
const idx = this._populateFullChanges[response.storageKey].requestIds.indexOf(response.id);
|
|
991
1477
|
if (idx !== -1) {
|
|
992
|
-
this._populateFullChanges[response.
|
|
993
|
-
this._populateFullChanges[response.
|
|
994
|
-
if (this._populateFullChanges[response.
|
|
995
|
-
await this._populateFullChanges[response.
|
|
1478
|
+
this._populateFullChanges[response.storageKey].requestIds.splice(idx, 1);
|
|
1479
|
+
this._populateFullChanges[response.storageKey].entities[response.id] = response.entity;
|
|
1480
|
+
if (this._populateFullChanges[response.storageKey].requestIds.length === 0) {
|
|
1481
|
+
await this._populateFullChanges[response.storageKey].completeCallback();
|
|
996
1482
|
}
|
|
997
1483
|
}
|
|
998
1484
|
}
|
|
@@ -1004,20 +1490,20 @@ class RemoteSyncStateHelper {
|
|
|
1004
1490
|
*/
|
|
1005
1491
|
class SynchronisedStorageService {
|
|
1006
1492
|
/**
|
|
1007
|
-
* The default interval to check for entity updates
|
|
1493
|
+
* The default interval to check for entity updates.
|
|
1008
1494
|
* @internal
|
|
1009
1495
|
*/
|
|
1010
|
-
static
|
|
1496
|
+
static _DEFAULT_ENTITY_UPDATE_INTERVAL_MINUTES = 5;
|
|
1011
1497
|
/**
|
|
1012
|
-
* The default interval to perform consolidation
|
|
1498
|
+
* The default interval to perform consolidation.
|
|
1013
1499
|
* @internal
|
|
1014
1500
|
*/
|
|
1015
|
-
static
|
|
1501
|
+
static _DEFAULT_CONSOLIDATION_INTERVAL_MINUTES = 60;
|
|
1016
1502
|
/**
|
|
1017
1503
|
* The default size of a consolidation batch.
|
|
1018
1504
|
* @internal
|
|
1019
1505
|
*/
|
|
1020
|
-
static _DEFAULT_CONSOLIDATION_BATCH_SIZE =
|
|
1506
|
+
static _DEFAULT_CONSOLIDATION_BATCH_SIZE = 100;
|
|
1021
1507
|
/**
|
|
1022
1508
|
* Runtime name for the class.
|
|
1023
1509
|
*/
|
|
@@ -1032,16 +1518,21 @@ class SynchronisedStorageService {
|
|
|
1032
1518
|
* @internal
|
|
1033
1519
|
*/
|
|
1034
1520
|
_eventBusComponent;
|
|
1521
|
+
/**
|
|
1522
|
+
* The vault connector.
|
|
1523
|
+
* @internal
|
|
1524
|
+
*/
|
|
1525
|
+
_vaultConnector;
|
|
1035
1526
|
/**
|
|
1036
1527
|
* The storage connector for the sync snapshot entries.
|
|
1037
1528
|
* @internal
|
|
1038
1529
|
*/
|
|
1039
1530
|
_localSyncSnapshotEntryEntityStorage;
|
|
1040
1531
|
/**
|
|
1041
|
-
* The blob storage
|
|
1532
|
+
* The blob storage connector to use for remote sync states.
|
|
1042
1533
|
* @internal
|
|
1043
1534
|
*/
|
|
1044
|
-
|
|
1535
|
+
_blobStorageConnector;
|
|
1045
1536
|
/**
|
|
1046
1537
|
* The verifiable storage connector to use for storing sync pointers.
|
|
1047
1538
|
* @internal
|
|
@@ -1052,11 +1543,21 @@ class SynchronisedStorageService {
|
|
|
1052
1543
|
* @internal
|
|
1053
1544
|
*/
|
|
1054
1545
|
_identityConnector;
|
|
1546
|
+
/**
|
|
1547
|
+
* The task scheduler component.
|
|
1548
|
+
* @internal
|
|
1549
|
+
*/
|
|
1550
|
+
_taskSchedulerComponent;
|
|
1055
1551
|
/**
|
|
1056
1552
|
* The synchronised storage service to use when this is not a trusted node.
|
|
1057
1553
|
* @internal
|
|
1058
1554
|
*/
|
|
1059
1555
|
_trustedSynchronisedStorageComponent;
|
|
1556
|
+
/**
|
|
1557
|
+
* The blob storage helper.
|
|
1558
|
+
* @internal
|
|
1559
|
+
*/
|
|
1560
|
+
_blobStorageHelper;
|
|
1060
1561
|
/**
|
|
1061
1562
|
* The change set helper.
|
|
1062
1563
|
* @internal
|
|
@@ -1078,15 +1579,20 @@ class SynchronisedStorageService {
|
|
|
1078
1579
|
*/
|
|
1079
1580
|
_config;
|
|
1080
1581
|
/**
|
|
1081
|
-
* The
|
|
1582
|
+
* The synchronised storage key to use for the remote synchronised storage.
|
|
1082
1583
|
* @internal
|
|
1083
1584
|
*/
|
|
1084
|
-
|
|
1585
|
+
_synchronisedStorageKey;
|
|
1085
1586
|
/**
|
|
1086
|
-
* The
|
|
1587
|
+
* The flag to determine if the service has been started.
|
|
1087
1588
|
* @internal
|
|
1088
1589
|
*/
|
|
1089
|
-
|
|
1590
|
+
_serviceStarted;
|
|
1591
|
+
/**
|
|
1592
|
+
* The active storage keys for the synchronised storage service.
|
|
1593
|
+
* @internal
|
|
1594
|
+
*/
|
|
1595
|
+
_activeStorageKeys;
|
|
1090
1596
|
/**
|
|
1091
1597
|
* The identity of the node this connector is running on.
|
|
1092
1598
|
* @internal
|
|
@@ -1101,21 +1607,26 @@ class SynchronisedStorageService {
|
|
|
1101
1607
|
Guards.object(this.CLASS_NAME, "options.config", options.config);
|
|
1102
1608
|
this._eventBusComponent = ComponentFactory.get(options.eventBusComponentType ?? "event-bus");
|
|
1103
1609
|
this._logging = LoggingConnectorFactory.getIfExists(options.loggingConnectorType ?? "logging");
|
|
1610
|
+
this._vaultConnector = VaultConnectorFactory.get(options.vaultConnectorType ?? "vault");
|
|
1104
1611
|
this._localSyncSnapshotEntryEntityStorage = EntityStorageConnectorFactory.get(options.syncSnapshotStorageConnectorType ?? "sync-snapshot-entry");
|
|
1105
1612
|
this._verifiableSyncPointerStorageConnector = VerifiableStorageConnectorFactory.get(options.verifiableStorageConnectorType ?? "verifiable-storage");
|
|
1106
|
-
this.
|
|
1613
|
+
this._blobStorageConnector = BlobStorageConnectorFactory.get(options.blobStorageConnectorType ?? "blob-storage");
|
|
1107
1614
|
this._identityConnector = IdentityConnectorFactory.get(options.identityConnectorType ?? "identity");
|
|
1615
|
+
this._taskSchedulerComponent = ComponentFactory.get(options.taskSchedulerComponentType ?? "task-scheduler");
|
|
1108
1616
|
this._config = {
|
|
1109
|
-
synchronisedStorageKey: options.config.synchronisedStorageKey,
|
|
1110
1617
|
synchronisedStorageMethodId: options.config.synchronisedStorageMethodId ?? "synchronised-storage-assertion",
|
|
1111
|
-
|
|
1112
|
-
SynchronisedStorageService.
|
|
1618
|
+
entityUpdateIntervalMinutes: options.config.entityUpdateIntervalMinutes ??
|
|
1619
|
+
SynchronisedStorageService._DEFAULT_ENTITY_UPDATE_INTERVAL_MINUTES,
|
|
1113
1620
|
isTrustedNode: options.config.isTrustedNode ?? false,
|
|
1114
|
-
|
|
1115
|
-
SynchronisedStorageService.
|
|
1621
|
+
consolidationIntervalMinutes: options.config.consolidationIntervalMinutes ??
|
|
1622
|
+
SynchronisedStorageService._DEFAULT_CONSOLIDATION_INTERVAL_MINUTES,
|
|
1116
1623
|
consolidationBatchSize: options.config.consolidationBatchSize ??
|
|
1117
|
-
SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE
|
|
1624
|
+
SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE,
|
|
1625
|
+
blobStorageEncryptionKeyId: options.config.blobStorageEncryptionKeyId ?? "synchronised-storage-blob-encryption-key",
|
|
1626
|
+
verifiableStorageKeyId: options.config.verifiableStorageKeyId
|
|
1118
1627
|
};
|
|
1628
|
+
this._synchronisedStorageKey =
|
|
1629
|
+
verifiableStorageKeys[options.config.verifiableStorageKeyId] ?? options.config.verifiableStorageKeyId;
|
|
1119
1630
|
// If this is not a trusted node, we need to use a synchronised storage service
|
|
1120
1631
|
// to synchronise with a trusted node.
|
|
1121
1632
|
if (!this._config.isTrustedNode) {
|
|
@@ -1123,13 +1634,14 @@ class SynchronisedStorageService {
|
|
|
1123
1634
|
this._trustedSynchronisedStorageComponent =
|
|
1124
1635
|
ComponentFactory.get(options.trustedSynchronisedStorageComponentType);
|
|
1125
1636
|
}
|
|
1126
|
-
this.
|
|
1637
|
+
this._blobStorageHelper = new BlobStorageHelper(this._logging, this._vaultConnector, this._blobStorageConnector, this._config.blobStorageEncryptionKeyId, this._config.isTrustedNode);
|
|
1638
|
+
this._changeSetHelper = new ChangeSetHelper(this._logging, this._eventBusComponent, this._identityConnector, this._blobStorageHelper, this._config.synchronisedStorageMethodId);
|
|
1127
1639
|
this._localSyncStateHelper = new LocalSyncStateHelper(this._logging, this._localSyncSnapshotEntryEntityStorage, this._changeSetHelper);
|
|
1128
|
-
this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._logging, this._eventBusComponent, this.
|
|
1129
|
-
this.
|
|
1130
|
-
this.
|
|
1131
|
-
this._eventBusComponent.subscribe(SynchronisedStorageTopics.
|
|
1132
|
-
this._eventBusComponent.subscribe(SynchronisedStorageTopics.LocalItemChange, async (event) => this._localSyncStateHelper.addLocalChange(event.data.
|
|
1640
|
+
this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._logging, this._eventBusComponent, this._verifiableSyncPointerStorageConnector, this._blobStorageHelper, this._changeSetHelper, this._config.isTrustedNode);
|
|
1641
|
+
this._serviceStarted = false;
|
|
1642
|
+
this._activeStorageKeys = {};
|
|
1643
|
+
this._eventBusComponent.subscribe(SynchronisedStorageTopics.RegisterStorageKey, async (event) => this.registerStorageKey(event.data));
|
|
1644
|
+
this._eventBusComponent.subscribe(SynchronisedStorageTopics.LocalItemChange, async (event) => this._localSyncStateHelper.addLocalChange(event.data.storageKey, event.data.operation, event.data.id));
|
|
1133
1645
|
}
|
|
1134
1646
|
/**
|
|
1135
1647
|
* The component needs to be started when the node is initialized.
|
|
@@ -1141,6 +1653,20 @@ class SynchronisedStorageService {
|
|
|
1141
1653
|
async start(nodeIdentity, nodeLoggingConnectorType, componentState) {
|
|
1142
1654
|
this._nodeIdentity = nodeIdentity;
|
|
1143
1655
|
this._remoteSyncStateHelper.setNodeIdentity(nodeIdentity);
|
|
1656
|
+
this._changeSetHelper.setNodeIdentity(nodeIdentity);
|
|
1657
|
+
this._remoteSyncStateHelper.setSynchronisedStorageKey(this._synchronisedStorageKey);
|
|
1658
|
+
this._serviceStarted = true;
|
|
1659
|
+
// If this is not a trusted node we need to request the decryption key from a trusted node
|
|
1660
|
+
if (!this._config.isTrustedNode && !Is.empty(this._trustedSynchronisedStorageComponent)) {
|
|
1661
|
+
const proof = await this._identityConnector.createProof(this._nodeIdentity, DocumentHelper.joinId(this._nodeIdentity, this._config.synchronisedStorageMethodId), ProofTypes.DataIntegrityProof, { nodeIdentity });
|
|
1662
|
+
const decryptionKey = await this._trustedSynchronisedStorageComponent.getDecryptionKey(this._nodeIdentity, proof);
|
|
1663
|
+
// We don't have the private key so instead we store the key as a secret in the vault
|
|
1664
|
+
await this._vaultConnector.setSecret(this._config.blobStorageEncryptionKeyId, decryptionKey);
|
|
1665
|
+
}
|
|
1666
|
+
// If there are already storage keys registered, we need to activate them
|
|
1667
|
+
for (const storageKey in this._activeStorageKeys) {
|
|
1668
|
+
await this.activateStorageKey(storageKey);
|
|
1669
|
+
}
|
|
1144
1670
|
}
|
|
1145
1671
|
/**
|
|
1146
1672
|
* The component needs to be stopped when the node is closed.
|
|
@@ -1150,48 +1676,88 @@ class SynchronisedStorageService {
|
|
|
1150
1676
|
* @returns Nothing.
|
|
1151
1677
|
*/
|
|
1152
1678
|
async stop(nodeIdentity, nodeLoggingConnectorType, componentState) {
|
|
1153
|
-
for (const
|
|
1154
|
-
|
|
1155
|
-
|
|
1679
|
+
for (const storageKey in this._activeStorageKeys) {
|
|
1680
|
+
this._activeStorageKeys[storageKey] = false;
|
|
1681
|
+
this._taskSchedulerComponent.removeTask(`synchronised-storage-update-${storageKey}`);
|
|
1682
|
+
this._taskSchedulerComponent.removeTask(`synchronised-storage-consolidation-${storageKey}`);
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
/**
|
|
1686
|
+
* Get the decryption key for the synchronised storage.
|
|
1687
|
+
* This is used to decrypt the data stored in the synchronised storage.
|
|
1688
|
+
* @param nodeIdentity The identity of the node requesting the decryption key.
|
|
1689
|
+
* @param proof The proof of the request so we know the request is from the specified node.
|
|
1690
|
+
* @returns The decryption key.
|
|
1691
|
+
*/
|
|
1692
|
+
async getDecryptionKey(nodeIdentity, proof) {
|
|
1693
|
+
if (!this._config.isTrustedNode) {
|
|
1694
|
+
throw new GeneralError(this.CLASS_NAME, "notTrustedNode");
|
|
1156
1695
|
}
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1696
|
+
Guards.stringValue(this.CLASS_NAME, "nodeIdentity", nodeIdentity);
|
|
1697
|
+
Guards.object(this.CLASS_NAME, "proof", proof);
|
|
1698
|
+
const isValid = await this._identityConnector.verifyProof({ nodeIdentity }, proof);
|
|
1699
|
+
if (!isValid) {
|
|
1700
|
+
throw new UnauthorizedError(this.CLASS_NAME, "invalidProof");
|
|
1160
1701
|
}
|
|
1702
|
+
// TODO: We need to check if the node has permissions to access the decryption key
|
|
1703
|
+
// using rights-management
|
|
1704
|
+
const key = await this._vaultConnector.getKey(this._config.blobStorageEncryptionKeyId);
|
|
1705
|
+
if (Is.undefined(key.publicKey)) {
|
|
1706
|
+
throw new UnauthorizedError(this.CLASS_NAME, "decryptionKeyNotFound");
|
|
1707
|
+
}
|
|
1708
|
+
return Converter.bytesToBase64(key.publicKey);
|
|
1161
1709
|
}
|
|
1162
1710
|
/**
|
|
1163
|
-
* Synchronise a
|
|
1164
|
-
* @param
|
|
1711
|
+
* Synchronise a set of changes from an untrusted node, assumes this is a trusted node.
|
|
1712
|
+
* @param syncChangeSet The change set to synchronise.
|
|
1165
1713
|
* @returns Nothing.
|
|
1166
1714
|
*/
|
|
1167
|
-
async syncChangeSet(
|
|
1715
|
+
async syncChangeSet(syncChangeSet) {
|
|
1168
1716
|
if (!this._config.isTrustedNode) {
|
|
1169
1717
|
throw new GeneralError(this.CLASS_NAME, "notTrustedNode");
|
|
1170
1718
|
}
|
|
1171
|
-
|
|
1172
|
-
|
|
1719
|
+
Guards.object(this.CLASS_NAME, "syncChangeSet", syncChangeSet);
|
|
1720
|
+
await this._logging?.log({
|
|
1721
|
+
level: "info",
|
|
1722
|
+
source: this.CLASS_NAME,
|
|
1723
|
+
message: "syncChangeSetForRemoteNode",
|
|
1724
|
+
data: {
|
|
1725
|
+
changeSetStorageId: syncChangeSet.id
|
|
1726
|
+
}
|
|
1727
|
+
});
|
|
1173
1728
|
// TODO: The change set has a proof signed by the originating node identity
|
|
1174
1729
|
// The proof is verified that the change set is valid and has not been tampered with.
|
|
1175
1730
|
// but we also need to check that the originating node has permissions
|
|
1176
1731
|
// to store the change set in the synchronised storage.
|
|
1177
1732
|
// This will be performed using rights-management
|
|
1178
|
-
const
|
|
1179
|
-
if (!Is.empty(
|
|
1180
|
-
|
|
1733
|
+
const copy = await this._changeSetHelper.copyChangeset(syncChangeSet);
|
|
1734
|
+
if (!Is.empty(copy) && Is.stringValue(this._nodeIdentity)) {
|
|
1735
|
+
// Apply the changes to this node
|
|
1736
|
+
await this._changeSetHelper.applyChangeset(copy.syncChangeSet);
|
|
1737
|
+
// And update the sync state with the latest changes
|
|
1738
|
+
await this._remoteSyncStateHelper.addChangeSetToSyncState(copy.syncChangeSet.storageKey, copy.changeSetStorageId);
|
|
1181
1739
|
}
|
|
1182
1740
|
}
|
|
1183
1741
|
/**
|
|
1184
1742
|
* Start the sync with further updates after an interval.
|
|
1185
|
-
* @param
|
|
1743
|
+
* @param storageKey The storage key to sync.
|
|
1186
1744
|
* @returns Nothing.
|
|
1187
1745
|
* @internal
|
|
1188
1746
|
*/
|
|
1189
|
-
async startEntitySync(
|
|
1747
|
+
async startEntitySync(storageKey) {
|
|
1190
1748
|
try {
|
|
1749
|
+
await this._logging?.log({
|
|
1750
|
+
level: "info",
|
|
1751
|
+
source: this.CLASS_NAME,
|
|
1752
|
+
message: "startEntitySync",
|
|
1753
|
+
data: {
|
|
1754
|
+
storageKey
|
|
1755
|
+
}
|
|
1756
|
+
});
|
|
1191
1757
|
// First we check for remote changes
|
|
1192
|
-
await this.updateFromRemoteSyncState(
|
|
1758
|
+
await this.updateFromRemoteSyncState(storageKey);
|
|
1193
1759
|
// Now send any updates we have to the remote storage
|
|
1194
|
-
await this.updateFromLocalSyncState(
|
|
1760
|
+
await this.updateFromLocalSyncState(storageKey);
|
|
1195
1761
|
}
|
|
1196
1762
|
catch (error) {
|
|
1197
1763
|
await this._logging?.log({
|
|
@@ -1201,27 +1767,31 @@ class SynchronisedStorageService {
|
|
|
1201
1767
|
error: BaseError.fromError(error)
|
|
1202
1768
|
});
|
|
1203
1769
|
}
|
|
1204
|
-
finally {
|
|
1205
|
-
// Set a timer to check for updates again
|
|
1206
|
-
this._entityUpdateTimers[schemaType] = setTimeout(async () => this.startEntitySync(schemaType), this._config.entityUpdateIntervalMs);
|
|
1207
|
-
}
|
|
1208
1770
|
}
|
|
1209
1771
|
/**
|
|
1210
1772
|
* Check for updates in the remote storage.
|
|
1211
|
-
* @param
|
|
1773
|
+
* @param storageKey The storage key to check for updates.
|
|
1212
1774
|
* @returns Nothing.
|
|
1213
1775
|
* @internal
|
|
1214
1776
|
*/
|
|
1215
|
-
async updateFromRemoteSyncState(
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1777
|
+
async updateFromRemoteSyncState(storageKey) {
|
|
1778
|
+
await this._logging?.log({
|
|
1779
|
+
level: "info",
|
|
1780
|
+
source: this.CLASS_NAME,
|
|
1781
|
+
message: "updateFromRemoteSyncState",
|
|
1782
|
+
data: {
|
|
1783
|
+
storageKey
|
|
1784
|
+
}
|
|
1785
|
+
});
|
|
1786
|
+
// Get the verifiable sync pointer store from the verifiable storage
|
|
1787
|
+
const verifiableSyncPointerStore = await this._remoteSyncStateHelper.getVerifiableSyncPointerStore();
|
|
1788
|
+
if (!Is.empty(verifiableSyncPointerStore.syncPointers[storageKey])) {
|
|
1219
1789
|
// Load the sync state from the remote blob storage using the sync pointer
|
|
1220
1790
|
// to load the sync state
|
|
1221
|
-
const remoteSyncState = await this._remoteSyncStateHelper.getRemoteSyncState(
|
|
1791
|
+
const remoteSyncState = await this._remoteSyncStateHelper.getRemoteSyncState(verifiableSyncPointerStore.syncPointers[storageKey]);
|
|
1222
1792
|
// If we got the sync state we can try and sync from it
|
|
1223
1793
|
if (!Is.undefined(remoteSyncState)) {
|
|
1224
|
-
await this._localSyncStateHelper.
|
|
1794
|
+
await this._localSyncStateHelper.applySyncState(storageKey, remoteSyncState);
|
|
1225
1795
|
}
|
|
1226
1796
|
}
|
|
1227
1797
|
}
|
|
@@ -1230,47 +1800,94 @@ class SynchronisedStorageService {
|
|
|
1230
1800
|
* @returns Nothing.
|
|
1231
1801
|
* @internal
|
|
1232
1802
|
*/
|
|
1233
|
-
async updateFromLocalSyncState(
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1803
|
+
async updateFromLocalSyncState(storageKey) {
|
|
1804
|
+
await this._logging?.log({
|
|
1805
|
+
level: "info",
|
|
1806
|
+
source: this.CLASS_NAME,
|
|
1807
|
+
message: "updateFromLocalSyncState",
|
|
1808
|
+
data: {
|
|
1809
|
+
storageKey
|
|
1810
|
+
}
|
|
1811
|
+
});
|
|
1812
|
+
const localChangeSnapshot = await this._localSyncStateHelper.getLocalChangeSnapshot(storageKey);
|
|
1813
|
+
if (Is.arrayValue(localChangeSnapshot.changes)) {
|
|
1814
|
+
await this._remoteSyncStateHelper.buildChangeSet(storageKey, localChangeSnapshot.changes, async (syncChangeSet, changeSetStorageId) => {
|
|
1815
|
+
if (Is.empty(syncChangeSet) && Is.empty(changeSetStorageId)) {
|
|
1816
|
+
await this._logging?.log({
|
|
1817
|
+
level: "info",
|
|
1818
|
+
source: this.CLASS_NAME,
|
|
1819
|
+
message: "builtStorageChangeSetNone",
|
|
1820
|
+
data: {
|
|
1821
|
+
storageKey
|
|
1243
1822
|
}
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1823
|
+
});
|
|
1824
|
+
}
|
|
1825
|
+
else {
|
|
1826
|
+
await this._logging?.log({
|
|
1827
|
+
level: "info",
|
|
1828
|
+
source: this.CLASS_NAME,
|
|
1829
|
+
message: "builtStorageChangeSet",
|
|
1830
|
+
data: {
|
|
1831
|
+
storageKey,
|
|
1832
|
+
changeSetStorageId
|
|
1247
1833
|
}
|
|
1834
|
+
});
|
|
1835
|
+
// Send the local changes to the remote storage if we are a trusted node
|
|
1836
|
+
if (this._config.isTrustedNode && Is.stringValue(changeSetStorageId)) {
|
|
1837
|
+
// If we are a trusted node, we can add the change set to the sync state
|
|
1838
|
+
// and remove the local change snapshot
|
|
1839
|
+
await this._remoteSyncStateHelper.addChangeSetToSyncState(storageKey, changeSetStorageId);
|
|
1248
1840
|
await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
|
|
1249
1841
|
}
|
|
1250
|
-
|
|
1251
|
-
|
|
1842
|
+
else if (!Is.empty(this._trustedSynchronisedStorageComponent) &&
|
|
1843
|
+
Is.object(syncChangeSet)) {
|
|
1844
|
+
// If we are not a trusted node, we need to send the changes to the trusted node
|
|
1845
|
+
// and then remove the local change snapshot
|
|
1846
|
+
await this._logging?.log({
|
|
1847
|
+
level: "info",
|
|
1848
|
+
source: this.CLASS_NAME,
|
|
1849
|
+
message: "sendingChangeSetToTrustedNode",
|
|
1850
|
+
data: {
|
|
1851
|
+
storageKey,
|
|
1852
|
+
changeSetStorageId
|
|
1853
|
+
}
|
|
1854
|
+
});
|
|
1855
|
+
await this._trustedSynchronisedStorageComponent.syncChangeSet(syncChangeSet);
|
|
1856
|
+
await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
|
|
1857
|
+
}
|
|
1858
|
+
}
|
|
1859
|
+
});
|
|
1860
|
+
}
|
|
1861
|
+
else {
|
|
1862
|
+
await this._logging?.log({
|
|
1863
|
+
level: "info",
|
|
1864
|
+
source: this.CLASS_NAME,
|
|
1865
|
+
message: "updateFromLocalSyncStateNoChanges",
|
|
1866
|
+
data: {
|
|
1867
|
+
storageKey
|
|
1868
|
+
}
|
|
1869
|
+
});
|
|
1252
1870
|
}
|
|
1253
1871
|
}
|
|
1254
1872
|
/**
|
|
1255
1873
|
* Start the consolidation sync.
|
|
1256
|
-
* @param
|
|
1874
|
+
* @param storageKey The storage key to consolidate.
|
|
1257
1875
|
* @returns Nothing.
|
|
1258
1876
|
* @internal
|
|
1259
1877
|
*/
|
|
1260
|
-
async startConsolidationSync(
|
|
1878
|
+
async startConsolidationSync(storageKey) {
|
|
1261
1879
|
let localChangeSnapshot;
|
|
1262
1880
|
try {
|
|
1263
|
-
// If we are performing a consolidation, we can remove the local
|
|
1264
|
-
|
|
1881
|
+
// If we are performing a consolidation, we can remove the local change snapshot
|
|
1882
|
+
// as we are going to create a complete changeset from the DB
|
|
1883
|
+
localChangeSnapshot = await this._localSyncStateHelper.getLocalChangeSnapshot(storageKey);
|
|
1265
1884
|
if (!Is.empty(localChangeSnapshot)) {
|
|
1266
1885
|
await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
|
|
1267
1886
|
}
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
localChangeSnapshot = undefined;
|
|
1273
|
-
}
|
|
1887
|
+
await this._remoteSyncStateHelper.consolidationStart(storageKey, this._config.consolidationBatchSize ??
|
|
1888
|
+
SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE);
|
|
1889
|
+
// The consolidation was successful, so we can remove the local change snapshot permanently
|
|
1890
|
+
localChangeSnapshot = undefined;
|
|
1274
1891
|
}
|
|
1275
1892
|
catch (error) {
|
|
1276
1893
|
if (localChangeSnapshot) {
|
|
@@ -1284,32 +1901,62 @@ class SynchronisedStorageService {
|
|
|
1284
1901
|
error: BaseError.fromError(error)
|
|
1285
1902
|
});
|
|
1286
1903
|
}
|
|
1287
|
-
finally {
|
|
1288
|
-
// Set a timer to perform the consolidation again
|
|
1289
|
-
this._consolidationTimers[schemaType] = setTimeout(async () => this.startConsolidationSync(schemaType), this._config.consolidationIntervalMs);
|
|
1290
|
-
}
|
|
1291
1904
|
}
|
|
1292
1905
|
/**
|
|
1293
1906
|
* Register a new sync type.
|
|
1294
|
-
* @param
|
|
1907
|
+
* @param syncRegisterStorageKey The sync register type to register.
|
|
1295
1908
|
* @internal
|
|
1296
1909
|
*/
|
|
1297
|
-
async
|
|
1910
|
+
async registerStorageKey(syncRegisterStorageKey) {
|
|
1298
1911
|
await this._logging?.log({
|
|
1299
1912
|
level: "info",
|
|
1300
1913
|
source: this.CLASS_NAME,
|
|
1301
|
-
message: "
|
|
1914
|
+
message: "registerStorageKey",
|
|
1302
1915
|
data: {
|
|
1303
|
-
|
|
1916
|
+
storageKey: syncRegisterStorageKey.storageKey
|
|
1304
1917
|
}
|
|
1305
1918
|
});
|
|
1306
|
-
if (this.
|
|
1307
|
-
|
|
1919
|
+
if (Is.empty(this._activeStorageKeys[syncRegisterStorageKey.storageKey])) {
|
|
1920
|
+
this._activeStorageKeys[syncRegisterStorageKey.storageKey] = false;
|
|
1921
|
+
if (this._serviceStarted) {
|
|
1922
|
+
await this.activateStorageKey(syncRegisterStorageKey.storageKey);
|
|
1923
|
+
}
|
|
1308
1924
|
}
|
|
1309
|
-
|
|
1310
|
-
|
|
1925
|
+
}
|
|
1926
|
+
/**
|
|
1927
|
+
* Activate a storage key.
|
|
1928
|
+
* @param storageKey The storage key to activate.
|
|
1929
|
+
* @internal
|
|
1930
|
+
*/
|
|
1931
|
+
async activateStorageKey(storageKey) {
|
|
1932
|
+
if (!Is.empty(this._activeStorageKeys[storageKey]) && !this._activeStorageKeys[storageKey]) {
|
|
1933
|
+
await this._logging?.log({
|
|
1934
|
+
level: "info",
|
|
1935
|
+
source: this.CLASS_NAME,
|
|
1936
|
+
message: "activateStorageKey",
|
|
1937
|
+
data: {
|
|
1938
|
+
storageKey
|
|
1939
|
+
}
|
|
1940
|
+
});
|
|
1941
|
+
this._activeStorageKeys[storageKey] = true;
|
|
1942
|
+
if (this._config.entityUpdateIntervalMinutes > 0) {
|
|
1943
|
+
await this._taskSchedulerComponent.addTask(`synchronised-storage-update-${storageKey}`, [
|
|
1944
|
+
{
|
|
1945
|
+
nextTriggerTime: Date.now(),
|
|
1946
|
+
intervalMinutes: this._config.entityUpdateIntervalMinutes
|
|
1947
|
+
}
|
|
1948
|
+
], async () => this.startEntitySync(storageKey));
|
|
1949
|
+
}
|
|
1950
|
+
if (this._config.isTrustedNode && this._config.consolidationIntervalMinutes > 0) {
|
|
1951
|
+
await this._taskSchedulerComponent.addTask(`synchronised-storage-consolidation-${storageKey}`, [
|
|
1952
|
+
{
|
|
1953
|
+
nextTriggerTime: Date.now(),
|
|
1954
|
+
intervalMinutes: this._config.consolidationIntervalMinutes
|
|
1955
|
+
}
|
|
1956
|
+
], async () => this.startConsolidationSync(storageKey));
|
|
1957
|
+
}
|
|
1311
1958
|
}
|
|
1312
1959
|
}
|
|
1313
1960
|
}
|
|
1314
1961
|
|
|
1315
|
-
export { SyncSnapshotEntry, SynchronisedStorageService, generateRestRoutesSynchronisedStorage, initSchema, restEntryPoints, synchronisedStorageSyncChangeSetRequest, tagsSynchronisedStorage };
|
|
1962
|
+
export { SyncSnapshotEntry, SynchronisedStorageService, generateRestRoutesSynchronisedStorage, initSchema, restEntryPoints, synchronisedStorageGetDecryptionKeyRequest, synchronisedStorageSyncChangeSetRequest, tagsSynchronisedStorage };
|