@twin.org/synchronised-storage-service 0.0.1-next.2 → 0.0.1-next.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +962 -314
- package/dist/esm/index.mjs +963 -316
- package/dist/types/entities/syncSnapshotEntry.d.ts +4 -5
- package/dist/types/helpers/blobStorageHelper.d.ts +33 -0
- package/dist/types/helpers/changeSetHelper.d.ts +20 -7
- package/dist/types/helpers/localSyncStateHelper.d.ts +13 -28
- package/dist/types/helpers/remoteSyncStateHelper.d.ts +26 -21
- package/dist/types/helpers/versions.d.ts +3 -0
- package/dist/types/index.d.ts +4 -2
- package/dist/types/models/ISyncPointerStore.d.ts +15 -0
- package/dist/types/models/ISyncSnapshot.d.ts +5 -1
- package/dist/types/models/ISyncState.d.ts +4 -0
- package/dist/types/models/ISynchronisedStorageServiceConfig.d.ts +17 -11
- package/dist/types/models/ISynchronisedStorageServiceConstructorOptions.d.ts +11 -2
- package/dist/types/synchronisedStorageRoutes.d.ts +9 -1
- package/dist/types/synchronisedStorageService.d.ts +13 -4
- package/docs/architecture.md +125 -0
- package/docs/changelog.md +29 -0
- package/docs/open-api/spec.json +244 -18
- package/docs/reference/classes/SyncSnapshotEntry.md +5 -5
- package/docs/reference/classes/SynchronisedStorageService.md +38 -5
- package/docs/reference/functions/synchronisedStorageGetDecryptionKeyRequest.md +31 -0
- package/docs/reference/index.md +4 -1
- package/docs/reference/interfaces/ISyncPointerStore.md +23 -0
- package/docs/reference/interfaces/ISyncSnapshot.md +43 -0
- package/docs/reference/interfaces/ISyncState.md +19 -0
- package/docs/reference/interfaces/ISynchronisedStorageServiceConfig.md +37 -17
- package/docs/reference/interfaces/ISynchronisedStorageServiceConstructorOptions.md +25 -3
- package/locales/en.json +62 -14
- package/package.json +4 -2
- package/dist/types/models/ISyncChange.d.ts +0 -18
- package/dist/types/models/ISyncChangeSet.d.ts +0 -36
- package/dist/types/models/ISyncPointer.d.ts +0 -9
- package/docs/reference/interfaces/ISyncChange.md +0 -33
package/dist/cjs/index.cjs
CHANGED
|
@@ -3,13 +3,15 @@
|
|
|
3
3
|
var entity = require('@twin.org/entity');
|
|
4
4
|
var core = require('@twin.org/core');
|
|
5
5
|
var web = require('@twin.org/web');
|
|
6
|
+
var blobStorageModels = require('@twin.org/blob-storage-models');
|
|
6
7
|
var entityStorageModels = require('@twin.org/entity-storage-models');
|
|
7
8
|
var identityModels = require('@twin.org/identity-models');
|
|
8
9
|
var loggingModels = require('@twin.org/logging-models');
|
|
10
|
+
var standardsW3cDid = require('@twin.org/standards-w3c-did');
|
|
9
11
|
var synchronisedStorageModels = require('@twin.org/synchronised-storage-models');
|
|
12
|
+
var vaultModels = require('@twin.org/vault-models');
|
|
10
13
|
var verifiableStorageModels = require('@twin.org/verifiable-storage-models');
|
|
11
|
-
var
|
|
12
|
-
var standardsW3cDid = require('@twin.org/standards-w3c-did');
|
|
14
|
+
var crypto = require('@twin.org/crypto');
|
|
13
15
|
|
|
14
16
|
// Copyright 2024 IOTA Stiftung.
|
|
15
17
|
// SPDX-License-Identifier: Apache-2.0.
|
|
@@ -22,9 +24,9 @@ exports.SyncSnapshotEntry = class SyncSnapshotEntry {
|
|
|
22
24
|
*/
|
|
23
25
|
id;
|
|
24
26
|
/**
|
|
25
|
-
* The
|
|
27
|
+
* The storage key for the snapshot i.e. which entity is being synchronized.
|
|
26
28
|
*/
|
|
27
|
-
|
|
29
|
+
storageKey;
|
|
28
30
|
/**
|
|
29
31
|
* The date the snapshot was created.
|
|
30
32
|
*/
|
|
@@ -44,7 +46,7 @@ exports.SyncSnapshotEntry = class SyncSnapshotEntry {
|
|
|
44
46
|
/**
|
|
45
47
|
* The changes that were made in this snapshot, if this is a local snapshot.
|
|
46
48
|
*/
|
|
47
|
-
|
|
49
|
+
changes;
|
|
48
50
|
};
|
|
49
51
|
__decorate([
|
|
50
52
|
entity.property({ type: "string", isPrimary: true }),
|
|
@@ -53,7 +55,7 @@ __decorate([
|
|
|
53
55
|
__decorate([
|
|
54
56
|
entity.property({ type: "string", isSecondary: true }),
|
|
55
57
|
__metadata("design:type", String)
|
|
56
|
-
], exports.SyncSnapshotEntry.prototype, "
|
|
58
|
+
], exports.SyncSnapshotEntry.prototype, "storageKey", void 0);
|
|
57
59
|
__decorate([
|
|
58
60
|
entity.property({ type: "string" }),
|
|
59
61
|
__metadata("design:type", String)
|
|
@@ -73,7 +75,7 @@ __decorate([
|
|
|
73
75
|
__decorate([
|
|
74
76
|
entity.property({ type: "array", itemType: "object", optional: true }),
|
|
75
77
|
__metadata("design:type", Array)
|
|
76
|
-
], exports.SyncSnapshotEntry.prototype, "
|
|
78
|
+
], exports.SyncSnapshotEntry.prototype, "changes", void 0);
|
|
77
79
|
exports.SyncSnapshotEntry = __decorate([
|
|
78
80
|
entity.entity()
|
|
79
81
|
], exports.SyncSnapshotEntry);
|
|
@@ -102,8 +104,8 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
|
|
|
102
104
|
operationId: "synchronisedStorageSyncChangeSetRequest",
|
|
103
105
|
summary: "Request that the node perform a sync request for a changeset.",
|
|
104
106
|
tag: tagsSynchronisedStorage[0].name,
|
|
105
|
-
method: "
|
|
106
|
-
path: `${baseRouteName}
|
|
107
|
+
method: "POST",
|
|
108
|
+
path: `${baseRouteName}/sync-changeset`,
|
|
107
109
|
handler: async (httpRequestContext, request) => synchronisedStorageSyncChangeSetRequest(httpRequestContext, componentName, request),
|
|
108
110
|
requestType: {
|
|
109
111
|
type: "ISyncChangeSetRequest",
|
|
@@ -111,8 +113,29 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
|
|
|
111
113
|
{
|
|
112
114
|
id: "synchronisedStorageSyncChangeSetRequestExample",
|
|
113
115
|
request: {
|
|
114
|
-
|
|
115
|
-
|
|
116
|
+
body: {
|
|
117
|
+
id: "0909090909090909090909090909090909090909090909090909090909090909",
|
|
118
|
+
dateCreated: "2025-05-29T01:00:00.000Z",
|
|
119
|
+
nodeIdentity: "did:entity-storage:0xd2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2",
|
|
120
|
+
changes: [
|
|
121
|
+
{
|
|
122
|
+
entity: {
|
|
123
|
+
dateModified: "2025-01-01T00:00:00.000Z"
|
|
124
|
+
},
|
|
125
|
+
id: "test-id-1",
|
|
126
|
+
operation: "set"
|
|
127
|
+
}
|
|
128
|
+
],
|
|
129
|
+
proof: {
|
|
130
|
+
"@context": "https://www.w3.org/ns/credentials/v2",
|
|
131
|
+
created: "2025-05-29T01:00:00.000Z",
|
|
132
|
+
cryptosuite: "eddsa-jcs-2022",
|
|
133
|
+
proofPurpose: "assertionMethod",
|
|
134
|
+
proofValue: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3",
|
|
135
|
+
type: "DataIntegrityProof",
|
|
136
|
+
verificationMethod: "did:entity-storage:0xd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0#synchronised-storage-assertion"
|
|
137
|
+
},
|
|
138
|
+
storageKey: "test-type"
|
|
116
139
|
}
|
|
117
140
|
}
|
|
118
141
|
}
|
|
@@ -122,9 +145,61 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
|
|
|
122
145
|
{
|
|
123
146
|
type: "INoContentResponse"
|
|
124
147
|
}
|
|
125
|
-
]
|
|
148
|
+
],
|
|
149
|
+
// Authentication is provided by the proof in the request body.
|
|
150
|
+
skipAuth: true
|
|
126
151
|
};
|
|
127
|
-
|
|
152
|
+
const getDecryptionKeyRoute = {
|
|
153
|
+
operationId: "synchronisedStorageGetDecryptionKeyRequest",
|
|
154
|
+
summary: "Request the decryption key.",
|
|
155
|
+
tag: tagsSynchronisedStorage[0].name,
|
|
156
|
+
method: "POST",
|
|
157
|
+
path: `${baseRouteName}/decryption-key`,
|
|
158
|
+
handler: async (httpRequestContext, request) => synchronisedStorageGetDecryptionKeyRequest(httpRequestContext, componentName, request),
|
|
159
|
+
requestType: {
|
|
160
|
+
type: "ISyncChangeSetRequest",
|
|
161
|
+
examples: [
|
|
162
|
+
{
|
|
163
|
+
id: "synchronisedStorageSyncGetDecryptionKeyRequestExample",
|
|
164
|
+
request: {
|
|
165
|
+
body: {
|
|
166
|
+
nodeIdentity: "did:entity-storage:0xd2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2",
|
|
167
|
+
proof: {
|
|
168
|
+
"@context": "https://www.w3.org/ns/credentials/v2",
|
|
169
|
+
created: "2025-05-29T01:00:00.000Z",
|
|
170
|
+
cryptosuite: "eddsa-jcs-2022",
|
|
171
|
+
proofPurpose: "assertionMethod",
|
|
172
|
+
proofValue: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3",
|
|
173
|
+
type: "DataIntegrityProof",
|
|
174
|
+
verificationMethod: "did:entity-storage:0xd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0#synchronised-storage-assertion"
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
]
|
|
180
|
+
},
|
|
181
|
+
responseType: [
|
|
182
|
+
{
|
|
183
|
+
type: "ISyncDecryptionKeyResponse",
|
|
184
|
+
examples: [
|
|
185
|
+
{
|
|
186
|
+
id: "synchronisedStorageSyncGetDecryptionKeyResponseExample",
|
|
187
|
+
response: {
|
|
188
|
+
body: {
|
|
189
|
+
decryptionKey: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3"
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
]
|
|
194
|
+
},
|
|
195
|
+
{
|
|
196
|
+
type: "IUnauthorizedResponse"
|
|
197
|
+
}
|
|
198
|
+
],
|
|
199
|
+
// Authentication is provided by the proof in the request body.
|
|
200
|
+
skipAuth: true
|
|
201
|
+
};
|
|
202
|
+
return [syncChangeSetRoute, getDecryptionKeyRoute];
|
|
128
203
|
}
|
|
129
204
|
/**
|
|
130
205
|
* Perform the sync change set operation.
|
|
@@ -135,13 +210,31 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
|
|
|
135
210
|
*/
|
|
136
211
|
async function synchronisedStorageSyncChangeSetRequest(httpRequestContext, componentName, request) {
|
|
137
212
|
core.Guards.object(ROUTES_SOURCE, "request", request);
|
|
138
|
-
core.Guards.object(ROUTES_SOURCE, "request.
|
|
213
|
+
core.Guards.object(ROUTES_SOURCE, "request.body", request.body);
|
|
139
214
|
const component = core.ComponentFactory.get(componentName);
|
|
140
|
-
await component.syncChangeSet(request.
|
|
215
|
+
await component.syncChangeSet(request.body);
|
|
141
216
|
return {
|
|
142
217
|
statusCode: web.HttpStatusCode.noContent
|
|
143
218
|
};
|
|
144
219
|
}
|
|
220
|
+
/**
|
|
221
|
+
* Request the decryption key.
|
|
222
|
+
* @param httpRequestContext The request context for the API.
|
|
223
|
+
* @param componentName The name of the component to use in the routes.
|
|
224
|
+
* @param request The request.
|
|
225
|
+
* @returns The response object with additional http response properties.
|
|
226
|
+
*/
|
|
227
|
+
async function synchronisedStorageGetDecryptionKeyRequest(httpRequestContext, componentName, request) {
|
|
228
|
+
core.Guards.object(ROUTES_SOURCE, "request", request);
|
|
229
|
+
core.Guards.object(ROUTES_SOURCE, "request.body", request.body);
|
|
230
|
+
const component = core.ComponentFactory.get(componentName);
|
|
231
|
+
const key = await component.getDecryptionKey(request.body.nodeIdentity, request.body.proof);
|
|
232
|
+
return {
|
|
233
|
+
body: {
|
|
234
|
+
decryptionKey: key
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
}
|
|
145
238
|
|
|
146
239
|
const restEntryPoints = [
|
|
147
240
|
{
|
|
@@ -161,6 +254,163 @@ function initSchema() {
|
|
|
161
254
|
entity.EntitySchemaFactory.register("SyncSnapshotEntry", () => entity.EntitySchemaHelper.getSchema(exports.SyncSnapshotEntry));
|
|
162
255
|
}
|
|
163
256
|
|
|
257
|
+
var mainnet = "";
|
|
258
|
+
var testnet = "";
|
|
259
|
+
var devnet = "";
|
|
260
|
+
var verifiableStorageKeys = {
|
|
261
|
+
mainnet: mainnet,
|
|
262
|
+
testnet: testnet,
|
|
263
|
+
devnet: devnet
|
|
264
|
+
};
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Class for performing blob storage operations.
|
|
268
|
+
*/
|
|
269
|
+
class BlobStorageHelper {
|
|
270
|
+
/**
|
|
271
|
+
* Runtime name for the class.
|
|
272
|
+
*/
|
|
273
|
+
CLASS_NAME = "BlobStorageHelper";
|
|
274
|
+
/**
|
|
275
|
+
* The logging connector to use for logging.
|
|
276
|
+
* @internal
|
|
277
|
+
*/
|
|
278
|
+
_logging;
|
|
279
|
+
/**
|
|
280
|
+
* The vault connector.
|
|
281
|
+
* @internal
|
|
282
|
+
*/
|
|
283
|
+
_vaultConnector;
|
|
284
|
+
/**
|
|
285
|
+
* The blob storage connector to use.
|
|
286
|
+
* @internal
|
|
287
|
+
*/
|
|
288
|
+
_blobStorageConnector;
|
|
289
|
+
/**
|
|
290
|
+
* The id of the vault key to use for encrypting/decrypting blobs.
|
|
291
|
+
* @internal
|
|
292
|
+
*/
|
|
293
|
+
_blobStorageEncryptionKeyId;
|
|
294
|
+
/**
|
|
295
|
+
* Is this a trusted node.
|
|
296
|
+
* @internal
|
|
297
|
+
*/
|
|
298
|
+
_isTrustedNode;
|
|
299
|
+
/**
|
|
300
|
+
* Create a new instance of BlobStorageHelper.
|
|
301
|
+
* @param logging The logging connector to use for logging.
|
|
302
|
+
* @param vaultConnector The vault connector to use for for the encryption key.
|
|
303
|
+
* @param blobStorageConnector The blob storage component to use.
|
|
304
|
+
* @param blobStorageEncryptionKeyId The id of the vault key to use for encrypting/decrypting blobs.
|
|
305
|
+
* @param isTrustedNode Is this a trusted node.
|
|
306
|
+
*/
|
|
307
|
+
constructor(logging, vaultConnector, blobStorageConnector, blobStorageEncryptionKeyId, isTrustedNode) {
|
|
308
|
+
this._logging = logging;
|
|
309
|
+
this._vaultConnector = vaultConnector;
|
|
310
|
+
this._blobStorageConnector = blobStorageConnector;
|
|
311
|
+
this._blobStorageEncryptionKeyId = blobStorageEncryptionKeyId;
|
|
312
|
+
this._isTrustedNode = isTrustedNode;
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* Load a blob from storage.
|
|
316
|
+
* @param blobId The id of the blob to apply.
|
|
317
|
+
* @returns The blob.
|
|
318
|
+
*/
|
|
319
|
+
async load(blobId) {
|
|
320
|
+
await this._logging?.log({
|
|
321
|
+
level: "info",
|
|
322
|
+
source: this.CLASS_NAME,
|
|
323
|
+
message: "loadBlob",
|
|
324
|
+
data: {
|
|
325
|
+
blobId
|
|
326
|
+
}
|
|
327
|
+
});
|
|
328
|
+
try {
|
|
329
|
+
const encryptedBlob = await this._blobStorageConnector.get(blobId);
|
|
330
|
+
if (core.Is.uint8Array(encryptedBlob)) {
|
|
331
|
+
let compressedBlob;
|
|
332
|
+
// If this is a trusted node, we can decrypt the blob using the vault
|
|
333
|
+
if (this._isTrustedNode) {
|
|
334
|
+
compressedBlob = await this._vaultConnector.decrypt(this._blobStorageEncryptionKeyId, vaultModels.VaultEncryptionType.Rsa2048, encryptedBlob);
|
|
335
|
+
}
|
|
336
|
+
else {
|
|
337
|
+
// Otherwise we need the public key stored as a secret in the vault
|
|
338
|
+
const key = await this._vaultConnector.getSecret(this._blobStorageEncryptionKeyId);
|
|
339
|
+
const rsa = new crypto.RSA(core.Converter.base64ToBytes(key));
|
|
340
|
+
compressedBlob = rsa.decrypt(encryptedBlob);
|
|
341
|
+
}
|
|
342
|
+
const decompressedBlob = await core.Compression.decompress(compressedBlob, core.CompressionType.Gzip);
|
|
343
|
+
await this._logging?.log({
|
|
344
|
+
level: "info",
|
|
345
|
+
source: this.CLASS_NAME,
|
|
346
|
+
message: "loadedBlob",
|
|
347
|
+
data: {
|
|
348
|
+
blobId
|
|
349
|
+
}
|
|
350
|
+
});
|
|
351
|
+
return core.ObjectHelper.fromBytes(decompressedBlob);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
catch (error) {
|
|
355
|
+
await this._logging?.log({
|
|
356
|
+
level: "error",
|
|
357
|
+
source: this.CLASS_NAME,
|
|
358
|
+
message: "loadBlobFailed",
|
|
359
|
+
data: {
|
|
360
|
+
blobId
|
|
361
|
+
},
|
|
362
|
+
error: core.BaseError.fromError(error)
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
await this._logging?.log({
|
|
366
|
+
level: "info",
|
|
367
|
+
source: this.CLASS_NAME,
|
|
368
|
+
message: "loadBlobEmpty",
|
|
369
|
+
data: {
|
|
370
|
+
blobId
|
|
371
|
+
}
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
/**
|
|
375
|
+
* Save a blob.
|
|
376
|
+
* @param blob The blob to save.
|
|
377
|
+
* @returns The id of the blob.
|
|
378
|
+
*/
|
|
379
|
+
async saveBlob(blob) {
|
|
380
|
+
await this._logging?.log({
|
|
381
|
+
level: "info",
|
|
382
|
+
source: this.CLASS_NAME,
|
|
383
|
+
message: "saveBlob"
|
|
384
|
+
});
|
|
385
|
+
if (!this._isTrustedNode) {
|
|
386
|
+
throw new core.GeneralError(this.CLASS_NAME, "notTrustedNode");
|
|
387
|
+
}
|
|
388
|
+
const compressedBlob = await core.Compression.compress(core.ObjectHelper.toBytes(blob), core.CompressionType.Gzip);
|
|
389
|
+
const encryptedBlob = await this._vaultConnector.encrypt(this._blobStorageEncryptionKeyId, vaultModels.VaultEncryptionType.Rsa2048, compressedBlob);
|
|
390
|
+
try {
|
|
391
|
+
const blobId = await this._blobStorageConnector.set(encryptedBlob);
|
|
392
|
+
await this._logging?.log({
|
|
393
|
+
level: "info",
|
|
394
|
+
source: this.CLASS_NAME,
|
|
395
|
+
message: "savedBlob",
|
|
396
|
+
data: {
|
|
397
|
+
blobId
|
|
398
|
+
}
|
|
399
|
+
});
|
|
400
|
+
return blobId;
|
|
401
|
+
}
|
|
402
|
+
catch (error) {
|
|
403
|
+
await this._logging?.log({
|
|
404
|
+
level: "error",
|
|
405
|
+
source: this.CLASS_NAME,
|
|
406
|
+
message: "saveBlobFailed",
|
|
407
|
+
error: core.BaseError.fromError(error)
|
|
408
|
+
});
|
|
409
|
+
throw error;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
|
|
164
414
|
// Copyright 2024 IOTA Stiftung.
|
|
165
415
|
// SPDX-License-Identifier: Apache-2.0.
|
|
166
416
|
/**
|
|
@@ -182,10 +432,10 @@ class ChangeSetHelper {
|
|
|
182
432
|
*/
|
|
183
433
|
_eventBusComponent;
|
|
184
434
|
/**
|
|
185
|
-
* The blob storage
|
|
435
|
+
* The blob storage helper to use for remote sync states.
|
|
186
436
|
* @internal
|
|
187
437
|
*/
|
|
188
|
-
|
|
438
|
+
_blobStorageHelper;
|
|
189
439
|
/**
|
|
190
440
|
* The identity connector to use for signing/verifying changesets.
|
|
191
441
|
* @internal
|
|
@@ -196,50 +446,85 @@ class ChangeSetHelper {
|
|
|
196
446
|
* @internal
|
|
197
447
|
*/
|
|
198
448
|
_decentralisedStorageMethodId;
|
|
449
|
+
/**
|
|
450
|
+
* The identity of the node that is performing the update.
|
|
451
|
+
* @internal
|
|
452
|
+
*/
|
|
453
|
+
_nodeIdentity;
|
|
199
454
|
/**
|
|
200
455
|
* Create a new instance of ChangeSetHelper.
|
|
201
456
|
* @param logging The logging connector to use for logging.
|
|
202
457
|
* @param eventBusComponent The event bus component to use for events.
|
|
203
|
-
* @param blobStorageComponent The blob storage component to use for remote sync states.
|
|
204
458
|
* @param identityConnector The identity connector to use for signing/verifying changesets.
|
|
459
|
+
* @param blobStorageHelper The blob storage component to use for remote sync states.
|
|
205
460
|
* @param decentralisedStorageMethodId The id of the identity method to use when signing/verifying changesets.
|
|
206
461
|
*/
|
|
207
|
-
constructor(logging, eventBusComponent,
|
|
462
|
+
constructor(logging, eventBusComponent, identityConnector, blobStorageHelper, decentralisedStorageMethodId) {
|
|
208
463
|
this._logging = logging;
|
|
209
464
|
this._eventBusComponent = eventBusComponent;
|
|
210
465
|
this._decentralisedStorageMethodId = decentralisedStorageMethodId;
|
|
211
|
-
this.
|
|
466
|
+
this._blobStorageHelper = blobStorageHelper;
|
|
212
467
|
this._identityConnector = identityConnector;
|
|
213
468
|
}
|
|
469
|
+
/**
|
|
470
|
+
* Set the node identity to use for signing changesets.
|
|
471
|
+
* @param nodeIdentity The identity of the node that is performing the update.
|
|
472
|
+
*/
|
|
473
|
+
setNodeIdentity(nodeIdentity) {
|
|
474
|
+
this._nodeIdentity = nodeIdentity;
|
|
475
|
+
}
|
|
214
476
|
/**
|
|
215
477
|
* Get and verify a changeset.
|
|
216
478
|
* @param changeSetStorageId The id of the sync changeset to apply.
|
|
217
479
|
* @returns The changeset if it was verified.
|
|
218
480
|
*/
|
|
219
481
|
async getAndVerifyChangeset(changeSetStorageId) {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
482
|
+
await this._logging?.log({
|
|
483
|
+
level: "info",
|
|
484
|
+
source: this.CLASS_NAME,
|
|
485
|
+
message: "getChangeSet",
|
|
486
|
+
data: {
|
|
487
|
+
changeSetStorageId
|
|
488
|
+
}
|
|
224
489
|
});
|
|
225
|
-
|
|
226
|
-
const
|
|
227
|
-
|
|
228
|
-
|
|
490
|
+
try {
|
|
491
|
+
const syncChangeSet = await this._blobStorageHelper.load(changeSetStorageId);
|
|
492
|
+
if (core.Is.object(syncChangeSet)) {
|
|
493
|
+
const verified = await this.verifyChangesetProof(syncChangeSet);
|
|
494
|
+
return verified ? syncChangeSet : undefined;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
catch (error) {
|
|
498
|
+
await this._logging?.log({
|
|
499
|
+
level: "warn",
|
|
500
|
+
source: this.CLASS_NAME,
|
|
501
|
+
message: "getChangeSetError",
|
|
502
|
+
data: {
|
|
503
|
+
changeSetStorageId
|
|
504
|
+
},
|
|
505
|
+
error: core.BaseError.fromError(error)
|
|
506
|
+
});
|
|
229
507
|
}
|
|
508
|
+
await this._logging?.log({
|
|
509
|
+
level: "info",
|
|
510
|
+
source: this.CLASS_NAME,
|
|
511
|
+
message: "getChangeSetEmpty",
|
|
512
|
+
data: {
|
|
513
|
+
changeSetStorageId
|
|
514
|
+
}
|
|
515
|
+
});
|
|
230
516
|
}
|
|
231
517
|
/**
|
|
232
518
|
* Apply a sync changeset.
|
|
233
519
|
* @param changeSetStorageId The id of the sync changeset to apply.
|
|
234
|
-
* @returns
|
|
520
|
+
* @returns The changeset if it existed.
|
|
235
521
|
*/
|
|
236
522
|
async getAndApplyChangeset(changeSetStorageId) {
|
|
237
523
|
const syncChangeset = await this.getAndVerifyChangeset(changeSetStorageId);
|
|
238
524
|
if (!core.Is.empty(syncChangeset)) {
|
|
239
525
|
await this.applyChangeset(syncChangeset);
|
|
240
|
-
return true;
|
|
241
526
|
}
|
|
242
|
-
return
|
|
527
|
+
return syncChangeset;
|
|
243
528
|
}
|
|
244
529
|
/**
|
|
245
530
|
* Apply a sync changeset.
|
|
@@ -261,20 +546,25 @@ class ChangeSetHelper {
|
|
|
261
546
|
switch (change.operation) {
|
|
262
547
|
case synchronisedStorageModels.SyncChangeOperation.Set:
|
|
263
548
|
if (!core.Is.empty(change.entity)) {
|
|
264
|
-
// The
|
|
549
|
+
// The id was stripped from the entity as it is part of the operation
|
|
550
|
+
// we make sure we reinstate it in the publish
|
|
551
|
+
// Also the node identity was stripped when stored in the changeset
|
|
265
552
|
// as the changeset is signed with the node identity.
|
|
266
553
|
// so we need to restore it here.
|
|
267
|
-
change.entity.nodeIdentity = syncChangeset.nodeIdentity;
|
|
268
554
|
await this._eventBusComponent.publish(synchronisedStorageModels.SynchronisedStorageTopics.RemoteItemSet, {
|
|
269
|
-
|
|
270
|
-
entity:
|
|
555
|
+
storageKey: syncChangeset.storageKey,
|
|
556
|
+
entity: {
|
|
557
|
+
...change.entity,
|
|
558
|
+
id: change.id,
|
|
559
|
+
nodeIdentity: syncChangeset.nodeIdentity
|
|
560
|
+
}
|
|
271
561
|
});
|
|
272
562
|
}
|
|
273
563
|
break;
|
|
274
564
|
case synchronisedStorageModels.SyncChangeOperation.Delete:
|
|
275
565
|
if (!core.Is.empty(change.id)) {
|
|
276
566
|
await this._eventBusComponent.publish(synchronisedStorageModels.SynchronisedStorageTopics.RemoteItemRemove, {
|
|
277
|
-
|
|
567
|
+
storageKey: syncChangeset.storageKey,
|
|
278
568
|
id: change.id
|
|
279
569
|
});
|
|
280
570
|
}
|
|
@@ -297,12 +587,7 @@ class ChangeSetHelper {
|
|
|
297
587
|
id: syncChangeSet.id
|
|
298
588
|
}
|
|
299
589
|
});
|
|
300
|
-
|
|
301
|
-
// the blob storage also needs to be publicly accessible so that other nodes can retrieve it
|
|
302
|
-
return this._blobStorageComponent.create(core.Converter.bytesToBase64(core.ObjectHelper.toBytes(syncChangeSet)), undefined, undefined, undefined, {
|
|
303
|
-
disableEncryption: true,
|
|
304
|
-
compress: blobStorageModels.BlobStorageCompressionType.Gzip
|
|
305
|
-
});
|
|
590
|
+
return this._blobStorageHelper.saveBlob(syncChangeSet);
|
|
306
591
|
}
|
|
307
592
|
/**
|
|
308
593
|
* Verify the proof of a sync changeset.
|
|
@@ -321,6 +606,32 @@ class ChangeSetHelper {
|
|
|
321
606
|
});
|
|
322
607
|
return false;
|
|
323
608
|
}
|
|
609
|
+
// If the proof or verification method is missing, the proof is invalid
|
|
610
|
+
const verificationMethod = syncChangeset.proof?.verificationMethod;
|
|
611
|
+
if (!core.Is.stringValue(verificationMethod)) {
|
|
612
|
+
await this._logging?.log({
|
|
613
|
+
level: "error",
|
|
614
|
+
source: this.CLASS_NAME,
|
|
615
|
+
message: "verifyChangeSetProofMissing",
|
|
616
|
+
data: {
|
|
617
|
+
id: syncChangeset.id
|
|
618
|
+
}
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
// Parse the verification method and extract the node identity
|
|
622
|
+
// this should match the node identity of the changeset
|
|
623
|
+
// otherwise you could sign a changeset for another node
|
|
624
|
+
const changeSetNodeIdentity = identityModels.DocumentHelper.parseId(verificationMethod ?? "");
|
|
625
|
+
if (changeSetNodeIdentity.id !== syncChangeset.nodeIdentity) {
|
|
626
|
+
await this._logging?.log({
|
|
627
|
+
level: "error",
|
|
628
|
+
source: this.CLASS_NAME,
|
|
629
|
+
message: "verifyChangeSetProofNodeIdentityMismatch",
|
|
630
|
+
data: {
|
|
631
|
+
id: syncChangeset.id
|
|
632
|
+
}
|
|
633
|
+
});
|
|
634
|
+
}
|
|
324
635
|
const changeSetWithoutProof = core.ObjectHelper.clone(syncChangeset);
|
|
325
636
|
delete changeSetWithoutProof.proof;
|
|
326
637
|
const isValid = await this._identityConnector.verifyProof(changeSetWithoutProof, syncChangeset.proof);
|
|
@@ -352,11 +663,12 @@ class ChangeSetHelper {
|
|
|
352
663
|
* @returns The proof.
|
|
353
664
|
*/
|
|
354
665
|
async createChangeSetProof(syncChangeset) {
|
|
666
|
+
core.Guards.stringValue(this.CLASS_NAME, "nodeIdentity", this._nodeIdentity);
|
|
355
667
|
const changeSetWithoutProof = core.ObjectHelper.clone(syncChangeset);
|
|
356
668
|
delete changeSetWithoutProof.proof;
|
|
357
|
-
const proof = await this._identityConnector.createProof(
|
|
669
|
+
const proof = await this._identityConnector.createProof(this._nodeIdentity, identityModels.DocumentHelper.joinId(this._nodeIdentity, this._decentralisedStorageMethodId), standardsW3cDid.ProofTypes.DataIntegrityProof, changeSetWithoutProof);
|
|
358
670
|
await this._logging?.log({
|
|
359
|
-
level: "
|
|
671
|
+
level: "info",
|
|
360
672
|
source: this.CLASS_NAME,
|
|
361
673
|
message: "createdChangeSetProof",
|
|
362
674
|
data: {
|
|
@@ -366,6 +678,35 @@ class ChangeSetHelper {
|
|
|
366
678
|
});
|
|
367
679
|
return proof;
|
|
368
680
|
}
|
|
681
|
+
/**
|
|
682
|
+
* Copy a change set.
|
|
683
|
+
* @param syncChangeSet The sync changeset to copy.
|
|
684
|
+
* @returns The id of the updated change set.
|
|
685
|
+
*/
|
|
686
|
+
async copyChangeset(syncChangeSet) {
|
|
687
|
+
if (core.Is.stringValue(this._nodeIdentity)) {
|
|
688
|
+
const verified = await this.verifyChangesetProof(syncChangeSet);
|
|
689
|
+
if (verified) {
|
|
690
|
+
await this._logging?.log({
|
|
691
|
+
level: "info",
|
|
692
|
+
source: this.CLASS_NAME,
|
|
693
|
+
message: "copyChangeSet",
|
|
694
|
+
data: {
|
|
695
|
+
changeSetStorageId: syncChangeSet.id
|
|
696
|
+
}
|
|
697
|
+
});
|
|
698
|
+
// Allocate a new id to the changeset copy and re-create a proof using this nodes identity
|
|
699
|
+
const copy = core.ObjectHelper.clone(syncChangeSet);
|
|
700
|
+
copy.id = core.Converter.bytesToHex(core.RandomHelper.generate(32));
|
|
701
|
+
copy.proof = await this.createChangeSetProof(copy);
|
|
702
|
+
// Store the copy
|
|
703
|
+
return {
|
|
704
|
+
syncChangeSet: copy,
|
|
705
|
+
changeSetStorageId: await this.storeChangeSet(copy)
|
|
706
|
+
};
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
}
|
|
369
710
|
}
|
|
370
711
|
|
|
371
712
|
// Copyright 2024 IOTA Stiftung.
|
|
@@ -387,7 +728,7 @@ class LocalSyncStateHelper {
|
|
|
387
728
|
* The storage connector for the sync snapshot entries.
|
|
388
729
|
* @internal
|
|
389
730
|
*/
|
|
390
|
-
|
|
731
|
+
_snapshotEntryEntityStorage;
|
|
391
732
|
/**
|
|
392
733
|
* The change set helper to use for applying changesets.
|
|
393
734
|
* @internal
|
|
@@ -396,44 +737,62 @@ class LocalSyncStateHelper {
|
|
|
396
737
|
/**
|
|
397
738
|
* Create a new instance of LocalSyncStateHelper.
|
|
398
739
|
* @param logging The logging connector to use for logging.
|
|
399
|
-
* @param
|
|
740
|
+
* @param snapshotEntryEntityStorage The storage connector for the sync snapshot entries.
|
|
400
741
|
* @param changeSetHelper The change set helper to use for applying changesets.
|
|
401
742
|
*/
|
|
402
|
-
constructor(logging,
|
|
743
|
+
constructor(logging, snapshotEntryEntityStorage, changeSetHelper) {
|
|
403
744
|
this._logging = logging;
|
|
404
|
-
this.
|
|
745
|
+
this._snapshotEntryEntityStorage = snapshotEntryEntityStorage;
|
|
405
746
|
this._changeSetHelper = changeSetHelper;
|
|
406
747
|
}
|
|
407
748
|
/**
|
|
408
749
|
* Add a new change to the local snapshot.
|
|
409
|
-
* @param
|
|
750
|
+
* @param storageKey The storage key of the snapshot to add the change for.
|
|
410
751
|
* @param operation The operation to perform.
|
|
411
752
|
* @param id The id of the entity to add the change for.
|
|
412
753
|
* @returns Nothing.
|
|
413
754
|
*/
|
|
414
|
-
async addLocalChange(
|
|
415
|
-
|
|
416
|
-
|
|
755
|
+
async addLocalChange(storageKey, operation, id) {
|
|
756
|
+
await this._logging?.log({
|
|
757
|
+
level: "info",
|
|
758
|
+
source: this.CLASS_NAME,
|
|
759
|
+
message: "addLocalChange",
|
|
760
|
+
data: {
|
|
761
|
+
storageKey,
|
|
762
|
+
operation,
|
|
763
|
+
id
|
|
764
|
+
}
|
|
765
|
+
});
|
|
766
|
+
const localChangeSnapshot = await this.getLocalChangeSnapshot(storageKey);
|
|
767
|
+
localChangeSnapshot.changes ??= [];
|
|
417
768
|
// If we already have a change for this id we are
|
|
418
769
|
// about to supersede it, we remove the previous change
|
|
419
770
|
// to avoid having multiple changes for the same id
|
|
420
|
-
const previousChangeIndex = localChangeSnapshot.
|
|
771
|
+
const previousChangeIndex = localChangeSnapshot.changes.findIndex(change => change.id === id);
|
|
421
772
|
if (previousChangeIndex !== -1) {
|
|
422
|
-
localChangeSnapshot.
|
|
773
|
+
localChangeSnapshot.changes.splice(previousChangeIndex, 1);
|
|
423
774
|
}
|
|
424
|
-
if (localChangeSnapshot.
|
|
775
|
+
if (localChangeSnapshot.changes.length > 0) {
|
|
425
776
|
localChangeSnapshot.dateModified = new Date(Date.now()).toISOString();
|
|
426
777
|
}
|
|
427
|
-
localChangeSnapshot.
|
|
778
|
+
localChangeSnapshot.changes.push({ operation, id });
|
|
428
779
|
await this.setLocalChangeSnapshot(localChangeSnapshot);
|
|
429
780
|
}
|
|
430
781
|
/**
|
|
431
|
-
* Get the current local snapshot.
|
|
432
|
-
* @param
|
|
782
|
+
* Get the current local snapshot which contains just the changes for this node.
|
|
783
|
+
* @param storageKey The storage key of the snapshot to get.
|
|
433
784
|
* @returns The local snapshot entry.
|
|
434
785
|
*/
|
|
435
|
-
async getLocalChangeSnapshot(
|
|
436
|
-
|
|
786
|
+
async getLocalChangeSnapshot(storageKey) {
|
|
787
|
+
await this._logging?.log({
|
|
788
|
+
level: "info",
|
|
789
|
+
source: this.CLASS_NAME,
|
|
790
|
+
message: "getLocalChangeSnapshot",
|
|
791
|
+
data: {
|
|
792
|
+
storageKey
|
|
793
|
+
}
|
|
794
|
+
});
|
|
795
|
+
const queryResult = await this._snapshotEntryEntityStorage.query({
|
|
437
796
|
conditions: [
|
|
438
797
|
{
|
|
439
798
|
property: "isLocalSnapshot",
|
|
@@ -441,33 +800,57 @@ class LocalSyncStateHelper {
|
|
|
441
800
|
comparison: entity.ComparisonOperator.Equals
|
|
442
801
|
},
|
|
443
802
|
{
|
|
444
|
-
property: "
|
|
445
|
-
value:
|
|
803
|
+
property: "storageKey",
|
|
804
|
+
value: storageKey,
|
|
446
805
|
comparison: entity.ComparisonOperator.Equals
|
|
447
806
|
}
|
|
448
807
|
]
|
|
449
808
|
});
|
|
450
809
|
if (queryResult.entities.length > 0) {
|
|
810
|
+
await this._logging?.log({
|
|
811
|
+
level: "info",
|
|
812
|
+
source: this.CLASS_NAME,
|
|
813
|
+
message: "localChangeSnapshotExists",
|
|
814
|
+
data: {
|
|
815
|
+
storageKey
|
|
816
|
+
}
|
|
817
|
+
});
|
|
451
818
|
return queryResult.entities[0];
|
|
452
819
|
}
|
|
820
|
+
await this._logging?.log({
|
|
821
|
+
level: "info",
|
|
822
|
+
source: this.CLASS_NAME,
|
|
823
|
+
message: "localChangeSnapshotDoesNotExist",
|
|
824
|
+
data: {
|
|
825
|
+
storageKey
|
|
826
|
+
}
|
|
827
|
+
});
|
|
453
828
|
return {
|
|
454
829
|
id: core.Converter.bytesToHex(core.RandomHelper.generate(32)),
|
|
455
|
-
|
|
830
|
+
storageKey,
|
|
456
831
|
dateCreated: new Date(Date.now()).toISOString(),
|
|
457
832
|
changeSetStorageIds: [],
|
|
458
833
|
isLocalSnapshot: true
|
|
459
834
|
};
|
|
460
835
|
}
|
|
461
836
|
/**
|
|
462
|
-
* Set the current local snapshot.
|
|
837
|
+
* Set the current local snapshot with changes for this node.
|
|
463
838
|
* @param localChangeSnapshot The local change snapshot to set.
|
|
464
839
|
* @returns Nothing.
|
|
465
840
|
*/
|
|
466
841
|
async setLocalChangeSnapshot(localChangeSnapshot) {
|
|
467
|
-
await this.
|
|
842
|
+
await this._logging?.log({
|
|
843
|
+
level: "info",
|
|
844
|
+
source: this.CLASS_NAME,
|
|
845
|
+
message: "setLocalChangeSnapshot",
|
|
846
|
+
data: {
|
|
847
|
+
storageKey: localChangeSnapshot.storageKey
|
|
848
|
+
}
|
|
849
|
+
});
|
|
850
|
+
await this._snapshotEntryEntityStorage.set(localChangeSnapshot);
|
|
468
851
|
}
|
|
469
852
|
/**
|
|
470
|
-
* Get the current local snapshot.
|
|
853
|
+
* Get the current local snapshot with the changes for this node.
|
|
471
854
|
* @param localChangeSnapshot The local change snapshot to remove.
|
|
472
855
|
* @returns Nothing.
|
|
473
856
|
*/
|
|
@@ -480,47 +863,47 @@ class LocalSyncStateHelper {
|
|
|
480
863
|
snapshotId: localChangeSnapshot.id
|
|
481
864
|
}
|
|
482
865
|
});
|
|
483
|
-
await this.
|
|
866
|
+
await this._snapshotEntryEntityStorage.remove(localChangeSnapshot.id);
|
|
484
867
|
}
|
|
485
868
|
/**
|
|
486
|
-
*
|
|
487
|
-
* @param
|
|
488
|
-
* @param
|
|
869
|
+
* Apply a sync state to the local node.
|
|
870
|
+
* @param storageKey The storage key of the snapshot to sync with.
|
|
871
|
+
* @param syncState The sync state to sync with.
|
|
489
872
|
* @returns Nothing.
|
|
490
873
|
*/
|
|
491
|
-
async
|
|
874
|
+
async applySyncState(storageKey, syncState) {
|
|
492
875
|
await this._logging?.log({
|
|
493
876
|
level: "info",
|
|
494
877
|
source: this.CLASS_NAME,
|
|
495
|
-
message: "
|
|
878
|
+
message: "applySyncState",
|
|
496
879
|
data: {
|
|
497
|
-
snapshotCount:
|
|
880
|
+
snapshotCount: syncState.snapshots.length
|
|
498
881
|
}
|
|
499
882
|
});
|
|
500
883
|
// Sort from newest to oldest
|
|
501
|
-
const
|
|
884
|
+
const sortedSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
|
|
502
885
|
const newSnapshots = [];
|
|
503
886
|
const modifiedSnapshots = [];
|
|
504
|
-
for (const
|
|
887
|
+
for (const snapshot of sortedSnapshots) {
|
|
505
888
|
await this._logging?.log({
|
|
506
889
|
level: "info",
|
|
507
890
|
source: this.CLASS_NAME,
|
|
508
|
-
message: "
|
|
891
|
+
message: "applySnapshot",
|
|
509
892
|
data: {
|
|
510
|
-
snapshotId:
|
|
511
|
-
dateCreated: new Date(
|
|
893
|
+
snapshotId: snapshot.id,
|
|
894
|
+
dateCreated: new Date(snapshot.dateCreated).toISOString()
|
|
512
895
|
}
|
|
513
896
|
});
|
|
514
|
-
const localSnapshot = await this.
|
|
897
|
+
const localSnapshot = await this._snapshotEntryEntityStorage.get(snapshot.id);
|
|
515
898
|
const remoteSnapshotWithContext = {
|
|
516
|
-
...
|
|
517
|
-
|
|
899
|
+
...snapshot,
|
|
900
|
+
storageKey
|
|
518
901
|
};
|
|
519
902
|
if (core.Is.empty(localSnapshot)) {
|
|
520
903
|
// We don't have the snapshot locally, so we need to process it
|
|
521
904
|
newSnapshots.push(remoteSnapshotWithContext);
|
|
522
905
|
}
|
|
523
|
-
else if (localSnapshot.dateModified !==
|
|
906
|
+
else if (localSnapshot.dateModified !== snapshot.dateModified) {
|
|
524
907
|
// If the local snapshot has a different dateModified, we need to update it
|
|
525
908
|
modifiedSnapshots.push({
|
|
526
909
|
localSnapshot,
|
|
@@ -542,13 +925,14 @@ class LocalSyncStateHelper {
|
|
|
542
925
|
* Process the modified snapshots and store them in the local storage.
|
|
543
926
|
* @param modifiedSnapshots The modified snapshots to process.
|
|
544
927
|
* @returns Nothing.
|
|
928
|
+
* @internal
|
|
545
929
|
*/
|
|
546
930
|
async processModifiedSnapshots(modifiedSnapshots) {
|
|
547
931
|
for (const modifiedSnapshot of modifiedSnapshots) {
|
|
548
932
|
await this._logging?.log({
|
|
549
933
|
level: "info",
|
|
550
934
|
source: this.CLASS_NAME,
|
|
551
|
-
message: "
|
|
935
|
+
message: "processModifiedSnapshot",
|
|
552
936
|
data: {
|
|
553
937
|
snapshotId: modifiedSnapshot.remoteSnapshot.id,
|
|
554
938
|
localModified: new Date(modifiedSnapshot.localSnapshot.dateModified ??
|
|
@@ -567,20 +951,21 @@ class LocalSyncStateHelper {
|
|
|
567
951
|
}
|
|
568
952
|
}
|
|
569
953
|
}
|
|
570
|
-
await this.
|
|
954
|
+
await this._snapshotEntryEntityStorage.set(modifiedSnapshot.remoteSnapshot);
|
|
571
955
|
}
|
|
572
956
|
}
|
|
573
957
|
/**
|
|
574
958
|
* Process the new snapshots and store them in the local storage.
|
|
575
959
|
* @param newSnapshots The new snapshots to process.
|
|
576
960
|
* @returns Nothing.
|
|
961
|
+
* @internal
|
|
577
962
|
*/
|
|
578
963
|
async processNewSnapshots(newSnapshots) {
|
|
579
964
|
for (const newSnapshot of newSnapshots) {
|
|
580
965
|
await this._logging?.log({
|
|
581
966
|
level: "info",
|
|
582
967
|
source: this.CLASS_NAME,
|
|
583
|
-
message: "
|
|
968
|
+
message: "processNewSnapshot",
|
|
584
969
|
data: {
|
|
585
970
|
snapshotId: newSnapshot.id,
|
|
586
971
|
localModified: new Date(newSnapshot.dateCreated).toISOString()
|
|
@@ -592,11 +977,17 @@ class LocalSyncStateHelper {
|
|
|
592
977
|
await this._changeSetHelper.getAndApplyChangeset(storageId);
|
|
593
978
|
}
|
|
594
979
|
}
|
|
595
|
-
await this.
|
|
980
|
+
await this._snapshotEntryEntityStorage.set(newSnapshot);
|
|
596
981
|
}
|
|
597
982
|
}
|
|
598
983
|
}
|
|
599
984
|
|
|
985
|
+
// Copyright 2024 IOTA Stiftung.
|
|
986
|
+
// SPDX-License-Identifier: Apache-2.0.
|
|
987
|
+
const SYNC_STATE_VERSION = "1";
|
|
988
|
+
const SYNC_POINTER_STORE_VERSION = "1";
|
|
989
|
+
const SYNC_SNAPSHOT_VERSION = "1";
|
|
990
|
+
|
|
600
991
|
// Copyright 2024 IOTA Stiftung.
|
|
601
992
|
// SPDX-License-Identifier: Apache-2.0.
|
|
602
993
|
/**
|
|
@@ -618,10 +1009,10 @@ class RemoteSyncStateHelper {
|
|
|
618
1009
|
*/
|
|
619
1010
|
_eventBusComponent;
|
|
620
1011
|
/**
|
|
621
|
-
* The blob storage
|
|
1012
|
+
* The blob storage helper.
|
|
622
1013
|
* @internal
|
|
623
1014
|
*/
|
|
624
|
-
|
|
1015
|
+
_blobStorageHelper;
|
|
625
1016
|
/**
|
|
626
1017
|
* The verifiable storage connector to use for storing sync pointers.
|
|
627
1018
|
* @internal
|
|
@@ -633,12 +1024,12 @@ class RemoteSyncStateHelper {
|
|
|
633
1024
|
*/
|
|
634
1025
|
_changeSetHelper;
|
|
635
1026
|
/**
|
|
636
|
-
* The storage ids of the batch responses for each
|
|
1027
|
+
* The storage ids of the batch responses for each storage key.
|
|
637
1028
|
* @internal
|
|
638
1029
|
*/
|
|
639
1030
|
_batchResponseStorageIds;
|
|
640
1031
|
/**
|
|
641
|
-
* The full changes for each
|
|
1032
|
+
* The full changes for each storage key.
|
|
642
1033
|
* @internal
|
|
643
1034
|
*/
|
|
644
1035
|
_populateFullChanges;
|
|
@@ -652,22 +1043,27 @@ class RemoteSyncStateHelper {
|
|
|
652
1043
|
* @internal
|
|
653
1044
|
*/
|
|
654
1045
|
_nodeIdentity;
|
|
1046
|
+
/**
|
|
1047
|
+
* Whether the node is trusted or not.
|
|
1048
|
+
* @internal
|
|
1049
|
+
*/
|
|
1050
|
+
_isTrustedNode;
|
|
655
1051
|
/**
|
|
656
1052
|
* Create a new instance of DecentralisedEntityStorageConnector.
|
|
657
1053
|
* @param logging The logging connector to use for logging.
|
|
658
1054
|
* @param eventBusComponent The event bus component to use for events.
|
|
659
|
-
* @param blobStorageComponent The blob storage component to use for remote sync states.
|
|
660
1055
|
* @param verifiableSyncPointerStorageConnector The verifiable storage connector to use for storing sync pointers.
|
|
1056
|
+
* @param blobStorageHelper The blob storage helper to use for remote sync states.
|
|
661
1057
|
* @param changeSetHelper The change set helper to use for managing changesets.
|
|
662
|
-
* @param
|
|
1058
|
+
* @param isTrustedNode Whether the node is trusted or not.
|
|
663
1059
|
*/
|
|
664
|
-
constructor(logging, eventBusComponent,
|
|
1060
|
+
constructor(logging, eventBusComponent, verifiableSyncPointerStorageConnector, blobStorageHelper, changeSetHelper, isTrustedNode) {
|
|
665
1061
|
this._logging = logging;
|
|
666
1062
|
this._eventBusComponent = eventBusComponent;
|
|
667
|
-
this._blobStorageComponent = blobStorageComponent;
|
|
668
1063
|
this._verifiableSyncPointerStorageConnector = verifiableSyncPointerStorageConnector;
|
|
669
1064
|
this._changeSetHelper = changeSetHelper;
|
|
670
|
-
this.
|
|
1065
|
+
this._blobStorageHelper = blobStorageHelper;
|
|
1066
|
+
this._isTrustedNode = isTrustedNode;
|
|
671
1067
|
this._batchResponseStorageIds = {};
|
|
672
1068
|
this._populateFullChanges = {};
|
|
673
1069
|
this._eventBusComponent.subscribe(synchronisedStorageModels.SynchronisedStorageTopics.BatchResponse, async (response) => {
|
|
@@ -685,72 +1081,119 @@ class RemoteSyncStateHelper {
|
|
|
685
1081
|
this._nodeIdentity = nodeIdentity;
|
|
686
1082
|
}
|
|
687
1083
|
/**
|
|
688
|
-
*
|
|
689
|
-
* @param
|
|
1084
|
+
* Set the synchronised storage key.
|
|
1085
|
+
* @param synchronisedStorageKey The synchronised storage key to use.
|
|
1086
|
+
*/
|
|
1087
|
+
setSynchronisedStorageKey(synchronisedStorageKey) {
|
|
1088
|
+
this._synchronisedStorageKey = synchronisedStorageKey;
|
|
1089
|
+
}
|
|
1090
|
+
/**
|
|
1091
|
+
* Build a changeset.
|
|
1092
|
+
* @param storageKey The storage key of the change set.
|
|
690
1093
|
* @param changes The changes to apply.
|
|
691
1094
|
* @param completeCallback The callback to call when the changeset is created and stored.
|
|
692
1095
|
* @returns The storage id of the change set if created.
|
|
693
1096
|
*/
|
|
694
|
-
async
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
const setChanges = changes.filter(c => c.operation === synchronisedStorageModels.SyncChangeOperation.Set);
|
|
703
|
-
if (setChanges.length === 0) {
|
|
704
|
-
// If we don't need to request any full details, we can just call the complete callback
|
|
705
|
-
await this.finaliseFullChanges(schemaType, completeCallback);
|
|
706
|
-
}
|
|
707
|
-
else {
|
|
708
|
-
// Otherwise we need to request the full details for each change
|
|
709
|
-
this._populateFullChanges[schemaType].requestIds = setChanges.map(change => change.id);
|
|
710
|
-
// Once all the requests are handled the callback will be called
|
|
711
|
-
for (const change of setChanges) {
|
|
712
|
-
// Create a request for each change to populate the full details
|
|
713
|
-
this._eventBusComponent.publish(synchronisedStorageModels.SynchronisedStorageTopics.LocalItemRequest, {
|
|
714
|
-
schemaType,
|
|
715
|
-
id: change.id
|
|
716
|
-
});
|
|
717
|
-
}
|
|
1097
|
+
async buildChangeSet(storageKey, changes, completeCallback) {
|
|
1098
|
+
await this._logging?.log({
|
|
1099
|
+
level: "info",
|
|
1100
|
+
source: this.CLASS_NAME,
|
|
1101
|
+
message: "buildingChangeSet",
|
|
1102
|
+
data: {
|
|
1103
|
+
storageKey,
|
|
1104
|
+
changeCount: changes.length
|
|
718
1105
|
}
|
|
1106
|
+
});
|
|
1107
|
+
this._populateFullChanges[storageKey] = {
|
|
1108
|
+
changes,
|
|
1109
|
+
entities: {},
|
|
1110
|
+
requestIds: [],
|
|
1111
|
+
completeCallback: async () => this.finaliseFullChanges(storageKey, completeCallback)
|
|
1112
|
+
};
|
|
1113
|
+
const setChanges = changes.filter(c => c.operation === synchronisedStorageModels.SyncChangeOperation.Set);
|
|
1114
|
+
if (setChanges.length === 0) {
|
|
1115
|
+
// If we don't need to request any full details, we can just call the complete callback
|
|
1116
|
+
await this.finaliseFullChanges(storageKey, completeCallback);
|
|
719
1117
|
}
|
|
720
1118
|
else {
|
|
721
|
-
|
|
1119
|
+
// Otherwise we need to request the full details for each change
|
|
1120
|
+
this._populateFullChanges[storageKey].requestIds = setChanges.map(change => change.id);
|
|
1121
|
+
// Once all the requests are handled the callback will be called
|
|
1122
|
+
for (const change of setChanges) {
|
|
1123
|
+
// Create a request for each change to populate the full details
|
|
1124
|
+
await this._logging?.log({
|
|
1125
|
+
level: "info",
|
|
1126
|
+
source: this.CLASS_NAME,
|
|
1127
|
+
message: "createChangeSetRequestingItem",
|
|
1128
|
+
data: {
|
|
1129
|
+
storageKey,
|
|
1130
|
+
id: change.id
|
|
1131
|
+
}
|
|
1132
|
+
});
|
|
1133
|
+
this._eventBusComponent.publish(synchronisedStorageModels.SynchronisedStorageTopics.LocalItemRequest, {
|
|
1134
|
+
storageKey,
|
|
1135
|
+
id: change.id
|
|
1136
|
+
});
|
|
1137
|
+
}
|
|
722
1138
|
}
|
|
723
1139
|
}
|
|
724
1140
|
/**
|
|
725
1141
|
* Finalise the full details for the sync change set.
|
|
726
|
-
* @param
|
|
1142
|
+
* @param storageKey The storage key of the change set.
|
|
727
1143
|
* @param completeCallback The callback to call when the changeset is populated.
|
|
728
1144
|
* @returns Nothing.
|
|
729
1145
|
*/
|
|
730
|
-
async finaliseFullChanges(
|
|
1146
|
+
async finaliseFullChanges(storageKey, completeCallback) {
|
|
1147
|
+
await this._logging?.log({
|
|
1148
|
+
level: "info",
|
|
1149
|
+
source: this.CLASS_NAME,
|
|
1150
|
+
message: "finalisingSyncChanges",
|
|
1151
|
+
data: {
|
|
1152
|
+
storageKey
|
|
1153
|
+
}
|
|
1154
|
+
});
|
|
731
1155
|
if (core.Is.stringValue(this._nodeIdentity)) {
|
|
732
|
-
const changes = this._populateFullChanges[
|
|
1156
|
+
const changes = this._populateFullChanges[storageKey].changes;
|
|
733
1157
|
for (const change of changes) {
|
|
734
|
-
change.entity = this._populateFullChanges[
|
|
1158
|
+
change.entity = this._populateFullChanges[storageKey].entities[change.id] ?? change.entity;
|
|
735
1159
|
if (change.operation === synchronisedStorageModels.SyncChangeOperation.Set && core.Is.objectValue(change.entity)) {
|
|
1160
|
+
// Remove the id from the entity as this is stored in the operation
|
|
1161
|
+
// and will be reinstated when the changeset is reconstituted
|
|
1162
|
+
core.ObjectHelper.propertyDelete(change.entity, "id");
|
|
736
1163
|
// Remove the node identity as the changeset has this stored at the top level
|
|
737
1164
|
// and we do not want to store it in the change itself to reduce redundancy
|
|
738
1165
|
core.ObjectHelper.propertyDelete(change.entity, "nodeIdentity");
|
|
739
1166
|
}
|
|
740
1167
|
}
|
|
741
|
-
// Add the changeset to the current snapshot
|
|
742
1168
|
const syncChangeSet = {
|
|
743
1169
|
id: core.Converter.bytesToHex(core.RandomHelper.generate(32)),
|
|
744
1170
|
dateCreated: new Date(Date.now()).toISOString(),
|
|
745
|
-
|
|
1171
|
+
storageKey,
|
|
746
1172
|
changes,
|
|
747
1173
|
nodeIdentity: this._nodeIdentity
|
|
748
1174
|
};
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
1175
|
+
try {
|
|
1176
|
+
// And sign it with the node identity
|
|
1177
|
+
syncChangeSet.proof = await this._changeSetHelper.createChangeSetProof(syncChangeSet);
|
|
1178
|
+
// If this is a trusted node, we also store the changeset
|
|
1179
|
+
let changeSetStorageId;
|
|
1180
|
+
if (this._isTrustedNode) {
|
|
1181
|
+
changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
|
|
1182
|
+
}
|
|
1183
|
+
await completeCallback(syncChangeSet, changeSetStorageId);
|
|
1184
|
+
}
|
|
1185
|
+
catch (err) {
|
|
1186
|
+
await this._logging?.log({
|
|
1187
|
+
level: "error",
|
|
1188
|
+
source: this.CLASS_NAME,
|
|
1189
|
+
message: "finalisingSyncChangesFailed",
|
|
1190
|
+
data: {
|
|
1191
|
+
storageKey
|
|
1192
|
+
},
|
|
1193
|
+
error: core.BaseError.fromError(err)
|
|
1194
|
+
});
|
|
1195
|
+
await completeCallback();
|
|
1196
|
+
}
|
|
754
1197
|
}
|
|
755
1198
|
else {
|
|
756
1199
|
await completeCallback();
|
|
@@ -758,122 +1201,138 @@ class RemoteSyncStateHelper {
|
|
|
758
1201
|
}
|
|
759
1202
|
/**
|
|
760
1203
|
* Add a new changeset into the sync state.
|
|
1204
|
+
* @param storageKey The storage key of the change set to add.
|
|
761
1205
|
* @param changeSetStorageId The id of the change set to add the the current state
|
|
762
1206
|
* @returns Nothing.
|
|
763
1207
|
*/
|
|
764
|
-
async addChangeSetToSyncState(changeSetStorageId) {
|
|
765
|
-
|
|
766
|
-
|
|
1208
|
+
async addChangeSetToSyncState(storageKey, changeSetStorageId) {
|
|
1209
|
+
await this._logging?.log({
|
|
1210
|
+
level: "info",
|
|
1211
|
+
source: this.CLASS_NAME,
|
|
1212
|
+
message: "addChangeSetToSyncState",
|
|
1213
|
+
data: {
|
|
1214
|
+
storageKey,
|
|
1215
|
+
changeSetStorageId
|
|
1216
|
+
}
|
|
1217
|
+
});
|
|
1218
|
+
// First load the sync pointer store to get the current sync pointer for the storage key
|
|
1219
|
+
const syncPointerStore = await this.getVerifiableSyncPointerStore();
|
|
767
1220
|
let syncState;
|
|
768
|
-
if (!core.Is.empty(
|
|
769
|
-
syncState = await this.getRemoteSyncState(
|
|
1221
|
+
if (!core.Is.empty(syncPointerStore.syncPointers[storageKey])) {
|
|
1222
|
+
syncState = await this.getRemoteSyncState(syncPointerStore.syncPointers[storageKey]);
|
|
770
1223
|
}
|
|
771
1224
|
// No current sync state, so we create a new one
|
|
772
1225
|
if (core.Is.empty(syncState)) {
|
|
773
|
-
syncState = { snapshots: [] };
|
|
1226
|
+
syncState = { version: SYNC_STATE_VERSION, snapshots: [] };
|
|
774
1227
|
}
|
|
775
1228
|
// Sort the snapshots so the newest snapshot is last in the array
|
|
776
1229
|
const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
|
|
777
1230
|
// Get the current snapshot, if it does not exist we create a new one
|
|
778
1231
|
let currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
|
|
1232
|
+
const now = new Date(Date.now()).toISOString();
|
|
779
1233
|
if (core.Is.empty(currentSnapshot)) {
|
|
780
1234
|
currentSnapshot = {
|
|
1235
|
+
version: SYNC_SNAPSHOT_VERSION,
|
|
781
1236
|
id: core.Converter.bytesToHex(core.RandomHelper.generate(32)),
|
|
782
|
-
dateCreated:
|
|
1237
|
+
dateCreated: now,
|
|
1238
|
+
dateModified: now,
|
|
783
1239
|
changeSetStorageIds: []
|
|
784
1240
|
};
|
|
785
1241
|
syncState.snapshots.push(currentSnapshot);
|
|
786
1242
|
}
|
|
787
1243
|
else {
|
|
788
1244
|
// Snapshot exists, we update the dateModified
|
|
789
|
-
currentSnapshot.dateModified =
|
|
1245
|
+
currentSnapshot.dateModified = now;
|
|
790
1246
|
}
|
|
791
1247
|
// Add the changeset storage id to the current snapshot
|
|
792
1248
|
currentSnapshot.changeSetStorageIds.push(changeSetStorageId);
|
|
793
1249
|
// Store the sync state in the blob storage
|
|
794
|
-
|
|
795
|
-
// Store the verifiable sync pointer in the verifiable storage
|
|
796
|
-
await this.
|
|
1250
|
+
syncPointerStore.syncPointers[storageKey] = await this.storeRemoteSyncState(syncState);
|
|
1251
|
+
// Store the verifiable sync pointer store in the verifiable storage
|
|
1252
|
+
await this.storeVerifiableSyncPointerStore(syncPointerStore);
|
|
797
1253
|
}
|
|
798
1254
|
/**
|
|
799
1255
|
* Create a consolidated snapshot for the entire storage.
|
|
800
|
-
* @param
|
|
1256
|
+
* @param storageKey The storage key of the snapshot to create.
|
|
801
1257
|
* @param batchSize The batch size to use for consolidation.
|
|
802
1258
|
* @returns Nothing.
|
|
803
1259
|
*/
|
|
804
|
-
async
|
|
1260
|
+
async consolidationStart(storageKey, batchSize) {
|
|
805
1261
|
await this._logging?.log({
|
|
806
1262
|
level: "info",
|
|
807
1263
|
source: this.CLASS_NAME,
|
|
808
1264
|
message: "consolidationStarting"
|
|
809
1265
|
});
|
|
810
|
-
|
|
1266
|
+
// Perform a batch request to start the consolidation
|
|
1267
|
+
await this._eventBusComponent.publish(synchronisedStorageModels.SynchronisedStorageTopics.BatchRequest, { storageKey, batchSize });
|
|
811
1268
|
}
|
|
812
1269
|
/**
|
|
813
|
-
* Get the sync pointer.
|
|
814
|
-
* @returns The sync pointer.
|
|
1270
|
+
* Get the sync pointer store.
|
|
1271
|
+
* @returns The sync pointer store.
|
|
815
1272
|
*/
|
|
816
|
-
async
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
level: "info",
|
|
820
|
-
source: this.CLASS_NAME,
|
|
821
|
-
message: "verifiableSyncPointerRetrieving",
|
|
822
|
-
data: {
|
|
823
|
-
key: this._synchronisedStorageKey
|
|
824
|
-
}
|
|
825
|
-
});
|
|
826
|
-
const syncPointerStore = await this._verifiableSyncPointerStorageConnector.get(this._synchronisedStorageKey, { includeData: true });
|
|
827
|
-
if (core.Is.uint8Array(syncPointerStore.data)) {
|
|
828
|
-
const syncPointer = core.ObjectHelper.fromBytes(syncPointerStore.data);
|
|
1273
|
+
async getVerifiableSyncPointerStore() {
|
|
1274
|
+
if (core.Is.stringValue(this._synchronisedStorageKey)) {
|
|
1275
|
+
try {
|
|
829
1276
|
await this._logging?.log({
|
|
830
1277
|
level: "info",
|
|
831
1278
|
source: this.CLASS_NAME,
|
|
832
|
-
message: "
|
|
1279
|
+
message: "verifiableSyncPointerStoreRetrieving",
|
|
833
1280
|
data: {
|
|
834
|
-
key: this._synchronisedStorageKey
|
|
835
|
-
syncPointerId: syncPointer.syncPointerId
|
|
1281
|
+
key: this._synchronisedStorageKey
|
|
836
1282
|
}
|
|
837
1283
|
});
|
|
838
|
-
|
|
1284
|
+
const syncPointerStore = await this._verifiableSyncPointerStorageConnector.get(this._synchronisedStorageKey, { includeData: true });
|
|
1285
|
+
if (core.Is.uint8Array(syncPointerStore.data)) {
|
|
1286
|
+
const syncPointer = core.ObjectHelper.fromBytes(syncPointerStore.data);
|
|
1287
|
+
await this._logging?.log({
|
|
1288
|
+
level: "info",
|
|
1289
|
+
source: this.CLASS_NAME,
|
|
1290
|
+
message: "verifiableSyncPointerStoreRetrieved",
|
|
1291
|
+
data: {
|
|
1292
|
+
key: this._synchronisedStorageKey
|
|
1293
|
+
}
|
|
1294
|
+
});
|
|
1295
|
+
return syncPointer;
|
|
1296
|
+
}
|
|
839
1297
|
}
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
1298
|
+
catch (err) {
|
|
1299
|
+
if (!core.BaseError.someErrorName(err, core.NotFoundError.CLASS_NAME)) {
|
|
1300
|
+
throw err;
|
|
1301
|
+
}
|
|
844
1302
|
}
|
|
1303
|
+
await this._logging?.log({
|
|
1304
|
+
level: "info",
|
|
1305
|
+
source: this.CLASS_NAME,
|
|
1306
|
+
message: "verifiableSyncPointerStoreNotFound",
|
|
1307
|
+
data: {
|
|
1308
|
+
key: this._synchronisedStorageKey
|
|
1309
|
+
}
|
|
1310
|
+
});
|
|
845
1311
|
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
key: this._synchronisedStorageKey
|
|
852
|
-
}
|
|
853
|
-
});
|
|
1312
|
+
// If no sync pointer store exists, we return an empty one
|
|
1313
|
+
return {
|
|
1314
|
+
version: SYNC_POINTER_STORE_VERSION,
|
|
1315
|
+
syncPointers: {}
|
|
1316
|
+
};
|
|
854
1317
|
}
|
|
855
1318
|
/**
|
|
856
1319
|
* Store the verifiable sync pointer in the verifiable storage.
|
|
857
|
-
* @param
|
|
1320
|
+
* @param syncPointerStore The sync pointer store to store.
|
|
858
1321
|
* @returns Nothing.
|
|
859
1322
|
*/
|
|
860
|
-
async
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
});
|
|
874
|
-
// Store the verifiable sync pointer in the verifiable storage
|
|
875
|
-
await this._verifiableSyncPointerStorageConnector.create(this._synchronisedStorageKey, core.ObjectHelper.toBytes(verifiableSyncPointer));
|
|
876
|
-
return verifiableSyncPointer;
|
|
1323
|
+
async storeVerifiableSyncPointerStore(syncPointerStore) {
|
|
1324
|
+
if (core.Is.stringValue(this._nodeIdentity) && core.Is.stringValue(this._synchronisedStorageKey)) {
|
|
1325
|
+
await this._logging?.log({
|
|
1326
|
+
level: "info",
|
|
1327
|
+
source: this.CLASS_NAME,
|
|
1328
|
+
message: "verifiableSyncPointerStoreStoring",
|
|
1329
|
+
data: {
|
|
1330
|
+
key: this._synchronisedStorageKey
|
|
1331
|
+
}
|
|
1332
|
+
});
|
|
1333
|
+
// Store the verifiable sync pointer in the verifiable storage
|
|
1334
|
+
await this._verifiableSyncPointerStorageConnector.update(this._nodeIdentity, this._synchronisedStorageKey, core.ObjectHelper.toBytes(syncPointerStore));
|
|
1335
|
+
}
|
|
877
1336
|
}
|
|
878
1337
|
/**
|
|
879
1338
|
* Store the remote sync state.
|
|
@@ -889,9 +1348,7 @@ class RemoteSyncStateHelper {
|
|
|
889
1348
|
snapshotCount: syncState.snapshots.length
|
|
890
1349
|
}
|
|
891
1350
|
});
|
|
892
|
-
|
|
893
|
-
// the blob storage also needs to be publicly accessible so that other nodes can retrieve it
|
|
894
|
-
return this._blobStorageComponent.create(core.Converter.bytesToBase64(core.ObjectHelper.toBytes(syncState)), undefined, undefined, undefined, { disableEncryption: true, compress: blobStorageModels.BlobStorageCompressionType.Gzip });
|
|
1351
|
+
return this._blobStorageHelper.saveBlob(syncState);
|
|
895
1352
|
}
|
|
896
1353
|
/**
|
|
897
1354
|
* Get the remote sync state.
|
|
@@ -908,11 +1365,8 @@ class RemoteSyncStateHelper {
|
|
|
908
1365
|
syncPointerId
|
|
909
1366
|
}
|
|
910
1367
|
});
|
|
911
|
-
const
|
|
912
|
-
|
|
913
|
-
});
|
|
914
|
-
if (core.Is.stringBase64(blobEntry.blob)) {
|
|
915
|
-
const syncState = core.ObjectHelper.fromBytes(core.Converter.base64ToBytes(blobEntry.blob));
|
|
1368
|
+
const syncState = await this._blobStorageHelper.load(syncPointerId);
|
|
1369
|
+
if (core.Is.object(syncState)) {
|
|
916
1370
|
await this._logging?.log({
|
|
917
1371
|
level: "info",
|
|
918
1372
|
source: this.CLASS_NAME,
|
|
@@ -925,10 +1379,16 @@ class RemoteSyncStateHelper {
|
|
|
925
1379
|
return syncState;
|
|
926
1380
|
}
|
|
927
1381
|
}
|
|
928
|
-
catch (
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
1382
|
+
catch (error) {
|
|
1383
|
+
await this._logging?.log({
|
|
1384
|
+
level: "warn",
|
|
1385
|
+
source: this.CLASS_NAME,
|
|
1386
|
+
message: "getSyncStateError",
|
|
1387
|
+
data: {
|
|
1388
|
+
syncPointerId
|
|
1389
|
+
},
|
|
1390
|
+
error: core.BaseError.fromError(error)
|
|
1391
|
+
});
|
|
932
1392
|
}
|
|
933
1393
|
await this._logging?.log({
|
|
934
1394
|
level: "info",
|
|
@@ -940,20 +1400,22 @@ class RemoteSyncStateHelper {
|
|
|
940
1400
|
});
|
|
941
1401
|
}
|
|
942
1402
|
/**
|
|
943
|
-
* Handle the batch response.
|
|
1403
|
+
* Handle the batch response which is triggered from a consolidation request.
|
|
944
1404
|
* @param response The batch response to handle.
|
|
945
1405
|
*/
|
|
946
1406
|
async handleBatchResponse(response) {
|
|
947
1407
|
if (core.Is.stringValue(this._nodeIdentity)) {
|
|
1408
|
+
const now = new Date(Date.now()).toISOString();
|
|
948
1409
|
// Create a new snapshot entry for the current batch
|
|
949
1410
|
const syncChangeSet = {
|
|
950
1411
|
id: core.Converter.bytesToHex(core.RandomHelper.generate(32)),
|
|
951
|
-
dateCreated:
|
|
1412
|
+
dateCreated: now,
|
|
1413
|
+
dateModified: now,
|
|
952
1414
|
changes: response.entities.map(change => ({
|
|
953
1415
|
operation: synchronisedStorageModels.SyncChangeOperation.Set,
|
|
954
|
-
id: change
|
|
1416
|
+
id: change.id
|
|
955
1417
|
})),
|
|
956
|
-
|
|
1418
|
+
storageKey: response.storageKey,
|
|
957
1419
|
nodeIdentity: this._nodeIdentity
|
|
958
1420
|
};
|
|
959
1421
|
// And sign it with the node identity
|
|
@@ -961,20 +1423,35 @@ class RemoteSyncStateHelper {
|
|
|
961
1423
|
// Store the changeset in the blob storage
|
|
962
1424
|
const changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
|
|
963
1425
|
// Add the changeset storage id to the snapshot ids
|
|
964
|
-
this._batchResponseStorageIds[response.
|
|
965
|
-
this._batchResponseStorageIds[response.
|
|
1426
|
+
this._batchResponseStorageIds[response.storageKey] ??= [];
|
|
1427
|
+
this._batchResponseStorageIds[response.storageKey].push(changeSetStorageId);
|
|
1428
|
+
// If this is the last entry in the batch response, we can create the consolidated snapshot
|
|
966
1429
|
if (response.lastEntry) {
|
|
967
|
-
|
|
1430
|
+
// Get the current sync pointer store
|
|
1431
|
+
const syncPointerStore = await this.getVerifiableSyncPointerStore();
|
|
1432
|
+
let syncState;
|
|
1433
|
+
if (core.Is.stringValue(syncPointerStore.syncPointers[response.storageKey])) {
|
|
1434
|
+
// If the sync pointer exists, we load the current sync state
|
|
1435
|
+
syncState = await this.getRemoteSyncState(syncPointerStore.syncPointers[response.storageKey]);
|
|
1436
|
+
}
|
|
1437
|
+
// If the sync state does not exist, we create a new one
|
|
1438
|
+
syncState ??= { version: SYNC_STATE_VERSION, snapshots: [] };
|
|
968
1439
|
const batchSnapshot = {
|
|
1440
|
+
version: SYNC_SNAPSHOT_VERSION,
|
|
969
1441
|
id: core.Converter.bytesToHex(core.RandomHelper.generate(32)),
|
|
970
|
-
dateCreated:
|
|
971
|
-
|
|
1442
|
+
dateCreated: now,
|
|
1443
|
+
dateModified: now,
|
|
1444
|
+
changeSetStorageIds: this._batchResponseStorageIds[response.storageKey]
|
|
972
1445
|
};
|
|
973
1446
|
syncState.snapshots.push(batchSnapshot);
|
|
974
1447
|
// Store the sync state in the blob storage
|
|
975
1448
|
const syncStateId = await this.storeRemoteSyncState(syncState);
|
|
1449
|
+
syncPointerStore.syncPointers[response.storageKey] = syncStateId;
|
|
976
1450
|
// Store the verifiable sync pointer in the verifiable storage
|
|
977
|
-
await this.
|
|
1451
|
+
await this.storeVerifiableSyncPointerStore(syncPointerStore);
|
|
1452
|
+
// Remove the batch response storage ids for the storage key
|
|
1453
|
+
// as we have consolidated the changes
|
|
1454
|
+
delete this._batchResponseStorageIds[response.storageKey];
|
|
978
1455
|
await this._logging?.log({
|
|
979
1456
|
level: "info",
|
|
980
1457
|
source: this.CLASS_NAME,
|
|
@@ -988,13 +1465,22 @@ class RemoteSyncStateHelper {
|
|
|
988
1465
|
* @param response The item response to handle.
|
|
989
1466
|
*/
|
|
990
1467
|
async handleLocalItemResponse(response) {
|
|
991
|
-
|
|
992
|
-
|
|
1468
|
+
await this._logging?.log({
|
|
1469
|
+
level: "info",
|
|
1470
|
+
source: this.CLASS_NAME,
|
|
1471
|
+
message: "createChangeSetRespondingItem",
|
|
1472
|
+
data: {
|
|
1473
|
+
storageKey: response.storageKey,
|
|
1474
|
+
id: response.id
|
|
1475
|
+
}
|
|
1476
|
+
});
|
|
1477
|
+
if (!core.Is.empty(this._populateFullChanges[response.storageKey])) {
|
|
1478
|
+
const idx = this._populateFullChanges[response.storageKey].requestIds.indexOf(response.id);
|
|
993
1479
|
if (idx !== -1) {
|
|
994
|
-
this._populateFullChanges[response.
|
|
995
|
-
this._populateFullChanges[response.
|
|
996
|
-
if (this._populateFullChanges[response.
|
|
997
|
-
await this._populateFullChanges[response.
|
|
1480
|
+
this._populateFullChanges[response.storageKey].requestIds.splice(idx, 1);
|
|
1481
|
+
this._populateFullChanges[response.storageKey].entities[response.id] = response.entity;
|
|
1482
|
+
if (this._populateFullChanges[response.storageKey].requestIds.length === 0) {
|
|
1483
|
+
await this._populateFullChanges[response.storageKey].completeCallback();
|
|
998
1484
|
}
|
|
999
1485
|
}
|
|
1000
1486
|
}
|
|
@@ -1006,20 +1492,20 @@ class RemoteSyncStateHelper {
|
|
|
1006
1492
|
*/
|
|
1007
1493
|
class SynchronisedStorageService {
|
|
1008
1494
|
/**
|
|
1009
|
-
* The default interval to check for entity updates
|
|
1495
|
+
* The default interval to check for entity updates.
|
|
1010
1496
|
* @internal
|
|
1011
1497
|
*/
|
|
1012
|
-
static
|
|
1498
|
+
static _DEFAULT_ENTITY_UPDATE_INTERVAL_MINUTES = 5;
|
|
1013
1499
|
/**
|
|
1014
|
-
* The default interval to perform consolidation
|
|
1500
|
+
* The default interval to perform consolidation.
|
|
1015
1501
|
* @internal
|
|
1016
1502
|
*/
|
|
1017
|
-
static
|
|
1503
|
+
static _DEFAULT_CONSOLIDATION_INTERVAL_MINUTES = 60;
|
|
1018
1504
|
/**
|
|
1019
1505
|
* The default size of a consolidation batch.
|
|
1020
1506
|
* @internal
|
|
1021
1507
|
*/
|
|
1022
|
-
static _DEFAULT_CONSOLIDATION_BATCH_SIZE =
|
|
1508
|
+
static _DEFAULT_CONSOLIDATION_BATCH_SIZE = 100;
|
|
1023
1509
|
/**
|
|
1024
1510
|
* Runtime name for the class.
|
|
1025
1511
|
*/
|
|
@@ -1034,16 +1520,21 @@ class SynchronisedStorageService {
|
|
|
1034
1520
|
* @internal
|
|
1035
1521
|
*/
|
|
1036
1522
|
_eventBusComponent;
|
|
1523
|
+
/**
|
|
1524
|
+
* The vault connector.
|
|
1525
|
+
* @internal
|
|
1526
|
+
*/
|
|
1527
|
+
_vaultConnector;
|
|
1037
1528
|
/**
|
|
1038
1529
|
* The storage connector for the sync snapshot entries.
|
|
1039
1530
|
* @internal
|
|
1040
1531
|
*/
|
|
1041
1532
|
_localSyncSnapshotEntryEntityStorage;
|
|
1042
1533
|
/**
|
|
1043
|
-
* The blob storage
|
|
1534
|
+
* The blob storage connector to use for remote sync states.
|
|
1044
1535
|
* @internal
|
|
1045
1536
|
*/
|
|
1046
|
-
|
|
1537
|
+
_blobStorageConnector;
|
|
1047
1538
|
/**
|
|
1048
1539
|
* The verifiable storage connector to use for storing sync pointers.
|
|
1049
1540
|
* @internal
|
|
@@ -1054,11 +1545,21 @@ class SynchronisedStorageService {
|
|
|
1054
1545
|
* @internal
|
|
1055
1546
|
*/
|
|
1056
1547
|
_identityConnector;
|
|
1548
|
+
/**
|
|
1549
|
+
* The task scheduler component.
|
|
1550
|
+
* @internal
|
|
1551
|
+
*/
|
|
1552
|
+
_taskSchedulerComponent;
|
|
1057
1553
|
/**
|
|
1058
1554
|
* The synchronised storage service to use when this is not a trusted node.
|
|
1059
1555
|
* @internal
|
|
1060
1556
|
*/
|
|
1061
1557
|
_trustedSynchronisedStorageComponent;
|
|
1558
|
+
/**
|
|
1559
|
+
* The blob storage helper.
|
|
1560
|
+
* @internal
|
|
1561
|
+
*/
|
|
1562
|
+
_blobStorageHelper;
|
|
1062
1563
|
/**
|
|
1063
1564
|
* The change set helper.
|
|
1064
1565
|
* @internal
|
|
@@ -1080,15 +1581,20 @@ class SynchronisedStorageService {
|
|
|
1080
1581
|
*/
|
|
1081
1582
|
_config;
|
|
1082
1583
|
/**
|
|
1083
|
-
* The
|
|
1584
|
+
* The synchronised storage key to use for the remote synchronised storage.
|
|
1084
1585
|
* @internal
|
|
1085
1586
|
*/
|
|
1086
|
-
|
|
1587
|
+
_synchronisedStorageKey;
|
|
1087
1588
|
/**
|
|
1088
|
-
* The
|
|
1589
|
+
* The flag to determine if the service has been started.
|
|
1089
1590
|
* @internal
|
|
1090
1591
|
*/
|
|
1091
|
-
|
|
1592
|
+
_serviceStarted;
|
|
1593
|
+
/**
|
|
1594
|
+
* The active storage keys for the synchronised storage service.
|
|
1595
|
+
* @internal
|
|
1596
|
+
*/
|
|
1597
|
+
_activeStorageKeys;
|
|
1092
1598
|
/**
|
|
1093
1599
|
* The identity of the node this connector is running on.
|
|
1094
1600
|
* @internal
|
|
@@ -1103,21 +1609,26 @@ class SynchronisedStorageService {
|
|
|
1103
1609
|
core.Guards.object(this.CLASS_NAME, "options.config", options.config);
|
|
1104
1610
|
this._eventBusComponent = core.ComponentFactory.get(options.eventBusComponentType ?? "event-bus");
|
|
1105
1611
|
this._logging = loggingModels.LoggingConnectorFactory.getIfExists(options.loggingConnectorType ?? "logging");
|
|
1612
|
+
this._vaultConnector = vaultModels.VaultConnectorFactory.get(options.vaultConnectorType ?? "vault");
|
|
1106
1613
|
this._localSyncSnapshotEntryEntityStorage = entityStorageModels.EntityStorageConnectorFactory.get(options.syncSnapshotStorageConnectorType ?? "sync-snapshot-entry");
|
|
1107
1614
|
this._verifiableSyncPointerStorageConnector = verifiableStorageModels.VerifiableStorageConnectorFactory.get(options.verifiableStorageConnectorType ?? "verifiable-storage");
|
|
1108
|
-
this.
|
|
1615
|
+
this._blobStorageConnector = blobStorageModels.BlobStorageConnectorFactory.get(options.blobStorageConnectorType ?? "blob-storage");
|
|
1109
1616
|
this._identityConnector = identityModels.IdentityConnectorFactory.get(options.identityConnectorType ?? "identity");
|
|
1617
|
+
this._taskSchedulerComponent = core.ComponentFactory.get(options.taskSchedulerComponentType ?? "task-scheduler");
|
|
1110
1618
|
this._config = {
|
|
1111
|
-
synchronisedStorageKey: options.config.synchronisedStorageKey,
|
|
1112
1619
|
synchronisedStorageMethodId: options.config.synchronisedStorageMethodId ?? "synchronised-storage-assertion",
|
|
1113
|
-
|
|
1114
|
-
SynchronisedStorageService.
|
|
1620
|
+
entityUpdateIntervalMinutes: options.config.entityUpdateIntervalMinutes ??
|
|
1621
|
+
SynchronisedStorageService._DEFAULT_ENTITY_UPDATE_INTERVAL_MINUTES,
|
|
1115
1622
|
isTrustedNode: options.config.isTrustedNode ?? false,
|
|
1116
|
-
|
|
1117
|
-
SynchronisedStorageService.
|
|
1623
|
+
consolidationIntervalMinutes: options.config.consolidationIntervalMinutes ??
|
|
1624
|
+
SynchronisedStorageService._DEFAULT_CONSOLIDATION_INTERVAL_MINUTES,
|
|
1118
1625
|
consolidationBatchSize: options.config.consolidationBatchSize ??
|
|
1119
|
-
SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE
|
|
1626
|
+
SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE,
|
|
1627
|
+
blobStorageEncryptionKeyId: options.config.blobStorageEncryptionKeyId ?? "synchronised-storage-blob-encryption-key",
|
|
1628
|
+
verifiableStorageKeyId: options.config.verifiableStorageKeyId
|
|
1120
1629
|
};
|
|
1630
|
+
this._synchronisedStorageKey =
|
|
1631
|
+
verifiableStorageKeys[options.config.verifiableStorageKeyId] ?? options.config.verifiableStorageKeyId;
|
|
1121
1632
|
// If this is not a trusted node, we need to use a synchronised storage service
|
|
1122
1633
|
// to synchronise with a trusted node.
|
|
1123
1634
|
if (!this._config.isTrustedNode) {
|
|
@@ -1125,13 +1636,14 @@ class SynchronisedStorageService {
|
|
|
1125
1636
|
this._trustedSynchronisedStorageComponent =
|
|
1126
1637
|
core.ComponentFactory.get(options.trustedSynchronisedStorageComponentType);
|
|
1127
1638
|
}
|
|
1128
|
-
this.
|
|
1639
|
+
this._blobStorageHelper = new BlobStorageHelper(this._logging, this._vaultConnector, this._blobStorageConnector, this._config.blobStorageEncryptionKeyId, this._config.isTrustedNode);
|
|
1640
|
+
this._changeSetHelper = new ChangeSetHelper(this._logging, this._eventBusComponent, this._identityConnector, this._blobStorageHelper, this._config.synchronisedStorageMethodId);
|
|
1129
1641
|
this._localSyncStateHelper = new LocalSyncStateHelper(this._logging, this._localSyncSnapshotEntryEntityStorage, this._changeSetHelper);
|
|
1130
|
-
this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._logging, this._eventBusComponent, this.
|
|
1131
|
-
this.
|
|
1132
|
-
this.
|
|
1133
|
-
this._eventBusComponent.subscribe(synchronisedStorageModels.SynchronisedStorageTopics.
|
|
1134
|
-
this._eventBusComponent.subscribe(synchronisedStorageModels.SynchronisedStorageTopics.LocalItemChange, async (event) => this._localSyncStateHelper.addLocalChange(event.data.
|
|
1642
|
+
this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._logging, this._eventBusComponent, this._verifiableSyncPointerStorageConnector, this._blobStorageHelper, this._changeSetHelper, this._config.isTrustedNode);
|
|
1643
|
+
this._serviceStarted = false;
|
|
1644
|
+
this._activeStorageKeys = {};
|
|
1645
|
+
this._eventBusComponent.subscribe(synchronisedStorageModels.SynchronisedStorageTopics.RegisterStorageKey, async (event) => this.registerStorageKey(event.data));
|
|
1646
|
+
this._eventBusComponent.subscribe(synchronisedStorageModels.SynchronisedStorageTopics.LocalItemChange, async (event) => this._localSyncStateHelper.addLocalChange(event.data.storageKey, event.data.operation, event.data.id));
|
|
1135
1647
|
}
|
|
1136
1648
|
/**
|
|
1137
1649
|
* The component needs to be started when the node is initialized.
|
|
@@ -1143,6 +1655,20 @@ class SynchronisedStorageService {
|
|
|
1143
1655
|
async start(nodeIdentity, nodeLoggingConnectorType, componentState) {
|
|
1144
1656
|
this._nodeIdentity = nodeIdentity;
|
|
1145
1657
|
this._remoteSyncStateHelper.setNodeIdentity(nodeIdentity);
|
|
1658
|
+
this._changeSetHelper.setNodeIdentity(nodeIdentity);
|
|
1659
|
+
this._remoteSyncStateHelper.setSynchronisedStorageKey(this._synchronisedStorageKey);
|
|
1660
|
+
this._serviceStarted = true;
|
|
1661
|
+
// If this is not a trusted node we need to request the decryption key from a trusted node
|
|
1662
|
+
if (!this._config.isTrustedNode && !core.Is.empty(this._trustedSynchronisedStorageComponent)) {
|
|
1663
|
+
const proof = await this._identityConnector.createProof(this._nodeIdentity, identityModels.DocumentHelper.joinId(this._nodeIdentity, this._config.synchronisedStorageMethodId), standardsW3cDid.ProofTypes.DataIntegrityProof, { nodeIdentity });
|
|
1664
|
+
const decryptionKey = await this._trustedSynchronisedStorageComponent.getDecryptionKey(this._nodeIdentity, proof);
|
|
1665
|
+
// We don't have the private key so instead we store the key as a secret in the vault
|
|
1666
|
+
await this._vaultConnector.setSecret(this._config.blobStorageEncryptionKeyId, decryptionKey);
|
|
1667
|
+
}
|
|
1668
|
+
// If there are already storage keys registered, we need to activate them
|
|
1669
|
+
for (const storageKey in this._activeStorageKeys) {
|
|
1670
|
+
await this.activateStorageKey(storageKey);
|
|
1671
|
+
}
|
|
1146
1672
|
}
|
|
1147
1673
|
/**
|
|
1148
1674
|
* The component needs to be stopped when the node is closed.
|
|
@@ -1152,48 +1678,88 @@ class SynchronisedStorageService {
|
|
|
1152
1678
|
* @returns Nothing.
|
|
1153
1679
|
*/
|
|
1154
1680
|
async stop(nodeIdentity, nodeLoggingConnectorType, componentState) {
|
|
1155
|
-
for (const
|
|
1156
|
-
|
|
1157
|
-
|
|
1681
|
+
for (const storageKey in this._activeStorageKeys) {
|
|
1682
|
+
this._activeStorageKeys[storageKey] = false;
|
|
1683
|
+
this._taskSchedulerComponent.removeTask(`synchronised-storage-update-${storageKey}`);
|
|
1684
|
+
this._taskSchedulerComponent.removeTask(`synchronised-storage-consolidation-${storageKey}`);
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
/**
|
|
1688
|
+
* Get the decryption key for the synchronised storage.
|
|
1689
|
+
* This is used to decrypt the data stored in the synchronised storage.
|
|
1690
|
+
* @param nodeIdentity The identity of the node requesting the decryption key.
|
|
1691
|
+
* @param proof The proof of the request so we know the request is from the specified node.
|
|
1692
|
+
* @returns The decryption key.
|
|
1693
|
+
*/
|
|
1694
|
+
async getDecryptionKey(nodeIdentity, proof) {
|
|
1695
|
+
if (!this._config.isTrustedNode) {
|
|
1696
|
+
throw new core.GeneralError(this.CLASS_NAME, "notTrustedNode");
|
|
1158
1697
|
}
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1698
|
+
core.Guards.stringValue(this.CLASS_NAME, "nodeIdentity", nodeIdentity);
|
|
1699
|
+
core.Guards.object(this.CLASS_NAME, "proof", proof);
|
|
1700
|
+
const isValid = await this._identityConnector.verifyProof({ nodeIdentity }, proof);
|
|
1701
|
+
if (!isValid) {
|
|
1702
|
+
throw new core.UnauthorizedError(this.CLASS_NAME, "invalidProof");
|
|
1162
1703
|
}
|
|
1704
|
+
// TODO: We need to check if the node has permissions to access the decryption key
|
|
1705
|
+
// using rights-management
|
|
1706
|
+
const key = await this._vaultConnector.getKey(this._config.blobStorageEncryptionKeyId);
|
|
1707
|
+
if (core.Is.undefined(key.publicKey)) {
|
|
1708
|
+
throw new core.UnauthorizedError(this.CLASS_NAME, "decryptionKeyNotFound");
|
|
1709
|
+
}
|
|
1710
|
+
return core.Converter.bytesToBase64(key.publicKey);
|
|
1163
1711
|
}
|
|
1164
1712
|
/**
|
|
1165
|
-
* Synchronise a
|
|
1166
|
-
* @param
|
|
1713
|
+
* Synchronise a set of changes from an untrusted node, assumes this is a trusted node.
|
|
1714
|
+
* @param syncChangeSet The change set to synchronise.
|
|
1167
1715
|
* @returns Nothing.
|
|
1168
1716
|
*/
|
|
1169
|
-
async syncChangeSet(
|
|
1717
|
+
async syncChangeSet(syncChangeSet) {
|
|
1170
1718
|
if (!this._config.isTrustedNode) {
|
|
1171
1719
|
throw new core.GeneralError(this.CLASS_NAME, "notTrustedNode");
|
|
1172
1720
|
}
|
|
1173
|
-
|
|
1174
|
-
|
|
1721
|
+
core.Guards.object(this.CLASS_NAME, "syncChangeSet", syncChangeSet);
|
|
1722
|
+
await this._logging?.log({
|
|
1723
|
+
level: "info",
|
|
1724
|
+
source: this.CLASS_NAME,
|
|
1725
|
+
message: "syncChangeSetForRemoteNode",
|
|
1726
|
+
data: {
|
|
1727
|
+
changeSetStorageId: syncChangeSet.id
|
|
1728
|
+
}
|
|
1729
|
+
});
|
|
1175
1730
|
// TODO: The change set has a proof signed by the originating node identity
|
|
1176
1731
|
// The proof is verified that the change set is valid and has not been tampered with.
|
|
1177
1732
|
// but we also need to check that the originating node has permissions
|
|
1178
1733
|
// to store the change set in the synchronised storage.
|
|
1179
1734
|
// This will be performed using rights-management
|
|
1180
|
-
const
|
|
1181
|
-
if (!core.Is.empty(
|
|
1182
|
-
|
|
1735
|
+
const copy = await this._changeSetHelper.copyChangeset(syncChangeSet);
|
|
1736
|
+
if (!core.Is.empty(copy) && core.Is.stringValue(this._nodeIdentity)) {
|
|
1737
|
+
// Apply the changes to this node
|
|
1738
|
+
await this._changeSetHelper.applyChangeset(copy.syncChangeSet);
|
|
1739
|
+
// And update the sync state with the latest changes
|
|
1740
|
+
await this._remoteSyncStateHelper.addChangeSetToSyncState(copy.syncChangeSet.storageKey, copy.changeSetStorageId);
|
|
1183
1741
|
}
|
|
1184
1742
|
}
|
|
1185
1743
|
/**
|
|
1186
1744
|
* Start the sync with further updates after an interval.
|
|
1187
|
-
* @param
|
|
1745
|
+
* @param storageKey The storage key to sync.
|
|
1188
1746
|
* @returns Nothing.
|
|
1189
1747
|
* @internal
|
|
1190
1748
|
*/
|
|
1191
|
-
async startEntitySync(
|
|
1749
|
+
async startEntitySync(storageKey) {
|
|
1192
1750
|
try {
|
|
1751
|
+
await this._logging?.log({
|
|
1752
|
+
level: "info",
|
|
1753
|
+
source: this.CLASS_NAME,
|
|
1754
|
+
message: "startEntitySync",
|
|
1755
|
+
data: {
|
|
1756
|
+
storageKey
|
|
1757
|
+
}
|
|
1758
|
+
});
|
|
1193
1759
|
// First we check for remote changes
|
|
1194
|
-
await this.updateFromRemoteSyncState(
|
|
1760
|
+
await this.updateFromRemoteSyncState(storageKey);
|
|
1195
1761
|
// Now send any updates we have to the remote storage
|
|
1196
|
-
await this.updateFromLocalSyncState(
|
|
1762
|
+
await this.updateFromLocalSyncState(storageKey);
|
|
1197
1763
|
}
|
|
1198
1764
|
catch (error) {
|
|
1199
1765
|
await this._logging?.log({
|
|
@@ -1203,27 +1769,31 @@ class SynchronisedStorageService {
|
|
|
1203
1769
|
error: core.BaseError.fromError(error)
|
|
1204
1770
|
});
|
|
1205
1771
|
}
|
|
1206
|
-
finally {
|
|
1207
|
-
// Set a timer to check for updates again
|
|
1208
|
-
this._entityUpdateTimers[schemaType] = setTimeout(async () => this.startEntitySync(schemaType), this._config.entityUpdateIntervalMs);
|
|
1209
|
-
}
|
|
1210
1772
|
}
|
|
1211
1773
|
/**
|
|
1212
1774
|
* Check for updates in the remote storage.
|
|
1213
|
-
* @param
|
|
1775
|
+
* @param storageKey The storage key to check for updates.
|
|
1214
1776
|
* @returns Nothing.
|
|
1215
1777
|
* @internal
|
|
1216
1778
|
*/
|
|
1217
|
-
async updateFromRemoteSyncState(
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1779
|
+
async updateFromRemoteSyncState(storageKey) {
|
|
1780
|
+
await this._logging?.log({
|
|
1781
|
+
level: "info",
|
|
1782
|
+
source: this.CLASS_NAME,
|
|
1783
|
+
message: "updateFromRemoteSyncState",
|
|
1784
|
+
data: {
|
|
1785
|
+
storageKey
|
|
1786
|
+
}
|
|
1787
|
+
});
|
|
1788
|
+
// Get the verifiable sync pointer store from the verifiable storage
|
|
1789
|
+
const verifiableSyncPointerStore = await this._remoteSyncStateHelper.getVerifiableSyncPointerStore();
|
|
1790
|
+
if (!core.Is.empty(verifiableSyncPointerStore.syncPointers[storageKey])) {
|
|
1221
1791
|
// Load the sync state from the remote blob storage using the sync pointer
|
|
1222
1792
|
// to load the sync state
|
|
1223
|
-
const remoteSyncState = await this._remoteSyncStateHelper.getRemoteSyncState(
|
|
1793
|
+
const remoteSyncState = await this._remoteSyncStateHelper.getRemoteSyncState(verifiableSyncPointerStore.syncPointers[storageKey]);
|
|
1224
1794
|
// If we got the sync state we can try and sync from it
|
|
1225
1795
|
if (!core.Is.undefined(remoteSyncState)) {
|
|
1226
|
-
await this._localSyncStateHelper.
|
|
1796
|
+
await this._localSyncStateHelper.applySyncState(storageKey, remoteSyncState);
|
|
1227
1797
|
}
|
|
1228
1798
|
}
|
|
1229
1799
|
}
|
|
@@ -1232,47 +1802,94 @@ class SynchronisedStorageService {
|
|
|
1232
1802
|
* @returns Nothing.
|
|
1233
1803
|
* @internal
|
|
1234
1804
|
*/
|
|
1235
|
-
async updateFromLocalSyncState(
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1805
|
+
async updateFromLocalSyncState(storageKey) {
|
|
1806
|
+
await this._logging?.log({
|
|
1807
|
+
level: "info",
|
|
1808
|
+
source: this.CLASS_NAME,
|
|
1809
|
+
message: "updateFromLocalSyncState",
|
|
1810
|
+
data: {
|
|
1811
|
+
storageKey
|
|
1812
|
+
}
|
|
1813
|
+
});
|
|
1814
|
+
const localChangeSnapshot = await this._localSyncStateHelper.getLocalChangeSnapshot(storageKey);
|
|
1815
|
+
if (core.Is.arrayValue(localChangeSnapshot.changes)) {
|
|
1816
|
+
await this._remoteSyncStateHelper.buildChangeSet(storageKey, localChangeSnapshot.changes, async (syncChangeSet, changeSetStorageId) => {
|
|
1817
|
+
if (core.Is.empty(syncChangeSet) && core.Is.empty(changeSetStorageId)) {
|
|
1818
|
+
await this._logging?.log({
|
|
1819
|
+
level: "info",
|
|
1820
|
+
source: this.CLASS_NAME,
|
|
1821
|
+
message: "builtStorageChangeSetNone",
|
|
1822
|
+
data: {
|
|
1823
|
+
storageKey
|
|
1245
1824
|
}
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1825
|
+
});
|
|
1826
|
+
}
|
|
1827
|
+
else {
|
|
1828
|
+
await this._logging?.log({
|
|
1829
|
+
level: "info",
|
|
1830
|
+
source: this.CLASS_NAME,
|
|
1831
|
+
message: "builtStorageChangeSet",
|
|
1832
|
+
data: {
|
|
1833
|
+
storageKey,
|
|
1834
|
+
changeSetStorageId
|
|
1249
1835
|
}
|
|
1836
|
+
});
|
|
1837
|
+
// Send the local changes to the remote storage if we are a trusted node
|
|
1838
|
+
if (this._config.isTrustedNode && core.Is.stringValue(changeSetStorageId)) {
|
|
1839
|
+
// If we are a trusted node, we can add the change set to the sync state
|
|
1840
|
+
// and remove the local change snapshot
|
|
1841
|
+
await this._remoteSyncStateHelper.addChangeSetToSyncState(storageKey, changeSetStorageId);
|
|
1250
1842
|
await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
|
|
1251
1843
|
}
|
|
1252
|
-
|
|
1253
|
-
|
|
1844
|
+
else if (!core.Is.empty(this._trustedSynchronisedStorageComponent) &&
|
|
1845
|
+
core.Is.object(syncChangeSet)) {
|
|
1846
|
+
// If we are not a trusted node, we need to send the changes to the trusted node
|
|
1847
|
+
// and then remove the local change snapshot
|
|
1848
|
+
await this._logging?.log({
|
|
1849
|
+
level: "info",
|
|
1850
|
+
source: this.CLASS_NAME,
|
|
1851
|
+
message: "sendingChangeSetToTrustedNode",
|
|
1852
|
+
data: {
|
|
1853
|
+
storageKey,
|
|
1854
|
+
changeSetStorageId
|
|
1855
|
+
}
|
|
1856
|
+
});
|
|
1857
|
+
await this._trustedSynchronisedStorageComponent.syncChangeSet(syncChangeSet);
|
|
1858
|
+
await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
});
|
|
1862
|
+
}
|
|
1863
|
+
else {
|
|
1864
|
+
await this._logging?.log({
|
|
1865
|
+
level: "info",
|
|
1866
|
+
source: this.CLASS_NAME,
|
|
1867
|
+
message: "updateFromLocalSyncStateNoChanges",
|
|
1868
|
+
data: {
|
|
1869
|
+
storageKey
|
|
1870
|
+
}
|
|
1871
|
+
});
|
|
1254
1872
|
}
|
|
1255
1873
|
}
|
|
1256
1874
|
/**
|
|
1257
1875
|
* Start the consolidation sync.
|
|
1258
|
-
* @param
|
|
1876
|
+
* @param storageKey The storage key to consolidate.
|
|
1259
1877
|
* @returns Nothing.
|
|
1260
1878
|
* @internal
|
|
1261
1879
|
*/
|
|
1262
|
-
async startConsolidationSync(
|
|
1880
|
+
async startConsolidationSync(storageKey) {
|
|
1263
1881
|
let localChangeSnapshot;
|
|
1264
1882
|
try {
|
|
1265
|
-
// If we are performing a consolidation, we can remove the local
|
|
1266
|
-
|
|
1883
|
+
// If we are performing a consolidation, we can remove the local change snapshot
|
|
1884
|
+
// as we are going to create a complete changeset from the DB
|
|
1885
|
+
localChangeSnapshot = await this._localSyncStateHelper.getLocalChangeSnapshot(storageKey);
|
|
1267
1886
|
if (!core.Is.empty(localChangeSnapshot)) {
|
|
1268
1887
|
await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
|
|
1269
1888
|
}
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
localChangeSnapshot = undefined;
|
|
1275
|
-
}
|
|
1889
|
+
await this._remoteSyncStateHelper.consolidationStart(storageKey, this._config.consolidationBatchSize ??
|
|
1890
|
+
SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE);
|
|
1891
|
+
// The consolidation was successful, so we can remove the local change snapshot permanently
|
|
1892
|
+
localChangeSnapshot = undefined;
|
|
1276
1893
|
}
|
|
1277
1894
|
catch (error) {
|
|
1278
1895
|
if (localChangeSnapshot) {
|
|
@@ -1286,30 +1903,60 @@ class SynchronisedStorageService {
|
|
|
1286
1903
|
error: core.BaseError.fromError(error)
|
|
1287
1904
|
});
|
|
1288
1905
|
}
|
|
1289
|
-
finally {
|
|
1290
|
-
// Set a timer to perform the consolidation again
|
|
1291
|
-
this._consolidationTimers[schemaType] = setTimeout(async () => this.startConsolidationSync(schemaType), this._config.consolidationIntervalMs);
|
|
1292
|
-
}
|
|
1293
1906
|
}
|
|
1294
1907
|
/**
|
|
1295
1908
|
* Register a new sync type.
|
|
1296
|
-
* @param
|
|
1909
|
+
* @param syncRegisterStorageKey The sync register type to register.
|
|
1297
1910
|
* @internal
|
|
1298
1911
|
*/
|
|
1299
|
-
async
|
|
1912
|
+
async registerStorageKey(syncRegisterStorageKey) {
|
|
1300
1913
|
await this._logging?.log({
|
|
1301
1914
|
level: "info",
|
|
1302
1915
|
source: this.CLASS_NAME,
|
|
1303
|
-
message: "
|
|
1916
|
+
message: "registerStorageKey",
|
|
1304
1917
|
data: {
|
|
1305
|
-
|
|
1918
|
+
storageKey: syncRegisterStorageKey.storageKey
|
|
1306
1919
|
}
|
|
1307
1920
|
});
|
|
1308
|
-
if (this.
|
|
1309
|
-
|
|
1921
|
+
if (core.Is.empty(this._activeStorageKeys[syncRegisterStorageKey.storageKey])) {
|
|
1922
|
+
this._activeStorageKeys[syncRegisterStorageKey.storageKey] = false;
|
|
1923
|
+
if (this._serviceStarted) {
|
|
1924
|
+
await this.activateStorageKey(syncRegisterStorageKey.storageKey);
|
|
1925
|
+
}
|
|
1310
1926
|
}
|
|
1311
|
-
|
|
1312
|
-
|
|
1927
|
+
}
|
|
1928
|
+
/**
|
|
1929
|
+
* Activate a storage key.
|
|
1930
|
+
* @param storageKey The storage key to activate.
|
|
1931
|
+
* @internal
|
|
1932
|
+
*/
|
|
1933
|
+
async activateStorageKey(storageKey) {
|
|
1934
|
+
if (!core.Is.empty(this._activeStorageKeys[storageKey]) && !this._activeStorageKeys[storageKey]) {
|
|
1935
|
+
await this._logging?.log({
|
|
1936
|
+
level: "info",
|
|
1937
|
+
source: this.CLASS_NAME,
|
|
1938
|
+
message: "activateStorageKey",
|
|
1939
|
+
data: {
|
|
1940
|
+
storageKey
|
|
1941
|
+
}
|
|
1942
|
+
});
|
|
1943
|
+
this._activeStorageKeys[storageKey] = true;
|
|
1944
|
+
if (this._config.entityUpdateIntervalMinutes > 0) {
|
|
1945
|
+
await this._taskSchedulerComponent.addTask(`synchronised-storage-update-${storageKey}`, [
|
|
1946
|
+
{
|
|
1947
|
+
nextTriggerTime: Date.now(),
|
|
1948
|
+
intervalMinutes: this._config.entityUpdateIntervalMinutes
|
|
1949
|
+
}
|
|
1950
|
+
], async () => this.startEntitySync(storageKey));
|
|
1951
|
+
}
|
|
1952
|
+
if (this._config.isTrustedNode && this._config.consolidationIntervalMinutes > 0) {
|
|
1953
|
+
await this._taskSchedulerComponent.addTask(`synchronised-storage-consolidation-${storageKey}`, [
|
|
1954
|
+
{
|
|
1955
|
+
nextTriggerTime: Date.now(),
|
|
1956
|
+
intervalMinutes: this._config.consolidationIntervalMinutes
|
|
1957
|
+
}
|
|
1958
|
+
], async () => this.startConsolidationSync(storageKey));
|
|
1959
|
+
}
|
|
1313
1960
|
}
|
|
1314
1961
|
}
|
|
1315
1962
|
}
|
|
@@ -1318,5 +1965,6 @@ exports.SynchronisedStorageService = SynchronisedStorageService;
|
|
|
1318
1965
|
exports.generateRestRoutesSynchronisedStorage = generateRestRoutesSynchronisedStorage;
|
|
1319
1966
|
exports.initSchema = initSchema;
|
|
1320
1967
|
exports.restEntryPoints = restEntryPoints;
|
|
1968
|
+
exports.synchronisedStorageGetDecryptionKeyRequest = synchronisedStorageGetDecryptionKeyRequest;
|
|
1321
1969
|
exports.synchronisedStorageSyncChangeSetRequest = synchronisedStorageSyncChangeSetRequest;
|
|
1322
1970
|
exports.tagsSynchronisedStorage = tagsSynchronisedStorage;
|