@twin.org/synchronised-storage-service 0.0.1-next.2 → 0.0.1-next.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/cjs/index.cjs +962 -314
  2. package/dist/esm/index.mjs +963 -316
  3. package/dist/types/entities/syncSnapshotEntry.d.ts +4 -5
  4. package/dist/types/helpers/blobStorageHelper.d.ts +33 -0
  5. package/dist/types/helpers/changeSetHelper.d.ts +20 -7
  6. package/dist/types/helpers/localSyncStateHelper.d.ts +13 -28
  7. package/dist/types/helpers/remoteSyncStateHelper.d.ts +26 -21
  8. package/dist/types/helpers/versions.d.ts +3 -0
  9. package/dist/types/index.d.ts +4 -2
  10. package/dist/types/models/ISyncPointerStore.d.ts +15 -0
  11. package/dist/types/models/ISyncSnapshot.d.ts +5 -1
  12. package/dist/types/models/ISyncState.d.ts +4 -0
  13. package/dist/types/models/ISynchronisedStorageServiceConfig.d.ts +17 -11
  14. package/dist/types/models/ISynchronisedStorageServiceConstructorOptions.d.ts +11 -2
  15. package/dist/types/synchronisedStorageRoutes.d.ts +9 -1
  16. package/dist/types/synchronisedStorageService.d.ts +13 -4
  17. package/docs/architecture.md +125 -0
  18. package/docs/changelog.md +29 -0
  19. package/docs/open-api/spec.json +244 -18
  20. package/docs/reference/classes/SyncSnapshotEntry.md +5 -5
  21. package/docs/reference/classes/SynchronisedStorageService.md +38 -5
  22. package/docs/reference/functions/synchronisedStorageGetDecryptionKeyRequest.md +31 -0
  23. package/docs/reference/index.md +4 -1
  24. package/docs/reference/interfaces/ISyncPointerStore.md +23 -0
  25. package/docs/reference/interfaces/ISyncSnapshot.md +43 -0
  26. package/docs/reference/interfaces/ISyncState.md +19 -0
  27. package/docs/reference/interfaces/ISynchronisedStorageServiceConfig.md +37 -17
  28. package/docs/reference/interfaces/ISynchronisedStorageServiceConstructorOptions.md +25 -3
  29. package/locales/en.json +62 -14
  30. package/package.json +4 -2
  31. package/dist/types/models/ISyncChange.d.ts +0 -18
  32. package/dist/types/models/ISyncChangeSet.d.ts +0 -36
  33. package/dist/types/models/ISyncPointer.d.ts +0 -9
  34. package/docs/reference/interfaces/ISyncChange.md +0 -33
@@ -1,13 +1,15 @@
1
1
  import { property, entity, EntitySchemaFactory, EntitySchemaHelper, ComparisonOperator } from '@twin.org/entity';
2
- import { Guards, ComponentFactory, Is, ObjectHelper, Converter, RandomHelper, BaseError, NotFoundError, GeneralError } from '@twin.org/core';
2
+ import { Guards, ComponentFactory, Is, Converter, Compression, CompressionType, ObjectHelper, BaseError, GeneralError, RandomHelper, NotFoundError, UnauthorizedError } from '@twin.org/core';
3
3
  import { HttpStatusCode } from '@twin.org/web';
4
+ import { BlobStorageConnectorFactory } from '@twin.org/blob-storage-models';
4
5
  import { EntityStorageConnectorFactory } from '@twin.org/entity-storage-models';
5
6
  import { DocumentHelper, IdentityConnectorFactory } from '@twin.org/identity-models';
6
7
  import { LoggingConnectorFactory } from '@twin.org/logging-models';
8
+ import { ProofTypes } from '@twin.org/standards-w3c-did';
7
9
  import { SyncChangeOperation, SynchronisedStorageTopics } from '@twin.org/synchronised-storage-models';
10
+ import { VaultEncryptionType, VaultConnectorFactory } from '@twin.org/vault-models';
8
11
  import { VerifiableStorageConnectorFactory } from '@twin.org/verifiable-storage-models';
9
- import { BlobStorageCompressionType } from '@twin.org/blob-storage-models';
10
- import { ProofTypes } from '@twin.org/standards-w3c-did';
12
+ import { RSA } from '@twin.org/crypto';
11
13
 
12
14
  // Copyright 2024 IOTA Stiftung.
13
15
  // SPDX-License-Identifier: Apache-2.0.
@@ -20,9 +22,9 @@ let SyncSnapshotEntry = class SyncSnapshotEntry {
20
22
  */
21
23
  id;
22
24
  /**
23
- * The schema type for the snapshot i.e. which entity is being synchronized.
25
+ * The storage key for the snapshot i.e. which entity is being synchronized.
24
26
  */
25
- schemaType;
27
+ storageKey;
26
28
  /**
27
29
  * The date the snapshot was created.
28
30
  */
@@ -42,7 +44,7 @@ let SyncSnapshotEntry = class SyncSnapshotEntry {
42
44
  /**
43
45
  * The changes that were made in this snapshot, if this is a local snapshot.
44
46
  */
45
- localChanges;
47
+ changes;
46
48
  };
47
49
  __decorate([
48
50
  property({ type: "string", isPrimary: true }),
@@ -51,7 +53,7 @@ __decorate([
51
53
  __decorate([
52
54
  property({ type: "string", isSecondary: true }),
53
55
  __metadata("design:type", String)
54
- ], SyncSnapshotEntry.prototype, "schemaType", void 0);
56
+ ], SyncSnapshotEntry.prototype, "storageKey", void 0);
55
57
  __decorate([
56
58
  property({ type: "string" }),
57
59
  __metadata("design:type", String)
@@ -71,7 +73,7 @@ __decorate([
71
73
  __decorate([
72
74
  property({ type: "array", itemType: "object", optional: true }),
73
75
  __metadata("design:type", Array)
74
- ], SyncSnapshotEntry.prototype, "localChanges", void 0);
76
+ ], SyncSnapshotEntry.prototype, "changes", void 0);
75
77
  SyncSnapshotEntry = __decorate([
76
78
  entity()
77
79
  ], SyncSnapshotEntry);
@@ -100,8 +102,8 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
100
102
  operationId: "synchronisedStorageSyncChangeSetRequest",
101
103
  summary: "Request that the node perform a sync request for a changeset.",
102
104
  tag: tagsSynchronisedStorage[0].name,
103
- method: "GET",
104
- path: `${baseRouteName}/`,
105
+ method: "POST",
106
+ path: `${baseRouteName}/sync-changeset`,
105
107
  handler: async (httpRequestContext, request) => synchronisedStorageSyncChangeSetRequest(httpRequestContext, componentName, request),
106
108
  requestType: {
107
109
  type: "ISyncChangeSetRequest",
@@ -109,8 +111,29 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
109
111
  {
110
112
  id: "synchronisedStorageSyncChangeSetRequestExample",
111
113
  request: {
112
- query: {
113
- changeSetStorageId: "12345"
114
+ body: {
115
+ id: "0909090909090909090909090909090909090909090909090909090909090909",
116
+ dateCreated: "2025-05-29T01:00:00.000Z",
117
+ nodeIdentity: "did:entity-storage:0xd2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2",
118
+ changes: [
119
+ {
120
+ entity: {
121
+ dateModified: "2025-01-01T00:00:00.000Z"
122
+ },
123
+ id: "test-id-1",
124
+ operation: "set"
125
+ }
126
+ ],
127
+ proof: {
128
+ "@context": "https://www.w3.org/ns/credentials/v2",
129
+ created: "2025-05-29T01:00:00.000Z",
130
+ cryptosuite: "eddsa-jcs-2022",
131
+ proofPurpose: "assertionMethod",
132
+ proofValue: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3",
133
+ type: "DataIntegrityProof",
134
+ verificationMethod: "did:entity-storage:0xd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0#synchronised-storage-assertion"
135
+ },
136
+ storageKey: "test-type"
114
137
  }
115
138
  }
116
139
  }
@@ -120,9 +143,61 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
120
143
  {
121
144
  type: "INoContentResponse"
122
145
  }
123
- ]
146
+ ],
147
+ // Authentication is provided by the proof in the request body.
148
+ skipAuth: true
124
149
  };
125
- return [syncChangeSetRoute];
150
+ const getDecryptionKeyRoute = {
151
+ operationId: "synchronisedStorageGetDecryptionKeyRequest",
152
+ summary: "Request the decryption key.",
153
+ tag: tagsSynchronisedStorage[0].name,
154
+ method: "POST",
155
+ path: `${baseRouteName}/decryption-key`,
156
+ handler: async (httpRequestContext, request) => synchronisedStorageGetDecryptionKeyRequest(httpRequestContext, componentName, request),
157
+ requestType: {
158
+ type: "ISyncChangeSetRequest",
159
+ examples: [
160
+ {
161
+ id: "synchronisedStorageSyncGetDecryptionKeyRequestExample",
162
+ request: {
163
+ body: {
164
+ nodeIdentity: "did:entity-storage:0xd2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2",
165
+ proof: {
166
+ "@context": "https://www.w3.org/ns/credentials/v2",
167
+ created: "2025-05-29T01:00:00.000Z",
168
+ cryptosuite: "eddsa-jcs-2022",
169
+ proofPurpose: "assertionMethod",
170
+ proofValue: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3",
171
+ type: "DataIntegrityProof",
172
+ verificationMethod: "did:entity-storage:0xd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0#synchronised-storage-assertion"
173
+ }
174
+ }
175
+ }
176
+ }
177
+ ]
178
+ },
179
+ responseType: [
180
+ {
181
+ type: "ISyncDecryptionKeyResponse",
182
+ examples: [
183
+ {
184
+ id: "synchronisedStorageSyncGetDecryptionKeyResponseExample",
185
+ response: {
186
+ body: {
187
+ decryptionKey: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3"
188
+ }
189
+ }
190
+ }
191
+ ]
192
+ },
193
+ {
194
+ type: "IUnauthorizedResponse"
195
+ }
196
+ ],
197
+ // Authentication is provided by the proof in the request body.
198
+ skipAuth: true
199
+ };
200
+ return [syncChangeSetRoute, getDecryptionKeyRoute];
126
201
  }
127
202
  /**
128
203
  * Perform the sync change set operation.
@@ -133,13 +208,31 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
133
208
  */
134
209
  async function synchronisedStorageSyncChangeSetRequest(httpRequestContext, componentName, request) {
135
210
  Guards.object(ROUTES_SOURCE, "request", request);
136
- Guards.object(ROUTES_SOURCE, "request.query", request.query);
211
+ Guards.object(ROUTES_SOURCE, "request.body", request.body);
137
212
  const component = ComponentFactory.get(componentName);
138
- await component.syncChangeSet(request.query.changeSetStorageId);
213
+ await component.syncChangeSet(request.body);
139
214
  return {
140
215
  statusCode: HttpStatusCode.noContent
141
216
  };
142
217
  }
218
+ /**
219
+ * Request the decryption key.
220
+ * @param httpRequestContext The request context for the API.
221
+ * @param componentName The name of the component to use in the routes.
222
+ * @param request The request.
223
+ * @returns The response object with additional http response properties.
224
+ */
225
+ async function synchronisedStorageGetDecryptionKeyRequest(httpRequestContext, componentName, request) {
226
+ Guards.object(ROUTES_SOURCE, "request", request);
227
+ Guards.object(ROUTES_SOURCE, "request.body", request.body);
228
+ const component = ComponentFactory.get(componentName);
229
+ const key = await component.getDecryptionKey(request.body.nodeIdentity, request.body.proof);
230
+ return {
231
+ body: {
232
+ decryptionKey: key
233
+ }
234
+ };
235
+ }
143
236
 
144
237
  const restEntryPoints = [
145
238
  {
@@ -159,6 +252,163 @@ function initSchema() {
159
252
  EntitySchemaFactory.register("SyncSnapshotEntry", () => EntitySchemaHelper.getSchema(SyncSnapshotEntry));
160
253
  }
161
254
 
255
+ var mainnet = "";
256
+ var testnet = "";
257
+ var devnet = "";
258
+ var verifiableStorageKeys = {
259
+ mainnet: mainnet,
260
+ testnet: testnet,
261
+ devnet: devnet
262
+ };
263
+
264
+ /**
265
+ * Class for performing blob storage operations.
266
+ */
267
+ class BlobStorageHelper {
268
+ /**
269
+ * Runtime name for the class.
270
+ */
271
+ CLASS_NAME = "BlobStorageHelper";
272
+ /**
273
+ * The logging connector to use for logging.
274
+ * @internal
275
+ */
276
+ _logging;
277
+ /**
278
+ * The vault connector.
279
+ * @internal
280
+ */
281
+ _vaultConnector;
282
+ /**
283
+ * The blob storage connector to use.
284
+ * @internal
285
+ */
286
+ _blobStorageConnector;
287
+ /**
288
+ * The id of the vault key to use for encrypting/decrypting blobs.
289
+ * @internal
290
+ */
291
+ _blobStorageEncryptionKeyId;
292
+ /**
293
+ * Is this a trusted node.
294
+ * @internal
295
+ */
296
+ _isTrustedNode;
297
+ /**
298
+ * Create a new instance of BlobStorageHelper.
299
+ * @param logging The logging connector to use for logging.
300
+ * @param vaultConnector The vault connector to use for for the encryption key.
301
+ * @param blobStorageConnector The blob storage component to use.
302
+ * @param blobStorageEncryptionKeyId The id of the vault key to use for encrypting/decrypting blobs.
303
+ * @param isTrustedNode Is this a trusted node.
304
+ */
305
+ constructor(logging, vaultConnector, blobStorageConnector, blobStorageEncryptionKeyId, isTrustedNode) {
306
+ this._logging = logging;
307
+ this._vaultConnector = vaultConnector;
308
+ this._blobStorageConnector = blobStorageConnector;
309
+ this._blobStorageEncryptionKeyId = blobStorageEncryptionKeyId;
310
+ this._isTrustedNode = isTrustedNode;
311
+ }
312
+ /**
313
+ * Load a blob from storage.
314
+ * @param blobId The id of the blob to apply.
315
+ * @returns The blob.
316
+ */
317
+ async load(blobId) {
318
+ await this._logging?.log({
319
+ level: "info",
320
+ source: this.CLASS_NAME,
321
+ message: "loadBlob",
322
+ data: {
323
+ blobId
324
+ }
325
+ });
326
+ try {
327
+ const encryptedBlob = await this._blobStorageConnector.get(blobId);
328
+ if (Is.uint8Array(encryptedBlob)) {
329
+ let compressedBlob;
330
+ // If this is a trusted node, we can decrypt the blob using the vault
331
+ if (this._isTrustedNode) {
332
+ compressedBlob = await this._vaultConnector.decrypt(this._blobStorageEncryptionKeyId, VaultEncryptionType.Rsa2048, encryptedBlob);
333
+ }
334
+ else {
335
+ // Otherwise we need the public key stored as a secret in the vault
336
+ const key = await this._vaultConnector.getSecret(this._blobStorageEncryptionKeyId);
337
+ const rsa = new RSA(Converter.base64ToBytes(key));
338
+ compressedBlob = rsa.decrypt(encryptedBlob);
339
+ }
340
+ const decompressedBlob = await Compression.decompress(compressedBlob, CompressionType.Gzip);
341
+ await this._logging?.log({
342
+ level: "info",
343
+ source: this.CLASS_NAME,
344
+ message: "loadedBlob",
345
+ data: {
346
+ blobId
347
+ }
348
+ });
349
+ return ObjectHelper.fromBytes(decompressedBlob);
350
+ }
351
+ }
352
+ catch (error) {
353
+ await this._logging?.log({
354
+ level: "error",
355
+ source: this.CLASS_NAME,
356
+ message: "loadBlobFailed",
357
+ data: {
358
+ blobId
359
+ },
360
+ error: BaseError.fromError(error)
361
+ });
362
+ }
363
+ await this._logging?.log({
364
+ level: "info",
365
+ source: this.CLASS_NAME,
366
+ message: "loadBlobEmpty",
367
+ data: {
368
+ blobId
369
+ }
370
+ });
371
+ }
372
+ /**
373
+ * Save a blob.
374
+ * @param blob The blob to save.
375
+ * @returns The id of the blob.
376
+ */
377
+ async saveBlob(blob) {
378
+ await this._logging?.log({
379
+ level: "info",
380
+ source: this.CLASS_NAME,
381
+ message: "saveBlob"
382
+ });
383
+ if (!this._isTrustedNode) {
384
+ throw new GeneralError(this.CLASS_NAME, "notTrustedNode");
385
+ }
386
+ const compressedBlob = await Compression.compress(ObjectHelper.toBytes(blob), CompressionType.Gzip);
387
+ const encryptedBlob = await this._vaultConnector.encrypt(this._blobStorageEncryptionKeyId, VaultEncryptionType.Rsa2048, compressedBlob);
388
+ try {
389
+ const blobId = await this._blobStorageConnector.set(encryptedBlob);
390
+ await this._logging?.log({
391
+ level: "info",
392
+ source: this.CLASS_NAME,
393
+ message: "savedBlob",
394
+ data: {
395
+ blobId
396
+ }
397
+ });
398
+ return blobId;
399
+ }
400
+ catch (error) {
401
+ await this._logging?.log({
402
+ level: "error",
403
+ source: this.CLASS_NAME,
404
+ message: "saveBlobFailed",
405
+ error: BaseError.fromError(error)
406
+ });
407
+ throw error;
408
+ }
409
+ }
410
+ }
411
+
162
412
  // Copyright 2024 IOTA Stiftung.
163
413
  // SPDX-License-Identifier: Apache-2.0.
164
414
  /**
@@ -180,10 +430,10 @@ class ChangeSetHelper {
180
430
  */
181
431
  _eventBusComponent;
182
432
  /**
183
- * The blob storage component to use for remote sync states.
433
+ * The blob storage helper to use for remote sync states.
184
434
  * @internal
185
435
  */
186
- _blobStorageComponent;
436
+ _blobStorageHelper;
187
437
  /**
188
438
  * The identity connector to use for signing/verifying changesets.
189
439
  * @internal
@@ -194,50 +444,85 @@ class ChangeSetHelper {
194
444
  * @internal
195
445
  */
196
446
  _decentralisedStorageMethodId;
447
+ /**
448
+ * The identity of the node that is performing the update.
449
+ * @internal
450
+ */
451
+ _nodeIdentity;
197
452
  /**
198
453
  * Create a new instance of ChangeSetHelper.
199
454
  * @param logging The logging connector to use for logging.
200
455
  * @param eventBusComponent The event bus component to use for events.
201
- * @param blobStorageComponent The blob storage component to use for remote sync states.
202
456
  * @param identityConnector The identity connector to use for signing/verifying changesets.
457
+ * @param blobStorageHelper The blob storage component to use for remote sync states.
203
458
  * @param decentralisedStorageMethodId The id of the identity method to use when signing/verifying changesets.
204
459
  */
205
- constructor(logging, eventBusComponent, blobStorageComponent, identityConnector, decentralisedStorageMethodId) {
460
+ constructor(logging, eventBusComponent, identityConnector, blobStorageHelper, decentralisedStorageMethodId) {
206
461
  this._logging = logging;
207
462
  this._eventBusComponent = eventBusComponent;
208
463
  this._decentralisedStorageMethodId = decentralisedStorageMethodId;
209
- this._blobStorageComponent = blobStorageComponent;
464
+ this._blobStorageHelper = blobStorageHelper;
210
465
  this._identityConnector = identityConnector;
211
466
  }
467
+ /**
468
+ * Set the node identity to use for signing changesets.
469
+ * @param nodeIdentity The identity of the node that is performing the update.
470
+ */
471
+ setNodeIdentity(nodeIdentity) {
472
+ this._nodeIdentity = nodeIdentity;
473
+ }
212
474
  /**
213
475
  * Get and verify a changeset.
214
476
  * @param changeSetStorageId The id of the sync changeset to apply.
215
477
  * @returns The changeset if it was verified.
216
478
  */
217
479
  async getAndVerifyChangeset(changeSetStorageId) {
218
- // Changesets are not encrypted as they are signed with the node identity
219
- // and they are publicly accessible so that other nodes can retrieve them.
220
- const blobEntry = await this._blobStorageComponent.get(changeSetStorageId, {
221
- includeContent: true
480
+ await this._logging?.log({
481
+ level: "info",
482
+ source: this.CLASS_NAME,
483
+ message: "getChangeSet",
484
+ data: {
485
+ changeSetStorageId
486
+ }
222
487
  });
223
- if (Is.stringBase64(blobEntry.blob)) {
224
- const syncChangeset = ObjectHelper.fromBytes(Converter.base64ToBytes(blobEntry.blob));
225
- const verified = await this.verifyChangesetProof(syncChangeset);
226
- return verified ? syncChangeset : undefined;
488
+ try {
489
+ const syncChangeSet = await this._blobStorageHelper.load(changeSetStorageId);
490
+ if (Is.object(syncChangeSet)) {
491
+ const verified = await this.verifyChangesetProof(syncChangeSet);
492
+ return verified ? syncChangeSet : undefined;
493
+ }
494
+ }
495
+ catch (error) {
496
+ await this._logging?.log({
497
+ level: "warn",
498
+ source: this.CLASS_NAME,
499
+ message: "getChangeSetError",
500
+ data: {
501
+ changeSetStorageId
502
+ },
503
+ error: BaseError.fromError(error)
504
+ });
227
505
  }
506
+ await this._logging?.log({
507
+ level: "info",
508
+ source: this.CLASS_NAME,
509
+ message: "getChangeSetEmpty",
510
+ data: {
511
+ changeSetStorageId
512
+ }
513
+ });
228
514
  }
229
515
  /**
230
516
  * Apply a sync changeset.
231
517
  * @param changeSetStorageId The id of the sync changeset to apply.
232
- * @returns True if the change was applied.
518
+ * @returns The changeset if it existed.
233
519
  */
234
520
  async getAndApplyChangeset(changeSetStorageId) {
235
521
  const syncChangeset = await this.getAndVerifyChangeset(changeSetStorageId);
236
522
  if (!Is.empty(syncChangeset)) {
237
523
  await this.applyChangeset(syncChangeset);
238
- return true;
239
524
  }
240
- return false;
525
+ return syncChangeset;
241
526
  }
242
527
  /**
243
528
  * Apply a sync changeset.
@@ -259,20 +544,25 @@ class ChangeSetHelper {
259
544
  switch (change.operation) {
260
545
  case SyncChangeOperation.Set:
261
546
  if (!Is.empty(change.entity)) {
262
- // The node identity was stripped when stored in the changeset
547
+ // The id was stripped from the entity as it is part of the operation
548
+ // we make sure we reinstate it in the publish
549
+ // Also the node identity was stripped when stored in the changeset
263
550
  // as the changeset is signed with the node identity.
264
551
  // so we need to restore it here.
265
- change.entity.nodeIdentity = syncChangeset.nodeIdentity;
266
552
  await this._eventBusComponent.publish(SynchronisedStorageTopics.RemoteItemSet, {
267
- schemaType: syncChangeset.schemaType,
268
- entity: change.entity
553
+ storageKey: syncChangeset.storageKey,
554
+ entity: {
555
+ ...change.entity,
556
+ id: change.id,
557
+ nodeIdentity: syncChangeset.nodeIdentity
558
+ }
269
559
  });
270
560
  }
271
561
  break;
272
562
  case SyncChangeOperation.Delete:
273
563
  if (!Is.empty(change.id)) {
274
564
  await this._eventBusComponent.publish(SynchronisedStorageTopics.RemoteItemRemove, {
275
- schemaType: syncChangeset.schemaType,
565
+ storageKey: syncChangeset.storageKey,
276
566
  id: change.id
277
567
  });
278
568
  }
@@ -295,12 +585,7 @@ class ChangeSetHelper {
295
585
  id: syncChangeSet.id
296
586
  }
297
587
  });
298
- // We don't want to encrypt the sync state as no other nodes would be able to read it
299
- // the blob storage also needs to be publicly accessible so that other nodes can retrieve it
300
- return this._blobStorageComponent.create(Converter.bytesToBase64(ObjectHelper.toBytes(syncChangeSet)), undefined, undefined, undefined, {
301
- disableEncryption: true,
302
- compress: BlobStorageCompressionType.Gzip
303
- });
588
+ return this._blobStorageHelper.saveBlob(syncChangeSet);
304
589
  }
305
590
  /**
306
591
  * Verify the proof of a sync changeset.
@@ -319,6 +604,32 @@ class ChangeSetHelper {
319
604
  });
320
605
  return false;
321
606
  }
607
+ // If the proof or verification method is missing, the proof is invalid
608
+ const verificationMethod = syncChangeset.proof?.verificationMethod;
609
+ if (!Is.stringValue(verificationMethod)) {
610
+ await this._logging?.log({
611
+ level: "error",
612
+ source: this.CLASS_NAME,
613
+ message: "verifyChangeSetProofMissing",
614
+ data: {
615
+ id: syncChangeset.id
616
+ }
617
+ });
618
+ }
619
+ // Parse the verification method and extract the node identity
620
+ // this should match the node identity of the changeset
621
+ // otherwise you could sign a changeset for another node
622
+ const changeSetNodeIdentity = DocumentHelper.parseId(verificationMethod ?? "");
623
+ if (changeSetNodeIdentity.id !== syncChangeset.nodeIdentity) {
624
+ await this._logging?.log({
625
+ level: "error",
626
+ source: this.CLASS_NAME,
627
+ message: "verifyChangeSetProofNodeIdentityMismatch",
628
+ data: {
629
+ id: syncChangeset.id
630
+ }
631
+ });
632
+ }
322
633
  const changeSetWithoutProof = ObjectHelper.clone(syncChangeset);
323
634
  delete changeSetWithoutProof.proof;
324
635
  const isValid = await this._identityConnector.verifyProof(changeSetWithoutProof, syncChangeset.proof);
@@ -350,11 +661,12 @@ class ChangeSetHelper {
350
661
  * @returns The proof.
351
662
  */
352
663
  async createChangeSetProof(syncChangeset) {
664
+ Guards.stringValue(this.CLASS_NAME, "nodeIdentity", this._nodeIdentity);
353
665
  const changeSetWithoutProof = ObjectHelper.clone(syncChangeset);
354
666
  delete changeSetWithoutProof.proof;
355
- const proof = await this._identityConnector.createProof(syncChangeset.nodeIdentity, DocumentHelper.joinId(syncChangeset.nodeIdentity, this._decentralisedStorageMethodId), ProofTypes.DataIntegrityProof, changeSetWithoutProof);
667
+ const proof = await this._identityConnector.createProof(this._nodeIdentity, DocumentHelper.joinId(this._nodeIdentity, this._decentralisedStorageMethodId), ProofTypes.DataIntegrityProof, changeSetWithoutProof);
356
668
  await this._logging?.log({
357
- level: "error",
669
+ level: "info",
358
670
  source: this.CLASS_NAME,
359
671
  message: "createdChangeSetProof",
360
672
  data: {
@@ -364,6 +676,35 @@ class ChangeSetHelper {
364
676
  });
365
677
  return proof;
366
678
  }
679
+ /**
680
+ * Copy a change set.
681
+ * @param syncChangeSet The sync changeset to copy.
682
+ * @returns The id of the updated change set.
683
+ */
684
+ async copyChangeset(syncChangeSet) {
685
+ if (Is.stringValue(this._nodeIdentity)) {
686
+ const verified = await this.verifyChangesetProof(syncChangeSet);
687
+ if (verified) {
688
+ await this._logging?.log({
689
+ level: "info",
690
+ source: this.CLASS_NAME,
691
+ message: "copyChangeSet",
692
+ data: {
693
+ changeSetStorageId: syncChangeSet.id
694
+ }
695
+ });
696
+ // Allocate a new id to the changeset copy and re-create a proof using this nodes identity
697
+ const copy = ObjectHelper.clone(syncChangeSet);
698
+ copy.id = Converter.bytesToHex(RandomHelper.generate(32));
699
+ copy.proof = await this.createChangeSetProof(copy);
700
+ // Store the copy
701
+ return {
702
+ syncChangeSet: copy,
703
+ changeSetStorageId: await this.storeChangeSet(copy)
704
+ };
705
+ }
706
+ }
707
+ }
367
708
  }
368
709
 
369
710
  // Copyright 2024 IOTA Stiftung.
@@ -385,7 +726,7 @@ class LocalSyncStateHelper {
385
726
  * The storage connector for the sync snapshot entries.
386
727
  * @internal
387
728
  */
388
- _localSyncSnapshotEntryEntityStorage;
729
+ _snapshotEntryEntityStorage;
389
730
  /**
390
731
  * The change set helper to use for applying changesets.
391
732
  * @internal
@@ -394,44 +735,62 @@ class LocalSyncStateHelper {
394
735
  /**
395
736
  * Create a new instance of LocalSyncStateHelper.
396
737
  * @param logging The logging connector to use for logging.
397
- * @param localSyncSnapshotEntryEntityStorage The storage connector for the local sync snapshot entries.
738
+ * @param snapshotEntryEntityStorage The storage connector for the sync snapshot entries.
398
739
  * @param changeSetHelper The change set helper to use for applying changesets.
399
740
  */
400
- constructor(logging, localSyncSnapshotEntryEntityStorage, changeSetHelper) {
741
+ constructor(logging, snapshotEntryEntityStorage, changeSetHelper) {
401
742
  this._logging = logging;
402
- this._localSyncSnapshotEntryEntityStorage = localSyncSnapshotEntryEntityStorage;
743
+ this._snapshotEntryEntityStorage = snapshotEntryEntityStorage;
403
744
  this._changeSetHelper = changeSetHelper;
404
745
  }
405
746
  /**
406
747
  * Add a new change to the local snapshot.
407
- * @param schemaType The schema type of the snapshot to add the change for.
748
+ * @param storageKey The storage key of the snapshot to add the change for.
408
749
  * @param operation The operation to perform.
409
750
  * @param id The id of the entity to add the change for.
410
751
  * @returns Nothing.
411
752
  */
412
- async addLocalChange(schemaType, operation, id) {
413
- const localChangeSnapshot = await this.getLocalChangeSnapshot(schemaType);
414
- localChangeSnapshot.localChanges ??= [];
753
+ async addLocalChange(storageKey, operation, id) {
754
+ await this._logging?.log({
755
+ level: "info",
756
+ source: this.CLASS_NAME,
757
+ message: "addLocalChange",
758
+ data: {
759
+ storageKey,
760
+ operation,
761
+ id
762
+ }
763
+ });
764
+ const localChangeSnapshot = await this.getLocalChangeSnapshot(storageKey);
765
+ localChangeSnapshot.changes ??= [];
415
766
  // If we already have a change for this id we are
416
767
  // about to supersede it, we remove the previous change
417
768
  // to avoid having multiple changes for the same id
418
- const previousChangeIndex = localChangeSnapshot.localChanges.findIndex(change => change.id === id);
769
+ const previousChangeIndex = localChangeSnapshot.changes.findIndex(change => change.id === id);
419
770
  if (previousChangeIndex !== -1) {
420
- localChangeSnapshot.localChanges.splice(previousChangeIndex, 1);
771
+ localChangeSnapshot.changes.splice(previousChangeIndex, 1);
421
772
  }
422
- if (localChangeSnapshot.localChanges.length > 0) {
773
+ if (localChangeSnapshot.changes.length > 0) {
423
774
  localChangeSnapshot.dateModified = new Date(Date.now()).toISOString();
424
775
  }
425
- localChangeSnapshot.localChanges.push({ operation, id });
776
+ localChangeSnapshot.changes.push({ operation, id });
426
777
  await this.setLocalChangeSnapshot(localChangeSnapshot);
427
778
  }
428
779
  /**
429
- * Get the current local snapshot.
430
- * @param schemaType The schema type of the snapshot to get.
780
+ * Get the current local snapshot which contains just the changes for this node.
781
+ * @param storageKey The storage key of the snapshot to get.
431
782
  * @returns The local snapshot entry.
432
783
  */
433
- async getLocalChangeSnapshot(schemaType) {
434
- const queryResult = await this._localSyncSnapshotEntryEntityStorage.query({
784
+ async getLocalChangeSnapshot(storageKey) {
785
+ await this._logging?.log({
786
+ level: "info",
787
+ source: this.CLASS_NAME,
788
+ message: "getLocalChangeSnapshot",
789
+ data: {
790
+ storageKey
791
+ }
792
+ });
793
+ const queryResult = await this._snapshotEntryEntityStorage.query({
435
794
  conditions: [
436
795
  {
437
796
  property: "isLocalSnapshot",
@@ -439,33 +798,57 @@ class LocalSyncStateHelper {
439
798
  comparison: ComparisonOperator.Equals
440
799
  },
441
800
  {
442
- property: "schemaType",
443
- value: schemaType,
801
+ property: "storageKey",
802
+ value: storageKey,
444
803
  comparison: ComparisonOperator.Equals
445
804
  }
446
805
  ]
447
806
  });
448
807
  if (queryResult.entities.length > 0) {
808
+ await this._logging?.log({
809
+ level: "info",
810
+ source: this.CLASS_NAME,
811
+ message: "localChangeSnapshotExists",
812
+ data: {
813
+ storageKey
814
+ }
815
+ });
449
816
  return queryResult.entities[0];
450
817
  }
818
+ await this._logging?.log({
819
+ level: "info",
820
+ source: this.CLASS_NAME,
821
+ message: "localChangeSnapshotDoesNotExist",
822
+ data: {
823
+ storageKey
824
+ }
825
+ });
451
826
  return {
452
827
  id: Converter.bytesToHex(RandomHelper.generate(32)),
453
- schemaType,
828
+ storageKey,
454
829
  dateCreated: new Date(Date.now()).toISOString(),
455
830
  changeSetStorageIds: [],
456
831
  isLocalSnapshot: true
457
832
  };
458
833
  }
459
834
  /**
460
- * Set the current local snapshot.
835
+ * Set the current local snapshot with changes for this node.
461
836
  * @param localChangeSnapshot The local change snapshot to set.
462
837
  * @returns Nothing.
463
838
  */
464
839
  async setLocalChangeSnapshot(localChangeSnapshot) {
465
- await this._localSyncSnapshotEntryEntityStorage.set(localChangeSnapshot);
840
+ await this._logging?.log({
841
+ level: "info",
842
+ source: this.CLASS_NAME,
843
+ message: "setLocalChangeSnapshot",
844
+ data: {
845
+ storageKey: localChangeSnapshot.storageKey
846
+ }
847
+ });
848
+ await this._snapshotEntryEntityStorage.set(localChangeSnapshot);
466
849
  }
467
850
  /**
468
- * Get the current local snapshot.
851
+ * Get the current local snapshot with the changes for this node.
469
852
  * @param localChangeSnapshot The local change snapshot to remove.
470
853
  * @returns Nothing.
471
854
  */
@@ -478,47 +861,47 @@ class LocalSyncStateHelper {
478
861
  snapshotId: localChangeSnapshot.id
479
862
  }
480
863
  });
481
- await this._localSyncSnapshotEntryEntityStorage.remove(localChangeSnapshot.id);
864
+ await this._snapshotEntryEntityStorage.remove(localChangeSnapshot.id);
482
865
  }
483
866
  /**
484
- * Sync local data using a remote sync state.
485
- * @param schemaType The schema type of the snapshot to sync with.
486
- * @param remoteSyncState The sync state to sync with.
867
+ * Apply a sync state to the local node.
868
+ * @param storageKey The storage key of the snapshot to sync with.
869
+ * @param syncState The sync state to sync with.
487
870
  * @returns Nothing.
488
871
  */
489
- async syncFromRemote(schemaType, remoteSyncState) {
872
+ async applySyncState(storageKey, syncState) {
490
873
  await this._logging?.log({
491
874
  level: "info",
492
875
  source: this.CLASS_NAME,
493
- message: "remoteSyncSynchronisation",
876
+ message: "applySyncState",
494
877
  data: {
495
- snapshotCount: remoteSyncState.snapshots.length
878
+ snapshotCount: syncState.snapshots.length
496
879
  }
497
880
  });
498
881
  // Sort from newest to oldest
499
- const sortedRemoteSnapshots = remoteSyncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
882
+ const sortedSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
500
883
  const newSnapshots = [];
501
884
  const modifiedSnapshots = [];
502
- for (const remoteSnapshot of sortedRemoteSnapshots) {
885
+ for (const snapshot of sortedSnapshots) {
503
886
  await this._logging?.log({
504
887
  level: "info",
505
888
  source: this.CLASS_NAME,
506
- message: "remoteSyncSnapshotProcessing",
889
+ message: "applySnapshot",
507
890
  data: {
508
- snapshotId: remoteSnapshot.id,
509
- dateCreated: new Date(remoteSnapshot.dateCreated).toISOString()
891
+ snapshotId: snapshot.id,
892
+ dateCreated: new Date(snapshot.dateCreated).toISOString()
510
893
  }
511
894
  });
512
- const localSnapshot = await this._localSyncSnapshotEntryEntityStorage.get(remoteSnapshot.id);
895
+ const localSnapshot = await this._snapshotEntryEntityStorage.get(snapshot.id);
513
896
  const remoteSnapshotWithContext = {
514
- ...remoteSnapshot,
515
- schemaType
897
+ ...snapshot,
898
+ storageKey
516
899
  };
517
900
  if (Is.empty(localSnapshot)) {
518
901
  // We don't have the snapshot locally, so we need to process it
519
902
  newSnapshots.push(remoteSnapshotWithContext);
520
903
  }
521
- else if (localSnapshot.dateModified !== remoteSnapshot.dateModified) {
904
+ else if (localSnapshot.dateModified !== snapshot.dateModified) {
522
905
  // If the local snapshot has a different dateModified, we need to update it
523
906
  modifiedSnapshots.push({
524
907
  localSnapshot,
@@ -540,13 +923,14 @@ class LocalSyncStateHelper {
540
923
  * Process the modified snapshots and store them in the local storage.
541
924
  * @param modifiedSnapshots The modified snapshots to process.
542
925
  * @returns Nothing.
926
+ * @internal
543
927
  */
544
928
  async processModifiedSnapshots(modifiedSnapshots) {
545
929
  for (const modifiedSnapshot of modifiedSnapshots) {
546
930
  await this._logging?.log({
547
931
  level: "info",
548
932
  source: this.CLASS_NAME,
549
- message: "remoteSyncSnapshotModified",
933
+ message: "processModifiedSnapshot",
550
934
  data: {
551
935
  snapshotId: modifiedSnapshot.remoteSnapshot.id,
552
936
  localModified: new Date(modifiedSnapshot.localSnapshot.dateModified ??
@@ -565,20 +949,21 @@ class LocalSyncStateHelper {
565
949
  }
566
950
  }
567
951
  }
568
- await this._localSyncSnapshotEntryEntityStorage.set(modifiedSnapshot.remoteSnapshot);
952
+ await this._snapshotEntryEntityStorage.set(modifiedSnapshot.remoteSnapshot);
569
953
  }
570
954
  }
571
955
  /**
572
956
  * Process the new snapshots and store them in the local storage.
573
957
  * @param newSnapshots The new snapshots to process.
574
958
  * @returns Nothing.
959
+ * @internal
575
960
  */
576
961
  async processNewSnapshots(newSnapshots) {
577
962
  for (const newSnapshot of newSnapshots) {
578
963
  await this._logging?.log({
579
964
  level: "info",
580
965
  source: this.CLASS_NAME,
581
- message: "remoteSyncSnapshotNew",
966
+ message: "processNewSnapshot",
582
967
  data: {
583
968
  snapshotId: newSnapshot.id,
584
969
  localModified: new Date(newSnapshot.dateCreated).toISOString()
@@ -590,11 +975,17 @@ class LocalSyncStateHelper {
590
975
  await this._changeSetHelper.getAndApplyChangeset(storageId);
591
976
  }
592
977
  }
593
- await this._localSyncSnapshotEntryEntityStorage.set(newSnapshot);
978
+ await this._snapshotEntryEntityStorage.set(newSnapshot);
594
979
  }
595
980
  }
596
981
  }
597
982
 
983
+ // Copyright 2024 IOTA Stiftung.
984
+ // SPDX-License-Identifier: Apache-2.0.
985
+ const SYNC_STATE_VERSION = "1";
986
+ const SYNC_POINTER_STORE_VERSION = "1";
987
+ const SYNC_SNAPSHOT_VERSION = "1";
988
+
598
989
  // Copyright 2024 IOTA Stiftung.
599
990
  // SPDX-License-Identifier: Apache-2.0.
600
991
  /**
@@ -616,10 +1007,10 @@ class RemoteSyncStateHelper {
616
1007
  */
617
1008
  _eventBusComponent;
618
1009
  /**
619
- * The blob storage component to use for remote sync states.
1010
+ * The blob storage helper.
620
1011
  * @internal
621
1012
  */
622
- _blobStorageComponent;
1013
+ _blobStorageHelper;
623
1014
  /**
624
1015
  * The verifiable storage connector to use for storing sync pointers.
625
1016
  * @internal
@@ -631,12 +1022,12 @@ class RemoteSyncStateHelper {
631
1022
  */
632
1023
  _changeSetHelper;
633
1024
  /**
634
- * The storage ids of the batch responses for each schema type.
1025
+ * The storage ids of the batch responses for each storage key.
635
1026
  * @internal
636
1027
  */
637
1028
  _batchResponseStorageIds;
638
1029
  /**
639
- * The full changes for each schema type.
1030
+ * The full changes for each storage key.
640
1031
  * @internal
641
1032
  */
642
1033
  _populateFullChanges;
@@ -650,22 +1041,27 @@ class RemoteSyncStateHelper {
650
1041
  * @internal
651
1042
  */
652
1043
  _nodeIdentity;
1044
+ /**
1045
+ * Whether the node is trusted or not.
1046
+ * @internal
1047
+ */
1048
+ _isTrustedNode;
653
1049
  /**
654
1050
  * Create a new instance of DecentralisedEntityStorageConnector.
655
1051
  * @param logging The logging connector to use for logging.
656
1052
  * @param eventBusComponent The event bus component to use for events.
657
- * @param blobStorageComponent The blob storage component to use for remote sync states.
658
1053
  * @param verifiableSyncPointerStorageConnector The verifiable storage connector to use for storing sync pointers.
1054
+ * @param blobStorageHelper The blob storage helper to use for remote sync states.
659
1055
  * @param changeSetHelper The change set helper to use for managing changesets.
660
- * @param synchronisedStorageKey The synchronised storage key to use for verified storage operations.
1056
+ * @param isTrustedNode Whether the node is trusted or not.
661
1057
  */
662
- constructor(logging, eventBusComponent, blobStorageComponent, verifiableSyncPointerStorageConnector, changeSetHelper, synchronisedStorageKey) {
1058
+ constructor(logging, eventBusComponent, verifiableSyncPointerStorageConnector, blobStorageHelper, changeSetHelper, isTrustedNode) {
663
1059
  this._logging = logging;
664
1060
  this._eventBusComponent = eventBusComponent;
665
- this._blobStorageComponent = blobStorageComponent;
666
1061
  this._verifiableSyncPointerStorageConnector = verifiableSyncPointerStorageConnector;
667
1062
  this._changeSetHelper = changeSetHelper;
668
- this._synchronisedStorageKey = synchronisedStorageKey;
1063
+ this._blobStorageHelper = blobStorageHelper;
1064
+ this._isTrustedNode = isTrustedNode;
669
1065
  this._batchResponseStorageIds = {};
670
1066
  this._populateFullChanges = {};
671
1067
  this._eventBusComponent.subscribe(SynchronisedStorageTopics.BatchResponse, async (response) => {
@@ -683,72 +1079,119 @@ class RemoteSyncStateHelper {
683
1079
  this._nodeIdentity = nodeIdentity;
684
1080
  }
685
1081
  /**
686
- * Create and store a change set.
687
- * @param schemaType The schema type of the change set.
1082
+ * Set the synchronised storage key.
1083
+ * @param synchronisedStorageKey The synchronised storage key to use.
1084
+ */
1085
+ setSynchronisedStorageKey(synchronisedStorageKey) {
1086
+ this._synchronisedStorageKey = synchronisedStorageKey;
1087
+ }
1088
+ /**
1089
+ * Build a changeset.
1090
+ * @param storageKey The storage key of the change set.
688
1091
  * @param changes The changes to apply.
689
1092
  * @param completeCallback The callback to call when the changeset is created and stored.
690
1093
  * @returns The storage id of the change set if created.
691
1094
  */
692
- async createAndStoreChangeSet(schemaType, changes, completeCallback) {
693
- if (Is.arrayValue(changes)) {
694
- this._populateFullChanges[schemaType] = {
695
- changes,
696
- entities: {},
697
- requestIds: [],
698
- completeCallback: async () => this.finaliseFullChanges(schemaType, completeCallback)
699
- };
700
- const setChanges = changes.filter(c => c.operation === SyncChangeOperation.Set);
701
- if (setChanges.length === 0) {
702
- // If we don't need to request any full details, we can just call the complete callback
703
- await this.finaliseFullChanges(schemaType, completeCallback);
704
- }
705
- else {
706
- // Otherwise we need to request the full details for each change
707
- this._populateFullChanges[schemaType].requestIds = setChanges.map(change => change.id);
708
- // Once all the requests are handled the callback will be called
709
- for (const change of setChanges) {
710
- // Create a request for each change to populate the full details
711
- this._eventBusComponent.publish(SynchronisedStorageTopics.LocalItemRequest, {
712
- schemaType,
713
- id: change.id
714
- });
715
- }
1095
+ async buildChangeSet(storageKey, changes, completeCallback) {
1096
+ await this._logging?.log({
1097
+ level: "info",
1098
+ source: this.CLASS_NAME,
1099
+ message: "buildingChangeSet",
1100
+ data: {
1101
+ storageKey,
1102
+ changeCount: changes.length
716
1103
  }
1104
+ });
1105
+ this._populateFullChanges[storageKey] = {
1106
+ changes,
1107
+ entities: {},
1108
+ requestIds: [],
1109
+ completeCallback: async () => this.finaliseFullChanges(storageKey, completeCallback)
1110
+ };
1111
+ const setChanges = changes.filter(c => c.operation === SyncChangeOperation.Set);
1112
+ if (setChanges.length === 0) {
1113
+ // If we don't need to request any full details, we can just call the complete callback
1114
+ await this.finaliseFullChanges(storageKey, completeCallback);
717
1115
  }
718
1116
  else {
719
- await completeCallback();
1117
+ // Otherwise we need to request the full details for each change
1118
+ this._populateFullChanges[storageKey].requestIds = setChanges.map(change => change.id);
1119
+ // Once all the requests are handled the callback will be called
1120
+ for (const change of setChanges) {
1121
+ // Create a request for each change to populate the full details
1122
+ await this._logging?.log({
1123
+ level: "info",
1124
+ source: this.CLASS_NAME,
1125
+ message: "createChangeSetRequestingItem",
1126
+ data: {
1127
+ storageKey,
1128
+ id: change.id
1129
+ }
1130
+ });
1131
+ this._eventBusComponent.publish(SynchronisedStorageTopics.LocalItemRequest, {
1132
+ storageKey,
1133
+ id: change.id
1134
+ });
1135
+ }
720
1136
  }
721
1137
  }
722
1138
  /**
723
1139
  * Finalise the full details for the sync change set.
724
- * @param schemaType The schema type of the change set.
1140
+ * @param storageKey The storage key of the change set.
725
1141
  * @param completeCallback The callback to call when the changeset is populated.
726
1142
  * @returns Nothing.
727
1143
  */
728
- async finaliseFullChanges(schemaType, completeCallback) {
1144
+ async finaliseFullChanges(storageKey, completeCallback) {
1145
+ await this._logging?.log({
1146
+ level: "info",
1147
+ source: this.CLASS_NAME,
1148
+ message: "finalisingSyncChanges",
1149
+ data: {
1150
+ storageKey
1151
+ }
1152
+ });
729
1153
  if (Is.stringValue(this._nodeIdentity)) {
730
- const changes = this._populateFullChanges[schemaType].changes;
1154
+ const changes = this._populateFullChanges[storageKey].changes;
731
1155
  for (const change of changes) {
732
- change.entity = this._populateFullChanges[schemaType].entities[change.id] ?? change.entity;
1156
+ change.entity = this._populateFullChanges[storageKey].entities[change.id] ?? change.entity;
733
1157
  if (change.operation === SyncChangeOperation.Set && Is.objectValue(change.entity)) {
1158
+ // Remove the id from the entity as this is stored in the operation
1159
+ // and will be reinstated when the changeset is reconstituted
1160
+ ObjectHelper.propertyDelete(change.entity, "id");
734
1161
  // Remove the node identity as the changeset has this stored at the top level
735
1162
  // and we do not want to store it in the change itself to reduce redundancy
736
1163
  ObjectHelper.propertyDelete(change.entity, "nodeIdentity");
737
1164
  }
738
1165
  }
739
- // Add the changeset to the current snapshot
740
1166
  const syncChangeSet = {
741
1167
  id: Converter.bytesToHex(RandomHelper.generate(32)),
742
1168
  dateCreated: new Date(Date.now()).toISOString(),
743
- schemaType,
1169
+ storageKey,
744
1170
  changes,
745
1171
  nodeIdentity: this._nodeIdentity
746
1172
  };
747
- // And sign it with the node identity
748
- syncChangeSet.proof = await this._changeSetHelper.createChangeSetProof(syncChangeSet);
749
- // Store the changeset in the blob storage
750
- const changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
751
- await completeCallback(changeSetStorageId);
1173
+ try {
1174
+ // And sign it with the node identity
1175
+ syncChangeSet.proof = await this._changeSetHelper.createChangeSetProof(syncChangeSet);
1176
+ // If this is a trusted node, we also store the changeset
1177
+ let changeSetStorageId;
1178
+ if (this._isTrustedNode) {
1179
+ changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
1180
+ }
1181
+ await completeCallback(syncChangeSet, changeSetStorageId);
1182
+ }
1183
+ catch (err) {
1184
+ await this._logging?.log({
1185
+ level: "error",
1186
+ source: this.CLASS_NAME,
1187
+ message: "finalisingSyncChangesFailed",
1188
+ data: {
1189
+ storageKey
1190
+ },
1191
+ error: BaseError.fromError(err)
1192
+ });
1193
+ await completeCallback();
1194
+ }
752
1195
  }
753
1196
  else {
754
1197
  await completeCallback();
@@ -756,122 +1199,138 @@ class RemoteSyncStateHelper {
756
1199
  }
757
1200
  /**
758
1201
  * Add a new changeset into the sync state.
1202
+ * @param storageKey The storage key of the change set to add.
759
1203
  * @param changeSetStorageId The id of the change set to add the the current state
760
1204
  * @returns Nothing.
761
1205
  */
762
- async addChangeSetToSyncState(changeSetStorageId) {
763
- // First load the current sync state if there is one
764
- const syncStatePointer = await this.getVerifiableSyncPointer();
1206
+ async addChangeSetToSyncState(storageKey, changeSetStorageId) {
1207
+ await this._logging?.log({
1208
+ level: "info",
1209
+ source: this.CLASS_NAME,
1210
+ message: "addChangeSetToSyncState",
1211
+ data: {
1212
+ storageKey,
1213
+ changeSetStorageId
1214
+ }
1215
+ });
1216
+ // First load the sync pointer store to get the current sync pointer for the storage key
1217
+ const syncPointerStore = await this.getVerifiableSyncPointerStore();
765
1218
  let syncState;
766
- if (!Is.empty(syncStatePointer?.syncPointerId)) {
767
- syncState = await this.getRemoteSyncState(syncStatePointer.syncPointerId);
1219
+ if (!Is.empty(syncPointerStore.syncPointers[storageKey])) {
1220
+ syncState = await this.getRemoteSyncState(syncPointerStore.syncPointers[storageKey]);
768
1221
  }
769
1222
  // No current sync state, so we create a new one
770
1223
  if (Is.empty(syncState)) {
771
- syncState = { snapshots: [] };
1224
+ syncState = { version: SYNC_STATE_VERSION, snapshots: [] };
772
1225
  }
773
1226
  // Sort the snapshots so the newest snapshot is last in the array
774
1227
  const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
775
1228
  // Get the current snapshot, if it does not exist we create a new one
776
1229
  let currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
1230
+ const now = new Date(Date.now()).toISOString();
777
1231
  if (Is.empty(currentSnapshot)) {
778
1232
  currentSnapshot = {
1233
+ version: SYNC_SNAPSHOT_VERSION,
779
1234
  id: Converter.bytesToHex(RandomHelper.generate(32)),
780
- dateCreated: new Date(Date.now()).toISOString(),
1235
+ dateCreated: now,
1236
+ dateModified: now,
781
1237
  changeSetStorageIds: []
782
1238
  };
783
1239
  syncState.snapshots.push(currentSnapshot);
784
1240
  }
785
1241
  else {
786
1242
  // Snapshot exists, we update the dateModified
787
- currentSnapshot.dateModified = new Date(Date.now()).toISOString();
1243
+ currentSnapshot.dateModified = now;
788
1244
  }
789
1245
  // Add the changeset storage id to the current snapshot
790
1246
  currentSnapshot.changeSetStorageIds.push(changeSetStorageId);
791
1247
  // Store the sync state in the blob storage
792
- const syncStateId = await this.storeRemoteSyncState(syncState);
793
- // Store the verifiable sync pointer in the verifiable storage
794
- await this.storeVerifiableSyncPointer(syncStateId);
1248
+ syncPointerStore.syncPointers[storageKey] = await this.storeRemoteSyncState(syncState);
1249
+ // Store the verifiable sync pointer store in the verifiable storage
1250
+ await this.storeVerifiableSyncPointerStore(syncPointerStore);
795
1251
  }
796
1252
  /**
797
1253
  * Create a consolidated snapshot for the entire storage.
798
- * @param schemaType The schema type of the snapshot to create.
1254
+ * @param storageKey The storage key of the snapshot to create.
799
1255
  * @param batchSize The batch size to use for consolidation.
800
1256
  * @returns Nothing.
801
1257
  */
802
- async consolidateFromLocal(schemaType, batchSize) {
1258
+ async consolidationStart(storageKey, batchSize) {
803
1259
  await this._logging?.log({
804
1260
  level: "info",
805
1261
  source: this.CLASS_NAME,
806
1262
  message: "consolidationStarting"
807
1263
  });
808
- await this._eventBusComponent.publish(SynchronisedStorageTopics.BatchRequest, { schemaType, batchSize });
1264
+ // Perform a batch request to start the consolidation
1265
+ await this._eventBusComponent.publish(SynchronisedStorageTopics.BatchRequest, { storageKey, batchSize });
809
1266
  }
810
1267
  /**
811
- * Get the sync pointer.
812
- * @returns The sync pointer.
1268
+ * Get the sync pointer store.
1269
+ * @returns The sync pointer store.
813
1270
  */
814
- async getVerifiableSyncPointer() {
815
- try {
816
- await this._logging?.log({
817
- level: "info",
818
- source: this.CLASS_NAME,
819
- message: "verifiableSyncPointerRetrieving",
820
- data: {
821
- key: this._synchronisedStorageKey
822
- }
823
- });
824
- const syncPointerStore = await this._verifiableSyncPointerStorageConnector.get(this._synchronisedStorageKey, { includeData: true });
825
- if (Is.uint8Array(syncPointerStore.data)) {
826
- const syncPointer = ObjectHelper.fromBytes(syncPointerStore.data);
1271
+ async getVerifiableSyncPointerStore() {
1272
+ if (Is.stringValue(this._synchronisedStorageKey)) {
1273
+ try {
827
1274
  await this._logging?.log({
828
1275
  level: "info",
829
1276
  source: this.CLASS_NAME,
830
- message: "verifiableSyncPointerRetrieved",
1277
+ message: "verifiableSyncPointerStoreRetrieving",
831
1278
  data: {
832
- key: this._synchronisedStorageKey,
833
- syncPointerId: syncPointer.syncPointerId
1279
+ key: this._synchronisedStorageKey
834
1280
  }
835
1281
  });
836
- return syncPointer;
1282
+ const syncPointerStore = await this._verifiableSyncPointerStorageConnector.get(this._synchronisedStorageKey, { includeData: true });
1283
+ if (Is.uint8Array(syncPointerStore.data)) {
1284
+ const syncPointer = ObjectHelper.fromBytes(syncPointerStore.data);
1285
+ await this._logging?.log({
1286
+ level: "info",
1287
+ source: this.CLASS_NAME,
1288
+ message: "verifiableSyncPointerStoreRetrieved",
1289
+ data: {
1290
+ key: this._synchronisedStorageKey
1291
+ }
1292
+ });
1293
+ return syncPointer;
1294
+ }
837
1295
  }
838
- }
839
- catch (err) {
840
- if (!BaseError.someErrorName(err, NotFoundError.CLASS_NAME)) {
841
- throw err;
1296
+ catch (err) {
1297
+ if (!BaseError.someErrorName(err, NotFoundError.CLASS_NAME)) {
1298
+ throw err;
1299
+ }
842
1300
  }
1301
+ await this._logging?.log({
1302
+ level: "info",
1303
+ source: this.CLASS_NAME,
1304
+ message: "verifiableSyncPointerStoreNotFound",
1305
+ data: {
1306
+ key: this._synchronisedStorageKey
1307
+ }
1308
+ });
843
1309
  }
844
- await this._logging?.log({
845
- level: "info",
846
- source: this.CLASS_NAME,
847
- message: "verifiableSyncPointerNotFound",
848
- data: {
849
- key: this._synchronisedStorageKey
850
- }
851
- });
1310
+ // If no sync pointer store exists, we return an empty one
1311
+ return {
1312
+ version: SYNC_POINTER_STORE_VERSION,
1313
+ syncPointers: {}
1314
+ };
852
1315
  }
853
1316
  /**
854
1317
  * Store the verifiable sync pointer in the verifiable storage.
855
- * @param syncStateId The id of the sync state to store.
1318
+ * @param syncPointerStore The sync pointer store to store.
856
1319
  * @returns Nothing.
857
1320
  */
858
- async storeVerifiableSyncPointer(syncStateId) {
859
- // Create a new verifiable sync pointer object pointing to the sync state
860
- const verifiableSyncPointer = {
861
- syncPointerId: syncStateId
862
- };
863
- await this._logging?.log({
864
- level: "info",
865
- source: this.CLASS_NAME,
866
- message: "verifiableSyncPointerStoring",
867
- data: {
868
- key: this._synchronisedStorageKey,
869
- syncPointerId: verifiableSyncPointer.syncPointerId
870
- }
871
- });
872
- // Store the verifiable sync pointer in the verifiable storage
873
- await this._verifiableSyncPointerStorageConnector.create(this._synchronisedStorageKey, ObjectHelper.toBytes(verifiableSyncPointer));
874
- return verifiableSyncPointer;
1321
+ async storeVerifiableSyncPointerStore(syncPointerStore) {
1322
+ if (Is.stringValue(this._nodeIdentity) && Is.stringValue(this._synchronisedStorageKey)) {
1323
+ await this._logging?.log({
1324
+ level: "info",
1325
+ source: this.CLASS_NAME,
1326
+ message: "verifiableSyncPointerStoreStoring",
1327
+ data: {
1328
+ key: this._synchronisedStorageKey
1329
+ }
1330
+ });
1331
+ // Store the verifiable sync pointer in the verifiable storage
1332
+ await this._verifiableSyncPointerStorageConnector.update(this._nodeIdentity, this._synchronisedStorageKey, ObjectHelper.toBytes(syncPointerStore));
1333
+ }
875
1334
  }
876
1335
  /**
877
1336
  * Store the remote sync state.
@@ -887,9 +1346,7 @@ class RemoteSyncStateHelper {
887
1346
  snapshotCount: syncState.snapshots.length
888
1347
  }
889
1348
  });
890
- // We don't want to encrypt the sync state as no other nodes would be able to read it
891
- // the blob storage also needs to be publicly accessible so that other nodes can retrieve it
892
- return this._blobStorageComponent.create(Converter.bytesToBase64(ObjectHelper.toBytes(syncState)), undefined, undefined, undefined, { disableEncryption: true, compress: BlobStorageCompressionType.Gzip });
1349
+ return this._blobStorageHelper.saveBlob(syncState);
893
1350
  }
894
1351
  /**
895
1352
  * Get the remote sync state.
@@ -906,11 +1363,8 @@ class RemoteSyncStateHelper {
906
1363
  syncPointerId
907
1364
  }
908
1365
  });
909
- const blobEntry = await this._blobStorageComponent.get(syncPointerId, {
910
- includeContent: true
911
- });
912
- if (Is.stringBase64(blobEntry.blob)) {
913
- const syncState = ObjectHelper.fromBytes(Converter.base64ToBytes(blobEntry.blob));
1366
+ const syncState = await this._blobStorageHelper.load(syncPointerId);
1367
+ if (Is.object(syncState)) {
914
1368
  await this._logging?.log({
915
1369
  level: "info",
916
1370
  source: this.CLASS_NAME,
@@ -923,10 +1377,16 @@ class RemoteSyncStateHelper {
923
1377
  return syncState;
924
1378
  }
925
1379
  }
926
- catch (err) {
927
- if (!BaseError.someErrorName(err, NotFoundError.CLASS_NAME)) {
928
- throw err;
929
- }
1380
+ catch (error) {
1381
+ await this._logging?.log({
1382
+ level: "warn",
1383
+ source: this.CLASS_NAME,
1384
+ message: "getSyncStateError",
1385
+ data: {
1386
+ syncPointerId
1387
+ },
1388
+ error: BaseError.fromError(error)
1389
+ });
930
1390
  }
931
1391
  await this._logging?.log({
932
1392
  level: "info",
@@ -938,20 +1398,22 @@ class RemoteSyncStateHelper {
938
1398
  });
939
1399
  }
940
1400
  /**
941
- * Handle the batch response.
1401
+ * Handle the batch response which is triggered from a consolidation request.
942
1402
  * @param response The batch response to handle.
943
1403
  */
944
1404
  async handleBatchResponse(response) {
945
1405
  if (Is.stringValue(this._nodeIdentity)) {
1406
+ const now = new Date(Date.now()).toISOString();
946
1407
  // Create a new snapshot entry for the current batch
947
1408
  const syncChangeSet = {
948
1409
  id: Converter.bytesToHex(RandomHelper.generate(32)),
949
- dateCreated: new Date(Date.now()).toISOString(),
1410
+ dateCreated: now,
1411
+ dateModified: now,
950
1412
  changes: response.entities.map(change => ({
951
1413
  operation: SyncChangeOperation.Set,
952
- id: change[response.primaryKey]
1414
+ id: change.id
953
1415
  })),
954
- schemaType: response.schemaType,
1416
+ storageKey: response.storageKey,
955
1417
  nodeIdentity: this._nodeIdentity
956
1418
  };
957
1419
  // And sign it with the node identity
@@ -959,20 +1421,35 @@ class RemoteSyncStateHelper {
959
1421
  // Store the changeset in the blob storage
960
1422
  const changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
961
1423
  // Add the changeset storage id to the snapshot ids
962
- this._batchResponseStorageIds[response.schemaType] ??= [];
963
- this._batchResponseStorageIds[response.schemaType].push(changeSetStorageId);
1424
+ this._batchResponseStorageIds[response.storageKey] ??= [];
1425
+ this._batchResponseStorageIds[response.storageKey].push(changeSetStorageId);
1426
+ // If this is the last entry in the batch response, we can create the consolidated snapshot
964
1427
  if (response.lastEntry) {
965
- const syncState = { snapshots: [] };
1428
+ // Get the current sync pointer store
1429
+ const syncPointerStore = await this.getVerifiableSyncPointerStore();
1430
+ let syncState;
1431
+ if (Is.stringValue(syncPointerStore.syncPointers[response.storageKey])) {
1432
+ // If the sync pointer exists, we load the current sync state
1433
+ syncState = await this.getRemoteSyncState(syncPointerStore.syncPointers[response.storageKey]);
1434
+ }
1435
+ // If the sync state does not exist, we create a new one
1436
+ syncState ??= { version: SYNC_STATE_VERSION, snapshots: [] };
966
1437
  const batchSnapshot = {
1438
+ version: SYNC_SNAPSHOT_VERSION,
967
1439
  id: Converter.bytesToHex(RandomHelper.generate(32)),
968
- dateCreated: new Date(Date.now()).toISOString(),
969
- changeSetStorageIds: this._batchResponseStorageIds[response.schemaType]
1440
+ dateCreated: now,
1441
+ dateModified: now,
1442
+ changeSetStorageIds: this._batchResponseStorageIds[response.storageKey]
970
1443
  };
971
1444
  syncState.snapshots.push(batchSnapshot);
972
1445
  // Store the sync state in the blob storage
973
1446
  const syncStateId = await this.storeRemoteSyncState(syncState);
1447
+ syncPointerStore.syncPointers[response.storageKey] = syncStateId;
974
1448
  // Store the verifiable sync pointer in the verifiable storage
975
- await this.storeVerifiableSyncPointer(syncStateId);
1449
+ await this.storeVerifiableSyncPointerStore(syncPointerStore);
1450
+ // Remove the batch response storage ids for the storage key
1451
+ // as we have consolidated the changes
1452
+ delete this._batchResponseStorageIds[response.storageKey];
976
1453
  await this._logging?.log({
977
1454
  level: "info",
978
1455
  source: this.CLASS_NAME,
@@ -986,13 +1463,22 @@ class RemoteSyncStateHelper {
986
1463
  * @param response The item response to handle.
987
1464
  */
988
1465
  async handleLocalItemResponse(response) {
989
- if (!Is.empty(this._populateFullChanges[response.schemaType])) {
990
- const idx = this._populateFullChanges[response.schemaType].requestIds.indexOf(response.id);
1466
+ await this._logging?.log({
1467
+ level: "info",
1468
+ source: this.CLASS_NAME,
1469
+ message: "createChangeSetRespondingItem",
1470
+ data: {
1471
+ storageKey: response.storageKey,
1472
+ id: response.id
1473
+ }
1474
+ });
1475
+ if (!Is.empty(this._populateFullChanges[response.storageKey])) {
1476
+ const idx = this._populateFullChanges[response.storageKey].requestIds.indexOf(response.id);
991
1477
  if (idx !== -1) {
992
- this._populateFullChanges[response.schemaType].requestIds.splice(idx, 1);
993
- this._populateFullChanges[response.schemaType].entities[response.id] = response.entity;
994
- if (this._populateFullChanges[response.schemaType].requestIds.length === 0) {
995
- await this._populateFullChanges[response.schemaType].completeCallback();
1478
+ this._populateFullChanges[response.storageKey].requestIds.splice(idx, 1);
1479
+ this._populateFullChanges[response.storageKey].entities[response.id] = response.entity;
1480
+ if (this._populateFullChanges[response.storageKey].requestIds.length === 0) {
1481
+ await this._populateFullChanges[response.storageKey].completeCallback();
996
1482
  }
997
1483
  }
998
1484
  }
@@ -1004,20 +1490,20 @@ class RemoteSyncStateHelper {
1004
1490
  */
1005
1491
  class SynchronisedStorageService {
1006
1492
  /**
1007
- * The default interval to check for entity updates, defaults to 5 mins.
1493
+ * The default interval to check for entity updates.
1008
1494
  * @internal
1009
1495
  */
1010
- static _DEFAULT_ENTITY_UPDATE_INTERVAL_MS = 300000;
1496
+ static _DEFAULT_ENTITY_UPDATE_INTERVAL_MINUTES = 5;
1011
1497
  /**
1012
- * The default interval to perform consolidation, defaults to 60 mins.
1498
+ * The default interval to perform consolidation.
1013
1499
  * @internal
1014
1500
  */
1015
- static _DEFAULT_CONSOLIDATION_INTERVAL_MS = 3600000;
1501
+ static _DEFAULT_CONSOLIDATION_INTERVAL_MINUTES = 60;
1016
1502
  /**
1017
1503
  * The default size of a consolidation batch.
1018
1504
  * @internal
1019
1505
  */
1020
- static _DEFAULT_CONSOLIDATION_BATCH_SIZE = 1000;
1506
+ static _DEFAULT_CONSOLIDATION_BATCH_SIZE = 100;
1021
1507
  /**
1022
1508
  * Runtime name for the class.
1023
1509
  */
@@ -1032,16 +1518,21 @@ class SynchronisedStorageService {
1032
1518
  * @internal
1033
1519
  */
1034
1520
  _eventBusComponent;
1521
+ /**
1522
+ * The vault connector.
1523
+ * @internal
1524
+ */
1525
+ _vaultConnector;
1035
1526
  /**
1036
1527
  * The storage connector for the sync snapshot entries.
1037
1528
  * @internal
1038
1529
  */
1039
1530
  _localSyncSnapshotEntryEntityStorage;
1040
1531
  /**
1041
- * The blob storage component to use for remote sync states.
1532
+ * The blob storage connector to use for remote sync states.
1042
1533
  * @internal
1043
1534
  */
1044
- _blobStorageComponent;
1535
+ _blobStorageConnector;
1045
1536
  /**
1046
1537
  * The verifiable storage connector to use for storing sync pointers.
1047
1538
  * @internal
@@ -1052,11 +1543,21 @@ class SynchronisedStorageService {
1052
1543
  * @internal
1053
1544
  */
1054
1545
  _identityConnector;
1546
+ /**
1547
+ * The task scheduler component.
1548
+ * @internal
1549
+ */
1550
+ _taskSchedulerComponent;
1055
1551
  /**
1056
1552
  * The synchronised storage service to use when this is not a trusted node.
1057
1553
  * @internal
1058
1554
  */
1059
1555
  _trustedSynchronisedStorageComponent;
1556
+ /**
1557
+ * The blob storage helper.
1558
+ * @internal
1559
+ */
1560
+ _blobStorageHelper;
1060
1561
  /**
1061
1562
  * The change set helper.
1062
1563
  * @internal
@@ -1078,15 +1579,20 @@ class SynchronisedStorageService {
1078
1579
  */
1079
1580
  _config;
1080
1581
  /**
1081
- * The timer ids for checking for entity updates.
1582
+ * The synchronised storage key to use for the remote synchronised storage.
1082
1583
  * @internal
1083
1584
  */
1084
- _entityUpdateTimers;
1585
+ _synchronisedStorageKey;
1085
1586
  /**
1086
- * The timer ids for consolidation.
1587
+ * The flag to determine if the service has been started.
1087
1588
  * @internal
1088
1589
  */
1089
- _consolidationTimers;
1590
+ _serviceStarted;
1591
+ /**
1592
+ * The active storage keys for the synchronised storage service.
1593
+ * @internal
1594
+ */
1595
+ _activeStorageKeys;
1090
1596
  /**
1091
1597
  * The identity of the node this connector is running on.
1092
1598
  * @internal
@@ -1101,21 +1607,26 @@ class SynchronisedStorageService {
1101
1607
  Guards.object(this.CLASS_NAME, "options.config", options.config);
1102
1608
  this._eventBusComponent = ComponentFactory.get(options.eventBusComponentType ?? "event-bus");
1103
1609
  this._logging = LoggingConnectorFactory.getIfExists(options.loggingConnectorType ?? "logging");
1610
+ this._vaultConnector = VaultConnectorFactory.get(options.vaultConnectorType ?? "vault");
1104
1611
  this._localSyncSnapshotEntryEntityStorage = EntityStorageConnectorFactory.get(options.syncSnapshotStorageConnectorType ?? "sync-snapshot-entry");
1105
1612
  this._verifiableSyncPointerStorageConnector = VerifiableStorageConnectorFactory.get(options.verifiableStorageConnectorType ?? "verifiable-storage");
1106
- this._blobStorageComponent = ComponentFactory.get(options.blobStorageComponentType ?? "blob-storage");
1613
+ this._blobStorageConnector = BlobStorageConnectorFactory.get(options.blobStorageConnectorType ?? "blob-storage");
1107
1614
  this._identityConnector = IdentityConnectorFactory.get(options.identityConnectorType ?? "identity");
1615
+ this._taskSchedulerComponent = ComponentFactory.get(options.taskSchedulerComponentType ?? "task-scheduler");
1108
1616
  this._config = {
1109
- synchronisedStorageKey: options.config.synchronisedStorageKey,
1110
1617
  synchronisedStorageMethodId: options.config.synchronisedStorageMethodId ?? "synchronised-storage-assertion",
1111
- entityUpdateIntervalMs: options.config.entityUpdateIntervalMs ??
1112
- SynchronisedStorageService._DEFAULT_ENTITY_UPDATE_INTERVAL_MS,
1618
+ entityUpdateIntervalMinutes: options.config.entityUpdateIntervalMinutes ??
1619
+ SynchronisedStorageService._DEFAULT_ENTITY_UPDATE_INTERVAL_MINUTES,
1113
1620
  isTrustedNode: options.config.isTrustedNode ?? false,
1114
- consolidationIntervalMs: options.config.consolidationIntervalMs ??
1115
- SynchronisedStorageService._DEFAULT_CONSOLIDATION_INTERVAL_MS,
1621
+ consolidationIntervalMinutes: options.config.consolidationIntervalMinutes ??
1622
+ SynchronisedStorageService._DEFAULT_CONSOLIDATION_INTERVAL_MINUTES,
1116
1623
  consolidationBatchSize: options.config.consolidationBatchSize ??
1117
- SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE
1624
+ SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE,
1625
+ blobStorageEncryptionKeyId: options.config.blobStorageEncryptionKeyId ?? "synchronised-storage-blob-encryption-key",
1626
+ verifiableStorageKeyId: options.config.verifiableStorageKeyId
1118
1627
  };
1628
+ this._synchronisedStorageKey =
1629
+ verifiableStorageKeys[options.config.verifiableStorageKeyId] ?? options.config.verifiableStorageKeyId;
1119
1630
  // If this is not a trusted node, we need to use a synchronised storage service
1120
1631
  // to synchronise with a trusted node.
1121
1632
  if (!this._config.isTrustedNode) {
@@ -1123,13 +1634,14 @@ class SynchronisedStorageService {
1123
1634
  this._trustedSynchronisedStorageComponent =
1124
1635
  ComponentFactory.get(options.trustedSynchronisedStorageComponentType);
1125
1636
  }
1126
- this._changeSetHelper = new ChangeSetHelper(this._logging, this._eventBusComponent, this._blobStorageComponent, this._identityConnector, this._config.synchronisedStorageMethodId);
1637
+ this._blobStorageHelper = new BlobStorageHelper(this._logging, this._vaultConnector, this._blobStorageConnector, this._config.blobStorageEncryptionKeyId, this._config.isTrustedNode);
1638
+ this._changeSetHelper = new ChangeSetHelper(this._logging, this._eventBusComponent, this._identityConnector, this._blobStorageHelper, this._config.synchronisedStorageMethodId);
1127
1639
  this._localSyncStateHelper = new LocalSyncStateHelper(this._logging, this._localSyncSnapshotEntryEntityStorage, this._changeSetHelper);
1128
- this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._logging, this._eventBusComponent, this._blobStorageComponent, this._verifiableSyncPointerStorageConnector, this._changeSetHelper, this._config.synchronisedStorageKey);
1129
- this._consolidationTimers = {};
1130
- this._entityUpdateTimers = {};
1131
- this._eventBusComponent.subscribe(SynchronisedStorageTopics.RegisterSchemaType, async (event) => this.registerType(event.data));
1132
- this._eventBusComponent.subscribe(SynchronisedStorageTopics.LocalItemChange, async (event) => this._localSyncStateHelper.addLocalChange(event.data.schemaType, event.data.operation, event.data.id));
1640
+ this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._logging, this._eventBusComponent, this._verifiableSyncPointerStorageConnector, this._blobStorageHelper, this._changeSetHelper, this._config.isTrustedNode);
1641
+ this._serviceStarted = false;
1642
+ this._activeStorageKeys = {};
1643
+ this._eventBusComponent.subscribe(SynchronisedStorageTopics.RegisterStorageKey, async (event) => this.registerStorageKey(event.data));
1644
+ this._eventBusComponent.subscribe(SynchronisedStorageTopics.LocalItemChange, async (event) => this._localSyncStateHelper.addLocalChange(event.data.storageKey, event.data.operation, event.data.id));
1133
1645
  }
1134
1646
  /**
1135
1647
  * The component needs to be started when the node is initialized.
@@ -1141,6 +1653,20 @@ class SynchronisedStorageService {
1141
1653
  async start(nodeIdentity, nodeLoggingConnectorType, componentState) {
1142
1654
  this._nodeIdentity = nodeIdentity;
1143
1655
  this._remoteSyncStateHelper.setNodeIdentity(nodeIdentity);
1656
+ this._changeSetHelper.setNodeIdentity(nodeIdentity);
1657
+ this._remoteSyncStateHelper.setSynchronisedStorageKey(this._synchronisedStorageKey);
1658
+ this._serviceStarted = true;
1659
+ // If this is not a trusted node we need to request the decryption key from a trusted node
1660
+ if (!this._config.isTrustedNode && !Is.empty(this._trustedSynchronisedStorageComponent)) {
1661
+ const proof = await this._identityConnector.createProof(this._nodeIdentity, DocumentHelper.joinId(this._nodeIdentity, this._config.synchronisedStorageMethodId), ProofTypes.DataIntegrityProof, { nodeIdentity });
1662
+ const decryptionKey = await this._trustedSynchronisedStorageComponent.getDecryptionKey(this._nodeIdentity, proof);
1663
+ // We don't have the private key so instead we store the key as a secret in the vault
1664
+ await this._vaultConnector.setSecret(this._config.blobStorageEncryptionKeyId, decryptionKey);
1665
+ }
1666
+ // If there are already storage keys registered, we need to activate them
1667
+ for (const storageKey in this._activeStorageKeys) {
1668
+ await this.activateStorageKey(storageKey);
1669
+ }
1144
1670
  }
1145
1671
  /**
1146
1672
  * The component needs to be stopped when the node is closed.
@@ -1150,48 +1676,88 @@ class SynchronisedStorageService {
1150
1676
  * @returns Nothing.
1151
1677
  */
1152
1678
  async stop(nodeIdentity, nodeLoggingConnectorType, componentState) {
1153
- for (const schemaType in this._entityUpdateTimers) {
1154
- clearTimeout(this._entityUpdateTimers[schemaType]);
1155
- delete this._entityUpdateTimers[schemaType];
1679
+ for (const storageKey in this._activeStorageKeys) {
1680
+ this._activeStorageKeys[storageKey] = false;
1681
+ this._taskSchedulerComponent.removeTask(`synchronised-storage-update-${storageKey}`);
1682
+ this._taskSchedulerComponent.removeTask(`synchronised-storage-consolidation-${storageKey}`);
1683
+ }
1684
+ }
1685
+ /**
1686
+ * Get the decryption key for the synchronised storage.
1687
+ * This is used to decrypt the data stored in the synchronised storage.
1688
+ * @param nodeIdentity The identity of the node requesting the decryption key.
1689
+ * @param proof The proof of the request so we know the request is from the specified node.
1690
+ * @returns The decryption key.
1691
+ */
1692
+ async getDecryptionKey(nodeIdentity, proof) {
1693
+ if (!this._config.isTrustedNode) {
1694
+ throw new GeneralError(this.CLASS_NAME, "notTrustedNode");
1156
1695
  }
1157
- for (const schemaType in this._consolidationTimers) {
1158
- clearTimeout(this._consolidationTimers[schemaType]);
1159
- delete this._consolidationTimers[schemaType];
1696
+ Guards.stringValue(this.CLASS_NAME, "nodeIdentity", nodeIdentity);
1697
+ Guards.object(this.CLASS_NAME, "proof", proof);
1698
+ const isValid = await this._identityConnector.verifyProof({ nodeIdentity }, proof);
1699
+ if (!isValid) {
1700
+ throw new UnauthorizedError(this.CLASS_NAME, "invalidProof");
1160
1701
  }
1702
+ // TODO: We need to check if the node has permissions to access the decryption key
1703
+ // using rights-management
1704
+ const key = await this._vaultConnector.getKey(this._config.blobStorageEncryptionKeyId);
1705
+ if (Is.undefined(key.publicKey)) {
1706
+ throw new UnauthorizedError(this.CLASS_NAME, "decryptionKeyNotFound");
1707
+ }
1708
+ return Converter.bytesToBase64(key.publicKey);
1161
1709
  }
1162
1710
  /**
1163
- * Synchronise a complete set of changes, assumes this is a trusted node.
1164
- * @param changeSetStorageId The id of the change set to synchronise in blob storage.
1711
+ * Synchronise a set of changes from an untrusted node, assumes this is a trusted node.
1712
+ * @param syncChangeSet The change set to synchronise.
1165
1713
  * @returns Nothing.
1166
1714
  */
1167
- async syncChangeSet(changeSetStorageId) {
1715
+ async syncChangeSet(syncChangeSet) {
1168
1716
  if (!this._config.isTrustedNode) {
1169
1717
  throw new GeneralError(this.CLASS_NAME, "notTrustedNode");
1170
1718
  }
1171
- // This method is called by non trusted nodes to synchronise changes
1172
- Guards.stringValue(this.CLASS_NAME, "changeSetStorageId", changeSetStorageId);
1719
+ Guards.object(this.CLASS_NAME, "syncChangeSet", syncChangeSet);
1720
+ await this._logging?.log({
1721
+ level: "info",
1722
+ source: this.CLASS_NAME,
1723
+ message: "syncChangeSetForRemoteNode",
1724
+ data: {
1725
+ changeSetStorageId: syncChangeSet.id
1726
+ }
1727
+ });
1173
1728
  // TODO: The change set has a proof signed by the originating node identity
1174
1729
  // The proof is verified that the change set is valid and has not been tampered with.
1175
1730
  // but we also need to check that the originating node has permissions
1176
1731
  // to store the change set in the synchronised storage.
1177
1732
  // This will be performed using rights-management
1178
- const changeSet = await this._changeSetHelper.getAndApplyChangeset(changeSetStorageId);
1179
- if (!Is.empty(changeSet)) {
1180
- await this._remoteSyncStateHelper.addChangeSetToSyncState(changeSetStorageId);
1733
+ const copy = await this._changeSetHelper.copyChangeset(syncChangeSet);
1734
+ if (!Is.empty(copy) && Is.stringValue(this._nodeIdentity)) {
1735
+ // Apply the changes to this node
1736
+ await this._changeSetHelper.applyChangeset(copy.syncChangeSet);
1737
+ // And update the sync state with the latest changes
1738
+ await this._remoteSyncStateHelper.addChangeSetToSyncState(copy.syncChangeSet.storageKey, copy.changeSetStorageId);
1181
1739
  }
1182
1740
  }
1183
1741
  /**
1184
1742
  * Start the sync with further updates after an interval.
1185
- * @param schemaType The schema type to sync.
1743
+ * @param storageKey The storage key to sync.
1186
1744
  * @returns Nothing.
1187
1745
  * @internal
1188
1746
  */
1189
- async startEntitySync(schemaType) {
1747
+ async startEntitySync(storageKey) {
1190
1748
  try {
1749
+ await this._logging?.log({
1750
+ level: "info",
1751
+ source: this.CLASS_NAME,
1752
+ message: "startEntitySync",
1753
+ data: {
1754
+ storageKey
1755
+ }
1756
+ });
1191
1757
  // First we check for remote changes
1192
- await this.updateFromRemoteSyncState(schemaType);
1758
+ await this.updateFromRemoteSyncState(storageKey);
1193
1759
  // Now send any updates we have to the remote storage
1194
- await this.updateFromLocalSyncState(schemaType);
1760
+ await this.updateFromLocalSyncState(storageKey);
1195
1761
  }
1196
1762
  catch (error) {
1197
1763
  await this._logging?.log({
@@ -1201,27 +1767,31 @@ class SynchronisedStorageService {
1201
1767
  error: BaseError.fromError(error)
1202
1768
  });
1203
1769
  }
1204
- finally {
1205
- // Set a timer to check for updates again
1206
- this._entityUpdateTimers[schemaType] = setTimeout(async () => this.startEntitySync(schemaType), this._config.entityUpdateIntervalMs);
1207
- }
1208
1770
  }
1209
1771
  /**
1210
1772
  * Check for updates in the remote storage.
1211
- * @param schemaType The schema type to check for updates.
1773
+ * @param storageKey The storage key to check for updates.
1212
1774
  * @returns Nothing.
1213
1775
  * @internal
1214
1776
  */
1215
- async updateFromRemoteSyncState(schemaType) {
1216
- // Get the verifiable sync pointer from the verifiable storage
1217
- const verifiableSyncPointer = await this._remoteSyncStateHelper.getVerifiableSyncPointer();
1218
- if (!Is.empty(verifiableSyncPointer)) {
1777
+ async updateFromRemoteSyncState(storageKey) {
1778
+ await this._logging?.log({
1779
+ level: "info",
1780
+ source: this.CLASS_NAME,
1781
+ message: "updateFromRemoteSyncState",
1782
+ data: {
1783
+ storageKey
1784
+ }
1785
+ });
1786
+ // Get the verifiable sync pointer store from the verifiable storage
1787
+ const verifiableSyncPointerStore = await this._remoteSyncStateHelper.getVerifiableSyncPointerStore();
1788
+ if (!Is.empty(verifiableSyncPointerStore.syncPointers[storageKey])) {
1219
1789
  // Load the sync state from the remote blob storage using the sync pointer
1220
1790
  // to load the sync state
1221
- const remoteSyncState = await this._remoteSyncStateHelper.getRemoteSyncState(verifiableSyncPointer.syncPointerId);
1791
+ const remoteSyncState = await this._remoteSyncStateHelper.getRemoteSyncState(verifiableSyncPointerStore.syncPointers[storageKey]);
1222
1792
  // If we got the sync state we can try and sync from it
1223
1793
  if (!Is.undefined(remoteSyncState)) {
1224
- await this._localSyncStateHelper.syncFromRemote(schemaType, remoteSyncState);
1794
+ await this._localSyncStateHelper.applySyncState(storageKey, remoteSyncState);
1225
1795
  }
1226
1796
  }
1227
1797
  }
@@ -1230,47 +1800,94 @@ class SynchronisedStorageService {
1230
1800
  * @returns Nothing.
1231
1801
  * @internal
1232
1802
  */
1233
- async updateFromLocalSyncState(schemaType) {
1234
- if (Is.stringValue(this._nodeIdentity)) {
1235
- // Ge the current local change snapshot
1236
- const localChangeSnapshot = await this._localSyncStateHelper.getLocalChangeSnapshot(schemaType);
1237
- if (!Is.empty(localChangeSnapshot)) {
1238
- await this._remoteSyncStateHelper.createAndStoreChangeSet(schemaType, localChangeSnapshot.localChanges, async (changeSetStorageId) => {
1239
- if (Is.stringValue(changeSetStorageId)) {
1240
- // Send the local changes to the remote storage if we are a trusted node
1241
- if (this._config.isTrustedNode) {
1242
- await this._remoteSyncStateHelper.addChangeSetToSyncState(changeSetStorageId);
1803
+ async updateFromLocalSyncState(storageKey) {
1804
+ await this._logging?.log({
1805
+ level: "info",
1806
+ source: this.CLASS_NAME,
1807
+ message: "updateFromLocalSyncState",
1808
+ data: {
1809
+ storageKey
1810
+ }
1811
+ });
1812
+ const localChangeSnapshot = await this._localSyncStateHelper.getLocalChangeSnapshot(storageKey);
1813
+ if (Is.arrayValue(localChangeSnapshot.changes)) {
1814
+ await this._remoteSyncStateHelper.buildChangeSet(storageKey, localChangeSnapshot.changes, async (syncChangeSet, changeSetStorageId) => {
1815
+ if (Is.empty(syncChangeSet) && Is.empty(changeSetStorageId)) {
1816
+ await this._logging?.log({
1817
+ level: "info",
1818
+ source: this.CLASS_NAME,
1819
+ message: "builtStorageChangeSetNone",
1820
+ data: {
1821
+ storageKey
1243
1822
  }
1244
- else if (!Is.empty(this._trustedSynchronisedStorageComponent)) {
1245
- // If we are not a trusted node, we need to send the changes to the trusted node
1246
- await this._trustedSynchronisedStorageComponent.syncChangeSet(changeSetStorageId);
1823
+ });
1824
+ }
1825
+ else {
1826
+ await this._logging?.log({
1827
+ level: "info",
1828
+ source: this.CLASS_NAME,
1829
+ message: "builtStorageChangeSet",
1830
+ data: {
1831
+ storageKey,
1832
+ changeSetStorageId
1247
1833
  }
1834
+ });
1835
+ // Send the local changes to the remote storage if we are a trusted node
1836
+ if (this._config.isTrustedNode && Is.stringValue(changeSetStorageId)) {
1837
+ // If we are a trusted node, we can add the change set to the sync state
1838
+ // and remove the local change snapshot
1839
+ await this._remoteSyncStateHelper.addChangeSetToSyncState(storageKey, changeSetStorageId);
1248
1840
  await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
1249
1841
  }
1250
- });
1251
- }
1842
+ else if (!Is.empty(this._trustedSynchronisedStorageComponent) &&
1843
+ Is.object(syncChangeSet)) {
1844
+ // If we are not a trusted node, we need to send the changes to the trusted node
1845
+ // and then remove the local change snapshot
1846
+ await this._logging?.log({
1847
+ level: "info",
1848
+ source: this.CLASS_NAME,
1849
+ message: "sendingChangeSetToTrustedNode",
1850
+ data: {
1851
+ storageKey,
1852
+ changeSetStorageId
1853
+ }
1854
+ });
1855
+ await this._trustedSynchronisedStorageComponent.syncChangeSet(syncChangeSet);
1856
+ await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
1857
+ }
1858
+ }
1859
+ });
1860
+ }
1861
+ else {
1862
+ await this._logging?.log({
1863
+ level: "info",
1864
+ source: this.CLASS_NAME,
1865
+ message: "updateFromLocalSyncStateNoChanges",
1866
+ data: {
1867
+ storageKey
1868
+ }
1869
+ });
1252
1870
  }
1253
1871
  }
1254
1872
  /**
1255
1873
  * Start the consolidation sync.
1256
- * @param schemaType The schema type to consolidate.
1874
+ * @param storageKey The storage key to consolidate.
1257
1875
  * @returns Nothing.
1258
1876
  * @internal
1259
1877
  */
1260
- async startConsolidationSync(schemaType) {
1878
+ async startConsolidationSync(storageKey) {
1261
1879
  let localChangeSnapshot;
1262
1880
  try {
1263
- // If we are performing a consolidation, we can remove the local changes
1264
- await this._localSyncStateHelper.getLocalChangeSnapshot(schemaType);
1881
+ // If we are performing a consolidation, we can remove the local change snapshot
1882
+ // as we are going to create a complete changeset from the DB
1883
+ localChangeSnapshot = await this._localSyncStateHelper.getLocalChangeSnapshot(storageKey);
1265
1884
  if (!Is.empty(localChangeSnapshot)) {
1266
1885
  await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
1267
1886
  }
1268
- if (Is.stringValue(this._nodeIdentity)) {
1269
- await this._remoteSyncStateHelper.consolidateFromLocal(schemaType, this._config.consolidationBatchSize ??
1270
- SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE);
1271
- // The consolidation was successful, so we can remove the local change snapshot permanently
1272
- localChangeSnapshot = undefined;
1273
- }
1887
+ await this._remoteSyncStateHelper.consolidationStart(storageKey, this._config.consolidationBatchSize ??
1888
+ SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE);
1889
+ // The consolidation was successful, so we can remove the local change snapshot permanently
1890
+ localChangeSnapshot = undefined;
1274
1891
  }
1275
1892
  catch (error) {
1276
1893
  if (localChangeSnapshot) {
@@ -1284,32 +1901,62 @@ class SynchronisedStorageService {
1284
1901
  error: BaseError.fromError(error)
1285
1902
  });
1286
1903
  }
1287
- finally {
1288
- // Set a timer to perform the consolidation again
1289
- this._consolidationTimers[schemaType] = setTimeout(async () => this.startConsolidationSync(schemaType), this._config.consolidationIntervalMs);
1290
- }
1291
1904
  }
1292
1905
  /**
1293
1906
  * Register a new sync type.
1294
- * @param syncRegisterType The sync register type to register.
1907
+ * @param syncRegisterStorageKey The sync register type to register.
1295
1908
  * @internal
1296
1909
  */
1297
- async registerType(syncRegisterType) {
1910
+ async registerStorageKey(syncRegisterStorageKey) {
1298
1911
  await this._logging?.log({
1299
1912
  level: "info",
1300
1913
  source: this.CLASS_NAME,
1301
- message: "registerType",
1914
+ message: "registerStorageKey",
1302
1915
  data: {
1303
- schemaType: syncRegisterType.schemaType
1916
+ storageKey: syncRegisterStorageKey.storageKey
1304
1917
  }
1305
1918
  });
1306
- if (this._config.entityUpdateIntervalMs > 0) {
1307
- await this.startEntitySync(syncRegisterType.schemaType);
1919
+ if (Is.empty(this._activeStorageKeys[syncRegisterStorageKey.storageKey])) {
1920
+ this._activeStorageKeys[syncRegisterStorageKey.storageKey] = false;
1921
+ if (this._serviceStarted) {
1922
+ await this.activateStorageKey(syncRegisterStorageKey.storageKey);
1923
+ }
1308
1924
  }
1309
- if (this._config.isTrustedNode && this._config.consolidationIntervalMs > 0) {
1310
- await this.startConsolidationSync(syncRegisterType.schemaType);
1925
+ }
1926
+ /**
1927
+ * Activate a storage key.
1928
+ * @param storageKey The storage key to activate.
1929
+ * @internal
1930
+ */
1931
+ async activateStorageKey(storageKey) {
1932
+ if (!Is.empty(this._activeStorageKeys[storageKey]) && !this._activeStorageKeys[storageKey]) {
1933
+ await this._logging?.log({
1934
+ level: "info",
1935
+ source: this.CLASS_NAME,
1936
+ message: "activateStorageKey",
1937
+ data: {
1938
+ storageKey
1939
+ }
1940
+ });
1941
+ this._activeStorageKeys[storageKey] = true;
1942
+ if (this._config.entityUpdateIntervalMinutes > 0) {
1943
+ await this._taskSchedulerComponent.addTask(`synchronised-storage-update-${storageKey}`, [
1944
+ {
1945
+ nextTriggerTime: Date.now(),
1946
+ intervalMinutes: this._config.entityUpdateIntervalMinutes
1947
+ }
1948
+ ], async () => this.startEntitySync(storageKey));
1949
+ }
1950
+ if (this._config.isTrustedNode && this._config.consolidationIntervalMinutes > 0) {
1951
+ await this._taskSchedulerComponent.addTask(`synchronised-storage-consolidation-${storageKey}`, [
1952
+ {
1953
+ nextTriggerTime: Date.now(),
1954
+ intervalMinutes: this._config.consolidationIntervalMinutes
1955
+ }
1956
+ ], async () => this.startConsolidationSync(storageKey));
1957
+ }
1311
1958
  }
1312
1959
  }
1313
1960
  }
1314
1961
 
1315
- export { SyncSnapshotEntry, SynchronisedStorageService, generateRestRoutesSynchronisedStorage, initSchema, restEntryPoints, synchronisedStorageSyncChangeSetRequest, tagsSynchronisedStorage };
1962
+ export { SyncSnapshotEntry, SynchronisedStorageService, generateRestRoutesSynchronisedStorage, initSchema, restEntryPoints, synchronisedStorageGetDecryptionKeyRequest, synchronisedStorageSyncChangeSetRequest, tagsSynchronisedStorage };