@twin.org/synchronised-storage-service 0.0.1-next.9 → 0.0.3-next.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/dist/es/data/verifiableStorageKeys.json +5 -0
  2. package/dist/es/entities/syncSnapshotEntry.js +93 -0
  3. package/dist/es/entities/syncSnapshotEntry.js.map +1 -0
  4. package/dist/es/helpers/blobStorageHelper.js +185 -0
  5. package/dist/es/helpers/blobStorageHelper.js.map +1 -0
  6. package/dist/es/helpers/changeSetHelper.js +215 -0
  7. package/dist/es/helpers/changeSetHelper.js.map +1 -0
  8. package/dist/es/helpers/localSyncStateHelper.js +384 -0
  9. package/dist/es/helpers/localSyncStateHelper.js.map +1 -0
  10. package/dist/es/helpers/remoteSyncStateHelper.js +560 -0
  11. package/dist/es/helpers/remoteSyncStateHelper.js.map +1 -0
  12. package/dist/es/helpers/versions.js +6 -0
  13. package/dist/es/helpers/versions.js.map +1 -0
  14. package/dist/es/index.js +13 -0
  15. package/dist/es/index.js.map +1 -0
  16. package/dist/es/models/ISyncPointerStore.js +4 -0
  17. package/dist/es/models/ISyncPointerStore.js.map +1 -0
  18. package/dist/es/models/ISyncSnapshot.js +4 -0
  19. package/dist/es/models/ISyncSnapshot.js.map +1 -0
  20. package/dist/es/models/ISyncState.js +2 -0
  21. package/dist/es/models/ISyncState.js.map +1 -0
  22. package/dist/es/models/ISynchronisedStorageServiceConfig.js +4 -0
  23. package/dist/es/models/ISynchronisedStorageServiceConfig.js.map +1 -0
  24. package/dist/es/models/ISynchronisedStorageServiceConstructorOptions.js +2 -0
  25. package/dist/es/models/ISynchronisedStorageServiceConstructorOptions.js.map +1 -0
  26. package/dist/es/restEntryPoints.js +10 -0
  27. package/dist/es/restEntryPoints.js.map +1 -0
  28. package/dist/es/schema.js +11 -0
  29. package/dist/es/schema.js.map +1 -0
  30. package/dist/es/synchronisedStorageRoutes.js +142 -0
  31. package/dist/es/synchronisedStorageRoutes.js.map +1 -0
  32. package/dist/es/synchronisedStorageService.js +512 -0
  33. package/dist/es/synchronisedStorageService.js.map +1 -0
  34. package/dist/types/entities/syncSnapshotEntry.d.ts +3 -3
  35. package/dist/types/helpers/blobStorageHelper.d.ts +3 -3
  36. package/dist/types/helpers/changeSetHelper.d.ts +16 -32
  37. package/dist/types/helpers/localSyncStateHelper.d.ts +11 -11
  38. package/dist/types/helpers/remoteSyncStateHelper.d.ts +18 -14
  39. package/dist/types/index.d.ts +10 -10
  40. package/dist/types/models/ISyncState.d.ts +1 -1
  41. package/dist/types/models/ISynchronisedStorageServiceConfig.d.ts +7 -8
  42. package/dist/types/models/ISynchronisedStorageServiceConstructorOptions.d.ts +6 -6
  43. package/dist/types/synchronisedStorageRoutes.d.ts +1 -1
  44. package/dist/types/synchronisedStorageService.d.ts +17 -21
  45. package/docs/architecture.md +168 -12
  46. package/docs/changelog.md +149 -0
  47. package/docs/open-api/spec.json +62 -57
  48. package/docs/reference/classes/SyncSnapshotEntry.md +4 -10
  49. package/docs/reference/classes/SynchronisedStorageService.md +38 -50
  50. package/docs/reference/interfaces/ISynchronisedStorageServiceConfig.md +11 -17
  51. package/docs/reference/interfaces/ISynchronisedStorageServiceConstructorOptions.md +8 -8
  52. package/locales/en.json +7 -15
  53. package/package.json +26 -9
  54. package/dist/cjs/index.cjs +0 -2235
  55. package/dist/esm/index.mjs +0 -2227
@@ -1,2227 +0,0 @@
1
- import { property, entity, EntitySchemaFactory, EntitySchemaHelper, ComparisonOperator } from '@twin.org/entity';
2
- import { Guards, ComponentFactory, Is, Converter, Compression, CompressionType, ObjectHelper, BaseError, GeneralError, RandomHelper, NotFoundError, UnauthorizedError } from '@twin.org/core';
3
- import { HttpStatusCode } from '@twin.org/web';
4
- import { BlobStorageConnectorFactory } from '@twin.org/blob-storage-models';
5
- import { EntityStorageConnectorFactory } from '@twin.org/entity-storage-models';
6
- import { DocumentHelper, IdentityConnectorFactory } from '@twin.org/identity-models';
7
- import { ProofTypes } from '@twin.org/standards-w3c-did';
8
- import { SyncChangeOperation, SynchronisedStorageTopics, SyncNodeIdentityMode } from '@twin.org/synchronised-storage-models';
9
- import { VaultEncryptionType, VaultConnectorFactory } from '@twin.org/vault-models';
10
- import { VerifiableStorageConnectorFactory } from '@twin.org/verifiable-storage-models';
11
- import { RSA } from '@twin.org/crypto';
12
-
13
- // Copyright 2024 IOTA Stiftung.
14
- // SPDX-License-Identifier: Apache-2.0.
15
- /**
16
- * Class representing an entry for the sync snapshot.
17
- */
18
- let SyncSnapshotEntry = class SyncSnapshotEntry {
19
- /**
20
- * The id for the snapshot.
21
- */
22
- id;
23
- /**
24
- * The version for the snapshot.
25
- */
26
- version;
27
- /**
28
- * The storage key for the snapshot i.e. which entity is being synchronized.
29
- */
30
- storageKey;
31
- /**
32
- * The date the snapshot was created.
33
- */
34
- dateCreated;
35
- /**
36
- * The date the snapshot was last modified.
37
- */
38
- dateModified;
39
- /**
40
- * The flag to determine if this is the snapshot is the local one containing changes for this node.
41
- */
42
- isLocal;
43
- /**
44
- * The flag to determine if this is a consolidated snapshot.
45
- */
46
- isConsolidated;
47
- /**
48
- * The epoch for the changeset.
49
- */
50
- epoch;
51
- /**
52
- * The ids of the storage for the change sets in the snapshot, if this is not a local snapshot.
53
- */
54
- changeSetStorageIds;
55
- /**
56
- * The changes that were made in this snapshot, if this is a local snapshot.
57
- */
58
- changes;
59
- };
60
- __decorate([
61
- property({ type: "string", isPrimary: true }),
62
- __metadata("design:type", String)
63
- ], SyncSnapshotEntry.prototype, "id", void 0);
64
- __decorate([
65
- property({ type: "string" }),
66
- __metadata("design:type", String)
67
- ], SyncSnapshotEntry.prototype, "version", void 0);
68
- __decorate([
69
- property({ type: "string", isSecondary: true }),
70
- __metadata("design:type", String)
71
- ], SyncSnapshotEntry.prototype, "storageKey", void 0);
72
- __decorate([
73
- property({ type: "string" }),
74
- __metadata("design:type", String)
75
- ], SyncSnapshotEntry.prototype, "dateCreated", void 0);
76
- __decorate([
77
- property({ type: "string" }),
78
- __metadata("design:type", String)
79
- ], SyncSnapshotEntry.prototype, "dateModified", void 0);
80
- __decorate([
81
- property({ type: "boolean" }),
82
- __metadata("design:type", Boolean)
83
- ], SyncSnapshotEntry.prototype, "isLocal", void 0);
84
- __decorate([
85
- property({ type: "boolean" }),
86
- __metadata("design:type", Boolean)
87
- ], SyncSnapshotEntry.prototype, "isConsolidated", void 0);
88
- __decorate([
89
- property({ type: "number" }),
90
- __metadata("design:type", Number)
91
- ], SyncSnapshotEntry.prototype, "epoch", void 0);
92
- __decorate([
93
- property({ type: "array", itemType: "string", optional: true }),
94
- __metadata("design:type", Array)
95
- ], SyncSnapshotEntry.prototype, "changeSetStorageIds", void 0);
96
- __decorate([
97
- property({ type: "array", itemType: "object", optional: true }),
98
- __metadata("design:type", Array)
99
- ], SyncSnapshotEntry.prototype, "changes", void 0);
100
- SyncSnapshotEntry = __decorate([
101
- entity()
102
- ], SyncSnapshotEntry);
103
-
104
- /**
105
- * The source used when communicating about these routes.
106
- */
107
- const ROUTES_SOURCE = "synchronisedStorageRoutes";
108
- /**
109
- * The tag to associate with the routes.
110
- */
111
- const tagsSynchronisedStorage = [
112
- {
113
- name: "Synchronised Storage",
114
- description: "Endpoints which are modelled to access a synchronised storage contract."
115
- }
116
- ];
117
- /**
118
- * The REST routes for synchronised storage.
119
- * @param baseRouteName Prefix to prepend to the paths.
120
- * @param componentName The name of the component to use in the routes stored in the ComponentFactory.
121
- * @returns The generated routes.
122
- */
123
- function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
124
- const syncChangeSetRoute = {
125
- operationId: "synchronisedStorageSyncChangeSetRequest",
126
- summary: "Request that the node perform a sync request for a changeset.",
127
- tag: tagsSynchronisedStorage[0].name,
128
- method: "POST",
129
- path: `${baseRouteName}/sync-changeset`,
130
- handler: async (httpRequestContext, request) => synchronisedStorageSyncChangeSetRequest(httpRequestContext, componentName, request),
131
- requestType: {
132
- type: "ISyncChangeSetRequest",
133
- examples: [
134
- {
135
- id: "synchronisedStorageSyncChangeSetRequestExample",
136
- request: {
137
- body: {
138
- id: "0909090909090909090909090909090909090909090909090909090909090909",
139
- dateCreated: "2025-05-29T01:00:00.000Z",
140
- dateModified: "2025-05-29T01:00:00.000Z",
141
- nodeIdentity: "did:entity-storage:0xd2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2",
142
- changes: [
143
- {
144
- entity: {
145
- dateModified: "2025-01-01T00:00:00.000Z"
146
- },
147
- id: "test-id-1",
148
- operation: "set"
149
- }
150
- ],
151
- proof: {
152
- "@context": "https://www.w3.org/ns/credentials/v2",
153
- created: "2025-05-29T01:00:00.000Z",
154
- cryptosuite: "eddsa-jcs-2022",
155
- proofPurpose: "assertionMethod",
156
- proofValue: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3",
157
- type: "DataIntegrityProof",
158
- verificationMethod: "did:entity-storage:0xd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0#synchronised-storage-assertion"
159
- },
160
- storageKey: "test-type"
161
- }
162
- }
163
- }
164
- ]
165
- },
166
- responseType: [
167
- {
168
- type: "INoContentResponse"
169
- }
170
- ],
171
- // Authentication is provided by the proof in the request body.
172
- skipAuth: true
173
- };
174
- const getDecryptionKeyRoute = {
175
- operationId: "synchronisedStorageGetDecryptionKeyRequest",
176
- summary: "Request the decryption key.",
177
- tag: tagsSynchronisedStorage[0].name,
178
- method: "POST",
179
- path: `${baseRouteName}/decryption-key`,
180
- handler: async (httpRequestContext, request) => synchronisedStorageGetDecryptionKeyRequest(httpRequestContext, componentName, request),
181
- requestType: {
182
- type: "ISyncChangeSetRequest",
183
- examples: [
184
- {
185
- id: "synchronisedStorageSyncGetDecryptionKeyRequestExample",
186
- request: {
187
- body: {
188
- nodeIdentity: "did:entity-storage:0xd2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2",
189
- proof: {
190
- "@context": "https://www.w3.org/ns/credentials/v2",
191
- created: "2025-05-29T01:00:00.000Z",
192
- cryptosuite: "eddsa-jcs-2022",
193
- proofPurpose: "assertionMethod",
194
- proofValue: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3",
195
- type: "DataIntegrityProof",
196
- verificationMethod: "did:entity-storage:0xd0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0#synchronised-storage-assertion"
197
- }
198
- }
199
- }
200
- }
201
- ]
202
- },
203
- responseType: [
204
- {
205
- type: "ISyncDecryptionKeyResponse",
206
- examples: [
207
- {
208
- id: "synchronisedStorageSyncGetDecryptionKeyResponseExample",
209
- response: {
210
- body: {
211
- decryptionKey: "z5efBErQs3YBLZoH7jgKMQaRc9YjAxA5XSYKmW3FmTBDw9WionT2NS2x1SMvcRyBvw53cSSoaCT1xQH9tkWngGCX3"
212
- }
213
- }
214
- }
215
- ]
216
- },
217
- {
218
- type: "IUnauthorizedResponse"
219
- }
220
- ],
221
- // Authentication is provided by the proof in the request body.
222
- skipAuth: true
223
- };
224
- return [syncChangeSetRoute, getDecryptionKeyRoute];
225
- }
226
- /**
227
- * Perform the sync change set operation.
228
- * @param httpRequestContext The request context for the API.
229
- * @param componentName The name of the component to use in the routes.
230
- * @param request The request.
231
- * @returns The response object with additional http response properties.
232
- */
233
- async function synchronisedStorageSyncChangeSetRequest(httpRequestContext, componentName, request) {
234
- Guards.object(ROUTES_SOURCE, "request", request);
235
- Guards.object(ROUTES_SOURCE, "request.body", request.body);
236
- const component = ComponentFactory.get(componentName);
237
- await component.syncChangeSet(request.body);
238
- return {
239
- statusCode: HttpStatusCode.noContent
240
- };
241
- }
242
- /**
243
- * Request the decryption key.
244
- * @param httpRequestContext The request context for the API.
245
- * @param componentName The name of the component to use in the routes.
246
- * @param request The request.
247
- * @returns The response object with additional http response properties.
248
- */
249
- async function synchronisedStorageGetDecryptionKeyRequest(httpRequestContext, componentName, request) {
250
- Guards.object(ROUTES_SOURCE, "request", request);
251
- Guards.object(ROUTES_SOURCE, "request.body", request.body);
252
- const component = ComponentFactory.get(componentName);
253
- const key = await component.getDecryptionKey(request.body.nodeIdentity, request.body.proof);
254
- return {
255
- body: {
256
- decryptionKey: key
257
- }
258
- };
259
- }
260
-
261
- const restEntryPoints = [
262
- {
263
- name: "synchronised-storage",
264
- defaultBaseRoute: "synchronised-storage",
265
- tags: tagsSynchronisedStorage,
266
- generateRoutes: generateRestRoutesSynchronisedStorage
267
- }
268
- ];
269
-
270
- // Copyright 2024 IOTA Stiftung.
271
- // SPDX-License-Identifier: Apache-2.0.
272
- /**
273
- * Initialize the schema for the synchronised service.
274
- */
275
- function initSchema() {
276
- EntitySchemaFactory.register("SyncSnapshotEntry", () => EntitySchemaHelper.getSchema(SyncSnapshotEntry));
277
- }
278
-
279
- var mainnet = "verifiable:iota:0x0000000000000000000000000000000000000000000000000000000000000000:0x0000000000000000000000000000000000000000000000000000000000000000";
280
- var testnet = "verifiable:iota:0x0000000000000000000000000000000000000000000000000000000000000000:0x0000000000000000000000000000000000000000000000000000000000000000";
281
- var devnet = "verifiable:iota:0x0000000000000000000000000000000000000000000000000000000000000000:0x0000000000000000000000000000000000000000000000000000000000000000";
282
- var verifiableStorageKeys = {
283
- mainnet: mainnet,
284
- testnet: testnet,
285
- devnet: devnet
286
- };
287
-
288
- /**
289
- * Class for performing blob storage operations.
290
- */
291
- class BlobStorageHelper {
292
- /**
293
- * Runtime name for the class.
294
- */
295
- CLASS_NAME = "BlobStorageHelper";
296
- /**
297
- * The logging component to use for logging.
298
- * @internal
299
- */
300
- _loggingComponent;
301
- /**
302
- * The vault connector.
303
- * @internal
304
- */
305
- _vaultConnector;
306
- /**
307
- * The blob storage connector to use.
308
- * @internal
309
- */
310
- _blobStorageConnector;
311
- /**
312
- * The id of the vault key to use for encrypting/decrypting blobs.
313
- * @internal
314
- */
315
- _blobStorageEncryptionKeyId;
316
- /**
317
- * Is this a trusted node.
318
- * @internal
319
- */
320
- _isTrustedNode;
321
- /**
322
- * Create a new instance of BlobStorageHelper.
323
- * @param loggingComponent The logging connector to use for logging.
324
- * @param vaultConnector The vault connector to use for for the encryption key.
325
- * @param blobStorageConnector The blob storage component to use.
326
- * @param blobStorageEncryptionKeyId The id of the vault key to use for encrypting/decrypting blobs.
327
- * @param isTrustedNode Is this a trusted node.
328
- */
329
- constructor(loggingComponent, vaultConnector, blobStorageConnector, blobStorageEncryptionKeyId, isTrustedNode) {
330
- this._loggingComponent = loggingComponent;
331
- this._vaultConnector = vaultConnector;
332
- this._blobStorageConnector = blobStorageConnector;
333
- this._blobStorageEncryptionKeyId = blobStorageEncryptionKeyId;
334
- this._isTrustedNode = isTrustedNode;
335
- }
336
- /**
337
- * Load a blob from storage.
338
- * @param blobId The id of the blob to apply.
339
- * @returns The blob.
340
- */
341
- async loadBlob(blobId) {
342
- await this._loggingComponent?.log({
343
- level: "info",
344
- source: this.CLASS_NAME,
345
- message: "loadBlob",
346
- data: {
347
- blobId
348
- }
349
- });
350
- try {
351
- const encryptedBlob = await this._blobStorageConnector.get(blobId);
352
- if (Is.uint8Array(encryptedBlob)) {
353
- let compressedBlob;
354
- // If this is a trusted node, we can decrypt the blob using the vault
355
- if (this._isTrustedNode) {
356
- compressedBlob = await this._vaultConnector.decrypt(this._blobStorageEncryptionKeyId, VaultEncryptionType.Rsa2048, encryptedBlob);
357
- }
358
- else {
359
- // Otherwise we need the public key stored as a secret in the vault
360
- const key = await this._vaultConnector.getSecret(this._blobStorageEncryptionKeyId);
361
- const rsa = new RSA(Converter.base64ToBytes(key));
362
- compressedBlob = rsa.decrypt(encryptedBlob);
363
- }
364
- const decompressedBlob = await Compression.decompress(compressedBlob, CompressionType.Gzip);
365
- await this._loggingComponent?.log({
366
- level: "info",
367
- source: this.CLASS_NAME,
368
- message: "loadedBlob",
369
- data: {
370
- blobId
371
- }
372
- });
373
- return ObjectHelper.fromBytes(decompressedBlob);
374
- }
375
- }
376
- catch (error) {
377
- await this._loggingComponent?.log({
378
- level: "error",
379
- source: this.CLASS_NAME,
380
- message: "loadBlobFailed",
381
- data: {
382
- blobId
383
- },
384
- error: BaseError.fromError(error)
385
- });
386
- }
387
- await this._loggingComponent?.log({
388
- level: "info",
389
- source: this.CLASS_NAME,
390
- message: "loadBlobEmpty",
391
- data: {
392
- blobId
393
- }
394
- });
395
- }
396
- /**
397
- * Save a blob.
398
- * @param blob The blob to save.
399
- * @returns The id of the blob.
400
- */
401
- async saveBlob(blob) {
402
- await this._loggingComponent?.log({
403
- level: "info",
404
- source: this.CLASS_NAME,
405
- message: "saveBlob"
406
- });
407
- if (!this._isTrustedNode) {
408
- throw new GeneralError(this.CLASS_NAME, "notTrustedNode");
409
- }
410
- const compressedBlob = await Compression.compress(ObjectHelper.toBytes(blob), CompressionType.Gzip);
411
- const encryptedBlob = await this._vaultConnector.encrypt(this._blobStorageEncryptionKeyId, VaultEncryptionType.Rsa2048, compressedBlob);
412
- try {
413
- const blobId = await this._blobStorageConnector.set(encryptedBlob);
414
- await this._loggingComponent?.log({
415
- level: "info",
416
- source: this.CLASS_NAME,
417
- message: "savedBlob",
418
- data: {
419
- blobId
420
- }
421
- });
422
- return blobId;
423
- }
424
- catch (error) {
425
- await this._loggingComponent?.log({
426
- level: "error",
427
- source: this.CLASS_NAME,
428
- message: "saveBlobFailed",
429
- error: BaseError.fromError(error)
430
- });
431
- throw error;
432
- }
433
- }
434
- /**
435
- * Remove a blob from storage.
436
- * @param blobId The id of the blob to remove.
437
- * @returns Nothing.
438
- */
439
- async removeBlob(blobId) {
440
- await this._loggingComponent?.log({
441
- level: "info",
442
- source: this.CLASS_NAME,
443
- message: "removeBlob",
444
- data: {
445
- blobId
446
- }
447
- });
448
- try {
449
- await this._blobStorageConnector.remove(blobId);
450
- await this._loggingComponent?.log({
451
- level: "info",
452
- source: this.CLASS_NAME,
453
- message: "removedBlob",
454
- data: {
455
- blobId
456
- }
457
- });
458
- }
459
- catch (error) {
460
- await this._loggingComponent?.log({
461
- level: "error",
462
- source: this.CLASS_NAME,
463
- message: "removeBlobFailed",
464
- data: {
465
- blobId
466
- },
467
- error: BaseError.fromError(error)
468
- });
469
- }
470
- await this._loggingComponent?.log({
471
- level: "info",
472
- source: this.CLASS_NAME,
473
- message: "removeBlobEmpty",
474
- data: {
475
- blobId
476
- }
477
- });
478
- }
479
- }
480
-
481
- // Copyright 2024 IOTA Stiftung.
482
- // SPDX-License-Identifier: Apache-2.0.
483
- /**
484
- * Class for performing change set operations.
485
- */
486
- class ChangeSetHelper {
487
- /**
488
- * Runtime name for the class.
489
- */
490
- CLASS_NAME = "ChangeSetHelper";
491
- /**
492
- * The logging component to use for logging.
493
- * @internal
494
- */
495
- _loggingComponent;
496
- /**
497
- * The event bus component.
498
- * @internal
499
- */
500
- _eventBusComponent;
501
- /**
502
- * The blob storage helper to use for remote sync states.
503
- * @internal
504
- */
505
- _blobStorageHelper;
506
- /**
507
- * The identity connector to use for signing/verifying changesets.
508
- * @internal
509
- */
510
- _identityConnector;
511
- /**
512
- * The id of the identity method to use when signing/verifying changesets.
513
- * @internal
514
- */
515
- _decentralisedStorageMethodId;
516
- /**
517
- * The identity of the node that is performing the update.
518
- * @internal
519
- */
520
- _nodeIdentity;
521
- /**
522
- * Create a new instance of ChangeSetHelper.
523
- * @param loggingComponent The logging connector to use for logging.
524
- * @param eventBusComponent The event bus component to use for events.
525
- * @param identityConnector The identity connector to use for signing/verifying changesets.
526
- * @param blobStorageHelper The blob storage component to use for remote sync states.
527
- * @param decentralisedStorageMethodId The id of the identity method to use when signing/verifying changesets.
528
- */
529
- constructor(loggingComponent, eventBusComponent, identityConnector, blobStorageHelper, decentralisedStorageMethodId) {
530
- this._loggingComponent = loggingComponent;
531
- this._eventBusComponent = eventBusComponent;
532
- this._decentralisedStorageMethodId = decentralisedStorageMethodId;
533
- this._blobStorageHelper = blobStorageHelper;
534
- this._identityConnector = identityConnector;
535
- }
536
- /**
537
- * Set the node identity to use for signing changesets.
538
- * @param nodeIdentity The identity of the node that is performing the update.
539
- */
540
- setNodeIdentity(nodeIdentity) {
541
- this._nodeIdentity = nodeIdentity;
542
- }
543
- /**
544
- * Get and verify a changeset.
545
- * @param changeSetStorageId The id of the sync changeset to apply.
546
- * @returns The changeset if it was verified.
547
- */
548
- async getAndVerifyChangeset(changeSetStorageId) {
549
- await this._loggingComponent?.log({
550
- level: "info",
551
- source: this.CLASS_NAME,
552
- message: "getChangeSet",
553
- data: {
554
- changeSetStorageId
555
- }
556
- });
557
- try {
558
- const syncChangeSet = await this._blobStorageHelper.loadBlob(changeSetStorageId);
559
- if (Is.object(syncChangeSet)) {
560
- const verified = await this.verifyChangesetProof(syncChangeSet);
561
- return verified ? syncChangeSet : undefined;
562
- }
563
- }
564
- catch (error) {
565
- await this._loggingComponent?.log({
566
- level: "warn",
567
- source: this.CLASS_NAME,
568
- message: "getChangeSetError",
569
- data: {
570
- changeSetStorageId
571
- },
572
- error: BaseError.fromError(error)
573
- });
574
- }
575
- await this._loggingComponent?.log({
576
- level: "info",
577
- source: this.CLASS_NAME,
578
- message: "getChangeSetEmpty",
579
- data: {
580
- changeSetStorageId
581
- }
582
- });
583
- }
584
- /**
585
- * Apply a sync changeset.
586
- * @param changeSetStorageId The id of the sync changeset to apply.
587
- * @returns The changeset if it existed.
588
- */
589
- async getAndApplyChangeset(changeSetStorageId) {
590
- const syncChangeset = await this.getAndVerifyChangeset(changeSetStorageId);
591
- // Only apply changesets from other nodes, we don't want to overwrite
592
- // any changes we have made to local entity storage
593
- if (!Is.empty(syncChangeset) && syncChangeset.nodeIdentity !== this._nodeIdentity) {
594
- await this.applyChangeset(syncChangeset);
595
- }
596
- return syncChangeset;
597
- }
598
- /**
599
- * Apply a sync changeset.
600
- * @param syncChangeset The sync changeset to apply.
601
- * @returns Nothing.
602
- */
603
- async applyChangeset(syncChangeset) {
604
- if (Is.arrayValue(syncChangeset.changes)) {
605
- for (const change of syncChangeset.changes) {
606
- await this._loggingComponent?.log({
607
- level: "info",
608
- source: this.CLASS_NAME,
609
- message: "changeSetApplyingChange",
610
- data: {
611
- operation: change.operation,
612
- id: change.id
613
- }
614
- });
615
- switch (change.operation) {
616
- case SyncChangeOperation.Set:
617
- if (!Is.empty(change.entity)) {
618
- // The id was stripped from the entity as it is part of the operation
619
- // we make sure we reinstate it in the publish
620
- // Also the node identity was stripped when stored in the changeset
621
- // as the changeset is signed with the node identity.
622
- // so we need to restore it here.
623
- await this._eventBusComponent.publish(SynchronisedStorageTopics.RemoteItemSet, {
624
- storageKey: syncChangeset.storageKey,
625
- entity: {
626
- ...change.entity,
627
- id: change.id,
628
- nodeIdentity: syncChangeset.nodeIdentity
629
- }
630
- });
631
- }
632
- break;
633
- case SyncChangeOperation.Delete:
634
- if (!Is.empty(change.id)) {
635
- await this._eventBusComponent.publish(SynchronisedStorageTopics.RemoteItemRemove, {
636
- storageKey: syncChangeset.storageKey,
637
- id: change.id,
638
- nodeIdentity: syncChangeset.nodeIdentity
639
- });
640
- }
641
- break;
642
- }
643
- }
644
- }
645
- }
646
- /**
647
- * Store the changeset.
648
- * @param syncChangeSet The sync change set to store.
649
- * @returns The id of the change set.
650
- */
651
- async storeChangeSet(syncChangeSet) {
652
- await this._loggingComponent?.log({
653
- level: "info",
654
- source: this.CLASS_NAME,
655
- message: "changeSetStoring",
656
- data: {
657
- id: syncChangeSet.id
658
- }
659
- });
660
- return this._blobStorageHelper.saveBlob(syncChangeSet);
661
- }
662
- /**
663
- * Verify the proof of a sync changeset.
664
- * @param syncChangeset The sync changeset to verify.
665
- * @returns True if the proof is valid, false otherwise.
666
- */
667
- async verifyChangesetProof(syncChangeset) {
668
- if (Is.empty(syncChangeset.proof)) {
669
- await this._loggingComponent?.log({
670
- level: "info",
671
- source: this.CLASS_NAME,
672
- message: "verifyChangeSetProofMissing",
673
- data: {
674
- snapshotId: syncChangeset.id
675
- }
676
- });
677
- return false;
678
- }
679
- // If the proof or verification method is missing, the proof is invalid
680
- const verificationMethod = syncChangeset.proof?.verificationMethod;
681
- if (!Is.stringValue(verificationMethod)) {
682
- await this._loggingComponent?.log({
683
- level: "error",
684
- source: this.CLASS_NAME,
685
- message: "verifyChangeSetProofMissing",
686
- data: {
687
- id: syncChangeset.id
688
- }
689
- });
690
- }
691
- // Parse the verification method and extract the node identity
692
- // this should match the node identity of the changeset
693
- // otherwise you could sign a changeset for another node
694
- const changeSetNodeIdentity = DocumentHelper.parseId(verificationMethod ?? "");
695
- if (changeSetNodeIdentity.id !== syncChangeset.nodeIdentity) {
696
- await this._loggingComponent?.log({
697
- level: "error",
698
- source: this.CLASS_NAME,
699
- message: "verifyChangeSetProofNodeIdentityMismatch",
700
- data: {
701
- id: syncChangeset.id
702
- }
703
- });
704
- }
705
- const changeSetWithoutProof = ObjectHelper.clone(syncChangeset);
706
- delete changeSetWithoutProof.proof;
707
- const isValid = await this._identityConnector.verifyProof(changeSetWithoutProof, syncChangeset.proof);
708
- if (!isValid) {
709
- await this._loggingComponent?.log({
710
- level: "error",
711
- source: this.CLASS_NAME,
712
- message: "verifyChangeSetProofInvalid",
713
- data: {
714
- id: syncChangeset.id
715
- }
716
- });
717
- }
718
- else {
719
- await this._loggingComponent?.log({
720
- level: "error",
721
- source: this.CLASS_NAME,
722
- message: "verifyChangeSetProofValid",
723
- data: {
724
- id: syncChangeset.id
725
- }
726
- });
727
- }
728
- return isValid;
729
- }
730
- /**
731
- * Create the proof of a sync change set.
732
- * @param syncChangeset The sync changeset to create the proof for.
733
- * @returns The proof.
734
- */
735
- async createChangeSetProof(syncChangeset) {
736
- Guards.stringValue(this.CLASS_NAME, "nodeIdentity", this._nodeIdentity);
737
- const changeSetWithoutProof = ObjectHelper.clone(syncChangeset);
738
- delete changeSetWithoutProof.proof;
739
- const proof = await this._identityConnector.createProof(this._nodeIdentity, DocumentHelper.joinId(this._nodeIdentity, this._decentralisedStorageMethodId), ProofTypes.DataIntegrityProof, changeSetWithoutProof);
740
- await this._loggingComponent?.log({
741
- level: "info",
742
- source: this.CLASS_NAME,
743
- message: "createdChangeSetProof",
744
- data: {
745
- id: syncChangeset.id,
746
- ...proof
747
- }
748
- });
749
- return proof;
750
- }
751
- /**
752
- * Copy a change set.
753
- * @param syncChangeSet The sync changeset to copy.
754
- * @returns The id of the updated change set.
755
- */
756
- async copyChangeset(syncChangeSet) {
757
- if (Is.stringValue(this._nodeIdentity)) {
758
- const verified = await this.verifyChangesetProof(syncChangeSet);
759
- if (verified) {
760
- await this._loggingComponent?.log({
761
- level: "info",
762
- source: this.CLASS_NAME,
763
- message: "copyChangeSet",
764
- data: {
765
- changeSetStorageId: syncChangeSet.id
766
- }
767
- });
768
- // Allocate a new id to the changeset copy and re-create a proof using this nodes identity
769
- const copy = ObjectHelper.clone(syncChangeSet);
770
- copy.id = Converter.bytesToHex(RandomHelper.generate(32));
771
- copy.proof = await this.createChangeSetProof(copy);
772
- // Store the copy
773
- return {
774
- syncChangeSet: copy,
775
- changeSetStorageId: await this.storeChangeSet(copy)
776
- };
777
- }
778
- }
779
- }
780
- /**
781
- * Reset the storage for a given storage key.
782
- * @param storageKey The key of the storage to reset.
783
- * @param resetMode The reset mode, this will use the nodeIdentity in the entities to determine which are local/remote.
784
- * @returns Nothing.
785
- */
786
- async reset(storageKey, resetMode) {
787
- // If we are applying a consolidation we need to reset the local db
788
- // but keep any entries from the local node, as they might have been updated
789
- await this._loggingComponent?.log({
790
- level: "info",
791
- source: this.CLASS_NAME,
792
- message: "storageReset",
793
- data: {
794
- storageKey
795
- }
796
- });
797
- await this._eventBusComponent.publish(SynchronisedStorageTopics.Reset, {
798
- storageKey,
799
- resetMode
800
- });
801
- }
802
- }
803
-
804
- // Copyright 2024 IOTA Stiftung.
805
- // SPDX-License-Identifier: Apache-2.0.
806
- const SYNC_STATE_VERSION = "1";
807
- const SYNC_POINTER_STORE_VERSION = "1";
808
- const SYNC_SNAPSHOT_VERSION = "1";
809
-
810
- // Copyright 2024 IOTA Stiftung.
811
- // SPDX-License-Identifier: Apache-2.0.
812
- /**
813
- * Class for performing entity storage operations in decentralised storage.
814
- */
815
- class LocalSyncStateHelper {
816
- /**
817
- * Runtime name for the class.
818
- */
819
- CLASS_NAME = "LocalSyncStateHelper";
820
- /**
821
- * The logging component to use for logging.
822
- * @internal
823
- */
824
- _loggingComponent;
825
- /**
826
- * The storage connector for the sync snapshot entries.
827
- * @internal
828
- */
829
- _snapshotEntryEntityStorage;
830
- /**
831
- * The change set helper to use for applying changesets.
832
- * @internal
833
- */
834
- _changeSetHelper;
835
- /**
836
- * Create a new instance of LocalSyncStateHelper.
837
- * @param loggingComponent The logging connector to use for logging.
838
- * @param snapshotEntryEntityStorage The storage connector for the sync snapshot entries.
839
- * @param changeSetHelper The change set helper to use for applying changesets.
840
- */
841
- constructor(loggingComponent, snapshotEntryEntityStorage, changeSetHelper) {
842
- this._loggingComponent = loggingComponent;
843
- this._snapshotEntryEntityStorage = snapshotEntryEntityStorage;
844
- this._changeSetHelper = changeSetHelper;
845
- }
846
- /**
847
- * Add a new change to the local snapshot.
848
- * @param storageKey The storage key of the snapshot to add the change for.
849
- * @param operation The operation to perform.
850
- * @param id The id of the entity to add the change for.
851
- * @returns Nothing.
852
- */
853
- async addLocalChange(storageKey, operation, id) {
854
- await this._loggingComponent?.log({
855
- level: "info",
856
- source: this.CLASS_NAME,
857
- message: "addLocalChange",
858
- data: {
859
- storageKey,
860
- operation,
861
- id
862
- }
863
- });
864
- const localChangeSnapshots = await this.getSnapshots(storageKey, true);
865
- if (localChangeSnapshots.length > 0) {
866
- const localChangeSnapshot = localChangeSnapshots[0];
867
- localChangeSnapshot.changes ??= [];
868
- // If we already have a change for this id we are
869
- // about to supersede it, we remove the previous change
870
- // to avoid having multiple changes for the same id
871
- const previousChangeIndex = localChangeSnapshot.changes.findIndex(change => change.id === id);
872
- if (previousChangeIndex !== -1) {
873
- localChangeSnapshot.changes.splice(previousChangeIndex, 1);
874
- }
875
- // If we already have changes from previous updates
876
- // then make sure we update the dateModified, otherwise
877
- // we assume this is the first change and setting modified is not necessary
878
- if (localChangeSnapshot.changes.length > 0) {
879
- localChangeSnapshot.dateModified = new Date(Date.now()).toISOString();
880
- }
881
- localChangeSnapshot.changes.push({ operation, id });
882
- await this.setLocalChangeSnapshot(localChangeSnapshot);
883
- }
884
- }
885
- /**
886
- * Get the snapshot which contains just the changes for this node.
887
- * @param storageKey The storage key of the snapshot to get.
888
- * @param isLocal Whether to get the local snapshot or not.
889
- * @returns The local snapshot entry.
890
- */
891
- async getSnapshots(storageKey, isLocal) {
892
- await this._loggingComponent?.log({
893
- level: "info",
894
- source: this.CLASS_NAME,
895
- message: "getSnapshots",
896
- data: {
897
- storageKey
898
- }
899
- });
900
- const queryResult = await this._snapshotEntryEntityStorage.query({
901
- conditions: [
902
- {
903
- property: "isLocal",
904
- value: isLocal,
905
- comparison: ComparisonOperator.Equals
906
- },
907
- {
908
- property: "storageKey",
909
- value: storageKey,
910
- comparison: ComparisonOperator.Equals
911
- }
912
- ]
913
- });
914
- if (queryResult.entities.length > 0) {
915
- await this._loggingComponent?.log({
916
- level: "info",
917
- source: this.CLASS_NAME,
918
- message: "getSnapshotsExists",
919
- data: {
920
- storageKey
921
- }
922
- });
923
- return queryResult.entities;
924
- }
925
- await this._loggingComponent?.log({
926
- level: "info",
927
- source: this.CLASS_NAME,
928
- message: "getSnapshotsDoesNotExist",
929
- data: {
930
- storageKey
931
- }
932
- });
933
- const now = new Date(Date.now()).toISOString();
934
- return [
935
- {
936
- version: SYNC_SNAPSHOT_VERSION,
937
- id: Converter.bytesToHex(RandomHelper.generate(32)),
938
- storageKey,
939
- dateCreated: now,
940
- dateModified: now,
941
- changeSetStorageIds: [],
942
- isLocal,
943
- isConsolidated: false,
944
- epoch: 0
945
- }
946
- ];
947
- }
948
- /**
949
- * Set the current local snapshot with changes for this node.
950
- * @param localChangeSnapshot The local change snapshot to set.
951
- * @returns Nothing.
952
- */
953
- async setLocalChangeSnapshot(localChangeSnapshot) {
954
- await this._loggingComponent?.log({
955
- level: "info",
956
- source: this.CLASS_NAME,
957
- message: "setLocalChangeSnapshot",
958
- data: {
959
- storageKey: localChangeSnapshot.storageKey
960
- }
961
- });
962
- await this._snapshotEntryEntityStorage.set(localChangeSnapshot);
963
- }
964
- /**
965
- * Get the current local snapshot with the changes for this node.
966
- * @param localChangeSnapshot The local change snapshot to remove.
967
- * @returns Nothing.
968
- */
969
- async removeLocalChangeSnapshot(localChangeSnapshot) {
970
- await this._loggingComponent?.log({
971
- level: "info",
972
- source: this.CLASS_NAME,
973
- message: "removeLocalChangeSnapshot",
974
- data: {
975
- snapshotId: localChangeSnapshot.id
976
- }
977
- });
978
- await this._snapshotEntryEntityStorage.remove(localChangeSnapshot.id);
979
- }
980
- /**
981
- * Apply a sync state to the local node.
982
- * @param storageKey The storage key of the snapshot to sync with.
983
- * @param syncState The sync state to sync with.
984
- * @returns Nothing.
985
- */
986
- async applySyncState(storageKey, syncState) {
987
- await this._loggingComponent?.log({
988
- level: "info",
989
- source: this.CLASS_NAME,
990
- message: "applySyncState",
991
- data: {
992
- snapshotCount: syncState.snapshots.length
993
- }
994
- });
995
- // Get all the existing snapshots that we have processed previously
996
- let existingSnapshots = await this.getSnapshots(storageKey, false);
997
- // Sort from newest to oldest
998
- existingSnapshots = existingSnapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
999
- // Sort from newest to oldest
1000
- const syncStateSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
1001
- // Get the newest epoch from the local storage
1002
- const newestExistingEpoch = existingSnapshots[0]?.epoch ?? 0;
1003
- // Get the oldest epoch from the remote storage
1004
- const oldestSyncStateEpoch = syncStateSnapshots[syncStateSnapshots.length - 1]?.epoch ?? 0;
1005
- // If there is a gap between the largest epoch we have locally
1006
- // and the smallest epoch we have remotely then we have missed
1007
- // data so we need to perform a full sync
1008
- const hasEpochGap = newestExistingEpoch + 1 < oldestSyncStateEpoch;
1009
- // If we have an epoch gap or no existing snapshots then we need to apply
1010
- // a full sync from a consolidation
1011
- if (!existingSnapshots.some(s => s.isConsolidated) || hasEpochGap) {
1012
- await this._loggingComponent?.log({
1013
- level: "info",
1014
- source: this.CLASS_NAME,
1015
- message: "applySnapshotNoExisting",
1016
- data: {
1017
- storageKey
1018
- }
1019
- });
1020
- const mostRecentConsolidation = syncStateSnapshots.findIndex(snapshot => snapshot.isConsolidated);
1021
- if (mostRecentConsolidation !== -1) {
1022
- // We found the most recent consolidated snapshot, we can use it
1023
- await this._loggingComponent?.log({
1024
- level: "info",
1025
- source: this.CLASS_NAME,
1026
- message: "applySnapshotFoundConsolidated",
1027
- data: {
1028
- storageKey,
1029
- snapshotId: syncStateSnapshots[mostRecentConsolidation].id
1030
- }
1031
- });
1032
- // We need to reset the entity storage and remove all the remote items
1033
- // so that we use just the ones from the consolidation, since
1034
- // we don't have any existing there shouldn't be any remote entries
1035
- // but we reset nonetheless
1036
- await this._changeSetHelper.reset(storageKey, SyncNodeIdentityMode.Remote);
1037
- // We need to process the most recent consolidation and all changes
1038
- // that were made since then, from newest to oldest (so newer changes override older ones)
1039
- // Process snapshots from the consolidation point (most recent) back to the newest
1040
- for (let i = mostRecentConsolidation; i >= 0; i--) {
1041
- await this.processNewSnapshots([
1042
- {
1043
- ...syncStateSnapshots[i],
1044
- storageKey,
1045
- isLocal: false
1046
- }
1047
- ]);
1048
- }
1049
- }
1050
- else {
1051
- await this._loggingComponent?.log({
1052
- level: "info",
1053
- source: this.CLASS_NAME,
1054
- message: "applySnapshotNoConsolidated",
1055
- data: {
1056
- storageKey
1057
- }
1058
- });
1059
- }
1060
- }
1061
- else {
1062
- // We have existing consolidated remote snapshots, so we can assume that we have
1063
- // applied at least one consolidation snapshot, in this case we need to look at the changes since
1064
- // then and apply them if we haven't already
1065
- // We don't need to apply any additional consolidated snapshots, just the changesets
1066
- // Create a lookup map for the existing snapshots
1067
- const existingSnapshotsMap = {};
1068
- for (const snapshot of existingSnapshots) {
1069
- existingSnapshotsMap[snapshot.id] = snapshot;
1070
- }
1071
- const newSnapshots = [];
1072
- const modifiedSnapshots = [];
1073
- const referencedExistingSnapshots = Object.keys(existingSnapshotsMap);
1074
- let completedProcessing = false;
1075
- for (const snapshot of syncStateSnapshots) {
1076
- await this._loggingComponent?.log({
1077
- level: "info",
1078
- source: this.CLASS_NAME,
1079
- message: "applySnapshot",
1080
- data: {
1081
- snapshotId: snapshot.id,
1082
- dateCreated: new Date(snapshot.dateCreated).toISOString()
1083
- }
1084
- });
1085
- // See if we have the snapshot stored locally
1086
- const currentSnapshot = existingSnapshotsMap[snapshot.id];
1087
- // As we are referencing an existing snapshot, we need to remove it from the list
1088
- // to allow us to cleanup any unreferenced snapshots later
1089
- const idx = referencedExistingSnapshots.indexOf(snapshot.id);
1090
- if (idx !== -1) {
1091
- referencedExistingSnapshots.splice(idx, 1);
1092
- }
1093
- // No need to apply consolidated snapshots
1094
- if (!snapshot.isConsolidated && !completedProcessing) {
1095
- const updatedSnapshot = {
1096
- ...snapshot,
1097
- storageKey,
1098
- isLocal: false
1099
- };
1100
- if (Is.empty(currentSnapshot)) {
1101
- // We don't have the snapshot locally, so we need to process all of it
1102
- newSnapshots.push(updatedSnapshot);
1103
- }
1104
- else if (currentSnapshot.dateModified !== snapshot.dateModified) {
1105
- // If the local snapshot has a different dateModified, we need to update it
1106
- modifiedSnapshots.push({
1107
- currentSnapshot,
1108
- updatedSnapshot
1109
- });
1110
- }
1111
- else {
1112
- // we sorted the snapshots from newest to oldest, so if we found a local snapshot
1113
- // with the same dateModified as the remote snapshot, we can stop processing further
1114
- completedProcessing = true;
1115
- }
1116
- }
1117
- }
1118
- // We reverse the order of the snapshots to process them from oldest to newest
1119
- // because we want to apply the changes in the order they were created
1120
- await this.processModifiedSnapshots(modifiedSnapshots.reverse());
1121
- await this.processNewSnapshots(newSnapshots.reverse());
1122
- // Any ids remaining in this list are no longer referenced in the global state
1123
- // so we should remove them from the local storage as they will never be updated again
1124
- for (const referencedSnapshotId of referencedExistingSnapshots) {
1125
- await this._snapshotEntryEntityStorage.remove(referencedSnapshotId);
1126
- }
1127
- }
1128
- }
1129
- /**
1130
- * Process the modified snapshots and store them in the local storage.
1131
- * @param modifiedSnapshots The modified snapshots to process.
1132
- * @returns Nothing.
1133
- * @internal
1134
- */
1135
- async processModifiedSnapshots(modifiedSnapshots) {
1136
- for (const modifiedSnapshot of modifiedSnapshots) {
1137
- await this._loggingComponent?.log({
1138
- level: "info",
1139
- source: this.CLASS_NAME,
1140
- message: "processModifiedSnapshot",
1141
- data: {
1142
- snapshotId: modifiedSnapshot.updatedSnapshot.id,
1143
- localModified: new Date(modifiedSnapshot.currentSnapshot.dateModified ??
1144
- modifiedSnapshot.currentSnapshot.dateCreated).toISOString(),
1145
- remoteModified: new Date(modifiedSnapshot.updatedSnapshot.dateModified ??
1146
- modifiedSnapshot.updatedSnapshot.dateCreated).toISOString()
1147
- }
1148
- });
1149
- const remoteChangeSetStorageIds = modifiedSnapshot.updatedSnapshot.changeSetStorageIds;
1150
- const localChangeSetStorageIds = modifiedSnapshot.currentSnapshot.changeSetStorageIds ?? [];
1151
- if (Is.arrayValue(remoteChangeSetStorageIds)) {
1152
- for (const storageId of remoteChangeSetStorageIds) {
1153
- // Check if the local snapshot does not have the storageId
1154
- if (!localChangeSetStorageIds.includes(storageId)) {
1155
- await this._changeSetHelper.getAndApplyChangeset(storageId);
1156
- }
1157
- }
1158
- }
1159
- await this._snapshotEntryEntityStorage.set(modifiedSnapshot.updatedSnapshot);
1160
- }
1161
- }
1162
- /**
1163
- * Process the new snapshots and store them in the local storage.
1164
- * @param newSnapshots The new snapshots to process.
1165
- * @returns Nothing.
1166
- * @internal
1167
- */
1168
- async processNewSnapshots(newSnapshots) {
1169
- for (const newSnapshot of newSnapshots) {
1170
- await this._loggingComponent?.log({
1171
- level: "info",
1172
- source: this.CLASS_NAME,
1173
- message: "processNewSnapshot",
1174
- data: {
1175
- snapshotId: newSnapshot.id,
1176
- dateCreated: newSnapshot.dateCreated
1177
- }
1178
- });
1179
- const newSnapshotChangeSetStorageIds = newSnapshot.changeSetStorageIds ?? [];
1180
- if (Is.arrayValue(newSnapshotChangeSetStorageIds)) {
1181
- for (const storageId of newSnapshotChangeSetStorageIds) {
1182
- await this._changeSetHelper.getAndApplyChangeset(storageId);
1183
- }
1184
- }
1185
- await this._snapshotEntryEntityStorage.set(newSnapshot);
1186
- }
1187
- }
1188
- }
1189
-
1190
- // Copyright 2024 IOTA Stiftung.
1191
- // SPDX-License-Identifier: Apache-2.0.
1192
- /**
1193
- * Class for performing entity storage operations in decentralised storage.
1194
- */
1195
- class RemoteSyncStateHelper {
1196
- /**
1197
- * Runtime name for the class.
1198
- */
1199
- CLASS_NAME = "RemoteSyncStateHelper";
1200
- /**
1201
- * The logging component to use for logging.
1202
- * @internal
1203
- */
1204
- _loggingComponent;
1205
- /**
1206
- * The event bus component.
1207
- * @internal
1208
- */
1209
- _eventBusComponent;
1210
- /**
1211
- * The blob storage helper.
1212
- * @internal
1213
- */
1214
- _blobStorageHelper;
1215
- /**
1216
- * The verifiable storage connector to use for storing sync pointers.
1217
- * @internal
1218
- */
1219
- _verifiableSyncPointerStorageConnector;
1220
- /**
1221
- * The change set helper to use for applying changesets.
1222
- * @internal
1223
- */
1224
- _changeSetHelper;
1225
- /**
1226
- * The storage ids of the batch responses for each storage key.
1227
- * @internal
1228
- */
1229
- _batchResponseStorageIds;
1230
- /**
1231
- * The full changes for each storage key.
1232
- * @internal
1233
- */
1234
- _populateFullChanges;
1235
- /**
1236
- * The synchronised storage key to use for verified storage operations.
1237
- * @internal
1238
- */
1239
- _synchronisedStorageKey;
1240
- /**
1241
- * The identity of the node that is performing the update.
1242
- * @internal
1243
- */
1244
- _nodeIdentity;
1245
- /**
1246
- * Whether the node is trusted or not.
1247
- * @internal
1248
- */
1249
- _isTrustedNode;
1250
- /**
1251
- * Maximum number of consolidations to keep in storage.
1252
- * @internal
1253
- */
1254
- _maxConsolidations;
1255
- /**
1256
- * Create a new instance of DecentralisedEntityStorageConnector.
1257
- * @param loggingComponent The logging component to use for logging.
1258
- * @param eventBusComponent The event bus component to use for events.
1259
- * @param verifiableSyncPointerStorageConnector The verifiable storage connector to use for storing sync pointers.
1260
- * @param blobStorageHelper The blob storage helper to use for remote sync states.
1261
- * @param changeSetHelper The change set helper to use for managing changesets.
1262
- * @param isTrustedNode Whether the node is trusted or not.
1263
- * @param maxConsolidations The maximum number of consolidations to keep in storage.
1264
- */
1265
- constructor(loggingComponent, eventBusComponent, verifiableSyncPointerStorageConnector, blobStorageHelper, changeSetHelper, isTrustedNode, maxConsolidations) {
1266
- this._loggingComponent = loggingComponent;
1267
- this._eventBusComponent = eventBusComponent;
1268
- this._verifiableSyncPointerStorageConnector = verifiableSyncPointerStorageConnector;
1269
- this._changeSetHelper = changeSetHelper;
1270
- this._blobStorageHelper = blobStorageHelper;
1271
- this._isTrustedNode = isTrustedNode;
1272
- this._maxConsolidations = maxConsolidations;
1273
- this._batchResponseStorageIds = {};
1274
- this._populateFullChanges = {};
1275
- this._eventBusComponent.subscribe(SynchronisedStorageTopics.BatchResponse, async (response) => {
1276
- await this.handleBatchResponse(response.data);
1277
- });
1278
- this._eventBusComponent.subscribe(SynchronisedStorageTopics.LocalItemResponse, async (response) => {
1279
- await this.handleLocalItemResponse(response.data);
1280
- });
1281
- }
1282
- /**
1283
- * Set the node identity to use for signing changesets.
1284
- * @param nodeIdentity The identity of the node that is performing the update.
1285
- */
1286
- setNodeIdentity(nodeIdentity) {
1287
- this._nodeIdentity = nodeIdentity;
1288
- }
1289
- /**
1290
- * Set the synchronised storage key.
1291
- * @param synchronisedStorageKey The synchronised storage key to use.
1292
- */
1293
- setSynchronisedStorageKey(synchronisedStorageKey) {
1294
- this._synchronisedStorageKey = synchronisedStorageKey;
1295
- }
1296
- /**
1297
- * Build a changeset.
1298
- * @param storageKey The storage key of the change set.
1299
- * @param changes The changes to apply.
1300
- * @param completeCallback The callback to call when the changeset is created and stored.
1301
- * @returns The storage id of the change set if created.
1302
- */
1303
- async buildChangeSet(storageKey, changes, completeCallback) {
1304
- await this._loggingComponent?.log({
1305
- level: "info",
1306
- source: this.CLASS_NAME,
1307
- message: "buildingChangeSet",
1308
- data: {
1309
- storageKey,
1310
- changeCount: changes.length
1311
- }
1312
- });
1313
- this._populateFullChanges[storageKey] = {
1314
- changes,
1315
- entities: {},
1316
- requestIds: [],
1317
- completeCallback: async () => this.finaliseFullChanges(storageKey, completeCallback)
1318
- };
1319
- const setChanges = changes.filter(c => c.operation === SyncChangeOperation.Set);
1320
- if (setChanges.length === 0) {
1321
- // If we don't need to request any full details, we can just call the complete callback
1322
- await this.finaliseFullChanges(storageKey, completeCallback);
1323
- }
1324
- else {
1325
- // Otherwise we need to request the full details for each change
1326
- this._populateFullChanges[storageKey].requestIds = setChanges.map(change => change.id);
1327
- // Once all the requests are handled the callback will be called
1328
- for (const change of setChanges) {
1329
- // Create a request for each change to populate the full details
1330
- await this._loggingComponent?.log({
1331
- level: "info",
1332
- source: this.CLASS_NAME,
1333
- message: "createChangeSetRequestingItem",
1334
- data: {
1335
- storageKey,
1336
- id: change.id
1337
- }
1338
- });
1339
- this._eventBusComponent.publish(SynchronisedStorageTopics.LocalItemRequest, {
1340
- storageKey,
1341
- id: change.id
1342
- });
1343
- }
1344
- }
1345
- }
1346
- /**
1347
- * Finalise the full details for the sync change set.
1348
- * @param storageKey The storage key of the change set.
1349
- * @param completeCallback The callback to call when the changeset is populated.
1350
- * @returns Nothing.
1351
- */
1352
- async finaliseFullChanges(storageKey, completeCallback) {
1353
- await this._loggingComponent?.log({
1354
- level: "info",
1355
- source: this.CLASS_NAME,
1356
- message: "finalisingSyncChanges",
1357
- data: {
1358
- storageKey
1359
- }
1360
- });
1361
- if (Is.stringValue(this._nodeIdentity)) {
1362
- const changes = this._populateFullChanges[storageKey].changes;
1363
- for (const change of changes) {
1364
- change.entity = this._populateFullChanges[storageKey].entities[change.id] ?? change.entity;
1365
- if (change.operation === SyncChangeOperation.Set && Is.objectValue(change.entity)) {
1366
- // Remove the id from the entity as this is stored in the operation
1367
- // and will be reinstated when the changeset is reconstituted
1368
- ObjectHelper.propertyDelete(change.entity, "id");
1369
- // Remove the node identity as the changeset has this stored at the top level
1370
- // and we do not want to store it in the change itself to reduce redundancy
1371
- ObjectHelper.propertyDelete(change.entity, "nodeIdentity");
1372
- }
1373
- }
1374
- const now = new Date(Date.now()).toISOString();
1375
- const syncChangeSet = {
1376
- id: Converter.bytesToHex(RandomHelper.generate(32)),
1377
- dateCreated: now,
1378
- dateModified: now,
1379
- storageKey,
1380
- changes,
1381
- nodeIdentity: this._nodeIdentity
1382
- };
1383
- try {
1384
- // And sign it with the node identity
1385
- syncChangeSet.proof = await this._changeSetHelper.createChangeSetProof(syncChangeSet);
1386
- // If this is a trusted node, we also store the changeset
1387
- let changeSetStorageId;
1388
- if (this._isTrustedNode) {
1389
- changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
1390
- }
1391
- await completeCallback(syncChangeSet, changeSetStorageId);
1392
- }
1393
- catch (err) {
1394
- await this._loggingComponent?.log({
1395
- level: "error",
1396
- source: this.CLASS_NAME,
1397
- message: "finalisingSyncChangesFailed",
1398
- data: {
1399
- storageKey
1400
- },
1401
- error: BaseError.fromError(err)
1402
- });
1403
- await completeCallback();
1404
- }
1405
- }
1406
- else {
1407
- await completeCallback();
1408
- }
1409
- }
1410
- /**
1411
- * Add a new changeset into the sync state.
1412
- * @param storageKey The storage key of the change set to add.
1413
- * @param changeSetStorageId The id of the change set to add the the current state
1414
- * @returns Nothing.
1415
- */
1416
- async addChangeSetToSyncState(storageKey, changeSetStorageId) {
1417
- await this._loggingComponent?.log({
1418
- level: "info",
1419
- source: this.CLASS_NAME,
1420
- message: "addChangeSetToSyncState",
1421
- data: {
1422
- storageKey,
1423
- changeSetStorageId
1424
- }
1425
- });
1426
- // First load the sync pointer store to get the current sync pointer for the storage key
1427
- const syncPointerStore = await this.getVerifiableSyncPointerStore();
1428
- let syncState;
1429
- if (!Is.empty(syncPointerStore.syncPointers[storageKey])) {
1430
- syncState = await this.getSyncState(syncPointerStore.syncPointers[storageKey]);
1431
- }
1432
- // No current sync state, so we create a new one
1433
- if (Is.empty(syncState)) {
1434
- syncState = { version: SYNC_STATE_VERSION, storageKey, snapshots: [] };
1435
- }
1436
- // Sort the snapshots so the newest snapshot is last in the array
1437
- const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
1438
- // Get the current snapshot, if it does not exist we create a new one
1439
- let currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
1440
- const currentEpoch = currentSnapshot?.epoch ?? 0;
1441
- const now = new Date(Date.now()).toISOString();
1442
- // If there is no snapshot or the current one is a consolidation
1443
- // we start a new snapshot
1444
- if (Is.empty(currentSnapshot) || currentSnapshot.isConsolidated) {
1445
- currentSnapshot = {
1446
- version: SYNC_SNAPSHOT_VERSION,
1447
- id: Converter.bytesToHex(RandomHelper.generate(32)),
1448
- dateCreated: now,
1449
- dateModified: now,
1450
- isConsolidated: false,
1451
- epoch: currentEpoch + 1,
1452
- changeSetStorageIds: []
1453
- };
1454
- syncState.snapshots.push(currentSnapshot);
1455
- }
1456
- else {
1457
- // Snapshot exists, we update the dateModified
1458
- currentSnapshot.dateModified = now;
1459
- }
1460
- // Add the changeset storage id to the current snapshot
1461
- currentSnapshot.changeSetStorageIds.push(changeSetStorageId);
1462
- // Store the sync state in the blob storage
1463
- syncPointerStore.syncPointers[storageKey] = await this.storeRemoteSyncState(syncState);
1464
- // Store the verifiable sync pointer store in the verifiable storage
1465
- await this.storeVerifiableSyncPointerStore(syncPointerStore);
1466
- }
1467
- /**
1468
- * Create a consolidated snapshot for the entire storage.
1469
- * @param storageKey The storage key of the snapshot to create.
1470
- * @param batchSize The batch size to use for consolidation.
1471
- * @returns Nothing.
1472
- */
1473
- async consolidationStart(storageKey, batchSize) {
1474
- await this._loggingComponent?.log({
1475
- level: "info",
1476
- source: this.CLASS_NAME,
1477
- message: "consolidationStarting"
1478
- });
1479
- // Perform a batch request to start the consolidation
1480
- await this._eventBusComponent.publish(SynchronisedStorageTopics.BatchRequest, { storageKey, batchSize, requestMode: SyncNodeIdentityMode.All });
1481
- }
1482
- /**
1483
- * Get the sync pointer store.
1484
- * @returns The sync pointer store.
1485
- */
1486
- async getVerifiableSyncPointerStore() {
1487
- if (Is.stringValue(this._synchronisedStorageKey)) {
1488
- try {
1489
- await this._loggingComponent?.log({
1490
- level: "info",
1491
- source: this.CLASS_NAME,
1492
- message: "verifiableSyncPointerStoreRetrieving",
1493
- data: {
1494
- key: this._synchronisedStorageKey
1495
- }
1496
- });
1497
- const syncPointerStore = await this._verifiableSyncPointerStorageConnector.get(this._synchronisedStorageKey, { includeData: true });
1498
- if (Is.uint8Array(syncPointerStore.data)) {
1499
- const syncPointer = ObjectHelper.fromBytes(syncPointerStore.data);
1500
- await this._loggingComponent?.log({
1501
- level: "info",
1502
- source: this.CLASS_NAME,
1503
- message: "verifiableSyncPointerStoreRetrieved",
1504
- data: {
1505
- key: this._synchronisedStorageKey
1506
- }
1507
- });
1508
- return syncPointer;
1509
- }
1510
- }
1511
- catch (err) {
1512
- if (!BaseError.someErrorName(err, NotFoundError.CLASS_NAME)) {
1513
- throw err;
1514
- }
1515
- }
1516
- await this._loggingComponent?.log({
1517
- level: "info",
1518
- source: this.CLASS_NAME,
1519
- message: "verifiableSyncPointerStoreNotFound",
1520
- data: {
1521
- key: this._synchronisedStorageKey
1522
- }
1523
- });
1524
- }
1525
- // If no sync pointer store exists, we return an empty one
1526
- return {
1527
- version: SYNC_POINTER_STORE_VERSION,
1528
- syncPointers: {}
1529
- };
1530
- }
1531
- /**
1532
- * Store the verifiable sync pointer in the verifiable storage.
1533
- * @param syncPointerStore The sync pointer store to store.
1534
- * @returns Nothing.
1535
- */
1536
- async storeVerifiableSyncPointerStore(syncPointerStore) {
1537
- if (Is.stringValue(this._nodeIdentity) && Is.stringValue(this._synchronisedStorageKey)) {
1538
- await this._loggingComponent?.log({
1539
- level: "info",
1540
- source: this.CLASS_NAME,
1541
- message: "verifiableSyncPointerStoreStoring",
1542
- data: {
1543
- key: this._synchronisedStorageKey
1544
- }
1545
- });
1546
- // Store the verifiable sync pointer in the verifiable storage
1547
- await this._verifiableSyncPointerStorageConnector.update(this._nodeIdentity, this._synchronisedStorageKey, ObjectHelper.toBytes(syncPointerStore));
1548
- }
1549
- }
1550
- /**
1551
- * Store the remote sync state.
1552
- * @param syncState The sync state to store.
1553
- * @returns The id of the sync state.
1554
- */
1555
- async storeRemoteSyncState(syncState) {
1556
- await this._loggingComponent?.log({
1557
- level: "info",
1558
- source: this.CLASS_NAME,
1559
- message: "syncStateStoring",
1560
- data: {
1561
- snapshotCount: syncState.snapshots.length
1562
- }
1563
- });
1564
- // Limits the number of consolidations in the list so that we can shrink decentralised
1565
- // storage requirements, sort from newest to oldest so that we can easily find the
1566
- // oldest snapshots to remove.
1567
- const snapshots = syncState.snapshots.sort((a, b) => new Date(a.dateCreated).getTime() - new Date(b.dateCreated).getTime());
1568
- // Find all the consolidation indexes
1569
- const consolidationIndexes = [];
1570
- for (let i = 0; i < snapshots.length; i++) {
1571
- const snapshot = snapshots[i];
1572
- if (snapshot.isConsolidated) {
1573
- consolidationIndexes.push(i);
1574
- }
1575
- }
1576
- if (consolidationIndexes.length > this._maxConsolidations) {
1577
- // Once we have reached the max for consolidations we need to remove
1578
- // all the snapshots, including non consolidated ones, beyond this point
1579
- const toRemove = snapshots.slice(consolidationIndexes[this._maxConsolidations - 1] + 1);
1580
- syncState.snapshots = snapshots.slice(0, consolidationIndexes[this._maxConsolidations - 1] + 1);
1581
- for (const snapshot of toRemove) {
1582
- // We need to remove all the storage ids associated with the snapshot
1583
- if (Is.arrayValue(snapshot.changeSetStorageIds)) {
1584
- for (const storageId of snapshot.changeSetStorageIds) {
1585
- await this._blobStorageHelper.removeBlob(storageId);
1586
- }
1587
- }
1588
- }
1589
- }
1590
- return this._blobStorageHelper.saveBlob(syncState);
1591
- }
1592
- /**
1593
- * Get the remote sync state.
1594
- * @param syncPointerId The id of the sync pointer to retrieve the state for.
1595
- * @returns The remote sync state.
1596
- */
1597
- async getSyncState(syncPointerId) {
1598
- try {
1599
- await this._loggingComponent?.log({
1600
- level: "info",
1601
- source: this.CLASS_NAME,
1602
- message: "syncStateRetrieving",
1603
- data: {
1604
- syncPointerId
1605
- }
1606
- });
1607
- const syncState = await this._blobStorageHelper.loadBlob(syncPointerId);
1608
- if (Is.object(syncState)) {
1609
- await this._loggingComponent?.log({
1610
- level: "info",
1611
- source: this.CLASS_NAME,
1612
- message: "syncStateRetrieved",
1613
- data: {
1614
- syncPointerId,
1615
- snapshotCount: syncState.snapshots.length
1616
- }
1617
- });
1618
- return syncState;
1619
- }
1620
- }
1621
- catch (error) {
1622
- await this._loggingComponent?.log({
1623
- level: "warn",
1624
- source: this.CLASS_NAME,
1625
- message: "getSyncStateError",
1626
- data: {
1627
- syncPointerId
1628
- },
1629
- error: BaseError.fromError(error)
1630
- });
1631
- }
1632
- await this._loggingComponent?.log({
1633
- level: "info",
1634
- source: this.CLASS_NAME,
1635
- message: "syncStateNotFound",
1636
- data: {
1637
- syncPointerId
1638
- }
1639
- });
1640
- }
1641
- /**
1642
- * Handle the batch response which is triggered from a consolidation request.
1643
- * @param response The batch response to handle.
1644
- */
1645
- async handleBatchResponse(response) {
1646
- if (Is.stringValue(this._nodeIdentity)) {
1647
- const now = new Date(Date.now()).toISOString();
1648
- // Create a new snapshot entry for the current batch
1649
- const syncChangeSet = {
1650
- id: Converter.bytesToHex(RandomHelper.generate(32)),
1651
- dateCreated: now,
1652
- dateModified: now,
1653
- changes: response.entities.map(change => ({
1654
- operation: SyncChangeOperation.Set,
1655
- id: change.id
1656
- })),
1657
- storageKey: response.storageKey,
1658
- nodeIdentity: this._nodeIdentity
1659
- };
1660
- // And sign it with the node identity
1661
- syncChangeSet.proof = await this._changeSetHelper.createChangeSetProof(syncChangeSet);
1662
- // Store the changeset in the blob storage
1663
- const changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
1664
- // Add the changeset storage id to the snapshot ids
1665
- this._batchResponseStorageIds[response.storageKey] ??= [];
1666
- this._batchResponseStorageIds[response.storageKey].push(changeSetStorageId);
1667
- // If this is the last entry in the batch response, we can create the consolidated snapshot
1668
- if (response.lastEntry) {
1669
- // Get the current sync pointer store
1670
- const syncPointerStore = await this.getVerifiableSyncPointerStore();
1671
- let syncState;
1672
- if (Is.stringValue(syncPointerStore.syncPointers[response.storageKey])) {
1673
- // If the sync pointer exists, we load the current sync state
1674
- syncState = await this.getSyncState(syncPointerStore.syncPointers[response.storageKey]);
1675
- }
1676
- // If the sync state does not exist, we create a new one
1677
- syncState ??= {
1678
- version: SYNC_STATE_VERSION,
1679
- storageKey: response.storageKey,
1680
- snapshots: []
1681
- };
1682
- // Sort the snapshots so the newest snapshot is last in the array
1683
- const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
1684
- const currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
1685
- const currentEpoch = currentSnapshot?.epoch ?? 0;
1686
- const batchSnapshot = {
1687
- version: SYNC_SNAPSHOT_VERSION,
1688
- id: Converter.bytesToHex(RandomHelper.generate(32)),
1689
- dateCreated: now,
1690
- dateModified: now,
1691
- isConsolidated: true,
1692
- epoch: currentEpoch + 1,
1693
- changeSetStorageIds: this._batchResponseStorageIds[response.storageKey]
1694
- };
1695
- syncState.snapshots.push(batchSnapshot);
1696
- // Store the updated sync state
1697
- const syncStateId = await this.storeRemoteSyncState(syncState);
1698
- syncPointerStore.syncPointers[response.storageKey] = syncStateId;
1699
- // Store the verifiable sync pointer in the verifiable storage
1700
- await this.storeVerifiableSyncPointerStore(syncPointerStore);
1701
- // Remove the batch response storage ids for the storage key
1702
- // as we have consolidated the changes
1703
- delete this._batchResponseStorageIds[response.storageKey];
1704
- await this._loggingComponent?.log({
1705
- level: "info",
1706
- source: this.CLASS_NAME,
1707
- message: "consolidationCompleted"
1708
- });
1709
- }
1710
- }
1711
- }
1712
- /**
1713
- * Handle the item response.
1714
- * @param response The item response to handle.
1715
- */
1716
- async handleLocalItemResponse(response) {
1717
- await this._loggingComponent?.log({
1718
- level: "info",
1719
- source: this.CLASS_NAME,
1720
- message: "createChangeSetRespondingItem",
1721
- data: {
1722
- storageKey: response.storageKey,
1723
- id: response.id
1724
- }
1725
- });
1726
- // We have received a response to an item request, find the right storage
1727
- // for the request id
1728
- if (!Is.empty(this._populateFullChanges[response.storageKey])) {
1729
- const idx = this._populateFullChanges[response.storageKey].requestIds.indexOf(response.id);
1730
- if (idx !== -1) {
1731
- this._populateFullChanges[response.storageKey].requestIds.splice(idx, 1);
1732
- this._populateFullChanges[response.storageKey].entities[response.id] = response.entity;
1733
- // If there are no request ids remaining we can complete the population
1734
- if (this._populateFullChanges[response.storageKey].requestIds.length === 0) {
1735
- await this._populateFullChanges[response.storageKey].completeCallback();
1736
- }
1737
- }
1738
- }
1739
- }
1740
- }
1741
-
1742
- /**
1743
- * Class for performing synchronised storage operations.
1744
- */
1745
- class SynchronisedStorageService {
1746
- /**
1747
- * The default interval to check for entity updates.
1748
- * @internal
1749
- */
1750
- static _DEFAULT_ENTITY_UPDATE_INTERVAL_MINUTES = 5;
1751
- /**
1752
- * The default interval to perform consolidation.
1753
- * @internal
1754
- */
1755
- static _DEFAULT_CONSOLIDATION_INTERVAL_MINUTES = 60;
1756
- /**
1757
- * The default size of a consolidation batch.
1758
- * @internal
1759
- */
1760
- static _DEFAULT_CONSOLIDATION_BATCH_SIZE = 100;
1761
- /**
1762
- * The default max number of consolidations to keep in storage.
1763
- * @internal
1764
- */
1765
- static _DEFAULT_MAX_CONSOLIDATIONS = 5;
1766
- /**
1767
- * Runtime name for the class.
1768
- */
1769
- CLASS_NAME = "SynchronisedStorageService";
1770
- /**
1771
- * The logging component to use for logging.
1772
- * @internal
1773
- */
1774
- _loggingComponent;
1775
- /**
1776
- * The event bus component.
1777
- * @internal
1778
- */
1779
- _eventBusComponent;
1780
- /**
1781
- * The vault connector.
1782
- * @internal
1783
- */
1784
- _vaultConnector;
1785
- /**
1786
- * The storage connector for the sync snapshot entries.
1787
- * @internal
1788
- */
1789
- _localSyncSnapshotEntryEntityStorage;
1790
- /**
1791
- * The blob storage connector to use for remote sync states.
1792
- * @internal
1793
- */
1794
- _blobStorageConnector;
1795
- /**
1796
- * The verifiable storage connector to use for storing sync pointers.
1797
- * @internal
1798
- */
1799
- _verifiableSyncPointerStorageConnector;
1800
- /**
1801
- * The identity connector to use for signing/verifying changesets.
1802
- * @internal
1803
- */
1804
- _identityConnector;
1805
- /**
1806
- * The task scheduler component.
1807
- * @internal
1808
- */
1809
- _taskSchedulerComponent;
1810
- /**
1811
- * The synchronised storage service to use when this is not a trusted node.
1812
- * @internal
1813
- */
1814
- _trustedSynchronisedStorageComponent;
1815
- /**
1816
- * The blob storage helper.
1817
- * @internal
1818
- */
1819
- _blobStorageHelper;
1820
- /**
1821
- * The change set helper.
1822
- * @internal
1823
- */
1824
- _changeSetHelper;
1825
- /**
1826
- * The local sync state helper to use for applying changesets.
1827
- * @internal
1828
- */
1829
- _localSyncStateHelper;
1830
- /**
1831
- * The remote sync state helper to use for applying changesets.
1832
- * @internal
1833
- */
1834
- _remoteSyncStateHelper;
1835
- /**
1836
- * The options for the connector.
1837
- * @internal
1838
- */
1839
- _config;
1840
- /**
1841
- * The synchronised storage key to use for the remote synchronised storage.
1842
- * @internal
1843
- */
1844
- _synchronisedStorageKey;
1845
- /**
1846
- * The flag to determine if the service has been started.
1847
- * @internal
1848
- */
1849
- _serviceStarted;
1850
- /**
1851
- * The active storage keys for the synchronised storage service.
1852
- * @internal
1853
- */
1854
- _activeStorageKeys;
1855
- /**
1856
- * The identity of the node this connector is running on.
1857
- * @internal
1858
- */
1859
- _nodeIdentity;
1860
- /**
1861
- * Create a new instance of SynchronisedStorageService.
1862
- * @param options The options for the service.
1863
- */
1864
- constructor(options) {
1865
- Guards.object(this.CLASS_NAME, "options", options);
1866
- Guards.object(this.CLASS_NAME, "options.config", options.config);
1867
- Guards.stringValue(this.CLASS_NAME, "options.config.verifiableStorageKeyId", options.config.verifiableStorageKeyId);
1868
- this._eventBusComponent = ComponentFactory.get(options.eventBusComponentType ?? "event-bus");
1869
- this._loggingComponent = ComponentFactory.getIfExists(options.loggingComponentType ?? "logging");
1870
- this._vaultConnector = VaultConnectorFactory.get(options.vaultConnectorType ?? "vault");
1871
- this._localSyncSnapshotEntryEntityStorage = EntityStorageConnectorFactory.get(options.syncSnapshotStorageConnectorType ?? "sync-snapshot-entry");
1872
- this._verifiableSyncPointerStorageConnector = VerifiableStorageConnectorFactory.get(options.verifiableStorageConnectorType ?? "verifiable-storage");
1873
- this._blobStorageConnector = BlobStorageConnectorFactory.get(options.blobStorageConnectorType ?? "blob-storage");
1874
- this._identityConnector = IdentityConnectorFactory.get(options.identityConnectorType ?? "identity");
1875
- this._taskSchedulerComponent = ComponentFactory.get(options.taskSchedulerComponentType ?? "task-scheduler");
1876
- // If this is empty we assume the local node has the rights to write to the verifiable storage.
1877
- let isTrustedNode = true;
1878
- if (!Is.empty(options.trustedSynchronisedStorageComponentType)) {
1879
- isTrustedNode = false;
1880
- // If it is set then we used the trusted component to send changesets to
1881
- this._trustedSynchronisedStorageComponent =
1882
- ComponentFactory.get(options.trustedSynchronisedStorageComponentType);
1883
- }
1884
- this._config = {
1885
- synchronisedStorageMethodId: options.config.synchronisedStorageMethodId ?? "synchronised-storage-assertion",
1886
- entityUpdateIntervalMinutes: options.config.entityUpdateIntervalMinutes ??
1887
- SynchronisedStorageService._DEFAULT_ENTITY_UPDATE_INTERVAL_MINUTES,
1888
- consolidationIntervalMinutes: options.config.consolidationIntervalMinutes ??
1889
- SynchronisedStorageService._DEFAULT_CONSOLIDATION_INTERVAL_MINUTES,
1890
- consolidationBatchSize: options.config.consolidationBatchSize ??
1891
- SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE,
1892
- maxConsolidations: options.config.maxConsolidations ?? SynchronisedStorageService._DEFAULT_MAX_CONSOLIDATIONS,
1893
- blobStorageEncryptionKeyId: options.config.blobStorageEncryptionKeyId ?? "synchronised-storage-blob-encryption-key",
1894
- verifiableStorageKeyId: options.config.verifiableStorageKeyId
1895
- };
1896
- this._synchronisedStorageKey =
1897
- verifiableStorageKeys[options.config.verifiableStorageKeyId] ?? options.config.verifiableStorageKeyId;
1898
- Guards.stringValue(this.CLASS_NAME, "synchronisedStorageKey", this._synchronisedStorageKey);
1899
- this._blobStorageHelper = new BlobStorageHelper(this._loggingComponent, this._vaultConnector, this._blobStorageConnector, this._config.blobStorageEncryptionKeyId, isTrustedNode);
1900
- this._changeSetHelper = new ChangeSetHelper(this._loggingComponent, this._eventBusComponent, this._identityConnector, this._blobStorageHelper, this._config.synchronisedStorageMethodId);
1901
- this._localSyncStateHelper = new LocalSyncStateHelper(this._loggingComponent, this._localSyncSnapshotEntryEntityStorage, this._changeSetHelper);
1902
- this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._loggingComponent, this._eventBusComponent, this._verifiableSyncPointerStorageConnector, this._blobStorageHelper, this._changeSetHelper, isTrustedNode, this._config.maxConsolidations);
1903
- this._serviceStarted = false;
1904
- this._activeStorageKeys = {};
1905
- this._eventBusComponent.subscribe(SynchronisedStorageTopics.RegisterStorageKey, async (event) => this.registerStorageKey(event.data));
1906
- this._eventBusComponent.subscribe(SynchronisedStorageTopics.LocalItemChange, async (event) => {
1907
- // Make sure the change event is from this node
1908
- if (Is.stringValue(this._nodeIdentity) && this._nodeIdentity === event.data.nodeIdentity) {
1909
- await this._localSyncStateHelper.addLocalChange(event.data.storageKey, event.data.operation, event.data.id);
1910
- }
1911
- });
1912
- }
1913
- /**
1914
- * The component needs to be started when the node is initialized.
1915
- * @param nodeIdentity The identity of the node starting the component.
1916
- * @param nodeLoggingConnectorType The node logging connector type, defaults to "node-logging".
1917
- * @param componentState A persistent state which can be modified by the method.
1918
- * @returns Nothing.
1919
- */
1920
- async start(nodeIdentity, nodeLoggingConnectorType, componentState) {
1921
- this._nodeIdentity = nodeIdentity;
1922
- this._remoteSyncStateHelper.setNodeIdentity(nodeIdentity);
1923
- this._changeSetHelper.setNodeIdentity(nodeIdentity);
1924
- this._remoteSyncStateHelper.setSynchronisedStorageKey(this._synchronisedStorageKey);
1925
- this._serviceStarted = true;
1926
- // If this is not a trusted node we need to request the decryption key from a trusted node
1927
- if (!Is.empty(this._trustedSynchronisedStorageComponent)) {
1928
- const proof = await this._identityConnector.createProof(this._nodeIdentity, DocumentHelper.joinId(this._nodeIdentity, this._config.synchronisedStorageMethodId), ProofTypes.DataIntegrityProof, { nodeIdentity });
1929
- const decryptionKey = await this._trustedSynchronisedStorageComponent.getDecryptionKey(this._nodeIdentity, proof);
1930
- // We don't have the private key so instead we store the key as a secret in the vault
1931
- await this._vaultConnector.setSecret(this._config.blobStorageEncryptionKeyId, decryptionKey);
1932
- }
1933
- // If there are already storage keys registered, we need to activate them
1934
- for (const storageKey in this._activeStorageKeys) {
1935
- await this.activateStorageKey(storageKey);
1936
- }
1937
- }
1938
- /**
1939
- * The component needs to be stopped when the node is closed.
1940
- * @param nodeIdentity The identity of the node stopping the component.
1941
- * @param nodeLoggingConnectorType The node logging connector type, defaults to "node-logging".
1942
- * @param componentState A persistent state which can be modified by the method.
1943
- * @returns Nothing.
1944
- */
1945
- async stop(nodeIdentity, nodeLoggingConnectorType, componentState) {
1946
- for (const storageKey in this._activeStorageKeys) {
1947
- this._activeStorageKeys[storageKey] = false;
1948
- this._taskSchedulerComponent.removeTask(`synchronised-storage-update-${storageKey}`);
1949
- this._taskSchedulerComponent.removeTask(`synchronised-storage-consolidation-${storageKey}`);
1950
- }
1951
- }
1952
- /**
1953
- * Get the decryption key for the synchronised storage.
1954
- * This is used to decrypt the data stored in the synchronised storage.
1955
- * @param nodeIdentity The identity of the node requesting the decryption key.
1956
- * @param proof The proof of the request so we know the request is from the specified node.
1957
- * @returns The decryption key.
1958
- */
1959
- async getDecryptionKey(nodeIdentity, proof) {
1960
- if (!Is.empty(this._trustedSynchronisedStorageComponent)) {
1961
- throw new GeneralError(this.CLASS_NAME, "notTrustedNode");
1962
- }
1963
- Guards.stringValue(this.CLASS_NAME, "nodeIdentity", nodeIdentity);
1964
- Guards.object(this.CLASS_NAME, "proof", proof);
1965
- const isValid = await this._identityConnector.verifyProof({ nodeIdentity }, proof);
1966
- if (!isValid) {
1967
- throw new UnauthorizedError(this.CLASS_NAME, "invalidProof");
1968
- }
1969
- // TODO: We need to check if the node has permissions to access the decryption key
1970
- // using rights-management
1971
- const key = await this._vaultConnector.getKey(this._config.blobStorageEncryptionKeyId);
1972
- if (Is.undefined(key.publicKey)) {
1973
- throw new UnauthorizedError(this.CLASS_NAME, "decryptionKeyNotFound");
1974
- }
1975
- return Converter.bytesToBase64(key.publicKey);
1976
- }
1977
- /**
1978
- * Synchronise a set of changes from an untrusted node, assumes this is a trusted node.
1979
- * @param syncChangeSet The change set to synchronise.
1980
- * @returns Nothing.
1981
- */
1982
- async syncChangeSet(syncChangeSet) {
1983
- if (!Is.empty(this._trustedSynchronisedStorageComponent)) {
1984
- throw new GeneralError(this.CLASS_NAME, "notTrustedNode");
1985
- }
1986
- Guards.object(this.CLASS_NAME, "syncChangeSet", syncChangeSet);
1987
- await this._loggingComponent?.log({
1988
- level: "info",
1989
- source: this.CLASS_NAME,
1990
- message: "syncChangeSetForRemoteNode",
1991
- data: {
1992
- changeSetStorageId: syncChangeSet.id
1993
- }
1994
- });
1995
- // TODO: The change set has a proof signed by the originating node identity
1996
- // The proof is verified that the change set is valid and has not been tampered with.
1997
- // but we also need to check that the originating node has permissions
1998
- // to store the change set in the synchronised storage.
1999
- // This will be performed using rights-management
2000
- const copy = await this._changeSetHelper.copyChangeset(syncChangeSet);
2001
- if (!Is.empty(copy)) {
2002
- // Apply the changes to this node
2003
- await this._changeSetHelper.applyChangeset(copy.syncChangeSet);
2004
- // And update the sync state with the latest changes
2005
- await this._remoteSyncStateHelper.addChangeSetToSyncState(copy.syncChangeSet.storageKey, copy.changeSetStorageId);
2006
- }
2007
- }
2008
- /**
2009
- * Start the sync with further updates after an interval.
2010
- * @param storageKey The storage key to sync.
2011
- * @returns Nothing.
2012
- * @internal
2013
- */
2014
- async startEntitySync(storageKey) {
2015
- try {
2016
- await this._loggingComponent?.log({
2017
- level: "info",
2018
- source: this.CLASS_NAME,
2019
- message: "startEntitySync",
2020
- data: {
2021
- storageKey
2022
- }
2023
- });
2024
- // First we check for remote changes
2025
- await this.updateFromRemoteSyncState(storageKey);
2026
- // Now send any updates we have to the remote storage
2027
- await this.updateFromLocalSyncState(storageKey);
2028
- }
2029
- catch (error) {
2030
- await this._loggingComponent?.log({
2031
- level: "error",
2032
- source: this.CLASS_NAME,
2033
- message: "entitySyncFailed",
2034
- error: BaseError.fromError(error)
2035
- });
2036
- }
2037
- }
2038
- /**
2039
- * Check for updates in the remote storage.
2040
- * @param storageKey The storage key to check for updates.
2041
- * @returns Nothing.
2042
- * @internal
2043
- */
2044
- async updateFromRemoteSyncState(storageKey) {
2045
- await this._loggingComponent?.log({
2046
- level: "info",
2047
- source: this.CLASS_NAME,
2048
- message: "updateFromRemoteSyncState",
2049
- data: {
2050
- storageKey
2051
- }
2052
- });
2053
- // Get the verifiable sync pointer store from the verifiable storage
2054
- const verifiableSyncPointerStore = await this._remoteSyncStateHelper.getVerifiableSyncPointerStore();
2055
- if (!Is.empty(verifiableSyncPointerStore.syncPointers[storageKey])) {
2056
- // Load the sync state from the remote blob storage using the sync pointer
2057
- // to load the sync state
2058
- const remoteSyncState = await this._remoteSyncStateHelper.getSyncState(verifiableSyncPointerStore.syncPointers[storageKey]);
2059
- // If we got the sync state we can try and sync from it
2060
- if (!Is.undefined(remoteSyncState)) {
2061
- await this._localSyncStateHelper.applySyncState(storageKey, remoteSyncState);
2062
- }
2063
- }
2064
- }
2065
- /**
2066
- * Find any local updates and send them to the remote storage.
2067
- * @returns Nothing.
2068
- * @internal
2069
- */
2070
- async updateFromLocalSyncState(storageKey) {
2071
- await this._loggingComponent?.log({
2072
- level: "info",
2073
- source: this.CLASS_NAME,
2074
- message: "updateFromLocalSyncState",
2075
- data: {
2076
- storageKey
2077
- }
2078
- });
2079
- const localChangeSnapshots = await this._localSyncStateHelper.getSnapshots(storageKey, true);
2080
- if (localChangeSnapshots.length > 0) {
2081
- const localChangeSnapshot = localChangeSnapshots[0];
2082
- if (Is.arrayValue(localChangeSnapshot.changes)) {
2083
- await this._remoteSyncStateHelper.buildChangeSet(storageKey, localChangeSnapshot.changes, async (syncChangeSet, changeSetStorageId) => {
2084
- if (Is.empty(syncChangeSet) && Is.empty(changeSetStorageId)) {
2085
- await this._loggingComponent?.log({
2086
- level: "info",
2087
- source: this.CLASS_NAME,
2088
- message: "builtStorageChangeSetNone",
2089
- data: {
2090
- storageKey
2091
- }
2092
- });
2093
- }
2094
- else {
2095
- await this._loggingComponent?.log({
2096
- level: "info",
2097
- source: this.CLASS_NAME,
2098
- message: "builtStorageChangeSet",
2099
- data: {
2100
- storageKey,
2101
- changeSetStorageId
2102
- }
2103
- });
2104
- // Send the local changes to the remote storage if we are a trusted node
2105
- if (Is.empty(this._trustedSynchronisedStorageComponent) &&
2106
- Is.stringValue(changeSetStorageId)) {
2107
- // If we are a trusted node, we can add the change set to the sync state
2108
- // and remove the local change snapshot
2109
- await this._remoteSyncStateHelper.addChangeSetToSyncState(storageKey, changeSetStorageId);
2110
- await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
2111
- }
2112
- else if (!Is.empty(this._trustedSynchronisedStorageComponent) &&
2113
- Is.object(syncChangeSet)) {
2114
- // If we are not a trusted node, we need to send the changes to the trusted node
2115
- // and then remove the local change snapshot
2116
- await this._loggingComponent?.log({
2117
- level: "info",
2118
- source: this.CLASS_NAME,
2119
- message: "sendingChangeSetToTrustedNode",
2120
- data: {
2121
- storageKey,
2122
- changeSetStorageId
2123
- }
2124
- });
2125
- await this._trustedSynchronisedStorageComponent.syncChangeSet(syncChangeSet);
2126
- await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
2127
- }
2128
- }
2129
- });
2130
- }
2131
- else {
2132
- await this._loggingComponent?.log({
2133
- level: "info",
2134
- source: this.CLASS_NAME,
2135
- message: "updateFromLocalSyncStateNoChanges",
2136
- data: {
2137
- storageKey
2138
- }
2139
- });
2140
- }
2141
- }
2142
- }
2143
- /**
2144
- * Start the consolidation sync.
2145
- * @param storageKey The storage key to consolidate.
2146
- * @returns Nothing.
2147
- * @internal
2148
- */
2149
- async startConsolidationSync(storageKey) {
2150
- try {
2151
- // If we are going to perform a consolidation first take any local updates
2152
- // we have and create a changeset from them, so that anybody applying
2153
- // just changes since a consolidation can use the changeset
2154
- // and skip the consolidation
2155
- await this.updateFromLocalSyncState(storageKey);
2156
- // Now start the consolidation
2157
- await this._remoteSyncStateHelper.consolidationStart(storageKey, this._config.consolidationBatchSize ??
2158
- SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE);
2159
- }
2160
- catch (error) {
2161
- await this._loggingComponent?.log({
2162
- level: "error",
2163
- source: this.CLASS_NAME,
2164
- message: "consolidationSyncFailed",
2165
- error: BaseError.fromError(error)
2166
- });
2167
- }
2168
- }
2169
- /**
2170
- * Register a new sync type.
2171
- * @param syncRegisterStorageKey The sync register type to register.
2172
- * @internal
2173
- */
2174
- async registerStorageKey(syncRegisterStorageKey) {
2175
- await this._loggingComponent?.log({
2176
- level: "info",
2177
- source: this.CLASS_NAME,
2178
- message: "registerStorageKey",
2179
- data: {
2180
- storageKey: syncRegisterStorageKey.storageKey
2181
- }
2182
- });
2183
- if (Is.empty(this._activeStorageKeys[syncRegisterStorageKey.storageKey])) {
2184
- this._activeStorageKeys[syncRegisterStorageKey.storageKey] = false;
2185
- if (this._serviceStarted) {
2186
- await this.activateStorageKey(syncRegisterStorageKey.storageKey);
2187
- }
2188
- }
2189
- }
2190
- /**
2191
- * Activate a storage key.
2192
- * @param storageKey The storage key to activate.
2193
- * @internal
2194
- */
2195
- async activateStorageKey(storageKey) {
2196
- if (!Is.empty(this._activeStorageKeys[storageKey]) && !this._activeStorageKeys[storageKey]) {
2197
- await this._loggingComponent?.log({
2198
- level: "info",
2199
- source: this.CLASS_NAME,
2200
- message: "activateStorageKey",
2201
- data: {
2202
- storageKey
2203
- }
2204
- });
2205
- this._activeStorageKeys[storageKey] = true;
2206
- if (this._config.entityUpdateIntervalMinutes > 0) {
2207
- await this._taskSchedulerComponent.addTask(`synchronised-storage-update-${storageKey}`, [
2208
- {
2209
- nextTriggerTime: Date.now(),
2210
- intervalMinutes: this._config.entityUpdateIntervalMinutes
2211
- }
2212
- ], async () => this.startEntitySync(storageKey));
2213
- }
2214
- if (!Is.empty(this._trustedSynchronisedStorageComponent) &&
2215
- this._config.consolidationIntervalMinutes > 0) {
2216
- await this._taskSchedulerComponent.addTask(`synchronised-storage-consolidation-${storageKey}`, [
2217
- {
2218
- nextTriggerTime: Date.now(),
2219
- intervalMinutes: this._config.consolidationIntervalMinutes
2220
- }
2221
- ], async () => this.startConsolidationSync(storageKey));
2222
- }
2223
- }
2224
- }
2225
- }
2226
-
2227
- export { SyncSnapshotEntry, SynchronisedStorageService, generateRestRoutesSynchronisedStorage, initSchema, restEntryPoints, synchronisedStorageGetDecryptionKeyRequest, synchronisedStorageSyncChangeSetRequest, tagsSynchronisedStorage };