@twin.org/synchronised-storage-service 0.0.1-next.5 → 0.0.1-next.7

package/dist/esm/index.mjs

@@ -4,7 +4,6 @@ import { HttpStatusCode } from '@twin.org/web';
 import { BlobStorageConnectorFactory } from '@twin.org/blob-storage-models';
 import { EntityStorageConnectorFactory } from '@twin.org/entity-storage-models';
 import { DocumentHelper, IdentityConnectorFactory } from '@twin.org/identity-models';
-import { LoggingConnectorFactory } from '@twin.org/logging-models';
 import { ProofTypes } from '@twin.org/standards-w3c-did';
 import { SyncChangeOperation, SynchronisedStorageTopics, SyncNodeIdentityMode } from '@twin.org/synchronised-storage-models';
 import { VaultEncryptionType, VaultConnectorFactory } from '@twin.org/vault-models';
@@ -45,6 +44,10 @@ let SyncSnapshotEntry = class SyncSnapshotEntry {
      * The flag to determine if this is a consolidated snapshot.
      */
     isConsolidated;
+    /**
+     * The epoch for the changeset.
+     */
+    epoch;
     /**
      * The ids of the storage for the change sets in the snapshot, if this is not a local snapshot.
      */
@@ -82,6 +85,10 @@ __decorate([
     property({ type: "boolean" }),
     __metadata("design:type", Boolean)
 ], SyncSnapshotEntry.prototype, "isConsolidated", void 0);
+__decorate([
+    property({ type: "number" }),
+    __metadata("design:type", Number)
+], SyncSnapshotEntry.prototype, "epoch", void 0);
 __decorate([
     property({ type: "array", itemType: "string", optional: true }),
     __metadata("design:type", Array)
@@ -287,10 +294,10 @@ class BlobStorageHelper {
      */
     CLASS_NAME = "BlobStorageHelper";
     /**
-     * The logging connector to use for logging.
+     * The logging component to use for logging.
      * @internal
      */
-    _logging;
+    _loggingComponent;
     /**
      * The vault connector.
      * @internal
@@ -313,14 +320,14 @@ class BlobStorageHelper {
     _isTrustedNode;
     /**
      * Create a new instance of BlobStorageHelper.
-     * @param logging The logging connector to use for logging.
+     * @param loggingComponent The logging connector to use for logging.
      * @param vaultConnector The vault connector to use for for the encryption key.
      * @param blobStorageConnector The blob storage component to use.
      * @param blobStorageEncryptionKeyId The id of the vault key to use for encrypting/decrypting blobs.
      * @param isTrustedNode Is this a trusted node.
      */
-    constructor(logging, vaultConnector, blobStorageConnector, blobStorageEncryptionKeyId, isTrustedNode) {
-        this._logging = logging;
+    constructor(loggingComponent, vaultConnector, blobStorageConnector, blobStorageEncryptionKeyId, isTrustedNode) {
+        this._loggingComponent = loggingComponent;
         this._vaultConnector = vaultConnector;
         this._blobStorageConnector = blobStorageConnector;
         this._blobStorageEncryptionKeyId = blobStorageEncryptionKeyId;
@@ -332,7 +339,7 @@ class BlobStorageHelper {
      * @returns The blob.
      */
     async loadBlob(blobId) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "loadBlob",
@@ -355,7 +362,7 @@ class BlobStorageHelper {
                     compressedBlob = rsa.decrypt(encryptedBlob);
                 }
                 const decompressedBlob = await Compression.decompress(compressedBlob, CompressionType.Gzip);
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "loadedBlob",
@@ -367,7 +374,7 @@ class BlobStorageHelper {
             }
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "loadBlobFailed",
@@ -377,7 +384,7 @@ class BlobStorageHelper {
                 error: BaseError.fromError(error)
             });
         }
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "loadBlobEmpty",
@@ -392,7 +399,7 @@ class BlobStorageHelper {
      * @returns The id of the blob.
      */
     async saveBlob(blob) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "saveBlob"
@@ -404,7 +411,7 @@ class BlobStorageHelper {
         const encryptedBlob = await this._vaultConnector.encrypt(this._blobStorageEncryptionKeyId, VaultEncryptionType.Rsa2048, compressedBlob);
         try {
             const blobId = await this._blobStorageConnector.set(encryptedBlob);
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "savedBlob",
@@ -415,7 +422,7 @@ class BlobStorageHelper {
             return blobId;
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "saveBlobFailed",
@@ -430,7 +437,7 @@ class BlobStorageHelper {
      * @returns Nothing.
      */
     async removeBlob(blobId) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "removeBlob",
@@ -440,7 +447,7 @@ class BlobStorageHelper {
         });
         try {
             await this._blobStorageConnector.remove(blobId);
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "removedBlob",
@@ -450,7 +457,7 @@ class BlobStorageHelper {
             });
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "removeBlobFailed",
@@ -460,7 +467,7 @@ class BlobStorageHelper {
                 error: BaseError.fromError(error)
             });
         }
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "removeBlobEmpty",
@@ -482,10 +489,10 @@ class ChangeSetHelper {
      */
     CLASS_NAME = "ChangeSetHelper";
     /**
-     * The logging connector to use for logging.
+     * The logging component to use for logging.
      * @internal
      */
-    _logging;
+    _loggingComponent;
     /**
      * The event bus component.
      * @internal
@@ -513,14 +520,14 @@ class ChangeSetHelper {
     _nodeIdentity;
     /**
      * Create a new instance of ChangeSetHelper.
-     * @param logging The logging connector to use for logging.
+     * @param loggingComponent The logging connector to use for logging.
      * @param eventBusComponent The event bus component to use for events.
      * @param identityConnector The identity connector to use for signing/verifying changesets.
      * @param blobStorageHelper The blob storage component to use for remote sync states.
      * @param decentralisedStorageMethodId The id of the identity method to use when signing/verifying changesets.
      */
-    constructor(logging, eventBusComponent, identityConnector, blobStorageHelper, decentralisedStorageMethodId) {
-        this._logging = logging;
+    constructor(loggingComponent, eventBusComponent, identityConnector, blobStorageHelper, decentralisedStorageMethodId) {
+        this._loggingComponent = loggingComponent;
         this._eventBusComponent = eventBusComponent;
         this._decentralisedStorageMethodId = decentralisedStorageMethodId;
         this._blobStorageHelper = blobStorageHelper;
@@ -539,7 +546,7 @@ class ChangeSetHelper {
      * @returns The changeset if it was verified.
      */
     async getAndVerifyChangeset(changeSetStorageId) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "getChangeSet",
@@ -555,7 +562,7 @@ class ChangeSetHelper {
             }
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "warn",
                 source: this.CLASS_NAME,
                 message: "getChangeSetError",
@@ -565,7 +572,7 @@ class ChangeSetHelper {
                 error: BaseError.fromError(error)
             });
         }
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "getChangeSetEmpty",
@@ -596,7 +603,7 @@ class ChangeSetHelper {
     async applyChangeset(syncChangeset) {
         if (Is.arrayValue(syncChangeset.changes)) {
             for (const change of syncChangeset.changes) {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "changeSetApplyingChange",
@@ -642,7 +649,7 @@ class ChangeSetHelper {
      * @returns The id of the change set.
      */
     async storeChangeSet(syncChangeSet) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "changeSetStoring",
@@ -659,7 +666,7 @@ class ChangeSetHelper {
      */
     async verifyChangesetProof(syncChangeset) {
         if (Is.empty(syncChangeset.proof)) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "verifyChangeSetProofMissing",
@@ -672,7 +679,7 @@ class ChangeSetHelper {
         // If the proof or verification method is missing, the proof is invalid
         const verificationMethod = syncChangeset.proof?.verificationMethod;
         if (!Is.stringValue(verificationMethod)) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "verifyChangeSetProofMissing",
@@ -686,7 +693,7 @@ class ChangeSetHelper {
         // otherwise you could sign a changeset for another node
         const changeSetNodeIdentity = DocumentHelper.parseId(verificationMethod ?? "");
         if (changeSetNodeIdentity.id !== syncChangeset.nodeIdentity) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "verifyChangeSetProofNodeIdentityMismatch",
@@ -699,7 +706,7 @@ class ChangeSetHelper {
         delete changeSetWithoutProof.proof;
         const isValid = await this._identityConnector.verifyProof(changeSetWithoutProof, syncChangeset.proof);
         if (!isValid) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "verifyChangeSetProofInvalid",
@@ -709,7 +716,7 @@ class ChangeSetHelper {
             });
         }
         else {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "verifyChangeSetProofValid",
@@ -730,7 +737,7 @@ class ChangeSetHelper {
         const changeSetWithoutProof = ObjectHelper.clone(syncChangeset);
         delete changeSetWithoutProof.proof;
         const proof = await this._identityConnector.createProof(this._nodeIdentity, DocumentHelper.joinId(this._nodeIdentity, this._decentralisedStorageMethodId), ProofTypes.DataIntegrityProof, changeSetWithoutProof);
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "createdChangeSetProof",
@@ -750,7 +757,7 @@ class ChangeSetHelper {
         if (Is.stringValue(this._nodeIdentity)) {
             const verified = await this.verifyChangesetProof(syncChangeSet);
             if (verified) {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "copyChangeSet",
@@ -779,7 +786,7 @@ class ChangeSetHelper {
     async reset(storageKey, resetMode) {
         // If we are applying a consolidation we need to reset the local db
         // but keep any entries from the local node, as they might have been updated
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "storageReset",
@@ -811,10 +818,10 @@ class LocalSyncStateHelper {
      */
     CLASS_NAME = "LocalSyncStateHelper";
     /**
-     * The logging connector to use for logging.
+     * The logging component to use for logging.
      * @internal
      */
-    _logging;
+    _loggingComponent;
     /**
      * The storage connector for the sync snapshot entries.
      * @internal
@@ -827,12 +834,12 @@ class LocalSyncStateHelper {
     _changeSetHelper;
     /**
      * Create a new instance of LocalSyncStateHelper.
-     * @param logging The logging connector to use for logging.
+     * @param loggingComponent The logging connector to use for logging.
      * @param snapshotEntryEntityStorage The storage connector for the sync snapshot entries.
      * @param changeSetHelper The change set helper to use for applying changesets.
      */
-    constructor(logging, snapshotEntryEntityStorage, changeSetHelper) {
-        this._logging = logging;
+    constructor(loggingComponent, snapshotEntryEntityStorage, changeSetHelper) {
+        this._loggingComponent = loggingComponent;
         this._snapshotEntryEntityStorage = snapshotEntryEntityStorage;
         this._changeSetHelper = changeSetHelper;
     }
@@ -844,7 +851,7 @@ class LocalSyncStateHelper {
      * @returns Nothing.
      */
     async addLocalChange(storageKey, operation, id) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "addLocalChange",
@@ -865,6 +872,9 @@ class LocalSyncStateHelper {
             if (previousChangeIndex !== -1) {
                 localChangeSnapshot.changes.splice(previousChangeIndex, 1);
             }
+            // If we already have changes from previous updates
+            // then make sure we update the dateModified, otherwise
+            // we assume this is the first change and setting modified is not necessary
             if (localChangeSnapshot.changes.length > 0) {
                 localChangeSnapshot.dateModified = new Date(Date.now()).toISOString();
             }
@@ -879,7 +889,7 @@ class LocalSyncStateHelper {
      * @returns The local snapshot entry.
      */
     async getSnapshots(storageKey, isLocal) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "getSnapshots",
@@ -902,7 +912,7 @@ class LocalSyncStateHelper {
             ]
         });
         if (queryResult.entities.length > 0) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "getSnapshotsExists",
@@ -912,7 +922,7 @@ class LocalSyncStateHelper {
             });
             return queryResult.entities;
         }
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "getSnapshotsDoesNotExist",
@@ -930,7 +940,8 @@ class LocalSyncStateHelper {
                 dateModified: now,
                 changeSetStorageIds: [],
                 isLocal,
-                isConsolidated: false
+                isConsolidated: false,
+                epoch: 0
             }
         ];
     }
@@ -940,7 +951,7 @@ class LocalSyncStateHelper {
      * @returns Nothing.
      */
     async setLocalChangeSnapshot(localChangeSnapshot) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "setLocalChangeSnapshot",
@@ -956,7 +967,7 @@ class LocalSyncStateHelper {
      * @returns Nothing.
      */
     async removeLocalChangeSnapshot(localChangeSnapshot) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "removeLocalChangeSnapshot",
@@ -973,7 +984,7 @@ class LocalSyncStateHelper {
      * @returns Nothing.
      */
     async applySyncState(storageKey, syncState) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "applySyncState",
@@ -982,14 +993,23 @@ class LocalSyncStateHelper {
             }
         });
         // Get all the existing snapshots that we have processed previously
-        const existingRemoteSnapshots = await this.getSnapshots(storageKey, false);
+        let existingSnapshots = await this.getSnapshots(storageKey, false);
         // Sort from newest to oldest
-        const sortedSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
-        // If we have no existing snapshots we can't have yet synced
-        // in this case we need to find the most recent consolidation
-        // and use that to build a complete DB table
-        if (existingRemoteSnapshots.length === 0) {
-            await this._logging?.log({
+        existingSnapshots = existingSnapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
+        // Sort from newest to oldest
+        const syncStateSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
+        // Get the newest epoch from the local storage
+        const newestExistingEpoch = existingSnapshots[0]?.epoch ?? 0;
+        // Get the oldest epoch from the remote storage
+        const oldestSyncStateEpoch = syncStateSnapshots[syncStateSnapshots.length - 1]?.epoch ?? 0;
+        // If there is a gap between the largest epoch we have locally
+        // and the smallest epoch we have remotely then we have missed
+        // data so we need to perform a full sync
+        const hasEpochGap = newestExistingEpoch + 1 < oldestSyncStateEpoch;
+        // If we have an epoch gap or no existing snapshots then we need to apply
+        // a full sync from a consolidation
+        if (!existingSnapshots.some(s => s.isConsolidated) || hasEpochGap) {
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "applySnapshotNoExisting",
@@ -997,31 +1017,38 @@ class LocalSyncStateHelper {
                     storageKey
                 }
             });
-            const firstConsolidated = sortedSnapshots.find(snapshot => snapshot.isConsolidated);
-            if (firstConsolidated) {
-                // We found a consolidated snapshot, we can use it
-                await this._logging?.log({
+            const mostRecentConsolidation = syncStateSnapshots.findIndex(snapshot => snapshot.isConsolidated);
+            if (mostRecentConsolidation !== -1) {
+                // We found the most recent consolidated snapshot, we can use it
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "applySnapshotFoundConsolidated",
                     data: {
                         storageKey,
-                        snapshotId: firstConsolidated.id
+                        snapshotId: syncStateSnapshots[mostRecentConsolidation].id
                     }
                 });
                 // We need to reset the entity storage and remove all the remote items
-                // so that we use just the ones from the consolidation
+                // so that we use just the ones from the consolidation, since
+                // we don't have any existing there shouldn't be any remote entries
+                // but we reset nonetheless
                 await this._changeSetHelper.reset(storageKey, SyncNodeIdentityMode.Remote);
-                await this.processNewSnapshots([
-                    {
-                        ...firstConsolidated,
-                        storageKey,
-                        isLocal: false
-                    }
-                ]);
+                // We need to process the most recent consolidation and all changes
+                // that were made since then, from newest to oldest (so newer changes override older ones)
+                // Process snapshots from the consolidation point (most recent) back to the newest
+                for (let i = mostRecentConsolidation; i >= 0; i--) {
+                    await this.processNewSnapshots([
+                        {
+                            ...syncStateSnapshots[i],
+                            storageKey,
+                            isLocal: false
+                        }
+                    ]);
+                }
             }
             else {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "applySnapshotNoConsolidated",
@@ -1032,16 +1059,21 @@ class LocalSyncStateHelper {
             }
         }
         else {
+            // We have existing consolidated remote snapshots, so we can assume that we have
+            // applied at least one consolidation snapshot, in this case we need to look at the changes since
+            // then and apply them if we haven't already
+            // We don't need to apply any additional consolidated snapshots, just the changesets
             // Create a lookup map for the existing snapshots
-            const existingSnapshots = {};
-            for (const snapshot of existingRemoteSnapshots) {
-                existingSnapshots[snapshot.id] = snapshot;
+            const existingSnapshotsMap = {};
+            for (const snapshot of existingSnapshots) {
+                existingSnapshotsMap[snapshot.id] = snapshot;
             }
             const newSnapshots = [];
             const modifiedSnapshots = [];
-            const referencedExistingSnapshots = Object.keys(existingSnapshots);
-            for (const snapshot of sortedSnapshots) {
-                await this._logging?.log({
+            const referencedExistingSnapshots = Object.keys(existingSnapshotsMap);
+            let completedProcessing = false;
+            for (const snapshot of syncStateSnapshots) {
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "applySnapshot",
@@ -1050,34 +1082,37 @@ class LocalSyncStateHelper {
                         dateCreated: new Date(snapshot.dateCreated).toISOString()
                     }
                 });
-                // See if we have the local snapshot
-                const currentSnapshot = existingSnapshots[snapshot.id];
+                // See if we have the snapshot stored locally
+                const currentSnapshot = existingSnapshotsMap[snapshot.id];
                 // As we are referencing an existing snapshot, we need to remove it from the list
                 // to allow us to cleanup any unreferenced snapshots later
                 const idx = referencedExistingSnapshots.indexOf(snapshot.id);
                 if (idx !== -1) {
                     referencedExistingSnapshots.splice(idx, 1);
                 }
-                const updatedSnapshot = {
-                    ...snapshot,
-                    storageKey,
-                    isLocal: false
-                };
-                if (Is.empty(currentSnapshot)) {
-                    // We don't have the snapshot locally, so we need to process it
-                    newSnapshots.push(updatedSnapshot);
-                }
-                else if (currentSnapshot.dateModified !== snapshot.dateModified) {
-                    // If the local snapshot has a different dateModified, we need to update it
-                    modifiedSnapshots.push({
-                        currentSnapshot,
-                        updatedSnapshot
-                    });
-                }
-                else {
-                    // we sorted the snapshots from newest to oldest, so if we found a local snapshot
-                    // with the same dateModified as the remote snapshot, we can stop processing further
-                    break;
+                // No need to apply consolidated snapshots
+                if (!snapshot.isConsolidated && !completedProcessing) {
+                    const updatedSnapshot = {
+                        ...snapshot,
+                        storageKey,
+                        isLocal: false
+                    };
+                    if (Is.empty(currentSnapshot)) {
+                        // We don't have the snapshot locally, so we need to process all of it
+                        newSnapshots.push(updatedSnapshot);
+                    }
+                    else if (currentSnapshot.dateModified !== snapshot.dateModified) {
+                        // If the local snapshot has a different dateModified, we need to update it
+                        modifiedSnapshots.push({
+                            currentSnapshot,
+                            updatedSnapshot
+                        });
+                    }
+                    else {
+                        // we sorted the snapshots from newest to oldest, so if we found a local snapshot
+                        // with the same dateModified as the remote snapshot, we can stop processing further
+                        completedProcessing = true;
+                    }
                 }
             }
             // We reverse the order of the snapshots to process them from oldest to newest
@@ -1099,7 +1134,7 @@ class LocalSyncStateHelper {
      */
     async processModifiedSnapshots(modifiedSnapshots) {
         for (const modifiedSnapshot of modifiedSnapshots) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "processModifiedSnapshot",
@@ -1132,7 +1167,7 @@ class LocalSyncStateHelper {
      */
     async processNewSnapshots(newSnapshots) {
         for (const newSnapshot of newSnapshots) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "processNewSnapshot",
@@ -1163,10 +1198,10 @@ class RemoteSyncStateHelper {
      */
     CLASS_NAME = "RemoteSyncStateHelper";
     /**
-     * The logging connector to use for logging.
+     * The logging component to use for logging.
      * @internal
      */
-    _logging;
+    _loggingComponent;
     /**
      * The event bus component.
      * @internal
@@ -1219,7 +1254,7 @@ class RemoteSyncStateHelper {
     _maxConsolidations;
     /**
      * Create a new instance of DecentralisedEntityStorageConnector.
-     * @param logging The logging connector to use for logging.
+     * @param loggingComponent The logging component to use for logging.
      * @param eventBusComponent The event bus component to use for events.
      * @param verifiableSyncPointerStorageConnector The verifiable storage connector to use for storing sync pointers.
      * @param blobStorageHelper The blob storage helper to use for remote sync states.
@@ -1227,8 +1262,8 @@ class RemoteSyncStateHelper {
      * @param isTrustedNode Whether the node is trusted or not.
      * @param maxConsolidations The maximum number of consolidations to keep in storage.
      */
-    constructor(logging, eventBusComponent, verifiableSyncPointerStorageConnector, blobStorageHelper, changeSetHelper, isTrustedNode, maxConsolidations) {
-        this._logging = logging;
+    constructor(loggingComponent, eventBusComponent, verifiableSyncPointerStorageConnector, blobStorageHelper, changeSetHelper, isTrustedNode, maxConsolidations) {
+        this._loggingComponent = loggingComponent;
         this._eventBusComponent = eventBusComponent;
         this._verifiableSyncPointerStorageConnector = verifiableSyncPointerStorageConnector;
         this._changeSetHelper = changeSetHelper;
@@ -1266,7 +1301,7 @@ class RemoteSyncStateHelper {
      * @returns The storage id of the change set if created.
      */
     async buildChangeSet(storageKey, changes, completeCallback) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "buildingChangeSet",
@@ -1292,7 +1327,7 @@ class RemoteSyncStateHelper {
             // Once all the requests are handled the callback will be called
             for (const change of setChanges) {
                 // Create a request for each change to populate the full details
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "createChangeSetRequestingItem",
@@ -1315,7 +1350,7 @@ class RemoteSyncStateHelper {
      * @returns Nothing.
      */
     async finaliseFullChanges(storageKey, completeCallback) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "finalisingSyncChanges",
@@ -1356,7 +1391,7 @@ class RemoteSyncStateHelper {
                 await completeCallback(syncChangeSet, changeSetStorageId);
             }
             catch (err) {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "error",
                     source: this.CLASS_NAME,
                     message: "finalisingSyncChangesFailed",
@@ -1379,7 +1414,7 @@ class RemoteSyncStateHelper {
      * @returns Nothing.
      */
     async addChangeSetToSyncState(storageKey, changeSetStorageId) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "addChangeSetToSyncState",
@@ -1402,6 +1437,7 @@ class RemoteSyncStateHelper {
         const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
         // Get the current snapshot, if it does not exist we create a new one
         let currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+        const currentEpoch = currentSnapshot?.epoch ?? 0;
         const now = new Date(Date.now()).toISOString();
         // If there is no snapshot or the current one is a consolidation
         // we start a new snapshot
@@ -1412,6 +1448,7 @@ class RemoteSyncStateHelper {
                 dateCreated: now,
                 dateModified: now,
                 isConsolidated: false,
+                epoch: currentEpoch + 1,
                 changeSetStorageIds: []
             };
             syncState.snapshots.push(currentSnapshot);
@@ -1434,7 +1471,7 @@ class RemoteSyncStateHelper {
      * @returns Nothing.
      */
     async consolidationStart(storageKey, batchSize) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "consolidationStarting"
@@ -1449,7 +1486,7 @@ class RemoteSyncStateHelper {
     async getVerifiableSyncPointerStore() {
         if (Is.stringValue(this._synchronisedStorageKey)) {
             try {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "verifiableSyncPointerStoreRetrieving",
@@ -1460,7 +1497,7 @@ class RemoteSyncStateHelper {
                 const syncPointerStore = await this._verifiableSyncPointerStorageConnector.get(this._synchronisedStorageKey, { includeData: true });
                 if (Is.uint8Array(syncPointerStore.data)) {
                     const syncPointer = ObjectHelper.fromBytes(syncPointerStore.data);
-                    await this._logging?.log({
+                    await this._loggingComponent?.log({
                         level: "info",
                         source: this.CLASS_NAME,
                         message: "verifiableSyncPointerStoreRetrieved",
@@ -1476,7 +1513,7 @@ class RemoteSyncStateHelper {
                     throw err;
                 }
             }
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "verifiableSyncPointerStoreNotFound",
@@ -1498,7 +1535,7 @@ class RemoteSyncStateHelper {
      */
     async storeVerifiableSyncPointerStore(syncPointerStore) {
         if (Is.stringValue(this._nodeIdentity) && Is.stringValue(this._synchronisedStorageKey)) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "verifiableSyncPointerStoreStoring",
@@ -1516,7 +1553,7 @@ class RemoteSyncStateHelper {
      * @returns The id of the sync state.
      */
     async storeRemoteSyncState(syncState) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "syncStateStoring",
@@ -1542,7 +1579,12 @@ class RemoteSyncStateHelper {
             const toRemove = snapshots.slice(consolidationIndexes[this._maxConsolidations - 1] + 1);
             syncState.snapshots = snapshots.slice(0, consolidationIndexes[this._maxConsolidations - 1] + 1);
             for (const snapshot of toRemove) {
-                await this._blobStorageHelper.removeBlob(snapshot.id);
+                // We need to remove all the storage ids associated with the snapshot
+                if (Is.arrayValue(snapshot.changeSetStorageIds)) {
+                    for (const storageId of snapshot.changeSetStorageIds) {
+                        await this._blobStorageHelper.removeBlob(storageId);
+                    }
+                }
             }
         }
         return this._blobStorageHelper.saveBlob(syncState);
@@ -1554,7 +1596,7 @@ class RemoteSyncStateHelper {
      */
     async getSyncState(syncPointerId) {
         try {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "syncStateRetrieving",
@@ -1564,7 +1606,7 @@ class RemoteSyncStateHelper {
             });
             const syncState = await this._blobStorageHelper.loadBlob(syncPointerId);
             if (Is.object(syncState)) {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "syncStateRetrieved",
@@ -1577,7 +1619,7 @@ class RemoteSyncStateHelper {
             }
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "warn",
                 source: this.CLASS_NAME,
                 message: "getSyncStateError",
@@ -1587,7 +1629,7 @@ class RemoteSyncStateHelper {
                 error: BaseError.fromError(error)
             });
         }
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "syncStateNotFound",
@@ -1637,12 +1679,17 @@ class RemoteSyncStateHelper {
                     storageKey: response.storageKey,
                     snapshots: []
                 };
+                // Sort the snapshots so the newest snapshot is last in the array
+                const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
+                const currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+                const currentEpoch = currentSnapshot?.epoch ?? 0;
                 const batchSnapshot = {
                     version: SYNC_SNAPSHOT_VERSION,
                     id: Converter.bytesToHex(RandomHelper.generate(32)),
                     dateCreated: now,
                     dateModified: now,
                     isConsolidated: true,
+                    epoch: currentEpoch + 1,
                     changeSetStorageIds: this._batchResponseStorageIds[response.storageKey]
                 };
                 syncState.snapshots.push(batchSnapshot);
@@ -1654,7 +1701,7 @@ class RemoteSyncStateHelper {
                 // Remove the batch response storage ids for the storage key
                 // as we have consolidated the changes
                 delete this._batchResponseStorageIds[response.storageKey];
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "consolidationCompleted"
@@ -1667,7 +1714,7 @@ class RemoteSyncStateHelper {
      * @param response The item response to handle.
      */
     async handleLocalItemResponse(response) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "createChangeSetRespondingItem",
@@ -1721,10 +1768,10 @@ class SynchronisedStorageService {
      */
     CLASS_NAME = "SynchronisedStorageService";
     /**
-     * The logging connector to use for logging.
+     * The logging component to use for logging.
      * @internal
      */
-    _logging;
+    _loggingComponent;
     /**
      * The event bus component.
      * @internal
@@ -1818,7 +1865,7 @@ class SynchronisedStorageService {
         Guards.object(this.CLASS_NAME, "options", options);
         Guards.object(this.CLASS_NAME, "options.config", options.config);
         this._eventBusComponent = ComponentFactory.get(options.eventBusComponentType ?? "event-bus");
-        this._logging = LoggingConnectorFactory.getIfExists(options.loggingConnectorType ?? "logging");
+        this._loggingComponent = ComponentFactory.getIfExists(options.loggingComponentType ?? "logging");
         this._vaultConnector = VaultConnectorFactory.get(options.vaultConnectorType ?? "vault");
         this._localSyncSnapshotEntryEntityStorage = EntityStorageConnectorFactory.get(options.syncSnapshotStorageConnectorType ?? "sync-snapshot-entry");
         this._verifiableSyncPointerStorageConnector = VerifiableStorageConnectorFactory.get(options.verifiableStorageConnectorType ?? "verifiable-storage");
@@ -1847,10 +1894,10 @@ class SynchronisedStorageService {
             this._trustedSynchronisedStorageComponent =
                 ComponentFactory.get(options.trustedSynchronisedStorageComponentType);
         }
-        this._blobStorageHelper = new BlobStorageHelper(this._logging, this._vaultConnector, this._blobStorageConnector, this._config.blobStorageEncryptionKeyId, this._config.isTrustedNode);
-        this._changeSetHelper = new ChangeSetHelper(this._logging, this._eventBusComponent, this._identityConnector, this._blobStorageHelper, this._config.synchronisedStorageMethodId);
-        this._localSyncStateHelper = new LocalSyncStateHelper(this._logging, this._localSyncSnapshotEntryEntityStorage, this._changeSetHelper);
-        this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._logging, this._eventBusComponent, this._verifiableSyncPointerStorageConnector, this._blobStorageHelper, this._changeSetHelper, this._config.isTrustedNode, this._config.maxConsolidations);
+        this._blobStorageHelper = new BlobStorageHelper(this._loggingComponent, this._vaultConnector, this._blobStorageConnector, this._config.blobStorageEncryptionKeyId, this._config.isTrustedNode);
+        this._changeSetHelper = new ChangeSetHelper(this._loggingComponent, this._eventBusComponent, this._identityConnector, this._blobStorageHelper, this._config.synchronisedStorageMethodId);
+        this._localSyncStateHelper = new LocalSyncStateHelper(this._loggingComponent, this._localSyncSnapshotEntryEntityStorage, this._changeSetHelper);
+        this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._loggingComponent, this._eventBusComponent, this._verifiableSyncPointerStorageConnector, this._blobStorageHelper, this._changeSetHelper, this._config.isTrustedNode, this._config.maxConsolidations);
         this._serviceStarted = false;
         this._activeStorageKeys = {};
         this._eventBusComponent.subscribe(SynchronisedStorageTopics.RegisterStorageKey, async (event) => this.registerStorageKey(event.data));
@@ -1935,7 +1982,7 @@ class SynchronisedStorageService {
             throw new GeneralError(this.CLASS_NAME, "notTrustedNode");
         }
         Guards.object(this.CLASS_NAME, "syncChangeSet", syncChangeSet);
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "syncChangeSetForRemoteNode",
@@ -1964,7 +2011,7 @@ class SynchronisedStorageService {
      */
     async startEntitySync(storageKey) {
         try {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "startEntitySync",
@@ -1978,7 +2025,7 @@ class SynchronisedStorageService {
             await this.updateFromLocalSyncState(storageKey);
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "entitySyncFailed",
@@ -1993,7 +2040,7 @@ class SynchronisedStorageService {
      * @internal
      */
     async updateFromRemoteSyncState(storageKey) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "updateFromRemoteSyncState",
@@ -2019,7 +2066,7 @@ class SynchronisedStorageService {
      * @internal
      */
     async updateFromLocalSyncState(storageKey) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "updateFromLocalSyncState",
@@ -2033,7 +2080,7 @@ class SynchronisedStorageService {
             if (Is.arrayValue(localChangeSnapshot.changes)) {
                 await this._remoteSyncStateHelper.buildChangeSet(storageKey, localChangeSnapshot.changes, async (syncChangeSet, changeSetStorageId) => {
                     if (Is.empty(syncChangeSet) && Is.empty(changeSetStorageId)) {
-                        await this._logging?.log({
+                        await this._loggingComponent?.log({
                             level: "info",
                             source: this.CLASS_NAME,
                             message: "builtStorageChangeSetNone",
@@ -2043,7 +2090,7 @@ class SynchronisedStorageService {
                         });
                     }
                     else {
-                        await this._logging?.log({
+                        await this._loggingComponent?.log({
                             level: "info",
                             source: this.CLASS_NAME,
                             message: "builtStorageChangeSet",
@@ -2063,7 +2110,7 @@ class SynchronisedStorageService {
                             Is.object(syncChangeSet)) {
                             // If we are not a trusted node, we need to send the changes to the trusted node
                             // and then remove the local change snapshot
-                            await this._logging?.log({
+                            await this._loggingComponent?.log({
                                 level: "info",
                                 source: this.CLASS_NAME,
                                 message: "sendingChangeSetToTrustedNode",
@@ -2079,7 +2126,7 @@ class SynchronisedStorageService {
                 });
             }
             else {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "updateFromLocalSyncStateNoChanges",
@@ -2097,26 +2144,18 @@ class SynchronisedStorageService {
      * @internal
      */
     async startConsolidationSync(storageKey) {
-        let localChangeSnapshot;
         try {
-            // If we are performing a consolidation, we can remove the local change snapshot
-            // as we are going to create a complete changeset from the DB
-            const localChangeSnapshots = await this._localSyncStateHelper.getSnapshots(storageKey, true);
-            localChangeSnapshot = localChangeSnapshots[0];
-            if (!Is.empty(localChangeSnapshot)) {
-                await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
-            }
+            // If we are going to perform a consolidation first take any local updates
+            // we have and create a changeset from them, so that anybody applying
+            // just changes since a consolidation can use the changeset
+            // and skip the consolidation
+            await this.updateFromLocalSyncState(storageKey);
+            // Now start the consolidation
             await this._remoteSyncStateHelper.consolidationStart(storageKey, this._config.consolidationBatchSize ??
                 SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE);
-            // The consolidation was successful, so we can remove the local change snapshot permanently
-            localChangeSnapshot = undefined;
         }
         catch (error) {
-            if (localChangeSnapshot) {
-                // If the consolidation failed, we can keep the local change snapshot
-                await this._localSyncStateHelper.setLocalChangeSnapshot(localChangeSnapshot);
-            }
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "consolidationSyncFailed",
@@ -2130,7 +2169,7 @@ class SynchronisedStorageService {
      * @internal
      */
     async registerStorageKey(syncRegisterStorageKey) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "registerStorageKey",
@@ -2152,7 +2191,7 @@ class SynchronisedStorageService {
      */
     async activateStorageKey(storageKey) {
         if (!Is.empty(this._activeStorageKeys[storageKey]) && !this._activeStorageKeys[storageKey]) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "activateStorageKey",