@twin.org/synchronised-storage-service 0.0.1-next.5 → 0.0.1-next.7

package/dist/cjs/index.cjs

@@ -6,7 +6,6 @@ var web = require('@twin.org/web');
 var blobStorageModels = require('@twin.org/blob-storage-models');
 var entityStorageModels = require('@twin.org/entity-storage-models');
 var identityModels = require('@twin.org/identity-models');
-var loggingModels = require('@twin.org/logging-models');
 var standardsW3cDid = require('@twin.org/standards-w3c-did');
 var synchronisedStorageModels = require('@twin.org/synchronised-storage-models');
 var vaultModels = require('@twin.org/vault-models');
@@ -47,6 +46,10 @@ exports.SyncSnapshotEntry = class SyncSnapshotEntry {
      * The flag to determine if this is a consolidated snapshot.
      */
     isConsolidated;
+    /**
+     * The epoch for the changeset.
+     */
+    epoch;
     /**
      * The ids of the storage for the change sets in the snapshot, if this is not a local snapshot.
      */
@@ -84,6 +87,10 @@ __decorate([
     entity.property({ type: "boolean" }),
     __metadata("design:type", Boolean)
 ], exports.SyncSnapshotEntry.prototype, "isConsolidated", void 0);
+__decorate([
+    entity.property({ type: "number" }),
+    __metadata("design:type", Number)
+], exports.SyncSnapshotEntry.prototype, "epoch", void 0);
 __decorate([
     entity.property({ type: "array", itemType: "string", optional: true }),
     __metadata("design:type", Array)
@@ -289,10 +296,10 @@ class BlobStorageHelper {
      */
     CLASS_NAME = "BlobStorageHelper";
     /**
-     * The logging connector to use for logging.
+     * The logging component to use for logging.
      * @internal
      */
-    _logging;
+    _loggingComponent;
     /**
      * The vault connector.
      * @internal
@@ -315,14 +322,14 @@ class BlobStorageHelper {
     _isTrustedNode;
     /**
      * Create a new instance of BlobStorageHelper.
-     * @param logging The logging connector to use for logging.
+     * @param loggingComponent The logging connector to use for logging.
      * @param vaultConnector The vault connector to use for for the encryption key.
      * @param blobStorageConnector The blob storage component to use.
      * @param blobStorageEncryptionKeyId The id of the vault key to use for encrypting/decrypting blobs.
      * @param isTrustedNode Is this a trusted node.
      */
-    constructor(logging, vaultConnector, blobStorageConnector, blobStorageEncryptionKeyId, isTrustedNode) {
-        this._logging = logging;
+    constructor(loggingComponent, vaultConnector, blobStorageConnector, blobStorageEncryptionKeyId, isTrustedNode) {
+        this._loggingComponent = loggingComponent;
         this._vaultConnector = vaultConnector;
         this._blobStorageConnector = blobStorageConnector;
         this._blobStorageEncryptionKeyId = blobStorageEncryptionKeyId;
@@ -334,7 +341,7 @@ class BlobStorageHelper {
      * @returns The blob.
      */
     async loadBlob(blobId) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "loadBlob",
@@ -357,7 +364,7 @@ class BlobStorageHelper {
                     compressedBlob = rsa.decrypt(encryptedBlob);
                 }
                 const decompressedBlob = await core.Compression.decompress(compressedBlob, core.CompressionType.Gzip);
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "loadedBlob",
@@ -369,7 +376,7 @@ class BlobStorageHelper {
             }
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "loadBlobFailed",
@@ -379,7 +386,7 @@ class BlobStorageHelper {
                 error: core.BaseError.fromError(error)
             });
         }
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "loadBlobEmpty",
@@ -394,7 +401,7 @@ class BlobStorageHelper {
      * @returns The id of the blob.
      */
     async saveBlob(blob) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "saveBlob"
@@ -406,7 +413,7 @@ class BlobStorageHelper {
         const encryptedBlob = await this._vaultConnector.encrypt(this._blobStorageEncryptionKeyId, vaultModels.VaultEncryptionType.Rsa2048, compressedBlob);
         try {
             const blobId = await this._blobStorageConnector.set(encryptedBlob);
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "savedBlob",
@@ -417,7 +424,7 @@ class BlobStorageHelper {
             return blobId;
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "saveBlobFailed",
@@ -432,7 +439,7 @@ class BlobStorageHelper {
      * @returns Nothing.
      */
     async removeBlob(blobId) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "removeBlob",
@@ -442,7 +449,7 @@ class BlobStorageHelper {
         });
         try {
             await this._blobStorageConnector.remove(blobId);
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "removedBlob",
@@ -452,7 +459,7 @@ class BlobStorageHelper {
             });
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "removeBlobFailed",
@@ -462,7 +469,7 @@ class BlobStorageHelper {
                 error: core.BaseError.fromError(error)
             });
         }
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "removeBlobEmpty",
@@ -484,10 +491,10 @@ class ChangeSetHelper {
      */
     CLASS_NAME = "ChangeSetHelper";
     /**
-     * The logging connector to use for logging.
+     * The logging component to use for logging.
      * @internal
      */
-    _logging;
+    _loggingComponent;
     /**
      * The event bus component.
      * @internal
@@ -515,14 +522,14 @@ class ChangeSetHelper {
     _nodeIdentity;
     /**
      * Create a new instance of ChangeSetHelper.
-     * @param logging The logging connector to use for logging.
+     * @param loggingComponent The logging connector to use for logging.
      * @param eventBusComponent The event bus component to use for events.
      * @param identityConnector The identity connector to use for signing/verifying changesets.
      * @param blobStorageHelper The blob storage component to use for remote sync states.
      * @param decentralisedStorageMethodId The id of the identity method to use when signing/verifying changesets.
      */
-    constructor(logging, eventBusComponent, identityConnector, blobStorageHelper, decentralisedStorageMethodId) {
-        this._logging = logging;
+    constructor(loggingComponent, eventBusComponent, identityConnector, blobStorageHelper, decentralisedStorageMethodId) {
+        this._loggingComponent = loggingComponent;
         this._eventBusComponent = eventBusComponent;
         this._decentralisedStorageMethodId = decentralisedStorageMethodId;
         this._blobStorageHelper = blobStorageHelper;
@@ -541,7 +548,7 @@ class ChangeSetHelper {
      * @returns The changeset if it was verified.
      */
     async getAndVerifyChangeset(changeSetStorageId) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "getChangeSet",
@@ -557,7 +564,7 @@ class ChangeSetHelper {
             }
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "warn",
                 source: this.CLASS_NAME,
                 message: "getChangeSetError",
@@ -567,7 +574,7 @@ class ChangeSetHelper {
                 error: core.BaseError.fromError(error)
             });
         }
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "getChangeSetEmpty",
@@ -598,7 +605,7 @@ class ChangeSetHelper {
     async applyChangeset(syncChangeset) {
         if (core.Is.arrayValue(syncChangeset.changes)) {
             for (const change of syncChangeset.changes) {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "changeSetApplyingChange",
@@ -644,7 +651,7 @@ class ChangeSetHelper {
      * @returns The id of the change set.
      */
     async storeChangeSet(syncChangeSet) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "changeSetStoring",
@@ -661,7 +668,7 @@ class ChangeSetHelper {
      */
     async verifyChangesetProof(syncChangeset) {
         if (core.Is.empty(syncChangeset.proof)) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "verifyChangeSetProofMissing",
@@ -674,7 +681,7 @@ class ChangeSetHelper {
         // If the proof or verification method is missing, the proof is invalid
         const verificationMethod = syncChangeset.proof?.verificationMethod;
         if (!core.Is.stringValue(verificationMethod)) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "verifyChangeSetProofMissing",
@@ -688,7 +695,7 @@ class ChangeSetHelper {
         // otherwise you could sign a changeset for another node
         const changeSetNodeIdentity = identityModels.DocumentHelper.parseId(verificationMethod ?? "");
         if (changeSetNodeIdentity.id !== syncChangeset.nodeIdentity) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "verifyChangeSetProofNodeIdentityMismatch",
@@ -701,7 +708,7 @@ class ChangeSetHelper {
         delete changeSetWithoutProof.proof;
         const isValid = await this._identityConnector.verifyProof(changeSetWithoutProof, syncChangeset.proof);
         if (!isValid) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "verifyChangeSetProofInvalid",
@@ -711,7 +718,7 @@ class ChangeSetHelper {
             });
         }
         else {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "verifyChangeSetProofValid",
@@ -732,7 +739,7 @@ class ChangeSetHelper {
         const changeSetWithoutProof = core.ObjectHelper.clone(syncChangeset);
         delete changeSetWithoutProof.proof;
         const proof = await this._identityConnector.createProof(this._nodeIdentity, identityModels.DocumentHelper.joinId(this._nodeIdentity, this._decentralisedStorageMethodId), standardsW3cDid.ProofTypes.DataIntegrityProof, changeSetWithoutProof);
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "createdChangeSetProof",
@@ -752,7 +759,7 @@ class ChangeSetHelper {
         if (core.Is.stringValue(this._nodeIdentity)) {
             const verified = await this.verifyChangesetProof(syncChangeSet);
             if (verified) {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "copyChangeSet",
@@ -781,7 +788,7 @@ class ChangeSetHelper {
     async reset(storageKey, resetMode) {
         // If we are applying a consolidation we need to reset the local db
         // but keep any entries from the local node, as they might have been updated
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "storageReset",
@@ -813,10 +820,10 @@ class LocalSyncStateHelper {
      */
     CLASS_NAME = "LocalSyncStateHelper";
     /**
-     * The logging connector to use for logging.
+     * The logging component to use for logging.
      * @internal
      */
-    _logging;
+    _loggingComponent;
     /**
      * The storage connector for the sync snapshot entries.
      * @internal
@@ -829,12 +836,12 @@ class LocalSyncStateHelper {
     _changeSetHelper;
     /**
      * Create a new instance of LocalSyncStateHelper.
-     * @param logging The logging connector to use for logging.
+     * @param loggingComponent The logging connector to use for logging.
      * @param snapshotEntryEntityStorage The storage connector for the sync snapshot entries.
      * @param changeSetHelper The change set helper to use for applying changesets.
      */
-    constructor(logging, snapshotEntryEntityStorage, changeSetHelper) {
-        this._logging = logging;
+    constructor(loggingComponent, snapshotEntryEntityStorage, changeSetHelper) {
+        this._loggingComponent = loggingComponent;
         this._snapshotEntryEntityStorage = snapshotEntryEntityStorage;
         this._changeSetHelper = changeSetHelper;
     }
@@ -846,7 +853,7 @@ class LocalSyncStateHelper {
      * @returns Nothing.
      */
     async addLocalChange(storageKey, operation, id) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "addLocalChange",
@@ -867,6 +874,9 @@ class LocalSyncStateHelper {
             if (previousChangeIndex !== -1) {
                 localChangeSnapshot.changes.splice(previousChangeIndex, 1);
             }
+            // If we already have changes from previous updates
+            // then make sure we update the dateModified, otherwise
+            // we assume this is the first change and setting modified is not necessary
             if (localChangeSnapshot.changes.length > 0) {
                 localChangeSnapshot.dateModified = new Date(Date.now()).toISOString();
             }
@@ -881,7 +891,7 @@ class LocalSyncStateHelper {
      * @returns The local snapshot entry.
      */
     async getSnapshots(storageKey, isLocal) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "getSnapshots",
@@ -904,7 +914,7 @@ class LocalSyncStateHelper {
             ]
         });
         if (queryResult.entities.length > 0) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "getSnapshotsExists",
@@ -914,7 +924,7 @@ class LocalSyncStateHelper {
             });
             return queryResult.entities;
         }
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "getSnapshotsDoesNotExist",
@@ -932,7 +942,8 @@ class LocalSyncStateHelper {
                 dateModified: now,
                 changeSetStorageIds: [],
                 isLocal,
-                isConsolidated: false
+                isConsolidated: false,
+                epoch: 0
             }
         ];
     }
@@ -942,7 +953,7 @@ class LocalSyncStateHelper {
      * @returns Nothing.
      */
     async setLocalChangeSnapshot(localChangeSnapshot) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "setLocalChangeSnapshot",
@@ -958,7 +969,7 @@ class LocalSyncStateHelper {
      * @returns Nothing.
      */
     async removeLocalChangeSnapshot(localChangeSnapshot) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "removeLocalChangeSnapshot",
@@ -975,7 +986,7 @@ class LocalSyncStateHelper {
      * @returns Nothing.
      */
     async applySyncState(storageKey, syncState) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "applySyncState",
@@ -984,14 +995,23 @@ class LocalSyncStateHelper {
             }
         });
         // Get all the existing snapshots that we have processed previously
-        const existingRemoteSnapshots = await this.getSnapshots(storageKey, false);
+        let existingSnapshots = await this.getSnapshots(storageKey, false);
         // Sort from newest to oldest
-        const sortedSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
-        // If we have no existing snapshots we can't have yet synced
-        // in this case we need to find the most recent consolidation
-        // and use that to build a complete DB table
-        if (existingRemoteSnapshots.length === 0) {
-            await this._logging?.log({
+        existingSnapshots = existingSnapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
+        // Sort from newest to oldest
+        const syncStateSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
+        // Get the newest epoch from the local storage
+        const newestExistingEpoch = existingSnapshots[0]?.epoch ?? 0;
+        // Get the oldest epoch from the remote storage
+        const oldestSyncStateEpoch = syncStateSnapshots[syncStateSnapshots.length - 1]?.epoch ?? 0;
+        // If there is a gap between the largest epoch we have locally
+        // and the smallest epoch we have remotely then we have missed
+        // data so we need to perform a full sync
+        const hasEpochGap = newestExistingEpoch + 1 < oldestSyncStateEpoch;
+        // If we have an epoch gap or no existing snapshots then we need to apply
+        // a full sync from a consolidation
+        if (!existingSnapshots.some(s => s.isConsolidated) || hasEpochGap) {
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "applySnapshotNoExisting",
@@ -999,31 +1019,38 @@ class LocalSyncStateHelper {
                     storageKey
                 }
             });
-            const firstConsolidated = sortedSnapshots.find(snapshot => snapshot.isConsolidated);
-            if (firstConsolidated) {
-                // We found a consolidated snapshot, we can use it
-                await this._logging?.log({
+            const mostRecentConsolidation = syncStateSnapshots.findIndex(snapshot => snapshot.isConsolidated);
+            if (mostRecentConsolidation !== -1) {
+                // We found the most recent consolidated snapshot, we can use it
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "applySnapshotFoundConsolidated",
                     data: {
                         storageKey,
-                        snapshotId: firstConsolidated.id
+                        snapshotId: syncStateSnapshots[mostRecentConsolidation].id
                     }
                 });
                 // We need to reset the entity storage and remove all the remote items
-                // so that we use just the ones from the consolidation
+                // so that we use just the ones from the consolidation, since
+                // we don't have any existing there shouldn't be any remote entries
+                // but we reset nonetheless
                 await this._changeSetHelper.reset(storageKey, synchronisedStorageModels.SyncNodeIdentityMode.Remote);
-                await this.processNewSnapshots([
-                    {
-                        ...firstConsolidated,
-                        storageKey,
-                        isLocal: false
-                    }
-                ]);
+                // We need to process the most recent consolidation and all changes
+                // that were made since then, from newest to oldest (so newer changes override older ones)
+                // Process snapshots from the consolidation point (most recent) back to the newest
+                for (let i = mostRecentConsolidation; i >= 0; i--) {
+                    await this.processNewSnapshots([
+                        {
+                            ...syncStateSnapshots[i],
+                            storageKey,
+                            isLocal: false
+                        }
+                    ]);
+                }
             }
             else {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "applySnapshotNoConsolidated",
@@ -1034,16 +1061,21 @@ class LocalSyncStateHelper {
             }
         }
         else {
+            // We have existing consolidated remote snapshots, so we can assume that we have
+            // applied at least one consolidation snapshot, in this case we need to look at the changes since
+            // then and apply them if we haven't already
+            // We don't need to apply any additional consolidated snapshots, just the changesets
             // Create a lookup map for the existing snapshots
-            const existingSnapshots = {};
-            for (const snapshot of existingRemoteSnapshots) {
-                existingSnapshots[snapshot.id] = snapshot;
+            const existingSnapshotsMap = {};
+            for (const snapshot of existingSnapshots) {
+                existingSnapshotsMap[snapshot.id] = snapshot;
             }
             const newSnapshots = [];
             const modifiedSnapshots = [];
-            const referencedExistingSnapshots = Object.keys(existingSnapshots);
-            for (const snapshot of sortedSnapshots) {
-                await this._logging?.log({
+            const referencedExistingSnapshots = Object.keys(existingSnapshotsMap);
+            let completedProcessing = false;
+            for (const snapshot of syncStateSnapshots) {
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "applySnapshot",
@@ -1052,34 +1084,37 @@ class LocalSyncStateHelper {
                         dateCreated: new Date(snapshot.dateCreated).toISOString()
                     }
                 });
-                // See if we have the local snapshot
-                const currentSnapshot = existingSnapshots[snapshot.id];
+                // See if we have the snapshot stored locally
+                const currentSnapshot = existingSnapshotsMap[snapshot.id];
                 // As we are referencing an existing snapshot, we need to remove it from the list
                 // to allow us to cleanup any unreferenced snapshots later
                 const idx = referencedExistingSnapshots.indexOf(snapshot.id);
                 if (idx !== -1) {
                     referencedExistingSnapshots.splice(idx, 1);
                 }
-                const updatedSnapshot = {
-                    ...snapshot,
-                    storageKey,
-                    isLocal: false
-                };
-                if (core.Is.empty(currentSnapshot)) {
-                    // We don't have the snapshot locally, so we need to process it
-                    newSnapshots.push(updatedSnapshot);
-                }
-                else if (currentSnapshot.dateModified !== snapshot.dateModified) {
-                    // If the local snapshot has a different dateModified, we need to update it
-                    modifiedSnapshots.push({
-                        currentSnapshot,
-                        updatedSnapshot
-                    });
-                }
-                else {
-                    // we sorted the snapshots from newest to oldest, so if we found a local snapshot
-                    // with the same dateModified as the remote snapshot, we can stop processing further
-                    break;
+                // No need to apply consolidated snapshots
+                if (!snapshot.isConsolidated && !completedProcessing) {
+                    const updatedSnapshot = {
+                        ...snapshot,
+                        storageKey,
+                        isLocal: false
+                    };
+                    if (core.Is.empty(currentSnapshot)) {
+                        // We don't have the snapshot locally, so we need to process all of it
+                        newSnapshots.push(updatedSnapshot);
+                    }
+                    else if (currentSnapshot.dateModified !== snapshot.dateModified) {
+                        // If the local snapshot has a different dateModified, we need to update it
+                        modifiedSnapshots.push({
+                            currentSnapshot,
+                            updatedSnapshot
+                        });
+                    }
+                    else {
+                        // we sorted the snapshots from newest to oldest, so if we found a local snapshot
+                        // with the same dateModified as the remote snapshot, we can stop processing further
+                        completedProcessing = true;
+                    }
                 }
             }
             // We reverse the order of the snapshots to process them from oldest to newest
@@ -1101,7 +1136,7 @@ class LocalSyncStateHelper {
      */
     async processModifiedSnapshots(modifiedSnapshots) {
         for (const modifiedSnapshot of modifiedSnapshots) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "processModifiedSnapshot",
@@ -1134,7 +1169,7 @@ class LocalSyncStateHelper {
      */
     async processNewSnapshots(newSnapshots) {
         for (const newSnapshot of newSnapshots) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "processNewSnapshot",
@@ -1165,10 +1200,10 @@ class RemoteSyncStateHelper {
      */
     CLASS_NAME = "RemoteSyncStateHelper";
     /**
-     * The logging connector to use for logging.
+     * The logging component to use for logging.
      * @internal
      */
-    _logging;
+    _loggingComponent;
     /**
      * The event bus component.
      * @internal
@@ -1221,7 +1256,7 @@ class RemoteSyncStateHelper {
     _maxConsolidations;
     /**
      * Create a new instance of DecentralisedEntityStorageConnector.
-     * @param logging The logging connector to use for logging.
+     * @param loggingComponent The logging component to use for logging.
      * @param eventBusComponent The event bus component to use for events.
      * @param verifiableSyncPointerStorageConnector The verifiable storage connector to use for storing sync pointers.
      * @param blobStorageHelper The blob storage helper to use for remote sync states.
@@ -1229,8 +1264,8 @@ class RemoteSyncStateHelper {
      * @param isTrustedNode Whether the node is trusted or not.
      * @param maxConsolidations The maximum number of consolidations to keep in storage.
      */
-    constructor(logging, eventBusComponent, verifiableSyncPointerStorageConnector, blobStorageHelper, changeSetHelper, isTrustedNode, maxConsolidations) {
-        this._logging = logging;
+    constructor(loggingComponent, eventBusComponent, verifiableSyncPointerStorageConnector, blobStorageHelper, changeSetHelper, isTrustedNode, maxConsolidations) {
+        this._loggingComponent = loggingComponent;
         this._eventBusComponent = eventBusComponent;
         this._verifiableSyncPointerStorageConnector = verifiableSyncPointerStorageConnector;
         this._changeSetHelper = changeSetHelper;
@@ -1268,7 +1303,7 @@ class RemoteSyncStateHelper {
      * @returns The storage id of the change set if created.
      */
     async buildChangeSet(storageKey, changes, completeCallback) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "buildingChangeSet",
@@ -1294,7 +1329,7 @@ class RemoteSyncStateHelper {
             // Once all the requests are handled the callback will be called
             for (const change of setChanges) {
                 // Create a request for each change to populate the full details
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "createChangeSetRequestingItem",
@@ -1317,7 +1352,7 @@ class RemoteSyncStateHelper {
      * @returns Nothing.
      */
     async finaliseFullChanges(storageKey, completeCallback) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "finalisingSyncChanges",
@@ -1358,7 +1393,7 @@ class RemoteSyncStateHelper {
                 await completeCallback(syncChangeSet, changeSetStorageId);
             }
             catch (err) {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "error",
                     source: this.CLASS_NAME,
                     message: "finalisingSyncChangesFailed",
@@ -1381,7 +1416,7 @@ class RemoteSyncStateHelper {
      * @returns Nothing.
      */
     async addChangeSetToSyncState(storageKey, changeSetStorageId) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "addChangeSetToSyncState",
@@ -1404,6 +1439,7 @@ class RemoteSyncStateHelper {
         const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
         // Get the current snapshot, if it does not exist we create a new one
         let currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+        const currentEpoch = currentSnapshot?.epoch ?? 0;
         const now = new Date(Date.now()).toISOString();
         // If there is no snapshot or the current one is a consolidation
         // we start a new snapshot
@@ -1414,6 +1450,7 @@ class RemoteSyncStateHelper {
                 dateCreated: now,
                 dateModified: now,
                 isConsolidated: false,
+                epoch: currentEpoch + 1,
                 changeSetStorageIds: []
             };
             syncState.snapshots.push(currentSnapshot);
@@ -1436,7 +1473,7 @@ class RemoteSyncStateHelper {
      * @returns Nothing.
      */
     async consolidationStart(storageKey, batchSize) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "consolidationStarting"
@@ -1451,7 +1488,7 @@ class RemoteSyncStateHelper {
     async getVerifiableSyncPointerStore() {
         if (core.Is.stringValue(this._synchronisedStorageKey)) {
             try {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "verifiableSyncPointerStoreRetrieving",
@@ -1462,7 +1499,7 @@ class RemoteSyncStateHelper {
                 const syncPointerStore = await this._verifiableSyncPointerStorageConnector.get(this._synchronisedStorageKey, { includeData: true });
                 if (core.Is.uint8Array(syncPointerStore.data)) {
                     const syncPointer = core.ObjectHelper.fromBytes(syncPointerStore.data);
-                    await this._logging?.log({
+                    await this._loggingComponent?.log({
                         level: "info",
                         source: this.CLASS_NAME,
                         message: "verifiableSyncPointerStoreRetrieved",
@@ -1478,7 +1515,7 @@ class RemoteSyncStateHelper {
                     throw err;
                 }
             }
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "verifiableSyncPointerStoreNotFound",
@@ -1500,7 +1537,7 @@ class RemoteSyncStateHelper {
      */
     async storeVerifiableSyncPointerStore(syncPointerStore) {
         if (core.Is.stringValue(this._nodeIdentity) && core.Is.stringValue(this._synchronisedStorageKey)) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "verifiableSyncPointerStoreStoring",
@@ -1518,7 +1555,7 @@ class RemoteSyncStateHelper {
      * @returns The id of the sync state.
      */
     async storeRemoteSyncState(syncState) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "syncStateStoring",
@@ -1544,7 +1581,12 @@ class RemoteSyncStateHelper {
             const toRemove = snapshots.slice(consolidationIndexes[this._maxConsolidations - 1] + 1);
             syncState.snapshots = snapshots.slice(0, consolidationIndexes[this._maxConsolidations - 1] + 1);
             for (const snapshot of toRemove) {
-                await this._blobStorageHelper.removeBlob(snapshot.id);
+                // We need to remove all the storage ids associated with the snapshot
+                if (core.Is.arrayValue(snapshot.changeSetStorageIds)) {
+                    for (const storageId of snapshot.changeSetStorageIds) {
+                        await this._blobStorageHelper.removeBlob(storageId);
+                    }
+                }
             }
         }
         return this._blobStorageHelper.saveBlob(syncState);
@@ -1556,7 +1598,7 @@ class RemoteSyncStateHelper {
      */
     async getSyncState(syncPointerId) {
         try {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "syncStateRetrieving",
@@ -1566,7 +1608,7 @@ class RemoteSyncStateHelper {
             });
             const syncState = await this._blobStorageHelper.loadBlob(syncPointerId);
             if (core.Is.object(syncState)) {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "syncStateRetrieved",
@@ -1579,7 +1621,7 @@ class RemoteSyncStateHelper {
             }
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "warn",
                 source: this.CLASS_NAME,
                 message: "getSyncStateError",
@@ -1589,7 +1631,7 @@ class RemoteSyncStateHelper {
                 error: core.BaseError.fromError(error)
             });
         }
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "syncStateNotFound",
@@ -1639,12 +1681,17 @@ class RemoteSyncStateHelper {
                     storageKey: response.storageKey,
                     snapshots: []
                 };
+                // Sort the snapshots so the newest snapshot is last in the array
+                const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
+                const currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+                const currentEpoch = currentSnapshot?.epoch ?? 0;
                 const batchSnapshot = {
                     version: SYNC_SNAPSHOT_VERSION,
                     id: core.Converter.bytesToHex(core.RandomHelper.generate(32)),
                     dateCreated: now,
                     dateModified: now,
                     isConsolidated: true,
+                    epoch: currentEpoch + 1,
                     changeSetStorageIds: this._batchResponseStorageIds[response.storageKey]
                 };
                 syncState.snapshots.push(batchSnapshot);
@@ -1656,7 +1703,7 @@ class RemoteSyncStateHelper {
                 // Remove the batch response storage ids for the storage key
                 // as we have consolidated the changes
                 delete this._batchResponseStorageIds[response.storageKey];
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "consolidationCompleted"
@@ -1669,7 +1716,7 @@ class RemoteSyncStateHelper {
      * @param response The item response to handle.
      */
     async handleLocalItemResponse(response) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "createChangeSetRespondingItem",
@@ -1723,10 +1770,10 @@ class SynchronisedStorageService {
      */
     CLASS_NAME = "SynchronisedStorageService";
     /**
-     * The logging connector to use for logging.
+     * The logging component to use for logging.
      * @internal
      */
-    _logging;
+    _loggingComponent;
     /**
      * The event bus component.
      * @internal
@@ -1820,7 +1867,7 @@ class SynchronisedStorageService {
         core.Guards.object(this.CLASS_NAME, "options", options);
         core.Guards.object(this.CLASS_NAME, "options.config", options.config);
         this._eventBusComponent = core.ComponentFactory.get(options.eventBusComponentType ?? "event-bus");
-        this._logging = loggingModels.LoggingConnectorFactory.getIfExists(options.loggingConnectorType ?? "logging");
+        this._loggingComponent = core.ComponentFactory.getIfExists(options.loggingComponentType ?? "logging");
         this._vaultConnector = vaultModels.VaultConnectorFactory.get(options.vaultConnectorType ?? "vault");
         this._localSyncSnapshotEntryEntityStorage = entityStorageModels.EntityStorageConnectorFactory.get(options.syncSnapshotStorageConnectorType ?? "sync-snapshot-entry");
         this._verifiableSyncPointerStorageConnector = verifiableStorageModels.VerifiableStorageConnectorFactory.get(options.verifiableStorageConnectorType ?? "verifiable-storage");
@@ -1849,10 +1896,10 @@ class SynchronisedStorageService {
             this._trustedSynchronisedStorageComponent =
                 core.ComponentFactory.get(options.trustedSynchronisedStorageComponentType);
         }
-        this._blobStorageHelper = new BlobStorageHelper(this._logging, this._vaultConnector, this._blobStorageConnector, this._config.blobStorageEncryptionKeyId, this._config.isTrustedNode);
-        this._changeSetHelper = new ChangeSetHelper(this._logging, this._eventBusComponent, this._identityConnector, this._blobStorageHelper, this._config.synchronisedStorageMethodId);
-        this._localSyncStateHelper = new LocalSyncStateHelper(this._logging, this._localSyncSnapshotEntryEntityStorage, this._changeSetHelper);
-        this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._logging, this._eventBusComponent, this._verifiableSyncPointerStorageConnector, this._blobStorageHelper, this._changeSetHelper, this._config.isTrustedNode, this._config.maxConsolidations);
+        this._blobStorageHelper = new BlobStorageHelper(this._loggingComponent, this._vaultConnector, this._blobStorageConnector, this._config.blobStorageEncryptionKeyId, this._config.isTrustedNode);
+        this._changeSetHelper = new ChangeSetHelper(this._loggingComponent, this._eventBusComponent, this._identityConnector, this._blobStorageHelper, this._config.synchronisedStorageMethodId);
+        this._localSyncStateHelper = new LocalSyncStateHelper(this._loggingComponent, this._localSyncSnapshotEntryEntityStorage, this._changeSetHelper);
+        this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._loggingComponent, this._eventBusComponent, this._verifiableSyncPointerStorageConnector, this._blobStorageHelper, this._changeSetHelper, this._config.isTrustedNode, this._config.maxConsolidations);
         this._serviceStarted = false;
         this._activeStorageKeys = {};
         this._eventBusComponent.subscribe(synchronisedStorageModels.SynchronisedStorageTopics.RegisterStorageKey, async (event) => this.registerStorageKey(event.data));
@@ -1937,7 +1984,7 @@ class SynchronisedStorageService {
             throw new core.GeneralError(this.CLASS_NAME, "notTrustedNode");
         }
         core.Guards.object(this.CLASS_NAME, "syncChangeSet", syncChangeSet);
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "syncChangeSetForRemoteNode",
@@ -1966,7 +2013,7 @@ class SynchronisedStorageService {
      */
     async startEntitySync(storageKey) {
         try {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "startEntitySync",
@@ -1980,7 +2027,7 @@ class SynchronisedStorageService {
             await this.updateFromLocalSyncState(storageKey);
         }
         catch (error) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "entitySyncFailed",
@@ -1995,7 +2042,7 @@ class SynchronisedStorageService {
      * @internal
      */
     async updateFromRemoteSyncState(storageKey) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "updateFromRemoteSyncState",
@@ -2021,7 +2068,7 @@ class SynchronisedStorageService {
      * @internal
      */
     async updateFromLocalSyncState(storageKey) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "updateFromLocalSyncState",
@@ -2035,7 +2082,7 @@ class SynchronisedStorageService {
             if (core.Is.arrayValue(localChangeSnapshot.changes)) {
                 await this._remoteSyncStateHelper.buildChangeSet(storageKey, localChangeSnapshot.changes, async (syncChangeSet, changeSetStorageId) => {
                     if (core.Is.empty(syncChangeSet) && core.Is.empty(changeSetStorageId)) {
-                        await this._logging?.log({
+                        await this._loggingComponent?.log({
                             level: "info",
                             source: this.CLASS_NAME,
                             message: "builtStorageChangeSetNone",
@@ -2045,7 +2092,7 @@ class SynchronisedStorageService {
                         });
                     }
                     else {
-                        await this._logging?.log({
+                        await this._loggingComponent?.log({
                             level: "info",
                             source: this.CLASS_NAME,
                             message: "builtStorageChangeSet",
@@ -2065,7 +2112,7 @@ class SynchronisedStorageService {
                             core.Is.object(syncChangeSet)) {
                             // If we are not a trusted node, we need to send the changes to the trusted node
                             // and then remove the local change snapshot
-                            await this._logging?.log({
+                            await this._loggingComponent?.log({
                                 level: "info",
                                 source: this.CLASS_NAME,
                                 message: "sendingChangeSetToTrustedNode",
@@ -2081,7 +2128,7 @@ class SynchronisedStorageService {
                 });
             }
             else {
-                await this._logging?.log({
+                await this._loggingComponent?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "updateFromLocalSyncStateNoChanges",
@@ -2099,26 +2146,18 @@ class SynchronisedStorageService {
      * @internal
      */
     async startConsolidationSync(storageKey) {
-        let localChangeSnapshot;
         try {
-            // If we are performing a consolidation, we can remove the local change snapshot
-            // as we are going to create a complete changeset from the DB
-            const localChangeSnapshots = await this._localSyncStateHelper.getSnapshots(storageKey, true);
-            localChangeSnapshot = localChangeSnapshots[0];
-            if (!core.Is.empty(localChangeSnapshot)) {
-                await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
-            }
+            // If we are going to perform a consolidation first take any local updates
+            // we have and create a changeset from them, so that anybody applying
+            // just changes since a consolidation can use the changeset
+            // and skip the consolidation
+            await this.updateFromLocalSyncState(storageKey);
+            // Now start the consolidation
             await this._remoteSyncStateHelper.consolidationStart(storageKey, this._config.consolidationBatchSize ??
                 SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE);
-            // The consolidation was successful, so we can remove the local change snapshot permanently
-            localChangeSnapshot = undefined;
         }
         catch (error) {
-            if (localChangeSnapshot) {
-                // If the consolidation failed, we can keep the local change snapshot
-                await this._localSyncStateHelper.setLocalChangeSnapshot(localChangeSnapshot);
-            }
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "error",
                 source: this.CLASS_NAME,
                 message: "consolidationSyncFailed",
@@ -2132,7 +2171,7 @@ class SynchronisedStorageService {
      * @internal
      */
     async registerStorageKey(syncRegisterStorageKey) {
-        await this._logging?.log({
+        await this._loggingComponent?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "registerStorageKey",
@@ -2154,7 +2193,7 @@ class SynchronisedStorageService {
      */
     async activateStorageKey(storageKey) {
         if (!core.Is.empty(this._activeStorageKeys[storageKey]) && !this._activeStorageKeys[storageKey]) {
-            await this._logging?.log({
+            await this._loggingComponent?.log({
                 level: "info",
                 source: this.CLASS_NAME,
                 message: "activateStorageKey",