@twin.org/synchronised-storage-service 0.0.1-next.4 → 0.0.1-next.6

package/dist/esm/index.mjs

@@ -6,7 +6,7 @@ import { EntityStorageConnectorFactory } from '@twin.org/entity-storage-models';
 import { DocumentHelper, IdentityConnectorFactory } from '@twin.org/identity-models';
 import { LoggingConnectorFactory } from '@twin.org/logging-models';
 import { ProofTypes } from '@twin.org/standards-w3c-did';
-import { SyncChangeOperation, SynchronisedStorageTopics } from '@twin.org/synchronised-storage-models';
+import { SyncChangeOperation, SynchronisedStorageTopics, SyncNodeIdentityMode } from '@twin.org/synchronised-storage-models';
 import { VaultEncryptionType, VaultConnectorFactory } from '@twin.org/vault-models';
 import { VerifiableStorageConnectorFactory } from '@twin.org/verifiable-storage-models';
 import { RSA } from '@twin.org/crypto';
@@ -21,6 +21,10 @@ let SyncSnapshotEntry = class SyncSnapshotEntry {
      * The id for the snapshot.
      */
     id;
+    /**
+     * The version for the snapshot.
+     */
+    version;
     /**
      * The storage key for the snapshot i.e. which entity is being synchronized.
      */
@@ -34,9 +38,17 @@ let SyncSnapshotEntry = class SyncSnapshotEntry {
      */
     dateModified;
     /**
-     * The flag to determine if this is the current local snapshot containing changes for this node.
+     * The flag to determine if this is the snapshot is the local one containing changes for this node.
+     */
+    isLocal;
+    /**
+     * The flag to determine if this is a consolidated snapshot.
      */
-    isLocalSnapshot;
+    isConsolidated;
+    /**
+     * The epoch for the changeset.
+     */
+    epoch;
     /**
      * The ids of the storage for the change sets in the snapshot, if this is not a local snapshot.
      */
@@ -50,6 +62,10 @@ __decorate([
     property({ type: "string", isPrimary: true }),
     __metadata("design:type", String)
 ], SyncSnapshotEntry.prototype, "id", void 0);
+__decorate([
+    property({ type: "string" }),
+    __metadata("design:type", String)
+], SyncSnapshotEntry.prototype, "version", void 0);
 __decorate([
     property({ type: "string", isSecondary: true }),
     __metadata("design:type", String)
@@ -59,13 +75,21 @@ __decorate([
     __metadata("design:type", String)
 ], SyncSnapshotEntry.prototype, "dateCreated", void 0);
 __decorate([
-    property({ type: "string", optional: true }),
+    property({ type: "string" }),
     __metadata("design:type", String)
 ], SyncSnapshotEntry.prototype, "dateModified", void 0);
 __decorate([
-    property({ type: "boolean", optional: true }),
+    property({ type: "boolean" }),
     __metadata("design:type", Boolean)
-], SyncSnapshotEntry.prototype, "isLocalSnapshot", void 0);
+], SyncSnapshotEntry.prototype, "isLocal", void 0);
+__decorate([
+    property({ type: "boolean" }),
+    __metadata("design:type", Boolean)
+], SyncSnapshotEntry.prototype, "isConsolidated", void 0);
+__decorate([
+    property({ type: "number" }),
+    __metadata("design:type", Number)
+], SyncSnapshotEntry.prototype, "epoch", void 0);
 __decorate([
     property({ type: "array", itemType: "string", optional: true }),
     __metadata("design:type", Array)
@@ -114,6 +138,7 @@ function generateRestRoutesSynchronisedStorage(baseRouteName, componentName) {
                         body: {
                             id: "0909090909090909090909090909090909090909090909090909090909090909",
                             dateCreated: "2025-05-29T01:00:00.000Z",
+                            dateModified: "2025-05-29T01:00:00.000Z",
                             nodeIdentity: "did:entity-storage:0xd2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2",
                             changes: [
                                 {
@@ -314,7 +339,7 @@ class BlobStorageHelper {
      * @param blobId The id of the blob to apply.
      * @returns The blob.
      */
-    async load(blobId) {
+    async loadBlob(blobId) {
         await this._logging?.log({
             level: "info",
             source: this.CLASS_NAME,
@@ -407,6 +432,51 @@ class BlobStorageHelper {
             throw error;
         }
     }
+    /**
+     * Remove a blob from storage.
+     * @param blobId The id of the blob to remove.
+     * @returns Nothing.
+     */
+    async removeBlob(blobId) {
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "removeBlob",
+            data: {
+                blobId
+            }
+        });
+        try {
+            await this._blobStorageConnector.remove(blobId);
+            await this._logging?.log({
+                level: "info",
+                source: this.CLASS_NAME,
+                message: "removedBlob",
+                data: {
+                    blobId
+                }
+            });
+        }
+        catch (error) {
+            await this._logging?.log({
+                level: "error",
+                source: this.CLASS_NAME,
+                message: "removeBlobFailed",
+                data: {
+                    blobId
+                },
+                error: BaseError.fromError(error)
+            });
+        }
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "removeBlobEmpty",
+            data: {
+                blobId
+            }
+        });
+    }
 }
 
 // Copyright 2024 IOTA Stiftung.
@@ -486,7 +556,7 @@ class ChangeSetHelper {
             }
         });
         try {
-            const syncChangeSet = await this._blobStorageHelper.load(changeSetStorageId);
+            const syncChangeSet = await this._blobStorageHelper.loadBlob(changeSetStorageId);
             if (Is.object(syncChangeSet)) {
                 const verified = await this.verifyChangesetProof(syncChangeSet);
                 return verified ? syncChangeSet : undefined;
@@ -519,7 +589,9 @@ class ChangeSetHelper {
      */
     async getAndApplyChangeset(changeSetStorageId) {
         const syncChangeset = await this.getAndVerifyChangeset(changeSetStorageId);
-        if (!Is.empty(syncChangeset)) {
+        // Only apply changesets from other nodes, we don't want to overwrite
+        // any changes we have made to local entity storage
+        if (!Is.empty(syncChangeset) && syncChangeset.nodeIdentity !== this._nodeIdentity) {
             await this.applyChangeset(syncChangeset);
         }
         return syncChangeset;
@@ -563,7 +635,8 @@ class ChangeSetHelper {
                         if (!Is.empty(change.id)) {
                             await this._eventBusComponent.publish(SynchronisedStorageTopics.RemoteItemRemove, {
                                 storageKey: syncChangeset.storageKey,
-                                id: change.id
+                                id: change.id,
+                                nodeIdentity: syncChangeset.nodeIdentity
                             });
                         }
                         break;
@@ -705,8 +778,36 @@ class ChangeSetHelper {
             }
         }
     }
+    /**
+     * Reset the storage for a given storage key.
+     * @param storageKey The key of the storage to reset.
+     * @param resetMode The reset mode, this will use the nodeIdentity in the entities to determine which are local/remote.
+     * @returns Nothing.
+     */
+    async reset(storageKey, resetMode) {
+        // If we are applying a consolidation we need to reset the local db
+        // but keep any entries from the local node, as they might have been updated
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "storageReset",
+            data: {
+                storageKey
+            }
+        });
+        await this._eventBusComponent.publish(SynchronisedStorageTopics.Reset, {
+            storageKey,
+            resetMode
+        });
+    }
 }
 
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+const SYNC_STATE_VERSION = "1";
+const SYNC_POINTER_STORE_VERSION = "1";
+const SYNC_SNAPSHOT_VERSION = "1";
+
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
 /**
@@ -761,31 +862,38 @@ class LocalSyncStateHelper {
                 id
             }
         });
-        const localChangeSnapshot = await this.getLocalChangeSnapshot(storageKey);
-        localChangeSnapshot.changes ??= [];
-        // If we already have a change for this id we are
-        // about to supersede it, we remove the previous change
-        // to avoid having multiple changes for the same id
-        const previousChangeIndex = localChangeSnapshot.changes.findIndex(change => change.id === id);
-        if (previousChangeIndex !== -1) {
-            localChangeSnapshot.changes.splice(previousChangeIndex, 1);
-        }
-        if (localChangeSnapshot.changes.length > 0) {
-            localChangeSnapshot.dateModified = new Date(Date.now()).toISOString();
+        const localChangeSnapshots = await this.getSnapshots(storageKey, true);
+        if (localChangeSnapshots.length > 0) {
+            const localChangeSnapshot = localChangeSnapshots[0];
+            localChangeSnapshot.changes ??= [];
+            // If we already have a change for this id we are
+            // about to supersede it, we remove the previous change
+            // to avoid having multiple changes for the same id
+            const previousChangeIndex = localChangeSnapshot.changes.findIndex(change => change.id === id);
+            if (previousChangeIndex !== -1) {
+                localChangeSnapshot.changes.splice(previousChangeIndex, 1);
+            }
+            // If we already have changes from previous updates
+            // then make sure we update the dateModified, otherwise
+            // we assume this is the first change and setting modified is not necessary
+            if (localChangeSnapshot.changes.length > 0) {
+                localChangeSnapshot.dateModified = new Date(Date.now()).toISOString();
+            }
+            localChangeSnapshot.changes.push({ operation, id });
+            await this.setLocalChangeSnapshot(localChangeSnapshot);
         }
-        localChangeSnapshot.changes.push({ operation, id });
-        await this.setLocalChangeSnapshot(localChangeSnapshot);
     }
     /**
-     * Get the current local snapshot which contains just the changes for this node.
+     * Get the snapshot which contains just the changes for this node.
      * @param storageKey The storage key of the snapshot to get.
+     * @param isLocal Whether to get the local snapshot or not.
      * @returns The local snapshot entry.
      */
-    async getLocalChangeSnapshot(storageKey) {
+    async getSnapshots(storageKey, isLocal) {
         await this._logging?.log({
             level: "info",
             source: this.CLASS_NAME,
-            message: "getLocalChangeSnapshot",
+            message: "getSnapshots",
             data: {
                 storageKey
             }
@@ -793,8 +901,8 @@ class LocalSyncStateHelper {
         const queryResult = await this._snapshotEntryEntityStorage.query({
             conditions: [
                 {
-                    property: "isLocalSnapshot",
-                    value: true,
+                    property: "isLocal",
+                    value: isLocal,
                     comparison: ComparisonOperator.Equals
                 },
                 {
@@ -808,28 +916,35 @@ class LocalSyncStateHelper {
             await this._logging?.log({
                 level: "info",
                 source: this.CLASS_NAME,
-                message: "localChangeSnapshotExists",
+                message: "getSnapshotsExists",
                 data: {
                     storageKey
                 }
             });
-            return queryResult.entities[0];
+            return queryResult.entities;
         }
         await this._logging?.log({
             level: "info",
             source: this.CLASS_NAME,
-            message: "localChangeSnapshotDoesNotExist",
+            message: "getSnapshotsDoesNotExist",
             data: {
                 storageKey
             }
         });
-        return {
-            id: Converter.bytesToHex(RandomHelper.generate(32)),
-            storageKey,
-            dateCreated: new Date(Date.now()).toISOString(),
-            changeSetStorageIds: [],
-            isLocalSnapshot: true
-        };
+        const now = new Date(Date.now()).toISOString();
+        return [
+            {
+                version: SYNC_SNAPSHOT_VERSION,
+                id: Converter.bytesToHex(RandomHelper.generate(32)),
+                storageKey,
+                dateCreated: now,
+                dateModified: now,
+                changeSetStorageIds: [],
+                isLocal,
+                isConsolidated: false,
+                epoch: 0
+            }
+        ];
     }
     /**
      * Set the current local snapshot with changes for this node.
@@ -878,46 +993,139 @@ class LocalSyncStateHelper {
                 snapshotCount: syncState.snapshots.length
             }
         });
+        // Get all the existing snapshots that we have processed previously
+        let existingSnapshots = await this.getSnapshots(storageKey, false);
         // Sort from newest to oldest
-        const sortedSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
-        const newSnapshots = [];
-        const modifiedSnapshots = [];
-        for (const snapshot of sortedSnapshots) {
+        existingSnapshots = existingSnapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
+        // Sort from newest to oldest
+        const syncStateSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
+        // Get the newest epoch from the local storage
+        const newestExistingEpoch = existingSnapshots[0]?.epoch ?? 0;
+        // Get the oldest epoch from the remote storage
+        const oldestSyncStateEpoch = syncStateSnapshots[syncStateSnapshots.length - 1]?.epoch ?? 0;
+        // If there is a gap between the largest epoch we have locally
+        // and the smallest epoch we have remotely then we have missed
+        // data so we need to perform a full sync
+        const hasEpochGap = newestExistingEpoch + 1 < oldestSyncStateEpoch;
+        // If we have an epoch gap or no existing snapshots then we need to apply
+        // a full sync from a consolidation
+        if (!existingSnapshots.some(s => s.isConsolidated) || hasEpochGap) {
             await this._logging?.log({
                 level: "info",
                 source: this.CLASS_NAME,
-                message: "applySnapshot",
+                message: "applySnapshotNoExisting",
                 data: {
-                    snapshotId: snapshot.id,
-                    dateCreated: new Date(snapshot.dateCreated).toISOString()
+                    storageKey
                 }
             });
-            const localSnapshot = await this._snapshotEntryEntityStorage.get(snapshot.id);
-            const remoteSnapshotWithContext = {
-                ...snapshot,
-                storageKey
-            };
-            if (Is.empty(localSnapshot)) {
-                // We don't have the snapshot locally, so we need to process it
-                newSnapshots.push(remoteSnapshotWithContext);
-            }
-            else if (localSnapshot.dateModified !== snapshot.dateModified) {
-                // If the local snapshot has a different dateModified, we need to update it
-                modifiedSnapshots.push({
-                    localSnapshot,
-                    remoteSnapshot: remoteSnapshotWithContext
+            const mostRecentConsolidation = syncStateSnapshots.findIndex(snapshot => snapshot.isConsolidated);
+            if (mostRecentConsolidation !== -1) {
+                // We found the most recent consolidated snapshot, we can use it
+                await this._logging?.log({
+                    level: "info",
+                    source: this.CLASS_NAME,
+                    message: "applySnapshotFoundConsolidated",
+                    data: {
+                        storageKey,
+                        snapshotId: syncStateSnapshots[mostRecentConsolidation].id
+                    }
                 });
+                // We need to reset the entity storage and remove all the remote items
+                // so that we use just the ones from the consolidation, since
+                // we don't have any existing there shouldn't be any remote entries
+                // but we reset nonetheless
+                await this._changeSetHelper.reset(storageKey, SyncNodeIdentityMode.Remote);
+                // We need to process the most recent consolidation and all changes
+                // that were made since then, from newest to oldest (so newer changes override older ones)
+                // Process snapshots from the consolidation point (most recent) back to the newest
+                for (let i = mostRecentConsolidation; i >= 0; i--) {
+                    await this.processNewSnapshots([
+                        {
+                            ...syncStateSnapshots[i],
+                            storageKey,
+                            isLocal: false
+                        }
+                    ]);
+                }
             }
             else {
-                // we sorted the snapshots from newest to oldest, so if we found a local snapshot
-                // with the same dateModified as the remote snapshot, we can stop processing further
-                break;
+                await this._logging?.log({
+                    level: "info",
+                    source: this.CLASS_NAME,
+                    message: "applySnapshotNoConsolidated",
+                    data: {
+                        storageKey
+                    }
+                });
+            }
+        }
+        else {
+            // We have existing consolidated remote snapshots, so we can assume that we have
+            // applied at least one consolidation snapshot, in this case we need to look at the changes since
+            // then and apply them if we haven't already
+            // We don't need to apply any additional consolidated snapshots, just the changesets
+            // Create a lookup map for the existing snapshots
+            const existingSnapshotsMap = {};
+            for (const snapshot of existingSnapshots) {
+                existingSnapshotsMap[snapshot.id] = snapshot;
+            }
+            const newSnapshots = [];
+            const modifiedSnapshots = [];
+            const referencedExistingSnapshots = Object.keys(existingSnapshotsMap);
+            let completedProcessing = false;
+            for (const snapshot of syncStateSnapshots) {
+                await this._logging?.log({
+                    level: "info",
+                    source: this.CLASS_NAME,
+                    message: "applySnapshot",
+                    data: {
+                        snapshotId: snapshot.id,
+                        dateCreated: new Date(snapshot.dateCreated).toISOString()
+                    }
+                });
+                // See if we have the snapshot stored locally
+                const currentSnapshot = existingSnapshotsMap[snapshot.id];
+                // As we are referencing an existing snapshot, we need to remove it from the list
+                // to allow us to cleanup any unreferenced snapshots later
+                const idx = referencedExistingSnapshots.indexOf(snapshot.id);
+                if (idx !== -1) {
+                    referencedExistingSnapshots.splice(idx, 1);
+                }
+                // No need to apply consolidated snapshots
+                if (!snapshot.isConsolidated && !completedProcessing) {
+                    const updatedSnapshot = {
+                        ...snapshot,
+                        storageKey,
+                        isLocal: false
+                    };
+                    if (Is.empty(currentSnapshot)) {
+                        // We don't have the snapshot locally, so we need to process all of it
+                        newSnapshots.push(updatedSnapshot);
+                    }
+                    else if (currentSnapshot.dateModified !== snapshot.dateModified) {
+                        // If the local snapshot has a different dateModified, we need to update it
+                        modifiedSnapshots.push({
+                            currentSnapshot,
+                            updatedSnapshot
+                        });
+                    }
+                    else {
+                        // we sorted the snapshots from newest to oldest, so if we found a local snapshot
+                        // with the same dateModified as the remote snapshot, we can stop processing further
+                        completedProcessing = true;
+                    }
+                }
+            }
+            // We reverse the order of the snapshots to process them from oldest to newest
+            // because we want to apply the changes in the order they were created
+            await this.processModifiedSnapshots(modifiedSnapshots.reverse());
+            await this.processNewSnapshots(newSnapshots.reverse());
+            // Any ids remaining in this list are no longer referenced in the global state
+            // so we should remove them from the local storage as they will never be updated again
+            for (const referencedSnapshotId of referencedExistingSnapshots) {
+                await this._snapshotEntryEntityStorage.remove(referencedSnapshotId);
             }
         }
-        // We reverse the order of the snapshots to process them from oldest to newest
-        // because we want to apply the changes in the order they were created
-        await this.processModifiedSnapshots(modifiedSnapshots.reverse());
-        await this.processNewSnapshots(newSnapshots.reverse());
     }
     /**
      * Process the modified snapshots and store them in the local storage.
@@ -932,15 +1140,15 @@ class LocalSyncStateHelper {
                 source: this.CLASS_NAME,
                 message: "processModifiedSnapshot",
                 data: {
-                    snapshotId: modifiedSnapshot.remoteSnapshot.id,
-                    localModified: new Date(modifiedSnapshot.localSnapshot.dateModified ??
-                        modifiedSnapshot.localSnapshot.dateCreated).toISOString(),
-                    remoteModified: new Date(modifiedSnapshot.remoteSnapshot.dateModified ??
-                        modifiedSnapshot.remoteSnapshot.dateCreated).toISOString()
+                    snapshotId: modifiedSnapshot.updatedSnapshot.id,
+                    localModified: new Date(modifiedSnapshot.currentSnapshot.dateModified ??
+                        modifiedSnapshot.currentSnapshot.dateCreated).toISOString(),
+                    remoteModified: new Date(modifiedSnapshot.updatedSnapshot.dateModified ??
+                        modifiedSnapshot.updatedSnapshot.dateCreated).toISOString()
                 }
             });
-            const remoteChangeSetStorageIds = modifiedSnapshot.remoteSnapshot.changeSetStorageIds;
-            const localChangeSetStorageIds = modifiedSnapshot.localSnapshot.changeSetStorageIds ?? [];
+            const remoteChangeSetStorageIds = modifiedSnapshot.updatedSnapshot.changeSetStorageIds;
+            const localChangeSetStorageIds = modifiedSnapshot.currentSnapshot.changeSetStorageIds ?? [];
             if (Is.arrayValue(remoteChangeSetStorageIds)) {
                 for (const storageId of remoteChangeSetStorageIds) {
                     // Check if the local snapshot does not have the storageId
@@ -949,7 +1157,7 @@ class LocalSyncStateHelper {
                     }
                 }
             }
-            await this._snapshotEntryEntityStorage.set(modifiedSnapshot.remoteSnapshot);
+            await this._snapshotEntryEntityStorage.set(modifiedSnapshot.updatedSnapshot);
         }
     }
     /**
@@ -966,7 +1174,7 @@ class LocalSyncStateHelper {
                 message: "processNewSnapshot",
                 data: {
                     snapshotId: newSnapshot.id,
-                    localModified: new Date(newSnapshot.dateCreated).toISOString()
+                    dateCreated: newSnapshot.dateCreated
                 }
             });
             const newSnapshotChangeSetStorageIds = newSnapshot.changeSetStorageIds ?? [];
@@ -980,12 +1188,6 @@ class LocalSyncStateHelper {
     }
 }
 
-// Copyright 2024 IOTA Stiftung.
-// SPDX-License-Identifier: Apache-2.0.
-const SYNC_STATE_VERSION = "1";
-const SYNC_POINTER_STORE_VERSION = "1";
-const SYNC_SNAPSHOT_VERSION = "1";
-
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
 /**
@@ -1046,6 +1248,11 @@ class RemoteSyncStateHelper {
      * @internal
      */
     _isTrustedNode;
+    /**
+     * Maximum number of consolidations to keep in storage.
+     * @internal
+     */
+    _maxConsolidations;
     /**
      * Create a new instance of DecentralisedEntityStorageConnector.
      * @param logging The logging connector to use for logging.
@@ -1054,14 +1261,16 @@ class RemoteSyncStateHelper {
      * @param blobStorageHelper The blob storage helper to use for remote sync states.
      * @param changeSetHelper The change set helper to use for managing changesets.
      * @param isTrustedNode Whether the node is trusted or not.
+     * @param maxConsolidations The maximum number of consolidations to keep in storage.
      */
-    constructor(logging, eventBusComponent, verifiableSyncPointerStorageConnector, blobStorageHelper, changeSetHelper, isTrustedNode) {
+    constructor(logging, eventBusComponent, verifiableSyncPointerStorageConnector, blobStorageHelper, changeSetHelper, isTrustedNode, maxConsolidations) {
         this._logging = logging;
         this._eventBusComponent = eventBusComponent;
         this._verifiableSyncPointerStorageConnector = verifiableSyncPointerStorageConnector;
         this._changeSetHelper = changeSetHelper;
         this._blobStorageHelper = blobStorageHelper;
         this._isTrustedNode = isTrustedNode;
+        this._maxConsolidations = maxConsolidations;
         this._batchResponseStorageIds = {};
         this._populateFullChanges = {};
         this._eventBusComponent.subscribe(SynchronisedStorageTopics.BatchResponse, async (response) => {
@@ -1163,9 +1372,11 @@ class RemoteSyncStateHelper {
                     ObjectHelper.propertyDelete(change.entity, "nodeIdentity");
                 }
             }
+            const now = new Date(Date.now()).toISOString();
             const syncChangeSet = {
                 id: Converter.bytesToHex(RandomHelper.generate(32)),
-                dateCreated: new Date(Date.now()).toISOString(),
+                dateCreated: now,
+                dateModified: now,
                 storageKey,
                 changes,
                 nodeIdentity: this._nodeIdentity
@@ -1217,23 +1428,28 @@ class RemoteSyncStateHelper {
         const syncPointerStore = await this.getVerifiableSyncPointerStore();
         let syncState;
         if (!Is.empty(syncPointerStore.syncPointers[storageKey])) {
-            syncState = await this.getRemoteSyncState(syncPointerStore.syncPointers[storageKey]);
+            syncState = await this.getSyncState(syncPointerStore.syncPointers[storageKey]);
         }
         // No current sync state, so we create a new one
         if (Is.empty(syncState)) {
-            syncState = { version: SYNC_STATE_VERSION, snapshots: [] };
+            syncState = { version: SYNC_STATE_VERSION, storageKey, snapshots: [] };
         }
         // Sort the snapshots so the newest snapshot is last in the array
         const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
         // Get the current snapshot, if it does not exist we create a new one
         let currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+        const currentEpoch = currentSnapshot?.epoch ?? 0;
         const now = new Date(Date.now()).toISOString();
-        if (Is.empty(currentSnapshot)) {
+        // If there is no snapshot or the current one is a consolidation
+        // we start a new snapshot
+        if (Is.empty(currentSnapshot) || currentSnapshot.isConsolidated) {
             currentSnapshot = {
                 version: SYNC_SNAPSHOT_VERSION,
                 id: Converter.bytesToHex(RandomHelper.generate(32)),
                 dateCreated: now,
                 dateModified: now,
+                isConsolidated: false,
+                epoch: currentEpoch + 1,
                 changeSetStorageIds: []
             };
             syncState.snapshots.push(currentSnapshot);
@@ -1262,7 +1478,7 @@ class RemoteSyncStateHelper {
             message: "consolidationStarting"
         });
         // Perform a batch request to start the consolidation
-        await this._eventBusComponent.publish(SynchronisedStorageTopics.BatchRequest, { storageKey, batchSize });
+        await this._eventBusComponent.publish(SynchronisedStorageTopics.BatchRequest, { storageKey, batchSize, requestMode: SyncNodeIdentityMode.All });
     }
     /**
      * Get the sync pointer store.
@@ -1341,11 +1557,37 @@ class RemoteSyncStateHelper {
         await this._logging?.log({
             level: "info",
             source: this.CLASS_NAME,
-            message: "remoteSyncStateStoring",
+            message: "syncStateStoring",
             data: {
                 snapshotCount: syncState.snapshots.length
             }
         });
+        // Limits the number of consolidations in the list so that we can shrink decentralised
+        // storage requirements, sort from newest to oldest so that we can easily find the
+        // oldest snapshots to remove.
+        const snapshots = syncState.snapshots.sort((a, b) => new Date(a.dateCreated).getTime() - new Date(b.dateCreated).getTime());
+        // Find all the consolidation indexes
+        const consolidationIndexes = [];
+        for (let i = 0; i < snapshots.length; i++) {
+            const snapshot = snapshots[i];
+            if (snapshot.isConsolidated) {
+                consolidationIndexes.push(i);
+            }
+        }
+        if (consolidationIndexes.length > this._maxConsolidations) {
+            // Once we have reached the max for consolidations we need to remove
+            // all the snapshots, including non consolidated ones, beyond this point
+            const toRemove = snapshots.slice(consolidationIndexes[this._maxConsolidations - 1] + 1);
+            syncState.snapshots = snapshots.slice(0, consolidationIndexes[this._maxConsolidations - 1] + 1);
+            for (const snapshot of toRemove) {
+                // We need to remove all the storage ids associated with the snapshot
+                if (Is.arrayValue(snapshot.changeSetStorageIds)) {
+                    for (const storageId of snapshot.changeSetStorageIds) {
+                        await this._blobStorageHelper.removeBlob(storageId);
+                    }
+                }
+            }
+        }
         return this._blobStorageHelper.saveBlob(syncState);
     }
     /**
@@ -1353,22 +1595,22 @@ class RemoteSyncStateHelper {
      * @param syncPointerId The id of the sync pointer to retrieve the state for.
      * @returns The remote sync state.
      */
-    async getRemoteSyncState(syncPointerId) {
+    async getSyncState(syncPointerId) {
         try {
             await this._logging?.log({
                 level: "info",
                 source: this.CLASS_NAME,
-                message: "remoteSyncStateRetrieving",
+                message: "syncStateRetrieving",
                 data: {
                     syncPointerId
                 }
             });
-            const syncState = await this._blobStorageHelper.load(syncPointerId);
+            const syncState = await this._blobStorageHelper.loadBlob(syncPointerId);
             if (Is.object(syncState)) {
                 await this._logging?.log({
                     level: "info",
                     source: this.CLASS_NAME,
-                    message: "remoteSyncStateRetrieved",
+                    message: "syncStateRetrieved",
                     data: {
                         syncPointerId,
                         snapshotCount: syncState.snapshots.length
@@ -1391,7 +1633,7 @@ class RemoteSyncStateHelper {
         await this._logging?.log({
             level: "info",
             source: this.CLASS_NAME,
-            message: "remoteSyncStateNotFound",
+            message: "syncStateNotFound",
             data: {
                 syncPointerId
             }
@@ -1430,19 +1672,29 @@ class RemoteSyncStateHelper {
                 let syncState;
                 if (Is.stringValue(syncPointerStore.syncPointers[response.storageKey])) {
                     // If the sync pointer exists, we load the current sync state
-                    syncState = await this.getRemoteSyncState(syncPointerStore.syncPointers[response.storageKey]);
+                    syncState = await this.getSyncState(syncPointerStore.syncPointers[response.storageKey]);
                 }
                 // If the sync state does not exist, we create a new one
-                syncState ??= { version: SYNC_STATE_VERSION, snapshots: [] };
+                syncState ??= {
+                    version: SYNC_STATE_VERSION,
+                    storageKey: response.storageKey,
+                    snapshots: []
+                };
+                // Sort the snapshots so the newest snapshot is last in the array
+                const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
+                const currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+                const currentEpoch = currentSnapshot?.epoch ?? 0;
                 const batchSnapshot = {
                     version: SYNC_SNAPSHOT_VERSION,
                     id: Converter.bytesToHex(RandomHelper.generate(32)),
                     dateCreated: now,
                     dateModified: now,
+                    isConsolidated: true,
+                    epoch: currentEpoch + 1,
                     changeSetStorageIds: this._batchResponseStorageIds[response.storageKey]
                 };
                 syncState.snapshots.push(batchSnapshot);
-                // Store the sync state in the blob storage
+                // Store the updated sync state
                 const syncStateId = await this.storeRemoteSyncState(syncState);
                 syncPointerStore.syncPointers[response.storageKey] = syncStateId;
                 // Store the verifiable sync pointer in the verifiable storage
@@ -1472,11 +1724,14 @@ class RemoteSyncStateHelper {
                 id: response.id
             }
         });
+        // We have received a response to an item request, find the right storage
+        // for the request id
         if (!Is.empty(this._populateFullChanges[response.storageKey])) {
             const idx = this._populateFullChanges[response.storageKey].requestIds.indexOf(response.id);
             if (idx !== -1) {
                 this._populateFullChanges[response.storageKey].requestIds.splice(idx, 1);
                 this._populateFullChanges[response.storageKey].entities[response.id] = response.entity;
+                // If there are no request ids remaining we can complete the population
                 if (this._populateFullChanges[response.storageKey].requestIds.length === 0) {
                     await this._populateFullChanges[response.storageKey].completeCallback();
                 }
@@ -1504,6 +1759,11 @@ class SynchronisedStorageService {
      * @internal
      */
     static _DEFAULT_CONSOLIDATION_BATCH_SIZE = 100;
+    /**
+     * The default max number of consolidations to keep in storage.
+     * @internal
+     */
+    static _DEFAULT_MAX_CONSOLIDATIONS = 5;
     /**
      * Runtime name for the class.
      */
@@ -1622,6 +1882,7 @@ class SynchronisedStorageService {
                 SynchronisedStorageService._DEFAULT_CONSOLIDATION_INTERVAL_MINUTES,
             consolidationBatchSize: options.config.consolidationBatchSize ??
                 SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE,
+            maxConsolidations: options.config.maxConsolidations ?? SynchronisedStorageService._DEFAULT_MAX_CONSOLIDATIONS,
             blobStorageEncryptionKeyId: options.config.blobStorageEncryptionKeyId ?? "synchronised-storage-blob-encryption-key",
             verifiableStorageKeyId: options.config.verifiableStorageKeyId
         };
@@ -1637,11 +1898,16 @@ class SynchronisedStorageService {
         this._blobStorageHelper = new BlobStorageHelper(this._logging, this._vaultConnector, this._blobStorageConnector, this._config.blobStorageEncryptionKeyId, this._config.isTrustedNode);
         this._changeSetHelper = new ChangeSetHelper(this._logging, this._eventBusComponent, this._identityConnector, this._blobStorageHelper, this._config.synchronisedStorageMethodId);
         this._localSyncStateHelper = new LocalSyncStateHelper(this._logging, this._localSyncSnapshotEntryEntityStorage, this._changeSetHelper);
-        this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._logging, this._eventBusComponent, this._verifiableSyncPointerStorageConnector, this._blobStorageHelper, this._changeSetHelper, this._config.isTrustedNode);
+        this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._logging, this._eventBusComponent, this._verifiableSyncPointerStorageConnector, this._blobStorageHelper, this._changeSetHelper, this._config.isTrustedNode, this._config.maxConsolidations);
         this._serviceStarted = false;
         this._activeStorageKeys = {};
         this._eventBusComponent.subscribe(SynchronisedStorageTopics.RegisterStorageKey, async (event) => this.registerStorageKey(event.data));
-        this._eventBusComponent.subscribe(SynchronisedStorageTopics.LocalItemChange, async (event) => this._localSyncStateHelper.addLocalChange(event.data.storageKey, event.data.operation, event.data.id));
+        this._eventBusComponent.subscribe(SynchronisedStorageTopics.LocalItemChange, async (event) => {
+            // Make sure the change event is from this node
+            if (Is.stringValue(this._nodeIdentity) && this._nodeIdentity === event.data.nodeIdentity) {
+                await this._localSyncStateHelper.addLocalChange(event.data.storageKey, event.data.operation, event.data.id);
+            }
+        });
     }
     /**
      * The component needs to be started when the node is initialized.
@@ -1731,7 +1997,7 @@ class SynchronisedStorageService {
         // to store the change set in the synchronised storage.
         // This will be performed using rights-management
         const copy = await this._changeSetHelper.copyChangeset(syncChangeSet);
-        if (!Is.empty(copy) && Is.stringValue(this._nodeIdentity)) {
+        if (!Is.empty(copy)) {
             // Apply the changes to this node
             await this._changeSetHelper.applyChangeset(copy.syncChangeSet);
             // And update the sync state with the latest changes
@@ -1788,7 +2054,7 @@ class SynchronisedStorageService {
         if (!Is.empty(verifiableSyncPointerStore.syncPointers[storageKey])) {
             // Load the sync state from the remote blob storage using the sync pointer
             // to load the sync state
-            const remoteSyncState = await this._remoteSyncStateHelper.getRemoteSyncState(verifiableSyncPointerStore.syncPointers[storageKey]);
+            const remoteSyncState = await this._remoteSyncStateHelper.getSyncState(verifiableSyncPointerStore.syncPointers[storageKey]);
             // If we got the sync state we can try and sync from it
             if (!Is.undefined(remoteSyncState)) {
                 await this._localSyncStateHelper.applySyncState(storageKey, remoteSyncState);
@@ -1809,64 +2075,67 @@ class SynchronisedStorageService {
                 storageKey
             }
         });
-        const localChangeSnapshot = await this._localSyncStateHelper.getLocalChangeSnapshot(storageKey);
-        if (Is.arrayValue(localChangeSnapshot.changes)) {
-            await this._remoteSyncStateHelper.buildChangeSet(storageKey, localChangeSnapshot.changes, async (syncChangeSet, changeSetStorageId) => {
-                if (Is.empty(syncChangeSet) && Is.empty(changeSetStorageId)) {
-                    await this._logging?.log({
-                        level: "info",
-                        source: this.CLASS_NAME,
-                        message: "builtStorageChangeSetNone",
-                        data: {
-                            storageKey
-                        }
-                    });
-                }
-                else {
-                    await this._logging?.log({
-                        level: "info",
-                        source: this.CLASS_NAME,
-                        message: "builtStorageChangeSet",
-                        data: {
-                            storageKey,
-                            changeSetStorageId
-                        }
-                    });
-                    // Send the local changes to the remote storage if we are a trusted node
-                    if (this._config.isTrustedNode && Is.stringValue(changeSetStorageId)) {
-                        // If we are a trusted node, we can add the change set to the sync state
-                        // and remove the local change snapshot
-                        await this._remoteSyncStateHelper.addChangeSetToSyncState(storageKey, changeSetStorageId);
-                        await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
+        const localChangeSnapshots = await this._localSyncStateHelper.getSnapshots(storageKey, true);
+        if (localChangeSnapshots.length > 0) {
+            const localChangeSnapshot = localChangeSnapshots[0];
+            if (Is.arrayValue(localChangeSnapshot.changes)) {
+                await this._remoteSyncStateHelper.buildChangeSet(storageKey, localChangeSnapshot.changes, async (syncChangeSet, changeSetStorageId) => {
+                    if (Is.empty(syncChangeSet) && Is.empty(changeSetStorageId)) {
+                        await this._logging?.log({
+                            level: "info",
+                            source: this.CLASS_NAME,
+                            message: "builtStorageChangeSetNone",
+                            data: {
+                                storageKey
+                            }
+                        });
                     }
-                    else if (!Is.empty(this._trustedSynchronisedStorageComponent) &&
-                        Is.object(syncChangeSet)) {
-                        // If we are not a trusted node, we need to send the changes to the trusted node
-                        // and then remove the local change snapshot
+                    else {
                         await this._logging?.log({
                             level: "info",
                             source: this.CLASS_NAME,
-                            message: "sendingChangeSetToTrustedNode",
+                            message: "builtStorageChangeSet",
                             data: {
                                 storageKey,
                                 changeSetStorageId
                             }
                         });
-                        await this._trustedSynchronisedStorageComponent.syncChangeSet(syncChangeSet);
-                        await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
+                        // Send the local changes to the remote storage if we are a trusted node
+                        if (this._config.isTrustedNode && Is.stringValue(changeSetStorageId)) {
+                            // If we are a trusted node, we can add the change set to the sync state
+                            // and remove the local change snapshot
+                            await this._remoteSyncStateHelper.addChangeSetToSyncState(storageKey, changeSetStorageId);
+                            await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
+                        }
+                        else if (!Is.empty(this._trustedSynchronisedStorageComponent) &&
+                            Is.object(syncChangeSet)) {
+                            // If we are not a trusted node, we need to send the changes to the trusted node
+                            // and then remove the local change snapshot
+                            await this._logging?.log({
+                                level: "info",
+                                source: this.CLASS_NAME,
+                                message: "sendingChangeSetToTrustedNode",
+                                data: {
+                                    storageKey,
+                                    changeSetStorageId
+                                }
+                            });
+                            await this._trustedSynchronisedStorageComponent.syncChangeSet(syncChangeSet);
+                            await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
+                        }
                     }
-                }
-            });
-        }
-        else {
-            await this._logging?.log({
-                level: "info",
-                source: this.CLASS_NAME,
-                message: "updateFromLocalSyncStateNoChanges",
-                data: {
-                    storageKey
-                }
-            });
+                });
+            }
+            else {
+                await this._logging?.log({
+                    level: "info",
+                    source: this.CLASS_NAME,
+                    message: "updateFromLocalSyncStateNoChanges",
+                    data: {
+                        storageKey
+                    }
+                });
+            }
         }
     }
     /**
@@ -1876,24 +2145,17 @@ class SynchronisedStorageService {
      * @internal
      */
     async startConsolidationSync(storageKey) {
-        let localChangeSnapshot;
         try {
-            // If we are performing a consolidation, we can remove the local change snapshot
-            // as we are going to create a complete changeset from the DB
-            localChangeSnapshot = await this._localSyncStateHelper.getLocalChangeSnapshot(storageKey);
-            if (!Is.empty(localChangeSnapshot)) {
-                await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
-            }
+            // If we are going to perform a consolidation first take any local updates
+            // we have and create a changeset from them, so that anybody applying
+            // just changes since a consolidation can use the changeset
+            // and skip the consolidation
+            await this.updateFromLocalSyncState(storageKey);
+            // Now start the consolidation
             await this._remoteSyncStateHelper.consolidationStart(storageKey, this._config.consolidationBatchSize ??
                 SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE);
-            // The consolidation was successful, so we can remove the local change snapshot permanently
-            localChangeSnapshot = undefined;
         }
         catch (error) {
-            if (localChangeSnapshot) {
-                // If the consolidation failed, we can keep the local change snapshot
-                await this._localSyncStateHelper.setLocalChangeSnapshot(localChangeSnapshot);
-            }
             await this._logging?.log({
                 level: "error",
                 source: this.CLASS_NAME,