@twin.org/synchronised-storage-service 0.0.1-next.5 → 0.0.1-next.6

package/dist/cjs/index.cjs

@@ -47,6 +47,10 @@ exports.SyncSnapshotEntry = class SyncSnapshotEntry {
      * The flag to determine if this is a consolidated snapshot.
      */
     isConsolidated;
+    /**
+     * The epoch for the changeset.
+     */
+    epoch;
     /**
      * The ids of the storage for the change sets in the snapshot, if this is not a local snapshot.
      */
@@ -84,6 +88,10 @@ __decorate([
     entity.property({ type: "boolean" }),
     __metadata("design:type", Boolean)
 ], exports.SyncSnapshotEntry.prototype, "isConsolidated", void 0);
+__decorate([
+    entity.property({ type: "number" }),
+    __metadata("design:type", Number)
+], exports.SyncSnapshotEntry.prototype, "epoch", void 0);
 __decorate([
     entity.property({ type: "array", itemType: "string", optional: true }),
     __metadata("design:type", Array)
@@ -867,6 +875,9 @@ class LocalSyncStateHelper {
             if (previousChangeIndex !== -1) {
                 localChangeSnapshot.changes.splice(previousChangeIndex, 1);
             }
+            // If we already have changes from previous updates
+            // then make sure we update the dateModified, otherwise
+            // we assume this is the first change and setting modified is not necessary
             if (localChangeSnapshot.changes.length > 0) {
                 localChangeSnapshot.dateModified = new Date(Date.now()).toISOString();
             }
@@ -932,7 +943,8 @@ class LocalSyncStateHelper {
                 dateModified: now,
                 changeSetStorageIds: [],
                 isLocal,
-                isConsolidated: false
+                isConsolidated: false,
+                epoch: 0
             }
         ];
     }
@@ -984,13 +996,22 @@ class LocalSyncStateHelper {
             }
         });
         // Get all the existing snapshots that we have processed previously
-        const existingRemoteSnapshots = await this.getSnapshots(storageKey, false);
+        let existingSnapshots = await this.getSnapshots(storageKey, false);
         // Sort from newest to oldest
-        const sortedSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
-        // If we have no existing snapshots we can't have yet synced
-        // in this case we need to find the most recent consolidation
-        // and use that to build a complete DB table
-        if (existingRemoteSnapshots.length === 0) {
+        existingSnapshots = existingSnapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
+        // Sort from newest to oldest
+        const syncStateSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
+        // Get the newest epoch from the local storage
+        const newestExistingEpoch = existingSnapshots[0]?.epoch ?? 0;
+        // Get the oldest epoch from the remote storage
+        const oldestSyncStateEpoch = syncStateSnapshots[syncStateSnapshots.length - 1]?.epoch ?? 0;
+        // If there is a gap between the largest epoch we have locally
+        // and the smallest epoch we have remotely then we have missed
+        // data so we need to perform a full sync
+        const hasEpochGap = newestExistingEpoch + 1 < oldestSyncStateEpoch;
+        // If we have an epoch gap or no existing snapshots then we need to apply
+        // a full sync from a consolidation
+        if (!existingSnapshots.some(s => s.isConsolidated) || hasEpochGap) {
             await this._logging?.log({
                 level: "info",
                 source: this.CLASS_NAME,
@@ -999,28 +1020,35 @@ class LocalSyncStateHelper {
                     storageKey
                 }
             });
-            const firstConsolidated = sortedSnapshots.find(snapshot => snapshot.isConsolidated);
-            if (firstConsolidated) {
-                // We found a consolidated snapshot, we can use it
+            const mostRecentConsolidation = syncStateSnapshots.findIndex(snapshot => snapshot.isConsolidated);
+            if (mostRecentConsolidation !== -1) {
+                // We found the most recent consolidated snapshot, we can use it
                 await this._logging?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "applySnapshotFoundConsolidated",
                     data: {
                         storageKey,
-                        snapshotId: firstConsolidated.id
+                        snapshotId: syncStateSnapshots[mostRecentConsolidation].id
                     }
                 });
                 // We need to reset the entity storage and remove all the remote items
-                // so that we use just the ones from the consolidation
+                // so that we use just the ones from the consolidation, since
+                // we don't have any existing there shouldn't be any remote entries
+                // but we reset nonetheless
                 await this._changeSetHelper.reset(storageKey, synchronisedStorageModels.SyncNodeIdentityMode.Remote);
-                await this.processNewSnapshots([
-                    {
-                        ...firstConsolidated,
-                        storageKey,
-                        isLocal: false
-                    }
-                ]);
+                // We need to process the most recent consolidation and all changes
+                // that were made since then, from newest to oldest (so newer changes override older ones)
+                // Process snapshots from the consolidation point (most recent) back to the newest
+                for (let i = mostRecentConsolidation; i >= 0; i--) {
+                    await this.processNewSnapshots([
+                        {
+                            ...syncStateSnapshots[i],
+                            storageKey,
+                            isLocal: false
+                        }
+                    ]);
+                }
             }
             else {
                 await this._logging?.log({
@@ -1034,15 +1062,20 @@ class LocalSyncStateHelper {
             }
         }
         else {
+            // We have existing consolidated remote snapshots, so we can assume that we have
+            // applied at least one consolidation snapshot, in this case we need to look at the changes since
+            // then and apply them if we haven't already
+            // We don't need to apply any additional consolidated snapshots, just the changesets
             // Create a lookup map for the existing snapshots
-            const existingSnapshots = {};
-            for (const snapshot of existingRemoteSnapshots) {
-                existingSnapshots[snapshot.id] = snapshot;
+            const existingSnapshotsMap = {};
+            for (const snapshot of existingSnapshots) {
+                existingSnapshotsMap[snapshot.id] = snapshot;
             }
             const newSnapshots = [];
             const modifiedSnapshots = [];
-            const referencedExistingSnapshots = Object.keys(existingSnapshots);
-            for (const snapshot of sortedSnapshots) {
+            const referencedExistingSnapshots = Object.keys(existingSnapshotsMap);
+            let completedProcessing = false;
+            for (const snapshot of syncStateSnapshots) {
                 await this._logging?.log({
                     level: "info",
                     source: this.CLASS_NAME,
@@ -1052,34 +1085,37 @@ class LocalSyncStateHelper {
                         dateCreated: new Date(snapshot.dateCreated).toISOString()
                     }
                 });
-                // See if we have the local snapshot
-                const currentSnapshot = existingSnapshots[snapshot.id];
+                // See if we have the snapshot stored locally
+                const currentSnapshot = existingSnapshotsMap[snapshot.id];
                 // As we are referencing an existing snapshot, we need to remove it from the list
                 // to allow us to cleanup any unreferenced snapshots later
                 const idx = referencedExistingSnapshots.indexOf(snapshot.id);
                 if (idx !== -1) {
                     referencedExistingSnapshots.splice(idx, 1);
                 }
-                const updatedSnapshot = {
-                    ...snapshot,
-                    storageKey,
-                    isLocal: false
-                };
-                if (core.Is.empty(currentSnapshot)) {
-                    // We don't have the snapshot locally, so we need to process it
-                    newSnapshots.push(updatedSnapshot);
-                }
-                else if (currentSnapshot.dateModified !== snapshot.dateModified) {
-                    // If the local snapshot has a different dateModified, we need to update it
-                    modifiedSnapshots.push({
-                        currentSnapshot,
-                        updatedSnapshot
-                    });
-                }
-                else {
-                    // we sorted the snapshots from newest to oldest, so if we found a local snapshot
-                    // with the same dateModified as the remote snapshot, we can stop processing further
-                    break;
+                // No need to apply consolidated snapshots
+                if (!snapshot.isConsolidated && !completedProcessing) {
+                    const updatedSnapshot = {
+                        ...snapshot,
+                        storageKey,
+                        isLocal: false
+                    };
+                    if (core.Is.empty(currentSnapshot)) {
+                        // We don't have the snapshot locally, so we need to process all of it
+                        newSnapshots.push(updatedSnapshot);
+                    }
+                    else if (currentSnapshot.dateModified !== snapshot.dateModified) {
+                        // If the local snapshot has a different dateModified, we need to update it
+                        modifiedSnapshots.push({
+                            currentSnapshot,
+                            updatedSnapshot
+                        });
+                    }
+                    else {
+                        // we sorted the snapshots from newest to oldest, so if we found a local snapshot
+                        // with the same dateModified as the remote snapshot, we can stop processing further
+                        completedProcessing = true;
+                    }
                 }
             }
             // We reverse the order of the snapshots to process them from oldest to newest
@@ -1404,6 +1440,7 @@ class RemoteSyncStateHelper {
         const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
         // Get the current snapshot, if it does not exist we create a new one
         let currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+        const currentEpoch = currentSnapshot?.epoch ?? 0;
         const now = new Date(Date.now()).toISOString();
         // If there is no snapshot or the current one is a consolidation
         // we start a new snapshot
@@ -1414,6 +1451,7 @@ class RemoteSyncStateHelper {
                 dateCreated: now,
                 dateModified: now,
                 isConsolidated: false,
+                epoch: currentEpoch + 1,
                 changeSetStorageIds: []
             };
             syncState.snapshots.push(currentSnapshot);
@@ -1544,7 +1582,12 @@ class RemoteSyncStateHelper {
             const toRemove = snapshots.slice(consolidationIndexes[this._maxConsolidations - 1] + 1);
             syncState.snapshots = snapshots.slice(0, consolidationIndexes[this._maxConsolidations - 1] + 1);
             for (const snapshot of toRemove) {
-                await this._blobStorageHelper.removeBlob(snapshot.id);
+                // We need to remove all the storage ids associated with the snapshot
+                if (core.Is.arrayValue(snapshot.changeSetStorageIds)) {
+                    for (const storageId of snapshot.changeSetStorageIds) {
+                        await this._blobStorageHelper.removeBlob(storageId);
+                    }
+                }
             }
         }
         return this._blobStorageHelper.saveBlob(syncState);
@@ -1639,12 +1682,17 @@ class RemoteSyncStateHelper {
                     storageKey: response.storageKey,
                     snapshots: []
                 };
+                // Sort the snapshots so the newest snapshot is last in the array
+                const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
+                const currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+                const currentEpoch = currentSnapshot?.epoch ?? 0;
                 const batchSnapshot = {
                     version: SYNC_SNAPSHOT_VERSION,
                     id: core.Converter.bytesToHex(core.RandomHelper.generate(32)),
                     dateCreated: now,
                     dateModified: now,
                     isConsolidated: true,
+                    epoch: currentEpoch + 1,
                     changeSetStorageIds: this._batchResponseStorageIds[response.storageKey]
                 };
                 syncState.snapshots.push(batchSnapshot);
@@ -2099,25 +2147,17 @@ class SynchronisedStorageService {
      * @internal
      */
     async startConsolidationSync(storageKey) {
-        let localChangeSnapshot;
         try {
-            // If we are performing a consolidation, we can remove the local change snapshot
-            // as we are going to create a complete changeset from the DB
-            const localChangeSnapshots = await this._localSyncStateHelper.getSnapshots(storageKey, true);
-            localChangeSnapshot = localChangeSnapshots[0];
-            if (!core.Is.empty(localChangeSnapshot)) {
-                await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
-            }
+            // If we are going to perform a consolidation first take any local updates
+            // we have and create a changeset from them, so that anybody applying
+            // just changes since a consolidation can use the changeset
+            // and skip the consolidation
+            await this.updateFromLocalSyncState(storageKey);
+            // Now start the consolidation
             await this._remoteSyncStateHelper.consolidationStart(storageKey, this._config.consolidationBatchSize ??
                 SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE);
-            // The consolidation was successful, so we can remove the local change snapshot permanently
-            localChangeSnapshot = undefined;
         }
         catch (error) {
-            if (localChangeSnapshot) {
-                // If the consolidation failed, we can keep the local change snapshot
-                await this._localSyncStateHelper.setLocalChangeSnapshot(localChangeSnapshot);
-            }
             await this._logging?.log({
                 level: "error",
                 source: this.CLASS_NAME,

package/dist/esm/index.mjs

@@ -45,6 +45,10 @@ let SyncSnapshotEntry = class SyncSnapshotEntry {
      * The flag to determine if this is a consolidated snapshot.
      */
     isConsolidated;
+    /**
+     * The epoch for the changeset.
+     */
+    epoch;
     /**
      * The ids of the storage for the change sets in the snapshot, if this is not a local snapshot.
      */
@@ -82,6 +86,10 @@ __decorate([
     property({ type: "boolean" }),
     __metadata("design:type", Boolean)
 ], SyncSnapshotEntry.prototype, "isConsolidated", void 0);
+__decorate([
+    property({ type: "number" }),
+    __metadata("design:type", Number)
+], SyncSnapshotEntry.prototype, "epoch", void 0);
 __decorate([
     property({ type: "array", itemType: "string", optional: true }),
     __metadata("design:type", Array)
@@ -865,6 +873,9 @@ class LocalSyncStateHelper {
             if (previousChangeIndex !== -1) {
                 localChangeSnapshot.changes.splice(previousChangeIndex, 1);
             }
+            // If we already have changes from previous updates
+            // then make sure we update the dateModified, otherwise
+            // we assume this is the first change and setting modified is not necessary
             if (localChangeSnapshot.changes.length > 0) {
                 localChangeSnapshot.dateModified = new Date(Date.now()).toISOString();
             }
@@ -930,7 +941,8 @@ class LocalSyncStateHelper {
                 dateModified: now,
                 changeSetStorageIds: [],
                 isLocal,
-                isConsolidated: false
+                isConsolidated: false,
+                epoch: 0
             }
         ];
     }
@@ -982,13 +994,22 @@ class LocalSyncStateHelper {
             }
         });
         // Get all the existing snapshots that we have processed previously
-        const existingRemoteSnapshots = await this.getSnapshots(storageKey, false);
+        let existingSnapshots = await this.getSnapshots(storageKey, false);
         // Sort from newest to oldest
-        const sortedSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
-        // If we have no existing snapshots we can't have yet synced
-        // in this case we need to find the most recent consolidation
-        // and use that to build a complete DB table
-        if (existingRemoteSnapshots.length === 0) {
+        existingSnapshots = existingSnapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
+        // Sort from newest to oldest
+        const syncStateSnapshots = syncState.snapshots.sort((a, b) => new Date(b.dateCreated).getTime() - new Date(a.dateCreated).getTime());
+        // Get the newest epoch from the local storage
+        const newestExistingEpoch = existingSnapshots[0]?.epoch ?? 0;
+        // Get the oldest epoch from the remote storage
+        const oldestSyncStateEpoch = syncStateSnapshots[syncStateSnapshots.length - 1]?.epoch ?? 0;
+        // If there is a gap between the largest epoch we have locally
+        // and the smallest epoch we have remotely then we have missed
+        // data so we need to perform a full sync
+        const hasEpochGap = newestExistingEpoch + 1 < oldestSyncStateEpoch;
+        // If we have an epoch gap or no existing snapshots then we need to apply
+        // a full sync from a consolidation
+        if (!existingSnapshots.some(s => s.isConsolidated) || hasEpochGap) {
             await this._logging?.log({
                 level: "info",
                 source: this.CLASS_NAME,
@@ -997,28 +1018,35 @@ class LocalSyncStateHelper {
                     storageKey
                 }
             });
-            const firstConsolidated = sortedSnapshots.find(snapshot => snapshot.isConsolidated);
-            if (firstConsolidated) {
-                // We found a consolidated snapshot, we can use it
+            const mostRecentConsolidation = syncStateSnapshots.findIndex(snapshot => snapshot.isConsolidated);
+            if (mostRecentConsolidation !== -1) {
+                // We found the most recent consolidated snapshot, we can use it
                 await this._logging?.log({
                     level: "info",
                     source: this.CLASS_NAME,
                     message: "applySnapshotFoundConsolidated",
                     data: {
                         storageKey,
-                        snapshotId: firstConsolidated.id
+                        snapshotId: syncStateSnapshots[mostRecentConsolidation].id
                     }
                 });
                 // We need to reset the entity storage and remove all the remote items
-                // so that we use just the ones from the consolidation
+                // so that we use just the ones from the consolidation, since
+                // we don't have any existing there shouldn't be any remote entries
+                // but we reset nonetheless
                 await this._changeSetHelper.reset(storageKey, SyncNodeIdentityMode.Remote);
-                await this.processNewSnapshots([
-                    {
-                        ...firstConsolidated,
-                        storageKey,
-                        isLocal: false
-                    }
-                ]);
+                // We need to process the most recent consolidation and all changes
+                // that were made since then, from newest to oldest (so newer changes override older ones)
+                // Process snapshots from the consolidation point (most recent) back to the newest
+                for (let i = mostRecentConsolidation; i >= 0; i--) {
+                    await this.processNewSnapshots([
+                        {
+                            ...syncStateSnapshots[i],
+                            storageKey,
+                            isLocal: false
+                        }
+                    ]);
+                }
             }
             else {
                 await this._logging?.log({
@@ -1032,15 +1060,20 @@ class LocalSyncStateHelper {
             }
         }
         else {
+            // We have existing consolidated remote snapshots, so we can assume that we have
+            // applied at least one consolidation snapshot, in this case we need to look at the changes since
+            // then and apply them if we haven't already
+            // We don't need to apply any additional consolidated snapshots, just the changesets
             // Create a lookup map for the existing snapshots
-            const existingSnapshots = {};
-            for (const snapshot of existingRemoteSnapshots) {
-                existingSnapshots[snapshot.id] = snapshot;
+            const existingSnapshotsMap = {};
+            for (const snapshot of existingSnapshots) {
+                existingSnapshotsMap[snapshot.id] = snapshot;
             }
             const newSnapshots = [];
             const modifiedSnapshots = [];
-            const referencedExistingSnapshots = Object.keys(existingSnapshots);
-            for (const snapshot of sortedSnapshots) {
+            const referencedExistingSnapshots = Object.keys(existingSnapshotsMap);
+            let completedProcessing = false;
+            for (const snapshot of syncStateSnapshots) {
                 await this._logging?.log({
                     level: "info",
                     source: this.CLASS_NAME,
@@ -1050,34 +1083,37 @@ class LocalSyncStateHelper {
                         dateCreated: new Date(snapshot.dateCreated).toISOString()
                     }
                 });
-                // See if we have the local snapshot
-                const currentSnapshot = existingSnapshots[snapshot.id];
+                // See if we have the snapshot stored locally
+                const currentSnapshot = existingSnapshotsMap[snapshot.id];
                 // As we are referencing an existing snapshot, we need to remove it from the list
                 // to allow us to cleanup any unreferenced snapshots later
                 const idx = referencedExistingSnapshots.indexOf(snapshot.id);
                 if (idx !== -1) {
                     referencedExistingSnapshots.splice(idx, 1);
                 }
-                const updatedSnapshot = {
-                    ...snapshot,
-                    storageKey,
-                    isLocal: false
-                };
-                if (Is.empty(currentSnapshot)) {
-                    // We don't have the snapshot locally, so we need to process it
-                    newSnapshots.push(updatedSnapshot);
-                }
-                else if (currentSnapshot.dateModified !== snapshot.dateModified) {
-                    // If the local snapshot has a different dateModified, we need to update it
-                    modifiedSnapshots.push({
-                        currentSnapshot,
-                        updatedSnapshot
-                    });
-                }
-                else {
-                    // we sorted the snapshots from newest to oldest, so if we found a local snapshot
-                    // with the same dateModified as the remote snapshot, we can stop processing further
-                    break;
+                // No need to apply consolidated snapshots
+                if (!snapshot.isConsolidated && !completedProcessing) {
+                    const updatedSnapshot = {
+                        ...snapshot,
+                        storageKey,
+                        isLocal: false
+                    };
+                    if (Is.empty(currentSnapshot)) {
+                        // We don't have the snapshot locally, so we need to process all of it
+                        newSnapshots.push(updatedSnapshot);
+                    }
+                    else if (currentSnapshot.dateModified !== snapshot.dateModified) {
+                        // If the local snapshot has a different dateModified, we need to update it
+                        modifiedSnapshots.push({
+                            currentSnapshot,
+                            updatedSnapshot
+                        });
+                    }
+                    else {
+                        // we sorted the snapshots from newest to oldest, so if we found a local snapshot
+                        // with the same dateModified as the remote snapshot, we can stop processing further
+                        completedProcessing = true;
+                    }
                 }
             }
             // We reverse the order of the snapshots to process them from oldest to newest
@@ -1402,6 +1438,7 @@ class RemoteSyncStateHelper {
         const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
         // Get the current snapshot, if it does not exist we create a new one
         let currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+        const currentEpoch = currentSnapshot?.epoch ?? 0;
         const now = new Date(Date.now()).toISOString();
         // If there is no snapshot or the current one is a consolidation
         // we start a new snapshot
@@ -1412,6 +1449,7 @@ class RemoteSyncStateHelper {
                 dateCreated: now,
                 dateModified: now,
                 isConsolidated: false,
+                epoch: currentEpoch + 1,
                 changeSetStorageIds: []
             };
             syncState.snapshots.push(currentSnapshot);
@@ -1542,7 +1580,12 @@ class RemoteSyncStateHelper {
             const toRemove = snapshots.slice(consolidationIndexes[this._maxConsolidations - 1] + 1);
             syncState.snapshots = snapshots.slice(0, consolidationIndexes[this._maxConsolidations - 1] + 1);
             for (const snapshot of toRemove) {
-                await this._blobStorageHelper.removeBlob(snapshot.id);
+                // We need to remove all the storage ids associated with the snapshot
+                if (Is.arrayValue(snapshot.changeSetStorageIds)) {
+                    for (const storageId of snapshot.changeSetStorageIds) {
+                        await this._blobStorageHelper.removeBlob(storageId);
+                    }
+                }
             }
         }
         return this._blobStorageHelper.saveBlob(syncState);
@@ -1637,12 +1680,17 @@ class RemoteSyncStateHelper {
                     storageKey: response.storageKey,
                     snapshots: []
                 };
+                // Sort the snapshots so the newest snapshot is last in the array
+                const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
+                const currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+                const currentEpoch = currentSnapshot?.epoch ?? 0;
                 const batchSnapshot = {
                     version: SYNC_SNAPSHOT_VERSION,
                     id: Converter.bytesToHex(RandomHelper.generate(32)),
                     dateCreated: now,
                     dateModified: now,
                     isConsolidated: true,
+                    epoch: currentEpoch + 1,
                     changeSetStorageIds: this._batchResponseStorageIds[response.storageKey]
                 };
                 syncState.snapshots.push(batchSnapshot);
@@ -2097,25 +2145,17 @@ class SynchronisedStorageService {
      * @internal
      */
     async startConsolidationSync(storageKey) {
-        let localChangeSnapshot;
         try {
-            // If we are performing a consolidation, we can remove the local change snapshot
-            // as we are going to create a complete changeset from the DB
-            const localChangeSnapshots = await this._localSyncStateHelper.getSnapshots(storageKey, true);
-            localChangeSnapshot = localChangeSnapshots[0];
-            if (!Is.empty(localChangeSnapshot)) {
-                await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
-            }
+            // If we are going to perform a consolidation first take any local updates
+            // we have and create a changeset from them, so that anybody applying
+            // just changes since a consolidation can use the changeset
+            // and skip the consolidation
+            await this.updateFromLocalSyncState(storageKey);
+            // Now start the consolidation
             await this._remoteSyncStateHelper.consolidationStart(storageKey, this._config.consolidationBatchSize ??
                 SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE);
-            // The consolidation was successful, so we can remove the local change snapshot permanently
-            localChangeSnapshot = undefined;
         }
         catch (error) {
-            if (localChangeSnapshot) {
-                // If the consolidation failed, we can keep the local change snapshot
-                await this._localSyncStateHelper.setLocalChangeSnapshot(localChangeSnapshot);
-            }
             await this._logging?.log({
                 level: "error",
                 source: this.CLASS_NAME,

package/dist/types/entities/syncSnapshotEntry.d.ts

@@ -31,6 +31,10 @@ export declare class SyncSnapshotEntry<T extends ISynchronisedEntity = ISynchron
      * The flag to determine if this is a consolidated snapshot.
      */
     isConsolidated: boolean;
+    /**
+     * The epoch for the changeset.
+     */
+    epoch: number;
     /**
      * The ids of the storage for the change sets in the snapshot, if this is not a local snapshot.
      */

package/dist/types/models/ISyncSnapshot.d.ts

@@ -22,6 +22,10 @@ export interface ISyncSnapshot {
      * Is this a consolidated snapshot?
      */
     isConsolidated: boolean;
+    /**
+     * The epoch of the snapshot.
+     */
+    epoch: number;
     /**
      * The ids of the storage for the change sets in the snapshot.
      */

package/docs/changelog.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## [0.0.1-next.6](https://github.com/twinfoundation/synchronised-storage/compare/synchronised-storage-service-v0.0.1-next.5...synchronised-storage-service-v0.0.1-next.6) (2025-08-11)
+
+
+### Features
+
+* improve consolidation logic ([698232f](https://github.com/twinfoundation/synchronised-storage/commit/698232f57640f87642ecd323cb1e4670eda33343))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @twin.org/synchronised-storage-models bumped from 0.0.1-next.5 to 0.0.1-next.6
+
 ## [0.0.1-next.5](https://github.com/twinfoundation/synchronised-storage/compare/synchronised-storage-service-v0.0.1-next.4...synchronised-storage-service-v0.0.1-next.5) (2025-08-11)
 
 

package/docs/reference/classes/SyncSnapshotEntry.md

@@ -76,6 +76,14 @@ The flag to determine if this is a consolidated snapshot.
 
 ***
 
+### epoch
+
+> **epoch**: `number`
+
+The epoch for the changeset.
+
+***
+
 ### changeSetStorageIds?
 
 > `optional` **changeSetStorageIds**: `string`[]

package/docs/reference/interfaces/ISyncSnapshot.md

@@ -44,6 +44,14 @@ Is this a consolidated snapshot?
 
 ***
 
+### epoch
+
+> **epoch**: `number`
+
+The epoch of the snapshot.
+
+***
+
 ### changeSetStorageIds
 
 > **changeSetStorageIds**: `string`[]

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@twin.org/synchronised-storage-service",
-	"version": "0.0.1-next.5",
+	"version": "0.0.1-next.6",
 	"description": "Synchronised storage contract implementation and REST endpoint definitions",
 	"repository": {
 		"type": "git",
@@ -26,7 +26,7 @@
 		"@twin.org/logging-models": "next",
 		"@twin.org/nameof": "next",
 		"@twin.org/standards-w3c-did": "next",
-		"@twin.org/synchronised-storage-models": "0.0.1-next.5",
+		"@twin.org/synchronised-storage-models": "0.0.1-next.6",
 		"@twin.org/verifiable-storage-models": "next",
 		"@twin.org/web": "next"
 	},