@twin.org/synchronised-storage-service 0.0.1-next.2 → 0.0.1-next.3

package/dist/esm/index.mjs

@@ -20,9 +20,9 @@ let SyncSnapshotEntry = class SyncSnapshotEntry {
      */
     id;
     /**
-     * The schema type for the snapshot i.e. which entity is being synchronized.
+     * The storage key for the snapshot i.e. which entity is being synchronized.
      */
-    schemaType;
+    storageKey;
     /**
      * The date the snapshot was created.
      */
@@ -42,7 +42,7 @@ let SyncSnapshotEntry = class SyncSnapshotEntry {
     /**
      * The changes that were made in this snapshot, if this is a local snapshot.
      */
-    localChanges;
+    changes;
 };
 __decorate([
     property({ type: "string", isPrimary: true }),
@@ -51,7 +51,7 @@ __decorate([
 __decorate([
     property({ type: "string", isSecondary: true }),
     __metadata("design:type", String)
-], SyncSnapshotEntry.prototype, "schemaType", void 0);
+], SyncSnapshotEntry.prototype, "storageKey", void 0);
 __decorate([
     property({ type: "string" }),
     __metadata("design:type", String)
@@ -71,7 +71,7 @@ __decorate([
 __decorate([
     property({ type: "array", itemType: "object", optional: true }),
     __metadata("design:type", Array)
-], SyncSnapshotEntry.prototype, "localChanges", void 0);
+], SyncSnapshotEntry.prototype, "changes", void 0);
 SyncSnapshotEntry = __decorate([
     entity()
 ], SyncSnapshotEntry);
@@ -229,15 +229,14 @@ class ChangeSetHelper {
     /**
      * Apply a sync changeset.
      * @param changeSetStorageId The id of the sync changeset to apply.
-     * @returns True if the change was applied.
+     * @returns The changeset if it existed.
      */
     async getAndApplyChangeset(changeSetStorageId) {
         const syncChangeset = await this.getAndVerifyChangeset(changeSetStorageId);
         if (!Is.empty(syncChangeset)) {
             await this.applyChangeset(syncChangeset);
-            return true;
         }
-        return false;
+        return syncChangeset;
     }
     /**
      * Apply a sync changeset.
@@ -264,7 +263,7 @@ class ChangeSetHelper {
                             // so we need to restore it here.
                             change.entity.nodeIdentity = syncChangeset.nodeIdentity;
                             await this._eventBusComponent.publish(SynchronisedStorageTopics.RemoteItemSet, {
-                                schemaType: syncChangeset.schemaType,
+                                storageKey: syncChangeset.storageKey,
                                 entity: change.entity
                             });
                         }
@@ -272,7 +271,7 @@ class ChangeSetHelper {
                     case SyncChangeOperation.Delete:
                         if (!Is.empty(change.id)) {
                             await this._eventBusComponent.publish(SynchronisedStorageTopics.RemoteItemRemove, {
-                                schemaType: syncChangeset.schemaType,
+                                storageKey: syncChangeset.storageKey,
                                 id: change.id
                             });
                         }
@@ -284,9 +283,10 @@ class ChangeSetHelper {
     /**
      * Store the changeset.
      * @param syncChangeSet The sync change set to store.
+     * @param nodeIdentity The node identity to use for the changeset.
      * @returns The id of the change set.
      */
-    async storeChangeSet(syncChangeSet) {
+    async storeChangeSet(syncChangeSet, nodeIdentity) {
         await this._logging?.log({
             level: "info",
             source: this.CLASS_NAME,
@@ -300,7 +300,7 @@ class ChangeSetHelper {
         return this._blobStorageComponent.create(Converter.bytesToBase64(ObjectHelper.toBytes(syncChangeSet)), undefined, undefined, undefined, {
             disableEncryption: true,
             compress: BlobStorageCompressionType.Gzip
-        });
+        }, undefined, nodeIdentity);
     }
     /**
      * Verify the proof of a sync changeset.
@@ -354,7 +354,7 @@ class ChangeSetHelper {
         delete changeSetWithoutProof.proof;
         const proof = await this._identityConnector.createProof(syncChangeset.nodeIdentity, DocumentHelper.joinId(syncChangeset.nodeIdentity, this._decentralisedStorageMethodId), ProofTypes.DataIntegrityProof, changeSetWithoutProof);
         await this._logging?.log({
-            level: "error",
+            level: "info",
             source: this.CLASS_NAME,
             message: "createdChangeSetProof",
             data: {
@@ -404,33 +404,51 @@ class LocalSyncStateHelper {
     }
     /**
      * Add a new change to the local snapshot.
-     * @param schemaType The schema type of the snapshot to add the change for.
+     * @param storageKey The storage key of the snapshot to add the change for.
      * @param operation The operation to perform.
      * @param id The id of the entity to add the change for.
      * @returns Nothing.
      */
-    async addLocalChange(schemaType, operation, id) {
-        const localChangeSnapshot = await this.getLocalChangeSnapshot(schemaType);
-        localChangeSnapshot.localChanges ??= [];
+    async addLocalChange(storageKey, operation, id) {
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "addLocalChange",
+            data: {
+                storageKey,
+                operation,
+                id
+            }
+        });
+        const localChangeSnapshot = await this.getLocalChangeSnapshot(storageKey);
+        localChangeSnapshot.changes ??= [];
         // If we already have a change for this id we are
         // about to supersede it, we remove the previous change
         // to avoid having multiple changes for the same id
-        const previousChangeIndex = localChangeSnapshot.localChanges.findIndex(change => change.id === id);
+        const previousChangeIndex = localChangeSnapshot.changes.findIndex(change => change.id === id);
         if (previousChangeIndex !== -1) {
-            localChangeSnapshot.localChanges.splice(previousChangeIndex, 1);
+            localChangeSnapshot.changes.splice(previousChangeIndex, 1);
         }
-        if (localChangeSnapshot.localChanges.length > 0) {
+        if (localChangeSnapshot.changes.length > 0) {
             localChangeSnapshot.dateModified = new Date(Date.now()).toISOString();
         }
-        localChangeSnapshot.localChanges.push({ operation, id });
+        localChangeSnapshot.changes.push({ operation, id });
         await this.setLocalChangeSnapshot(localChangeSnapshot);
     }
     /**
      * Get the current local snapshot.
-     * @param schemaType The schema type of the snapshot to get.
+     * @param storageKey The storage key of the snapshot to get.
      * @returns The local snapshot entry.
      */
-    async getLocalChangeSnapshot(schemaType) {
+    async getLocalChangeSnapshot(storageKey) {
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "getLocalChangeSnapshot",
+            data: {
+                storageKey
+            }
+        });
         const queryResult = await this._localSyncSnapshotEntryEntityStorage.query({
             conditions: [
                 {
@@ -439,18 +457,34 @@ class LocalSyncStateHelper {
                     comparison: ComparisonOperator.Equals
                 },
                 {
-                    property: "schemaType",
-                    value: schemaType,
+                    property: "storageKey",
+                    value: storageKey,
                     comparison: ComparisonOperator.Equals
                 }
             ]
         });
         if (queryResult.entities.length > 0) {
+            await this._logging?.log({
+                level: "info",
+                source: this.CLASS_NAME,
+                message: "localChangeSnapshotExists",
+                data: {
+                    storageKey
+                }
+            });
             return queryResult.entities[0];
         }
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "localChangeSnapshotDoesNotExist",
+            data: {
+                storageKey
+            }
+        });
         return {
             id: Converter.bytesToHex(RandomHelper.generate(32)),
-            schemaType,
+            storageKey,
             dateCreated: new Date(Date.now()).toISOString(),
             changeSetStorageIds: [],
             isLocalSnapshot: true
@@ -462,6 +496,14 @@ class LocalSyncStateHelper {
      * @returns Nothing.
      */
     async setLocalChangeSnapshot(localChangeSnapshot) {
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "setLocalChangeSnapshot",
+            data: {
+                storageKey: localChangeSnapshot.storageKey
+            }
+        });
         await this._localSyncSnapshotEntryEntityStorage.set(localChangeSnapshot);
     }
     /**
@@ -482,11 +524,11 @@ class LocalSyncStateHelper {
     }
     /**
      * Sync local data using a remote sync state.
-     * @param schemaType The schema type of the snapshot to sync with.
+     * @param storageKey The storage key of the snapshot to sync with.
      * @param remoteSyncState The sync state to sync with.
      * @returns Nothing.
      */
-    async syncFromRemote(schemaType, remoteSyncState) {
+    async syncFromRemote(storageKey, remoteSyncState) {
         await this._logging?.log({
             level: "info",
             source: this.CLASS_NAME,
@@ -512,7 +554,7 @@ class LocalSyncStateHelper {
             const localSnapshot = await this._localSyncSnapshotEntryEntityStorage.get(remoteSnapshot.id);
             const remoteSnapshotWithContext = {
                 ...remoteSnapshot,
-                schemaType
+                storageKey
             };
             if (Is.empty(localSnapshot)) {
                 // We don't have the snapshot locally, so we need to process it
@@ -631,12 +673,12 @@ class RemoteSyncStateHelper {
      */
     _changeSetHelper;
     /**
-     * The storage ids of the batch responses for each schema type.
+     * The storage ids of the batch responses for each storage key.
      * @internal
      */
     _batchResponseStorageIds;
     /**
-     * The full changes for each schema type.
+     * The full changes for each storage key.
      * @internal
      */
     _populateFullChanges;
@@ -684,52 +726,73 @@ class RemoteSyncStateHelper {
     }
     /**
      * Create and store a change set.
-     * @param schemaType The schema type of the change set.
+     * @param storageKey The storage key of the change set.
      * @param changes The changes to apply.
      * @param completeCallback The callback to call when the changeset is created and stored.
      * @returns The storage id of the change set if created.
      */
-    async createAndStoreChangeSet(schemaType, changes, completeCallback) {
-        if (Is.arrayValue(changes)) {
-            this._populateFullChanges[schemaType] = {
-                changes,
-                entities: {},
-                requestIds: [],
-                completeCallback: async () => this.finaliseFullChanges(schemaType, completeCallback)
-            };
-            const setChanges = changes.filter(c => c.operation === SyncChangeOperation.Set);
-            if (setChanges.length === 0) {
-                // If we don't need to request any full details, we can just call the complete callback
-                await this.finaliseFullChanges(schemaType, completeCallback);
-            }
-            else {
-                // Otherwise we need to request the full details for each change
-                this._populateFullChanges[schemaType].requestIds = setChanges.map(change => change.id);
-                // Once all the requests are handled the callback will be called
-                for (const change of setChanges) {
-                    // Create a request for each change to populate the full details
-                    this._eventBusComponent.publish(SynchronisedStorageTopics.LocalItemRequest, {
-                        schemaType,
-                        id: change.id
-                    });
-                }
+    async createAndStoreChangeSet(storageKey, changes, completeCallback) {
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "createAndStoreChangeSet",
+            data: {
+                storageKey,
+                changeCount: changes.length
             }
+        });
+        this._populateFullChanges[storageKey] = {
+            changes,
+            entities: {},
+            requestIds: [],
+            completeCallback: async () => this.finaliseFullChanges(storageKey, completeCallback)
+        };
+        const setChanges = changes.filter(c => c.operation === SyncChangeOperation.Set);
+        if (setChanges.length === 0) {
+            // If we don't need to request any full details, we can just call the complete callback
+            await this.finaliseFullChanges(storageKey, completeCallback);
         }
         else {
-            await completeCallback();
+            // Otherwise we need to request the full details for each change
+            this._populateFullChanges[storageKey].requestIds = setChanges.map(change => change.id);
+            // Once all the requests are handled the callback will be called
+            for (const change of setChanges) {
+                // Create a request for each change to populate the full details
+                await this._logging?.log({
+                    level: "info",
+                    source: this.CLASS_NAME,
+                    message: "createChangeSetRequestingItem",
+                    data: {
+                        storageKey,
+                        id: change.id
+                    }
+                });
+                this._eventBusComponent.publish(SynchronisedStorageTopics.LocalItemRequest, {
+                    storageKey,
+                    id: change.id
+                });
+            }
         }
     }
     /**
      * Finalise the full details for the sync change set.
-     * @param schemaType The schema type of the change set.
+     * @param storageKey The storage key of the change set.
      * @param completeCallback The callback to call when the changeset is populated.
      * @returns Nothing.
      */
-    async finaliseFullChanges(schemaType, completeCallback) {
+    async finaliseFullChanges(storageKey, completeCallback) {
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "finalisingSyncChanges",
+            data: {
+                storageKey
+            }
+        });
         if (Is.stringValue(this._nodeIdentity)) {
-            const changes = this._populateFullChanges[schemaType].changes;
+            const changes = this._populateFullChanges[storageKey].changes;
             for (const change of changes) {
-                change.entity = this._populateFullChanges[schemaType].entities[change.id] ?? change.entity;
+                change.entity = this._populateFullChanges[storageKey].entities[change.id] ?? change.entity;
                 if (change.operation === SyncChangeOperation.Set && Is.objectValue(change.entity)) {
                     // Remove the node identity as the changeset has this stored at the top level
                     // and we do not want to store it in the change itself to reduce redundancy
@@ -740,15 +803,29 @@ class RemoteSyncStateHelper {
             const syncChangeSet = {
                 id: Converter.bytesToHex(RandomHelper.generate(32)),
                 dateCreated: new Date(Date.now()).toISOString(),
-                schemaType,
+                storageKey,
                 changes,
                 nodeIdentity: this._nodeIdentity
             };
-            // And sign it with the node identity
-            syncChangeSet.proof = await this._changeSetHelper.createChangeSetProof(syncChangeSet);
-            // Store the changeset in the blob storage
-            const changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
-            await completeCallback(changeSetStorageId);
+            try {
+                // And sign it with the node identity
+                syncChangeSet.proof = await this._changeSetHelper.createChangeSetProof(syncChangeSet);
+                // Store the changeset in the blob storage
+                const changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet, this._nodeIdentity);
+                await completeCallback(changeSetStorageId);
+            }
+            catch (err) {
+                await this._logging?.log({
+                    level: "error",
+                    source: this.CLASS_NAME,
+                    message: "finalisingSyncChangesFailed",
+                    data: {
+                        storageKey
+                    },
+                    error: BaseError.fromError(err)
+                });
+                await completeCallback();
+            }
         }
         else {
             await completeCallback();
@@ -756,15 +833,25 @@ class RemoteSyncStateHelper {
     }
     /**
      * Add a new changeset into the sync state.
+     * @param storageKey The storage key of the change set to add.
      * @param changeSetStorageId The id of the change set to add the the current state
      * @returns Nothing.
      */
-    async addChangeSetToSyncState(changeSetStorageId) {
-        // First load the current sync state if there is one
-        const syncStatePointer = await this.getVerifiableSyncPointer();
+    async addChangeSetToSyncState(storageKey, changeSetStorageId) {
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "addChangeSetToSyncState",
+            data: {
+                storageKey,
+                changeSetStorageId
+            }
+        });
+        // First load the sync pointer store to get the current sync pointer for the storage key
+        const syncPointerStore = await this.getVerifiableSyncPointerStore();
         let syncState;
-        if (!Is.empty(syncStatePointer?.syncPointerId)) {
-            syncState = await this.getRemoteSyncState(syncStatePointer.syncPointerId);
+        if (!Is.empty(syncPointerStore.syncPointers[storageKey])) {
+            syncState = await this.getRemoteSyncState(syncPointerStore.syncPointers[storageKey]);
         }
         // No current sync state, so we create a new one
         if (Is.empty(syncState)) {
@@ -789,34 +876,34 @@ class RemoteSyncStateHelper {
         // Add the changeset storage id to the current snapshot
         currentSnapshot.changeSetStorageIds.push(changeSetStorageId);
         // Store the sync state in the blob storage
-        const syncStateId = await this.storeRemoteSyncState(syncState);
-        // Store the verifiable sync pointer in the verifiable storage
-        await this.storeVerifiableSyncPointer(syncStateId);
+        syncPointerStore.syncPointers[storageKey] = await this.storeRemoteSyncState(syncState);
+        // Store the verifiable sync pointer store in the verifiable storage
+        await this.storeVerifiableSyncPointerStore(syncPointerStore);
     }
     /**
      * Create a consolidated snapshot for the entire storage.
-     * @param schemaType The schema type of the snapshot to create.
+     * @param storageKey The storage key of the snapshot to create.
      * @param batchSize The batch size to use for consolidation.
      * @returns Nothing.
      */
-    async consolidateFromLocal(schemaType, batchSize) {
+    async consolidateFromLocal(storageKey, batchSize) {
         await this._logging?.log({
             level: "info",
             source: this.CLASS_NAME,
             message: "consolidationStarting"
         });
-        await this._eventBusComponent.publish(SynchronisedStorageTopics.BatchRequest, { schemaType, batchSize });
+        await this._eventBusComponent.publish(SynchronisedStorageTopics.BatchRequest, { storageKey, batchSize });
     }
     /**
-     * Get the sync pointer.
-     * @returns The sync pointer.
+     * Get the sync pointer store.
+     * @returns The sync pointer store.
      */
-    async getVerifiableSyncPointer() {
+    async getVerifiableSyncPointerStore() {
         try {
             await this._logging?.log({
                 level: "info",
                 source: this.CLASS_NAME,
-                message: "verifiableSyncPointerRetrieving",
+                message: "verifiableSyncPointerStoreRetrieving",
                 data: {
                     key: this._synchronisedStorageKey
                 }
@@ -827,10 +914,9 @@ class RemoteSyncStateHelper {
                 await this._logging?.log({
                     level: "info",
                     source: this.CLASS_NAME,
-                    message: "verifiableSyncPointerRetrieved",
+                    message: "verifiableSyncPointerStoreRetrieved",
                     data: {
-                        key: this._synchronisedStorageKey,
-                        syncPointerId: syncPointer.syncPointerId
+                        key: this._synchronisedStorageKey
                     }
                 });
                 return syncPointer;
@@ -844,34 +930,34 @@ class RemoteSyncStateHelper {
         await this._logging?.log({
             level: "info",
             source: this.CLASS_NAME,
-            message: "verifiableSyncPointerNotFound",
+            message: "verifiableSyncPointerStoreNotFound",
             data: {
                 key: this._synchronisedStorageKey
             }
         });
+        // If no sync pointer store exists, we return an empty one
+        return {
+            syncPointers: {}
+        };
     }
     /**
      * Store the verifiable sync pointer in the verifiable storage.
-     * @param syncStateId The id of the sync state to store.
+     * @param syncPointerStore The sync pointer store to store.
      * @returns Nothing.
      */
-    async storeVerifiableSyncPointer(syncStateId) {
-        // Create a new verifiable sync pointer object pointing to the sync state
-        const verifiableSyncPointer = {
-            syncPointerId: syncStateId
-        };
-        await this._logging?.log({
-            level: "info",
-            source: this.CLASS_NAME,
-            message: "verifiableSyncPointerStoring",
-            data: {
-                key: this._synchronisedStorageKey,
-                syncPointerId: verifiableSyncPointer.syncPointerId
-            }
-        });
-        // Store the verifiable sync pointer in the verifiable storage
-        await this._verifiableSyncPointerStorageConnector.create(this._synchronisedStorageKey, ObjectHelper.toBytes(verifiableSyncPointer));
-        return verifiableSyncPointer;
+    async storeVerifiableSyncPointerStore(syncPointerStore) {
+        if (this._nodeIdentity) {
+            await this._logging?.log({
+                level: "info",
+                source: this.CLASS_NAME,
+                message: "verifiableSyncPointerStoreStoring",
+                data: {
+                    key: this._synchronisedStorageKey
+                }
+            });
+            // Store the verifiable sync pointer in the verifiable storage
+            await this._verifiableSyncPointerStorageConnector.update(this._nodeIdentity, this._synchronisedStorageKey, ObjectHelper.toBytes(syncPointerStore));
+        }
     }
     /**
      * Store the remote sync state.
@@ -889,7 +975,7 @@ class RemoteSyncStateHelper {
         });
         // We don't want to encrypt the sync state as no other nodes would be able to read it
         // the blob storage also needs to be publicly accessible so that other nodes can retrieve it
-        return this._blobStorageComponent.create(Converter.bytesToBase64(ObjectHelper.toBytes(syncState)), undefined, undefined, undefined, { disableEncryption: true, compress: BlobStorageCompressionType.Gzip });
+        return this._blobStorageComponent.create(Converter.bytesToBase64(ObjectHelper.toBytes(syncState)), undefined, undefined, undefined, { disableEncryption: true, compress: BlobStorageCompressionType.Gzip }, undefined, this._nodeIdentity);
     }
     /**
      * Get the remote sync state.
@@ -908,7 +994,7 @@ class RemoteSyncStateHelper {
             });
             const blobEntry = await this._blobStorageComponent.get(syncPointerId, {
                 includeContent: true
-            });
+            }, undefined, this._nodeIdentity);
             if (Is.stringBase64(blobEntry.blob)) {
                 const syncState = ObjectHelper.fromBytes(Converter.base64ToBytes(blobEntry.blob));
                 await this._logging?.log({
@@ -951,28 +1037,31 @@ class RemoteSyncStateHelper {
                     operation: SyncChangeOperation.Set,
                     id: change[response.primaryKey]
                 })),
-                schemaType: response.schemaType,
+                storageKey: response.storageKey,
                 nodeIdentity: this._nodeIdentity
             };
             // And sign it with the node identity
             syncChangeSet.proof = await this._changeSetHelper.createChangeSetProof(syncChangeSet);
             // Store the changeset in the blob storage
-            const changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
+            const changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet, this._nodeIdentity);
             // Add the changeset storage id to the snapshot ids
-            this._batchResponseStorageIds[response.schemaType] ??= [];
-            this._batchResponseStorageIds[response.schemaType].push(changeSetStorageId);
+            this._batchResponseStorageIds[response.storageKey] ??= [];
+            this._batchResponseStorageIds[response.storageKey].push(changeSetStorageId);
             if (response.lastEntry) {
                 const syncState = { snapshots: [] };
                 const batchSnapshot = {
                     id: Converter.bytesToHex(RandomHelper.generate(32)),
                     dateCreated: new Date(Date.now()).toISOString(),
-                    changeSetStorageIds: this._batchResponseStorageIds[response.schemaType]
+                    changeSetStorageIds: this._batchResponseStorageIds[response.storageKey]
                 };
                 syncState.snapshots.push(batchSnapshot);
                 // Store the sync state in the blob storage
                 const syncStateId = await this.storeRemoteSyncState(syncState);
+                // Get the current sync pointer store
+                const syncPointerStore = await this.getVerifiableSyncPointerStore();
+                syncPointerStore.syncPointers[response.storageKey] = syncStateId;
                 // Store the verifiable sync pointer in the verifiable storage
-                await this.storeVerifiableSyncPointer(syncStateId);
+                await this.storeVerifiableSyncPointerStore(syncPointerStore);
                 await this._logging?.log({
                     level: "info",
                     source: this.CLASS_NAME,
@@ -986,13 +1075,22 @@ class RemoteSyncStateHelper {
      * @param response The item response to handle.
      */
     async handleLocalItemResponse(response) {
-        if (!Is.empty(this._populateFullChanges[response.schemaType])) {
-            const idx = this._populateFullChanges[response.schemaType].requestIds.indexOf(response.id);
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "createChangeSetRespondingItem",
+            data: {
+                storageKey: response.storageKey,
+                id: response.id
+            }
+        });
+        if (!Is.empty(this._populateFullChanges[response.storageKey])) {
+            const idx = this._populateFullChanges[response.storageKey].requestIds.indexOf(response.id);
             if (idx !== -1) {
-                this._populateFullChanges[response.schemaType].requestIds.splice(idx, 1);
-                this._populateFullChanges[response.schemaType].entities[response.id] = response.entity;
-                if (this._populateFullChanges[response.schemaType].requestIds.length === 0) {
-                    await this._populateFullChanges[response.schemaType].completeCallback();
+                this._populateFullChanges[response.storageKey].requestIds.splice(idx, 1);
+                this._populateFullChanges[response.storageKey].entities[response.id] = response.entity;
+                if (this._populateFullChanges[response.storageKey].requestIds.length === 0) {
+                    await this._populateFullChanges[response.storageKey].completeCallback();
                 }
             }
         }
@@ -1004,20 +1102,20 @@ class RemoteSyncStateHelper {
  */
 class SynchronisedStorageService {
     /**
-     * The default interval to check for entity updates, defaults to 5 mins.
+     * The default interval to check for entity updates.
      * @internal
      */
-    static _DEFAULT_ENTITY_UPDATE_INTERVAL_MS = 300000;
+    static _DEFAULT_ENTITY_UPDATE_INTERVAL_MINUTES = 5;
     /**
-     * The default interval to perform consolidation, defaults to 60 mins.
+     * The default interval to perform consolidation.
      * @internal
      */
-    static _DEFAULT_CONSOLIDATION_INTERVAL_MS = 3600000;
+    static _DEFAULT_CONSOLIDATION_INTERVAL_MINUTES = 60;
     /**
      * The default size of a consolidation batch.
      * @internal
      */
-    static _DEFAULT_CONSOLIDATION_BATCH_SIZE = 1000;
+    static _DEFAULT_CONSOLIDATION_BATCH_SIZE = 100;
     /**
      * Runtime name for the class.
      */
@@ -1052,6 +1150,11 @@ class SynchronisedStorageService {
      * @internal
      */
     _identityConnector;
+    /**
+     * The task scheduler component.
+     * @internal
+     */
+    _taskSchedulerComponent;
     /**
      * The synchronised storage service to use when this is not a trusted node.
      * @internal
@@ -1078,15 +1181,15 @@ class SynchronisedStorageService {
      */
     _config;
     /**
-     * The timer ids for checking for entity updates.
+     * The flag to determine if the service has been started.
      * @internal
      */
-    _entityUpdateTimers;
+    _serviceStarted;
     /**
-     * The timer ids for consolidation.
+     * The active storage keys for the synchronised storage service.
      * @internal
      */
-    _consolidationTimers;
+    _activeStorageKeys;
     /**
      * The identity of the node this connector is running on.
      * @internal
@@ -1105,14 +1208,15 @@ class SynchronisedStorageService {
         this._verifiableSyncPointerStorageConnector = VerifiableStorageConnectorFactory.get(options.verifiableStorageConnectorType ?? "verifiable-storage");
         this._blobStorageComponent = ComponentFactory.get(options.blobStorageComponentType ?? "blob-storage");
         this._identityConnector = IdentityConnectorFactory.get(options.identityConnectorType ?? "identity");
+        this._taskSchedulerComponent = ComponentFactory.get(options.taskSchedulerComponentType ?? "task-scheduler");
         this._config = {
             synchronisedStorageKey: options.config.synchronisedStorageKey,
             synchronisedStorageMethodId: options.config.synchronisedStorageMethodId ?? "synchronised-storage-assertion",
-            entityUpdateIntervalMs: options.config.entityUpdateIntervalMs ??
-                SynchronisedStorageService._DEFAULT_ENTITY_UPDATE_INTERVAL_MS,
+            entityUpdateIntervalMinutes: options.config.entityUpdateIntervalMinutes ??
+                SynchronisedStorageService._DEFAULT_ENTITY_UPDATE_INTERVAL_MINUTES,
             isTrustedNode: options.config.isTrustedNode ?? false,
-            consolidationIntervalMs: options.config.consolidationIntervalMs ??
-                SynchronisedStorageService._DEFAULT_CONSOLIDATION_INTERVAL_MS,
+            consolidationIntervalMinutes: options.config.consolidationIntervalMinutes ??
+                SynchronisedStorageService._DEFAULT_CONSOLIDATION_INTERVAL_MINUTES,
             consolidationBatchSize: options.config.consolidationBatchSize ??
                 SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE
         };
@@ -1126,10 +1230,10 @@ class SynchronisedStorageService {
         this._changeSetHelper = new ChangeSetHelper(this._logging, this._eventBusComponent, this._blobStorageComponent, this._identityConnector, this._config.synchronisedStorageMethodId);
         this._localSyncStateHelper = new LocalSyncStateHelper(this._logging, this._localSyncSnapshotEntryEntityStorage, this._changeSetHelper);
         this._remoteSyncStateHelper = new RemoteSyncStateHelper(this._logging, this._eventBusComponent, this._blobStorageComponent, this._verifiableSyncPointerStorageConnector, this._changeSetHelper, this._config.synchronisedStorageKey);
-        this._consolidationTimers = {};
-        this._entityUpdateTimers = {};
-        this._eventBusComponent.subscribe(SynchronisedStorageTopics.RegisterSchemaType, async (event) => this.registerType(event.data));
-        this._eventBusComponent.subscribe(SynchronisedStorageTopics.LocalItemChange, async (event) => this._localSyncStateHelper.addLocalChange(event.data.schemaType, event.data.operation, event.data.id));
+        this._serviceStarted = false;
+        this._activeStorageKeys = {};
+        this._eventBusComponent.subscribe(SynchronisedStorageTopics.RegisterStorageKey, async (event) => this.registerType(event.data));
+        this._eventBusComponent.subscribe(SynchronisedStorageTopics.LocalItemChange, async (event) => this._localSyncStateHelper.addLocalChange(event.data.storageKey, event.data.operation, event.data.id));
     }
     /**
      * The component needs to be started when the node is initialized.
@@ -1141,6 +1245,11 @@ class SynchronisedStorageService {
     async start(nodeIdentity, nodeLoggingConnectorType, componentState) {
         this._nodeIdentity = nodeIdentity;
         this._remoteSyncStateHelper.setNodeIdentity(nodeIdentity);
+        this._serviceStarted = true;
+        // If there are already storage keys registered, we need to activate them
+        for (const storageKey in this._activeStorageKeys) {
+            await this.activateStorageKey(storageKey);
+        }
     }
     /**
      * The component needs to be stopped when the node is closed.
@@ -1150,13 +1259,10 @@ class SynchronisedStorageService {
      * @returns Nothing.
      */
     async stop(nodeIdentity, nodeLoggingConnectorType, componentState) {
-        for (const schemaType in this._entityUpdateTimers) {
-            clearTimeout(this._entityUpdateTimers[schemaType]);
-            delete this._entityUpdateTimers[schemaType];
-        }
-        for (const schemaType in this._consolidationTimers) {
-            clearTimeout(this._consolidationTimers[schemaType]);
-            delete this._consolidationTimers[schemaType];
+        for (const storageKey in this._activeStorageKeys) {
+            this._activeStorageKeys[storageKey] = false;
+            this._taskSchedulerComponent.removeTask(`synchronised-storage-update-${storageKey}`);
+            this._taskSchedulerComponent.removeTask(`synchronised-storage-consolidation-${storageKey}`);
         }
     }
     /**
@@ -1177,21 +1283,29 @@ class SynchronisedStorageService {
         // This will be performed using rights-management
         const changeSet = await this._changeSetHelper.getAndApplyChangeset(changeSetStorageId);
         if (!Is.empty(changeSet)) {
-            await this._remoteSyncStateHelper.addChangeSetToSyncState(changeSetStorageId);
+            await this._remoteSyncStateHelper.addChangeSetToSyncState(changeSet.storageKey, changeSetStorageId);
         }
     }
     /**
      * Start the sync with further updates after an interval.
-     * @param schemaType The schema type to sync.
+     * @param storageKey The storage key to sync.
      * @returns Nothing.
      * @internal
      */
-    async startEntitySync(schemaType) {
+    async startEntitySync(storageKey) {
         try {
+            await this._logging?.log({
+                level: "info",
+                source: this.CLASS_NAME,
+                message: "startEntitySync",
+                data: {
+                    storageKey
+                }
+            });
             // First we check for remote changes
-            await this.updateFromRemoteSyncState(schemaType);
+            await this.updateFromRemoteSyncState(storageKey);
             // Now send any updates we have to the remote storage
-            await this.updateFromLocalSyncState(schemaType);
+            await this.updateFromLocalSyncState(storageKey);
         }
         catch (error) {
             await this._logging?.log({
@@ -1201,27 +1315,31 @@ class SynchronisedStorageService {
                 error: BaseError.fromError(error)
             });
         }
-        finally {
-            // Set a timer to check for updates again
-            this._entityUpdateTimers[schemaType] = setTimeout(async () => this.startEntitySync(schemaType), this._config.entityUpdateIntervalMs);
-        }
     }
     /**
      * Check for updates in the remote storage.
-     * @param schemaType The schema type to check for updates.
+     * @param storageKey The storage key to check for updates.
      * @returns Nothing.
      * @internal
      */
-    async updateFromRemoteSyncState(schemaType) {
-        // Get the verifiable sync pointer from the verifiable storage
-        const verifiableSyncPointer = await this._remoteSyncStateHelper.getVerifiableSyncPointer();
-        if (!Is.empty(verifiableSyncPointer)) {
+    async updateFromRemoteSyncState(storageKey) {
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "updateFromRemoteSyncState",
+            data: {
+                storageKey
+            }
+        });
+        // Get the verifiable sync pointer store from the verifiable storage
+        const verifiableSyncPointerStore = await this._remoteSyncStateHelper.getVerifiableSyncPointerStore();
+        if (!Is.empty(verifiableSyncPointerStore.syncPointers[storageKey])) {
             // Load the sync state from the remote blob storage using the sync pointer
             // to load the sync state
-            const remoteSyncState = await this._remoteSyncStateHelper.getRemoteSyncState(verifiableSyncPointer.syncPointerId);
+            const remoteSyncState = await this._remoteSyncStateHelper.getRemoteSyncState(verifiableSyncPointerStore.syncPointers[storageKey]);
             // If we got the sync state we can try and sync from it
             if (!Is.undefined(remoteSyncState)) {
-                await this._localSyncStateHelper.syncFromRemote(schemaType, remoteSyncState);
+                await this._localSyncStateHelper.syncFromRemote(storageKey, remoteSyncState);
             }
         }
     }
@@ -1230,43 +1348,78 @@ class SynchronisedStorageService {
      * @returns Nothing.
      * @internal
      */
-    async updateFromLocalSyncState(schemaType) {
-        if (Is.stringValue(this._nodeIdentity)) {
-            // Ge the current local change snapshot
-            const localChangeSnapshot = await this._localSyncStateHelper.getLocalChangeSnapshot(schemaType);
-            if (!Is.empty(localChangeSnapshot)) {
-                await this._remoteSyncStateHelper.createAndStoreChangeSet(schemaType, localChangeSnapshot.localChanges, async (changeSetStorageId) => {
-                    if (Is.stringValue(changeSetStorageId)) {
-                        // Send the local changes to the remote storage if we are a trusted node
-                        if (this._config.isTrustedNode) {
-                            await this._remoteSyncStateHelper.addChangeSetToSyncState(changeSetStorageId);
-                        }
-                        else if (!Is.empty(this._trustedSynchronisedStorageComponent)) {
-                            // If we are not a trusted node, we need to send the changes to the trusted node
-                            await this._trustedSynchronisedStorageComponent.syncChangeSet(changeSetStorageId);
+    async updateFromLocalSyncState(storageKey) {
+        await this._logging?.log({
+            level: "info",
+            source: this.CLASS_NAME,
+            message: "updateFromLocalSyncState",
+            data: {
+                storageKey
+            }
+        });
+        const localChangeSnapshot = await this._localSyncStateHelper.getLocalChangeSnapshot(storageKey);
+        if (Is.arrayValue(localChangeSnapshot.changes)) {
+            await this._remoteSyncStateHelper.createAndStoreChangeSet(storageKey, localChangeSnapshot.changes, async (changeSetStorageId) => {
+                if (Is.stringValue(changeSetStorageId)) {
+                    await this._logging?.log({
+                        level: "info",
+                        source: this.CLASS_NAME,
+                        message: "createdStorageChangeSet",
+                        data: {
+                            storageKey,
+                            changeSetStorageId
                         }
+                    });
+                    // Send the local changes to the remote storage if we are a trusted node
+                    if (this._config.isTrustedNode) {
+                        await this._remoteSyncStateHelper.addChangeSetToSyncState(storageKey, changeSetStorageId);
                         await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
                     }
-                });
-            }
+                    else if (!Is.empty(this._trustedSynchronisedStorageComponent)) {
+                        // If we are not a trusted node, we need to send the changes to the trusted node
+                        await this._trustedSynchronisedStorageComponent.syncChangeSet(changeSetStorageId);
+                        await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
+                    }
+                }
+                else {
+                    await this._logging?.log({
+                        level: "info",
+                        source: this.CLASS_NAME,
+                        message: "createdStorageChangeSetNone",
+                        data: {
+                            storageKey
+                        }
+                    });
+                }
+            });
+        }
+        else {
+            await this._logging?.log({
+                level: "info",
+                source: this.CLASS_NAME,
+                message: "updateFromLocalSyncStateNoChanges",
+                data: {
+                    storageKey
+                }
+            });
         }
     }
     /**
      * Start the consolidation sync.
-     * @param schemaType The schema type to consolidate.
+     * @param storageKey The storage key to consolidate.
      * @returns Nothing.
      * @internal
      */
-    async startConsolidationSync(schemaType) {
+    async startConsolidationSync(storageKey) {
         let localChangeSnapshot;
         try {
             // If we are performing a consolidation, we can remove the local changes
-            await this._localSyncStateHelper.getLocalChangeSnapshot(schemaType);
+            await this._localSyncStateHelper.getLocalChangeSnapshot(storageKey);
             if (!Is.empty(localChangeSnapshot)) {
                 await this._localSyncStateHelper.removeLocalChangeSnapshot(localChangeSnapshot);
             }
             if (Is.stringValue(this._nodeIdentity)) {
-                await this._remoteSyncStateHelper.consolidateFromLocal(schemaType, this._config.consolidationBatchSize ??
+                await this._remoteSyncStateHelper.consolidateFromLocal(storageKey, this._config.consolidationBatchSize ??
                     SynchronisedStorageService._DEFAULT_CONSOLIDATION_BATCH_SIZE);
                 // The consolidation was successful, so we can remove the local change snapshot permanently
                 localChangeSnapshot = undefined;
@@ -1284,10 +1437,6 @@ class SynchronisedStorageService {
                 error: BaseError.fromError(error)
             });
         }
-        finally {
-            // Set a timer to perform the consolidation again
-            this._consolidationTimers[schemaType] = setTimeout(async () => this.startConsolidationSync(schemaType), this._config.consolidationIntervalMs);
-        }
     }
     /**
      * Register a new sync type.
@@ -1300,14 +1449,48 @@ class SynchronisedStorageService {
             source: this.CLASS_NAME,
             message: "registerType",
             data: {
-                schemaType: syncRegisterType.schemaType
+                storageKey: syncRegisterType.storageKey
             }
         });
-        if (this._config.entityUpdateIntervalMs > 0) {
-            await this.startEntitySync(syncRegisterType.schemaType);
+        if (Is.empty(this._activeStorageKeys[syncRegisterType.storageKey])) {
+            this._activeStorageKeys[syncRegisterType.storageKey] = false;
+            if (this._serviceStarted) {
+                await this.activateStorageKey(syncRegisterType.storageKey);
+            }
         }
-        if (this._config.isTrustedNode && this._config.consolidationIntervalMs > 0) {
-            await this.startConsolidationSync(syncRegisterType.schemaType);
+    }
+    /**
+     * Activate a storage key.
+     * @param storageKey The storage key to activate.
+     * @internal
+     */
+    async activateStorageKey(storageKey) {
+        if (!Is.empty(this._activeStorageKeys[storageKey]) && !this._activeStorageKeys[storageKey]) {
+            await this._logging?.log({
+                level: "info",
+                source: this.CLASS_NAME,
+                message: "activateType",
+                data: {
+                    storageKey
+                }
+            });
+            this._activeStorageKeys[storageKey] = true;
+            if (this._config.entityUpdateIntervalMinutes > 0) {
+                await this._taskSchedulerComponent.addTask(`synchronised-storage-update-${storageKey}`, [
+                    {
+                        nextTriggerTime: Date.now(),
+                        intervalMinutes: this._config.entityUpdateIntervalMinutes
+                    }
+                ], async () => this.startEntitySync(storageKey));
+            }
+            if (this._config.isTrustedNode && this._config.consolidationIntervalMinutes > 0) {
+                await this._taskSchedulerComponent.addTask(`synchronised-storage-consolidation-${storageKey}`, [
+                    {
+                        nextTriggerTime: Date.now(),
+                        intervalMinutes: this._config.consolidationIntervalMinutes
+                    }
+                ], async () => this.startConsolidationSync(storageKey));
+            }
         }
     }
 }