@twin.org/synchronised-storage-service 0.0.1-next.8 → 0.0.3-next.1

package/dist/es/helpers/remoteSyncStateHelper.js

@@ -0,0 +1,560 @@
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+import { BaseError, Converter, Is, NotFoundError, ObjectHelper, RandomHelper } from "@twin.org/core";
+import { SyncNodeIdMode, SyncChangeOperation, SynchronisedStorageTopics, SynchronisedStorageContexts, SynchronisedStorageTypes } from "@twin.org/synchronised-storage-models";
+import { SYNC_POINTER_STORE_VERSION, SYNC_SNAPSHOT_VERSION, SYNC_STATE_VERSION } from "./versions.js";
+/**
+ * Class for performing entity storage operations in decentralised storage.
+ */
+export class RemoteSyncStateHelper {
+    /**
+     * Runtime name for the class.
+     */
+    static CLASS_NAME = "RemoteSyncStateHelper";
+    /**
+     * The logging component to use for logging.
+     * @internal
+     */
+    _logging;
+    /**
+     * The event bus component.
+     * @internal
+     */
+    _eventBusComponent;
+    /**
+     * The blob storage helper.
+     * @internal
+     */
+    _blobStorageHelper;
+    /**
+     * The verifiable storage connector to use for storing sync pointers.
+     * @internal
+     */
+    _verifiableSyncPointerStorageConnector;
+    /**
+     * The change set helper to use for applying changesets.
+     * @internal
+     */
+    _changeSetHelper;
+    /**
+     * The storage ids of the batch responses for each storage key.
+     * @internal
+     */
+    _batchResponseStorageIds;
+    /**
+     * The full changes for each storage key.
+     * @internal
+     */
+    _populateFullChanges;
+    /**
+     * The synchronised storage key to use for verified storage operations.
+     * @internal
+     */
+    _synchronisedStorageKey;
+    /**
+     * The identity of the node that is performing the update.
+     * @internal
+     */
+    _nodeId;
+    /**
+     * Whether the node is trusted or not.
+     * @internal
+     */
+    _isTrustedNode;
+    /**
+     * Maximum number of consolidations to keep in storage.
+     * @internal
+     */
+    _maxConsolidations;
+    /**
+     * Create a new instance of RemoteSyncStateHelper.
+     * @param loggingComponent The logging component to use for logging.
+     * @param eventBusComponent The event bus component to use for events.
+     * @param verifiableSyncPointerStorageConnector The verifiable storage connector to use for storing sync pointers.
+     * @param blobStorageHelper The blob storage helper to use for remote sync states.
+     * @param changeSetHelper The change set helper to use for managing changesets.
+     * @param isTrustedNode Whether the node is trusted or not.
+     * @param maxConsolidations The maximum number of consolidations to keep in storage.
+     */
+    constructor(loggingComponent, eventBusComponent, verifiableSyncPointerStorageConnector, blobStorageHelper, changeSetHelper, isTrustedNode, maxConsolidations) {
+        this._logging = loggingComponent;
+        this._eventBusComponent = eventBusComponent;
+        this._verifiableSyncPointerStorageConnector = verifiableSyncPointerStorageConnector;
+        this._changeSetHelper = changeSetHelper;
+        this._blobStorageHelper = blobStorageHelper;
+        this._isTrustedNode = isTrustedNode;
+        this._maxConsolidations = maxConsolidations;
+        this._batchResponseStorageIds = {};
+        this._populateFullChanges = {};
+    }
+    /**
+     * Set the node identity to use for signing changesets.
+     * @param nodeId The identity of the node that is performing the update.
+     */
+    setNodeId(nodeId) {
+        this._nodeId = nodeId;
+    }
+    /**
+     * Start the remote sync state helper.
+     */
+    async start() {
+        await this._eventBusComponent.subscribe(SynchronisedStorageTopics.BatchResponse, async (response) => {
+            await this.handleBatchResponse(response.data);
+        });
+        await this._eventBusComponent.subscribe(SynchronisedStorageTopics.LocalItemResponse, async (response) => {
+            await this.handleLocalItemResponse(response.data);
+        });
+    }
+    /**
+     * Set the synchronised storage key.
+     * @param synchronisedStorageKey The synchronised storage key to use.
+     */
+    setSynchronisedStorageKey(synchronisedStorageKey) {
+        this._synchronisedStorageKey = synchronisedStorageKey;
+    }
+    /**
+     * Build a changeset.
+     * @param storageKey The storage key of the change set.
+     * @param changes The changes to apply.
+     * @param completeCallback The callback to call when the changeset is created and stored.
+     * @returns The storage id of the change set if created.
+     */
+    async buildChangeSet(storageKey, changes, completeCallback) {
+        await this._logging?.log({
+            level: "info",
+            source: RemoteSyncStateHelper.CLASS_NAME,
+            message: "buildingChangeSet",
+            data: {
+                storageKey,
+                changeCount: changes.length
+            }
+        });
+        this._populateFullChanges[storageKey] = {
+            changes,
+            entities: {},
+            requestIds: [],
+            completeCallback: async () => this.finaliseFullChanges(storageKey, completeCallback)
+        };
+        const setChanges = changes.filter(c => c.operation === SyncChangeOperation.Set);
+        if (setChanges.length === 0) {
+            // If we don't need to request any full details, we can just call the complete callback
+            await this.finaliseFullChanges(storageKey, completeCallback);
+        }
+        else {
+            // Otherwise we need to request the full details for each change
+            this._populateFullChanges[storageKey].requestIds = setChanges.map(change => change.id);
+            // Once all the requests are handled the callback will be called
+            for (const change of setChanges) {
+                // Create a request for each change to populate the full details
+                await this._logging?.log({
+                    level: "info",
+                    source: RemoteSyncStateHelper.CLASS_NAME,
+                    message: "createChangeSetRequestingItem",
+                    data: {
+                        storageKey,
+                        id: change.id
+                    }
+                });
+                await this._eventBusComponent.publish(SynchronisedStorageTopics.LocalItemRequest, {
+                    storageKey,
+                    id: change.id
+                });
+            }
+        }
+    }
+    /**
+     * Finalise the full details for the sync change set.
+     * @param storageKey The storage key of the change set.
+     * @param completeCallback The callback to call when the changeset is populated.
+     * @returns Nothing.
+     */
+    async finaliseFullChanges(storageKey, completeCallback) {
+        await this._logging?.log({
+            level: "info",
+            source: RemoteSyncStateHelper.CLASS_NAME,
+            message: "finalisingSyncChanges",
+            data: {
+                storageKey
+            }
+        });
+        if (Is.stringValue(this._nodeId)) {
+            const changes = this._populateFullChanges[storageKey].changes;
+            for (const change of changes) {
+                change.entity = this._populateFullChanges[storageKey].entities[change.id] ?? change.entity;
+                if (change.operation === SyncChangeOperation.Set && Is.objectValue(change.entity)) {
+                    // Remove the id from the entity as this is stored in the operation
+                    // and will be reinstated when the changeset is reconstituted
+                    ObjectHelper.propertyDelete(change.entity, "id");
+                    // Remove the node identity as the changeset has this stored at the top level
+                    // and we do not want to store it in the change itself to reduce redundancy
+                    ObjectHelper.propertyDelete(change.entity, "nodeId");
+                }
+            }
+            const now = new Date(Date.now()).toISOString();
+            const syncChangeSet = {
+                "@context": SynchronisedStorageContexts.ContextRoot,
+                type: SynchronisedStorageTypes.ChangeSet,
+                id: Converter.bytesToHex(RandomHelper.generate(32)),
+                dateCreated: now,
+                dateModified: now,
+                storageKey,
+                changes,
+                nodeId: this._nodeId
+            };
+            try {
+                // If this is a trusted node, we also store the changeset
+                let changeSetStorageId;
+                if (this._isTrustedNode) {
+                    changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
+                }
+                await completeCallback(syncChangeSet, changeSetStorageId);
+            }
+            catch (err) {
+                await this._logging?.log({
+                    level: "error",
+                    source: RemoteSyncStateHelper.CLASS_NAME,
+                    message: "finalisingSyncChangesFailed",
+                    data: {
+                        storageKey
+                    },
+                    error: BaseError.fromError(err)
+                });
+                await completeCallback();
+            }
+        }
+        else {
+            await completeCallback();
+        }
+    }
+    /**
+     * Add a new changeset into the sync state.
+     * @param storageKey The storage key of the change set to add.
+     * @param changeSetStorageId The id of the change set to add the current state
+     * @returns Nothing.
+     */
+    async addChangeSetToSyncState(storageKey, changeSetStorageId) {
+        await this._logging?.log({
+            level: "info",
+            source: RemoteSyncStateHelper.CLASS_NAME,
+            message: "addChangeSetToSyncState",
+            data: {
+                storageKey,
+                changeSetStorageId
+            }
+        });
+        // First load the sync pointer store to get the current sync pointer for the storage key
+        const syncPointerStore = await this.getVerifiableSyncPointerStore();
+        let syncState;
+        if (!Is.empty(syncPointerStore.syncPointers[storageKey])) {
+            syncState = await this.getSyncState(syncPointerStore.syncPointers[storageKey]);
+        }
+        // No current sync state, so we create a new one
+        if (Is.empty(syncState)) {
+            syncState = { version: SYNC_STATE_VERSION, storageKey, snapshots: [] };
+        }
+        // Sort the snapshots so the newest snapshot is last in the array
+        const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
+        // Get the current snapshot, if it does not exist we create a new one
+        let currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+        const currentEpoch = currentSnapshot?.epoch ?? 0;
+        const now = new Date(Date.now()).toISOString();
+        // If there is no snapshot or the current one is a consolidation
+        // we start a new snapshot
+        if (Is.empty(currentSnapshot) || currentSnapshot.isConsolidated) {
+            currentSnapshot = {
+                version: SYNC_SNAPSHOT_VERSION,
+                id: Converter.bytesToHex(RandomHelper.generate(32)),
+                dateCreated: now,
+                dateModified: now,
+                isConsolidated: false,
+                epoch: currentEpoch + 1,
+                changeSetStorageIds: []
+            };
+            syncState.snapshots.push(currentSnapshot);
+        }
+        else {
+            // Snapshot exists, we update the dateModified
+            currentSnapshot.dateModified = now;
+        }
+        // Add the changeset storage id to the current snapshot
+        currentSnapshot.changeSetStorageIds.push(changeSetStorageId);
+        // Store the sync state in the blob storage
+        syncPointerStore.syncPointers[storageKey] = await this.storeRemoteSyncState(syncState);
+        // Store the verifiable sync pointer store in the verifiable storage
+        await this.storeVerifiableSyncPointerStore(syncPointerStore);
+    }
+    /**
+     * Create a consolidated snapshot for the entire storage.
+     * @param storageKey The storage key of the snapshot to create.
+     * @param batchSize The batch size to use for consolidation.
+     * @returns Nothing.
+     */
+    async consolidationStart(storageKey, batchSize) {
+        await this._logging?.log({
+            level: "info",
+            source: RemoteSyncStateHelper.CLASS_NAME,
+            message: "consolidationStarting"
+        });
+        // Perform a batch request to start the consolidation
+        await this._eventBusComponent.publish(SynchronisedStorageTopics.BatchRequest, { storageKey, batchSize, requestMode: SyncNodeIdMode.All });
+    }
+    /**
+     * Get the sync pointer store.
+     * @returns The sync pointer store.
+     */
+    async getVerifiableSyncPointerStore() {
+        if (Is.stringValue(this._synchronisedStorageKey)) {
+            try {
+                await this._logging?.log({
+                    level: "info",
+                    source: RemoteSyncStateHelper.CLASS_NAME,
+                    message: "verifiableSyncPointerStoreRetrieving",
+                    data: {
+                        key: this._synchronisedStorageKey
+                    }
+                });
+                const syncPointerStore = await this._verifiableSyncPointerStorageConnector.get(this._synchronisedStorageKey, { includeData: true });
+                if (Is.uint8Array(syncPointerStore.data)) {
+                    const syncPointer = ObjectHelper.fromBytes(syncPointerStore.data);
+                    await this._logging?.log({
+                        level: "info",
+                        source: RemoteSyncStateHelper.CLASS_NAME,
+                        message: "verifiableSyncPointerStoreRetrieved",
+                        data: {
+                            key: this._synchronisedStorageKey
+                        }
+                    });
+                    return syncPointer;
+                }
+            }
+            catch (err) {
+                if (!BaseError.someErrorName(err, NotFoundError.CLASS_NAME)) {
+                    throw err;
+                }
+            }
+            await this._logging?.log({
+                level: "info",
+                source: RemoteSyncStateHelper.CLASS_NAME,
+                message: "verifiableSyncPointerStoreNotFound",
+                data: {
+                    key: this._synchronisedStorageKey
+                }
+            });
+        }
+        // If no sync pointer store exists, we return an empty one
+        return {
+            version: SYNC_POINTER_STORE_VERSION,
+            syncPointers: {}
+        };
+    }
+    /**
+     * Store the verifiable sync pointer in the verifiable storage.
+     * @param syncPointerStore The sync pointer store to store.
+     * @returns Nothing.
+     */
+    async storeVerifiableSyncPointerStore(syncPointerStore) {
+        if (Is.stringValue(this._nodeId) && Is.stringValue(this._synchronisedStorageKey)) {
+            await this._logging?.log({
+                level: "info",
+                source: RemoteSyncStateHelper.CLASS_NAME,
+                message: "verifiableSyncPointerStoreStoring",
+                data: {
+                    key: this._synchronisedStorageKey
+                }
+            });
+            // Store the verifiable sync pointer in the verifiable storage
+            await this._verifiableSyncPointerStorageConnector.update(this._nodeId, this._synchronisedStorageKey, ObjectHelper.toBytes(syncPointerStore));
+        }
+    }
+    /**
+     * Store the remote sync state.
+     * @param syncState The sync state to store.
+     * @returns The id of the sync state.
+     */
+    async storeRemoteSyncState(syncState) {
+        await this._logging?.log({
+            level: "info",
+            source: RemoteSyncStateHelper.CLASS_NAME,
+            message: "syncStateStoring",
+            data: {
+                snapshotCount: syncState.snapshots.length
+            }
+        });
+        // Limits the number of consolidations in the list so that we can shrink decentralised
+        // storage requirements, sort from newest to oldest so that we can easily find the
+        // oldest snapshots to remove.
+        const snapshots = syncState.snapshots.sort((a, b) => new Date(a.dateCreated).getTime() - new Date(b.dateCreated).getTime());
+        // Find all the consolidation indexes
+        const consolidationIndexes = [];
+        for (let i = 0; i < snapshots.length; i++) {
+            const snapshot = snapshots[i];
+            if (snapshot.isConsolidated) {
+                consolidationIndexes.push(i);
+            }
+        }
+        if (consolidationIndexes.length > this._maxConsolidations) {
+            // Once we have reached the max for consolidations we need to remove
+            // all the snapshots, including non consolidated ones, beyond this point
+            const toRemove = snapshots.slice(consolidationIndexes[this._maxConsolidations - 1] + 1);
+            syncState.snapshots = snapshots.slice(0, consolidationIndexes[this._maxConsolidations - 1] + 1);
+            for (const snapshot of toRemove) {
+                // We need to remove all the storage ids associated with the snapshot
+                if (Is.arrayValue(snapshot.changeSetStorageIds)) {
+                    for (const storageId of snapshot.changeSetStorageIds) {
+                        await this._blobStorageHelper.removeBlob(storageId);
+                    }
+                }
+            }
+        }
+        return this._blobStorageHelper.saveBlob(syncState);
+    }
+    /**
+     * Get the remote sync state.
+     * @param syncPointerId The id of the sync pointer to retrieve the state for.
+     * @returns The remote sync state.
+     */
+    async getSyncState(syncPointerId) {
+        try {
+            await this._logging?.log({
+                level: "info",
+                source: RemoteSyncStateHelper.CLASS_NAME,
+                message: "syncStateRetrieving",
+                data: {
+                    syncPointerId
+                }
+            });
+            const syncState = await this._blobStorageHelper.loadBlob(syncPointerId);
+            if (Is.object(syncState)) {
+                await this._logging?.log({
+                    level: "info",
+                    source: RemoteSyncStateHelper.CLASS_NAME,
+                    message: "syncStateRetrieved",
+                    data: {
+                        syncPointerId,
+                        snapshotCount: syncState.snapshots.length
+                    }
+                });
+                return syncState;
+            }
+        }
+        catch (error) {
+            await this._logging?.log({
+                level: "warn",
+                source: RemoteSyncStateHelper.CLASS_NAME,
+                message: "getSyncStateError",
+                data: {
+                    syncPointerId
+                },
+                error: BaseError.fromError(error)
+            });
+        }
+        await this._logging?.log({
+            level: "info",
+            source: RemoteSyncStateHelper.CLASS_NAME,
+            message: "syncStateNotFound",
+            data: {
+                syncPointerId
+            }
+        });
+    }
+    /**
+     * Handle the batch response which is triggered from a consolidation request.
+     * @param response The batch response to handle.
+     */
+    async handleBatchResponse(response) {
+        if (Is.stringValue(this._nodeId)) {
+            const now = new Date(Date.now()).toISOString();
+            // Create a new snapshot entry for the current batch
+            const syncChangeSet = {
+                "@context": SynchronisedStorageContexts.ContextRoot,
+                type: SynchronisedStorageTypes.ChangeSet,
+                id: Converter.bytesToHex(RandomHelper.generate(32)),
+                dateCreated: now,
+                dateModified: now,
+                changes: response.entities.map(change => ({
+                    operation: SyncChangeOperation.Set,
+                    id: change.id
+                })),
+                storageKey: response.storageKey,
+                nodeId: this._nodeId
+            };
+            // Store the changeset in the blob storage
+            const changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);
+            // Add the changeset storage id to the snapshot ids
+            this._batchResponseStorageIds[response.storageKey] ??= [];
+            this._batchResponseStorageIds[response.storageKey].push(changeSetStorageId);
+            // If this is the last entry in the batch response, we can create the consolidated snapshot
+            if (response.lastEntry) {
+                // Get the current sync pointer store
+                const syncPointerStore = await this.getVerifiableSyncPointerStore();
+                let syncState;
+                if (Is.stringValue(syncPointerStore.syncPointers[response.storageKey])) {
+                    // If the sync pointer exists, we load the current sync state
+                    syncState = await this.getSyncState(syncPointerStore.syncPointers[response.storageKey]);
+                }
+                // If the sync state does not exist, we create a new one
+                syncState ??= {
+                    version: SYNC_STATE_VERSION,
+                    storageKey: response.storageKey,
+                    snapshots: []
+                };
+                // Sort the snapshots so the newest snapshot is last in the array
+                const sortedSnapshots = syncState.snapshots.sort((a, b) => a.dateCreated.localeCompare(b.dateCreated));
+                const currentSnapshot = sortedSnapshots[sortedSnapshots.length - 1];
+                const currentEpoch = currentSnapshot?.epoch ?? 0;
+                const batchSnapshot = {
+                    version: SYNC_SNAPSHOT_VERSION,
+                    id: Converter.bytesToHex(RandomHelper.generate(32)),
+                    dateCreated: now,
+                    dateModified: now,
+                    isConsolidated: true,
+                    epoch: currentEpoch + 1,
+                    changeSetStorageIds: this._batchResponseStorageIds[response.storageKey]
+                };
+                syncState.snapshots.push(batchSnapshot);
+                // Store the updated sync state
+                const syncStateId = await this.storeRemoteSyncState(syncState);
+                syncPointerStore.syncPointers[response.storageKey] = syncStateId;
+                // Store the verifiable sync pointer in the verifiable storage
+                await this.storeVerifiableSyncPointerStore(syncPointerStore);
+                // Remove the batch response storage ids for the storage key
+                // as we have consolidated the changes
+                delete this._batchResponseStorageIds[response.storageKey];
+                await this._logging?.log({
+                    level: "info",
+                    source: RemoteSyncStateHelper.CLASS_NAME,
+                    message: "consolidationCompleted"
+                });
+            }
+        }
+    }
+    /**
+     * Handle the item response.
+     * @param response The item response to handle.
+     */
+    async handleLocalItemResponse(response) {
+        await this._logging?.log({
+            level: "info",
+            source: RemoteSyncStateHelper.CLASS_NAME,
+            message: "createChangeSetRespondingItem",
+            data: {
+                storageKey: response.storageKey,
+                id: response.id
+            }
+        });
+        // We have received a response to an item request, find the right storage
+        // for the request id
+        if (!Is.empty(this._populateFullChanges[response.storageKey])) {
+            const idx = this._populateFullChanges[response.storageKey].requestIds.indexOf(response.id);
+            if (idx !== -1) {
+                this._populateFullChanges[response.storageKey].requestIds.splice(idx, 1);
+                this._populateFullChanges[response.storageKey].entities[response.id] = response.entity;
+                // If there are no request ids remaining we can complete the population
+                if (this._populateFullChanges[response.storageKey].requestIds.length === 0) {
+                    await this._populateFullChanges[response.storageKey].completeCallback();
+                }
+            }
+        }
+    }
+}
+//# sourceMappingURL=remoteSyncStateHelper.js.map

package/dist/es/helpers/remoteSyncStateHelper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remoteSyncStateHelper.js","sourceRoot":"","sources":["../../../src/helpers/remoteSyncStateHelper.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,OAAO,EACN,SAAS,EACT,SAAS,EACT,EAAE,EACF,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,MAAM,gBAAgB,CAAC;AAIxB,OAAO,EAMN,cAAc,EACd,mBAAmB,EACnB,yBAAyB,EACzB,2BAA2B,EAC3B,wBAAwB,EAExB,MAAM,uCAAuC,CAAC;AAI/C,OAAO,EACN,0BAA0B,EAC1B,qBAAqB,EACrB,kBAAkB,EAClB,MAAM,eAAe,CAAC;AAKvB;;GAEG;AACH,MAAM,OAAO,qBAAqB;IACjC;;OAEG;IACI,MAAM,CAAU,UAAU,2BAA2C;IAE5E;;;OAGG;IACc,QAAQ,CAAqB;IAE9C;;;OAGG;IACc,kBAAkB,CAAqB;IAExD;;;OAGG;IACc,kBAAkB,CAAoB;IAEvD;;;OAGG;IACc,sCAAsC,CAA8B;IAErF;;;OAGG;IACc,gBAAgB,CAAkB;IAEnD;;;OAGG;IACc,wBAAwB,CAAqC;IAE9E;;;OAGG;IACc,oBAAoB,CAOnC;IAEF;;;OAGG;IACK,uBAAuB,CAAU;IAEzC;;;OAGG;IACK,OAAO,CAAU;IAEzB;;;OAGG;IACc,cAAc,CAAU;IAEzC;;;OAGG;IACc,kBAAkB,CAAS;IAE5C;;;;;;;;;OASG;IACH,YACC,gBAA+C,EAC/C,iBAAqC,EACrC,qCAAkE,EAClE,iBAAoC,EACpC,eAAgC,EAChC,aAAsB,EACtB,iBAAyB;QAEzB,IAAI,CAAC,QAAQ,GAAG,gBAAgB,CAAC;QACjC,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC;QAC5C,IAAI,CAAC,sCAAsC,GAAG,qCAAqC,CAAC;QACpF,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC;QAC5C,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC;QAE5C,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC;QACnC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;IAChC,CAAC;IAED;;;OAGG;IACI,SAAS,CAAC,MAAc;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,KAAK;QACjB,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CACtC,yBAAyB,CAAC,aAAa,EACvC,KAAK,EAAC,QAAQ,EAAC,EAAE;YAChB,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC,CACD,CAAC;QAEF,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CACtC,yBAAyB,CAAC,iBAAiB,EAC3C,KAAK,EAAC,QAAQ,EAAC,EAAE;YAChB,MAAM,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC,CACD,CAAC;IACH,CAAC;IAED;;;OAGG;IACI,yBAAyB,CAAC,sBAA8B;QAC9D,IAAI,CAAC,uBAAuB,GAAG,sBAAsB,CAAC;IACvD,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,cAAc,CAC1B,UAAkB,EAClB,OAAsB,EACtB,gBAAgF;QAEhF,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;YACxB,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;YACxC,OAAO,EAAE,mBAAmB;YAC5B,IAAI,EAAE;gBACL,UAAU;gBACV,WAAW,EAAE,OAAO,CAAC,MAAM;aAC3B;SACD,CAAC,CAAC;QAEH,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,GAAG;YACvC,OAAO;YACP,QAAQ,EAAE,EAAE;YACZ,UAAU,EAAE,EAAE;YACd,gBAAgB,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,gBAAgB,CAAC;SACpF,CAAC;QAEF,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC;QAChF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,uFAAuF;YACvF,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACP,gEAAgE;YAChE,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAEvF,gEAAgE;YAChE,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;gBACjC,gEAAgE;gBAChE,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;oBACxB,KAAK,EAAE,MAAM;oBACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;oBACxC,OAAO,EAAE,+BAA+B;oBACxC,IAAI,EAAE;wBACL,UAAU;wBACV,EAAE,EAAE,MAAM,CAAC,EAAE;qBACb;iBACD,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CACpC,yBAAyB,CAAC,gBAAgB,EAC1C;oBACC,UAAU;oBACV,EAAE,EAAE,MAAM,CAAC,EAAE;iBACb,CACD,CAAC;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,mBAAmB,CAC/B,UAAkB,EAClB,gBAAgF;QAEhF,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;YACxB,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;YACxC,OAAO,EAAE,uBAAuB;YAChC,IAAI,EAAE;gBACL,UAAU;aACV;SACD,CAAC,CAAC;QACH,IAAI,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC;YAC9D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;gBAE3F,IAAI,MAAM,CAAC,SAAS,KAAK,mBAAmB,CAAC,GAAG,IAAI,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;oBACnF,mEAAmE;oBACnE,6DAA6D;oBAC7D,YAAY,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;oBACjD,6EAA6E;oBAC7E,2EAA2E;oBAC3E,YAAY,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACtD,CAAC;YACF,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,aAAa,GAAmB;gBACrC,UAAU,EAAE,2BAA2B,CAAC,WAAW;gBACnD,IAAI,EAAE,wBAAwB,CAAC,SAAS;gBACxC,EAAE,EAAE,SAAS,CAAC,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACnD,WAAW,EAAE,GAAG;gBAChB,YAAY,EAAE,GAAG;gBACjB,UAAU;gBACV,OAAO;gBACP,MAAM,EAAE,IAAI,CAAC,OAAO;aACpB,CAAC;YAEF,IAAI,CAAC;gBACJ,yDAAyD;gBACzD,IAAI,kBAAkB,CAAC;gBACvB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBACzB,kBAAkB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;gBAChF,CAAC;gBAED,MAAM,gBAAgB,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC;YAC3D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;oBACxB,KAAK,EAAE,OAAO;oBACd,MAAM,EAAE,qBAAqB,CAAC,UAAU;oBACxC,OAAO,EAAE,6BAA6B;oBACtC,IAAI,EAAE;wBACL,UAAU;qBACV;oBACD,KAAK,EAAE,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC;iBAC/B,CAAC,CAAC;gBACH,MAAM,gBAAgB,EAAE,CAAC;YAC1B,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,gBAAgB,EAAE,CAAC;QAC1B,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,uBAAuB,CACnC,UAAkB,EAClB,kBAA0B;QAE1B,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;YACxB,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;YACxC,OAAO,EAAE,yBAAyB;YAClC,IAAI,EAAE;gBACL,UAAU;gBACV,kBAAkB;aAClB;SACD,CAAC,CAAC;QAEH,wFAAwF;QACxF,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,6BAA6B,EAAE,CAAC;QAEpE,IAAI,SAAiC,CAAC;QACtC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;YAC1D,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC;QAChF,CAAC;QAED,gDAAgD;QAChD,IAAI,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YACzB,SAAS,GAAG,EAAE,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QACxE,CAAC;QAED,iEAAiE;QACjE,MAAM,eAAe,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzD,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAC1C,CAAC;QAEF,qEAAqE;QACrE,IAAI,eAAe,GAA8B,eAAe,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC7F,MAAM,YAAY,GAAG,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC;QACjD,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAE/C,gEAAgE;QAChE,0BAA0B;QAC1B,IAAI,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,eAAe,CAAC,cAAc,EAAE,CAAC;YACjE,eAAe,GAAG;gBACjB,OAAO,EAAE,qBAAqB;gBAC9B,EAAE,EAAE,SAAS,CAAC,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACnD,WAAW,EAAE,GAAG;gBAChB,YAAY,EAAE,GAAG;gBACjB,cAAc,EAAE,KAAK;gBACrB,KAAK,EAAE,YAAY,GAAG,CAAC;gBACvB,mBAAmB,EAAE,EAAE;aACvB,CAAC;YACF,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACP,8CAA8C;YAC9C,eAAe,CAAC,YAAY,GAAG,GAAG,CAAC;QACpC,CAAC;QAED,uDAAuD;QACvD,eAAe,CAAC,mBAAmB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAE7D,2CAA2C;QAC3C,gBAAgB,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;QAEvF,oEAAoE;QACpE,MAAM,IAAI,CAAC,+BAA+B,CAAC,gBAAgB,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,kBAAkB,CAAC,UAAkB,EAAE,SAAiB;QACpE,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;YACxB,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;YACxC,OAAO,EAAE,uBAAuB;SAChC,CAAC,CAAC;QAEH,qDAAqD;QACrD,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CACpC,yBAAyB,CAAC,YAAY,EACtC,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,CAAC,GAAG,EAAE,CAC1D,CAAC;IACH,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,6BAA6B;QACzC,IAAI,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,uBAAuB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC;gBACJ,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;oBACxB,KAAK,EAAE,MAAM;oBACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;oBACxC,OAAO,EAAE,sCAAsC;oBAC/C,IAAI,EAAE;wBACL,GAAG,EAAE,IAAI,CAAC,uBAAuB;qBACjC;iBACD,CAAC,CAAC;gBACH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,sCAAsC,CAAC,GAAG,CAC7E,IAAI,CAAC,uBAAuB,EAC5B,EAAE,WAAW,EAAE,IAAI,EAAE,CACrB,CAAC;gBACF,IAAI,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC1C,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAoB,gBAAgB,CAAC,IAAI,CAAC,CAAC;oBACrF,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;wBACxB,KAAK,EAAE,MAAM;wBACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;wBACxC,OAAO,EAAE,qCAAqC;wBAC9C,IAAI,EAAE;4BACL,GAAG,EAAE,IAAI,CAAC,uBAAuB;yBACjC;qBACD,CAAC,CAAC;oBACH,OAAO,WAAW,CAAC;gBACpB,CAAC;YACF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,MAAM,GAAG,CAAC;gBACX,CAAC;YACF,CAAC;YAED,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;gBACxB,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;gBACxC,OAAO,EAAE,oCAAoC;gBAC7C,IAAI,EAAE;oBACL,GAAG,EAAE,IAAI,CAAC,uBAAuB;iBACjC;aACD,CAAC,CAAC;QACJ,CAAC;QAED,0DAA0D;QAC1D,OAAO;YACN,OAAO,EAAE,0BAA0B;YACnC,YAAY,EAAE,EAAE;SAChB,CAAC;IACH,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,+BAA+B,CAAC,gBAAmC;QAC/E,IAAI,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,uBAAuB,CAAC,EAAE,CAAC;YAClF,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;gBACxB,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;gBACxC,OAAO,EAAE,mCAAmC;gBAC5C,IAAI,EAAE;oBACL,GAAG,EAAE,IAAI,CAAC,uBAAuB;iBACjC;aACD,CAAC,CAAC;YAEH,8DAA8D;YAC9D,MAAM,IAAI,CAAC,sCAAsC,CAAC,MAAM,CACvD,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,uBAAuB,EAC5B,YAAY,CAAC,OAAO,CAAoB,gBAAgB,CAAC,CACzD,CAAC;QACH,CAAC;IACF,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,oBAAoB,CAAC,SAAqB;QACtD,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;YACxB,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;YACxC,OAAO,EAAE,kBAAkB;YAC3B,IAAI,EAAE;gBACL,aAAa,EAAE,SAAS,CAAC,SAAS,CAAC,MAAM;aACzC;SACD,CAAC,CAAC;QAEH,sFAAsF;QACtF,kFAAkF;QAClF,8BAA8B;QAC9B,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CACzC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAC/E,CAAC;QAEF,qCAAqC;QACrC,MAAM,oBAAoB,GAAG,EAAE,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAC9B,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;gBAC7B,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;QACF,CAAC;QAED,IAAI,oBAAoB,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC3D,oEAAoE;YACpE,wEAAwE;YACxE,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAExF,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,KAAK,CACpC,CAAC,EACD,oBAAoB,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,GAAG,CAAC,CACrD,CAAC;YAEF,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;gBACjC,qEAAqE;gBACrE,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;oBACjD,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;wBACtD,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;oBACrD,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,YAAY,CAAC,aAAqB;QAC9C,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;gBACxB,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;gBACxC,OAAO,EAAE,qBAAqB;gBAC9B,IAAI,EAAE;oBACL,aAAa;iBACb;aACD,CAAC,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAa,aAAa,CAAC,CAAC;YAEpF,IAAI,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;oBACxB,KAAK,EAAE,MAAM;oBACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;oBACxC,OAAO,EAAE,oBAAoB;oBAC7B,IAAI,EAAE;wBACL,aAAa;wBACb,aAAa,EAAE,SAAS,CAAC,SAAS,CAAC,MAAM;qBACzC;iBACD,CAAC,CAAC;gBACH,OAAO,SAAS,CAAC;YAClB,CAAC;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;gBACxB,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;gBACxC,OAAO,EAAE,mBAAmB;gBAC5B,IAAI,EAAE;oBACL,aAAa;iBACb;gBACD,KAAK,EAAE,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC;aACjC,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;YACxB,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;YACxC,OAAO,EAAE,mBAAmB;YAC5B,IAAI,EAAE;gBACL,aAAa;aACb;SACD,CAAC,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,mBAAmB,CAAC,QAA4B;QAC7D,IAAI,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAE/C,oDAAoD;YACpD,MAAM,aAAa,GAAmB;gBACrC,UAAU,EAAE,2BAA2B,CAAC,WAAW;gBACnD,IAAI,EAAE,wBAAwB,CAAC,SAAS;gBACxC,EAAE,EAAE,SAAS,CAAC,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACnD,WAAW,EAAE,GAAG;gBAChB,YAAY,EAAE,GAAG;gBACjB,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;oBACzC,SAAS,EAAE,mBAAmB,CAAC,GAAG;oBAClC,EAAE,EAAE,MAAM,CAAC,EAAE;iBACb,CAAC,CAAC;gBACH,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,MAAM,EAAE,IAAI,CAAC,OAAO;aACpB,CAAC;YAEF,0CAA0C;YAC1C,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;YAErF,mDAAmD;YACnD,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YAC1D,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAE5E,2FAA2F;YAC3F,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;gBACxB,qCAAqC;gBACrC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,6BAA6B,EAAE,CAAC;gBAEpE,IAAI,SAAiC,CAAC;gBAEtC,IAAI,EAAE,CAAC,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;oBACxE,6DAA6D;oBAC7D,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;gBACzF,CAAC;gBAED,wDAAwD;gBACxD,SAAS,KAAK;oBACb,OAAO,EAAE,kBAAkB;oBAC3B,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,SAAS,EAAE,EAAE;iBACb,CAAC;gBAEF,iEAAiE;gBACjE,MAAM,eAAe,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzD,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAC1C,CAAC;gBACF,MAAM,eAAe,GACpB,eAAe,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC7C,MAAM,YAAY,GAAG,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC;gBAEjD,MAAM,aAAa,GAAkB;oBACpC,OAAO,EAAE,qBAAqB;oBAC9B,EAAE,EAAE,SAAS,CAAC,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;oBACnD,WAAW,EAAE,GAAG;oBAChB,YAAY,EAAE,GAAG;oBACjB,cAAc,EAAE,IAAI;oBACpB,KAAK,EAAE,YAAY,GAAG,CAAC;oBACvB,mBAAmB,EAAE,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,UAAU,CAAC;iBACvE,CAAC;gBACF,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAExC,+BAA+B;gBAC/B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;gBAE/D,gBAAgB,CAAC,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,WAAW,CAAC;gBAEjE,8DAA8D;gBAC9D,MAAM,IAAI,CAAC,+BAA+B,CAAC,gBAAgB,CAAC,CAAC;gBAE7D,4DAA4D;gBAC5D,sCAAsC;gBACtC,OAAO,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;gBAE1D,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;oBACxB,KAAK,EAAE,MAAM;oBACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;oBACxC,OAAO,EAAE,wBAAwB;iBACjC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,uBAAuB,CAAC,QAA2B;QAChE,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;YACxB,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,qBAAqB,CAAC,UAAU;YACxC,OAAO,EAAE,+BAA+B;YACxC,IAAI,EAAE;gBACL,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,EAAE,EAAE,QAAQ,CAAC,EAAE;aACf;SACD,CAAC,CAAC;QACH,yEAAyE;QACzE,qBAAqB;QACrB,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;YAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAE3F,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;gBAChB,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;gBACzE,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;gBAEvF,uEAAuE;gBACvE,IAAI,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC5E,MAAM,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;gBACzE,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport {\n\tBaseError,\n\tConverter,\n\tIs,\n\tNotFoundError,\n\tObjectHelper,\n\tRandomHelper\n} from \"@twin.org/core\";\nimport type { IEventBusComponent } from \"@twin.org/event-bus-models\";\nimport type { ILoggingComponent } from \"@twin.org/logging-models\";\nimport { nameof } from \"@twin.org/nameof\";\nimport {\n\ttype ISyncBatchRequest,\n\ttype ISyncBatchResponse,\n\ttype ISyncChange,\n\ttype ISyncChangeSet,\n\ttype ISyncItemResponse,\n\tSyncNodeIdMode,\n\tSyncChangeOperation,\n\tSynchronisedStorageTopics,\n\tSynchronisedStorageContexts,\n\tSynchronisedStorageTypes,\n\ttype ISynchronisedEntity\n} from \"@twin.org/synchronised-storage-models\";\nimport type { IVerifiableStorageConnector } from \"@twin.org/verifiable-storage-models\";\nimport type { BlobStorageHelper } from \"./blobStorageHelper.js\";\nimport type { ChangeSetHelper } from \"./changeSetHelper.js\";\nimport {\n\tSYNC_POINTER_STORE_VERSION,\n\tSYNC_SNAPSHOT_VERSION,\n\tSYNC_STATE_VERSION\n} from \"./versions.js\";\nimport type { ISyncPointerStore } from \"../models/ISyncPointerStore.js\";\nimport type { ISyncSnapshot } from \"../models/ISyncSnapshot.js\";\nimport type { ISyncState } from \"../models/ISyncState.js\";\n\n/**\n * Class for performing entity storage operations in decentralised storage.\n */\nexport class RemoteSyncStateHelper {\n\t/**\n\t * Runtime name for the class.\n\t */\n\tpublic static readonly CLASS_NAME: string = nameof<RemoteSyncStateHelper>();\n\n\t/**\n\t * The logging component to use for logging.\n\t * @internal\n\t */\n\tprivate readonly _logging?: ILoggingComponent;\n\n\t/**\n\t * The event bus component.\n\t * @internal\n\t */\n\tprivate readonly _eventBusComponent: IEventBusComponent;\n\n\t/**\n\t * The blob storage helper.\n\t * @internal\n\t */\n\tprivate readonly _blobStorageHelper: BlobStorageHelper;\n\n\t/**\n\t * The verifiable storage connector to use for storing sync pointers.\n\t * @internal\n\t */\n\tprivate readonly _verifiableSyncPointerStorageConnector: IVerifiableStorageConnector;\n\n\t/**\n\t * The change set helper to use for applying changesets.\n\t * @internal\n\t */\n\tprivate readonly _changeSetHelper: ChangeSetHelper;\n\n\t/**\n\t * The storage ids of the batch responses for each storage key.\n\t * @internal\n\t */\n\tprivate readonly _batchResponseStorageIds: { [storageKey: string]: string[] };\n\n\t/**\n\t * The full changes for each storage key.\n\t * @internal\n\t */\n\tprivate readonly _populateFullChanges: {\n\t\t[storageKey: string]: {\n\t\t\tchanges: ISyncChange[];\n\t\t\tentities: { [id: string]: ISynchronisedEntity | undefined };\n\t\t\trequestIds: string[];\n\t\t\tcompleteCallback: (id?: string) => Promise<void>;\n\t\t};\n\t};\n\n\t/**\n\t * The synchronised storage key to use for verified storage operations.\n\t * @internal\n\t */\n\tprivate _synchronisedStorageKey?: string;\n\n\t/**\n\t * The identity of the node that is performing the update.\n\t * @internal\n\t */\n\tprivate _nodeId?: string;\n\n\t/**\n\t * Whether the node is trusted or not.\n\t * @internal\n\t */\n\tprivate readonly _isTrustedNode: boolean;\n\n\t/**\n\t * Maximum number of consolidations to keep in storage.\n\t * @internal\n\t */\n\tprivate readonly _maxConsolidations: number;\n\n\t/**\n\t * Create a new instance of RemoteSyncStateHelper.\n\t * @param loggingComponent The logging component to use for logging.\n\t * @param eventBusComponent The event bus component to use for events.\n\t * @param verifiableSyncPointerStorageConnector The verifiable storage connector to use for storing sync pointers.\n\t * @param blobStorageHelper The blob storage helper to use for remote sync states.\n\t * @param changeSetHelper The change set helper to use for managing changesets.\n\t * @param isTrustedNode Whether the node is trusted or not.\n\t * @param maxConsolidations The maximum number of consolidations to keep in storage.\n\t */\n\tconstructor(\n\t\tloggingComponent: ILoggingComponent | undefined,\n\t\teventBusComponent: IEventBusComponent,\n\t\tverifiableSyncPointerStorageConnector: IVerifiableStorageConnector,\n\t\tblobStorageHelper: BlobStorageHelper,\n\t\tchangeSetHelper: ChangeSetHelper,\n\t\tisTrustedNode: boolean,\n\t\tmaxConsolidations: number\n\t) {\n\t\tthis._logging = loggingComponent;\n\t\tthis._eventBusComponent = eventBusComponent;\n\t\tthis._verifiableSyncPointerStorageConnector = verifiableSyncPointerStorageConnector;\n\t\tthis._changeSetHelper = changeSetHelper;\n\t\tthis._blobStorageHelper = blobStorageHelper;\n\t\tthis._isTrustedNode = isTrustedNode;\n\t\tthis._maxConsolidations = maxConsolidations;\n\n\t\tthis._batchResponseStorageIds = {};\n\t\tthis._populateFullChanges = {};\n\t}\n\n\t/**\n\t * Set the node identity to use for signing changesets.\n\t * @param nodeId The identity of the node that is performing the update.\n\t */\n\tpublic setNodeId(nodeId: string): void {\n\t\tthis._nodeId = nodeId;\n\t}\n\n\t/**\n\t * Start the remote sync state helper.\n\t */\n\tpublic async start(): Promise<void> {\n\t\tawait this._eventBusComponent.subscribe<ISyncBatchResponse>(\n\t\t\tSynchronisedStorageTopics.BatchResponse,\n\t\t\tasync response => {\n\t\t\t\tawait this.handleBatchResponse(response.data);\n\t\t\t}\n\t\t);\n\n\t\tawait this._eventBusComponent.subscribe<ISyncItemResponse>(\n\t\t\tSynchronisedStorageTopics.LocalItemResponse,\n\t\t\tasync response => {\n\t\t\t\tawait this.handleLocalItemResponse(response.data);\n\t\t\t}\n\t\t);\n\t}\n\n\t/**\n\t * Set the synchronised storage key.\n\t * @param synchronisedStorageKey The synchronised storage key to use.\n\t */\n\tpublic setSynchronisedStorageKey(synchronisedStorageKey: string): void {\n\t\tthis._synchronisedStorageKey = synchronisedStorageKey;\n\t}\n\n\t/**\n\t * Build a changeset.\n\t * @param storageKey The storage key of the change set.\n\t * @param changes The changes to apply.\n\t * @param completeCallback The callback to call when the changeset is created and stored.\n\t * @returns The storage id of the change set if created.\n\t */\n\tpublic async buildChangeSet(\n\t\tstorageKey: string,\n\t\tchanges: ISyncChange[],\n\t\tcompleteCallback: (syncChangeSet?: ISyncChangeSet, id?: string) => Promise<void>\n\t): Promise<void> {\n\t\tawait this._logging?.log({\n\t\t\tlevel: \"info\",\n\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\tmessage: \"buildingChangeSet\",\n\t\t\tdata: {\n\t\t\t\tstorageKey,\n\t\t\t\tchangeCount: changes.length\n\t\t\t}\n\t\t});\n\n\t\tthis._populateFullChanges[storageKey] = {\n\t\t\tchanges,\n\t\t\tentities: {},\n\t\t\trequestIds: [],\n\t\t\tcompleteCallback: async () => this.finaliseFullChanges(storageKey, completeCallback)\n\t\t};\n\n\t\tconst setChanges = changes.filter(c => c.operation === SyncChangeOperation.Set);\n\t\tif (setChanges.length === 0) {\n\t\t\t// If we don't need to request any full details, we can just call the complete callback\n\t\t\tawait this.finaliseFullChanges(storageKey, completeCallback);\n\t\t} else {\n\t\t\t// Otherwise we need to request the full details for each change\n\t\t\tthis._populateFullChanges[storageKey].requestIds = setChanges.map(change => change.id);\n\n\t\t\t// Once all the requests are handled the callback will be called\n\t\t\tfor (const change of setChanges) {\n\t\t\t\t// Create a request for each change to populate the full details\n\t\t\t\tawait this._logging?.log({\n\t\t\t\t\tlevel: \"info\",\n\t\t\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\t\t\tmessage: \"createChangeSetRequestingItem\",\n\t\t\t\t\tdata: {\n\t\t\t\t\t\tstorageKey,\n\t\t\t\t\t\tid: change.id\n\t\t\t\t\t}\n\t\t\t\t});\n\t\t\t\tawait this._eventBusComponent.publish<ISyncItemResponse>(\n\t\t\t\t\tSynchronisedStorageTopics.LocalItemRequest,\n\t\t\t\t\t{\n\t\t\t\t\t\tstorageKey,\n\t\t\t\t\t\tid: change.id\n\t\t\t\t\t}\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Finalise the full details for the sync change set.\n\t * @param storageKey The storage key of the change set.\n\t * @param completeCallback The callback to call when the changeset is populated.\n\t * @returns Nothing.\n\t */\n\tpublic async finaliseFullChanges(\n\t\tstorageKey: string,\n\t\tcompleteCallback: (syncChangeSet?: ISyncChangeSet, id?: string) => Promise<void>\n\t): Promise<void> {\n\t\tawait this._logging?.log({\n\t\t\tlevel: \"info\",\n\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\tmessage: \"finalisingSyncChanges\",\n\t\t\tdata: {\n\t\t\t\tstorageKey\n\t\t\t}\n\t\t});\n\t\tif (Is.stringValue(this._nodeId)) {\n\t\t\tconst changes = this._populateFullChanges[storageKey].changes;\n\t\t\tfor (const change of changes) {\n\t\t\t\tchange.entity = this._populateFullChanges[storageKey].entities[change.id] ?? change.entity;\n\n\t\t\t\tif (change.operation === SyncChangeOperation.Set && Is.objectValue(change.entity)) {\n\t\t\t\t\t// Remove the id from the entity as this is stored in the operation\n\t\t\t\t\t// and will be reinstated when the changeset is reconstituted\n\t\t\t\t\tObjectHelper.propertyDelete(change.entity, \"id\");\n\t\t\t\t\t// Remove the node identity as the changeset has this stored at the top level\n\t\t\t\t\t// and we do not want to store it in the change itself to reduce redundancy\n\t\t\t\t\tObjectHelper.propertyDelete(change.entity, \"nodeId\");\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tconst now = new Date(Date.now()).toISOString();\n\t\t\tconst syncChangeSet: ISyncChangeSet = {\n\t\t\t\t\"@context\": SynchronisedStorageContexts.ContextRoot,\n\t\t\t\ttype: SynchronisedStorageTypes.ChangeSet,\n\t\t\t\tid: Converter.bytesToHex(RandomHelper.generate(32)),\n\t\t\t\tdateCreated: now,\n\t\t\t\tdateModified: now,\n\t\t\t\tstorageKey,\n\t\t\t\tchanges,\n\t\t\t\tnodeId: this._nodeId\n\t\t\t};\n\n\t\t\ttry {\n\t\t\t\t// If this is a trusted node, we also store the changeset\n\t\t\t\tlet changeSetStorageId;\n\t\t\t\tif (this._isTrustedNode) {\n\t\t\t\t\tchangeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);\n\t\t\t\t}\n\n\t\t\t\tawait completeCallback(syncChangeSet, changeSetStorageId);\n\t\t\t} catch (err) {\n\t\t\t\tawait this._logging?.log({\n\t\t\t\t\tlevel: \"error\",\n\t\t\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\t\t\tmessage: \"finalisingSyncChangesFailed\",\n\t\t\t\t\tdata: {\n\t\t\t\t\t\tstorageKey\n\t\t\t\t\t},\n\t\t\t\t\terror: BaseError.fromError(err)\n\t\t\t\t});\n\t\t\t\tawait completeCallback();\n\t\t\t}\n\t\t} else {\n\t\t\tawait completeCallback();\n\t\t}\n\t}\n\n\t/**\n\t * Add a new changeset into the sync state.\n\t * @param storageKey The storage key of the change set to add.\n\t * @param changeSetStorageId The id of the change set to add the current state\n\t * @returns Nothing.\n\t */\n\tpublic async addChangeSetToSyncState(\n\t\tstorageKey: string,\n\t\tchangeSetStorageId: string\n\t): Promise<void> {\n\t\tawait this._logging?.log({\n\t\t\tlevel: \"info\",\n\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\tmessage: \"addChangeSetToSyncState\",\n\t\t\tdata: {\n\t\t\t\tstorageKey,\n\t\t\t\tchangeSetStorageId\n\t\t\t}\n\t\t});\n\n\t\t// First load the sync pointer store to get the current sync pointer for the storage key\n\t\tconst syncPointerStore = await this.getVerifiableSyncPointerStore();\n\n\t\tlet syncState: ISyncState | undefined;\n\t\tif (!Is.empty(syncPointerStore.syncPointers[storageKey])) {\n\t\t\tsyncState = await this.getSyncState(syncPointerStore.syncPointers[storageKey]);\n\t\t}\n\n\t\t// No current sync state, so we create a new one\n\t\tif (Is.empty(syncState)) {\n\t\t\tsyncState = { version: SYNC_STATE_VERSION, storageKey, snapshots: [] };\n\t\t}\n\n\t\t// Sort the snapshots so the newest snapshot is last in the array\n\t\tconst sortedSnapshots = syncState.snapshots.sort((a, b) =>\n\t\t\ta.dateCreated.localeCompare(b.dateCreated)\n\t\t);\n\n\t\t// Get the current snapshot, if it does not exist we create a new one\n\t\tlet currentSnapshot: ISyncSnapshot | undefined = sortedSnapshots[sortedSnapshots.length - 1];\n\t\tconst currentEpoch = currentSnapshot?.epoch ?? 0;\n\t\tconst now = new Date(Date.now()).toISOString();\n\n\t\t// If there is no snapshot or the current one is a consolidation\n\t\t// we start a new snapshot\n\t\tif (Is.empty(currentSnapshot) || currentSnapshot.isConsolidated) {\n\t\t\tcurrentSnapshot = {\n\t\t\t\tversion: SYNC_SNAPSHOT_VERSION,\n\t\t\t\tid: Converter.bytesToHex(RandomHelper.generate(32)),\n\t\t\t\tdateCreated: now,\n\t\t\t\tdateModified: now,\n\t\t\t\tisConsolidated: false,\n\t\t\t\tepoch: currentEpoch + 1,\n\t\t\t\tchangeSetStorageIds: []\n\t\t\t};\n\t\t\tsyncState.snapshots.push(currentSnapshot);\n\t\t} else {\n\t\t\t// Snapshot exists, we update the dateModified\n\t\t\tcurrentSnapshot.dateModified = now;\n\t\t}\n\n\t\t// Add the changeset storage id to the current snapshot\n\t\tcurrentSnapshot.changeSetStorageIds.push(changeSetStorageId);\n\n\t\t// Store the sync state in the blob storage\n\t\tsyncPointerStore.syncPointers[storageKey] = await this.storeRemoteSyncState(syncState);\n\n\t\t// Store the verifiable sync pointer store in the verifiable storage\n\t\tawait this.storeVerifiableSyncPointerStore(syncPointerStore);\n\t}\n\n\t/**\n\t * Create a consolidated snapshot for the entire storage.\n\t * @param storageKey The storage key of the snapshot to create.\n\t * @param batchSize The batch size to use for consolidation.\n\t * @returns Nothing.\n\t */\n\tpublic async consolidationStart(storageKey: string, batchSize: number): Promise<void> {\n\t\tawait this._logging?.log({\n\t\t\tlevel: \"info\",\n\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\tmessage: \"consolidationStarting\"\n\t\t});\n\n\t\t// Perform a batch request to start the consolidation\n\t\tawait this._eventBusComponent.publish<ISyncBatchRequest>(\n\t\t\tSynchronisedStorageTopics.BatchRequest,\n\t\t\t{ storageKey, batchSize, requestMode: SyncNodeIdMode.All }\n\t\t);\n\t}\n\n\t/**\n\t * Get the sync pointer store.\n\t * @returns The sync pointer store.\n\t */\n\tpublic async getVerifiableSyncPointerStore(): Promise<ISyncPointerStore> {\n\t\tif (Is.stringValue(this._synchronisedStorageKey)) {\n\t\t\ttry {\n\t\t\t\tawait this._logging?.log({\n\t\t\t\t\tlevel: \"info\",\n\t\t\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\t\t\tmessage: \"verifiableSyncPointerStoreRetrieving\",\n\t\t\t\t\tdata: {\n\t\t\t\t\t\tkey: this._synchronisedStorageKey\n\t\t\t\t\t}\n\t\t\t\t});\n\t\t\t\tconst syncPointerStore = await this._verifiableSyncPointerStorageConnector.get(\n\t\t\t\t\tthis._synchronisedStorageKey,\n\t\t\t\t\t{ includeData: true }\n\t\t\t\t);\n\t\t\t\tif (Is.uint8Array(syncPointerStore.data)) {\n\t\t\t\t\tconst syncPointer = ObjectHelper.fromBytes<ISyncPointerStore>(syncPointerStore.data);\n\t\t\t\t\tawait this._logging?.log({\n\t\t\t\t\t\tlevel: \"info\",\n\t\t\t\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\t\t\t\tmessage: \"verifiableSyncPointerStoreRetrieved\",\n\t\t\t\t\t\tdata: {\n\t\t\t\t\t\t\tkey: this._synchronisedStorageKey\n\t\t\t\t\t\t}\n\t\t\t\t\t});\n\t\t\t\t\treturn syncPointer;\n\t\t\t\t}\n\t\t\t} catch (err) {\n\t\t\t\tif (!BaseError.someErrorName(err, NotFoundError.CLASS_NAME)) {\n\t\t\t\t\tthrow err;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tawait this._logging?.log({\n\t\t\t\tlevel: \"info\",\n\t\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\t\tmessage: \"verifiableSyncPointerStoreNotFound\",\n\t\t\t\tdata: {\n\t\t\t\t\tkey: this._synchronisedStorageKey\n\t\t\t\t}\n\t\t\t});\n\t\t}\n\n\t\t// If no sync pointer store exists, we return an empty one\n\t\treturn {\n\t\t\tversion: SYNC_POINTER_STORE_VERSION,\n\t\t\tsyncPointers: {}\n\t\t};\n\t}\n\n\t/**\n\t * Store the verifiable sync pointer in the verifiable storage.\n\t * @param syncPointerStore The sync pointer store to store.\n\t * @returns Nothing.\n\t */\n\tpublic async storeVerifiableSyncPointerStore(syncPointerStore: ISyncPointerStore): Promise<void> {\n\t\tif (Is.stringValue(this._nodeId) && Is.stringValue(this._synchronisedStorageKey)) {\n\t\t\tawait this._logging?.log({\n\t\t\t\tlevel: \"info\",\n\t\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\t\tmessage: \"verifiableSyncPointerStoreStoring\",\n\t\t\t\tdata: {\n\t\t\t\t\tkey: this._synchronisedStorageKey\n\t\t\t\t}\n\t\t\t});\n\n\t\t\t// Store the verifiable sync pointer in the verifiable storage\n\t\t\tawait this._verifiableSyncPointerStorageConnector.update(\n\t\t\t\tthis._nodeId,\n\t\t\t\tthis._synchronisedStorageKey,\n\t\t\t\tObjectHelper.toBytes<ISyncPointerStore>(syncPointerStore)\n\t\t\t);\n\t\t}\n\t}\n\n\t/**\n\t * Store the remote sync state.\n\t * @param syncState The sync state to store.\n\t * @returns The id of the sync state.\n\t */\n\tpublic async storeRemoteSyncState(syncState: ISyncState): Promise<string> {\n\t\tawait this._logging?.log({\n\t\t\tlevel: \"info\",\n\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\tmessage: \"syncStateStoring\",\n\t\t\tdata: {\n\t\t\t\tsnapshotCount: syncState.snapshots.length\n\t\t\t}\n\t\t});\n\n\t\t// Limits the number of consolidations in the list so that we can shrink decentralised\n\t\t// storage requirements, sort from newest to oldest so that we can easily find the\n\t\t// oldest snapshots to remove.\n\t\tconst snapshots = syncState.snapshots.sort(\n\t\t\t(a, b) => new Date(a.dateCreated).getTime() - new Date(b.dateCreated).getTime()\n\t\t);\n\n\t\t// Find all the consolidation indexes\n\t\tconst consolidationIndexes = [];\n\t\tfor (let i = 0; i < snapshots.length; i++) {\n\t\t\tconst snapshot = snapshots[i];\n\t\t\tif (snapshot.isConsolidated) {\n\t\t\t\tconsolidationIndexes.push(i);\n\t\t\t}\n\t\t}\n\n\t\tif (consolidationIndexes.length > this._maxConsolidations) {\n\t\t\t// Once we have reached the max for consolidations we need to remove\n\t\t\t// all the snapshots, including non consolidated ones, beyond this point\n\t\t\tconst toRemove = snapshots.slice(consolidationIndexes[this._maxConsolidations - 1] + 1);\n\n\t\t\tsyncState.snapshots = snapshots.slice(\n\t\t\t\t0,\n\t\t\t\tconsolidationIndexes[this._maxConsolidations - 1] + 1\n\t\t\t);\n\n\t\t\tfor (const snapshot of toRemove) {\n\t\t\t\t// We need to remove all the storage ids associated with the snapshot\n\t\t\t\tif (Is.arrayValue(snapshot.changeSetStorageIds)) {\n\t\t\t\t\tfor (const storageId of snapshot.changeSetStorageIds) {\n\t\t\t\t\t\tawait this._blobStorageHelper.removeBlob(storageId);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn this._blobStorageHelper.saveBlob(syncState);\n\t}\n\n\t/**\n\t * Get the remote sync state.\n\t * @param syncPointerId The id of the sync pointer to retrieve the state for.\n\t * @returns The remote sync state.\n\t */\n\tpublic async getSyncState(syncPointerId: string): Promise<ISyncState | undefined> {\n\t\ttry {\n\t\t\tawait this._logging?.log({\n\t\t\t\tlevel: \"info\",\n\t\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\t\tmessage: \"syncStateRetrieving\",\n\t\t\t\tdata: {\n\t\t\t\t\tsyncPointerId\n\t\t\t\t}\n\t\t\t});\n\t\t\tconst syncState = await this._blobStorageHelper.loadBlob<ISyncState>(syncPointerId);\n\n\t\t\tif (Is.object(syncState)) {\n\t\t\t\tawait this._logging?.log({\n\t\t\t\t\tlevel: \"info\",\n\t\t\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\t\t\tmessage: \"syncStateRetrieved\",\n\t\t\t\t\tdata: {\n\t\t\t\t\t\tsyncPointerId,\n\t\t\t\t\t\tsnapshotCount: syncState.snapshots.length\n\t\t\t\t\t}\n\t\t\t\t});\n\t\t\t\treturn syncState;\n\t\t\t}\n\t\t} catch (error) {\n\t\t\tawait this._logging?.log({\n\t\t\t\tlevel: \"warn\",\n\t\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\t\tmessage: \"getSyncStateError\",\n\t\t\t\tdata: {\n\t\t\t\t\tsyncPointerId\n\t\t\t\t},\n\t\t\t\terror: BaseError.fromError(error)\n\t\t\t});\n\t\t}\n\n\t\tawait this._logging?.log({\n\t\t\tlevel: \"info\",\n\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\tmessage: \"syncStateNotFound\",\n\t\t\tdata: {\n\t\t\t\tsyncPointerId\n\t\t\t}\n\t\t});\n\t}\n\n\t/**\n\t * Handle the batch response which is triggered from a consolidation request.\n\t * @param response The batch response to handle.\n\t */\n\tprivate async handleBatchResponse(response: ISyncBatchResponse): Promise<void> {\n\t\tif (Is.stringValue(this._nodeId)) {\n\t\t\tconst now = new Date(Date.now()).toISOString();\n\n\t\t\t// Create a new snapshot entry for the current batch\n\t\t\tconst syncChangeSet: ISyncChangeSet = {\n\t\t\t\t\"@context\": SynchronisedStorageContexts.ContextRoot,\n\t\t\t\ttype: SynchronisedStorageTypes.ChangeSet,\n\t\t\t\tid: Converter.bytesToHex(RandomHelper.generate(32)),\n\t\t\t\tdateCreated: now,\n\t\t\t\tdateModified: now,\n\t\t\t\tchanges: response.entities.map(change => ({\n\t\t\t\t\toperation: SyncChangeOperation.Set,\n\t\t\t\t\tid: change.id\n\t\t\t\t})),\n\t\t\t\tstorageKey: response.storageKey,\n\t\t\t\tnodeId: this._nodeId\n\t\t\t};\n\n\t\t\t// Store the changeset in the blob storage\n\t\t\tconst changeSetStorageId = await this._changeSetHelper.storeChangeSet(syncChangeSet);\n\n\t\t\t// Add the changeset storage id to the snapshot ids\n\t\t\tthis._batchResponseStorageIds[response.storageKey] ??= [];\n\t\t\tthis._batchResponseStorageIds[response.storageKey].push(changeSetStorageId);\n\n\t\t\t// If this is the last entry in the batch response, we can create the consolidated snapshot\n\t\t\tif (response.lastEntry) {\n\t\t\t\t// Get the current sync pointer store\n\t\t\t\tconst syncPointerStore = await this.getVerifiableSyncPointerStore();\n\n\t\t\t\tlet syncState: ISyncState | undefined;\n\n\t\t\t\tif (Is.stringValue(syncPointerStore.syncPointers[response.storageKey])) {\n\t\t\t\t\t// If the sync pointer exists, we load the current sync state\n\t\t\t\t\tsyncState = await this.getSyncState(syncPointerStore.syncPointers[response.storageKey]);\n\t\t\t\t}\n\n\t\t\t\t// If the sync state does not exist, we create a new one\n\t\t\t\tsyncState ??= {\n\t\t\t\t\tversion: SYNC_STATE_VERSION,\n\t\t\t\t\tstorageKey: response.storageKey,\n\t\t\t\t\tsnapshots: []\n\t\t\t\t};\n\n\t\t\t\t// Sort the snapshots so the newest snapshot is last in the array\n\t\t\t\tconst sortedSnapshots = syncState.snapshots.sort((a, b) =>\n\t\t\t\t\ta.dateCreated.localeCompare(b.dateCreated)\n\t\t\t\t);\n\t\t\t\tconst currentSnapshot: ISyncSnapshot | undefined =\n\t\t\t\t\tsortedSnapshots[sortedSnapshots.length - 1];\n\t\t\t\tconst currentEpoch = currentSnapshot?.epoch ?? 0;\n\n\t\t\t\tconst batchSnapshot: ISyncSnapshot = {\n\t\t\t\t\tversion: SYNC_SNAPSHOT_VERSION,\n\t\t\t\t\tid: Converter.bytesToHex(RandomHelper.generate(32)),\n\t\t\t\t\tdateCreated: now,\n\t\t\t\t\tdateModified: now,\n\t\t\t\t\tisConsolidated: true,\n\t\t\t\t\tepoch: currentEpoch + 1,\n\t\t\t\t\tchangeSetStorageIds: this._batchResponseStorageIds[response.storageKey]\n\t\t\t\t};\n\t\t\t\tsyncState.snapshots.push(batchSnapshot);\n\n\t\t\t\t// Store the updated sync state\n\t\t\t\tconst syncStateId = await this.storeRemoteSyncState(syncState);\n\n\t\t\t\tsyncPointerStore.syncPointers[response.storageKey] = syncStateId;\n\n\t\t\t\t// Store the verifiable sync pointer in the verifiable storage\n\t\t\t\tawait this.storeVerifiableSyncPointerStore(syncPointerStore);\n\n\t\t\t\t// Remove the batch response storage ids for the storage key\n\t\t\t\t// as we have consolidated the changes\n\t\t\t\tdelete this._batchResponseStorageIds[response.storageKey];\n\n\t\t\t\tawait this._logging?.log({\n\t\t\t\t\tlevel: \"info\",\n\t\t\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\t\t\tmessage: \"consolidationCompleted\"\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Handle the item response.\n\t * @param response The item response to handle.\n\t */\n\tprivate async handleLocalItemResponse(response: ISyncItemResponse): Promise<void> {\n\t\tawait this._logging?.log({\n\t\t\tlevel: \"info\",\n\t\t\tsource: RemoteSyncStateHelper.CLASS_NAME,\n\t\t\tmessage: \"createChangeSetRespondingItem\",\n\t\t\tdata: {\n\t\t\t\tstorageKey: response.storageKey,\n\t\t\t\tid: response.id\n\t\t\t}\n\t\t});\n\t\t// We have received a response to an item request, find the right storage\n\t\t// for the request id\n\t\tif (!Is.empty(this._populateFullChanges[response.storageKey])) {\n\t\t\tconst idx = this._populateFullChanges[response.storageKey].requestIds.indexOf(response.id);\n\n\t\t\tif (idx !== -1) {\n\t\t\t\tthis._populateFullChanges[response.storageKey].requestIds.splice(idx, 1);\n\t\t\t\tthis._populateFullChanges[response.storageKey].entities[response.id] = response.entity;\n\n\t\t\t\t// If there are no request ids remaining we can complete the population\n\t\t\t\tif (this._populateFullChanges[response.storageKey].requestIds.length === 0) {\n\t\t\t\t\tawait this._populateFullChanges[response.storageKey].completeCallback();\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n"]}

package/dist/es/helpers/versions.js

@@ -0,0 +1,6 @@
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+export const SYNC_STATE_VERSION = "1";
+export const SYNC_POINTER_STORE_VERSION = "1";
+export const SYNC_SNAPSHOT_VERSION = "1";
+//# sourceMappingURL=versions.js.map

package/dist/es/helpers/versions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"versions.js","sourceRoot":"","sources":["../../../src/helpers/versions.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,MAAM,CAAC,MAAM,kBAAkB,GAAW,GAAG,CAAC;AAC9C,MAAM,CAAC,MAAM,0BAA0B,GAAW,GAAG,CAAC;AACtD,MAAM,CAAC,MAAM,qBAAqB,GAAW,GAAG,CAAC","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nexport const SYNC_STATE_VERSION: string = \"1\";\nexport const SYNC_POINTER_STORE_VERSION: string = \"1\";\nexport const SYNC_SNAPSHOT_VERSION: string = \"1\";\n"]}