@twin.org/synchronised-storage-service 0.0.1-next.9 → 0.0.3-next.2

package/dist/types/helpers/changeSetHelper.d.ts

@@ -1,81 +1,65 @@
 import type { IEventBusComponent } from "@twin.org/event-bus-models";
-import { type IIdentityConnector } from "@twin.org/identity-models";
 import type { ILoggingComponent } from "@twin.org/logging-models";
-import { type IProof } from "@twin.org/standards-w3c-did";
-import { type ISyncChangeSet, type ISynchronisedEntity, type SyncNodeIdentityMode } from "@twin.org/synchronised-storage-models";
-import type { BlobStorageHelper } from "./blobStorageHelper";
+import { type ISyncChangeSet, type SyncNodeIdMode } from "@twin.org/synchronised-storage-models";
+import type { BlobStorageHelper } from "./blobStorageHelper.js";
 /**
  * Class for performing change set operations.
  */
-export declare class ChangeSetHelper<T extends ISynchronisedEntity = ISynchronisedEntity> {
+export declare class ChangeSetHelper {
     /**
      * Runtime name for the class.
      */
-    readonly CLASS_NAME: string;
+    static readonly CLASS_NAME: string;
     /**
      * Create a new instance of ChangeSetHelper.
-     * @param loggingComponent The logging connector to use for logging.
+     * @param logging The logging component to use for logging.
      * @param eventBusComponent The event bus component to use for events.
-     * @param identityConnector The identity connector to use for signing/verifying changesets.
      * @param blobStorageHelper The blob storage component to use for remote sync states.
-     * @param decentralisedStorageMethodId The id of the identity method to use when signing/verifying changesets.
      */
-    constructor(loggingComponent: ILoggingComponent | undefined, eventBusComponent: IEventBusComponent, identityConnector: IIdentityConnector, blobStorageHelper: BlobStorageHelper, decentralisedStorageMethodId: string);
+    constructor(logging: ILoggingComponent | undefined, eventBusComponent: IEventBusComponent, blobStorageHelper: BlobStorageHelper);
     /**
      * Set the node identity to use for signing changesets.
-     * @param nodeIdentity The identity of the node that is performing the update.
+     * @param nodeId The identity of the node that is performing the update.
      */
-    setNodeIdentity(nodeIdentity: string): void;
+    setNodeId(nodeId: string): void;
     /**
-     * Get and verify a changeset.
+     * Get a changeset.
      * @param changeSetStorageId The id of the sync changeset to apply.
      * @returns The changeset if it was verified.
      */
-    getAndVerifyChangeset(changeSetStorageId: string): Promise<ISyncChangeSet<T> | undefined>;
+    getChangeset(changeSetStorageId: string): Promise<ISyncChangeSet | undefined>;
     /**
      * Apply a sync changeset.
      * @param changeSetStorageId The id of the sync changeset to apply.
      * @returns The changeset if it existed.
      */
-    getAndApplyChangeset(changeSetStorageId: string): Promise<ISyncChangeSet<T> | undefined>;
+    getAndApplyChangeset(changeSetStorageId: string): Promise<ISyncChangeSet | undefined>;
     /**
      * Apply a sync changeset.
      * @param syncChangeset The sync changeset to apply.
      * @returns Nothing.
      */
-    applyChangeset(syncChangeset: ISyncChangeSet<T>): Promise<void>;
+    applyChangeset(syncChangeset: ISyncChangeSet): Promise<void>;
     /**
      * Store the changeset.
      * @param syncChangeSet The sync change set to store.
      * @returns The id of the change set.
      */
     storeChangeSet(syncChangeSet: ISyncChangeSet): Promise<string>;
-    /**
-     * Verify the proof of a sync changeset.
-     * @param syncChangeset The sync changeset to verify.
-     * @returns True if the proof is valid, false otherwise.
-     */
-    verifyChangesetProof(syncChangeset: ISyncChangeSet): Promise<boolean>;
-    /**
-     * Create the proof of a sync change set.
-     * @param syncChangeset The sync changeset to create the proof for.
-     * @returns The proof.
-     */
-    createChangeSetProof(syncChangeset: ISyncChangeSet): Promise<IProof>;
     /**
      * Copy a change set.
      * @param syncChangeSet The sync changeset to copy.
      * @returns The id of the updated change set.
      */
-    copyChangeset(syncChangeSet: ISyncChangeSet<T>): Promise<{
-        syncChangeSet: ISyncChangeSet<T>;
+    copyChangeset(syncChangeSet: ISyncChangeSet): Promise<{
+        syncChangeSet: ISyncChangeSet;
         changeSetStorageId: string;
     } | undefined>;
     /**
      * Reset the storage for a given storage key.
      * @param storageKey The key of the storage to reset.
-     * @param resetMode The reset mode, this will use the nodeIdentity in the entities to determine which are local/remote.
+     * @param resetMode The reset mode, this will use the nodeId in the entities to determine which are local/remote.
      * @returns Nothing.
      */
-    reset(storageKey: string, resetMode: SyncNodeIdentityMode): Promise<void>;
+    reset(storageKey: string, resetMode: SyncNodeIdMode): Promise<void>;
 }

package/dist/types/helpers/localSyncStateHelper.d.ts

@@ -1,24 +1,24 @@
 import type { IEntityStorageConnector } from "@twin.org/entity-storage-models";
 import type { ILoggingComponent } from "@twin.org/logging-models";
-import { type ISynchronisedEntity, type SyncChangeOperation } from "@twin.org/synchronised-storage-models";
-import type { ChangeSetHelper } from "./changeSetHelper";
-import type { SyncSnapshotEntry } from "../entities/syncSnapshotEntry";
-import type { ISyncState } from "../models/ISyncState";
+import { type SyncChangeOperation } from "@twin.org/synchronised-storage-models";
+import type { ChangeSetHelper } from "./changeSetHelper.js";
+import type { SyncSnapshotEntry } from "../entities/syncSnapshotEntry.js";
+import type { ISyncState } from "../models/ISyncState.js";
 /**
  * Class for performing entity storage operations in decentralised storage.
  */
-export declare class LocalSyncStateHelper<T extends ISynchronisedEntity = ISynchronisedEntity> {
+export declare class LocalSyncStateHelper {
     /**
      * Runtime name for the class.
      */
-    readonly CLASS_NAME: string;
+    static readonly CLASS_NAME: string;
     /**
      * Create a new instance of LocalSyncStateHelper.
-     * @param loggingComponent The logging connector to use for logging.
+     * @param logging The logging component to use for logging.
      * @param snapshotEntryEntityStorage The storage connector for the sync snapshot entries.
      * @param changeSetHelper The change set helper to use for applying changesets.
      */
-    constructor(loggingComponent: ILoggingComponent | undefined, snapshotEntryEntityStorage: IEntityStorageConnector<SyncSnapshotEntry<T>>, changeSetHelper: ChangeSetHelper<T>);
+    constructor(logging: ILoggingComponent | undefined, snapshotEntryEntityStorage: IEntityStorageConnector<SyncSnapshotEntry>, changeSetHelper: ChangeSetHelper);
     /**
      * Add a new change to the local snapshot.
      * @param storageKey The storage key of the snapshot to add the change for.
@@ -33,19 +33,19 @@ export declare class LocalSyncStateHelper<T extends ISynchronisedEntity = ISynch
      * @param isLocal Whether to get the local snapshot or not.
      * @returns The local snapshot entry.
      */
-    getSnapshots(storageKey: string, isLocal: boolean): Promise<SyncSnapshotEntry<T>[]>;
+    getSnapshots(storageKey: string, isLocal: boolean): Promise<SyncSnapshotEntry[]>;
     /**
      * Set the current local snapshot with changes for this node.
      * @param localChangeSnapshot The local change snapshot to set.
      * @returns Nothing.
      */
-    setLocalChangeSnapshot(localChangeSnapshot: SyncSnapshotEntry<T>): Promise<void>;
+    setLocalChangeSnapshot(localChangeSnapshot: SyncSnapshotEntry): Promise<void>;
     /**
      * Get the current local snapshot with the changes for this node.
      * @param localChangeSnapshot The local change snapshot to remove.
      * @returns Nothing.
      */
-    removeLocalChangeSnapshot(localChangeSnapshot: SyncSnapshotEntry<T>): Promise<void>;
+    removeLocalChangeSnapshot(localChangeSnapshot: SyncSnapshotEntry): Promise<void>;
     /**
      * Apply a sync state to the local node.
      * @param storageKey The storage key of the snapshot to sync with.

package/dist/types/helpers/remoteSyncStateHelper.d.ts

@@ -1,21 +1,21 @@
 import type { IEventBusComponent } from "@twin.org/event-bus-models";
 import type { ILoggingComponent } from "@twin.org/logging-models";
-import { type ISyncChange, type ISyncChangeSet, type ISynchronisedEntity } from "@twin.org/synchronised-storage-models";
+import { type ISyncChange, type ISyncChangeSet } from "@twin.org/synchronised-storage-models";
 import type { IVerifiableStorageConnector } from "@twin.org/verifiable-storage-models";
-import type { BlobStorageHelper } from "./blobStorageHelper";
-import type { ChangeSetHelper } from "./changeSetHelper";
-import type { ISyncPointerStore } from "../models/ISyncPointerStore";
-import type { ISyncState } from "../models/ISyncState";
+import type { BlobStorageHelper } from "./blobStorageHelper.js";
+import type { ChangeSetHelper } from "./changeSetHelper.js";
+import type { ISyncPointerStore } from "../models/ISyncPointerStore.js";
+import type { ISyncState } from "../models/ISyncState.js";
 /**
  * Class for performing entity storage operations in decentralised storage.
  */
-export declare class RemoteSyncStateHelper<T extends ISynchronisedEntity = ISynchronisedEntity> {
+export declare class RemoteSyncStateHelper {
     /**
      * Runtime name for the class.
      */
-    readonly CLASS_NAME: string;
+    static readonly CLASS_NAME: string;
     /**
-     * Create a new instance of DecentralisedEntityStorageConnector.
+     * Create a new instance of RemoteSyncStateHelper.
      * @param loggingComponent The logging component to use for logging.
      * @param eventBusComponent The event bus component to use for events.
      * @param verifiableSyncPointerStorageConnector The verifiable storage connector to use for storing sync pointers.
@@ -24,12 +24,16 @@ export declare class RemoteSyncStateHelper<T extends ISynchronisedEntity = ISync
      * @param isTrustedNode Whether the node is trusted or not.
      * @param maxConsolidations The maximum number of consolidations to keep in storage.
      */
-    constructor(loggingComponent: ILoggingComponent | undefined, eventBusComponent: IEventBusComponent, verifiableSyncPointerStorageConnector: IVerifiableStorageConnector, blobStorageHelper: BlobStorageHelper, changeSetHelper: ChangeSetHelper<T>, isTrustedNode: boolean, maxConsolidations: number);
+    constructor(loggingComponent: ILoggingComponent | undefined, eventBusComponent: IEventBusComponent, verifiableSyncPointerStorageConnector: IVerifiableStorageConnector, blobStorageHelper: BlobStorageHelper, changeSetHelper: ChangeSetHelper, isTrustedNode: boolean, maxConsolidations: number);
     /**
      * Set the node identity to use for signing changesets.
-     * @param nodeIdentity The identity of the node that is performing the update.
+     * @param nodeId The identity of the node that is performing the update.
      */
-    setNodeIdentity(nodeIdentity: string): void;
+    setNodeId(nodeId: string): void;
+    /**
+     * Start the remote sync state helper.
+     */
+    start(): Promise<void>;
     /**
      * Set the synchronised storage key.
      * @param synchronisedStorageKey The synchronised storage key to use.
@@ -42,18 +46,18 @@ export declare class RemoteSyncStateHelper<T extends ISynchronisedEntity = ISync
      * @param completeCallback The callback to call when the changeset is created and stored.
      * @returns The storage id of the change set if created.
      */
-    buildChangeSet(storageKey: string, changes: ISyncChange<T>[], completeCallback: (syncChangeSet?: ISyncChangeSet<T>, id?: string) => Promise<void>): Promise<void>;
+    buildChangeSet(storageKey: string, changes: ISyncChange[], completeCallback: (syncChangeSet?: ISyncChangeSet, id?: string) => Promise<void>): Promise<void>;
     /**
      * Finalise the full details for the sync change set.
      * @param storageKey The storage key of the change set.
      * @param completeCallback The callback to call when the changeset is populated.
      * @returns Nothing.
      */
-    finaliseFullChanges(storageKey: string, completeCallback: (syncChangeSet?: ISyncChangeSet<T>, id?: string) => Promise<void>): Promise<void>;
+    finaliseFullChanges(storageKey: string, completeCallback: (syncChangeSet?: ISyncChangeSet, id?: string) => Promise<void>): Promise<void>;
     /**
      * Add a new changeset into the sync state.
      * @param storageKey The storage key of the change set to add.
-     * @param changeSetStorageId The id of the change set to add the the current state
+     * @param changeSetStorageId The id of the change set to add the current state
      * @returns Nothing.
      */
     addChangeSetToSyncState(storageKey: string, changeSetStorageId: string): Promise<void>;

package/dist/types/index.d.ts

@@ -1,10 +1,10 @@
-export * from "./entities/syncSnapshotEntry";
-export * from "./models/ISynchronisedStorageServiceConfig";
-export * from "./models/ISynchronisedStorageServiceConstructorOptions";
-export * from "./models/ISyncPointerStore";
-export * from "./models/ISyncSnapshot";
-export * from "./models/ISyncState";
-export * from "./restEntryPoints";
-export * from "./schema";
-export * from "./synchronisedStorageRoutes";
-export * from "./synchronisedStorageService";
+export * from "./entities/syncSnapshotEntry.js";
+export * from "./models/ISynchronisedStorageServiceConfig.js";
+export * from "./models/ISynchronisedStorageServiceConstructorOptions.js";
+export * from "./models/ISyncPointerStore.js";
+export * from "./models/ISyncSnapshot.js";
+export * from "./models/ISyncState.js";
+export * from "./restEntryPoints.js";
+export * from "./schema.js";
+export * from "./synchronisedStorageRoutes.js";
+export * from "./synchronisedStorageService.js";

package/dist/types/models/ISyncState.d.ts

@@ -1,4 +1,4 @@
-import type { ISyncSnapshot } from "./ISyncSnapshot";
+import type { ISyncSnapshot } from "./ISyncSnapshot.js";
 /**
  * The object definition for a sync state.
  */

package/dist/types/models/ISynchronisedStorageServiceConfig.d.ts

@@ -2,28 +2,23 @@
  * Configuration for the Synchronised Storage Service.
  */
 export interface ISynchronisedStorageServiceConfig {
-    /**
-     * The id of the identity method to use when signing/verifying requests and changesets.
-     * @default synchronised-storage-assertion
-     */
-    synchronisedStorageMethodId?: string;
     /**
      * How often to check for entity updates in minutes.
      * @default 5
      */
     entityUpdateIntervalMinutes?: number;
     /**
-     * Interval to perform consolidation of changesets, only used if isTrustedNode is set.
+     * Interval to perform consolidation of changesets, only used if this is a trusted node.
      * @default 60
      */
     consolidationIntervalMinutes?: number;
     /**
-     * The number of entities to process in a single consolidation batch.
+     * The number of entities to process in a single consolidation batch, only used if this is a trusted node.
      * @default 1000
      */
     consolidationBatchSize?: number;
     /**
-     * The maximum number of consolidations to keep in storage.
+     * The maximum number of consolidations to keep in storage, only used if this is a trusted node.
      * @default 5
      */
     maxConsolidations?: number;
@@ -38,4 +33,8 @@ export interface ISynchronisedStorageServiceConfig {
      * @default local
      */
     verifiableStorageKeyId: "mainnet" | "testnet" | "devnet" | string;
+    /**
+     * Override the default trust generator.
+     */
+    overrideTrustGeneratorType?: string;
 }

package/dist/types/models/ISynchronisedStorageServiceConstructorOptions.d.ts

@@ -1,4 +1,4 @@
-import type { ISynchronisedStorageServiceConfig } from "./ISynchronisedStorageServiceConfig";
+import type { ISynchronisedStorageServiceConfig } from "./ISynchronisedStorageServiceConfig.js";
 /**
  * Options for the Synchronised Storage Service constructor.
  */
@@ -30,16 +30,16 @@ export interface ISynchronisedStorageServiceConstructorOptions {
      * @default verifiable-storage
      */
     verifiableStorageConnectorType?: string;
-    /**
-     * The identity connector.
-     * @default identity
-     */
-    identityConnectorType?: string;
     /**
      * The task scheduler component.
      * @default task-scheduler
      */
     taskSchedulerComponentType?: string;
+    /**
+     * The type of the trust component.
+     * @default trust
+     */
+    trustComponentType?: string;
     /**
      * The synchronised entity storage component type to use if this node is not trusted.
      * If this is set, this node uses it as the trusted node to store changesets.

package/dist/types/synchronisedStorageRoutes.d.ts

@@ -1,5 +1,5 @@
 import type { IHttpRequestContext, INoContentResponse, IRestRoute, ITag } from "@twin.org/api-models";
-import type { ISyncChangeSetRequest, ISyncDecryptionKeyRequest, ISyncDecryptionKeyResponse } from "@twin.org/synchronised-storage-models";
+import { type ISyncChangeSetRequest, type ISyncDecryptionKeyRequest, type ISyncDecryptionKeyResponse } from "@twin.org/synchronised-storage-models";
 /**
  * The tag to associate with the routes.
  */

package/dist/types/synchronisedStorageService.d.ts

@@ -1,51 +1,47 @@
-import { type IProof } from "@twin.org/standards-w3c-did";
-import { type ISyncChangeSet, type ISynchronisedEntity, type ISynchronisedStorageComponent } from "@twin.org/synchronised-storage-models";
-import type { ISynchronisedStorageServiceConstructorOptions } from "./models/ISynchronisedStorageServiceConstructorOptions";
+import { type ISyncChangeSet, type ISynchronisedStorageComponent } from "@twin.org/synchronised-storage-models";
+import type { ISynchronisedStorageServiceConstructorOptions } from "./models/ISynchronisedStorageServiceConstructorOptions.js";
 /**
  * Class for performing synchronised storage operations.
  */
-export declare class SynchronisedStorageService<T extends ISynchronisedEntity = ISynchronisedEntity> implements ISynchronisedStorageComponent {
+export declare class SynchronisedStorageService implements ISynchronisedStorageComponent {
     /**
      * Runtime name for the class.
      */
-    readonly CLASS_NAME: string;
+    static readonly CLASS_NAME: string;
     /**
      * Create a new instance of SynchronisedStorageService.
      * @param options The options for the service.
      */
     constructor(options: ISynchronisedStorageServiceConstructorOptions);
+    /**
+     * Returns the class name of the component.
+     * @returns The class name of the component.
+     */
+    className(): string;
     /**
      * The component needs to be started when the node is initialized.
-     * @param nodeIdentity The identity of the node starting the component.
-     * @param nodeLoggingConnectorType The node logging connector type, defaults to "node-logging".
-     * @param componentState A persistent state which can be modified by the method.
+     * @param nodeLoggingComponentType The node logging component type.
      * @returns Nothing.
      */
-    start(nodeIdentity: string, nodeLoggingConnectorType: string | undefined, componentState?: {
-        [id: string]: unknown;
-    }): Promise<void>;
+    start(nodeLoggingComponentType?: string): Promise<void>;
     /**
      * The component needs to be stopped when the node is closed.
-     * @param nodeIdentity The identity of the node stopping the component.
-     * @param nodeLoggingConnectorType The node logging connector type, defaults to "node-logging".
-     * @param componentState A persistent state which can be modified by the method.
+     * @param nodeLoggingComponentType The node logging component type.
      * @returns Nothing.
      */
-    stop(nodeIdentity: string, nodeLoggingConnectorType: string | undefined, componentState?: {
-        [id: string]: unknown;
-    }): Promise<void>;
+    stop(nodeLoggingComponentType?: string): Promise<void>;
     /**
      * Get the decryption key for the synchronised storage.
      * This is used to decrypt the data stored in the synchronised storage.
-     * @param nodeIdentity The identity of the node requesting the decryption key.
-     * @param proof The proof of the request so we know the request is from the specified node.
+     * @param trustPayload Trust payload to verify the requesters identity.
      * @returns The decryption key.
      */
-    getDecryptionKey(nodeIdentity: string, proof: IProof): Promise<string>;
+    getDecryptionKey(trustPayload: unknown): Promise<string>;
     /**
      * Synchronise a set of changes from an untrusted node, assumes this is a trusted node.
      * @param syncChangeSet The change set to synchronise.
+     * @param trustPayload Trust payload to verify the requesters identity.
      * @returns Nothing.
      */
-    syncChangeSet(syncChangeSet: ISyncChangeSet<T>): Promise<void>;
+    syncChangeSet(syncChangeSet: ISyncChangeSet, trustPayload: unknown): Promise<void>;
 }

package/docs/architecture.md

@@ -62,7 +62,6 @@ The Synchronised Storage Service implements a distributed, eventually-consistent
                     ┌───────────▼────────────┐
                     │  Verifiable Storage    │
                     │   • On-chain Pointers  │
-                    │   • State Root Hashes  │
                     │   • Access Control     │
                     └───────────┬────────────┘
                                 │
@@ -70,7 +69,6 @@ The Synchronised Storage Service implements a distributed, eventually-consistent
                     │ Decentralised Storage  │
                     │   • IPFS/Content Hash  │
                     │   • Immutable Blobs    │
-                    │   • Distributed Refs   │
                     └────────────────────────┘
 ```
 
@@ -89,10 +87,9 @@ The Synchronised Storage Service implements a distributed, eventually-consistent
 
 - **Multi-Node Architecture**: Multiple trusted nodes operate in parallel to distribute processing load and provide high availability
 - **Change Validation**: Verify cryptographic signatures and node permissions across the cluster
-- **Change Set Consolidation**: Periodic compaction of historical change logs distributed across trusted node instances
-- **Decentralised Storage Interface**: Upload/download operations to IPFS-like systems with load balancing
+- **Change Set Consolidation**: Periodic compaction of changes to create complete data sets
+- **Decentralised Storage Interface**: Upload/download operations to e.g. IPFS
 - **Verifiable Storage Updates**: Atomic updates to on-chain state pointers with consensus agreement
-- **Load Distribution**: Automatic distribution of regular node connections and change processing across available trusted nodes
 
 ## Data Flow Architecture
 
@@ -105,24 +102,183 @@ EntityStorage → ChangeCapture → LocalSnapshot → EventBus
 ### 2. Change Set Propagation
 
 ```typescript
-LocalNode → [Sign] → ChangeSet → TrustedNode → [Validation] → GlobalState
+RegularNode → [Sign] → ChangeSet → TrustedNode → [Verify] → GlobalState
 ```
 
 ### 3. Global State Persistence
 
 ```typescript
-TrustedNode → [Sign & Compression] → DecentralisedStorage → [StateRoot] → VerifiableStorage
+TrustedNode → [Encrypt & Compress] → DecentralisedStorage → [StateRoot] → VerifiableStorage
 ```
 
 ### 4. Remote Synchronisation
 
 ```typescript
-AnyNode → [Poll] → VerifiableStorage → [Fetch] → DecentralisedStorage → [Merge] → LocalState
+AnyNode → [Poll] → VerifiableStorage → [Fetch] → DecentralisedStorage → [Decrypt & Decompress] → LocalState
 ```
 
+## Synchronisation Algorithms
+
+### Apply Sync State Logic
+
+The `applySyncState` method in `LocalSyncStateHelper` implements the core synchronisation algorithm that reconciles remote sync state with local state. This algorithm handles both full and incremental synchronisation scenarios:
+
+#### Algorithm Overview
+
+**Input Processing:**
+
+- Retrieves existing local snapshots and remote sync state snapshots
+- Sorts both collections by creation date (newest to oldest) for temporal ordering
+- Extracts epoch information for gap detection analysis
+
+**Epoch Gap Detection:**
+
+If the node has not been running for a while or has failed communications it might start to miss data, so we use gap detection to determine if we need to use a full consolidation snapshot.
+
+```typescript
+const newestExistingEpoch = existingSnapshots[0]?.epoch ?? 0;
+const oldestSyncStateEpoch = syncStateSnapshots[syncStateSnapshots.length - 1]?.epoch ?? 0;
+const hasEpochGap = newestExistingEpoch + 1 < oldestSyncStateEpoch;
+```
+
+**Synchronisation Strategy Decision:**
+
+1. **Full Synchronisation Trigger:**
+   - No existing consolidated snapshots locally, OR
+   - Detected epoch gap between local and remote state
+
+2. **Incremental Synchronisation Trigger:**
+   - Existing consolidated snapshots present AND
+   - No epoch gaps detected
+
+#### Full Synchronisation Process
+
+When full sync is required:
+
+1. **Consolidation Search:** Locates the most recent consolidated snapshot in remote state
+2. **Entity Storage Reset:** Clears all remote entities from local storage to ensure clean state, but retains all local entries maintained by this node
+3. **Progressive Application:** Processes snapshots from the consolidation point forward to the newest, ensuring newer changes override older ones
+4. **Change Set Processing:** Applies all change sets from the consolidation and subsequent modifications
+
+#### Incremental Synchronisation Process
+
+When incremental sync is sufficient:
+
+1. **Snapshot Mapping:** Creates lookup map of existing local snapshots for efficient comparison
+2. **Change Detection:** Categorises remote snapshots into:
+   - **New Snapshots:** Not present locally
+   - **Modified Snapshots:** Present but with different `dateModified`
+   - **Unchanged Snapshots:** Identical local and remote versions
+
+3. **Processing Optimisation:** Stops processing when encountering an unchanged snapshot (temporal consistency guarantee)
+
+4. **Change Set Application:**
+   - Processes modified snapshots to apply incremental changes
+   - Processes new snapshots in chronological order (oldest to newest)
+   - Removes unreferenced local snapshots for storage clean-up
+
+#### Key Design Principles
+
+- **Temporal Consistency:** Changes are applied in chronological order to maintain causality
+- **Conflict Resolution:** Newer changes automatically override older ones within the same entity
+- **Storage Efficiency:** Only processes changed or new snapshots, avoiding redundant operations
+- **Data Integrity:** Maintains referential integrity by cleaning up orphaned snapshot references
+- **Gap Recovery:** Automatically triggers full synchronisation when data continuity is compromised
+
+#### Error Handling & Edge Cases
+
+- **Missing Consolidations:** Logs warning when full sync is needed but no consolidated snapshot is available
+- **Empty Remote State:** Handles scenarios where remote sync state contains no snapshots
+- **Epoch Inconsistencies:** Automatically recovers from epoch gaps through full synchronisation
+- **Processing Interruption:** Uses `completedProcessing` flag to optimise incremental sync execution
+
+This algorithm ensures eventually consistent data replication across distributed nodes while minimising network traffic and computational overhead through intelligent synchronisation strategy selection.
+
 ## Data Structures
 
-- Sync Pointer Store - stored in verifiable storage, for each storage type contains a storage id of the location for sync states
-- Sync State - stored in decentralised storage, contains all of the snapshots for a specific storage key
-- Sync Snapshot - contains a list of all the decentralised storage ids for the change sets which make up the snapshot
-- Change Set - contains a set of changes to be made to the entity storage
+### Sync Pointer Store
+
+Stored in verifiable storage, for each storage type contains a storage id of the location for sync states.
+
+```json
+{
+  "syncPointers": {
+    "my-type-1": "blob:ipfs:0x11...1111",
+    "my-type-2": "blob:ipfs:0x22...2222"
+  }
+}
+```
+
+### Sync State
+
+Stored in decentralised storage, contains all of the snapshots for a specific storage key
+
+```json
+{
+    "storageKey": "my-type-1",
+    "snapshots": [
+        {
+            "id": "0xaaa...aaa",
+            ...
+        },
+        {
+            "id": "0xbbb...bbb",
+            ...
+        }
+    ]
+}
+```
+
+### Sync Snapshot
+
+Contains a list of all the storage ids for the change sets which make up the snapshot
+
+```json
+{
+    "id": "0xaaa...aaa",
+    "dateCreated": "2025-05-29T01:00:00.000Z",
+    "dateModified": "2025-05-29T01:00:00.000Z",
+    "isConsolidated": false, // Determines if this contains a complete data set
+    "epoch": 123, // A counter that is incremented for each snapshot
+    "changeSetStorageIds": [
+        "blob:ipfs:0x111...1111",
+        "blob:ipfs:0x111...1112"
+        ...
+    ]
+}
+```
+
+### Sync Change Set
+
+Contains a set of changes to be made to the entity storage, the proof is generated by the node identity which guarantees the validity of the data from the node
+
+```json
+{
+    "id": "0xaaa...aaa",
+    "storageKey": "my-type-1",
+    "dateCreated": "2025-05-29T01:00:00.000Z",
+    "dateModified": "2025-05-29T01:00:00.000Z",
+    "changes": [
+        ...
+    ],
+    "nodeIdentity": "did:iota:0xccc.cccccc",
+    "proof": {
+        // Cryptographic proof signed by the node identity
+    }
+}
+```
+
+### Sync Change
+
+Contains a mutation for an entry in entity storage, either a `set` of `delete`. The `id` and `nodeIdentity` are omitted from the entity and stored at a higher level in the data structures to shrink the data set size.
+
+```json
+{
+  "operation": "set",
+  "id": "my-item-1",
+  "entity": {
+    "foo": true,
+    "bar": 123
+  }
+}
+```