@webex/plugin-meetings 3.12.0-mobius-socket.2 → 3.12.0-mobius-socket.4

package/src/hashTree/hashTreeParser.ts

@@ -1,11 +1,13 @@
 import {cloneDeep, isEmpty, zip} from 'lodash';
 import HashTree, {LeafDataItem} from './hashTree';
 import LoggerProxy from '../common/logs/logger-proxy';
+import Metrics from '../metrics';
+import BEHAVIORAL_METRICS from '../metrics/constants';
 import {Enum, HTTP_VERBS} from '../constants';
-import {DataSetNames, EMPTY_HASH} from './constants';
+import {DataSetNames, DATA_SET_INIT_PRIORITY, EMPTY_HASH} from './constants';
 import {ObjectType, HtMeta, HashTreeObject} from './types';
-import {LocusDTO} from '../locus-info/types';
-import {deleteNestedObjectsWithHtMeta, isMetadata} from './utils';
+import {LocusDTO, LocusErrorCodes} from '../locus-info/types';
+import {deleteNestedObjectsWithHtMeta, isMetadata, sortByInitPriority} from './utils';
 
 export interface DataSet {
   url: string;
@@ -54,13 +56,24 @@ type WebexRequestMethod = (options: Record<string, any>) => Promise<any>;
 export const LocusInfoUpdateType = {
   OBJECTS_UPDATED: 'OBJECTS_UPDATED',
   MEETING_ENDED: 'MEETING_ENDED',
+  LOCUS_NOT_FOUND: 'LOCUS_NOT_FOUND',
 } as const;
 
 export type LocusInfoUpdateType = Enum<typeof LocusInfoUpdateType>;
-export type LocusInfoUpdateCallback = (
-  updateType: LocusInfoUpdateType,
-  data?: {updatedObjects: HashTreeObject[]}
-) => void;
+
+interface LocusUpdatePayloads {
+  [LocusInfoUpdateType.OBJECTS_UPDATED]: {updatedObjects: HashTreeObject[]};
+  [LocusInfoUpdateType.MEETING_ENDED]: unknown; // No extra data
+  [LocusInfoUpdateType.LOCUS_NOT_FOUND]: unknown; // No extra data
+}
+
+export type LocusInfoUpdate = {
+  [K in keyof LocusUpdatePayloads]: {
+    updateType: K;
+  } & LocusUpdatePayloads[K];
+}[keyof LocusUpdatePayloads];
+
+export type LocusInfoUpdateCallback = (update: LocusInfoUpdate) => void;
 
 interface LeafInfo {
   type: ObjectType;
@@ -75,6 +88,13 @@ interface LeafInfo {
  */
 export class MeetingEndedError extends Error {}
 
+/**
+ * This error is thrown when a 404 is received from Locus hash tree endpoints, indicating that the locus URL
+ * is no longer valid (e.g. participant moved to a breakout room, or meeting ended).
+ * It's handled internally by HashTreeParser and results in LOCUS_NOT_FOUND being sent up.
+ */
+export class LocusNotFoundError extends Error {}
+
 /* Currently Locus always sends Metadata objects only in the "self" dataset.
  * If this ever changes, update all the code that relies on this constant.
  */
@@ -99,6 +119,10 @@ class HashTreeParser {
   heartbeatIntervalMs?: number;
   private excludedDataSets: string[];
   state: 'active' | 'stopped';
+  private syncQueue: Array<{dataSetName: string; reason: string; isInitialization?: boolean}> = [];
+  private isSyncInProgress = false;
+  private isSyncAllInProgress = false;
+  private syncQueueProcessingPromise: Promise<void> = Promise.resolve();
 
   /**
    * Constructor for HashTreeParser
@@ -224,16 +248,16 @@ class HashTreeParser {
    * @param {DataSet} dataSetInfo The new data set to be added
    * @returns {Promise}
    */
-  private initializeNewVisibleDataSet(
+  private async initializeNewVisibleDataSet(
     visibleDataSetInfo: VisibleDataSetInfo,
     dataSetInfo: DataSet
-  ): Promise<{updateType: LocusInfoUpdateType; updatedObjects?: HashTreeObject[]}> {
+  ): Promise<void> {
     if (this.isVisibleDataSet(dataSetInfo.name)) {
       LoggerProxy.logger.info(
         `HashTreeParser#initializeNewVisibleDataSet --> ${this.debugId} Data set "${dataSetInfo.name}" already exists, skipping init`
       );
 
-      return Promise.resolve({updateType: LocusInfoUpdateType.OBJECTS_UPDATED, updatedObjects: []});
+      return;
     }
 
     LoggerProxy.logger.info(
@@ -241,7 +265,7 @@ class HashTreeParser {
     );
 
     if (!this.addToVisibleDataSetsList(visibleDataSetInfo)) {
-      return Promise.resolve({updateType: LocusInfoUpdateType.OBJECTS_UPDATED, updatedObjects: []});
+      return;
     }
 
     const hashTree = new HashTree([], dataSetInfo.leafCount);
@@ -251,51 +275,8 @@ class HashTreeParser {
       hashTree,
     };
 
-    return this.sendInitializationSyncRequestToLocus(dataSetInfo.name, 'new visible data set');
-  }
-
-  /**
-   * Sends a special sync request to Locus with all leaves empty - this is a way to get all the data for a given dataset.
-   *
-   * @param {string} datasetName - name of the dataset for which to send the request
-   * @param {string} debugText - text to include in logs
-   * @returns {Promise}
-   */
-  private sendInitializationSyncRequestToLocus(
-    datasetName: string,
-    debugText: string
-  ): Promise<{updateType: LocusInfoUpdateType; updatedObjects?: HashTreeObject[]}> {
-    const dataset = this.dataSets[datasetName];
-
-    if (!dataset) {
-      LoggerProxy.logger.warn(
-        `HashTreeParser#sendInitializationSyncRequestToLocus --> ${this.debugId} No data set found for ${datasetName}, cannot send the request for leaf data`
-      );
-
-      return Promise.resolve(null);
-    }
-
-    const emptyLeavesData = new Array(dataset.leafCount).fill([]);
-
-    LoggerProxy.logger.info(
-      `HashTreeParser#sendInitializationSyncRequestToLocus --> ${this.debugId} Sending initial sync request to Locus for data set "${datasetName}" with empty leaf data`
-    );
-
-    return this.sendSyncRequestToLocus(this.dataSets[datasetName], emptyLeavesData).then(
-      (syncResponse) => {
-        if (syncResponse) {
-          return {
-            updateType: LocusInfoUpdateType.OBJECTS_UPDATED,
-            updatedObjects: this.parseMessage(
-              syncResponse,
-              `via empty leaves /sync API call for ${debugText}`
-            ),
-          };
-        }
-
-        return {updateType: LocusInfoUpdateType.OBJECTS_UPDATED, updatedObjects: []};
-      }
-    );
+    this.enqueueSyncForDataset(dataSetInfo.name, 'new visible data set initialization', true);
+    await this.syncQueueProcessingPromise;
   }
 
   /**
@@ -382,9 +363,8 @@ class HashTreeParser {
     if (this.state === 'stopped') {
       return;
     }
-    const updatedObjects: HashTreeObject[] = [];
 
-    for (const dataSet of visibleDataSets) {
+    for (const dataSet of sortByInitPriority(visibleDataSets, DATA_SET_INIT_PRIORITY)) {
       const {name, leafCount, url} = dataSet;
 
       if (!this.dataSets[name]) {
@@ -420,19 +400,12 @@ class HashTreeParser {
         );
         this.dataSets[name].hashTree = new HashTree([], leafCount);
 
-        // eslint-disable-next-line no-await-in-loop
-        const data = await this.sendInitializationSyncRequestToLocus(name, debugText);
-
-        if (data.updateType === LocusInfoUpdateType.OBJECTS_UPDATED) {
-          updatedObjects.push(...(data.updatedObjects || []));
-        }
+        this.enqueueSyncForDataset(name, `initialization sync for ${debugText}`, true);
       }
     }
 
-    this.callLocusInfoUpdateCallback({
-      updateType: LocusInfoUpdateType.OBJECTS_UPDATED,
-      updatedObjects,
-    });
+    // wait for all enqueued initialization syncs to complete
+    await this.syncQueueProcessingPromise;
   }
 
   /**
@@ -573,25 +546,38 @@ class HashTreeParser {
   private handleRootHashHeartBeatMessage(message: RootHashMessage): void {
     const {dataSets} = message;
 
-    LoggerProxy.logger.info(
-      `HashTreeParser#handleRootHashMessage --> ${
-        this.debugId
-      } Received heartbeat root hash message with data sets: ${JSON.stringify(
-        dataSets.map(({name, root, leafCount, version}) => ({
-          name,
-          root,
-          leafCount,
-          version,
-        }))
-      )}`
-    );
-
     dataSets.forEach((dataSet) => {
       this.updateDataSetInfo(dataSet);
       this.runSyncAlgorithm(dataSet);
     });
   }
 
+  /**
+   * Handles known errors that can happen during syncs
+   *
+   * @param {any} error - The error to handle
+   * @returns {boolean} true if the error was recognized and handled, false otherwise
+   */
+  private handleSyncErrors(error: any) {
+    if (error instanceof MeetingEndedError) {
+      this.callLocusInfoUpdateCallback({
+        updateType: LocusInfoUpdateType.MEETING_ENDED,
+      });
+
+      return true;
+    }
+    if (error instanceof LocusNotFoundError) {
+      this.callLocusInfoUpdateCallback({
+        updateType: LocusInfoUpdateType.LOCUS_NOT_FOUND,
+      });
+      this.stop();
+
+      return true;
+    }
+
+    return false;
+  }
+
   /**
    * Asynchronously initializes new visible data sets
    *
@@ -608,11 +594,7 @@ class HashTreeParser {
     );
     queueMicrotask(() => {
       this.initializeNewVisibleDataSets(dataSetsRequiringInitialization).catch((error) => {
-        if (error instanceof MeetingEndedError) {
-          this.callLocusInfoUpdateCallback({
-            updateType: LocusInfoUpdateType.MEETING_ENDED,
-          });
-        } else {
+        if (!this.handleSyncErrors(error)) {
           LoggerProxy.logger.warn(
             `HashTreeParser#queueInitForNewVisibleDataSets --> ${
               this.debugId
@@ -690,7 +672,14 @@ class HashTreeParser {
 
     const {dataSets, locus, metadata} = update;
 
+    LoggerProxy.logger.info(
+      `HashTreeParser#handleLocusUpdate --> ${this.debugId} received update with dataSets=${dataSets
+        ?.map((ds) => ds.name)
+        .join(',')} metadata=${metadata ? 'yes' : 'no'}`
+    );
+
     if (!dataSets) {
+      // this happens for example when we handle GET /loci response
       LoggerProxy.logger.info(
         `HashTreeParser#handleLocusUpdate --> ${this.debugId} received hash tree update without dataSets`
       );
@@ -794,6 +783,18 @@ class HashTreeParser {
     }
   }
 
+  /**
+   * Updates the leaf count for a data set, resizing its hash tree accordingly.
+   *
+   * @param {InternalDataSet} dataSet - The data set to update
+   * @param {number} newLeafCount - The new leaf count
+   * @returns {void}
+   */
+  private updateDataSetLeafCount(dataSet: InternalDataSet, newLeafCount: number): void {
+    dataSet.hashTree?.resize(newLeafCount);
+    dataSet.leafCount = newLeafCount;
+  }
+
   /**
    * Checks for changes in the visible data sets based on the updated objects.
    * @param {HashTreeObject[]} updatedObjects - The list of updated hash tree objects.
@@ -960,7 +961,7 @@ class HashTreeParser {
     }
     const allDataSets = await this.getAllVisibleDataSetsFromLocus();
 
-    for (const ds of addedDataSets) {
+    for (const ds of sortByInitPriority(addedDataSets, DATA_SET_INIT_PRIORITY)) {
       const dataSetInfo = allDataSets.find((d) => d.name === ds.name);
 
       LoggerProxy.logger.info(
@@ -972,12 +973,8 @@ class HashTreeParser {
           `HashTreeParser#initializeNewVisibleDataSets --> ${this.debugId} missing info about data set "${ds.name}" in Locus response from visibleDataSetsUrl`
         );
       } else {
-        // we're awaiting in a loop, because in practice there will be only one new data set at a time,
-        // so no point in trying to parallelize this
         // eslint-disable-next-line no-await-in-loop
-        const updates = await this.initializeNewVisibleDataSet(ds, dataSetInfo);
-
-        this.callLocusInfoUpdateCallback(updates);
+        await this.initializeNewVisibleDataSet(ds, dataSetInfo);
       }
     }
   }
@@ -997,14 +994,27 @@ class HashTreeParser {
     const {dataSets, visibleDataSetsUrl} = message;
 
     LoggerProxy.logger.info(
-      `HashTreeParser#parseMessage --> ${this.debugId} received message ${debugText || ''}:`,
-      message
+      `HashTreeParser#parseMessage --> ${this.debugId} ${
+        debugText || ''
+      } dataSets: ${message.dataSets
+        ?.map(({name, version}) => `${name}:${version}`)
+        .join(',')}, elements: ${message.locusStateElements
+        ?.map(
+          (el) =>
+            `${el.htMeta.elementId.type}:${el.htMeta.elementId.id}:${el.htMeta.elementId.version}${
+              el.data ? '+' : '-'
+            }`
+        )
+        .join(',')}`
     );
+
     if (message.locusStateElements?.length === 0) {
       LoggerProxy.logger.warn(
         `HashTreeParser#parseMessage --> ${this.debugId} got empty locusStateElements!!!`
       );
-      // todo: send a metric
+      Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.HASH_TREE_EMPTY_LOCUS_STATE_ELEMENTS, {
+        debugId: this.debugId,
+      });
     }
 
     // first, update our metadata about the datasets with info from the message
@@ -1015,7 +1025,7 @@ class HashTreeParser {
 
     // when we detect new visible datasets, it may be that the metadata about them is not
     // available in the message, they will require separate async initialization
-    let dataSetsRequiringInitialization = [];
+    let dataSetsRequiringInitialization: VisibleDataSetInfo[] = [];
 
     // first find out if there are any visible data set changes - they're signalled in Metadata object updates
     const metadataUpdates = (message.locusStateElements || []).filter((object) =>
@@ -1023,7 +1033,7 @@ class HashTreeParser {
     );
 
     if (metadataUpdates.length > 0) {
-      const updatedMetadataObjects = [];
+      const updatedMetadataObjects: HashTreeObject[] = [];
 
       metadataUpdates.forEach((object) => {
         // todo: once Locus supports it, we will use the "view" field here instead of dataSetNames
@@ -1052,7 +1062,7 @@ class HashTreeParser {
       }
     }
 
-    if (message.locusStateElements?.length > 0) {
+    if (message.locusStateElements && message.locusStateElements.length > 0) {
       // by this point we now have this.dataSets setup for data sets from this message
       // and hash trees created for the new visible data sets,
       // so we can now process all the updates from the message
@@ -1148,20 +1158,17 @@ class HashTreeParser {
    * @param {Object} updates parsed from a Locus message
    * @returns {void}
    */
-  private callLocusInfoUpdateCallback(updates: {
-    updateType: LocusInfoUpdateType;
-    updatedObjects?: HashTreeObject[];
-  }) {
+  private callLocusInfoUpdateCallback(updates: LocusInfoUpdate) {
     if (this.state === 'stopped') {
       return;
     }
 
-    const {updateType, updatedObjects} = updates;
+    const {updateType} = updates;
 
-    if (updateType === LocusInfoUpdateType.OBJECTS_UPDATED && updatedObjects?.length > 0) {
+    if (updateType === LocusInfoUpdateType.OBJECTS_UPDATED && updates.updatedObjects?.length > 0) {
       // Filter out updates for objects that already have a higher version in their datasets,
       // or removals for objects that still exist in any of their datasets
-      const filteredUpdates = updatedObjects.filter((object) => {
+      const filteredUpdates = updates.updatedObjects.filter((object) => {
         const {elementId} = object.htMeta;
         const {type, id, version} = elementId;
 
@@ -1198,10 +1205,10 @@ class HashTreeParser {
       });
 
       if (filteredUpdates.length > 0) {
-        this.locusInfoUpdateCallback(updateType, {updatedObjects: filteredUpdates});
+        this.locusInfoUpdateCallback({updateType, updatedObjects: filteredUpdates});
       }
     } else if (updateType !== LocusInfoUpdateType.OBJECTS_UPDATED) {
-      this.locusInfoUpdateCallback(updateType, {updatedObjects});
+      this.locusInfoUpdateCallback({updateType});
     }
   }
 
@@ -1223,76 +1230,89 @@ class HashTreeParser {
    * Performs a sync for the given data set.
    *
    * @param {InternalDataSet} dataSet - The data set to sync
-   * @param {string} rootHash - Our current root hash for this data set
    * @param {string} reason - The reason for the sync (used for logging)
+   * @param {boolean} [isInitialization] - Whether this is an initialization sync (sends empty leaves data instead of comparing hashes)
    * @returns {Promise<void>}
    */
   private async performSync(
     dataSet: InternalDataSet,
-    rootHash: string,
-    reason: string
+    reason: string,
+    isInitialization?: boolean
   ): Promise<void> {
     if (!dataSet.hashTree) {
       return;
     }
 
+    const {hashTree} = dataSet;
+    const rootHash = hashTree.getRootHash();
+
     try {
       LoggerProxy.logger.info(
         `HashTreeParser#performSync --> ${this.debugId} ${reason}, syncing data set "${dataSet.name}"`
       );
 
-      const mismatchedLeavesData: Record<number, LeafDataItem[]> = {};
+      let leavesData: Record<number, LeafDataItem[]> = {};
 
-      if (dataSet.leafCount !== 1) {
-        let receivedHashes;
+      if (!isInitialization) {
+        if (dataSet.leafCount !== 1) {
+          let receivedHashes;
 
-        try {
-          // request hashes from sender
-          const {hashes, dataSet: latestDataSetInfo} = await this.getHashesFromLocus(
-            dataSet.name,
-            rootHash
-          );
+          try {
+            // request hashes from sender
+            const hashesResult = await this.getHashesFromLocus(dataSet.name, rootHash);
 
-          receivedHashes = hashes;
+            if (!hashesResult) {
+              // hashes match, no sync needed
+              return;
+            }
 
-          dataSet.hashTree.resize(latestDataSetInfo.leafCount);
-        } catch (error) {
-          if (error.statusCode === 409) {
-            // this is a leaf count mismatch, we should do nothing, just wait for another heartbeat message from Locus
-            LoggerProxy.logger.info(
-              `HashTreeParser#getHashesFromLocus --> ${this.debugId} Got 409 when fetching hashes for data set "${dataSet.name}": ${error.message}`
-            );
+            receivedHashes = hashesResult.hashes;
+
+            this.updateDataSetLeafCount(dataSet, hashesResult.dataSet.leafCount);
+          } catch (error: any) {
+            if (error?.statusCode === 409) {
+              // this is a leaf count mismatch, we should do nothing, just wait for another heartbeat message from Locus
+              LoggerProxy.logger.info(
+                `HashTreeParser#getHashesFromLocus --> ${this.debugId} Got 409 when fetching hashes for data set "${dataSet.name}": ${error.message}`
+              );
 
-            return;
+              return;
+            }
+            throw error;
           }
-          throw error;
-        }
 
-        // identify mismatched leaves
-        const mismatchedLeaveIndexes = dataSet.hashTree.diffHashes(receivedHashes);
+          // identify mismatched leaves
+          const mismatchedLeaveIndexes = hashTree.diffHashes(receivedHashes);
 
-        mismatchedLeaveIndexes.forEach((index) => {
-          mismatchedLeavesData[index] = dataSet.hashTree.getLeafData(index);
-        });
-      } else {
-        mismatchedLeavesData[0] = dataSet.hashTree.getLeafData(0);
+          mismatchedLeaveIndexes.forEach((index) => {
+            leavesData[index] = hashTree.getLeafData(index);
+          });
+        } else {
+          leavesData = {0: hashTree.getLeafData(0)};
+        }
       }
       // request sync for mismatched leaves
-      if (Object.keys(mismatchedLeavesData).length > 0) {
-        const syncResponse = await this.sendSyncRequestToLocus(dataSet, mismatchedLeavesData);
+      let syncResponse: HashTreeMessage | null = null;
 
-        // sync API may return nothing (in that case data will arrive via messages)
-        // or it may return a response in the same format as messages
-        if (syncResponse) {
-          this.handleMessage(syncResponse, 'via sync API');
-        }
+      if (isInitialization) {
+        syncResponse = await this.sendSyncRequestToLocus(dataSet, {isInitialization: true});
+      } else if (Object.keys(leavesData).length > 0) {
+        syncResponse = await this.sendSyncRequestToLocus(dataSet, {
+          mismatchedLeavesData: leavesData,
+        });
+      }
+
+      // sync API may return nothing (in that case data will arrive via messages)
+      // or it may return a response in the same format as messages
+      // We still need to restart the sync timer as a safety net in case the messages don't arrive.
+      this.runSyncAlgorithm(dataSet);
+
+      if (syncResponse) {
+        // the format of sync response is the same as messages, so we can reuse the same handler
+        this.handleMessage(syncResponse, 'via sync API');
       }
     } catch (error) {
-      if (error instanceof MeetingEndedError) {
-        this.callLocusInfoUpdateCallback({
-          updateType: LocusInfoUpdateType.MEETING_ENDED,
-        });
-      } else {
+      if (!this.handleSyncErrors(error)) {
         LoggerProxy.logger.warn(
           `HashTreeParser#performSync --> ${this.debugId} error during sync for data set "${dataSet.name}":`,
           error
@@ -1301,6 +1321,105 @@ class HashTreeParser {
     }
   }
 
+  /**
+   * Enqueues a sync for the given data set. If the data set is already in the queue, the request is ignored.
+   * This ensures that all syncs are executed sequentially and no more than 1 sync runs at a time.
+   *
+   * @param {string} dataSetName - The name of the data set to sync
+   * @param {string} reason - The reason for the sync (used for logging)
+   * @param {boolean} [isInitialization=false] - Whether this is an initialization sync (uses empty leaves data instead of hash comparison)
+   * @returns {void}
+   */
+  private enqueueSyncForDataset(
+    dataSetName: string,
+    reason: string,
+    isInitialization = false
+  ): void {
+    if (this.state === 'stopped') return;
+
+    const existingEntry = this.syncQueue.find((entry) => entry.dataSetName === dataSetName);
+
+    if (existingEntry) {
+      if (isInitialization) {
+        existingEntry.isInitialization = true;
+      }
+      LoggerProxy.logger.info(
+        `HashTreeParser#enqueueSyncForDataset --> ${this.debugId} data set "${dataSetName}" already in sync queue, skipping`
+      );
+
+      return;
+    }
+
+    this.syncQueue.push({dataSetName, reason, isInitialization});
+
+    if (!this.isSyncInProgress) {
+      this.syncQueueProcessingPromise = this.processSyncQueue();
+    }
+  }
+
+  /**
+   * Processes the sync queue sequentially. Only one instance of this method runs at a time.
+   *
+   * @returns {Promise<void>}
+   */
+  private async processSyncQueue(): Promise<void> {
+    if (this.isSyncInProgress) return;
+
+    this.isSyncInProgress = true;
+    try {
+      while (this.syncQueue.length > 0 && this.state !== 'stopped') {
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const {dataSetName, reason, isInitialization} = this.syncQueue.shift()!;
+        const dataSet = this.dataSets[dataSetName];
+
+        if (!dataSet?.hashTree) {
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        // eslint-disable-next-line no-await-in-loop
+        await this.performSync(dataSet, reason, isInitialization);
+      }
+    } finally {
+      this.isSyncInProgress = false;
+    }
+  }
+
+  /**
+   * Syncs all data sets that have hash trees, one by one in sequence, using the priority order
+   * provided by sortByInitPriority(). Does nothing if the parser is stopped or if a syncAllDatasets
+   * call is already in progress.
+   *
+   * @returns {Promise<void>}
+   */
+  public async syncAllDatasets(): Promise<void> {
+    if (this.state === 'stopped') return;
+    if (this.isSyncAllInProgress) return;
+
+    this.isSyncAllInProgress = true;
+    try {
+      const dataSetsWithHashTrees = Object.values(this.dataSets)
+        .filter((dataSet) => dataSet?.hashTree)
+        .map((dataSet) => ({name: dataSet.name}));
+
+      const sorted = sortByInitPriority(dataSetsWithHashTrees, DATA_SET_INIT_PRIORITY);
+
+      LoggerProxy.logger.info(
+        `HashTreeParser#syncAllDatasets --> ${this.debugId} syncing datasets: ${sorted
+          .map((ds) => ds.name)
+          .join(', ')}`
+      );
+
+      for (const ds of sorted) {
+        this.enqueueSyncForDataset(ds.name, 'syncAllDatasets');
+      }
+
+      await this.syncQueueProcessingPromise;
+    } finally {
+      this.isSyncAllInProgress = false;
+    }
+  }
+
   /**
    * Runs the sync algorithm for the given data set.
    *
@@ -1319,21 +1438,14 @@ class HashTreeParser {
     }
 
     if (!dataSet.hashTree) {
-      LoggerProxy.logger.info(
-        `HashTreeParser#runSyncAlgorithm --> ${this.debugId} Data set "${dataSet.name}" has no hash tree, skipping sync algorithm`
-      );
+      // no hash tree, so no need to do any syncing
+      // we fall into this branch often, because Locus sends dataSets in messages that are not visible to us
 
       return;
     }
 
     dataSet.hashTree.resize(receivedDataSet.leafCount);
 
-    // temporary log for the workshop // todo: remove
-    const ourCurrentRootHash = dataSet.hashTree.getRootHash();
-    LoggerProxy.logger.info(
-      `HashTreeParser#runSyncAlgorithm --> ${this.debugId} dataSet="${dataSet.name}" version=${dataSet.version} hashes before starting timer: ours=${ourCurrentRootHash} Locus=${dataSet.root}`
-    );
-
     const delay = dataSet.idleMs + this.getWeightedBackoffTime(dataSet.backoff);
 
     if (delay > 0) {
@@ -1341,11 +1453,7 @@ class HashTreeParser {
         clearTimeout(dataSet.timer);
       }
 
-      LoggerProxy.logger.info(
-        `HashTreeParser#runSyncAlgorithm --> ${this.debugId} setting "${dataSet.name}" sync timer for ${delay}`
-      );
-
-      dataSet.timer = setTimeout(async () => {
+      dataSet.timer = setTimeout(() => {
         dataSet.timer = undefined;
 
         if (!dataSet.hashTree) {
@@ -1359,15 +1467,10 @@ class HashTreeParser {
         const rootHash = dataSet.hashTree.getRootHash();
 
         if (dataSet.root !== rootHash) {
-          await this.performSync(
-            dataSet,
-            rootHash,
+          this.enqueueSyncForDataset(
+            dataSet.name,
             `Root hash mismatch: received=${dataSet.root}, ours=${rootHash}`
           );
-        } else {
-          LoggerProxy.logger.info(
-            `HashTreeParser#runSyncAlgorithm --> ${this.debugId} "${dataSet.name}" root hash matching: ${rootHash}, version=${dataSet.version}`
-          );
         }
       }, delay);
     } else {
@@ -1407,18 +1510,20 @@ class HashTreeParser {
       const backoffTime = this.getWeightedBackoffTime(dataSet.backoff);
       const delay = this.heartbeatIntervalMs + backoffTime;
 
-      dataSet.heartbeatWatchdogTimer = setTimeout(async () => {
+      dataSet.heartbeatWatchdogTimer = setTimeout(() => {
         dataSet.heartbeatWatchdogTimer = undefined;
 
         LoggerProxy.logger.warn(
           `HashTreeParser#resetHeartbeatWatchdogs --> ${this.debugId} Heartbeat watchdog fired for data set "${dataSet.name}" - no heartbeat received within expected interval, initiating sync`
         );
 
-        await this.performSync(
-          dataSet,
-          dataSet.hashTree.getRootHash(),
-          `heartbeat watchdog expired`
-        );
+        Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.HASH_TREE_HEARTBEAT_WATCHDOG_EXPIRED, {
+          debugId: this.debugId,
+          dataSetName: dataSet.name,
+        });
+
+        this.enqueueSyncForDataset(dataSet.name, `heartbeat watchdog expired`);
+        this.resetHeartbeatWatchdogs([dataSet]);
       }, delay);
     }
   }
@@ -1451,6 +1556,7 @@ class HashTreeParser {
       `HashTreeParser#stop --> ${this.debugId} Stopping HashTreeParser, clearing timers and hash trees`
     );
     this.stopAllTimers();
+    this.syncQueue = [];
     Object.values(this.dataSets).forEach((dataSet) => {
       dataSet.hashTree = undefined;
     });
@@ -1459,17 +1565,27 @@ class HashTreeParser {
   }
 
   /**
-   * Resumes the HashTreeParser that was previously stopped.
+   * Cleans up the HashTreeParser, stopping all timers and clearing all internal state.
+   * After calling this, the parser should not be used anymore.
+   * @returns {void}
+   */
+  public cleanUp() {
+    this.stop();
+    this.dataSets = {};
+  }
+
+  /**
+   * Resumes the HashTreeParser that was previously stopped, using a hash tree message.
    * @param {HashTreeMessage} message - The message to resume with, it must contain metadata with visible data sets info
    * @returns {void}
    */
-  public resume(message: HashTreeMessage) {
+  public resumeFromMessage(message: HashTreeMessage) {
     // check that message contains metadata with visible data sets - this is essential to be able to resume
     const metadataObject = message.locusStateElements?.find((el) => isMetadata(el));
 
     if (!metadataObject?.data?.visibleDataSets) {
       LoggerProxy.logger.warn(
-        `HashTreeParser#resume --> ${this.debugId} Cannot resume HashTreeParser because the message is missing metadata with visible data sets info`
+        `HashTreeParser#resumeFromMessage --> ${this.debugId} Cannot resume HashTreeParser because the message is missing metadata with visible data sets info`
       );
 
       return;
@@ -1490,7 +1606,7 @@ class HashTreeParser {
       };
     }
     LoggerProxy.logger.info(
-      `HashTreeParser#resume --> ${
+      `HashTreeParser#resumeFromMessage --> ${
         this.debugId
       } Resuming HashTreeParser with data sets: ${Object.keys(this.dataSets).join(
         ', '
@@ -1501,18 +1617,47 @@ class HashTreeParser {
     this.handleMessage(message, 'on resume');
   }
 
+  /**
+   * Resumes the HashTreeParser that was previously stopped, using a Locus API response.
+   * Unlike resumeFromMessage(), this does not require metadata/dataSets in the input,
+   * as it fetches all necessary information from Locus via initializeFromGetLociResponse.
+   * @param {LocusDTO} locus - locus object from an API response
+   * @returns {Promise}
+   */
+  public async resumeFromApiResponse(locus: LocusDTO) {
+    this.state = 'active';
+    this.dataSets = {};
+
+    LoggerProxy.logger.info(
+      `HashTreeParser#resumeFromApiResponse --> ${this.debugId} Resuming HashTreeParser from API response`
+    );
+
+    await this.initializeFromGetLociResponse(locus);
+  }
+
   private checkForSentinelHttpResponse(error: any, dataSetName?: string) {
+    // 404 for any dataset means the locus is no longer available at this URL - could be replaced or ended
+    // if a dataset is just not visible, we would get a 400
+    if (error.statusCode === 404) {
+      LoggerProxy.logger.info(
+        `HashTreeParser#checkForSentinelHttpResponse --> ${this.debugId} Received 404 for data set "${dataSetName}", locus not found`
+      );
+      this.stopAllTimers();
+
+      throw new LocusNotFoundError();
+    }
+
     const isValidDataSetForSentinel =
       dataSetName === undefined ||
       PossibleSentinelMessageDataSetNames.includes(dataSetName.toLowerCase());
 
     if (
-      ((error.statusCode === 409 && error.body?.errorCode === 2403004) ||
-        error.statusCode === 404) &&
+      error.statusCode === 409 &&
+      error.body?.errorCode === LocusErrorCodes.LOCUS_INACTIVE &&
       isValidDataSetForSentinel
     ) {
       LoggerProxy.logger.info(
-        `HashTreeParser#checkForSentinelHttpResponse --> ${this.debugId} Received ${error.statusCode} for data set "${dataSetName}", indicating that the meeting has ended`
+        `HashTreeParser#checkForSentinelHttpResponse --> ${this.debugId} Received ${error.statusCode}/${error.body?.errorCode} for data set "${dataSetName}", indicating that the meeting has ended`
       );
       this.stopAllTimers();
 
@@ -1524,7 +1669,7 @@ class HashTreeParser {
    * Gets the current hashes from the locus for a specific data set.
    * @param {string} dataSetName
    * @param {string} currentRootHash
-   * @returns {string[]}
+   * @returns {Object|null} An object containing the hashes and leaf count, or null if the hashes match and no sync is needed
    */
   private getHashesFromLocus(dataSetName: string, currentRootHash: string) {
     LoggerProxy.logger.info(
@@ -1543,6 +1688,15 @@ class HashTreeParser {
       },
     })
       .then((response) => {
+        if (!response.body || isEmpty(response.body)) {
+          // 204 with empty body means our hashes match Locus, no sync needed
+          LoggerProxy.logger.info(
+            `HashTreeParser#getHashesFromLocus --> ${this.debugId} Got ${response.statusCode} with empty body for data set "${dataSetName}", hashes match - no sync needed`
+          );
+
+          return null;
+        }
+
         const hashes = response.body?.hashes as string[] | undefined;
         const dataSetFromResponse = response.body?.dataSet;
 
@@ -1571,6 +1725,13 @@ class HashTreeParser {
           error
         );
         this.checkForSentinelHttpResponse(error, dataSet.name);
+        Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.HASH_TREE_SYNC_FAILURE, {
+          debugId: this.debugId,
+          dataSetName,
+          request: 'GET /hashtree',
+          statusCode: error.statusCode,
+          reason: error.message,
+        });
 
         throw error;
       });
@@ -1580,29 +1741,43 @@ class HashTreeParser {
    * Sends a sync request to Locus for the specified data set.
    *
    * @param {InternalDataSet} dataSet The data set to sync.
-   * @param {Record<number, LeafDataItem[]>} mismatchedLeavesData The mismatched leaves data to include in the sync request.
+   * @param {Object} options Either `{ isInitialization: true }` for init syncs (uses leafCount=1 with empty leaf data) or `{ mismatchedLeavesData }` for normal syncs.
    * @returns {Promise<HashTreeMessage|null>}
    */
   private sendSyncRequestToLocus(
     dataSet: InternalDataSet,
-    mismatchedLeavesData: Record<number, LeafDataItem[]>
+    options: {isInitialization: true} | {mismatchedLeavesData: Record<number, LeafDataItem[]>}
   ): Promise<HashTreeMessage | null> {
     LoggerProxy.logger.info(
       `HashTreeParser#sendSyncRequestToLocus --> ${this.debugId} Sending sync request for data set "${dataSet.name}"`
     );
 
+    const isInitialization = 'isInitialization' in options;
+
     const url = `${dataSet.url}/sync`;
-    const body = {
-      leafCount: dataSet.leafCount,
+    const body: {
+      leafCount: number;
+      leafDataEntries: {leafIndex: number; elementIds: LeafDataItem[]}[];
+    } = {
+      leafCount: isInitialization ? 1 : dataSet.leafCount,
       leafDataEntries: [],
     };
 
-    Object.keys(mismatchedLeavesData).forEach((index) => {
-      body.leafDataEntries.push({
-        leafIndex: parseInt(index, 10),
-        elementIds: mismatchedLeavesData[index],
+    if (isInitialization) {
+      // initialization sync: Locus requires leafCount=1 with a single empty leaf
+      body.leafDataEntries.push({leafIndex: 0, elementIds: []});
+    } else {
+      const {mismatchedLeavesData} = options;
+
+      Object.keys(mismatchedLeavesData).forEach((index) => {
+        const leafIndex = parseInt(index, 10);
+
+        body.leafDataEntries.push({
+          leafIndex,
+          elementIds: mismatchedLeavesData[leafIndex],
+        });
       });
-    });
+    }
 
     const ourCurrentRootHash = dataSet.hashTree ? dataSet.hashTree.getRootHash() : EMPTY_HASH;
 
@@ -1615,10 +1790,6 @@ class HashTreeParser {
       body,
     })
       .then((resp) => {
-        LoggerProxy.logger.info(
-          `HashTreeParser#sendSyncRequestToLocus --> ${this.debugId} Sync request succeeded for "${dataSet.name}"`
-        );
-
         if (!resp.body || isEmpty(resp.body)) {
           LoggerProxy.logger.info(
             `HashTreeParser#sendSyncRequestToLocus --> ${this.debugId} Got ${resp.statusCode} with empty body for sync request for data set "${dataSet.name}", data should arrive via messages`
@@ -1635,6 +1806,13 @@ class HashTreeParser {
           error
         );
         this.checkForSentinelHttpResponse(error, dataSet.name);
+        Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.HASH_TREE_SYNC_FAILURE, {
+          debugId: this.debugId,
+          dataSetName: dataSet.name,
+          request: 'POST /sync',
+          statusCode: error.statusCode,
+          reason: error.message,
+        });
 
         throw error;
       });