@webex/plugin-meetings 3.12.0-next.7 → 3.12.0-next.71

package/src/hashTree/hashTreeParser.ts

@@ -1,11 +1,19 @@
 import {cloneDeep, isEmpty, zip} from 'lodash';
 import HashTree, {LeafDataItem} from './hashTree';
 import LoggerProxy from '../common/logs/logger-proxy';
+import Metrics from '../metrics';
+import BEHAVIORAL_METRICS from '../metrics/constants';
 import {Enum, HTTP_VERBS} from '../constants';
-import {DataSetNames, EMPTY_HASH} from './constants';
+import {DataSetNames, DATA_SET_INIT_PRIORITY, EMPTY_HASH, LLM_DATASET_NAMES} from './constants';
 import {ObjectType, HtMeta, HashTreeObject} from './types';
-import {LocusDTO} from '../locus-info/types';
-import {deleteNestedObjectsWithHtMeta, isMetadata} from './utils';
+import {LocusDTO, LocusErrorCodes} from '../locus-info/types';
+import {deleteNestedObjectsWithHtMeta, isMetadata, sleep, sortByInitPriority} from './utils';
+
+export enum SyncAllBackoffType {
+  NONE = 'none',
+  ONLY_LLM = 'onlyLLM',
+  ALL = 'all',
+}
 
 export interface DataSet {
   url: string;
@@ -18,10 +26,12 @@ export interface DataSet {
     maxMs: number;
     exponent: number;
   };
+  heartbeatIntervalMs?: number;
 }
 
 export interface RootHashMessage {
   dataSets: Array<DataSet>;
+  heartbeatIntervalMs?: number;
 }
 export interface HashTreeMessage {
   dataSets: Array<DataSet>;
@@ -47,6 +57,7 @@ interface InternalDataSet extends DataSet {
   hashTree?: HashTree; // set only for visible data sets
   timer?: ReturnType<typeof setTimeout>;
   heartbeatWatchdogTimer?: ReturnType<typeof setTimeout>;
+  syncAbortController?: AbortController;
 }
 
 type WebexRequestMethod = (options: Record<string, any>) => Promise<any>;
@@ -54,13 +65,24 @@ type WebexRequestMethod = (options: Record<string, any>) => Promise<any>;
 export const LocusInfoUpdateType = {
   OBJECTS_UPDATED: 'OBJECTS_UPDATED',
   MEETING_ENDED: 'MEETING_ENDED',
+  LOCUS_NOT_FOUND: 'LOCUS_NOT_FOUND',
 } as const;
 
 export type LocusInfoUpdateType = Enum<typeof LocusInfoUpdateType>;
-export type LocusInfoUpdateCallback = (
-  updateType: LocusInfoUpdateType,
-  data?: {updatedObjects: HashTreeObject[]}
-) => void;
+
+interface LocusUpdatePayloads {
+  [LocusInfoUpdateType.OBJECTS_UPDATED]: {updatedObjects: HashTreeObject[]};
+  [LocusInfoUpdateType.MEETING_ENDED]: unknown; // No extra data
+  [LocusInfoUpdateType.LOCUS_NOT_FOUND]: unknown; // No extra data
+}
+
+export type LocusInfoUpdate = {
+  [K in keyof LocusUpdatePayloads]: {
+    updateType: K;
+  } & LocusUpdatePayloads[K];
+}[keyof LocusUpdatePayloads];
+
+export type LocusInfoUpdateCallback = (update: LocusInfoUpdate) => void;
 
 interface LeafInfo {
   type: ObjectType;
@@ -75,6 +97,13 @@ interface LeafInfo {
  */
 export class MeetingEndedError extends Error {}
 
+/**
+ * This error is thrown when a 404 is received from Locus hash tree endpoints, indicating that the locus URL
+ * is no longer valid (e.g. participant moved to a breakout room, or meeting ended).
+ * It's handled internally by HashTreeParser and results in LOCUS_NOT_FOUND being sent up.
+ */
+export class LocusNotFoundError extends Error {}
+
 /* Currently Locus always sends Metadata objects only in the "self" dataset.
  * If this ever changes, update all the code that relies on this constant.
  */
@@ -96,9 +125,17 @@ class HashTreeParser {
   locusInfoUpdateCallback: LocusInfoUpdateCallback;
   visibleDataSets: VisibleDataSetInfo[];
   debugId: string;
-  heartbeatIntervalMs?: number;
   private excludedDataSets: string[];
   state: 'active' | 'stopped';
+  private syncQueue: Array<{dataSetName: string; reason: string; isInitialization?: boolean}> = [];
+  private isSyncInProgress = false;
+  // tracks whether syncAllDatasets is currently in its backoff delay phase and with what scope
+  private syncAllBackoffType: SyncAllBackoffType = SyncAllBackoffType.NONE;
+  // datasets that received messages during the syncAllDatasets backoff sleep and should be skipped
+  private dataSetsSyncedDuringBackoff: Set<string> = new Set();
+  private syncQueueProcessingPromise: Promise<void> = Promise.resolve();
+  // top-level heartbeat interval from the most recent message, used as fallback when dataset-level value is missing
+  private topLevelHeartbeatIntervalMs?: number;
 
   /**
    * Constructor for HashTreeParser
@@ -224,16 +261,16 @@ class HashTreeParser {
    * @param {DataSet} dataSetInfo The new data set to be added
    * @returns {Promise}
    */
-  private initializeNewVisibleDataSet(
+  private async initializeNewVisibleDataSet(
     visibleDataSetInfo: VisibleDataSetInfo,
     dataSetInfo: DataSet
-  ): Promise<{updateType: LocusInfoUpdateType; updatedObjects?: HashTreeObject[]}> {
+  ): Promise<void> {
     if (this.isVisibleDataSet(dataSetInfo.name)) {
       LoggerProxy.logger.info(
         `HashTreeParser#initializeNewVisibleDataSet --> ${this.debugId} Data set "${dataSetInfo.name}" already exists, skipping init`
       );
 
-      return Promise.resolve({updateType: LocusInfoUpdateType.OBJECTS_UPDATED, updatedObjects: []});
+      return;
     }
 
     LoggerProxy.logger.info(
@@ -241,7 +278,7 @@ class HashTreeParser {
     );
 
     if (!this.addToVisibleDataSetsList(visibleDataSetInfo)) {
-      return Promise.resolve({updateType: LocusInfoUpdateType.OBJECTS_UPDATED, updatedObjects: []});
+      return;
     }
 
     const hashTree = new HashTree([], dataSetInfo.leafCount);
@@ -251,51 +288,8 @@ class HashTreeParser {
       hashTree,
     };
 
-    return this.sendInitializationSyncRequestToLocus(dataSetInfo.name, 'new visible data set');
-  }
-
-  /**
-   * Sends a special sync request to Locus with all leaves empty - this is a way to get all the data for a given dataset.
-   *
-   * @param {string} datasetName - name of the dataset for which to send the request
-   * @param {string} debugText - text to include in logs
-   * @returns {Promise}
-   */
-  private sendInitializationSyncRequestToLocus(
-    datasetName: string,
-    debugText: string
-  ): Promise<{updateType: LocusInfoUpdateType; updatedObjects?: HashTreeObject[]}> {
-    const dataset = this.dataSets[datasetName];
-
-    if (!dataset) {
-      LoggerProxy.logger.warn(
-        `HashTreeParser#sendInitializationSyncRequestToLocus --> ${this.debugId} No data set found for ${datasetName}, cannot send the request for leaf data`
-      );
-
-      return Promise.resolve(null);
-    }
-
-    const emptyLeavesData = new Array(dataset.leafCount).fill([]);
-
-    LoggerProxy.logger.info(
-      `HashTreeParser#sendInitializationSyncRequestToLocus --> ${this.debugId} Sending initial sync request to Locus for data set "${datasetName}" with empty leaf data`
-    );
-
-    return this.sendSyncRequestToLocus(this.dataSets[datasetName], emptyLeavesData).then(
-      (syncResponse) => {
-        if (syncResponse) {
-          return {
-            updateType: LocusInfoUpdateType.OBJECTS_UPDATED,
-            updatedObjects: this.parseMessage(
-              syncResponse,
-              `via empty leaves /sync API call for ${debugText}`
-            ),
-          };
-        }
-
-        return {updateType: LocusInfoUpdateType.OBJECTS_UPDATED, updatedObjects: []};
-      }
-    );
+    this.enqueueSyncForDataset(dataSetInfo.name, 'new visible data set initialization', true);
+    await this.syncQueueProcessingPromise;
   }
 
   /**
@@ -382,9 +376,8 @@ class HashTreeParser {
     if (this.state === 'stopped') {
       return;
     }
-    const updatedObjects: HashTreeObject[] = [];
 
-    for (const dataSet of visibleDataSets) {
+    for (const dataSet of sortByInitPriority(visibleDataSets, DATA_SET_INIT_PRIORITY)) {
       const {name, leafCount, url} = dataSet;
 
       if (!this.dataSets[name]) {
@@ -420,19 +413,12 @@ class HashTreeParser {
         );
         this.dataSets[name].hashTree = new HashTree([], leafCount);
 
-        // eslint-disable-next-line no-await-in-loop
-        const data = await this.sendInitializationSyncRequestToLocus(name, debugText);
-
-        if (data.updateType === LocusInfoUpdateType.OBJECTS_UPDATED) {
-          updatedObjects.push(...(data.updatedObjects || []));
-        }
+        this.enqueueSyncForDataset(name, `initialization sync for ${debugText}`, true);
       }
     }
 
-    this.callLocusInfoUpdateCallback({
-      updateType: LocusInfoUpdateType.OBJECTS_UPDATED,
-      updatedObjects,
-    });
+    // wait for all enqueued initialization syncs to complete
+    await this.syncQueueProcessingPromise;
   }
 
   /**
@@ -586,12 +572,40 @@ class HashTreeParser {
       )}`
     );
 
+    this.cancelPendingSyncsForDataSets(dataSets.map((ds) => ds.name));
+
     dataSets.forEach((dataSet) => {
       this.updateDataSetInfo(dataSet);
       this.runSyncAlgorithm(dataSet);
     });
   }
 
+  /**
+   * Handles known errors that can happen during syncs
+   *
+   * @param {any} error - The error to handle
+   * @returns {boolean} true if the error was recognized and handled, false otherwise
+   */
+  private handleSyncErrors(error: any) {
+    if (error instanceof MeetingEndedError) {
+      this.callLocusInfoUpdateCallback({
+        updateType: LocusInfoUpdateType.MEETING_ENDED,
+      });
+
+      return true;
+    }
+    if (error instanceof LocusNotFoundError) {
+      this.callLocusInfoUpdateCallback({
+        updateType: LocusInfoUpdateType.LOCUS_NOT_FOUND,
+      });
+      this.stop();
+
+      return true;
+    }
+
+    return false;
+  }
+
   /**
    * Asynchronously initializes new visible data sets
    *
@@ -608,11 +622,7 @@ class HashTreeParser {
     );
     queueMicrotask(() => {
       this.initializeNewVisibleDataSets(dataSetsRequiringInitialization).catch((error) => {
-        if (error instanceof MeetingEndedError) {
-          this.callLocusInfoUpdateCallback({
-            updateType: LocusInfoUpdateType.MEETING_ENDED,
-          });
-        } else {
+        if (!this.handleSyncErrors(error)) {
           LoggerProxy.logger.warn(
             `HashTreeParser#queueInitForNewVisibleDataSets --> ${
               this.debugId
@@ -690,7 +700,14 @@ class HashTreeParser {
 
     const {dataSets, locus, metadata} = update;
 
+    LoggerProxy.logger.info(
+      `HashTreeParser#handleLocusUpdate --> ${this.debugId} received update with dataSets=${dataSets
+        ?.map((ds) => ds.name)
+        .join(',')} metadata=${metadata ? 'yes' : 'no'}`
+    );
+
     if (!dataSets) {
+      // this happens for example when we handle GET /loci response
       LoggerProxy.logger.info(
         `HashTreeParser#handleLocusUpdate --> ${this.debugId} received hash tree update without dataSets`
       );
@@ -788,12 +805,25 @@ class HashTreeParser {
         maxMs: receivedDataSet.backoff.maxMs,
         exponent: receivedDataSet.backoff.exponent,
       };
+      this.dataSets[receivedDataSet.name].heartbeatIntervalMs = receivedDataSet.heartbeatIntervalMs;
       LoggerProxy.logger.info(
         `HashTreeParser#updateDataSetInfo --> ${this.debugId} updated "${receivedDataSet.name}" dataset to version=${receivedDataSet.version}, root=${receivedDataSet.root}`
       );
     }
   }
 
+  /**
+   * Updates the leaf count for a data set, resizing its hash tree accordingly.
+   *
+   * @param {InternalDataSet} dataSet - The data set to update
+   * @param {number} newLeafCount - The new leaf count
+   * @returns {void}
+   */
+  private updateDataSetLeafCount(dataSet: InternalDataSet, newLeafCount: number): void {
+    dataSet.hashTree?.resize(newLeafCount);
+    dataSet.leafCount = newLeafCount;
+  }
+
   /**
    * Checks for changes in the visible data sets based on the updated objects.
    * @param {HashTreeObject[]} updatedObjects - The list of updated hash tree objects.
@@ -844,6 +874,8 @@ class HashTreeParser {
    */
   private deleteHashTree(dataSetName: string) {
     this.dataSets[dataSetName].hashTree = undefined;
+    this.dataSets[dataSetName].syncAbortController?.abort();
+    this.dataSets[dataSetName].syncAbortController = undefined;
 
     // we also need to stop the timers as there is no hash tree anymore to sync
     if (this.dataSets[dataSetName].timer) {
@@ -960,7 +992,7 @@ class HashTreeParser {
     }
     const allDataSets = await this.getAllVisibleDataSetsFromLocus();
 
-    for (const ds of addedDataSets) {
+    for (const ds of sortByInitPriority(addedDataSets, DATA_SET_INIT_PRIORITY)) {
       const dataSetInfo = allDataSets.find((d) => d.name === ds.name);
 
       LoggerProxy.logger.info(
@@ -972,12 +1004,8 @@ class HashTreeParser {
           `HashTreeParser#initializeNewVisibleDataSets --> ${this.debugId} missing info about data set "${ds.name}" in Locus response from visibleDataSetsUrl`
         );
       } else {
-        // we're awaiting in a loop, because in practice there will be only one new data set at a time,
-        // so no point in trying to parallelize this
         // eslint-disable-next-line no-await-in-loop
-        const updates = await this.initializeNewVisibleDataSet(ds, dataSetInfo);
-
-        this.callLocusInfoUpdateCallback(updates);
+        await this.initializeNewVisibleDataSet(ds, dataSetInfo);
       }
     }
   }
@@ -997,25 +1025,39 @@ class HashTreeParser {
     const {dataSets, visibleDataSetsUrl} = message;
 
     LoggerProxy.logger.info(
-      `HashTreeParser#parseMessage --> ${this.debugId} received message ${debugText || ''}:`,
-      message
+      `HashTreeParser#parseMessage --> ${this.debugId} ${
+        debugText || ''
+      } dataSets: ${message.dataSets
+        ?.map(({name, version}) => `${name}:${version}`)
+        .join(',')}, elements: ${message.locusStateElements
+        ?.map(
+          (el) =>
+            `${el.htMeta.elementId.type}:${el.htMeta.elementId.id}:${el.htMeta.elementId.version}${
+              el.data ? '+' : '-'
+            }`
+        )
+        .join(',')}`
     );
+
     if (message.locusStateElements?.length === 0) {
       LoggerProxy.logger.warn(
         `HashTreeParser#parseMessage --> ${this.debugId} got empty locusStateElements!!!`
       );
-      // todo: send a metric
+      Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.HASH_TREE_EMPTY_LOCUS_STATE_ELEMENTS, {
+        debugId: this.debugId,
+      });
     }
 
     // first, update our metadata about the datasets with info from the message
     this.visibleDataSetsUrl = visibleDataSetsUrl;
     dataSets.forEach((dataSet) => this.updateDataSetInfo(dataSet));
+    this.cancelPendingSyncsForDataSets(dataSets.map((ds) => ds.name));
 
     const updatedObjects: HashTreeObject[] = [];
 
     // when we detect new visible datasets, it may be that the metadata about them is not
     // available in the message, they will require separate async initialization
-    let dataSetsRequiringInitialization = [];
+    let dataSetsRequiringInitialization: VisibleDataSetInfo[] = [];
 
     // first find out if there are any visible data set changes - they're signalled in Metadata object updates
     const metadataUpdates = (message.locusStateElements || []).filter((object) =>
@@ -1023,7 +1065,7 @@ class HashTreeParser {
     );
 
     if (metadataUpdates.length > 0) {
-      const updatedMetadataObjects = [];
+      const updatedMetadataObjects: HashTreeObject[] = [];
 
       metadataUpdates.forEach((object) => {
         // todo: once Locus supports it, we will use the "view" field here instead of dataSetNames
@@ -1052,7 +1094,7 @@ class HashTreeParser {
       }
     }
 
-    if (message.locusStateElements?.length > 0) {
+    if (message.locusStateElements && message.locusStateElements.length > 0) {
       // by this point we now have this.dataSets setup for data sets from this message
       // and hash trees created for the new visible data sets,
       // so we can now process all the updates from the message
@@ -1118,9 +1160,10 @@ class HashTreeParser {
       return;
     }
 
-    if (message.heartbeatIntervalMs) {
-      this.heartbeatIntervalMs = message.heartbeatIntervalMs;
+    if (message.heartbeatIntervalMs !== undefined) {
+      this.topLevelHeartbeatIntervalMs = message.heartbeatIntervalMs;
     }
+
     if (this.isEndMessage(message)) {
       LoggerProxy.logger.info(
         `HashTreeParser#handleMessage --> ${this.debugId} received sentinel END MEETING message`
@@ -1148,20 +1191,17 @@ class HashTreeParser {
    * @param {Object} updates parsed from a Locus message
    * @returns {void}
    */
-  private callLocusInfoUpdateCallback(updates: {
-    updateType: LocusInfoUpdateType;
-    updatedObjects?: HashTreeObject[];
-  }) {
+  private callLocusInfoUpdateCallback(updates: LocusInfoUpdate) {
     if (this.state === 'stopped') {
       return;
     }
 
-    const {updateType, updatedObjects} = updates;
+    const {updateType} = updates;
 
-    if (updateType === LocusInfoUpdateType.OBJECTS_UPDATED && updatedObjects?.length > 0) {
+    if (updateType === LocusInfoUpdateType.OBJECTS_UPDATED && updates.updatedObjects?.length > 0) {
       // Filter out updates for objects that already have a higher version in their datasets,
       // or removals for objects that still exist in any of their datasets
-      const filteredUpdates = updatedObjects.filter((object) => {
+      const filteredUpdates = updates.updatedObjects.filter((object) => {
         const {elementId} = object.htMeta;
         const {type, id, version} = elementId;
 
@@ -1198,10 +1238,10 @@ class HashTreeParser {
       });
 
       if (filteredUpdates.length > 0) {
-        this.locusInfoUpdateCallback(updateType, {updatedObjects: filteredUpdates});
+        this.locusInfoUpdateCallback({updateType, updatedObjects: filteredUpdates});
       }
     } else if (updateType !== LocusInfoUpdateType.OBJECTS_UPDATED) {
-      this.locusInfoUpdateCallback(updateType, {updatedObjects});
+      this.locusInfoUpdateCallback({updateType});
     }
   }
 
@@ -1223,82 +1263,363 @@ class HashTreeParser {
    * Performs a sync for the given data set.
    *
    * @param {InternalDataSet} dataSet - The data set to sync
-   * @param {string} rootHash - Our current root hash for this data set
    * @param {string} reason - The reason for the sync (used for logging)
+   * @param {boolean} [isInitialization] - Whether this is an initialization sync (sends empty leaves data instead of comparing hashes)
    * @returns {Promise<void>}
    */
   private async performSync(
     dataSet: InternalDataSet,
-    rootHash: string,
-    reason: string
+    reason: string,
+    isInitialization?: boolean
   ): Promise<void> {
     if (!dataSet.hashTree) {
       return;
     }
 
+    const abortController = dataSet.syncAbortController ?? new AbortController();
+    dataSet.syncAbortController = abortController;
+
+    const {hashTree} = dataSet;
+    const rootHash = hashTree.getRootHash();
+
     try {
       LoggerProxy.logger.info(
         `HashTreeParser#performSync --> ${this.debugId} ${reason}, syncing data set "${dataSet.name}"`
       );
 
-      const mismatchedLeavesData: Record<number, LeafDataItem[]> = {};
+      let leavesData: Record<number, LeafDataItem[]> = {};
 
-      if (dataSet.leafCount !== 1) {
-        let receivedHashes;
+      if (!isInitialization) {
+        if (dataSet.leafCount !== 1) {
+          let receivedHashes;
 
-        try {
-          // request hashes from sender
-          const {hashes, dataSet: latestDataSetInfo} = await this.getHashesFromLocus(
-            dataSet.name,
-            rootHash
-          );
+          try {
+            // request hashes from sender
+            const hashesResult = await this.getHashesFromLocus(dataSet.name, rootHash);
 
-          receivedHashes = hashes;
+            if (!hashesResult) {
+              // hashes match, no sync needed
+              return;
+            }
 
-          dataSet.hashTree.resize(latestDataSetInfo.leafCount);
-        } catch (error) {
-          if (error.statusCode === 409) {
-            // this is a leaf count mismatch, we should do nothing, just wait for another heartbeat message from Locus
-            LoggerProxy.logger.info(
-              `HashTreeParser#getHashesFromLocus --> ${this.debugId} Got 409 when fetching hashes for data set "${dataSet.name}": ${error.message}`
-            );
+            receivedHashes = hashesResult.hashes;
+
+            this.updateDataSetLeafCount(dataSet, hashesResult.dataSet.leafCount);
+          } catch (error: any) {
+            if (error?.statusCode === 409) {
+              // this is a leaf count mismatch, we should do nothing, just wait for another heartbeat message from Locus
+              LoggerProxy.logger.info(
+                `HashTreeParser#getHashesFromLocus --> ${this.debugId} Got 409 when fetching hashes for data set "${dataSet.name}": ${error.message}`
+              );
 
-            return;
+              return;
+            }
+            throw error;
           }
-          throw error;
+
+          // identify mismatched leaves
+          const mismatchedLeaveIndexes = hashTree.diffHashes(receivedHashes);
+
+          mismatchedLeaveIndexes.forEach((index) => {
+            leavesData[index] = hashTree.getLeafData(index);
+          });
+        } else {
+          leavesData = {0: hashTree.getLeafData(0)};
         }
+      }
 
-        // identify mismatched leaves
-        const mismatchedLeaveIndexes = dataSet.hashTree.diffHashes(receivedHashes);
+      if (abortController.signal.aborted) {
+        LoggerProxy.logger.info(
+          `HashTreeParser#performSync --> ${this.debugId} abandoning sync for "${dataSet.name}" before /sync - message received during sync`
+        );
 
-        mismatchedLeaveIndexes.forEach((index) => {
-          mismatchedLeavesData[index] = dataSet.hashTree.getLeafData(index);
-        });
-      } else {
-        mismatchedLeavesData[0] = dataSet.hashTree.getLeafData(0);
+        return;
       }
       // request sync for mismatched leaves
-      if (Object.keys(mismatchedLeavesData).length > 0) {
-        const syncResponse = await this.sendSyncRequestToLocus(dataSet, mismatchedLeavesData);
+      let syncResponse: HashTreeMessage | null = null;
 
-        // sync API may return nothing (in that case data will arrive via messages)
-        // or it may return a response in the same format as messages
-        if (syncResponse) {
-          this.handleMessage(syncResponse, 'via sync API');
-        }
+      if (isInitialization) {
+        syncResponse = await this.sendSyncRequestToLocus(dataSet, {isInitialization: true});
+      } else if (Object.keys(leavesData).length > 0) {
+        syncResponse = await this.sendSyncRequestToLocus(dataSet, {
+          mismatchedLeavesData: leavesData,
+        });
+      }
+
+      // sync API may return nothing (in that case data will arrive via messages)
+      // or it may return a response in the same format as messages
+      // We still need to restart the sync timer as a safety net in case the messages don't arrive.
+      this.runSyncAlgorithm(dataSet);
+
+      if (syncResponse) {
+        // clear the abort controller before processing the response so that
+        // parseMessage() -> cancelPendingSyncsForDataSets() doesn't log a
+        // misleading "aborting sync" message for this already-completed sync
+        dataSet.syncAbortController = undefined;
+
+        // the format of sync response is the same as messages, so we can reuse the same handler
+        this.handleMessage(
+          syncResponse,
+          `via sync API (${
+            isInitialization ? 'init' : `${Object.keys(leavesData).length} mismatched leaves`
+          })`
+        );
       }
     } catch (error) {
-      if (error instanceof MeetingEndedError) {
-        this.callLocusInfoUpdateCallback({
-          updateType: LocusInfoUpdateType.MEETING_ENDED,
-        });
-      } else {
+      if (!this.handleSyncErrors(error)) {
         LoggerProxy.logger.warn(
           `HashTreeParser#performSync --> ${this.debugId} error during sync for data set "${dataSet.name}":`,
           error
         );
       }
+    } finally {
+      dataSet.syncAbortController = undefined;
+    }
+  }
+
+  /**
+   * Cancels any pending or in-flight syncs for the specified data sets.
+   * This removes matching entries from the sync queue and aborts any in-flight sync HTTP requests.
+   *
+   * @param {string[]} dataSetNames - The names of the data sets to cancel syncs for
+   * @returns {void}
+   */
+  private cancelPendingSyncsForDataSets(dataSetNames: string[]): void {
+    const previousLength = this.syncQueue.length;
+
+    this.syncQueue = this.syncQueue.filter((entry) => !dataSetNames.includes(entry.dataSetName));
+
+    if (previousLength !== this.syncQueue.length) {
+      LoggerProxy.logger.info(
+        `HashTreeParser#cancelPendingSyncsForDataSets --> ${this.debugId} removed ${
+          previousLength - this.syncQueue.length
+        } entries from sync queue for data sets: ${dataSetNames.join(', ')}`
+      );
+    }
+
+    this.markDataSetsForSyncAllBackoffSkip(dataSetNames);
+    this.abortInFlightSyncs(dataSetNames);
+  }
+
+  /**
+   * If a syncAllDatasets backoff sleep is in progress, marks the given data sets to be skipped
+   * after the sleep completes.
+   *
+   * @param {string[]} dataSetNames - The names of the data sets to mark
+   * @returns {void}
+   */
+  private markDataSetsForSyncAllBackoffSkip(dataSetNames: string[]): void {
+    if (this.syncAllBackoffType !== SyncAllBackoffType.NONE) {
+      for (const name of dataSetNames) {
+        this.dataSetsSyncedDuringBackoff.add(name);
+      }
+    }
+  }
+
+  /**
+   * Aborts any in-flight sync HTTP requests for the specified data sets.
+   *
+   * @param {string[]} dataSetNames - The names of the data sets whose syncs should be aborted
+   * @returns {void}
+   */
+  private abortInFlightSyncs(dataSetNames: string[]): void {
+    for (const name of dataSetNames) {
+      if (this.dataSets[name]?.syncAbortController) {
+        LoggerProxy.logger.info(
+          `HashTreeParser#cancelPendingSyncsForDataSets --> ${this.debugId} aborting in-flight sync for data set "${name}"`
+        );
+        this.dataSets[name].syncAbortController.abort();
+      }
+    }
+  }
+
+  /**
+   * Enqueues a sync for the given data set. If the data set is already in the queue, the request is ignored.
+   * This ensures that all syncs are executed sequentially and no more than 1 sync runs at a time.
+   *
+   * @param {string} dataSetName - The name of the data set to sync
+   * @param {string} reason - The reason for the sync (used for logging)
+   * @param {boolean} [isInitialization=false] - Whether this is an initialization sync (uses empty leaves data instead of hash comparison)
+   * @returns {void}
+   */
+  private enqueueSyncForDataset(
+    dataSetName: string,
+    reason: string,
+    isInitialization = false
+  ): void {
+    if (this.state === 'stopped') return;
+
+    const existingEntry = this.syncQueue.find((entry) => entry.dataSetName === dataSetName);
+
+    if (existingEntry) {
+      if (isInitialization) {
+        existingEntry.isInitialization = true;
+      }
+      LoggerProxy.logger.info(
+        `HashTreeParser#enqueueSyncForDataset --> ${this.debugId} data set "${dataSetName}" already in sync queue, skipping`
+      );
+
+      return;
+    }
+
+    this.syncQueue.push({dataSetName, reason, isInitialization});
+
+    if (!this.isSyncInProgress) {
+      this.syncQueueProcessingPromise = this.processSyncQueue();
+    }
+  }
+
+  /**
+   * Processes the sync queue sequentially. Only one instance of this method runs at a time.
+   *
+   * @returns {Promise<void>}
+   */
+  private async processSyncQueue(): Promise<void> {
+    if (this.isSyncInProgress) return;
+
+    this.isSyncInProgress = true;
+    try {
+      while (this.syncQueue.length > 0 && this.state !== 'stopped') {
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const {dataSetName, reason, isInitialization} = this.syncQueue.shift()!;
+        const dataSet = this.dataSets[dataSetName];
+
+        if (!dataSet?.hashTree) {
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        // eslint-disable-next-line no-await-in-loop
+        await this.performSync(dataSet, reason, isInitialization);
+      }
+    } finally {
+      this.isSyncInProgress = false;
+    }
+  }
+
+  /**
+   * sets the backoff type for syncAllDatasets calls, which determines the scope of datasets that will be synced after the backoff delay.
+   *
+   * @param {boolean} onlyLLM - Whether the backoff is for a syncAllDatasets call that is syncing only LLM datasets
+   * @returns {void}
+   */
+  private setSyncAllBackoffType(onlyLLM: boolean): void {
+    this.syncAllBackoffType = onlyLLM ? SyncAllBackoffType.ONLY_LLM : SyncAllBackoffType.ALL;
+  }
+
+  /**
+   * Checks if a syncAll backoff is already in progress. If so, upgrades the scope from
+   * onlyLLM to all datasets when the new call has a broader scope.
+   *
+   * @param {boolean} onlyLLM - Whether the current call is for LLM datasets only
+   * @returns {boolean} true if a backoff is already pending (caller should return early)
+   */
+  private tryUpgradePendingBackoff(onlyLLM: boolean): boolean {
+    if (this.syncAllBackoffType !== SyncAllBackoffType.NONE) {
+      if (!onlyLLM && this.syncAllBackoffType === SyncAllBackoffType.ONLY_LLM) {
+        this.setSyncAllBackoffType(false);
+        LoggerProxy.logger.info(
+          `HashTreeParser#syncAllDatasets --> ${this.debugId} upgraded pending syncAll from onlyLLM to all datasets`
+        );
+      }
+
+      return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Syncs all data sets that have hash trees, one by one in sequence, using the priority order
+   * provided by sortByInitPriority().
+   *
+   * If a call is already waiting in the backoff delay phase, a new call with a broader scope
+   * (onlyLLM=false) will upgrade the pending scope, and the dataset list will be computed after
+   * the backoff using the upgraded scope. After the backoff, the sync queue handles deduplication
+   * so no guard is needed.
+   *
+   * @param {Object} [options={}] - Options for syncing
+   * @param {boolean} [options.onlyLLM=false] - Whether to sync only LLM based data sets
+   * @returns {Promise<void>}
+   */
+  public async syncAllDatasets(options: {onlyLLM?: boolean} = {}): Promise<void> {
+    const {onlyLLM = false} = options;
+    if (this.state === 'stopped') return;
+
+    // if we're already in the backoff delay phase, try to upgrade the scope instead of starting a new one
+    if (this.tryUpgradePendingBackoff(onlyLLM)) {
+      return;
+    }
+
+    const dataSetsToSync = this.getSortedDataSetsWithHashTrees(onlyLLM);
+
+    if (dataSetsToSync.length === 0) return;
+
+    this.setSyncAllBackoffType(onlyLLM);
+
+    const delay = this.getWeightedBackoffTime(dataSetsToSync[0].backoff);
+
+    LoggerProxy.logger.info(
+      `HashTreeParser#syncAllDatasets --> ${this.debugId} starting backoff delay of ${delay}ms (onlyLLM=${onlyLLM})`
+    );
+
+    // delay the start of the syncs - this is a Locus requirement to avoid thundering herd issues
+    await sleep(delay);
+
+    // read the (possibly upgraded) scope and clear the backoff flag
+    const effectiveBackoffType = this.syncAllBackoffType;
+    const skippedDataSets = this.dataSetsSyncedDuringBackoff;
+
+    this.syncAllBackoffType = SyncAllBackoffType.NONE;
+    this.dataSetsSyncedDuringBackoff = new Set();
+
+    if ((this.state as string) === 'stopped') return;
+
+    // re-evaluate the dataset list after the sleep, since the scope may have been upgraded
+    // and exclude datasets that received messages during the backoff sleep
+    const effectiveDataSetsToSync = this.getSortedDataSetsWithHashTrees(
+      effectiveBackoffType === SyncAllBackoffType.ONLY_LLM
+    ).filter((ds) => !skippedDataSets.has(ds.name));
+
+    if (skippedDataSets.size > 0) {
+      LoggerProxy.logger.info(
+        `HashTreeParser#syncAllDatasets --> ${
+          this.debugId
+        } skipping datasets that received messages during backoff: ${[...skippedDataSets].join(
+          ', '
+        )}`
+      );
+    }
+
+    LoggerProxy.logger.info(
+      `HashTreeParser#syncAllDatasets --> ${this.debugId} syncing ${
+        effectiveBackoffType === SyncAllBackoffType.ONLY_LLM ? 'only LLM' : 'all'
+      } datasets: ${effectiveDataSetsToSync.map((ds) => ds.name).join(', ')}`
+    );
+
+    for (const ds of effectiveDataSetsToSync) {
+      this.enqueueSyncForDataset(ds.name, 'syncAllDatasets');
+    }
+
+    await this.syncQueueProcessingPromise;
+  }
+
+  /**
+   * Returns the list of data sets that have hash trees, sorted by the priority order provided by sortByInitPriority().
+   *
+   * @param {boolean} onlyLLM - Whether to include only LLM based data sets
+   * @returns {Array<{name: string, backoff: {maxMs: number, exponent: number}}>} The sorted list of data sets with their backoff configurations
+   */
+  private getSortedDataSetsWithHashTrees(onlyLLM: boolean) {
+    let dataSets = Object.values(this.dataSets)
+      .filter((dataSet) => dataSet?.hashTree)
+      .map((dataSet) => ({name: dataSet.name, backoff: dataSet.backoff}));
+
+    if (onlyLLM) {
+      dataSets = dataSets.filter((ds) => LLM_DATASET_NAMES.includes(ds.name));
     }
+
+    return sortByInitPriority(dataSets, DATA_SET_INIT_PRIORITY);
   }
 
   /**
@@ -1319,21 +1640,14 @@ class HashTreeParser {
     }
 
     if (!dataSet.hashTree) {
-      LoggerProxy.logger.info(
-        `HashTreeParser#runSyncAlgorithm --> ${this.debugId} Data set "${dataSet.name}" has no hash tree, skipping sync algorithm`
-      );
+      // no hash tree, so no need to do any syncing
+      // we fall into this branch often, because Locus sends dataSets in messages that are not visible to us
 
       return;
     }
 
     dataSet.hashTree.resize(receivedDataSet.leafCount);
 
-    // temporary log for the workshop // todo: remove
-    const ourCurrentRootHash = dataSet.hashTree.getRootHash();
-    LoggerProxy.logger.info(
-      `HashTreeParser#runSyncAlgorithm --> ${this.debugId} dataSet="${dataSet.name}" version=${dataSet.version} hashes before starting timer: ours=${ourCurrentRootHash} Locus=${dataSet.root}`
-    );
-
     const delay = dataSet.idleMs + this.getWeightedBackoffTime(dataSet.backoff);
 
     if (delay > 0) {
@@ -1341,11 +1655,7 @@ class HashTreeParser {
         clearTimeout(dataSet.timer);
       }
 
-      LoggerProxy.logger.info(
-        `HashTreeParser#runSyncAlgorithm --> ${this.debugId} setting "${dataSet.name}" sync timer for ${delay}`
-      );
-
-      dataSet.timer = setTimeout(async () => {
+      dataSet.timer = setTimeout(() => {
         dataSet.timer = undefined;
 
         if (!dataSet.hashTree) {
@@ -1359,15 +1669,10 @@ class HashTreeParser {
         const rootHash = dataSet.hashTree.getRootHash();
 
         if (dataSet.root !== rootHash) {
-          await this.performSync(
-            dataSet,
-            rootHash,
+          this.enqueueSyncForDataset(
+            dataSet.name,
             `Root hash mismatch: received=${dataSet.root}, ours=${rootHash}`
           );
-        } else {
-          LoggerProxy.logger.info(
-            `HashTreeParser#runSyncAlgorithm --> ${this.debugId} "${dataSet.name}" root hash matching: ${rootHash}, version=${dataSet.version}`
-          );
         }
       }, delay);
     } else {
@@ -1387,38 +1692,39 @@ class HashTreeParser {
    * @returns {void}
    */
   private resetHeartbeatWatchdogs(receivedDataSets: Array<DataSet>): void {
-    if (!this.heartbeatIntervalMs) {
-      return;
-    }
-
     for (const receivedDataSet of receivedDataSets) {
       const dataSet = this.dataSets[receivedDataSet.name];
 
-      if (!dataSet?.hashTree) {
-        // eslint-disable-next-line no-continue
-        continue;
-      }
-
       if (dataSet.heartbeatWatchdogTimer) {
         clearTimeout(dataSet.heartbeatWatchdogTimer);
         dataSet.heartbeatWatchdogTimer = undefined;
       }
 
+      // dataset-level heartbeatIntervalMs takes priority; fall back to top-level common value
+      const heartbeatIntervalMs = dataSet?.heartbeatIntervalMs ?? this.topLevelHeartbeatIntervalMs;
+
+      if (!dataSet?.hashTree || !heartbeatIntervalMs) {
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
       const backoffTime = this.getWeightedBackoffTime(dataSet.backoff);
-      const delay = this.heartbeatIntervalMs + backoffTime;
+      const delay = heartbeatIntervalMs + backoffTime;
 
-      dataSet.heartbeatWatchdogTimer = setTimeout(async () => {
+      dataSet.heartbeatWatchdogTimer = setTimeout(() => {
         dataSet.heartbeatWatchdogTimer = undefined;
 
         LoggerProxy.logger.warn(
           `HashTreeParser#resetHeartbeatWatchdogs --> ${this.debugId} Heartbeat watchdog fired for data set "${dataSet.name}" - no heartbeat received within expected interval, initiating sync`
         );
 
-        await this.performSync(
-          dataSet,
-          dataSet.hashTree.getRootHash(),
-          `heartbeat watchdog expired`
-        );
+        Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.HASH_TREE_HEARTBEAT_WATCHDOG_EXPIRED, {
+          debugId: this.debugId,
+          dataSetName: dataSet.name,
+        });
+
+        this.enqueueSyncForDataset(dataSet.name, `heartbeat watchdog expired`);
+        this.resetHeartbeatWatchdogs([dataSet]);
       }, delay);
     }
   }
@@ -1451,7 +1757,13 @@ class HashTreeParser {
       `HashTreeParser#stop --> ${this.debugId} Stopping HashTreeParser, clearing timers and hash trees`
     );
     this.stopAllTimers();
+    this.syncQueue = [];
+    this.topLevelHeartbeatIntervalMs = undefined;
+    this.syncAllBackoffType = SyncAllBackoffType.NONE;
+    this.dataSetsSyncedDuringBackoff = new Set();
     Object.values(this.dataSets).forEach((dataSet) => {
+      dataSet.syncAbortController?.abort();
+      dataSet.syncAbortController = undefined;
       dataSet.hashTree = undefined;
     });
     this.visibleDataSets = [];
@@ -1459,17 +1771,27 @@ class HashTreeParser {
   }
 
   /**
-   * Resumes the HashTreeParser that was previously stopped.
+   * Cleans up the HashTreeParser, stopping all timers and clearing all internal state.
+   * After calling this, the parser should not be used anymore.
+   * @returns {void}
+   */
+  public cleanUp() {
+    this.stop();
+    this.dataSets = {};
+  }
+
+  /**
+   * Resumes the HashTreeParser that was previously stopped, using a hash tree message.
    * @param {HashTreeMessage} message - The message to resume with, it must contain metadata with visible data sets info
    * @returns {void}
    */
-  public resume(message: HashTreeMessage) {
+  public resumeFromMessage(message: HashTreeMessage) {
     // check that message contains metadata with visible data sets - this is essential to be able to resume
     const metadataObject = message.locusStateElements?.find((el) => isMetadata(el));
 
     if (!metadataObject?.data?.visibleDataSets) {
       LoggerProxy.logger.warn(
-        `HashTreeParser#resume --> ${this.debugId} Cannot resume HashTreeParser because the message is missing metadata with visible data sets info`
+        `HashTreeParser#resumeFromMessage --> ${this.debugId} Cannot resume HashTreeParser because the message is missing metadata with visible data sets info`
       );
 
       return;
@@ -1490,7 +1812,7 @@ class HashTreeParser {
       };
     }
     LoggerProxy.logger.info(
-      `HashTreeParser#resume --> ${
+      `HashTreeParser#resumeFromMessage --> ${
         this.debugId
       } Resuming HashTreeParser with data sets: ${Object.keys(this.dataSets).join(
         ', '
@@ -1501,18 +1823,47 @@ class HashTreeParser {
     this.handleMessage(message, 'on resume');
   }
 
+  /**
+   * Resumes the HashTreeParser that was previously stopped, using a Locus API response.
+   * Unlike resumeFromMessage(), this does not require metadata/dataSets in the input,
+   * as it fetches all necessary information from Locus via initializeFromGetLociResponse.
+   * @param {LocusDTO} locus - locus object from an API response
+   * @returns {Promise}
+   */
+  public async resumeFromApiResponse(locus: LocusDTO) {
+    this.state = 'active';
+    this.dataSets = {};
+
+    LoggerProxy.logger.info(
+      `HashTreeParser#resumeFromApiResponse --> ${this.debugId} Resuming HashTreeParser from API response`
+    );
+
+    await this.initializeFromGetLociResponse(locus);
+  }
+
   private checkForSentinelHttpResponse(error: any, dataSetName?: string) {
+    // 404 for any dataset means the locus is no longer available at this URL - could be replaced or ended
+    // if a dataset is just not visible, we would get a 400
+    if (error.statusCode === 404) {
+      LoggerProxy.logger.info(
+        `HashTreeParser#checkForSentinelHttpResponse --> ${this.debugId} Received 404 for data set "${dataSetName}", locus not found`
+      );
+      this.stopAllTimers();
+
+      throw new LocusNotFoundError();
+    }
+
     const isValidDataSetForSentinel =
       dataSetName === undefined ||
       PossibleSentinelMessageDataSetNames.includes(dataSetName.toLowerCase());
 
     if (
-      ((error.statusCode === 409 && error.body?.errorCode === 2403004) ||
-        error.statusCode === 404) &&
+      error.statusCode === 409 &&
+      error.body?.errorCode === LocusErrorCodes.LOCUS_INACTIVE &&
       isValidDataSetForSentinel
     ) {
       LoggerProxy.logger.info(
-        `HashTreeParser#checkForSentinelHttpResponse --> ${this.debugId} Received ${error.statusCode} for data set "${dataSetName}", indicating that the meeting has ended`
+        `HashTreeParser#checkForSentinelHttpResponse --> ${this.debugId} Received ${error.statusCode}/${error.body?.errorCode} for data set "${dataSetName}", indicating that the meeting has ended`
       );
       this.stopAllTimers();
 
@@ -1524,7 +1875,7 @@ class HashTreeParser {
    * Gets the current hashes from the locus for a specific data set.
    * @param {string} dataSetName
    * @param {string} currentRootHash
-   * @returns {string[]}
+   * @returns {Object|null} An object containing the hashes and leaf count, or null if the hashes match and no sync is needed
    */
   private getHashesFromLocus(dataSetName: string, currentRootHash: string) {
     LoggerProxy.logger.info(
@@ -1543,6 +1894,15 @@ class HashTreeParser {
       },
     })
       .then((response) => {
+        if (!response.body || isEmpty(response.body)) {
+          // 204 with empty body means our hashes match Locus, no sync needed
+          LoggerProxy.logger.info(
+            `HashTreeParser#getHashesFromLocus --> ${this.debugId} Got ${response.statusCode} with empty body for data set "${dataSetName}", hashes match - no sync needed`
+          );
+
+          return null;
+        }
+
         const hashes = response.body?.hashes as string[] | undefined;
         const dataSetFromResponse = response.body?.dataSet;
 
@@ -1571,6 +1931,13 @@ class HashTreeParser {
           error
         );
         this.checkForSentinelHttpResponse(error, dataSet.name);
+        Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.HASH_TREE_SYNC_FAILURE, {
+          debugId: this.debugId,
+          dataSetName,
+          request: 'GET /hashtree',
+          statusCode: error.statusCode,
+          reason: error.message,
+        });
 
         throw error;
       });
@@ -1580,29 +1947,43 @@ class HashTreeParser {
    * Sends a sync request to Locus for the specified data set.
    *
    * @param {InternalDataSet} dataSet The data set to sync.
-   * @param {Record<number, LeafDataItem[]>} mismatchedLeavesData The mismatched leaves data to include in the sync request.
+   * @param {Object} options Either `{ isInitialization: true }` for init syncs (uses leafCount=1 with empty leaf data) or `{ mismatchedLeavesData }` for normal syncs.
    * @returns {Promise<HashTreeMessage|null>}
    */
   private sendSyncRequestToLocus(
     dataSet: InternalDataSet,
-    mismatchedLeavesData: Record<number, LeafDataItem[]>
+    options: {isInitialization: true} | {mismatchedLeavesData: Record<number, LeafDataItem[]>}
   ): Promise<HashTreeMessage | null> {
     LoggerProxy.logger.info(
       `HashTreeParser#sendSyncRequestToLocus --> ${this.debugId} Sending sync request for data set "${dataSet.name}"`
     );
 
+    const isInitialization = 'isInitialization' in options;
+
     const url = `${dataSet.url}/sync`;
-    const body = {
-      leafCount: dataSet.leafCount,
+    const body: {
+      leafCount: number;
+      leafDataEntries: {leafIndex: number; elementIds: LeafDataItem[]}[];
+    } = {
+      leafCount: isInitialization ? 1 : dataSet.leafCount,
       leafDataEntries: [],
     };
 
-    Object.keys(mismatchedLeavesData).forEach((index) => {
-      body.leafDataEntries.push({
-        leafIndex: parseInt(index, 10),
-        elementIds: mismatchedLeavesData[index],
+    if (isInitialization) {
+      // initialization sync: Locus requires leafCount=1 with a single empty leaf
+      body.leafDataEntries.push({leafIndex: 0, elementIds: []});
+    } else {
+      const {mismatchedLeavesData} = options;
+
+      Object.keys(mismatchedLeavesData).forEach((index) => {
+        const leafIndex = parseInt(index, 10);
+
+        body.leafDataEntries.push({
+          leafIndex,
+          elementIds: mismatchedLeavesData[leafIndex],
+        });
       });
-    });
+    }
 
     const ourCurrentRootHash = dataSet.hashTree ? dataSet.hashTree.getRootHash() : EMPTY_HASH;
 
@@ -1615,10 +1996,6 @@ class HashTreeParser {
       body,
     })
       .then((resp) => {
-        LoggerProxy.logger.info(
-          `HashTreeParser#sendSyncRequestToLocus --> ${this.debugId} Sync request succeeded for "${dataSet.name}"`
-        );
-
         if (!resp.body || isEmpty(resp.body)) {
           LoggerProxy.logger.info(
             `HashTreeParser#sendSyncRequestToLocus --> ${this.debugId} Got ${resp.statusCode} with empty body for sync request for data set "${dataSet.name}", data should arrive via messages`
@@ -1635,6 +2012,13 @@ class HashTreeParser {
           error
         );
         this.checkForSentinelHttpResponse(error, dataSet.name);
+        Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.HASH_TREE_SYNC_FAILURE, {
+          debugId: this.debugId,
+          dataSetName: dataSet.name,
+          request: 'POST /sync',
+          statusCode: error.statusCode,
+          reason: error.message,
+        });
 
         throw error;
       });