@webex/plugin-meetings 3.11.0-next.3 → 3.11.0-next.31

package/src/hashTree/hashTreeParser.ts

@@ -5,7 +5,7 @@ import {Enum, HTTP_VERBS} from '../constants';
 import {DataSetNames, EMPTY_HASH} from './constants';
 import {ObjectType, HtMeta, HashTreeObject} from './types';
 import {LocusDTO} from '../locus-info/types';
-import {deleteNestedObjectsWithHtMeta, isSelf} from './utils';
+import {deleteNestedObjectsWithHtMeta, isMetadata, isSelf} from './utils';
 
 export interface DataSet {
   url: string;
@@ -29,11 +29,24 @@ export interface HashTreeMessage {
   locusStateElements?: Array<HashTreeObject>;
   locusSessionId?: string;
   locusUrl: string;
+  heartbeatIntervalMs?: number;
+}
+
+export interface VisibleDataSetInfo {
+  name: string;
+  url: string;
+  dataChannelUrl?: string;
+}
+
+export interface Metadata {
+  htMeta: HtMeta;
+  visibleDataSets: VisibleDataSetInfo[];
 }
 
 interface InternalDataSet extends DataSet {
   hashTree?: HashTree; // set only for visible data sets
   timer?: ReturnType<typeof setTimeout>;
+  heartbeatWatchdogTimer?: ReturnType<typeof setTimeout>;
 }
 
 type WebexRequestMethod = (options: Record<string, any>) => Promise<any>;
@@ -49,12 +62,24 @@ export type LocusInfoUpdateCallback = (
   data?: {updatedObjects: HashTreeObject[]}
 ) => void;
 
+interface LeafInfo {
+  type: ObjectType;
+  id: number;
+  version: number;
+  data?: any;
+}
+
 /**
  * This error is thrown if we receive information that the meeting has ended while we're processing some hash messages.
  * It's handled internally by HashTreeParser and results in MEETING_ENDED being sent up.
  */
 class MeetingEndedError extends Error {}
 
+/* Currently Locus always sends Metadata objects only in the "self" dataset.
+ * If this ever changes, update all the code that relies on this constant.
+ */
+const MetadataDataSetName = DataSetNames.SELF;
+
 /**
  * Parses hash tree eventing locus data
  */
@@ -63,8 +88,10 @@ class HashTreeParser {
   visibleDataSetsUrl: string; // url from which we can get info about all data sets
   webexRequest: WebexRequestMethod;
   locusInfoUpdateCallback: LocusInfoUpdateCallback;
-  visibleDataSets: string[];
+  visibleDataSets: VisibleDataSetInfo[];
   debugId: string;
+  heartbeatIntervalMs?: number;
+  private excludedDataSets: string[];
 
   /**
    * Constructor for HashTreeParser
@@ -76,25 +103,36 @@ class HashTreeParser {
       dataSets: Array<DataSet>;
       locus: any;
     };
+    metadata: Metadata | null;
     webexRequest: WebexRequestMethod;
     locusInfoUpdateCallback: LocusInfoUpdateCallback;
     debugId: string;
+    excludedDataSets?: string[];
   }) {
     const {dataSets, locus} = options.initialLocus; // extract dataSets from initialLocus
 
     this.debugId = options.debugId;
     this.webexRequest = options.webexRequest;
     this.locusInfoUpdateCallback = options.locusInfoUpdateCallback;
-    this.visibleDataSets = locus?.self?.visibleDataSets || [];
+    this.excludedDataSets = options.excludedDataSets || [];
+    this.visibleDataSetsUrl = locus?.links?.resources?.visibleDataSets?.url;
+    this.visibleDataSets = (
+      cloneDeep(options.metadata?.visibleDataSets || []) as VisibleDataSetInfo[]
+    ).filter((vds) => !this.isExcludedDataSet(vds.name));
 
-    if (this.visibleDataSets.length === 0) {
+    if (options.metadata?.visibleDataSets?.length === 0) {
       LoggerProxy.logger.warn(
-        `HashTreeParser#constructor --> ${this.debugId} No visibleDataSets found in locus.self`
+        `HashTreeParser#constructor --> ${this.debugId} No visibleDataSets found in Metadata`
       );
     }
     // object mapping dataset names to arrays of leaf data
     const leafData = this.analyzeLocusHtMeta(locus);
 
+    if (options.metadata) {
+      // add also the metadata that's outside of locus object itself
+      this.analyzeMetadata(leafData, options.metadata);
+    }
+
     LoggerProxy.logger.info(
       `HashTreeParser#constructor --> creating HashTreeParser for datasets: ${JSON.stringify(
         dataSets.map((ds) => ds.name)
@@ -106,46 +144,87 @@ class HashTreeParser {
 
       this.dataSets[name] = {
         ...dataSet,
-        hashTree: this.visibleDataSets.includes(name)
+        hashTree: this.isVisibleDataSet(name)
           ? new HashTree(leafData[name] || [], leafCount)
           : undefined,
       };
     }
   }
 
+  /**
+   * Checks if the given data set name is in the list of visible data sets
+   * @param {string} dataSetName data set name to check
+   * @returns {Boolean} True if the data set is visible, false otherwise
+   */
+  private isVisibleDataSet(dataSetName: string): boolean {
+    return this.visibleDataSets.some((vds) => vds.name === dataSetName);
+  }
+
+  /**
+   * Checks if the given data set name is in the excluded list
+   * @param {string} dataSetName data set name to check
+   * @returns {boolean} True if the data set is excluded, false otherwise
+   */
+  private isExcludedDataSet(dataSetName: string): boolean {
+    return this.excludedDataSets.some((name) => name === dataSetName);
+  }
+
+  /**
+   * Adds a data set to the visible data sets list, unless it is in the excluded list.
+   * @param {VisibleDataSetInfo} dataSetInfo data set info to add
+   * @returns {boolean} True if the data set was added, false if it was excluded
+   */
+  private addToVisibleDataSetsList(dataSetInfo: VisibleDataSetInfo): boolean {
+    if (this.isExcludedDataSet(dataSetInfo.name)) {
+      LoggerProxy.logger.info(
+        `HashTreeParser#addToVisibleDataSetsList --> ${this.debugId} Data set "${dataSetInfo.name}" is in the excluded list, ignoring`
+      );
+
+      return false;
+    }
+
+    this.visibleDataSets.push(dataSetInfo);
+
+    return true;
+  }
+
   /**
    * Initializes a new visible data set by creating a hash tree for it, adding it to all the internal structures,
    * and sending an initial sync request to Locus with empty leaf data - that will trigger Locus to gives us all the data
    * from that dataset (in the response or via messages).
    *
-   * @param {DataSet} dataSet The new data set to be added
+   * @param {VisibleDataSetInfo} visibleDataSetInfo Information about the new visible data set
+   * @param {DataSet} dataSetInfo The new data set to be added
    * @returns {Promise}
    */
   private initializeNewVisibleDataSet(
-    dataSet: DataSet
+    visibleDataSetInfo: VisibleDataSetInfo,
+    dataSetInfo: DataSet
   ): Promise<{updateType: LocusInfoUpdateType; updatedObjects?: HashTreeObject[]}> {
-    if (this.visibleDataSets.includes(dataSet.name)) {
+    if (this.isVisibleDataSet(dataSetInfo.name)) {
       LoggerProxy.logger.info(
-        `HashTreeParser#initializeNewVisibleDataSet --> ${this.debugId} Data set "${dataSet.name}" already exists, skipping init`
+        `HashTreeParser#initializeNewVisibleDataSet --> ${this.debugId} Data set "${dataSetInfo.name}" already exists, skipping init`
       );
 
       return Promise.resolve({updateType: LocusInfoUpdateType.OBJECTS_UPDATED, updatedObjects: []});
     }
 
     LoggerProxy.logger.info(
-      `HashTreeParser#initializeNewVisibleDataSet --> ${this.debugId} Adding visible data set "${dataSet.name}"`
+      `HashTreeParser#initializeNewVisibleDataSet --> ${this.debugId} Adding visible data set "${dataSetInfo.name}"`
     );
 
-    this.visibleDataSets.push(dataSet.name);
+    if (!this.addToVisibleDataSetsList(visibleDataSetInfo)) {
+      return Promise.resolve({updateType: LocusInfoUpdateType.OBJECTS_UPDATED, updatedObjects: []});
+    }
 
-    const hashTree = new HashTree([], dataSet.leafCount);
+    const hashTree = new HashTree([], dataSetInfo.leafCount);
 
-    this.dataSets[dataSet.name] = {
-      ...dataSet,
+    this.dataSets[dataSetInfo.name] = {
+      ...dataSetInfo,
       hashTree,
     };
 
-    return this.sendInitializationSyncRequestToLocus(dataSet.name, 'new visible data set');
+    return this.sendInitializationSyncRequestToLocus(dataSetInfo.name, 'new visible data set');
   }
 
   /**
@@ -190,15 +269,22 @@ class HashTreeParser {
   }
 
   /**
-   * Queries Locus for information about all the data sets
+   * Queries Locus for all up-to-date information about all visible data sets
    *
-   * @param {string} url - url from which we can get info about all data sets
    * @returns {Promise}
    */
-  private getAllDataSetsMetadata(url) {
+  private getAllVisibleDataSetsFromLocus() {
+    if (!this.visibleDataSetsUrl) {
+      LoggerProxy.logger.warn(
+        `HashTreeParser#getAllVisibleDataSetsFromLocus --> ${this.debugId} No visibleDataSetsUrl, cannot get data sets information`
+      );
+
+      return Promise.resolve([]);
+    }
+
     return this.webexRequest({
       method: HTTP_VERBS.GET,
-      uri: url,
+      uri: this.visibleDataSetsUrl,
     }).then((response) => {
       return response.body.dataSets as Array<DataSet>;
     });
@@ -211,12 +297,14 @@ class HashTreeParser {
    * @returns {Promise}
    */
   async initializeFromMessage(message: HashTreeMessage) {
+    this.visibleDataSetsUrl = message.visibleDataSetsUrl;
+
     LoggerProxy.logger.info(
-      `HashTreeParser#initializeFromMessage --> ${this.debugId} visibleDataSetsUrl=${message.visibleDataSetsUrl}`
+      `HashTreeParser#initializeFromMessage --> ${this.debugId} visibleDataSetsUrl=${this.visibleDataSetsUrl}`
     );
-    const dataSets = await this.getAllDataSetsMetadata(message.visibleDataSetsUrl);
+    const visibleDataSets = await this.getAllVisibleDataSetsFromLocus();
 
-    await this.initializeDataSets(dataSets, 'initialization from message');
+    await this.initializeDataSets(visibleDataSets, 'initialization from message');
   }
 
   /**
@@ -236,28 +324,29 @@ class HashTreeParser {
 
       return;
     }
+    this.visibleDataSetsUrl = locus.links.resources.visibleDataSets.url;
 
     LoggerProxy.logger.info(
-      `HashTreeParser#initializeFromGetLociResponse --> ${this.debugId} visibleDataSets url: ${locus.links.resources.visibleDataSets.url}`
+      `HashTreeParser#initializeFromGetLociResponse --> ${this.debugId} visibleDataSets url: ${this.visibleDataSetsUrl}`
     );
 
-    const dataSets = await this.getAllDataSetsMetadata(locus.links.resources.visibleDataSets.url);
+    const visibleDataSets = await this.getAllVisibleDataSetsFromLocus();
 
-    await this.initializeDataSets(dataSets, 'initialization from GET /loci response');
+    await this.initializeDataSets(visibleDataSets, 'initialization from GET /loci response');
   }
 
   /**
    * Initializes data sets by doing an initialization sync on each visible data set that doesn't have a hash tree yet.
    *
-   * @param {DataSet[]} dataSets Array of DataSet objects to initialize
+   * @param {DataSet[]} visibleDataSets Array of visible DataSet objects to initialize
    * @param {string} debugText Text to include in logs for debugging purposes
    * @returns {Promise}
    */
-  private async initializeDataSets(dataSets: Array<DataSet>, debugText: string) {
+  private async initializeDataSets(visibleDataSets: Array<DataSet>, debugText: string) {
     const updatedObjects: HashTreeObject[] = [];
 
-    for (const dataSet of dataSets) {
-      const {name, leafCount} = dataSet;
+    for (const dataSet of visibleDataSets) {
+      const {name, leafCount, url} = dataSet;
 
       if (!this.dataSets[name]) {
         LoggerProxy.logger.info(
@@ -273,7 +362,20 @@ class HashTreeParser {
         );
       }
 
-      if (this.visibleDataSets.includes(name) && !this.dataSets[name].hashTree) {
+      if (!this.isVisibleDataSet(name)) {
+        if (
+          !this.addToVisibleDataSetsList({
+            name,
+            url,
+          })
+        ) {
+          // dataset is excluded, skip it
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+      }
+
+      if (!this.dataSets[name].hashTree) {
         LoggerProxy.logger.info(
           `HashTreeParser#initializeDataSets --> ${this.debugId} creating hash tree for visible dataset "${name}" (${debugText})`
         );
@@ -316,10 +418,7 @@ class HashTreeParser {
   private analyzeLocusHtMeta(locus: any, options?: {copyData?: boolean}) {
     const {copyData = false} = options || {};
     // object mapping dataset names to arrays of leaf data
-    const leafInfo: Record<
-      string,
-      Array<{type: ObjectType; id: number; version: number; data?: any}>
-    > = {};
+    const leafInfo: Record<string, Array<LeafInfo>> = {};
 
     const findAndStoreMetaData = (currentLocusPart: any) => {
       if (typeof currentLocusPart !== 'object' || currentLocusPart === null) {
@@ -329,7 +428,7 @@ class HashTreeParser {
       if (currentLocusPart.htMeta && currentLocusPart.htMeta.dataSetNames) {
         const {type, id, version} = currentLocusPart.htMeta.elementId;
         const {dataSetNames} = currentLocusPart.htMeta;
-        const newLeafInfo: {type: ObjectType; id: number; version: number; data?: any} = {
+        const newLeafInfo: LeafInfo = {
           type,
           id,
           version,
@@ -368,6 +467,43 @@ class HashTreeParser {
     return leafInfo;
   }
 
+  /**
+   * Analyzes the Metadata object that is sent outside of Locus object, and appends its data to passed in leafInfo
+   * structure.
+   *
+   * @param {Record<string, LeafInfo[]>} leafInfo the structure to which the Metadata info will be appended
+   * @param {Metadata} metadata Metadata object
+   * @returns {void}
+   */
+  private analyzeMetadata(leafInfo: Record<string, LeafInfo[]>, metadata: Metadata) {
+    const {htMeta} = metadata;
+
+    if (
+      htMeta?.dataSetNames?.length === 1 &&
+      htMeta.dataSetNames[0].toLowerCase() === MetadataDataSetName
+    ) {
+      const {type, id, version} = metadata.htMeta.elementId;
+
+      const dataSetName = htMeta.dataSetNames[0];
+
+      if (!leafInfo[dataSetName]) {
+        leafInfo[dataSetName] = [];
+      }
+
+      leafInfo[dataSetName].push({
+        type,
+        id,
+        version,
+      });
+    } else {
+      throw new Error(
+        `${this.debugId} Metadata htMeta has unexpected dataSetNames: ${
+          htMeta && htMeta.dataSetNames.join(',')
+        }`
+      );
+    }
+  }
+
   /**
    * Checks if the provided hash tree message indicates the end of the meeting and that there won't be any more updates.
    *
@@ -420,6 +556,58 @@ class HashTreeParser {
     });
   }
 
+  /**
+   * Handles updates to Metadata object that we receive from Locus via other means than messages. Right now
+   * that means only in the API response alongside locus object.
+   *
+   * @param {Metadata} metadata received in Locus update other than a message (for example in an API response)
+   * @param {HashTreeObject[]} updatedObjects a list of updated hash tree objects to which any updates resulting from new Metadata will be added
+   * @returns {void}
+   */
+  handleMetadataUpdate(metadata: Metadata, updatedObjects: HashTreeObject[]): void {
+    let dataSetsRequiringInitialization: VisibleDataSetInfo[] = [];
+
+    // current assumption based on Locus docs is that Metadata object lives always in "self" data set
+    const hashTree = this.dataSets[MetadataDataSetName]?.hashTree;
+
+    if (!hashTree) {
+      LoggerProxy.logger.warn(
+        `HashTreeParser#handleLocusUpdate --> ${this.debugId} received Metadata object but no hash tree for "${MetadataDataSetName}" data set exists`
+      );
+    } else {
+      const metadataUpdated = hashTree.putItem(metadata.htMeta.elementId);
+
+      if (metadataUpdated) {
+        // metadata in Locus API response is in a slightly different format than the objects in messages, so need to adapt it
+        const metadataObject: HashTreeObject = {
+          htMeta: metadata.htMeta,
+          data: metadata,
+        };
+
+        updatedObjects.push(metadataObject);
+
+        const {changeDetected, removedDataSets, addedDataSets} = this.checkForVisibleDataSetChanges(
+          [metadataObject]
+        );
+
+        if (changeDetected) {
+          dataSetsRequiringInitialization = this.processVisibleDataSetChanges(
+            removedDataSets,
+            addedDataSets,
+            updatedObjects
+          );
+        }
+
+        if (dataSetsRequiringInitialization.length > 0) {
+          // there are some data sets that we need to initialize asynchronously
+          queueMicrotask(() => {
+            this.initializeNewVisibleDataSets(dataSetsRequiringInitialization);
+          });
+        }
+      }
+    }
+  }
+
   /**
    * This method should be called when we receive a partial locus DTO that contains dataSets and htMeta information
    * It updates the hash trees with the new leaf data based on the received Locus
@@ -427,22 +615,28 @@ class HashTreeParser {
    * @param {Object} update - The locus update containing data sets and locus information
    * @returns {void}
    */
-  handleLocusUpdate(update: {dataSets?: Array<DataSet>; locus: any}): void {
-    const {dataSets, locus} = update;
+  handleLocusUpdate(update: {dataSets?: Array<DataSet>; locus: any; metadata?: Metadata}): void {
+    const {dataSets, locus, metadata} = update;
 
     if (!dataSets) {
-      LoggerProxy.logger.warn(
+      LoggerProxy.logger.info(
         `HashTreeParser#handleLocusUpdate --> ${this.debugId} received hash tree update without dataSets`
       );
-    }
-    for (const dataSet of dataSets) {
-      this.updateDataSetInfo(dataSet);
+    } else {
+      for (const dataSet of dataSets) {
+        this.updateDataSetInfo(dataSet);
+      }
     }
     const updatedObjects: HashTreeObject[] = [];
 
     // first, analyze the locus object to extract the hash tree objects' htMeta and data from it
     const leafInfo = this.analyzeLocusHtMeta(locus, {copyData: true});
 
+    // if we got metadata, process it (currently that means only potential visible data set list changes)
+    if (metadata) {
+      this.handleMetadataUpdate(metadata, updatedObjects);
+    }
+
     // then process the data in hash trees, if it is a new version, then add it to updatedObjects
     Object.keys(leafInfo).forEach((dataSetName) => {
       if (this.dataSets[dataSetName]) {
@@ -493,9 +687,6 @@ class HashTreeParser {
         updatedObjects,
       });
     }
-
-    // todo: once Locus design on how visible data sets will be communicated in subsequent API responses is confirmed,
-    // we'll need to check here if visible data sets have changed and update this.visibleDataSets, remove/create hash trees etc
   }
 
   /**
@@ -511,7 +702,7 @@ class HashTreeParser {
       };
 
       LoggerProxy.logger.info(
-        `HashTreeParser#handleMessage --> ${this.debugId} created entry for "${receivedDataSet.name}" dataset: version=${receivedDataSet.version}, root=${receivedDataSet.root}`
+        `HashTreeParser#updateDataSetInfo --> ${this.debugId} created entry for "${receivedDataSet.name}" dataset: version=${receivedDataSet.version}, root=${receivedDataSet.root}`
       );
 
       return;
@@ -526,7 +717,7 @@ class HashTreeParser {
         exponent: receivedDataSet.backoff.exponent,
       };
       LoggerProxy.logger.info(
-        `HashTreeParser#handleMessage --> ${this.debugId} updated "${receivedDataSet.name}" to version=${receivedDataSet.version}, root=${receivedDataSet.root}`
+        `HashTreeParser#updateDataSetInfo --> ${this.debugId} updated "${receivedDataSet.name}" dataset to version=${receivedDataSet.version}, root=${receivedDataSet.root}`
       );
     }
   }
@@ -537,25 +728,30 @@ class HashTreeParser {
    * @returns {Object} An object containing the removed and added visible data sets.
    */
   private checkForVisibleDataSetChanges(updatedObjects: HashTreeObject[]) {
-    let removedDataSets: string[] = [];
-    let addedDataSets: string[] = [];
+    let removedDataSets: VisibleDataSetInfo[] = [];
+    let addedDataSets: VisibleDataSetInfo[] = [];
 
-    // visibleDataSets can only be changed by self object updates
+    // visibleDataSets can only be changed by Metadata object updates
     updatedObjects.forEach((object) => {
-      // todo: in the future visibleDataSets will be in "Metadata" object, not in "self"
-      if (isSelf(object) && object.data?.visibleDataSets) {
-        const newVisibleDataSets = object.data.visibleDataSets;
+      if (isMetadata(object) && object.data?.visibleDataSets) {
+        const newVisibleDataSets = object.data.visibleDataSets.filter(
+          (vds) => !this.isExcludedDataSet(vds.name)
+        );
 
-        removedDataSets = this.visibleDataSets.filter((ds) => !newVisibleDataSets.includes(ds));
-        addedDataSets = newVisibleDataSets.filter((ds) => !this.visibleDataSets.includes(ds));
+        removedDataSets = this.visibleDataSets.filter(
+          (ds) => !newVisibleDataSets.some((nvs) => nvs.name === ds.name)
+        );
+        addedDataSets = newVisibleDataSets.filter((nvs) =>
+          this.visibleDataSets.every((ds) => ds.name !== nvs.name)
+        );
 
         if (removedDataSets.length > 0 || addedDataSets.length > 0) {
           LoggerProxy.logger.info(
             `HashTreeParser#checkForVisibleDataSetChanges --> ${
               this.debugId
-            } visible data sets change: removed: ${removedDataSets.join(
-              ', '
-            )}, added: ${addedDataSets.join(', ')}`
+            } visible data sets change: removed: ${removedDataSets
+              .map((ds) => ds.name)
+              .join(', ')}, added: ${addedDataSets.map((ds) => ds.name).join(', ')}`
           );
         }
       }
@@ -577,11 +773,15 @@ class HashTreeParser {
   private deleteHashTree(dataSetName: string) {
     this.dataSets[dataSetName].hashTree = undefined;
 
-    // we also need to stop the timer as there is no hash tree anymore to sync
+    // we also need to stop the timers as there is no hash tree anymore to sync
     if (this.dataSets[dataSetName].timer) {
       clearTimeout(this.dataSets[dataSetName].timer);
       this.dataSets[dataSetName].timer = undefined;
     }
+    if (this.dataSets[dataSetName].heartbeatWatchdogTimer) {
+      clearTimeout(this.dataSets[dataSetName].heartbeatWatchdogTimer);
+      this.dataSets[dataSetName].heartbeatWatchdogTimer = undefined;
+    }
   }
 
   /**
@@ -593,49 +793,51 @@ class HashTreeParser {
    * visible data sets and they require async initialization, the names of these data sets
    * are returned in an array.
    *
-   * @param {string[]} removedDataSets - The list of removed data sets.
-   * @param {string[]} addedDataSets - The list of added data sets.
+   * @param {VisibleDataSetInfo[]} removedDataSets - The list of removed data sets.
+   * @param {VisibleDataSetInfo[]} addedDataSets - The list of added data sets.
    * @param {HashTreeObject[]} updatedObjects - The list of updated hash tree objects to which changes will be added.
-   * @returns {string[]} names of data sets that couldn't be initialized synchronously
+   * @returns {VisibleDataSetInfo[]} list of data sets that couldn't be initialized synchronously
    */
   private processVisibleDataSetChanges(
-    removedDataSets: string[],
-    addedDataSets: string[],
+    removedDataSets: VisibleDataSetInfo[],
+    addedDataSets: VisibleDataSetInfo[],
     updatedObjects: HashTreeObject[]
-  ): string[] {
-    const dataSetsRequiringInitialization = [];
+  ): VisibleDataSetInfo[] {
+    const dataSetsRequiringInitialization: VisibleDataSetInfo[] = [];
 
     // if a visible data set was removed, we need to tell our client that all objects from it are removed
     const removedObjects: HashTreeObject[] = [];
 
     removedDataSets.forEach((ds) => {
-      if (this.dataSets[ds]?.hashTree) {
-        for (let i = 0; i < this.dataSets[ds].hashTree.numLeaves; i += 1) {
+      if (this.dataSets[ds.name]?.hashTree) {
+        for (let i = 0; i < this.dataSets[ds.name].hashTree.numLeaves; i += 1) {
           removedObjects.push(
-            ...this.dataSets[ds].hashTree.getLeafData(i).map((elementId) => ({
+            ...this.dataSets[ds.name].hashTree.getLeafData(i).map((elementId) => ({
               htMeta: {
                 elementId,
-                dataSetNames: [ds],
+                dataSetNames: [ds.name],
               },
               data: null,
             }))
           );
         }
 
-        this.deleteHashTree(ds);
+        this.deleteHashTree(ds.name);
       }
     });
-    this.visibleDataSets = this.visibleDataSets.filter((vds) => !removedDataSets.includes(vds));
+    this.visibleDataSets = this.visibleDataSets.filter(
+      (vds) => !removedDataSets.some((rds) => rds.name === vds.name)
+    );
     updatedObjects.push(...removedObjects);
 
     // now setup the new visible data sets
     for (const ds of addedDataSets) {
-      const dataSetInfo = this.dataSets[ds];
+      const dataSetInfo = this.dataSets[ds.name];
 
       if (dataSetInfo) {
-        if (this.visibleDataSets.includes(dataSetInfo.name)) {
+        if (this.isVisibleDataSet(dataSetInfo.name)) {
           LoggerProxy.logger.info(
-            `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} Data set "${ds}" is already visible, skipping`
+            `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} Data set "${ds.name}" is already visible, skipping`
           );
 
           // eslint-disable-next-line no-continue
@@ -643,10 +845,13 @@ class HashTreeParser {
         }
 
         LoggerProxy.logger.info(
-          `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} Adding visible data set "${ds}"`
+          `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} Adding visible data set "${ds.name}"`
         );
 
-        this.visibleDataSets.push(ds);
+        if (!this.addToVisibleDataSetsList(ds)) {
+          // eslint-disable-next-line no-continue
+          continue;
+        }
 
         const hashTree = new HashTree([], dataSetInfo.leafCount);
 
@@ -656,7 +861,7 @@ class HashTreeParser {
         };
       } else {
         LoggerProxy.logger.info(
-          `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} visible data set "${ds}" added but no info about it in our dataSets structures`
+          `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} visible data set "${ds.name}" added but no info about it in our dataSets structures`
         );
         // todo: add a metric here
         dataSetsRequiringInitialization.push(ds);
@@ -670,32 +875,28 @@ class HashTreeParser {
    * Adds entries to the passed in updateObjects array
    * for the changes that result from adding and removing visible data sets.
    *
-   * @param {HashTreeMessage} message - The hash tree message that triggered the visible data set changes.
-   * @param {string[]} addedDataSets - The list of added data sets.
+   * @param {VisibleDataSetInfo[]} addedDataSets - The list of added data sets.
    * @returns {Promise<void>}
    */
-  private async initializeNewVisibleDataSets(
-    message: HashTreeMessage,
-    addedDataSets: string[]
-  ): Promise<void> {
-    const allDataSets = await this.getAllDataSetsMetadata(message.visibleDataSetsUrl);
+  private async initializeNewVisibleDataSets(addedDataSets: VisibleDataSetInfo[]): Promise<void> {
+    const allDataSets = await this.getAllVisibleDataSetsFromLocus();
 
     for (const ds of addedDataSets) {
-      const dataSetInfo = allDataSets.find((d) => d.name === ds);
+      const dataSetInfo = allDataSets.find((d) => d.name === ds.name);
 
       LoggerProxy.logger.info(
-        `HashTreeParser#initializeNewVisibleDataSets --> ${this.debugId} initializing data set "${ds}"`
+        `HashTreeParser#initializeNewVisibleDataSets --> ${this.debugId} initializing data set "${ds.name}"`
       );
 
       if (!dataSetInfo) {
         LoggerProxy.logger.warn(
-          `HashTreeParser#handleHashTreeMessage --> ${this.debugId} missing info about data set "${ds}" in Locus response from visibleDataSetsUrl`
+          `HashTreeParser#handleHashTreeMessage --> ${this.debugId} missing info about data set "${ds.name}" in Locus response from visibleDataSetsUrl`
         );
       } else {
         // we're awaiting in a loop, because in practice there will be only one new data set at a time,
         // so no point in trying to parallelize this
         // eslint-disable-next-line no-await-in-loop
-        const updates = await this.initializeNewVisibleDataSet(dataSetInfo);
+        const updates = await this.initializeNewVisibleDataSet(ds, dataSetInfo);
 
         this.callLocusInfoUpdateCallback(updates);
       }
@@ -746,32 +947,31 @@ class HashTreeParser {
     // available in the message, they will require separate async initialization
     let dataSetsRequiringInitialization = [];
 
-    // first find out if there are any visible data set changes - they're signalled in SELF object updates
-    const selfUpdates = (message.locusStateElements || []).filter((object) =>
-      // todo: SPARK-744859 once Locus supports it, we will filter for "Metadata" type here instead of "self"
-      isSelf(object)
+    // first find out if there are any visible data set changes - they're signalled in Metadata object updates
+    const metadataUpdates = (message.locusStateElements || []).filter((object) =>
+      isMetadata(object)
     );
 
-    if (selfUpdates.length > 0) {
-      const updatedSelfObjects = [];
+    if (metadataUpdates.length > 0) {
+      const updatedMetadataObjects = [];
 
-      selfUpdates.forEach((object) => {
+      metadataUpdates.forEach((object) => {
         // todo: once Locus supports it, we will use the "view" field here instead of dataSetNames
         for (const dataSetName of object.htMeta.dataSetNames) {
           const hashTree = this.dataSets[dataSetName]?.hashTree;
 
           if (hashTree && object.data) {
             if (hashTree.putItem(object.htMeta.elementId)) {
-              updatedSelfObjects.push(object);
+              updatedMetadataObjects.push(object);
             }
           }
         }
       });
 
-      updatedObjects.push(...updatedSelfObjects);
+      updatedObjects.push(...updatedMetadataObjects);
 
       const {changeDetected, removedDataSets, addedDataSets} =
-        this.checkForVisibleDataSetChanges(updatedSelfObjects);
+        this.checkForVisibleDataSetChanges(updatedMetadataObjects);
 
       if (changeDetected) {
         dataSetsRequiringInitialization = this.processVisibleDataSetChanges(
@@ -782,48 +982,50 @@ class HashTreeParser {
       }
     }
 
-    // by this point we now have this.dataSets setup for data sets from this message
-    // and hash trees created for the new visible data sets,
-    // so we can now process all the updates from the message
-    dataSets.forEach((dataSet) => {
-      if (this.dataSets[dataSet.name]) {
-        const {hashTree} = this.dataSets[dataSet.name];
-
-        if (hashTree) {
-          const locusStateElementsForThisSet = message.locusStateElements.filter((object) =>
-            object.htMeta.dataSetNames.includes(dataSet.name)
-          );
-
-          const appliedChangesList = hashTree.updateItems(
-            locusStateElementsForThisSet.map((object) =>
-              object.data
-                ? {operation: 'update', item: object.htMeta.elementId}
-                : {operation: 'remove', item: object.htMeta.elementId}
-            )
-          );
-
-          zip(appliedChangesList, locusStateElementsForThisSet).forEach(
-            ([changeApplied, object]) => {
-              if (changeApplied) {
-                if (isSelf(object) && !object.data) {
-                  isRosterDropped = true;
+    if (message.locusStateElements?.length > 0) {
+      // by this point we now have this.dataSets setup for data sets from this message
+      // and hash trees created for the new visible data sets,
+      // so we can now process all the updates from the message
+      dataSets.forEach((dataSet) => {
+        if (this.dataSets[dataSet.name]) {
+          const {hashTree} = this.dataSets[dataSet.name];
+
+          if (hashTree) {
+            const locusStateElementsForThisSet = message.locusStateElements.filter((object) =>
+              object.htMeta.dataSetNames.includes(dataSet.name)
+            );
+
+            const appliedChangesList = hashTree.updateItems(
+              locusStateElementsForThisSet.map((object) =>
+                object.data
+                  ? {operation: 'update', item: object.htMeta.elementId}
+                  : {operation: 'remove', item: object.htMeta.elementId}
+              )
+            );
+
+            zip(appliedChangesList, locusStateElementsForThisSet).forEach(
+              ([changeApplied, object]) => {
+                if (changeApplied) {
+                  if (isSelf(object) && !object.data) {
+                    isRosterDropped = true;
+                  }
+                  // add to updatedObjects so that our locus DTO will get updated with the new object
+                  updatedObjects.push(object);
                 }
-                // add to updatedObjects so that our locus DTO will get updated with the new object
-                updatedObjects.push(object);
               }
-            }
-          );
-        } else {
-          LoggerProxy.logger.info(
-            `Locus-info:index#parseMessage --> ${this.debugId} unexpected (not visible) dataSet ${dataSet.name} received in hash tree message`
-          );
+            );
+          } else {
+            LoggerProxy.logger.info(
+              `Locus-info:index#parseMessage --> ${this.debugId} unexpected (not visible) dataSet ${dataSet.name} received in hash tree message`
+            );
+          }
         }
-      }
 
-      if (!isRosterDropped) {
-        this.runSyncAlgorithm(dataSet);
-      }
-    });
+        if (!isRosterDropped) {
+          this.runSyncAlgorithm(dataSet);
+        }
+      });
+    }
 
     if (isRosterDropped) {
       LoggerProxy.logger.info(
@@ -838,7 +1040,7 @@ class HashTreeParser {
     if (dataSetsRequiringInitialization.length > 0) {
       // there are some data sets that we need to initialize asynchronously
       queueMicrotask(() => {
-        this.initializeNewVisibleDataSets(message, dataSetsRequiringInitialization);
+        this.initializeNewVisibleDataSets(dataSetsRequiringInitialization);
       });
     }
 
@@ -859,11 +1061,20 @@ class HashTreeParser {
    * @returns {void}
    */
   async handleMessage(message: HashTreeMessage, debugText?: string): Promise<void> {
+    if (message.heartbeatIntervalMs) {
+      this.heartbeatIntervalMs = message.heartbeatIntervalMs;
+    }
     if (message.locusStateElements === undefined) {
       this.handleRootHashHeartBeatMessage(message);
+      this.resetHeartbeatWatchdogs(message.dataSets);
     } else {
       const updates = await this.parseMessage(message, debugText);
 
+      // Only reset watchdogs if the meeting hasn't ended
+      if (updates.updateType !== LocusInfoUpdateType.MEETING_ENDED) {
+        this.resetHeartbeatWatchdogs(message.dataSets);
+      }
+
       this.callLocusInfoUpdateCallback(updates);
     }
   }
@@ -880,7 +1091,49 @@ class HashTreeParser {
   }) {
     const {updateType, updatedObjects} = updates;
 
-    if (updateType !== LocusInfoUpdateType.OBJECTS_UPDATED || updatedObjects?.length > 0) {
+    if (updateType === LocusInfoUpdateType.OBJECTS_UPDATED && updatedObjects?.length > 0) {
+      // Filter out updates for objects that already have a higher version in their datasets,
+      // or removals for objects that still exist in any of their datasets
+      const filteredUpdates = updatedObjects.filter((object) => {
+        const {elementId} = object.htMeta;
+        const {type, id, version} = elementId;
+
+        // Check all datasets
+        for (const dataSetName of Object.keys(this.dataSets)) {
+          const dataSet = this.dataSets[dataSetName];
+
+          // only visible datasets have hash trees set
+          if (dataSet?.hashTree) {
+            const existingVersion = dataSet.hashTree.getItemVersion(id, type);
+            if (existingVersion !== undefined) {
+              if (object.data) {
+                // For updates: filter out if any dataset has a higher version
+                if (existingVersion > version) {
+                  LoggerProxy.logger.info(
+                    `HashTreeParser#callLocusInfoUpdateCallback --> ${this.debugId} Filtering out update for ${type}:${id} v${version} because dataset "${dataSetName}" has v${existingVersion}`
+                  );
+
+                  return false;
+                }
+              } else if (existingVersion >= version) {
+                // For removals: filter out if the object still exists in any dataset
+                LoggerProxy.logger.info(
+                  `HashTreeParser#callLocusInfoUpdateCallback --> ${this.debugId} Filtering out removal for ${type}:${id} v${version} because dataset "${dataSetName}" still has v${existingVersion}`
+                );
+
+                return false;
+              }
+            }
+          }
+        }
+
+        return true;
+      });
+
+      if (filteredUpdates.length > 0) {
+        this.locusInfoUpdateCallback(updateType, {updatedObjects: filteredUpdates});
+      }
+    } else if (updateType !== LocusInfoUpdateType.OBJECTS_UPDATED) {
       this.locusInfoUpdateCallback(updateType, {updatedObjects});
     }
   }
@@ -899,6 +1152,75 @@ class HashTreeParser {
     return Math.round(randomValue ** exponent * maxMs);
   }
 
+  /**
+   * Performs a sync for the given data set.
+   *
+   * @param {InternalDataSet} dataSet - The data set to sync
+   * @param {string} rootHash - Our current root hash for this data set
+   * @param {string} reason - The reason for the sync (used for logging)
+   * @returns {Promise<void>}
+   */
+  private async performSync(
+    dataSet: InternalDataSet,
+    rootHash: string,
+    reason: string
+  ): Promise<void> {
+    if (!dataSet.hashTree) {
+      return;
+    }
+
+    LoggerProxy.logger.info(
+      `HashTreeParser#performSync --> ${this.debugId} ${reason}, syncing data set "${dataSet.name}"`
+    );
+
+    const mismatchedLeavesData: Record<number, LeafDataItem[]> = {};
+
+    if (dataSet.leafCount !== 1) {
+      let receivedHashes;
+
+      try {
+        // request hashes from sender
+        const {hashes, dataSet: latestDataSetInfo} = await this.getHashesFromLocus(
+          dataSet.name,
+          rootHash
+        );
+
+        receivedHashes = hashes;
+
+        dataSet.hashTree.resize(latestDataSetInfo.leafCount);
+      } catch (error) {
+        if (error.statusCode === 409) {
+          // this is a leaf count mismatch, we should do nothing, just wait for another heartbeat message from Locus
+          LoggerProxy.logger.info(
+            `HashTreeParser#getHashesFromLocus --> ${this.debugId} Got 409 when fetching hashes for data set "${dataSet.name}": ${error.message}`
+          );
+
+          return;
+        }
+        throw error;
+      }
+
+      // identify mismatched leaves
+      const mismatchedLeaveIndexes = dataSet.hashTree.diffHashes(receivedHashes);
+
+      mismatchedLeaveIndexes.forEach((index) => {
+        mismatchedLeavesData[index] = dataSet.hashTree.getLeafData(index);
+      });
+    } else {
+      mismatchedLeavesData[0] = dataSet.hashTree.getLeafData(0);
+    }
+    // request sync for mismatched leaves
+    if (Object.keys(mismatchedLeavesData).length > 0) {
+      const syncResponse = await this.sendSyncRequestToLocus(dataSet, mismatchedLeavesData);
+
+      // sync API may return nothing (in that case data will arrive via messages)
+      // or it may return a response in the same format as messages
+      if (syncResponse) {
+        this.handleMessage(syncResponse, 'via sync API');
+      }
+    }
+  }
+
   /**
    * Runs the sync algorithm for the given data set.
    *
@@ -957,55 +1279,11 @@ class HashTreeParser {
         const rootHash = dataSet.hashTree.getRootHash();
 
         if (dataSet.root !== rootHash) {
-          LoggerProxy.logger.info(
-            `HashTreeParser#runSyncAlgorithm --> ${this.debugId} Root hash mismatch: received=${dataSet.root}, ours=${rootHash}, syncing data set "${dataSet.name}"`
+          await this.performSync(
+            dataSet,
+            rootHash,
+            `Root hash mismatch: received=${dataSet.root}, ours=${rootHash}`
           );
-
-          const mismatchedLeavesData: Record<number, LeafDataItem[]> = {};
-
-          if (dataSet.leafCount !== 1) {
-            let receivedHashes;
-
-            try {
-              // request hashes from sender
-              const {hashes, dataSet: latestDataSetInfo} = await this.getHashesFromLocus(
-                dataSet.name
-              );
-
-              receivedHashes = hashes;
-
-              dataSet.hashTree.resize(latestDataSetInfo.leafCount);
-            } catch (error) {
-              if (error.statusCode === 409) {
-                // this is a leaf count mismatch, we should do nothing, just wait for another heartbeat message from Locus
-                LoggerProxy.logger.info(
-                  `HashTreeParser#getHashesFromLocus --> ${this.debugId} Got 409 when fetching hashes for data set "${dataSet.name}": ${error.message}`
-                );
-
-                return;
-              }
-              throw error;
-            }
-
-            // identify mismatched leaves
-            const mismatchedLeaveIndexes = dataSet.hashTree.diffHashes(receivedHashes);
-
-            mismatchedLeaveIndexes.forEach((index) => {
-              mismatchedLeavesData[index] = dataSet.hashTree.getLeafData(index);
-            });
-          } else {
-            mismatchedLeavesData[0] = dataSet.hashTree.getLeafData(0);
-          }
-          // request sync for mismatched leaves
-          if (Object.keys(mismatchedLeavesData).length > 0) {
-            const syncResponse = await this.sendSyncRequestToLocus(dataSet, mismatchedLeavesData);
-
-            // sync API may return nothing (in that case data will arrive via messages)
-            // or it may return a response in the same format as messages
-            if (syncResponse) {
-              this.handleMessage(syncResponse, 'via sync API');
-            }
-          }
         } else {
           LoggerProxy.logger.info(
             `HashTreeParser#runSyncAlgorithm --> ${this.debugId} "${dataSet.name}" root hash matching: ${rootHash}, version=${dataSet.version}`
@@ -1019,6 +1297,52 @@ class HashTreeParser {
     }
   }
 
+  /**
+   * Resets the heartbeat watchdog timers for the specified data sets. Each data set has its own
+   * watchdog timer that monitors whether heartbeats are being received within the expected interval.
+   * If a heartbeat is not received for a specific data set within heartbeatIntervalMs plus
+   * a backoff-calculated time, the sync algorithm is initiated for that data set
+   *
+   * @param {Array<DataSet>} receivedDataSets - The data sets from the received message for which watchdog timers should be reset
+   * @returns {void}
+   */
+  private resetHeartbeatWatchdogs(receivedDataSets: Array<DataSet>): void {
+    if (!this.heartbeatIntervalMs) {
+      return;
+    }
+
+    for (const receivedDataSet of receivedDataSets) {
+      const dataSet = this.dataSets[receivedDataSet.name];
+
+      if (!dataSet?.hashTree) {
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
+      if (dataSet.heartbeatWatchdogTimer) {
+        clearTimeout(dataSet.heartbeatWatchdogTimer);
+        dataSet.heartbeatWatchdogTimer = undefined;
+      }
+
+      const backoffTime = this.getWeightedBackoffTime(dataSet.backoff);
+      const delay = this.heartbeatIntervalMs + backoffTime;
+
+      dataSet.heartbeatWatchdogTimer = setTimeout(async () => {
+        dataSet.heartbeatWatchdogTimer = undefined;
+
+        LoggerProxy.logger.warn(
+          `HashTreeParser#resetHeartbeatWatchdogs --> ${this.debugId} Heartbeat watchdog fired for data set "${dataSet.name}" - no heartbeat received within expected interval, initiating sync`
+        );
+
+        await this.performSync(
+          dataSet,
+          dataSet.hashTree.getRootHash(),
+          `heartbeat watchdog expired`
+        );
+      }, delay);
+    }
+  }
+
   /**
    * Stops all timers for the data sets to prevent any further sync attempts.
    * @returns {void}
@@ -1029,15 +1353,20 @@ class HashTreeParser {
         clearTimeout(dataSet.timer);
         dataSet.timer = undefined;
       }
+      if (dataSet.heartbeatWatchdogTimer) {
+        clearTimeout(dataSet.heartbeatWatchdogTimer);
+        dataSet.heartbeatWatchdogTimer = undefined;
+      }
     });
   }
 
   /**
    * Gets the current hashes from the locus for a specific data set.
    * @param {string} dataSetName
+   * @param {string} currentRootHash
    * @returns {string[]}
    */
-  private getHashesFromLocus(dataSetName: string) {
+  private getHashesFromLocus(dataSetName: string, currentRootHash: string) {
     LoggerProxy.logger.info(
       `HashTreeParser#getHashesFromLocus --> ${this.debugId} Requesting hashes for data set "${dataSetName}"`
     );
@@ -1049,6 +1378,9 @@ class HashTreeParser {
     return this.webexRequest({
       method: HTTP_VERBS.GET,
       uri: url,
+      qs: {
+        rootHash: currentRootHash,
+      },
     })
       .then((response) => {
         const hashes = response.body?.hashes as string[] | undefined;
@@ -1110,9 +1442,14 @@ class HashTreeParser {
       });
     });
 
+    const ourCurrentRootHash = dataSet.hashTree ? dataSet.hashTree.getRootHash() : EMPTY_HASH;
+
     return this.webexRequest({
       method: HTTP_VERBS.POST,
       uri: url,
+      qs: {
+        rootHash: ourCurrentRootHash,
+      },
       body,
     })
       .then((resp) => {