@webex/plugin-meetings 3.11.0-next.2 → 3.11.0-next.21

package/src/hashTree/hashTreeParser.ts

@@ -5,7 +5,7 @@ import {Enum, HTTP_VERBS} from '../constants';
 import {DataSetNames, EMPTY_HASH} from './constants';
 import {ObjectType, HtMeta, HashTreeObject} from './types';
 import {LocusDTO} from '../locus-info/types';
-import {deleteNestedObjectsWithHtMeta, isSelf} from './utils';
+import {deleteNestedObjectsWithHtMeta, isMetadata, isSelf} from './utils';
 
 export interface DataSet {
   url: string;
@@ -31,6 +31,17 @@ export interface HashTreeMessage {
   locusUrl: string;
 }
 
+export interface VisibleDataSetInfo {
+  name: string;
+  url: string;
+  dataChannelUrl?: string;
+}
+
+export interface Metadata {
+  htMeta: HtMeta;
+  visibleDataSets: VisibleDataSetInfo[];
+}
+
 interface InternalDataSet extends DataSet {
   hashTree?: HashTree; // set only for visible data sets
   timer?: ReturnType<typeof setTimeout>;
@@ -49,12 +60,24 @@ export type LocusInfoUpdateCallback = (
   data?: {updatedObjects: HashTreeObject[]}
 ) => void;
 
+interface LeafInfo {
+  type: ObjectType;
+  id: number;
+  version: number;
+  data?: any;
+}
+
 /**
  * This error is thrown if we receive information that the meeting has ended while we're processing some hash messages.
  * It's handled internally by HashTreeParser and results in MEETING_ENDED being sent up.
  */
 class MeetingEndedError extends Error {}
 
+/* Currently Locus always sends Metadata objects only in the "self" dataset.
+ * If this ever changes, update all the code that relies on this constant.
+ */
+const MetadataDataSetName = DataSetNames.SELF;
+
 /**
  * Parses hash tree eventing locus data
  */
@@ -63,7 +86,7 @@ class HashTreeParser {
   visibleDataSetsUrl: string; // url from which we can get info about all data sets
   webexRequest: WebexRequestMethod;
   locusInfoUpdateCallback: LocusInfoUpdateCallback;
-  visibleDataSets: string[];
+  visibleDataSets: VisibleDataSetInfo[];
   debugId: string;
 
   /**
@@ -76,6 +99,7 @@ class HashTreeParser {
       dataSets: Array<DataSet>;
       locus: any;
     };
+    metadata: Metadata | null;
     webexRequest: WebexRequestMethod;
     locusInfoUpdateCallback: LocusInfoUpdateCallback;
     debugId: string;
@@ -85,16 +109,22 @@ class HashTreeParser {
     this.debugId = options.debugId;
     this.webexRequest = options.webexRequest;
     this.locusInfoUpdateCallback = options.locusInfoUpdateCallback;
-    this.visibleDataSets = locus?.self?.visibleDataSets || [];
+    this.visibleDataSetsUrl = locus?.links?.resources?.visibleDataSets?.url;
+    this.visibleDataSets = cloneDeep(options.metadata?.visibleDataSets || []);
 
-    if (this.visibleDataSets.length === 0) {
+    if (options.metadata?.visibleDataSets?.length === 0) {
       LoggerProxy.logger.warn(
-        `HashTreeParser#constructor --> ${this.debugId} No visibleDataSets found in locus.self`
+        `HashTreeParser#constructor --> ${this.debugId} No visibleDataSets found in Metadata`
       );
     }
     // object mapping dataset names to arrays of leaf data
     const leafData = this.analyzeLocusHtMeta(locus);
 
+    if (options.metadata) {
+      // add also the metadata that's outside of locus object itself
+      this.analyzeMetadata(leafData, options.metadata);
+    }
+
     LoggerProxy.logger.info(
       `HashTreeParser#constructor --> creating HashTreeParser for datasets: ${JSON.stringify(
         dataSets.map((ds) => ds.name)
@@ -106,46 +136,57 @@ class HashTreeParser {
 
       this.dataSets[name] = {
         ...dataSet,
-        hashTree: this.visibleDataSets.includes(name)
+        hashTree: this.isVisibleDataSet(name)
           ? new HashTree(leafData[name] || [], leafCount)
           : undefined,
       };
     }
   }
 
+  /**
+   * Checks if the given data set name is in the list of visible data sets
+   * @param {string} dataSetName data set name to check
+   * @returns {Boolean} True if the data set is visible, false otherwise
+   */
+  private isVisibleDataSet(dataSetName: string): boolean {
+    return this.visibleDataSets.some((vds) => vds.name === dataSetName);
+  }
+
   /**
    * Initializes a new visible data set by creating a hash tree for it, adding it to all the internal structures,
    * and sending an initial sync request to Locus with empty leaf data - that will trigger Locus to gives us all the data
    * from that dataset (in the response or via messages).
    *
-   * @param {DataSet} dataSet The new data set to be added
+   * @param {VisibleDataSetInfo} visibleDataSetInfo Information about the new visible data set
+   * @param {DataSet} dataSetInfo The new data set to be added
    * @returns {Promise}
    */
   private initializeNewVisibleDataSet(
-    dataSet: DataSet
+    visibleDataSetInfo: VisibleDataSetInfo,
+    dataSetInfo: DataSet
   ): Promise<{updateType: LocusInfoUpdateType; updatedObjects?: HashTreeObject[]}> {
-    if (this.visibleDataSets.includes(dataSet.name)) {
+    if (this.isVisibleDataSet(dataSetInfo.name)) {
       LoggerProxy.logger.info(
-        `HashTreeParser#initializeNewVisibleDataSet --> ${this.debugId} Data set "${dataSet.name}" already exists, skipping init`
+        `HashTreeParser#initializeNewVisibleDataSet --> ${this.debugId} Data set "${dataSetInfo.name}" already exists, skipping init`
       );
 
       return Promise.resolve({updateType: LocusInfoUpdateType.OBJECTS_UPDATED, updatedObjects: []});
     }
 
     LoggerProxy.logger.info(
-      `HashTreeParser#initializeNewVisibleDataSet --> ${this.debugId} Adding visible data set "${dataSet.name}"`
+      `HashTreeParser#initializeNewVisibleDataSet --> ${this.debugId} Adding visible data set "${dataSetInfo.name}"`
     );
 
-    this.visibleDataSets.push(dataSet.name);
+    this.visibleDataSets.push(visibleDataSetInfo);
 
-    const hashTree = new HashTree([], dataSet.leafCount);
+    const hashTree = new HashTree([], dataSetInfo.leafCount);
 
-    this.dataSets[dataSet.name] = {
-      ...dataSet,
+    this.dataSets[dataSetInfo.name] = {
+      ...dataSetInfo,
       hashTree,
     };
 
-    return this.sendInitializationSyncRequestToLocus(dataSet.name, 'new visible data set');
+    return this.sendInitializationSyncRequestToLocus(dataSetInfo.name, 'new visible data set');
   }
 
   /**
@@ -190,15 +231,22 @@ class HashTreeParser {
   }
 
   /**
-   * Queries Locus for information about all the data sets
+   * Queries Locus for all up-to-date information about all visible data sets
    *
-   * @param {string} url - url from which we can get info about all data sets
    * @returns {Promise}
    */
-  private getAllDataSetsMetadata(url) {
+  private getAllVisibleDataSetsFromLocus() {
+    if (!this.visibleDataSetsUrl) {
+      LoggerProxy.logger.warn(
+        `HashTreeParser#getAllVisibleDataSetsFromLocus --> ${this.debugId} No visibleDataSetsUrl, cannot get data sets information`
+      );
+
+      return Promise.resolve([]);
+    }
+
     return this.webexRequest({
       method: HTTP_VERBS.GET,
-      uri: url,
+      uri: this.visibleDataSetsUrl,
     }).then((response) => {
       return response.body.dataSets as Array<DataSet>;
     });
@@ -211,12 +259,14 @@ class HashTreeParser {
    * @returns {Promise}
    */
   async initializeFromMessage(message: HashTreeMessage) {
+    this.visibleDataSetsUrl = message.visibleDataSetsUrl;
+
     LoggerProxy.logger.info(
-      `HashTreeParser#initializeFromMessage --> ${this.debugId} visibleDataSetsUrl=${message.visibleDataSetsUrl}`
+      `HashTreeParser#initializeFromMessage --> ${this.debugId} visibleDataSetsUrl=${this.visibleDataSetsUrl}`
     );
-    const dataSets = await this.getAllDataSetsMetadata(message.visibleDataSetsUrl);
+    const visibleDataSets = await this.getAllVisibleDataSetsFromLocus();
 
-    await this.initializeDataSets(dataSets, 'initialization from message');
+    await this.initializeDataSets(visibleDataSets, 'initialization from message');
   }
 
   /**
@@ -236,28 +286,29 @@ class HashTreeParser {
 
       return;
     }
+    this.visibleDataSetsUrl = locus.links.resources.visibleDataSets.url;
 
     LoggerProxy.logger.info(
-      `HashTreeParser#initializeFromGetLociResponse --> ${this.debugId} visibleDataSets url: ${locus.links.resources.visibleDataSets.url}`
+      `HashTreeParser#initializeFromGetLociResponse --> ${this.debugId} visibleDataSets url: ${this.visibleDataSetsUrl}`
     );
 
-    const dataSets = await this.getAllDataSetsMetadata(locus.links.resources.visibleDataSets.url);
+    const visibleDataSets = await this.getAllVisibleDataSetsFromLocus();
 
-    await this.initializeDataSets(dataSets, 'initialization from GET /loci response');
+    await this.initializeDataSets(visibleDataSets, 'initialization from GET /loci response');
   }
 
   /**
    * Initializes data sets by doing an initialization sync on each visible data set that doesn't have a hash tree yet.
    *
-   * @param {DataSet[]} dataSets Array of DataSet objects to initialize
+   * @param {DataSet[]} visibleDataSets Array of visible DataSet objects to initialize
    * @param {string} debugText Text to include in logs for debugging purposes
    * @returns {Promise}
    */
-  private async initializeDataSets(dataSets: Array<DataSet>, debugText: string) {
+  private async initializeDataSets(visibleDataSets: Array<DataSet>, debugText: string) {
     const updatedObjects: HashTreeObject[] = [];
 
-    for (const dataSet of dataSets) {
-      const {name, leafCount} = dataSet;
+    for (const dataSet of visibleDataSets) {
+      const {name, leafCount, url} = dataSet;
 
       if (!this.dataSets[name]) {
         LoggerProxy.logger.info(
@@ -273,7 +324,14 @@ class HashTreeParser {
         );
       }
 
-      if (this.visibleDataSets.includes(name) && !this.dataSets[name].hashTree) {
+      if (!this.isVisibleDataSet(name)) {
+        this.visibleDataSets.push({
+          name,
+          url,
+        });
+      }
+
+      if (!this.dataSets[name].hashTree) {
         LoggerProxy.logger.info(
           `HashTreeParser#initializeDataSets --> ${this.debugId} creating hash tree for visible dataset "${name}" (${debugText})`
         );
@@ -316,10 +374,7 @@ class HashTreeParser {
   private analyzeLocusHtMeta(locus: any, options?: {copyData?: boolean}) {
     const {copyData = false} = options || {};
     // object mapping dataset names to arrays of leaf data
-    const leafInfo: Record<
-      string,
-      Array<{type: ObjectType; id: number; version: number; data?: any}>
-    > = {};
+    const leafInfo: Record<string, Array<LeafInfo>> = {};
 
     const findAndStoreMetaData = (currentLocusPart: any) => {
       if (typeof currentLocusPart !== 'object' || currentLocusPart === null) {
@@ -329,7 +384,7 @@ class HashTreeParser {
       if (currentLocusPart.htMeta && currentLocusPart.htMeta.dataSetNames) {
         const {type, id, version} = currentLocusPart.htMeta.elementId;
         const {dataSetNames} = currentLocusPart.htMeta;
-        const newLeafInfo: {type: ObjectType; id: number; version: number; data?: any} = {
+        const newLeafInfo: LeafInfo = {
           type,
           id,
           version,
@@ -368,6 +423,43 @@ class HashTreeParser {
     return leafInfo;
   }
 
+  /**
+   * Analyzes the Metadata object that is sent outside of Locus object, and appends its data to passed in leafInfo
+   * structure.
+   *
+   * @param {Record<string, LeafInfo[]>} leafInfo the structure to which the Metadata info will be appended
+   * @param {Metadata} metadata Metadata object
+   * @returns {void}
+   */
+  private analyzeMetadata(leafInfo: Record<string, LeafInfo[]>, metadata: Metadata) {
+    const {htMeta} = metadata;
+
+    if (
+      htMeta?.dataSetNames?.length === 1 &&
+      htMeta.dataSetNames[0].toLowerCase() === MetadataDataSetName
+    ) {
+      const {type, id, version} = metadata.htMeta.elementId;
+
+      const dataSetName = htMeta.dataSetNames[0];
+
+      if (!leafInfo[dataSetName]) {
+        leafInfo[dataSetName] = [];
+      }
+
+      leafInfo[dataSetName].push({
+        type,
+        id,
+        version,
+      });
+    } else {
+      throw new Error(
+        `${this.debugId} Metadata htMeta has unexpected dataSetNames: ${
+          htMeta && htMeta.dataSetNames.join(',')
+        }`
+      );
+    }
+  }
+
   /**
    * Checks if the provided hash tree message indicates the end of the meeting and that there won't be any more updates.
    *
@@ -420,6 +512,58 @@ class HashTreeParser {
     });
   }
 
+  /**
+   * Handles updates to Metadata object that we receive from Locus via other means than messages. Right now
+   * that means only in the API response alongside locus object.
+   *
+   * @param {Metadata} metadata received in Locus update other than a message (for example in an API response)
+   * @param {HashTreeObject[]} updatedObjects a list of updated hash tree objects to which any updates resulting from new Metadata will be added
+   * @returns {void}
+   */
+  handleMetadataUpdate(metadata: Metadata, updatedObjects: HashTreeObject[]): void {
+    let dataSetsRequiringInitialization: VisibleDataSetInfo[] = [];
+
+    // current assumption based on Locus docs is that Metadata object lives always in "self" data set
+    const hashTree = this.dataSets[MetadataDataSetName]?.hashTree;
+
+    if (!hashTree) {
+      LoggerProxy.logger.warn(
+        `HashTreeParser#handleLocusUpdate --> ${this.debugId} received Metadata object but no hash tree for "${MetadataDataSetName}" data set exists`
+      );
+    } else {
+      const metadataUpdated = hashTree.putItem(metadata.htMeta.elementId);
+
+      if (metadataUpdated) {
+        // metadata in Locus API response is in a slightly different format than the objects in messages, so need to adapt it
+        const metadataObject: HashTreeObject = {
+          htMeta: metadata.htMeta,
+          data: metadata,
+        };
+
+        updatedObjects.push(metadataObject);
+
+        const {changeDetected, removedDataSets, addedDataSets} = this.checkForVisibleDataSetChanges(
+          [metadataObject]
+        );
+
+        if (changeDetected) {
+          dataSetsRequiringInitialization = this.processVisibleDataSetChanges(
+            removedDataSets,
+            addedDataSets,
+            updatedObjects
+          );
+        }
+
+        if (dataSetsRequiringInitialization.length > 0) {
+          // there are some data sets that we need to initialize asynchronously
+          queueMicrotask(() => {
+            this.initializeNewVisibleDataSets(dataSetsRequiringInitialization);
+          });
+        }
+      }
+    }
+  }
+
   /**
    * This method should be called when we receive a partial locus DTO that contains dataSets and htMeta information
    * It updates the hash trees with the new leaf data based on the received Locus
@@ -427,8 +571,8 @@ class HashTreeParser {
    * @param {Object} update - The locus update containing data sets and locus information
    * @returns {void}
    */
-  handleLocusUpdate(update: {dataSets?: Array<DataSet>; locus: any}): void {
-    const {dataSets, locus} = update;
+  handleLocusUpdate(update: {dataSets?: Array<DataSet>; locus: any; metadata?: Metadata}): void {
+    const {dataSets, locus, metadata} = update;
 
     if (!dataSets) {
       LoggerProxy.logger.warn(
@@ -443,6 +587,11 @@ class HashTreeParser {
     // first, analyze the locus object to extract the hash tree objects' htMeta and data from it
     const leafInfo = this.analyzeLocusHtMeta(locus, {copyData: true});
 
+    // if we got metadata, process it (currently that means only potential visible data set list changes)
+    if (metadata) {
+      this.handleMetadataUpdate(metadata, updatedObjects);
+    }
+
     // then process the data in hash trees, if it is a new version, then add it to updatedObjects
     Object.keys(leafInfo).forEach((dataSetName) => {
       if (this.dataSets[dataSetName]) {
@@ -493,9 +642,6 @@ class HashTreeParser {
         updatedObjects,
       });
     }
-
-    // todo: once Locus design on how visible data sets will be communicated in subsequent API responses is confirmed,
-    // we'll need to check here if visible data sets have changed and update this.visibleDataSets, remove/create hash trees etc
   }
 
   /**
@@ -511,7 +657,7 @@ class HashTreeParser {
       };
 
       LoggerProxy.logger.info(
-        `HashTreeParser#handleMessage --> ${this.debugId} created entry for "${receivedDataSet.name}" dataset: version=${receivedDataSet.version}, root=${receivedDataSet.root}`
+        `HashTreeParser#updateDataSetInfo --> ${this.debugId} created entry for "${receivedDataSet.name}" dataset: version=${receivedDataSet.version}, root=${receivedDataSet.root}`
       );
 
       return;
@@ -526,7 +672,7 @@ class HashTreeParser {
         exponent: receivedDataSet.backoff.exponent,
       };
       LoggerProxy.logger.info(
-        `HashTreeParser#handleMessage --> ${this.debugId} updated "${receivedDataSet.name}" to version=${receivedDataSet.version}, root=${receivedDataSet.root}`
+        `HashTreeParser#updateDataSetInfo --> ${this.debugId} updated "${receivedDataSet.name}" dataset to version=${receivedDataSet.version}, root=${receivedDataSet.root}`
       );
     }
   }
@@ -537,25 +683,28 @@ class HashTreeParser {
    * @returns {Object} An object containing the removed and added visible data sets.
    */
   private checkForVisibleDataSetChanges(updatedObjects: HashTreeObject[]) {
-    let removedDataSets: string[] = [];
-    let addedDataSets: string[] = [];
+    let removedDataSets: VisibleDataSetInfo[] = [];
+    let addedDataSets: VisibleDataSetInfo[] = [];
 
-    // visibleDataSets can only be changed by self object updates
+    // visibleDataSets can only be changed by Metadata object updates
     updatedObjects.forEach((object) => {
-      // todo: in the future visibleDataSets will be in "Metadata" object, not in "self"
-      if (isSelf(object) && object.data?.visibleDataSets) {
+      if (isMetadata(object) && object.data?.visibleDataSets) {
         const newVisibleDataSets = object.data.visibleDataSets;
 
-        removedDataSets = this.visibleDataSets.filter((ds) => !newVisibleDataSets.includes(ds));
-        addedDataSets = newVisibleDataSets.filter((ds) => !this.visibleDataSets.includes(ds));
+        removedDataSets = this.visibleDataSets.filter(
+          (ds) => !newVisibleDataSets.some((nvs) => nvs.name === ds.name)
+        );
+        addedDataSets = newVisibleDataSets.filter((nvs) =>
+          this.visibleDataSets.every((ds) => ds.name !== nvs.name)
+        );
 
         if (removedDataSets.length > 0 || addedDataSets.length > 0) {
           LoggerProxy.logger.info(
             `HashTreeParser#checkForVisibleDataSetChanges --> ${
               this.debugId
-            } visible data sets change: removed: ${removedDataSets.join(
-              ', '
-            )}, added: ${addedDataSets.join(', ')}`
+            } visible data sets change: removed: ${removedDataSets
+              .map((ds) => ds.name)
+              .join(', ')}, added: ${addedDataSets.map((ds) => ds.name).join(', ')}`
           );
         }
       }
@@ -593,49 +742,51 @@ class HashTreeParser {
    * visible data sets and they require async initialization, the names of these data sets
    * are returned in an array.
    *
-   * @param {string[]} removedDataSets - The list of removed data sets.
-   * @param {string[]} addedDataSets - The list of added data sets.
+   * @param {VisibleDataSetInfo[]} removedDataSets - The list of removed data sets.
+   * @param {VisibleDataSetInfo[]} addedDataSets - The list of added data sets.
    * @param {HashTreeObject[]} updatedObjects - The list of updated hash tree objects to which changes will be added.
-   * @returns {string[]} names of data sets that couldn't be initialized synchronously
+   * @returns {VisibleDataSetInfo[]} list of data sets that couldn't be initialized synchronously
    */
   private processVisibleDataSetChanges(
-    removedDataSets: string[],
-    addedDataSets: string[],
+    removedDataSets: VisibleDataSetInfo[],
+    addedDataSets: VisibleDataSetInfo[],
     updatedObjects: HashTreeObject[]
-  ): string[] {
-    const dataSetsRequiringInitialization = [];
+  ): VisibleDataSetInfo[] {
+    const dataSetsRequiringInitialization: VisibleDataSetInfo[] = [];
 
     // if a visible data set was removed, we need to tell our client that all objects from it are removed
     const removedObjects: HashTreeObject[] = [];
 
     removedDataSets.forEach((ds) => {
-      if (this.dataSets[ds]?.hashTree) {
-        for (let i = 0; i < this.dataSets[ds].hashTree.numLeaves; i += 1) {
+      if (this.dataSets[ds.name]?.hashTree) {
+        for (let i = 0; i < this.dataSets[ds.name].hashTree.numLeaves; i += 1) {
           removedObjects.push(
-            ...this.dataSets[ds].hashTree.getLeafData(i).map((elementId) => ({
+            ...this.dataSets[ds.name].hashTree.getLeafData(i).map((elementId) => ({
               htMeta: {
                 elementId,
-                dataSetNames: [ds],
+                dataSetNames: [ds.name],
               },
               data: null,
             }))
           );
         }
 
-        this.deleteHashTree(ds);
+        this.deleteHashTree(ds.name);
       }
     });
-    this.visibleDataSets = this.visibleDataSets.filter((vds) => !removedDataSets.includes(vds));
+    this.visibleDataSets = this.visibleDataSets.filter(
+      (vds) => !removedDataSets.some((rds) => rds.name === vds.name)
+    );
     updatedObjects.push(...removedObjects);
 
     // now setup the new visible data sets
     for (const ds of addedDataSets) {
-      const dataSetInfo = this.dataSets[ds];
+      const dataSetInfo = this.dataSets[ds.name];
 
       if (dataSetInfo) {
-        if (this.visibleDataSets.includes(dataSetInfo.name)) {
+        if (this.isVisibleDataSet(dataSetInfo.name)) {
           LoggerProxy.logger.info(
-            `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} Data set "${ds}" is already visible, skipping`
+            `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} Data set "${ds.name}" is already visible, skipping`
           );
 
           // eslint-disable-next-line no-continue
@@ -643,7 +794,7 @@ class HashTreeParser {
         }
 
         LoggerProxy.logger.info(
-          `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} Adding visible data set "${ds}"`
+          `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} Adding visible data set "${ds.name}"`
         );
 
         this.visibleDataSets.push(ds);
@@ -656,7 +807,7 @@ class HashTreeParser {
         };
       } else {
         LoggerProxy.logger.info(
-          `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} visible data set "${ds}" added but no info about it in our dataSets structures`
+          `HashTreeParser#processVisibleDataSetChanges --> ${this.debugId} visible data set "${ds.name}" added but no info about it in our dataSets structures`
         );
         // todo: add a metric here
         dataSetsRequiringInitialization.push(ds);
@@ -670,32 +821,28 @@ class HashTreeParser {
    * Adds entries to the passed in updateObjects array
    * for the changes that result from adding and removing visible data sets.
    *
-   * @param {HashTreeMessage} message - The hash tree message that triggered the visible data set changes.
-   * @param {string[]} addedDataSets - The list of added data sets.
+   * @param {VisibleDataSetInfo[]} addedDataSets - The list of added data sets.
    * @returns {Promise<void>}
    */
-  private async initializeNewVisibleDataSets(
-    message: HashTreeMessage,
-    addedDataSets: string[]
-  ): Promise<void> {
-    const allDataSets = await this.getAllDataSetsMetadata(message.visibleDataSetsUrl);
+  private async initializeNewVisibleDataSets(addedDataSets: VisibleDataSetInfo[]): Promise<void> {
+    const allDataSets = await this.getAllVisibleDataSetsFromLocus();
 
     for (const ds of addedDataSets) {
-      const dataSetInfo = allDataSets.find((d) => d.name === ds);
+      const dataSetInfo = allDataSets.find((d) => d.name === ds.name);
 
       LoggerProxy.logger.info(
-        `HashTreeParser#initializeNewVisibleDataSets --> ${this.debugId} initializing data set "${ds}"`
+        `HashTreeParser#initializeNewVisibleDataSets --> ${this.debugId} initializing data set "${ds.name}"`
       );
 
       if (!dataSetInfo) {
         LoggerProxy.logger.warn(
-          `HashTreeParser#handleHashTreeMessage --> ${this.debugId} missing info about data set "${ds}" in Locus response from visibleDataSetsUrl`
+          `HashTreeParser#handleHashTreeMessage --> ${this.debugId} missing info about data set "${ds.name}" in Locus response from visibleDataSetsUrl`
         );
       } else {
         // we're awaiting in a loop, because in practice there will be only one new data set at a time,
         // so no point in trying to parallelize this
         // eslint-disable-next-line no-await-in-loop
-        const updates = await this.initializeNewVisibleDataSet(dataSetInfo);
+        const updates = await this.initializeNewVisibleDataSet(ds, dataSetInfo);
 
         this.callLocusInfoUpdateCallback(updates);
       }
@@ -746,32 +893,31 @@ class HashTreeParser {
     // available in the message, they will require separate async initialization
     let dataSetsRequiringInitialization = [];
 
-    // first find out if there are any visible data set changes - they're signalled in SELF object updates
-    const selfUpdates = (message.locusStateElements || []).filter((object) =>
-      // todo: SPARK-744859 once Locus supports it, we will filter for "Metadata" type here instead of "self"
-      isSelf(object)
+    // first find out if there are any visible data set changes - they're signalled in Metadata object updates
+    const metadataUpdates = (message.locusStateElements || []).filter((object) =>
+      isMetadata(object)
     );
 
-    if (selfUpdates.length > 0) {
-      const updatedSelfObjects = [];
+    if (metadataUpdates.length > 0) {
+      const updatedMetadataObjects = [];
 
-      selfUpdates.forEach((object) => {
+      metadataUpdates.forEach((object) => {
         // todo: once Locus supports it, we will use the "view" field here instead of dataSetNames
         for (const dataSetName of object.htMeta.dataSetNames) {
           const hashTree = this.dataSets[dataSetName]?.hashTree;
 
           if (hashTree && object.data) {
             if (hashTree.putItem(object.htMeta.elementId)) {
-              updatedSelfObjects.push(object);
+              updatedMetadataObjects.push(object);
             }
           }
         }
       });
 
-      updatedObjects.push(...updatedSelfObjects);
+      updatedObjects.push(...updatedMetadataObjects);
 
       const {changeDetected, removedDataSets, addedDataSets} =
-        this.checkForVisibleDataSetChanges(updatedSelfObjects);
+        this.checkForVisibleDataSetChanges(updatedMetadataObjects);
 
       if (changeDetected) {
         dataSetsRequiringInitialization = this.processVisibleDataSetChanges(
@@ -838,7 +984,7 @@ class HashTreeParser {
     if (dataSetsRequiringInitialization.length > 0) {
       // there are some data sets that we need to initialize asynchronously
       queueMicrotask(() => {
-        this.initializeNewVisibleDataSets(message, dataSetsRequiringInitialization);
+        this.initializeNewVisibleDataSets(dataSetsRequiringInitialization);
       });
     }
 
@@ -880,7 +1026,49 @@ class HashTreeParser {
   }) {
     const {updateType, updatedObjects} = updates;
 
-    if (updateType !== LocusInfoUpdateType.OBJECTS_UPDATED || updatedObjects?.length > 0) {
+    if (updateType === LocusInfoUpdateType.OBJECTS_UPDATED && updatedObjects?.length > 0) {
+      // Filter out updates for objects that already have a higher version in their datasets,
+      // or removals for objects that still exist in any of their datasets
+      const filteredUpdates = updatedObjects.filter((object) => {
+        const {elementId} = object.htMeta;
+        const {type, id, version} = elementId;
+
+        // Check all datasets
+        for (const dataSetName of Object.keys(this.dataSets)) {
+          const dataSet = this.dataSets[dataSetName];
+
+          // only visible datasets have hash trees set
+          if (dataSet?.hashTree) {
+            const existingVersion = dataSet.hashTree.getItemVersion(id, type);
+            if (existingVersion !== undefined) {
+              if (object.data) {
+                // For updates: filter out if any dataset has a higher version
+                if (existingVersion > version) {
+                  LoggerProxy.logger.info(
+                    `HashTreeParser#callLocusInfoUpdateCallback --> ${this.debugId} Filtering out update for ${type}:${id} v${version} because dataset "${dataSetName}" has v${existingVersion}`
+                  );
+
+                  return false;
+                }
+              } else if (existingVersion >= version) {
+                // For removals: filter out if the object still exists in any dataset
+                LoggerProxy.logger.info(
+                  `HashTreeParser#callLocusInfoUpdateCallback --> ${this.debugId} Filtering out removal for ${type}:${id} v${version} because dataset "${dataSetName}" still has v${existingVersion}`
+                );
+
+                return false;
+              }
+            }
+          }
+        }
+
+        return true;
+      });
+
+      if (filteredUpdates.length > 0) {
+        this.locusInfoUpdateCallback(updateType, {updatedObjects: filteredUpdates});
+      }
+    } else if (updateType !== LocusInfoUpdateType.OBJECTS_UPDATED) {
       this.locusInfoUpdateCallback(updateType, {updatedObjects});
     }
   }
@@ -969,7 +1157,8 @@ class HashTreeParser {
             try {
               // request hashes from sender
               const {hashes, dataSet: latestDataSetInfo} = await this.getHashesFromLocus(
-                dataSet.name
+                dataSet.name,
+                rootHash
               );
 
               receivedHashes = hashes;
@@ -1035,9 +1224,10 @@ class HashTreeParser {
   /**
    * Gets the current hashes from the locus for a specific data set.
    * @param {string} dataSetName
+   * @param {string} currentRootHash
    * @returns {string[]}
    */
-  private getHashesFromLocus(dataSetName: string) {
+  private getHashesFromLocus(dataSetName: string, currentRootHash: string) {
     LoggerProxy.logger.info(
       `HashTreeParser#getHashesFromLocus --> ${this.debugId} Requesting hashes for data set "${dataSetName}"`
     );
@@ -1049,6 +1239,9 @@ class HashTreeParser {
     return this.webexRequest({
       method: HTTP_VERBS.GET,
       uri: url,
+      qs: {
+        rootHash: currentRootHash,
+      },
     })
       .then((response) => {
         const hashes = response.body?.hashes as string[] | undefined;
@@ -1110,9 +1303,14 @@ class HashTreeParser {
       });
     });
 
+    const ourCurrentRootHash = dataSet.hashTree ? dataSet.hashTree.getRootHash() : EMPTY_HASH;
+
     return this.webexRequest({
       method: HTTP_VERBS.POST,
       uri: url,
+      qs: {
+        rootHash: ourCurrentRootHash,
+      },
       body,
     })
       .then((resp) => {