@refinitiv-ui/efx-grid 6.0.53 → 6.0.55

package/lib/rt-grid/dist/rt-grid.js

@@ -10079,6 +10079,12 @@ Segment.prototype.getChildIds = function() {
 	return this._childCount ? Object.keys(this._children) : [];
 };
 /** @public
+* @return {!Object}
+*/
+Segment.prototype.getChildren = function() {
+	return this._children;
+};
+/** @public
 * @return {number}
 */
 Segment.prototype.getChildCount = function() {
@@ -12425,9 +12431,9 @@ DataTable.prototype.getSegmentChildIds = function(segmentId) {
 };
 /** Sort all of existing segments by multiple sort logics
 * @public
-* @param {Function|Array.<Function>} sortLogics
-* @param {Array.<number>} sortOrders
-* @param {Array.<string>} cids
+* @param {Function|Array.<Function>|Object} sortLogics
+* @param {Array.<number>=} sortOrders
+* @param {Array.<string>=} cids
 * @return {boolean}
 */
 DataTable.prototype.sortSeparators = function (sortLogics, sortOrders, cids) {
@@ -14466,9 +14472,9 @@ RowDefinition.dispose = function(rowDef) {
 * @const
 */
 var SYNAPSE_URL =
-	'/synapse/service/suggestions/suggest/?'
-	+ 'hits=1' // search only 1 result
-	+ '&profile=' + encodeURIComponent('Field Selector');
+	"/synapse/service/suggestions/suggest/?"
+	+ "hits=1" // search only 1 result
+	+ "&profile=" + encodeURIComponent("Field Selector");
 
 /* @namespace */
 var FieldDefinition = {};
@@ -14585,18 +14591,13 @@ FieldDefinition._timeSeriesChildren = {};
 * @private
 * @const
 */
-FieldDefinition._synapse = '';
+FieldDefinition._synapse = "";
 /**
 * @type {string}
 * @private
 * @const
 */
-FieldDefinition._lang = 'en';
-/**
-* @type {boolean}
-* @private
-*/
-FieldDefinition._caching = false;
+FieldDefinition._lang = "en";
 /**
 * @type {boolean}
 * @private
@@ -14623,17 +14624,22 @@ FieldDefinition.set = function(field, def) {
 		}
 	}
 };
-/** @public
+/** Get field definition object
+* @public
 * @function
 * @param {string} field
 * @return {Object}
 */
 FieldDefinition.get = function(field) {
-	if(this._caching) {
-		return FieldDefinition._defs[field];
-	}
-	return null;
+	return FieldDefinition._defs[field] || null;
 };
+/** Get field definition object
+* @public
+* @function
+* @param {string} field
+* @return {Object}
+*/
+FieldDefinition.getFieldInfo = FieldDefinition.get;
 
 /** @public
 * @function
@@ -14642,7 +14648,7 @@ FieldDefinition.get = function(field) {
 */
 FieldDefinition.hasFieldInfo = function(field) {
 	var val = FieldDefinition.get(field);
-	return val && val.field; // Already preventing an error caused by accessing a property on a null value
+	return (val && val.field) ? true : false; // Already preventing an error caused by accessing a property on a null value
 };
 
 /** @public
@@ -14678,15 +14684,6 @@ FieldDefinition.remove = function(field) {
 */
 FieldDefinition.setSynapseConfig = function (config) {
 	FieldDefinition._synapse = config;
-
-};
-
-/** @public
-* @function
-* @param {boolean} caching
-*/
-FieldDefinition.setFieldCaching = function (caching) {
-	FieldDefinition._caching = caching;
 };
 
 /** @public
@@ -14811,7 +14808,7 @@ FieldDefinition.getFieldProperty = function(field, propertyName) {
 	return null;
 };
 
-/** to get more info about field via synapse service
+/** To get more info about field via synapse service
 * @private
 * @param {string} field
 * @returns {Promise}
@@ -14843,12 +14840,12 @@ FieldDefinition.loadFieldInfo = function (field) {
 	// everything fine, call synapse service
 	else {
 		var queryUrl = SYNAPSE_URL
-			+ '&api-key=' + encodeURIComponent(synapse.apiKey)
-			+ '&contextApp=' + encodeURIComponent(synapse.contextApp)
-			+ '&language=' + encodeURIComponent(FieldDefinition._lang)
-			+ '&query=' + encodeURIComponent(field);
+			+ "&api-key=" + encodeURIComponent(synapse.apiKey)
+			+ "&contextApp=" + encodeURIComponent(synapse.contextApp)
+			+ "&language=" + encodeURIComponent(FieldDefinition._lang)
+			+ "&query=" + encodeURIComponent(field);
 		if (synapse.auth) {
-			queryUrl += '&auth=' + encodeURIComponent(synapse.auth);
+			queryUrl += "&auth=" + encodeURIComponent(synapse.auth);
 		}
 		// TODO: handle client timeout
 		var xmr = new XMLHttpRequest();
@@ -14857,9 +14854,9 @@ FieldDefinition.loadFieldInfo = function (field) {
 		FieldDefinition._loadingField[field] = defer;
 
 		// for now we care only successful case
-		xmr.addEventListener('loadend', onLoadEnd);
-		xmr.addEventListener('timeout', onError);
-		xmr.addEventListener('error', onError);
+		xmr.addEventListener("loadend", onLoadEnd);
+		xmr.addEventListener("timeout", onError);
+		xmr.addEventListener("error", onError);
 		xmr.open("GET", queryUrl, true); // true for asynchronous
 		xmr.send();
 	}
@@ -14887,7 +14884,7 @@ FieldDefinition._mockOnLoadEndData = function (field, defer) {
 	var fieldUpperCase = field.toUpperCase();
 	var fieldLowerCase = fieldUpperCase.toLowerCase();
 	var label;
-	if (fieldLowerCase.indexOf('_') === 2) { // transform XX_ABCD -> Abcd
+	if (fieldLowerCase.indexOf("_") === 2) { // transform XX_ABCD -> Abcd
 		label = fieldUpperCase[3] + fieldLowerCase.substring(4);
 	} else {
 		label = fieldUpperCase[0] + fieldLowerCase.substring(1);
@@ -14949,49 +14946,49 @@ FieldDefinition._mockOnLoadEndData = function (field, defer) {
 function onLoadEnd(e) {
 	var xmr = e.currentTarget;
 	var defer = xmr._defer;
-	var resolve = defer.resolve;
-	var reject = defer.reject;
+	var field = xmr._field;
 
-	delete FieldDefinition._loadingField[xmr._field];
+	delete FieldDefinition._loadingField[field];
 
 	if (xmr.status === 200) { // case success
-		var returnObj = {
-			field: xmr._field,
-			fieldDefinition: null
-		};
 		var res = JSON.parse(xmr.responseText);
+		var fieldDef = null;
 
 		var result = res.result && res.result[0];
 		if (result) {
 			var item = result.hits && result.hits[0];
 			if (item && item.p) {
 				var profile = item.p;
-				if (profile.fn.toUpperCase() === xmr._field.toUpperCase()) {
-					var fieldDef = FieldDefinition.get(xmr._field) || {};
+				if (profile.fn.toUpperCase() === field.toUpperCase()) {
+					fieldDef = FieldDefinition.get(field) || {};
 
 					fieldDef.field = profile.fn; // name
+					fieldDef.name = profile.fl; // label
 					fieldDef.rank = profile.Rank;
 					fieldDef.fieldDataType = profile.fdt; // data type
+					fieldDef.id = profile.fid;
 					fieldDef.IsMonitorOnlyField = profile.fimo;
 					fieldDef.IsRealtimePortfolioField = profile.firp;
 					fieldDef.IsRealtimeField = profile.firt;
-					fieldDef.name = profile.fl; // label
 					fieldDef.source = profile.fsrc;
 					fieldDef.sourceRank = profile.fsrnk;
 					fieldDef.description = item.title;
 
-					FieldDefinition.set(xmr._field, fieldDef);
-					returnObj.fieldDefinition = fieldDef;
+					FieldDefinition.set(field, fieldDef);
 				}
 			}
 		}
-		resolve(returnObj);
+		if(fieldDef) {
+			defer.resolve(fieldDef);
+		} else {
+			defer.reject("No definition for " + field);
+		}
 	} else if (xmr._error) { // case error && time out
-		xmr._error.field = xmr._field;
-		reject(xmr._error);
+		xmr._error.field = field;
+		defer.reject(xmr._error);
 	} else { // case status not 200
-		reject({
-			field: xmr._field,
+		defer.reject({
+			field: field,
 			status: xmr.status,
 			statusText: xmr.statusText,
 			request: xmr
@@ -16754,7 +16751,7 @@ GroupDefinitions.getLeafDescendants = function (groupMap, groupId) {
 		groupDef = unvisitedGroups[visitedCount++];
 		visitedMap[groupDef.id] = true;
 		var chdr = groupDef.children;
-		var len = chdr.length;
+		var len = chdr ? chdr.length : 0;
 		for(var i = 0; i < len; ++i) {
 			var childId = chdr[i];
 			groupDef = groupMap[childId];
@@ -16774,44 +16771,30 @@ GroupDefinitions.getLeafDescendants = function (groupMap, groupId) {
 /** @private
 * @function
 * @param {Object} obj
-* @return {Object}
+* @return {Object} Return a new object with guaranteed children property
 */
 GroupDefinitions._cloneObject = function(obj) {
 	var newObj = cloneObject(obj);
-	if(Array.isArray(newObj.children)) {
-		newObj.children = newObj.children;
+	if(Array.isArray(newObj.children)) { // This is to prevent modification from the outside source
+		newObj.children = newObj.children.slice();
 	} else {
 		newObj.children = [];
 	}
 	return newObj;
 };
-/** @private
-* @param {Object} groupMap
-* @return {!Object}
-*/
-GroupDefinitions._cloneGroupMap = function (groupMap) {
-	var outMap = {};
-	for(var groupId in groupMap) {
-		var groupDef = groupMap[groupId];
-		var obj = GroupDefinitions._cloneObject(groupDef);
-		obj.children = groupDef.children.slice();
-		outMap[groupId] = obj;
-	}
-	return outMap;
-};
 
 /** @private
 * @function
 * @param {Array.<string>|Object} obj
 * @param {string=} groupId
-* @return {Object}
+* @return {Object} Return a new object with guaranteed children property
 */
 GroupDefinitions._toGroupDefinition = function(obj, groupId) {
 	var groupDef = null;
 	if(obj) {
 		if(Array.isArray(obj)) {
 			groupDef = {
-				children: obj
+				children: obj.slice()
 			};
 		} else {
 			groupDef = GroupDefinitions._cloneObject(obj);
@@ -16891,7 +16874,43 @@ GroupDefinitions.prototype.getGroupMap = function () {
 * @return {!Object.<string, Object>}
 */
 GroupDefinitions.prototype.cloneGroupMap = function () {
-	return GroupDefinitions._cloneGroupMap(this._groupMap);
+	var groupMap = this._groupMap;
+	var outMap = {};
+	for(var groupId in groupMap) {
+		var groupDef = groupMap[groupId];
+		var obj = GroupDefinitions._cloneObject(groupDef);
+		outMap[groupId] = obj;
+	}
+	return outMap;
+};
+/** In case of a children array has been modified outside of GroupDefinitions, this method is needed to re-create internal maps to reflect existing structure
+* @public
+*/
+GroupDefinitions.prototype.rebuildMap = function () {
+	var groupMap = this._groupMap;
+	var childToParent = this._childToParent = {};
+
+	// Create child to parent map
+	var groupIds = Object.keys(groupMap);
+	var grpCount = groupIds.length;
+	var i, groupId;
+	for(i = 0; i < grpCount; ++i) {
+		groupId = groupIds[i];
+		var chdr = groupMap[groupId].children;
+		var childCount = chdr ? chdr.length : 0;
+		for (var j = 0; j < childCount; j++) {
+			childToParent[chdr[j]] = groupId;
+		}
+	}
+
+	// Apply a parent id to group definition to make it easier to find depth
+	for(i = 0; i < grpCount; ++i) {
+		groupId = groupIds[i];
+		var parentId = childToParent[groupId];
+		if(parentId) {
+			groupMap[groupId].parentId = parentId;
+		}
+	}
 };
 
 /** Get immediate child ids of the specified group. The returned array can contain both child ids and group ids
@@ -16971,7 +16990,7 @@ GroupDefinitions.prototype.getParentIds = function(childId) {
 };
 /** @public
 * @param {string} childId
-* @param {number=} groupLevel
+* @param {number=} groupLevel Default is to retrieve immediate parent id. Use 0 to get root (topmost) group id
 * @return {string}
 */
 GroupDefinitions.prototype.getParentId = function (childId, groupLevel) {
@@ -16999,47 +17018,25 @@ GroupDefinitions.prototype.removeAllGroups = function () {
 	}
 	return false;
 };
-/** Remove all existing group definitions and replace them with the given definitions.
+/** Remove all existing group definitions and replace them with the given definitions. Note that this method does not check for circular referencing nor duplication of ids
 * @public
 * @param {Array.<Object>=} groupDefs Use null or empty array to remove all existing groups
 */
 GroupDefinitions.prototype.setGroups = function (groupDefs) {
 	// Clear existing group structure
 	var groupMap = this._groupMap = {};
-	var childToParent = this._childToParent = {};
 
 	// Create group map
-	var i;
 	var grpCount = groupDefs ? groupDefs.length : 0;
-	var groupId = "";
-	for (i = 0; i < grpCount; i++) {
+	for (var i = 0; i < grpCount; i++) {
 		var groupDef = groupDefs[i];
-		groupId = groupDef.id;
+		var groupId = groupDef.id;
 		if(groupId) {
-			groupMap[groupId] = groupDef;
-		}
-	}
-
-	// Create child to parent map
-	var groupIds = Object.keys(groupMap);
-	grpCount = groupIds.length;
-	for(i = 0; i < grpCount; ++i) {
-		groupId = groupIds[i];
-		var children = groupMap[groupId].children;
-		var childCount = children.length;
-		for (var j = 0; j < childCount; j++) {
-			childToParent[children[j]] = groupId;
+			groupMap[groupId] = GroupDefinitions._cloneObject(groupDef);
 		}
 	}
 
-	// Apply a parent id to group definition to make it easier to find depth
-	for(i = 0; i < grpCount; ++i) {
-		groupId = groupIds[i];
-		var parentId = childToParent[groupId];
-		if(parentId) {
-			groupMap[groupId].parentId = parentId;
-		}
-	}
+	this.rebuildMap();
 };
 /** Add new group definition to existing group structure. Existing group with the same id will be replaced.
 * @public
@@ -17068,11 +17065,11 @@ GroupDefinitions.prototype.removeGroup = function (groupId) {
 	}
 	return false;
 };
-/** Replace and update existing group definition. New group is added if the given id has no match. Existing group will be removed of no new definition is given.
+/** Replace and update existing group definition. A new group will be added, only if the given id has no match. Existing group will be removed of no new definition is given.
 * @public
 * @param {string} groupId
-* @param {Array.<string>|Object=} groupDef
-* @return {string} Return group Id. Return empty string if nothing has been changed.
+* @param {(Array.<string>|Object)=} groupDef
+* @return {string} Return group Id. Return empty string, if nothing has been changed
 */
 GroupDefinitions.prototype.setGroup = function (groupId, groupDef) {
 	if(!groupId) {
@@ -17093,7 +17090,7 @@ GroupDefinitions.prototype.setGroup = function (groupId, groupDef) {
 		}
 		this._groupMap[groupId] = newDef;
 
-		var chdr = newDef.children;
+		var chdr = newDef.children; // newDef is guaranteed to have children property
 		var len = chdr.length;
 		for(var i = 0; i < len; ++i) {
 			var childId = chdr[i];
@@ -17125,7 +17122,8 @@ GroupDefinitions.prototype._ungroupChildren = function(children) {
 	}
 };
 
-/** @public
+/** Check if the given child id is an immediate child of the given group
+* @public
 * @param {string} parentId Group id
 * @param {string} childId
 * @return {boolean}
@@ -17135,7 +17133,30 @@ GroupDefinitions.prototype.hasGroupChild = function (parentId, childId) {
 	if(childId && groupDef) {
 		var chdr = groupDef.children;
 		if(chdr) {
-			return chdr.indexOf(childId) >= 0;
+			return chdr.indexOf(childId) >= 0; // TODO: Use childToParent map to improve performance
+		}
+	}
+	return false;
+};
+/** Check if the specified id is contained within the given group regardless of any group level that the id is in
+* @public
+* @param {string} groupId
+* @param {string} childId
+* @return {boolean}
+*/
+GroupDefinitions.prototype.contains = function (groupId, childId) {
+	if(groupId && childId) {
+		if(groupId === childId) {
+			return true;
+		}
+		var levelLimit = 20;
+		var parentId = this._childToParent[childId];
+		while(parentId && levelLimit) { // WARNING: Circular dependency could happen
+			if(groupId === parentId) {
+				return true;
+			}
+			--levelLimit;
+			parentId = this._childToParent[parentId];
 		}
 	}
 	return false;
@@ -17202,7 +17223,7 @@ GroupDefinitions.prototype.unsetParent = function (childId) {
 	var parentDef = this._groupMap[parentId];
 	if(parentDef) {
 		var chdr = parentDef.children;
-		if(chdr.length) {
+		if(chdr && chdr.length) {
 			var at = chdr.indexOf(childId);
 			if (at >= 0) {
 				chdr.splice(at, 1); // splice is slow
@@ -17220,7 +17241,7 @@ GroupDefinitions.prototype.removeAllChildren = function(groupId) {
 	var grpDef = this._groupMap[groupId];
 	if(grpDef) {
 		var chdr = grpDef.children;
-		var len = chdr.length;
+		var len = chdr ? chdr.length : 0;
 		if(len) {
 			grpDef.children = [];
 			for(var i = 0; i < len; ++i) {
@@ -28976,6 +28997,8 @@ var DataView = function(source) {
 	t._onRefreshTimeout = t._onRefreshTimeout.bind(t);
 	t._updateWrapCount = t._updateWrapCount.bind(t);
 
+	t._byRemovalMap = t._byRemovalMap.bind(t);
+
 	t._rids = [];
 	t._sortingDefs = [];
 	t._columnStats = {};
@@ -29029,6 +29052,15 @@ DataView.prototype._hiddenRids = null;
 * @type {Object.<string, boolean>}
 */
 DataView.prototype._collapsedRids = null; // for segmentation
+/** @private
+* @type {Object.<string, number>}
+*/
+DataView.prototype._excludedRids = null;
+/** @private
+* @type {boolean}
+*/
+DataView.prototype._emptySegmentFiltering = false;
+
 /** @private
 * @type {Object.<string, number>}
 */
@@ -29725,8 +29757,6 @@ DataView.prototype.hideRow = function(rId, hidden) {
  */
 DataView.prototype.hideRows = function(rowRefs, hidden) {
 	hidden = hidden !== false;
-	var dirty = false;
-	var rids = this._toRowIds(rowRefs);
 	var hiddenRids = this._hiddenRids;
 
 	if(hidden){
@@ -29737,11 +29767,19 @@ DataView.prototype.hideRows = function(rowRefs, hidden) {
 		return; // All rows are visible
 	}
 
+	var rids = this._toRowIds(rowRefs);
+	var dirty = false;
+
 	for(var i = rids.length; --i >= 0;) {
 		var rid = rids[i];
 		if(rid) { // undefined, null, and an empty string value are not a valid row id
-			if(!!hiddenRids[rid] !== hidden) {
-				hiddenRids[rid] = hidden;
+			if(hidden) {
+				if(!hiddenRids[rid]) {
+					hiddenRids[rid] = true;
+					dirty = true;
+				}
+			} else if(hiddenRids[rid]) {
+				delete hiddenRids[rid];
 				dirty = true;
 			}
 		}
@@ -29750,11 +29788,9 @@ DataView.prototype.hideRows = function(rowRefs, hidden) {
 	if(dirty) {
 		if(!hidden) {
 			var hasHiddenRow = false;
-			for(var key in hiddenRids) {
-				if(hiddenRids[key]) {
-					hasHiddenRow = true;
-					break;
-				}
+			for(var key in hiddenRids) { // eslint-disable-line
+				hasHiddenRow = true;
+				break;
 			}
 			if(!hasHiddenRow) {
 				hiddenRids = this._hiddenRids = null;
@@ -29762,6 +29798,7 @@ DataView.prototype.hideRows = function(rowRefs, hidden) {
 		}
 		this._refreshAndNotify(); // Very slow
 	}
+
 };
 /**
  * Show/hide rows in the data view
@@ -29843,23 +29880,14 @@ DataView.prototype.filterOut = function(cid, value) {
 */
 DataView.prototype.filterInOnce = function(cid, value, opt_filteringOut) {
 	var checker = this._getFilterLogic(cid, value);
-	if(!checker) { return; }
-
-	var filteringOut = (opt_filteringOut === true);
-	var rids = this._rids;
-	var dt = this._dt;
 	var removalMap = {};
-	var totalRem = 0;
-	var len = rids.length;
-	for(var i = len; --i >= 0;) {
-		var rid = rids[i];
-		var values = dt.getRowData(rid);
-		if (!values || checker(rid, values) === filteringOut) {
-			removalMap[rid] = true;
-			++totalRem;
-		}
+	if(!this._getRemovalMap(
+		removalMap,
+		checker,
+		(opt_filteringOut === true)
+	)) {
+		return;
 	}
-	if(totalRem <= 0) { return; }
 
 	var firstChange = this._removeRowIds(removalMap);
 
@@ -29882,13 +29910,13 @@ DataView.prototype.filterOutOnce = function(cid, value) {
 	this.filterInOnce(cid, value, true);
 };
 /** @private
-* @param {!Object.<string, *>} removalMap
+* @param {!Object} removalMap
 * @return {number}
 */
 DataView.prototype._removeRowIds = function(removalMap) {
-	var firstChange = this._removeArrayItems(this._rids, removalMap);
+	var firstChange = DataView._removeArrayItems(this._rids, removalMap);
 	if(this._groupView) {
-		firstChange = this._removeArrayItems(this._groupView, removalMap);
+		firstChange = DataView._removeArrayItems(this._groupView, removalMap);
 	}
 
 	if(this._groupMembers) {
@@ -29899,6 +29927,7 @@ DataView.prototype._removeRowIds = function(removalMap) {
 		}
 	}
 	return firstChange;
+
 };
 /** @public
 * @fires DataView#pageCountChanged
@@ -31452,9 +31481,9 @@ DataView.prototype.getSegmentChildIds = function(segmentRef) {
 };
 /** Sort all of existing segments by multiple sort logics
 * @public
-* @param {Array.<Function>} sortLogics
-* @param {Array.<number>} sortOrders
-* @param {Array.<string>} cids
+* @param {Function|Array.<Function>|Object} sortLogics
+* @param {Array.<number>=} sortOrders
+* @param {Array.<string>=} cids
 */
 DataView.prototype.sortSeparators = function (sortLogics, sortOrders, cids) {
 	this._dt.sortSeparators(sortLogics, sortOrders, cids);
@@ -31466,7 +31495,19 @@ DataView.prototype.sortSeparators = function (sortLogics, sortOrders, cids) {
 DataView.prototype.sortSegments = function (compare) {
 	this._dt.sortSegments(compare);
 };
-
+/** Automatically hide empty segment when all of its member are filtered out. An empty segment will not be hidden, if there is no active filter. Collapsed segment does not count as filtering.
+* @public
+* @param {boolean=} enabled
+*/
+DataView.prototype.enableEmptySegmentFiltering = function (enabled) {
+	enabled = enabled !== false;
+	if(this._emptySegmentFiltering !== enabled) {
+		this._emptySegmentFiltering = enabled;
+		if(this._userFilter) {
+			this._refreshAndNotify();
+		}
+	}
+};
 /**
 * @public
 * @param {string|number} segmentRef Row id or row index
@@ -31626,26 +31667,35 @@ DataView.prototype._updateRowIds = function(opt_rowIds) {
 	// Perform the following sequences: parent view cloning >> row hiding >> row filtering >> row grouping >> sorting >> paging
 	this._rids = opt_rowIds || this._parent.getAllRowIds(); // Get all data ids
 
-	if(this._hiddenRids) {
-		this._removeArrayItems(this._rids, this._hiddenRids);
-	}
+	this._dispatch("beforeFiltering", {});
+
+	this._excludedRids = {};
+	var exclusionCount = 0;
+	exclusionCount += DataView._copyObjectKeys(this._excludedRids, this._hiddenRids);
+
+	// Segment separators should not be filtered out (hidden)
 	var segments = this._dt._getSegmentSeparators();
+	var filterExceptions = segments ? segments.getSegments() : null;
+	var userRemoval = this._getRemovalMap(this._excludedRids, this._userFilter, this._filteringOut, filterExceptions);
+	exclusionCount += userRemoval;
+
 	this._collapsedRids = null;
-	var filterExceptions = null;
 	if(segments) {
-		filterExceptions = segments.getSegments(); // Segment separators should not be filtered out (hidden)
-		var collapsedRows = this._collapsedRids = segments.getCollapsedRows(); // Children of collapsed segments must be filtered out (hidden)
-		if(collapsedRows) {
-			this._removeArrayItems(this._rids, collapsedRows);
+		if(userRemoval && this._emptySegmentFiltering) {
+			exclusionCount += this._getEmptySegments(this._excludedRids, filterExceptions);
 		}
+		this._collapsedRids = segments.getCollapsedRows();
+		 // Children of collapsed segments must be filtered out (hidden)
+		exclusionCount += DataView._copyObjectKeys(this._excludedRids, this._collapsedRids);
 	}
 
-	this._dispatch("beforeFiltering", {});
-	this._quickFilter(this._userFilter, this._filteringOut, filterExceptions);
-
-	if(this._groupLevel > 0 && !opt_rowIds) {
-		this._quickFilter(this._groupFilterLogic, false); // Filter In
+	if(this._groupLevel > 0 && !opt_rowIds) { // WARNING: The line below is quite slow
+		exclusionCount += this._getRemovalMap(this._excludedRids, this._groupFilterLogic, false); // Filter In
 	}
+	if(exclusionCount) {
+		this._rids = this._rids.filter(this._byRemovalMap);
+	}
+	this._excludedRids = null;
 
 	if(this._groupMembers) { // Has grouping
 		this._populateGroups(); // View will be properly re-populate inside _populateGroups()
@@ -31917,9 +31967,9 @@ DataView.prototype._onRowUpdated = function(e) { // onUpdate
 
 		if(this._groupLevel > 0) {
 			if(processingFlag === 1) { // The row is moved to the other group
-				if(this._removeArrayItem(this._rids, rid) >= 0) {
+				if(DataView._removeArrayItem(this._rids, rid) >= 0) {
 					if(this._groupView) {
-						this._removeArrayItem(this._groupView, rid);
+						DataView._removeArrayItem(this._groupView, rid);
 					}
 				}
 				if(this._shared.multiGroupRow) {
@@ -32119,34 +32169,31 @@ DataView.prototype._getRowIndex = function(rids, opt_nextRid, opt_fallback) {
 * @returns {number}
 */
 DataView.prototype._removeDataRow = function(rid) {
-	var at = this._removeArrayItem(this._rids, rid);
+	var at = DataView._removeArrayItem(this._rids, rid);
 	if(this._groupView && at >= 0) {
-		at = this._removeArrayItem(this._groupView, rid);
+		at = DataView._removeArrayItem(this._groupView, rid);
 	}
 	return at;
 };
 /** @private
-* @param {Array} ary
+* @param {!Array} ary
 * @param {*} item
 * @return {number} Index of the removed item
 */
-DataView.prototype._removeArrayItem = function(ary, item) {
-	var len = ary.length;
-	for(var i = 0; i < len; ++i) {
-		if(ary[i] === item) {
-			ary.splice(i, 1);
-			return i;
-		}
+DataView._removeArrayItem = function(ary, item) {
+	var at = ary.indexOf(item);
+	if(at >= 0) {
+		ary.splice(at, 1);
 	}
-	return -1;
+	return at;
 };
 /** Remove multiple array items
 * @private
 * @param {Array.<string>} ary
-* @param {Object.<string, *>} items
+* @param {Object} items
 * @return {number} First item index that is being removed. NaN if no item is removed
 */
-DataView.prototype._removeArrayItems = function(ary, items) {
+DataView._removeArrayItems = function(ary, items) {
 	var f = NaN;
 	var c = 0;
 	for(var i = ary.length; --i >= 0;) {
@@ -32164,6 +32211,24 @@ DataView.prototype._removeArrayItems = function(ary, items) {
 	}
 	return f;
 };
+/** @private
+* @param {Object} baseObj
+* @param {Object} masterObj
+* @returns {number}
+*/
+DataView._copyObjectKeys = function(baseObj, masterObj) {
+	if(masterObj) {
+		var count = 0;
+
+		for(var key in masterObj) {
+			baseObj[key] = 1;
+			++count; // WARNING: duplicated key can be counted more than once
+		}
+		return count;
+	}
+	return 0;
+};
+
 /** @private
 * @param {string|null} rid
 * @param {Object} rowData
@@ -32198,9 +32263,6 @@ DataView.prototype.isRowFiltered = function(rid, rowData) {
 			return true;
 		}
 	}
-	if(this.isSegmentSeparator(rid)) {
-		return false; // Segment separator cannot be filtered
-	}
 	if(this._collapsedRids) {
 		if(this._collapsedRids[rid]) {
 			return true;
@@ -32242,41 +32304,92 @@ DataView.prototype._sort = function() {
 };
 
 /** @private
+* @param {string} rid
+* @returns {boolean}
+*/
+DataView.prototype._byRemovalMap = function(rid) {
+	return !this._excludedRids[rid];
+};
+/** @private
+* @param {Object} removalMap
 * @param {Function} checker
 * @param {boolean} filteringOut
 * @param {Object=} exceptions
+* @returns {number} Number of rids that would be filtered out
 */
-DataView.prototype._quickFilter = function(checker, filteringOut, exceptions) {
+DataView.prototype._getRemovalMap = function(removalMap, checker, filteringOut, exceptions) {
 	if(!checker) {
-		return;
+		return 0;
 	}
 
 	var rids = this._rids; // Make local variable to speed up the process
-	var len = rids.length;
 	var dt = this._dt;
-	var spliceCount = 0;
-	for(var i = len; --i >= 0;) {
-		var rid = rids[i];
-		var removed = false;
-		if(!exceptions || !exceptions[rid]) {
-			var values = dt.getRowData(rid);
+	var count = 0;
+	var rid, i, values;
+	var len = rids.length;
+
+	if(exceptions) {
+		for(i = len; --i >= 0;) {
+			rid = rids[i];
+			if(!exceptions[rid]) {
+				values = dt.getRowData(rid);
+				if (values) {
+					if(checker(rid, values) === filteringOut) {
+						removalMap[rid] = 1;
+						++count;
+					}
+				} else {
+					removalMap[rid] = 1;
+					++count;
+				}
+			}
+		}
+	} else {
+		for(i = len; --i >= 0;) {
+			rid = rids[i];
+			values = dt.getRowData(rid);
 			if (values) {
-				removed = checker(rid, values) === filteringOut;
+				if(checker(rid, values) === filteringOut) {
+					removalMap[rid] = 1;
+					++count;
+				}
 			} else {
-				removed = true;
+				removalMap[rid] = 1;
+				++count;
 			}
 		}
-		if(removed) {
-			++spliceCount;
-		} else if(spliceCount > 0) {
-			rids.splice(i + 1, spliceCount);
-			spliceCount = 0;
-		}
 	}
-	if(spliceCount > 0) {
-		rids.splice(0, spliceCount);
+	return count;
+};
+/** @private
+* @param {Object} removalMap
+* @param {Object} segmentRids
+* @returns {number} Number of rids that would be filtered out
+*/
+DataView.prototype._getEmptySegments = function(removalMap, segmentRids) {
+	var segments = this._dt._getSegmentSeparators();
+	var count = 0;
+	for(var segmentId in segmentRids) {
+		var segment = segments.getSegment(segmentId);
+		if(segment) {
+			var chdr = segment.getChildren();
+			var emptySegment = true;
+			for(var childId in chdr) {
+				if(!removalMap[childId]) {
+					emptySegment = false;
+					break;
+				}
+			}
+			if(emptySegment) {
+				removalMap[segmentId] = 1;
+				++count;
+			}
+		}
 	}
+
+	return count;
 };
+
 /** @private
 * @param {string|Function|undefined} cid
 * @param {*} value
@@ -32555,13 +32668,13 @@ DataView.prototype._removeGroupMember = function (groupIndex, groupId) {
 };
 /** @private
 * @param {string} cid
-* @param {Object.<string, *>} values
+* @param {Object} values
 * @return {Array.<string>}
 */
 DataView.prototype._defaultGroupCriteria = function(cid, values) {
 	var val = values[cid];
 	if(Array.isArray(val)) {
-		return val.map(function(data) {
+		return val.map(function(data) { // TODO: this is very slow
 			return data + "";
 		});
 	} else {
@@ -32570,11 +32683,11 @@ DataView.prototype._defaultGroupCriteria = function(cid, values) {
 };
 /** @private
 * @param {string|null} rid
-* @param {Object.<string, *>} values
+* @param {Object} values
 * @return {boolean}
 */
 DataView.prototype._groupFilterLogic = function(rid, values) {
-	var gids = this._groupCriteria[this._groupLevel - 1](values);
+	var gids = this._groupCriteria[this._groupLevel - 1](values); // TODO: this is very slow
 	return gids.indexOf(this._groupId) >= 0;
 };
 
@@ -36014,7 +36127,7 @@ Core.prototype._batches = null;
 * @return {string}
 */
 Core.getVersion = function () {
-	return "5.1.67";
+	return "5.1.71";
 };
 /** {@link ElementWrapper#dispose}
 * @override
@@ -39941,7 +40054,14 @@ Core.prototype.stopBatch = function (batchType) {
 	if(!batchType){
 		return false;
 	}
+	// The "columnVisibilityChanged" event is blocked while the "reset" batch operation is in progress, so this event will not be fired until the batch operation succeeds.
+	if(batchType === "reset") {
+		this._disableEvent("columnVisibilityChanged", true);
+	}
 	this._dispatch("afterBatchOperation", { batches: this._batches, batchType: batchType });
+	if(batchType === "reset") {
+		this._disableEvent("columnVisibilityChanged", false);
+	}
 
 	delete this._batches[batchType];
 	if(isEmptyObject(this._batches)){
@@ -40408,13 +40528,10 @@ Core.prototype._onSectionDataChanged = function (e) {
 				for (var r = fromR; r < toR; ++r) {
 					if(hasDataView) {
 						var rowData = rowDataCollection[r];
-						if(!rowData) { // This is a header row
-							continue;
-						}
 
 						e["rowData"] = rowData;
 						e["rowId"] = rids[r];
-						e["dataValue"] = rowData[cid];
+						e["dataValue"] = rowData ? rowData[cid] : null;
 					}
 					e["rowIndex"] = r;
 					e["cell"] = section["getCell"](c, r, false); // Accessing cell by using bracket allows extenal object to mock Section
@@ -42955,7 +43072,7 @@ SortableTitlePlugin.prototype.sortColumns = function (sortOptions, opt_arg) {
 				opt["sortOrder"] || opt["order"]
 			);
 			if (state) {
-				states[i] = state;
+				states.push(state);
 			}
 		}
 		this._sortColumn(states, opt_arg);
@@ -43537,10 +43654,12 @@ SortableTitlePlugin.prototype.sortSeparators = function (comparer) {
 		var sortOrders = [];
 		var sortFields = [];
 		var sortStateCount = this._sortStates.length;
+		var rowDefField = SortableTitlePlugin._toRowDefField();
 		for(var i = 0; i < sortStateCount; i++){
 			var sortState =  this._sortStates[i];
+			var field = this._rowDefMode ? rowDefField : sortState["field"];
 			sortOrders.push(sortState["sortOrder"]);
-			sortFields.push(sortState["field"]);
+			sortFields.push(field);
 		}
 		dv.sortSeparators(sortLogics, sortOrders, sortFields);
 	}
@@ -44241,15 +44360,6 @@ var cloneRowData = function(fromRowDef, toRowDef) {
 	}
 };
 
-/** @private
-* @param {string} sortField
-* @param {Object} elemData
-* @param {number} index
-*/
-var mapRowOrder = function (sortField, elemData, index) { // edit name
-	elemData[sortField] = index; // Make column for sort with user data array
-};
-
 /** @private
 * @param {RowDefinition} rowDef
 * @return {boolean}
@@ -44334,6 +44444,19 @@ var _hasFieldOrId = function(colDef, str) {
 	return (colDef.getField() === str) || (colDef.getId() === str);
 };
 
+/** Compare the difference in the 'id' property.
+ * @private
+ * @param {Object} obj1
+ * @param {Object} obj2
+ * @returns {boolean} If the id property of two objects is equal, the return will be true, otherwise it will be false.
+ */
+var _hasMatchingId = function(obj1, obj2) {
+	if(!obj1 || !obj2 || !obj1.id || !obj2.id) { // Handle nullable, if the object or id have null, it's means difference value
+		return false;
+	}
+	return obj1.id === obj2.id;
+};
+
 /** @constructor
 * @extends {EventDispatcher}
 * @param {(Element|null)=} placeholder
@@ -44883,7 +45006,6 @@ Grid.prototype.initialize = function(gridOption) {
 
 	if (gridOption["fieldCaching"]) {
 		t._fieldCaching = gridOption["fieldCaching"];
-		js_FieldDefinition.setFieldCaching(t._fieldCaching);
 	}
 
 	if(gridOption["timeSeriesExpansion"] != null) {
@@ -45633,8 +45755,8 @@ Grid.prototype.replaceColumn = function (columnOption, colRef) {
  * @param {Object} response
  */
 Grid.prototype._onFieldLoadedSuccess = function (field, colDef, response) {
-	if (response && response.fieldDefinition) {
-		var fieldDef = response.fieldDefinition;
+	if (response && response.id) {
+		var fieldDef = response;
 		if (colDef && colDef.getField() === field) {
 			if (colDef.isDefaultName() && fieldDef.name) {
 				colDef.setName(fieldDef.name);
@@ -45682,6 +45804,34 @@ Grid.prototype._setScrollbarParent = function (host) {
 	this._grid.getHScrollbar().attachToExternalElement(host);
 };
 
+/** Get stored field information. If field information has not been requested or no data has been received yet, null value is returned.
+* @public
+* @function
+* @param {string} field
+* @return {Object}
+*/
+Grid.prototype.getFieldInfo = function(field) {
+	return js_FieldDefinition.getFieldInfo(field);
+};
+/** Request field information from Synapse service. If field information already exists, a resolved promise is returned. Synapse config must be supplied before the request can be made.
+* @public
+* @function
+* @param {string} field
+* @return {Promise}
+* @example
+* var gridConfig = {
+*   synapse: { // define synapse configuration
+*     apiKey: "xxx",
+*     contextApp: "xxx",
+*     auth: "xxx" (optional)
+*   }
+* };
+* var promise = grid.loadFieldInfo("CF_LAST");
+*/
+Grid.prototype.loadFieldInfo = function(field) {
+	return js_FieldDefinition.loadFieldInfo(field);
+};
+
 /**
 * @private
 * @param {string} field
@@ -45740,8 +45890,9 @@ Grid.prototype.setColumns = function(columns) {
 /** Remove, add and keep column based on the given column data
 * @public
 * @param {Array.<Object>} columns Array of column options
+* @param {boolean=} byId=false, if enable it, this method will only check for differences in the 'id' property
 */
-Grid.prototype.restoreColumns = function(columns) {
+Grid.prototype.restoreColumns = function(columns, byId) {
 	var grid = this._grid;
 	grid.startBatch("reset");
 	var configObj = this.getConfigObject();
@@ -45750,6 +45901,7 @@ Grid.prototype.restoreColumns = function(columns) {
 	var preColLen = previousColumns.length;
 	var newColLen = columns.length;
 
+	var compareLogic = byId ? _hasMatchingId : deepEqual;
 	var removingFields = [];
 	var keepingColumns = [];
 	var columnOrdering = [];
@@ -45759,7 +45911,7 @@ Grid.prototype.restoreColumns = function(columns) {
 	for (i = 0; i < preColLen; i++) {
 		found = false;
 		for (j = 0; j < newColLen; j++) {
-			if (deepEqual(previousColumns[i], columns[j])) {
+			if(compareLogic(previousColumns[i], columns[j])) {
 				keepingColumns.push(previousColumns[i]);
 				found = true;
 				break;
@@ -45780,7 +45932,7 @@ Grid.prototype.restoreColumns = function(columns) {
 	for (i = 0; i < newColLen; i++) {
 		found = false;
 		for (j = 0; j < keepingLen; j++) { // loop only keeping column
-			if (deepEqual(columns[i], keepingColumns[j])) {
+			if(compareLogic(columns[i], keepingColumns[j])) {
 				found = true;
 				var colIndex = this.getColumnIndex(columns[i].field); // We cannot use 'i' (colIndex) in this case, as it will sort the columns. Instead, we need to obtain a new column index from the field.
 				columnOrdering.push(this.getColumnId(colIndex));
@@ -46459,20 +46611,11 @@ Grid.prototype.updateDataSet = function(records, rowIdentifier) {
 	}
 
 	// Map new data index
-	var newDataMap = {};
 	var recordCount = records.length;
-	var record, i;
-	for (i = 0; i < recordCount; i++) {
-		record = records[i];
-		newDataMap[record[rowIdentifier]] = record; // Assign a new data map to compare to the previous data
-	}
-
 	var fieldSorting = "ROW_ORDER"; // TODO: Should be config by options
-	records.forEach(mapRowOrder.bind(null, fieldSorting));
-
 	var oldDataMap = {};
-	var rowDef, id;
-	var rowDefs = this.getRowDefinitions(); // WARNING: Filtered and hidden rows are not included
+	var rowDef, id, record, i;
+	var rowDefs = this.getAllRowDefinitions(); // Include the filter/hidden rows
 	var rowDefCount = rowDefs.length;
 	for (i = 0; i < rowDefCount; i++) {
 		rowDef = rowDefs[i];
@@ -46486,26 +46629,35 @@ Grid.prototype.updateDataSet = function(records, rowIdentifier) {
 		}
 	}
 
-	// Check Remove previous data
-	for (var rowIdentifierName in oldDataMap) {
-		if (oldDataMap[rowIdentifierName] && !newDataMap[rowIdentifierName]) {
-			this.removeRow(oldDataMap[rowIdentifierName]); // Slow
-		}
-	}
-
 	// Check Update and Insert
-	for (i = 0; i < recordCount; i++) {
+	var idMap = {};
+	var newDataMap = {};
+	for (i = recordCount - 1; i >= 0; i--) {
 		record = records[i];
 		id = record[rowIdentifier];
 		rowDef = oldDataMap[id];
+		newDataMap[id] = record; // Assign a new data map to compare to the previous data
+		record[fieldSorting] = i;
+		if(idMap[id]) { // Prevent assign data in duplicate row
+			continue;
+		}
+		idMap[id] = true;
 		if (!rowDef) {
-			this.insertRow({ values: newDataMap[id]}); // Insert last position
+			this.insertRow({ values: record}); // Insert last position
 		} else {
 			rowDef.setRowData(record);
 		}
 	}
+
+	// Check Remove previous data
+	for (var rowIdentifierName in oldDataMap) {
+		if (oldDataMap[rowIdentifierName] && !newDataMap[rowIdentifierName]) {
+			this.removeRow(oldDataMap[rowIdentifierName]); // Slow
+		}
+	}
+
 	// Sorting
-	this._dt.sortOnce("ROW_DEF", "a", compareNumber, fieldSorting);
+	this._dt.sortOnce(ROW_DEF, "a", compareNumber, fieldSorting);
 };
 /** @private
 * @param {Array|Object} item
@@ -49674,7 +49826,11 @@ var _joinSubKeys = function(subA, subB) {
 */
 var _isDynamicChain = function(ric) {
 	// Dynamic chain will be 2 . (dot) for example .PG.PA
-	return ric.match(/\./g).length > 1;
+	var matching = ric.match(/\./g); // Can be null when doesn't found dot(.)
+	if(matching) {
+		return matching.length > 1;
+	}
+	return false;
 };
 
 /** @private