@finos/legend-application-repl 0.0.40 → 0.0.41

package/src/stores/dataCube/editor/DataCubeMutableConfiguration.ts

@@ -18,7 +18,8 @@ import {
   getDataType,
   type DataCubeColumnKind,
   type DataCubeFont,
-  type DataCubeAggregateFunction,
+  type DataCubeAggregateOperation,
+  type DataCubeOperationValue,
   type DataCubeNumberScale,
   type DataCubeSelectionStat,
   type DataCubeFontFormatUnderlineVariant,
@@ -50,6 +51,13 @@ import { buildDefaultColumnConfiguration } from '../core/DataCubeConfigurationBu
 export class DataCubeMutableColumnConfiguration extends DataCubeColumnConfiguration {
   readonly dataType!: DataCubeColumnDataType;
 
+  // NOTE: these configurations are synthesized from the data query, and materialized here
+  // to make editing more convenient. They should not be part of the persistent configuration
+  // to avoid duplication of information with the data query.
+  aggregateFunction?: DataCubeAggregateOperation | undefined;
+  aggregateFunctionParameters: DataCubeOperationValue[] = [];
+  excludedFromHorizontalPivot = false;
+
   static create(
     json: PlainObject<DataCubeColumnConfiguration>,
   ): DataCubeMutableColumnConfiguration {
@@ -79,6 +87,9 @@ export class DataCubeMutableColumnConfiguration extends DataCubeColumnConfigurat
       numberScale: observable,
       setNumberScale: action,
 
+      missingValueDisplayText: observable,
+      setMissingValueDisplayText: action,
+
       hPivotSortFunction: observable,
       setHPivotSortFunction: action,
 
@@ -136,12 +147,6 @@ export class DataCubeMutableColumnConfiguration extends DataCubeColumnConfigurat
       hideFromView: observable,
       setHideFromView: action,
 
-      aggregateFunction: observable,
-      setAggregateFunction: action,
-
-      excludedFromHorizontalPivot: observable,
-      setExcludedFromHorizontalPivot: action,
-
       fixedWidth: observable,
       setFixedWidth: action,
 
@@ -162,15 +167,21 @@ export class DataCubeMutableColumnConfiguration extends DataCubeColumnConfigurat
 
       isUsingDefaultStyling: computed,
       useDefaultStyling: action,
+
+      aggregateFunction: observable,
+      setAggregateFunction: action,
+
+      aggregateFunctionParameters: observable,
+      setAggregateFunctionParameters: action,
+
+      excludedFromHorizontalPivot: observable,
+      setExcludedFromHorizontalPivot: action,
     });
 
     return configuration;
   }
 
-  static createDefault(column: {
-    name: string;
-    type: string;
-  }): DataCubeMutableColumnConfiguration {
+  static createDefault(column: { name: string; type: string }) {
     return DataCubeMutableColumnConfiguration.create(
       DataCubeColumnConfiguration.serialization.toJson(
         buildDefaultColumnConfiguration(column),
@@ -178,11 +189,11 @@ export class DataCubeMutableColumnConfiguration extends DataCubeColumnConfigurat
     );
   }
 
-  serialize(): PlainObject<DataCubeColumnConfiguration> {
+  serialize() {
     return DataCubeColumnConfiguration.serialization.toJson(this);
   }
 
-  get isUsingDefaultStyling(): boolean {
+  get isUsingDefaultStyling() {
     return (
       this.fontFamily === undefined &&
       this.fontSize === undefined &&
@@ -202,7 +213,7 @@ export class DataCubeMutableColumnConfiguration extends DataCubeColumnConfigurat
     );
   }
 
-  useDefaultStyling(): void {
+  useDefaultStyling() {
     this.fontFamily = undefined;
     this.fontSize = undefined;
     this.fontBold = undefined;
@@ -220,139 +231,145 @@ export class DataCubeMutableColumnConfiguration extends DataCubeColumnConfigurat
     this.errorBackgroundColor = undefined;
   }
 
-  setKind(value: DataCubeColumnKind): void {
+  setKind(value: DataCubeColumnKind) {
     this.kind = value;
   }
 
-  setDisplayName(value: string | undefined): void {
+  setDisplayName(value: string | undefined) {
     this.displayName = value;
   }
 
-  setDecimals(value: number | undefined): void {
+  setDecimals(value: number | undefined) {
     this.decimals = value;
   }
 
-  setDisplayCommas(value: boolean): void {
+  setDisplayCommas(value: boolean) {
     this.displayCommas = value;
   }
 
-  setNegativeNumberInParens(value: boolean): void {
+  setNegativeNumberInParens(value: boolean) {
     this.negativeNumberInParens = value;
   }
 
-  setNumberScale(value: DataCubeNumberScale | undefined): void {
+  setNumberScale(value: DataCubeNumberScale | undefined) {
     this.numberScale = value;
   }
 
-  setHPivotSortFunction(value: string | undefined): void {
+  setMissingValueDisplayText(value: string | undefined) {
+    this.missingValueDisplayText = value;
+  }
+
+  setHPivotSortFunction(value: string | undefined) {
     this.hPivotSortFunction = value;
   }
 
-  setFontFamily(value: DataCubeFont | undefined): void {
+  setFontFamily(value: DataCubeFont | undefined) {
     this.fontFamily = value;
   }
 
-  setFontSize(value: number | undefined): void {
+  setFontSize(value: number | undefined) {
     this.fontSize = value;
   }
 
-  setFontBold(value: boolean | undefined): void {
+  setFontBold(value: boolean | undefined) {
     this.fontBold = value;
   }
 
-  setFontItalic(value: boolean | undefined): void {
+  setFontItalic(value: boolean | undefined) {
     this.fontItalic = value;
   }
 
-  setFontUnderline(
-    value: DataCubeFontFormatUnderlineVariant | undefined,
-  ): void {
+  setFontUnderline(value: DataCubeFontFormatUnderlineVariant | undefined) {
     this.fontUnderline = value;
   }
 
-  setFontStrikethrough(value: boolean | undefined): void {
+  setFontStrikethrough(value: boolean | undefined) {
     this.fontStrikethrough = value;
   }
 
-  setFontCase(value: DataCubeFontCase | undefined): void {
+  setFontCase(value: DataCubeFontCase | undefined) {
     this.fontCase = value;
   }
 
-  setTextAlign(value: DataCubeFontTextAlignment | undefined): void {
+  setTextAlign(value: DataCubeFontTextAlignment | undefined) {
     this.textAlign = value;
   }
 
-  setNormalForegroundColor(value: string | undefined): void {
+  setNormalForegroundColor(value: string | undefined) {
     this.normalForegroundColor = value;
   }
 
-  setNegativeForegroundColor(value: string | undefined): void {
+  setNegativeForegroundColor(value: string | undefined) {
     this.negativeForegroundColor = value;
   }
 
-  setZeroForegroundColor(value: string | undefined): void {
+  setZeroForegroundColor(value: string | undefined) {
     this.zeroForegroundColor = value;
   }
 
-  setErrorForegroundColor(value: string | undefined): void {
+  setErrorForegroundColor(value: string | undefined) {
     this.errorForegroundColor = value;
   }
 
-  setNormalBackgroundColor(value: string | undefined): void {
+  setNormalBackgroundColor(value: string | undefined) {
     this.normalBackgroundColor = value;
   }
 
-  setNegativeBackgroundColor(value: string | undefined): void {
+  setNegativeBackgroundColor(value: string | undefined) {
     this.negativeBackgroundColor = value;
   }
 
-  setZeroBackgroundColor(value: string | undefined): void {
+  setZeroBackgroundColor(value: string | undefined) {
     this.zeroBackgroundColor = value;
   }
 
-  setErrorBackgroundColor(value: string | undefined): void {
+  setErrorBackgroundColor(value: string | undefined) {
     this.errorBackgroundColor = value;
   }
 
-  setBlur(value: boolean): void {
+  setBlur(value: boolean) {
     this.blur = value;
   }
 
-  setHideFromView(value: boolean): void {
+  setHideFromView(value: boolean) {
     this.hideFromView = value;
   }
 
-  setFixedWidth(value: number | undefined): void {
+  setFixedWidth(value: number | undefined) {
     this.fixedWidth = value;
   }
 
-  setMinWidth(value: number | undefined): void {
+  setMinWidth(value: number | undefined) {
     this.minWidth = value;
   }
 
-  setMaxWidth(value: number | undefined): void {
+  setMaxWidth(value: number | undefined) {
     this.maxWidth = value;
   }
 
-  setPinned(value: DataCubeColumnPinPlacement | undefined): void {
+  setPinned(value: DataCubeColumnPinPlacement | undefined) {
     this.pinned = value;
   }
 
-  setDisplayAsLink(value: boolean): void {
+  setDisplayAsLink(value: boolean) {
     this.displayAsLink = value;
   }
 
-  setLinkLabelParameter(value: string | undefined): void {
+  setLinkLabelParameter(value: string | undefined) {
     this.linkLabelParameter = value;
   }
 
-  setAggregateFunction(value: DataCubeAggregateFunction | undefined): void {
+  setAggregateFunction(value: DataCubeAggregateOperation | undefined) {
     this.aggregateFunction = value;
   }
 
-  setExcludedFromHorizontalPivot(value: boolean): void {
+  setExcludedFromHorizontalPivot(value: boolean) {
     this.excludedFromHorizontalPivot = value;
   }
+
+  setAggregateFunctionParameters(value: DataCubeOperationValue[]) {
+    this.aggregateFunctionParameters = value;
+  }
 }
 
 export class DataCubeMutableConfiguration extends DataCubeConfiguration {
@@ -472,7 +489,7 @@ export class DataCubeMutableConfiguration extends DataCubeConfiguration {
     return configuration;
   }
 
-  get isUsingDefaultStyling(): boolean {
+  get isUsingDefaultStyling() {
     return (
       this.fontFamily === DEFAULT_FONT_FAMILY &&
       this.fontSize === DEFAULT_FONT_SIZE &&
@@ -492,7 +509,7 @@ export class DataCubeMutableConfiguration extends DataCubeConfiguration {
     );
   }
 
-  useDefaultStyling(): void {
+  useDefaultStyling() {
     this.fontFamily = DEFAULT_FONT_FAMILY;
     this.fontSize = DEFAULT_FONT_SIZE;
     this.fontBold = DEFAULT_FONT_BOLD;
@@ -510,141 +527,139 @@ export class DataCubeMutableConfiguration extends DataCubeConfiguration {
     this.errorBackgroundColor = DEFAULT_BACKGROUND_COLOR;
   }
 
-  serialize(): PlainObject<DataCubeConfiguration> {
+  serialize() {
     return DataCubeConfiguration.serialization.toJson(this);
   }
 
-  setDescription(value: string | undefined): void {
+  setDescription(value: string | undefined) {
     this.description = value;
   }
 
-  setShowTreeLines(value: boolean): void {
+  setShowTreeLines(value: boolean) {
     this.showTreeLines = value;
   }
 
-  setShowHorizontalGridLines(value: boolean): void {
+  setShowHorizontalGridLines(value: boolean) {
     this.showHorizontalGridLines = value;
   }
 
-  setShowVerticalGridLines(value: boolean): void {
+  setShowVerticalGridLines(value: boolean) {
     this.showVerticalGridLines = value;
   }
 
-  setGridLineColor(value: string): void {
+  setGridLineColor(value: string) {
     this.gridLineColor = value;
   }
 
-  setFontFamily(value: DataCubeFont): void {
+  setFontFamily(value: DataCubeFont) {
     this.fontFamily = value;
   }
 
-  setFontSize(value: number): void {
+  setFontSize(value: number) {
     this.fontSize = value;
   }
 
-  setFontBold(value: boolean): void {
+  setFontBold(value: boolean) {
     this.fontBold = value;
   }
 
-  setFontItalic(value: boolean): void {
+  setFontItalic(value: boolean) {
     this.fontItalic = value;
   }
 
-  setFontUnderline(
-    value: DataCubeFontFormatUnderlineVariant | undefined,
-  ): void {
+  setFontUnderline(value: DataCubeFontFormatUnderlineVariant | undefined) {
     this.fontUnderline = value;
   }
 
-  setFontStrikethrough(value: boolean): void {
+  setFontStrikethrough(value: boolean) {
     this.fontStrikethrough = value;
   }
 
-  setFontCase(value: DataCubeFontCase | undefined): void {
+  setFontCase(value: DataCubeFontCase | undefined) {
     this.fontCase = value;
   }
 
-  setTextAlign(value: DataCubeFontTextAlignment): void {
+  setTextAlign(value: DataCubeFontTextAlignment) {
     this.textAlign = value;
   }
 
-  setNormalForegroundColor(value: string): void {
+  setNormalForegroundColor(value: string) {
     this.normalForegroundColor = value;
   }
 
-  setNegativeForegroundColor(value: string): void {
+  setNegativeForegroundColor(value: string) {
     this.negativeForegroundColor = value;
   }
 
-  setZeroForegroundColor(value: string): void {
+  setZeroForegroundColor(value: string) {
     this.zeroForegroundColor = value;
   }
 
-  setErrorForegroundColor(value: string): void {
+  setErrorForegroundColor(value: string) {
     this.errorForegroundColor = value;
   }
 
-  setNormalBackgroundColor(value: string): void {
+  setNormalBackgroundColor(value: string) {
     this.normalBackgroundColor = value;
   }
 
-  setNegativeBackgroundColor(value: string): void {
+  setNegativeBackgroundColor(value: string) {
     this.negativeBackgroundColor = value;
   }
 
-  setZeroBackgroundColor(value: string): void {
+  setZeroBackgroundColor(value: string) {
     this.zeroBackgroundColor = value;
   }
 
-  setErrorBackgroundColor(value: string): void {
+  setErrorBackgroundColor(value: string) {
     this.errorBackgroundColor = value;
   }
 
-  setAlternateRows(value: boolean): void {
+  setAlternateRows(value: boolean) {
     this.alternateRows = value;
   }
 
-  setAlternateRowsColor(value: string): void {
+  setAlternateRowsColor(value: string) {
     this.alternateRowsColor = value;
   }
 
-  setAlternateRowsCount(value: number): void {
+  setAlternateRowsCount(value: number) {
     this.alternateRowsCount = value;
   }
 
-  setAlternateRowsStandardMode(value: boolean): void {
+  setAlternateRowsStandardMode(value: boolean) {
     this.alternateRowsStandardMode = value;
   }
 
-  setSelectionStats(value: DataCubeSelectionStat[]): void {
+  setSelectionStats(value: DataCubeSelectionStat[]) {
     this.selectionStats = value;
   }
 
-  setShowWarningForTruncatedResult(value: boolean): void {
+  setShowWarningForTruncatedResult(value: boolean) {
     this.showWarningForTruncatedResult = value;
   }
 
-  setInitialExpandLevel(value: number | undefined): void {
+  setInitialExpandLevel(value: number | undefined) {
     this.initialExpandLevel = value;
   }
 
-  setShowRootAggregation(value: boolean): void {
+  setShowRootAggregation(value: boolean) {
     this.showRootAggregation = value;
   }
 
-  setShowLeafCount(value: boolean): void {
+  setShowLeafCount(value: boolean) {
     this.showLeafCount = value;
   }
 
-  setAddPivotTotalColumn(value: boolean): void {
+  setAddPivotTotalColumn(value: boolean) {
     this.addPivotTotalColumn = value;
   }
 
-  setAddPivotTotalColumnOnLeft(value: boolean): void {
+  setAddPivotTotalColumnOnLeft(value: boolean) {
     this.addPivotTotalColumnOnLeft = value;
   }
 
-  setTreeGroupSortFunction(value: string | undefined): void {
+  setTreeGroupSortFunction(value: string | undefined) {
     this.treeGroupSortFunction = value;
   }
 }

package/src/stores/dataCube/grid/DataCubeGridClientEngine.ts

@@ -26,6 +26,7 @@ import {
   assertErrorThrown,
   guaranteeNonNullable,
   isBoolean,
+  isNonNullable,
 } from '@finos/legend-shared';
 import { buildExecutableQuery } from '../core/DataCubeQueryBuilder.js';
 import { type TabularDataSet, V1_Lambda } from '@finos/legend-graph';
@@ -34,6 +35,8 @@ import { buildQuerySnapshot } from './DataCubeGridQuerySnapshotBuilder.js';
 import { generateRowGroupingDrilldownExecutableQueryPostProcessor } from './DataCubeGridQueryBuilder.js';
 import { makeObservable, observable, runInAction } from 'mobx';
 import type { DataCubeConfigurationColorKey } from '../core/DataCubeConfiguration.js';
+import { AlertType } from '../../../components/repl/Alert.js';
+import { DEFAULT_LARGE_ALERT_WINDOW_CONFIG } from '../../LayoutManagerState.js';
 
 type GridClientCellValue = string | number | boolean | null | undefined;
 type GridClientRowData = {
@@ -92,12 +95,21 @@ export const generateBackgroundColorUtilityClassName = (
 ) =>
   `${INTERNAL__GridClientUtilityCssClassName.BACKGROUND_COLOR_PREFIX}${key}-${color.substring(1)}`;
 
+// Indicates how many rows for each block in the cache, i.e. how many rows returned from the server at a time.
+// ag-grid will dedicte space in advanced to store these rows. In server-side row model, this is used as the page size.
+// See https://www.ag-grid.com/react-data-grid/server-side-model-configuration/#server-side-cache
+export const INTERNAL__GRID_CLIENT_DEFAULT_CACHE_BLOCK_SIZE = 100;
+// NOTE: The cache block size is used by ag-grid to pre-allocate memory for the grid
+// so the value set must be reasonable, or else it can crash the application!
+export const INTERNAL__GRID_CLIENT_MAX_CACHE_BLOCK_SIZE = 1e5;
+
 export const INTERNAL__GRID_CLIENT_SIDE_BAR_WIDTH = 200;
 export const INTERNAL__GRID_CLIENT_COLUMN_MIN_WIDTH = 50;
 export const INTERNAL__GRID_CLIENT_HEADER_HEIGHT = 24;
 export const INTERNAL__GRID_CLIENT_ROW_HEIGHT = 20;
 export const INTERNAL__GRID_CLIENT_TOOLTIP_SHOW_DELAY = 1000;
 export const INTERNAL__GRID_CLIENT_AUTO_RESIZE_PADDING = 10;
+export const INTERNAL__GRID_CLIENT_MISSING_VALUE = '__MISSING';
 export const INTERNAL__GRID_CLIENT_TREE_COLUMN_ID = 'INTERNAL__tree';
 export const INTERNAL__GRID_CLIENT_ROW_GROUPING_COUNT_AGG_COLUMN_ID =
   'INTERNAL__count';
@@ -151,9 +163,12 @@ function TDStoRowData(tds: TabularDataSet): GridClientRowData[] {
     _row.values.forEach((value, colIdx) => {
       // `ag-grid` shows `false` value as empty string so we have
       // call `.toString()` to avoid this behavior.
-      row[cols[colIdx] as string] = isBoolean(value) ? String(value) : value;
+      row[cols[colIdx] as string] = isBoolean(value)
+        ? String(value)
+        : isNonNullable(value)
+          ? value
+          : INTERNAL__GRID_CLIENT_MISSING_VALUE;
     });
-    row.rowNumber = rowIdx;
     return row;
   });
 }
@@ -172,9 +187,7 @@ export class DataCubeGridClientServerSideDataSource
     this.grid = grid;
   }
 
-  async fetchRows(
-    params: IServerSideGetRowsParams<unknown, unknown>,
-  ): Promise<void> {
+  async fetchRows(params: IServerSideGetRowsParams<unknown, unknown>) {
     const task = this.grid.dataCube.newTask('Fetching data');
 
     // ------------------------------ GRID OPTIONS ------------------------------
@@ -229,27 +242,75 @@ export class DataCubeGridClientServerSideDataSource
       if (this.grid.dataCube.engine.enableDebugMode) {
         this.grid.dataCube.application.debugProcess(
           `Execution`,
+          `\nQuery: ${result.executedQuery}`,
           `\nConfig: pagination=${this.grid.isPaginationEnabled}`,
           `\nStats: ${rowData.length} rows, ${result.result.result.columns.length} columns`,
           `\nSQL: ${result.executedSQL}`,
         );
       }
+
       if (this.grid.isPaginationEnabled) {
         params.success({ rowData });
-        // Only update row count when loading the top-level drilldown data
+        // only update row count when loading the top-level drilldown data
         if (params.request.groupKeys.length === 0) {
           runInAction(() => {
             this.rowCount = (params.request.startRow ?? 0) + rowData.length;
           });
         }
       } else {
+        // NOTE: When pagination is disabled and the user currently scrolls to somewhere in the grid, as data is fetched
+        // and the operation does not force a scroll top (for example, grouping will always force scrolling to the
+        // top, while sorting will leave scroll position as is), the grid ends up showing the wrong data because
+        // the data being displayed does not take into account the scroll position, but by the start and end row
+        // which stay constant as pagination is disabled.
+        //
+        // In order to handle this, when pagination is disabled, we tune the start and end row by setting the cache block size
+        // to a high-enough value (100k-1m). However, ag-grid use cache block size to pre-allocate memory for the rows,
+        // which means we must cap/tune this value reasonably to prevent the app from crashing.
+        //
+        // When there are just too many rows (exceeding the maximum cache block size), we will fallback to a slightly less ideal
+        // behavior by forcing a scroll top for every data fetch and also reset the cache block size to the default value to save memory
+        if (rowData.length > INTERNAL__GRID_CLIENT_MAX_CACHE_BLOCK_SIZE) {
+          if (
+            !this.grid.dataCube.repl.dataCubeEngine.disableLargeDatasetWarning
+          ) {
+            this.grid.dataCube.repl.alert({
+              message: `Large dataset (>${INTERNAL__GRID_CLIENT_MAX_CACHE_BLOCK_SIZE} rows) detected!`,
+              text: `Overall app performance can be impacted by large dataset due to longer query execution time and increased memory usage. At its limit, the application can crash!\nTo boost performance, consider enabling pagination while working with large dataset.`,
+              type: AlertType.WARNING,
+              actions: [
+                {
+                  label: 'Enable Pagination',
+                  handler: () => {
+                    this.grid.setPaginationEnabled(true);
+                  },
+                },
+                {
+                  label: 'Dismiss Warning',
+                  handler: () => {
+                    // this.grid.setPaginationEnabled(true);
+                    this.grid.dataCube.repl.dataCubeEngine.setDisableLargeDatasetWarning(
+                      true,
+                    );
+                  },
+                },
+              ],
+              windowConfig: DEFAULT_LARGE_ALERT_WINDOW_CONFIG,
+            });
+          }
+
+          // NOTE: when drilldown occurs, we will scroll top until the drilldown row is reached
+          params.api.ensureIndexVisible(params.parentNode.rowIndex ?? 0, 'top');
+        }
+
         params.success({
           rowData,
-          // Setting row count to disable inifite-scrolling when pagination is disabled
-          // See https://www.ag-grid.com/javascript-data-grid/infinite-scrolling/#setting-last-row-index
+          // Setting row count to disable infinite-scrolling when pagination is disabled
+          // See https://www.ag-grid.com/react-data-grid/infinite-scrolling/#setting-last-row-index
           rowCount: rowData.length,
         });
-        // Only update row count when loading the top-level drilldown data
+
+        // only update row count when loading the top-level drilldown data
         if (params.request.groupKeys.length === 0) {
           runInAction(() => {
             this.rowCount = rowData.length;
@@ -258,18 +319,17 @@ export class DataCubeGridClientServerSideDataSource
       }
     } catch (error) {
       assertErrorThrown(error);
-      this.grid.dataCube.repl.notifyError(
-        error,
-        `Data fetch failure: ${error.message}`,
-        error.stack,
-      );
+      this.grid.dataCube.repl.alertError(error, {
+        message: `Data fetch failure: ${error.message}`,
+        text: error.stack,
+      });
       params.fail();
     } finally {
       this.grid.dataCube.endTask(task);
     }
   }
 
-  getRows(params: IServerSideGetRowsParams<unknown, unknown>): void {
+  getRows(params: IServerSideGetRowsParams<unknown, unknown>) {
     this.fetchRows(params).catch((error: unknown) => {
       assertErrorThrown(error);
       this.grid.dataCube.application.logService.error(

package/src/stores/dataCube/grid/DataCubeGridClientExportEngine.ts

@@ -89,11 +89,11 @@ export class DataCubeGridClientExportEngine {
     this.grid = grid;
   }
 
-  private generateFileName(): string {
+  private generateFileName() {
     return `${this.dataCube.core.name} - ${formatDate(new Date(), 'EEE MMM dd yyyy HH_mm_ss')}`;
   }
 
-  exportFile(format: DataCubeGridClientExportFormat): void {
+  exportFile(format: DataCubeGridClientExportFormat) {
     const fileName = this.generateFileName();
     switch (format) {
       case DataCubeGridClientExportFormat.CSV: {