@finos/legend-application-repl 0.0.58 → 0.0.59

package/src/stores/dataCube/editor/DataCubeEditorHorizontalPivotsPanelState.ts

@@ -19,9 +19,10 @@ import type { DataCubeState } from '../DataCubeState.js';
 import type { DataCubeConfiguration } from '../core/DataCubeConfiguration.js';
 import {
   DataCubeColumnKind,
-  PIVOT_COLUMN_NAME_VALUE_SEPARATOR,
+  isPivotResultColumnName,
 } from '../core/DataCubeQueryEngine.js';
 import {
+  _toCol,
   type DataCubeQuerySnapshot,
   type DataCubeQuerySnapshotColumn,
 } from '../core/DataCubeQuerySnapshot.js';
@@ -31,6 +32,7 @@ import {
 } from './DataCubeEditorColumnsSelectorState.js';
 import type { DataCubeQueryEditorPanelState } from './DataCubeEditorPanelState.js';
 import type { DataCubeEditorState } from './DataCubeEditorState.js';
+import { uniqBy } from '@finos/legend-shared';
 
 export class DataCubeEditorHorizontalPivotColumnsSelectorState extends DataCubeEditorColumnsSelectorState<DataCubeEditorColumnsSelectorColumnState> {
   override cloneColumn(
@@ -42,15 +44,14 @@ export class DataCubeEditorHorizontalPivotColumnsSelectorState extends DataCubeE
     );
   }
 
-  override get availableColumns(): DataCubeEditorColumnsSelectorColumnState[] {
-    return this.editor.columns.selector.selectedColumns
+  override get availableColumns() {
+    return this.editor.columnProperties.columns
       .filter(
-        (column) =>
-          this.editor.columnProperties.getColumnConfiguration(column.name)
-            ?.kind === DataCubeColumnKind.DIMENSION &&
+        (col) =>
+          col.kind === DataCubeColumnKind.DIMENSION &&
           // exclude group-level extended columns
-          !this.editor.columns.groupExtendColumns.find(
-            (col) => col.name === column.name,
+          !this.editor.groupExtendColumns.find(
+            (column) => column.name === col.name,
           ),
       )
       .map(
@@ -89,37 +90,37 @@ export class DataCubeEditorHorizontalPivotsPanelState
 
   get pivotResultColumns(): DataCubeQuerySnapshotColumn[] {
     return this.castColumns
-      .filter((col) => col.name.includes(PIVOT_COLUMN_NAME_VALUE_SEPARATOR))
-      .map((col) => ({ name: col.name, type: col.type }));
+      .filter((col) => isPivotResultColumnName(col.name))
+      .map((col) => _toCol(col));
   }
 
+  /**
+   * Due to the nature of pivot() operation, some base columns (i.e. source columns and leaf-level columns)
+   * will be "consumed" and not available for subsequent operations (e.g. sort, groupBy, etc.).
+   */
   get columnsConsumedByPivot(): DataCubeQuerySnapshotColumn[] {
     if (!this.selector.selectedColumns.length) {
       return [];
     }
-    return [
-      ...this.selector.selectedColumns,
-      ...this.editor.columnProperties.columns.filter(
-        (col) =>
-          col.kind === DataCubeColumnKind.MEASURE &&
-          !col.excludedFromHorizontalPivot,
-      ),
-      /** TODO: @datacube pivot - need to include columns used in complex aggregates (such as weighted-average) */
-    ].map((col) => ({ name: col.name, type: col.type }));
+    return uniqBy(
+      [
+        ...this.selector.selectedColumns,
+        ...this.editor.columnProperties.columns.filter(
+          (col) =>
+            col.isSelected &&
+            col.kind === DataCubeColumnKind.MEASURE &&
+            !col.excludedFromHorizontalPivot,
+        ),
+        /** TODO: @datacube pivot - need to include columns used in complex aggregates (such as weighted-average) */
+      ],
+      (col) => col.name,
+    ).map((col) => _toCol(col));
   }
 
   setCastColumns(value: DataCubeQuerySnapshotColumn[]) {
     this.castColumns = value;
   }
 
-  adaptPropagatedChanges() {
-    this.selector.setSelectedColumns(
-      this.selector.selectedColumns.filter((column) =>
-        this.selector.availableColumns.find((col) => col.name === column.name),
-      ),
-    );
-  }
-
   propagateChanges() {
     this.editor.verticalPivots.adaptPropagatedChanges();
     this.editor.sorts.adaptPropagatedChanges();
@@ -144,15 +145,13 @@ export class DataCubeEditorHorizontalPivotsPanelState
   ) {
     newSnapshot.data.pivot = this.selector.selectedColumns.length
       ? {
-          columns: this.selector.selectedColumns.map((column) => ({
-            name: column.name,
-            type: column.type,
-          })),
-          castColumns: this.castColumns.map((column) => ({
-            name: column.name,
-            type: column.type,
-          })),
+          columns: this.selector.selectedColumns.map((col) => _toCol(col)),
+          castColumns: this.castColumns.map((col) => _toCol(col)),
         }
       : undefined;
+    newSnapshot.data.selectColumns = uniqBy(
+      [...newSnapshot.data.selectColumns, ...this.selector.selectedColumns],
+      (col) => col.name,
+    ).map((col) => _toCol(col));
   }
 }

package/src/stores/dataCube/editor/DataCubeEditorMutableConfiguration.ts

@@ -143,12 +143,15 @@ export class DataCubeEditorMutableColumnConfiguration extends DataCubeColumnConf
       errorBackgroundColor: observable,
       setErrorBackgroundColor: action,
 
-      blur: observable,
-      setBlur: action,
+      isSelected: observable,
+      setIsSelected: action,
 
       hideFromView: observable,
       setHideFromView: action,
 
+      blur: observable,
+      setBlur: action,
+
       fixedWidth: observable,
       setFixedWidth: action,
 
@@ -334,14 +337,18 @@ export class DataCubeEditorMutableColumnConfiguration extends DataCubeColumnConf
     this.errorBackgroundColor = value;
   }
 
-  setBlur(value: boolean) {
-    this.blur = value;
+  setIsSelected(value: boolean) {
+    this.isSelected = value;
   }
 
   setHideFromView(value: boolean) {
     this.hideFromView = value;
   }
 
+  setBlur(value: boolean) {
+    this.blur = value;
+  }
+
   setFixedWidth(value: number | undefined) {
     this.fixedWidth = value;
   }

package/src/stores/dataCube/editor/DataCubeEditorSortsPanelState.ts

@@ -25,6 +25,7 @@ import {
 } from './DataCubeEditorColumnsSelectorState.js';
 import type { DataCubeEditorState } from './DataCubeEditorState.js';
 import type { DataCubeConfiguration } from '../core/DataCubeConfiguration.js';
+import { uniqBy } from '@finos/legend-shared';
 
 export class DataCubeEditorSortColumnState extends DataCubeEditorColumnsSelectorColumnState {
   operation: string;
@@ -55,19 +56,23 @@ export class DataCubeEditorSortColumnsSelectorState extends DataCubeEditorColumn
   }
 
   override get availableColumns() {
-    return [
-      ...this.editor.horizontalPivots.pivotResultColumns,
-      ...this.editor.columns.selector.selectedColumns.filter(
-        (column) =>
-          !this.editor.columns.groupExtendColumns.find(
-            (col) => col.name === column.name,
-          ) &&
-          !this.editor.horizontalPivots.columnsConsumedByPivot.find(
-            (col) => col.name === column.name,
-          ),
-      ),
-      ...this.editor.columns.groupExtendColumns,
-    ].map(
+    return uniqBy(
+      [
+        ...this.editor.horizontalPivots.pivotResultColumns,
+        ...[
+          ...this.editor.columns.selector.selectedColumns,
+          ...this.editor.horizontalPivots.selector.selectedColumns,
+          ...this.editor.verticalPivots.selector.selectedColumns,
+        ].filter(
+          (column) =>
+            !this.editor.horizontalPivots.columnsConsumedByPivot.find(
+              (col) => col.name === column.name,
+            ),
+        ),
+        ...this.editor.groupExtendColumns,
+      ],
+      (col) => col.name,
+    ).map(
       (col) =>
         new DataCubeEditorSortColumnState(
           col.name,

package/src/stores/dataCube/editor/DataCubeEditorState.tsx

@@ -18,7 +18,12 @@ import { action, makeObservable, observable } from 'mobx';
 import type { DataCubeState } from '../DataCubeState.js';
 import { DataCubeEditorSortsPanelState } from './DataCubeEditorSortsPanelState.js';
 import { DataCubeQuerySnapshotController } from '../core/DataCubeQuerySnapshotManager.js';
-import { type DataCubeQuerySnapshot } from '../core/DataCubeQuerySnapshot.js';
+import {
+  _toCol,
+  type DataCubeQuerySnapshot,
+  type DataCubeQuerySnapshotColumn,
+  type DataCubeQuerySnapshotExtendedColumn,
+} from '../core/DataCubeQuerySnapshot.js';
 import {
   ActionState,
   assertErrorThrown,
@@ -67,6 +72,10 @@ export class DataCubeEditorState extends DataCubeQuerySnapshotController {
 
   currentTab = DataCubeEditorTab.GENERAL_PROPERTIES;
 
+  sourceColumns: DataCubeQuerySnapshotColumn[] = [];
+  leafExtendColumns: DataCubeQuerySnapshotExtendedColumn[] = [];
+  groupExtendColumns: DataCubeQuerySnapshotExtendedColumn[] = [];
+
   constructor(dataCube: DataCubeState) {
     super(dataCube);
 
@@ -74,6 +83,11 @@ export class DataCubeEditorState extends DataCubeQuerySnapshotController {
       currentTab: observable,
       setCurrentTab: action,
 
+      sourceColumns: observable.ref,
+      leafExtendColumns: observable.ref,
+      groupExtendColumns: observable.ref,
+
+      applySnapshot: action,
       applyChanges: action,
     });
 
@@ -96,104 +110,99 @@ export class DataCubeEditorState extends DataCubeQuerySnapshotController {
     this.currentTab = val;
   }
 
+  override async applySnapshot(
+    snapshot: DataCubeQuerySnapshot,
+    previousSnapshot: DataCubeQuerySnapshot | undefined,
+  ) {
+    this.sourceColumns = snapshot.data.sourceColumns;
+    this.leafExtendColumns = snapshot.data.leafExtendedColumns;
+    this.groupExtendColumns = snapshot.data.groupExtendedColumns;
+
+    const configuration = DataCubeConfiguration.serialization.fromJson(
+      snapshot.data.configuration,
+    );
+    this.generalProperties.applySnaphot(snapshot, configuration);
+    this.columnProperties.applySnaphot(snapshot, configuration);
+
+    this.columns.applySnaphot(snapshot, configuration);
+    this.horizontalPivots.applySnaphot(snapshot, configuration);
+    this.verticalPivots.applySnaphot(snapshot, configuration);
+    this.sorts.applySnaphot(snapshot, configuration);
+  }
+
   async applyChanges(options?: { closeAfterApply?: boolean | undefined }) {
+    const task = this.dataCube.newTask('Validate query');
     this.finalizationState.inProgress();
 
     const baseSnapshot = guaranteeNonNullable(this.getLatestSnapshot());
     const newSnapshot = baseSnapshot.clone();
 
-    // NOTE: column selection must be processed first so necessary
-    // prunings can be done to make sure other panel stats are in sync
-    // with the current column selection
-    this.columns.buildSnapshot(newSnapshot, baseSnapshot);
-    this.horizontalPivots.buildSnapshot(newSnapshot, baseSnapshot);
-    this.verticalPivots.buildSnapshot(newSnapshot, baseSnapshot);
-    this.sorts.buildSnapshot(newSnapshot, baseSnapshot);
-
     // grid configuration must be processed before processing columns' configuration
     // to properly generate the container configuration
     this.generalProperties.buildSnapshot(newSnapshot, baseSnapshot);
     this.columnProperties.buildSnapshot(newSnapshot, baseSnapshot);
 
-    newSnapshot.finalize();
-    if (newSnapshot.hashCode === baseSnapshot.hashCode) {
-      if (options?.closeAfterApply) {
-        this.display.close();
-      }
-      this.finalizationState.complete();
-      return;
-    }
-
-    // NOTE: Compile the query to validate. This is a helpful check for a lot of different scenarios
-    // where the consistency of the query might be thrown off by changes from various parts that the
-    // editor does not have full control over (i.e. extended columns, pivot cast columns, etc.)
-    // e.g. when a column that group-level extended columns' expressions depend on has been unselected.
-    try {
-      const tempSnapshot = newSnapshot.clone();
-      // NOTE: in order to obtain the actual pivot result columns information, we need to execute
-      // the query which is expensive in certain cases, so here, we just compute the "optimistic" set
-      // of pivot result columns for casting to guarantee validation is not thronn off.
-      if (tempSnapshot.data.pivot) {
-        tempSnapshot.data.pivot.castColumns =
-          this.sorts.selector.availableColumns
-            .filter(
-              (col) =>
-                !tempSnapshot.data.groupExtendedColumns.find(
-                  (column) => column.name === col.name,
-                ),
-            )
-            .map((col) => ({
-              name: col.name,
-              type: col.type,
-            }));
-      }
-      await this.dataCube.engine.getQueryRelationType(
-        _lambda(
-          [],
-          [
-            buildExecutableQuery(
-              tempSnapshot,
-              this.dataCube.engine.filterOperations,
-              this.dataCube.engine.aggregateOperations,
-            ),
-          ],
-        ),
-      );
-    } catch (error) {
-      assertErrorThrown(error);
-      this.dataCube.repl.alertError(error, {
-        message: `Query Validation Failure: ${error.message}`,
-      });
-      return;
-    } finally {
-      this.finalizationState.complete();
-    }
+    // NOTE: column selection must be processed first since the snapshot
+    // processing of other parts of the query can be affected by column selection.
+    this.columns.buildSnapshot(newSnapshot, baseSnapshot);
+    this.horizontalPivots.buildSnapshot(newSnapshot, baseSnapshot);
+    this.verticalPivots.buildSnapshot(newSnapshot, baseSnapshot);
+    this.sorts.buildSnapshot(newSnapshot, baseSnapshot);
 
     // finalize
     newSnapshot.finalize();
     if (newSnapshot.hashCode !== baseSnapshot.hashCode) {
+      // NOTE: Compile the query to validate. This is a helpful check for a lot of different scenarios
+      // where the consistency of the query might be thrown off by changes from various parts that the
+      // editor does not have full control over (i.e. extended columns, pivot cast columns, etc.)
+      // e.g. when a column that group-level extended columns' expressions depend on has been unselected.
+      try {
+        const tempSnapshot = newSnapshot.clone();
+        // NOTE: in order to obtain the actual pivot result columns information, we need to execute
+        // the query which is expensive in certain cases, so here, we just compute the "optimistic" set
+        // of pivot result columns for casting to guarantee validation is not thronn off.
+        if (tempSnapshot.data.pivot) {
+          tempSnapshot.data.pivot.castColumns =
+            this.sorts.selector.availableColumns
+              .filter(
+                (col) =>
+                  !tempSnapshot.data.groupExtendedColumns.find(
+                    (column) => column.name === col.name,
+                  ),
+              )
+              .map((col) => _toCol(col));
+        }
+        await this.dataCube.engine.getQueryRelationType(
+          _lambda(
+            [],
+            [
+              buildExecutableQuery(
+                tempSnapshot,
+                this.dataCube.engine.filterOperations,
+                this.dataCube.engine.aggregateOperations,
+              ),
+            ],
+          ),
+        );
+      } catch (error) {
+        assertErrorThrown(error);
+        this.dataCube.repl.alertError(error, {
+          message: `Query Validation Failure: ${error.message}`,
+        });
+        this.dataCube.endTask(task);
+        return;
+      } finally {
+        this.finalizationState.complete();
+      }
+
       this.publishSnapshot(newSnapshot);
+    } else {
+      this.finalizationState.complete();
     }
 
     if (options?.closeAfterApply) {
       this.display.close();
     }
-  }
-
-  override async applySnapshot(
-    snapshot: DataCubeQuerySnapshot,
-    previousSnapshot: DataCubeQuerySnapshot | undefined,
-  ) {
-    const configuration = DataCubeConfiguration.serialization.fromJson(
-      snapshot.data.configuration,
-    );
-
-    this.columns.applySnaphot(snapshot, configuration);
-    this.horizontalPivots.applySnaphot(snapshot, configuration);
-    this.verticalPivots.applySnaphot(snapshot, configuration);
-    this.sorts.applySnaphot(snapshot, configuration);
-
-    this.generalProperties.applySnaphot(snapshot, configuration);
-    this.columnProperties.applySnaphot(snapshot, configuration);
+    this.dataCube.endTask(task);
   }
 }

package/src/stores/dataCube/editor/DataCubeEditorVerticalPivotsPanelState.ts

@@ -14,10 +14,14 @@
  * limitations under the License.
  */
 
+import { uniqBy } from '@finos/legend-shared';
 import type { DataCubeState } from '../DataCubeState.js';
 import type { DataCubeConfiguration } from '../core/DataCubeConfiguration.js';
 import { DataCubeColumnKind } from '../core/DataCubeQueryEngine.js';
-import { type DataCubeQuerySnapshot } from '../core/DataCubeQuerySnapshot.js';
+import {
+  _toCol,
+  type DataCubeQuerySnapshot,
+} from '../core/DataCubeQuerySnapshot.js';
 import {
   DataCubeEditorColumnsSelectorColumnState,
   DataCubeEditorColumnsSelectorState,
@@ -35,18 +39,17 @@ export class DataCubeEditorVerticalPivotColumnsSelectorState extends DataCubeEdi
     );
   }
 
-  override get availableColumns(): DataCubeEditorColumnsSelectorColumnState[] {
-    return this.editor.columns.selector.selectedColumns
+  override get availableColumns() {
+    return this.editor.columnProperties.columns
       .filter(
         (column) =>
-          this.editor.columnProperties.getColumnConfiguration(column.name)
-            ?.kind === DataCubeColumnKind.DIMENSION &&
+          column.kind === DataCubeColumnKind.DIMENSION &&
           // exclude group-level extended columns
-          !this.editor.columns.groupExtendColumns.find(
+          !this.editor.groupExtendColumns.find(
             (col) => col.name === column.name,
           ) &&
-          // prune available columns if h-pivot is present
-          !this.editor.horizontalPivots.columnsConsumedByPivot.find(
+          // exclude pivot columns
+          !this.editor.horizontalPivots.selector.selectedColumns.find(
             (col) => col.name === column.name,
           ),
       )
@@ -96,11 +99,12 @@ export class DataCubeEditorVerticalPivotsPanelState
   ) {
     newSnapshot.data.groupBy = this.selector.selectedColumns.length
       ? {
-          columns: this.selector.selectedColumns.map((column) => ({
-            name: column.name,
-            type: column.type,
-          })),
+          columns: this.selector.selectedColumns.map((col) => _toCol(col)),
         }
       : undefined;
+    newSnapshot.data.selectColumns = uniqBy(
+      [...newSnapshot.data.selectColumns, ...this.selector.selectedColumns],
+      (col) => col.name,
+    ).map((col) => _toCol(col));
   }
 }

package/src/stores/dataCube/extend/DataCubeExtendManagerState.tsx

@@ -15,10 +15,11 @@
  */
 
 import { action, computed, makeObservable, observable } from 'mobx';
-import type {
-  DataCubeQuerySnapshot,
-  DataCubeQuerySnapshotColumn,
-  DataCubeQuerySnapshotExtendedColumn,
+import {
+  _toCol,
+  type DataCubeQuerySnapshot,
+  type DataCubeQuerySnapshotColumn,
+  type DataCubeQuerySnapshotExtendedColumn,
 } from '../core/DataCubeQuerySnapshot.js';
 import { deleteEntry, guaranteeNonNullable, noop } from '@finos/legend-shared';
 import type { DataCubeState } from '../DataCubeState.js';
@@ -53,11 +54,14 @@ export class DataCubeQueryExtendedColumnState {
  */
 export class DataCubeExtendManagerState extends DataCubeQuerySnapshotController {
   configuration = new DataCubeConfiguration();
-  columnConfigurations: DataCubeColumnConfiguration[] = [];
 
+  columnConfigurations: DataCubeColumnConfiguration[] = [];
+  selectedColumns: DataCubeQuerySnapshotColumn[] = [];
   sourceColumns: DataCubeQuerySnapshotColumn[] = [];
+
   leafExtendedColumns: DataCubeQueryExtendedColumnState[] = [];
   groupExtendedColumns: DataCubeQueryExtendedColumnState[] = [];
+
   newColumnEditors: DataCubeNewColumnState[] = [];
   // TODO: existingColumnEditors
 
@@ -65,8 +69,10 @@ export class DataCubeExtendManagerState extends DataCubeQuerySnapshotController
     super(dataCube);
 
     makeObservable(this, {
-      sourceColumns: observable.ref,
       columnConfigurations: observable.struct,
+      selectedColumns: observable.struct,
+      sourceColumns: observable.ref,
+
       leafExtendedColumns: observable,
       groupExtendedColumns: observable,
 
@@ -113,6 +119,7 @@ export class DataCubeExtendManagerState extends DataCubeQuerySnapshotController
 
     this.columnConfigurations.push(columnConfiguration);
     deleteEntry(this.newColumnEditors, editor);
+    this.selectedColumns.push(_toCol(column));
     this.applyChanges();
   }
 
@@ -154,19 +161,7 @@ export class DataCubeExtendManagerState extends DataCubeQuerySnapshotController
     newSnapshot.data.groupExtendedColumns = this.groupExtendedColumns.map(
       (col) => col.data,
     );
-    // update the selected columns
-    // NOTE: group-level extended columns cannot be selected!
-    newSnapshot.data.selectColumns = this.columnConfigurations
-      .map((col) => ({
-        name: col.name,
-        type: col.type,
-      }))
-      .filter(
-        (col) =>
-          !newSnapshot.data.groupExtendedColumns.find(
-            (c) => c.name === col.name,
-          ),
-      );
+    newSnapshot.data.selectColumns = [...this.selectedColumns];
 
     // TODO: support edition, so v-pivots/h-pivots or column configuration that
     // depends on columns with name/type changed should be updated

package/src/stores/dataCube/grid/DataCubeGridClientEngine.ts

@@ -181,6 +181,7 @@ export function computeHashCodeForDataFetchManualTrigger(
   return hashObject(
     pruneObject({
       ...snapshot.data,
+      name: undefined,
       configuration: {
         initialExpandLevel: configuration.initialExpandLevel,
         showRootAggregation: configuration.showRootAggregation,
@@ -397,6 +398,17 @@ export class DataCubeGridClientServerSideDataSource
             this.rowCount = (params.request.startRow ?? 0) + rowData.length;
           });
         }
+
+        // toggle no-rows overlay
+        if (
+          params.request.groupKeys.length === 0 &&
+          params.request.startRow === 0 &&
+          rowData.length === 0
+        ) {
+          this.grid.client.showNoRowsOverlay();
+        } else {
+          this.grid.client.hideOverlay();
+        }
       } else {
         // NOTE: When pagination is disabled and the user currently scrolls to somewhere in the grid, as data is fetched
         // and the operation does not force a scroll top (for example, grouping will always force scrolling to the
@@ -412,7 +424,8 @@ export class DataCubeGridClientServerSideDataSource
         // behavior by forcing a scroll top for every data fetch and also reset the cache block size to the default value to save memory
         if (rowData.length > INTERNAL__GRID_CLIENT_MAX_CACHE_BLOCK_SIZE) {
           if (
-            !this.grid.dataCube.repl.dataCubeEngine.disableLargeDatasetWarning
+            !this.grid.dataCube.repl.dataCubeEngine
+              .gridClientSuppressLargeDatasetWarning
           ) {
             this.grid.dataCube.repl.alert({
               message: `Large dataset (>${INTERNAL__GRID_CLIENT_MAX_CACHE_BLOCK_SIZE} rows) detected!`,
@@ -429,7 +442,7 @@ export class DataCubeGridClientServerSideDataSource
                   label: 'Dismiss Warning',
                   handler: () => {
                     // this.grid.setPaginationEnabled(true);
-                    this.grid.dataCube.repl.dataCubeEngine.setDisableLargeDatasetWarning(
+                    this.grid.dataCube.repl.dataCubeEngine.setGridClientSuppressLargeDatasetWarning(
                       true,
                     );
                   },
@@ -456,6 +469,13 @@ export class DataCubeGridClientServerSideDataSource
             this.rowCount = rowData.length;
           });
         }
+
+        // toggle no-rows overlay
+        if (params.request.groupKeys.length === 0 && rowData.length === 0) {
+          this.grid.client.showNoRowsOverlay();
+        } else {
+          this.grid.client.hideOverlay();
+        }
       }
     } catch (error) {
       assertErrorThrown(error);