@finos/legend-application-repl 0.0.56 → 0.0.58

package/src/stores/dataCube/editor/DataCubeEditorState.tsx

@@ -17,7 +17,6 @@
 import { action, makeObservable, observable } from 'mobx';
 import type { DataCubeState } from '../DataCubeState.js';
 import { DataCubeEditorSortsPanelState } from './DataCubeEditorSortsPanelState.js';
-import { DataCubeEditorCodePanelState } from './DataCubeEditorCodePanelState.js';
 import { DataCubeQuerySnapshotController } from '../core/DataCubeQuerySnapshotManager.js';
 import { type DataCubeQuerySnapshot } from '../core/DataCubeQuerySnapshot.js';
 import {
@@ -43,7 +42,6 @@ export enum DataCubeEditorTab {
   VERTICAL_PIVOTS = 'Vertical Pivots',
   HORIZONTAL_PIVOTS = 'Horizontal Pivots',
   SORTS = 'Sorts',
-  CODE = 'Code',
 }
 
 /**
@@ -59,14 +57,12 @@ export class DataCubeEditorState extends DataCubeQuerySnapshotController {
   readonly display: DisplayState;
   readonly finalizationState = ActionState.create();
 
-  readonly code: DataCubeEditorCodePanelState;
-
   readonly generalProperties: DataCubeEditorGeneralPropertiesPanelState;
   readonly columnProperties: DataCubeEditorColumnPropertiesPanelState;
 
   readonly columns: DataCubeEditorColumnsPanelState;
-  readonly verticalPivots: DataCubeEditorVerticalPivotsPanelState;
   readonly horizontalPivots: DataCubeEditorHorizontalPivotsPanelState;
+  readonly verticalPivots: DataCubeEditorVerticalPivotsPanelState;
   readonly sorts: DataCubeEditorSortsPanelState;
 
   currentTab = DataCubeEditorTab.GENERAL_PROPERTIES;
@@ -91,10 +87,9 @@ export class DataCubeEditorState extends DataCubeQuerySnapshotController {
     );
     this.columnProperties = new DataCubeEditorColumnPropertiesPanelState(this);
     this.columns = new DataCubeEditorColumnsPanelState(this);
-    this.verticalPivots = new DataCubeEditorVerticalPivotsPanelState(this);
     this.horizontalPivots = new DataCubeEditorHorizontalPivotsPanelState(this);
+    this.verticalPivots = new DataCubeEditorVerticalPivotsPanelState(this);
     this.sorts = new DataCubeEditorSortsPanelState(this);
-    this.code = new DataCubeEditorCodePanelState(this);
   }
 
   setCurrentTab(val: DataCubeEditorTab) {
@@ -105,32 +100,59 @@ export class DataCubeEditorState extends DataCubeQuerySnapshotController {
     this.finalizationState.inProgress();
 
     const baseSnapshot = guaranteeNonNullable(this.getLatestSnapshot());
-    const snapshot = baseSnapshot.clone();
+    const newSnapshot = baseSnapshot.clone();
 
     // NOTE: column selection must be processed first so necessary
     // prunings can be done to make sure other panel stats are in sync
     // with the current column selection
-    this.columns.buildSnapshot(snapshot, baseSnapshot);
-    this.verticalPivots.buildSnapshot(snapshot, baseSnapshot);
-    this.horizontalPivots.buildSnapshot(snapshot, baseSnapshot);
-    this.sorts.buildSnapshot(snapshot, baseSnapshot);
+    this.columns.buildSnapshot(newSnapshot, baseSnapshot);
+    this.horizontalPivots.buildSnapshot(newSnapshot, baseSnapshot);
+    this.verticalPivots.buildSnapshot(newSnapshot, baseSnapshot);
+    this.sorts.buildSnapshot(newSnapshot, baseSnapshot);
 
     // grid configuration must be processed before processing columns' configuration
     // to properly generate the container configuration
-    this.generalProperties.buildSnapshot(snapshot, baseSnapshot);
-    this.columnProperties.buildSnapshot(snapshot, baseSnapshot);
+    this.generalProperties.buildSnapshot(newSnapshot, baseSnapshot);
+    this.columnProperties.buildSnapshot(newSnapshot, baseSnapshot);
+
+    newSnapshot.finalize();
+    if (newSnapshot.hashCode === baseSnapshot.hashCode) {
+      if (options?.closeAfterApply) {
+        this.display.close();
+      }
+      this.finalizationState.complete();
+      return;
+    }
 
-    // compile the query to validate it
-    // NOTE: This is a helpful check for a lot of different scenarios where the
-    // consistency of the query might be thrown off by changes
+    // NOTE: Compile the query to validate. This is a helpful check for a lot of different scenarios
+    // where the consistency of the query might be thrown off by changes from various parts that the
+    // editor does not have full control over (i.e. extended columns, pivot cast columns, etc.)
     // e.g. when a column that group-level extended columns' expressions depend on has been unselected.
     try {
+      const tempSnapshot = newSnapshot.clone();
+      // NOTE: in order to obtain the actual pivot result columns information, we need to execute
+      // the query which is expensive in certain cases, so here, we just compute the "optimistic" set
+      // of pivot result columns for casting to guarantee validation is not thronn off.
+      if (tempSnapshot.data.pivot) {
+        tempSnapshot.data.pivot.castColumns =
+          this.sorts.selector.availableColumns
+            .filter(
+              (col) =>
+                !tempSnapshot.data.groupExtendedColumns.find(
+                  (column) => column.name === col.name,
+                ),
+            )
+            .map((col) => ({
+              name: col.name,
+              type: col.type,
+            }));
+      }
       await this.dataCube.engine.getQueryRelationType(
         _lambda(
           [],
           [
             buildExecutableQuery(
-              snapshot,
+              tempSnapshot,
               this.dataCube.engine.filterOperations,
               this.dataCube.engine.aggregateOperations,
             ),
@@ -147,9 +169,10 @@ export class DataCubeEditorState extends DataCubeQuerySnapshotController {
       this.finalizationState.complete();
     }
 
-    snapshot.finalize();
-    if (snapshot.hashCode !== baseSnapshot.hashCode) {
-      this.publishSnapshot(snapshot);
+    // finalize
+    newSnapshot.finalize();
+    if (newSnapshot.hashCode !== baseSnapshot.hashCode) {
+      this.publishSnapshot(newSnapshot);
     }
 
     if (options?.closeAfterApply) {
@@ -166,8 +189,8 @@ export class DataCubeEditorState extends DataCubeQuerySnapshotController {
     );
 
     this.columns.applySnaphot(snapshot, configuration);
-    this.verticalPivots.applySnaphot(snapshot, configuration);
     this.horizontalPivots.applySnaphot(snapshot, configuration);
+    this.verticalPivots.applySnaphot(snapshot, configuration);
     this.sorts.applySnaphot(snapshot, configuration);
 
     this.generalProperties.applySnaphot(snapshot, configuration);

package/src/stores/dataCube/editor/DataCubeEditorVerticalPivotsPanelState.ts

@@ -44,6 +44,10 @@ export class DataCubeEditorVerticalPivotColumnsSelectorState extends DataCubeEdi
           // exclude group-level extended columns
           !this.editor.columns.groupExtendColumns.find(
             (col) => col.name === column.name,
+          ) &&
+          // prune available columns if h-pivot is present
+          !this.editor.horizontalPivots.columnsConsumedByPivot.find(
+            (col) => col.name === column.name,
           ),
       )
       .map(
@@ -66,6 +70,14 @@ export class DataCubeEditorVerticalPivotsPanelState
     this.selector = new DataCubeEditorVerticalPivotColumnsSelectorState(editor);
   }
 
+  adaptPropagatedChanges(): void {
+    this.selector.setSelectedColumns(
+      this.selector.selectedColumns.filter((column) =>
+        this.selector.availableColumns.find((col) => col.name === column.name),
+      ),
+    );
+  }
+
   applySnaphot(
     snapshot: DataCubeQuerySnapshot,
     configuration: DataCubeConfiguration,

package/src/stores/dataCube/extend/DataCubeColumnEditorState.tsx

@@ -350,6 +350,7 @@ export class DataCubeNewColumnState {
       },
       this.isGroupLevel,
       this.columnKind,
+      this,
     );
 
     this.close();

package/src/stores/dataCube/extend/DataCubeExtendManagerState.tsx

@@ -20,7 +20,7 @@ import type {
   DataCubeQuerySnapshotColumn,
   DataCubeQuerySnapshotExtendedColumn,
 } from '../core/DataCubeQuerySnapshot.js';
-import { guaranteeNonNullable } from '@finos/legend-shared';
+import { deleteEntry, guaranteeNonNullable, noop } from '@finos/legend-shared';
 import type { DataCubeState } from '../DataCubeState.js';
 import { DataCubeQuerySnapshotController } from '../core/DataCubeQuerySnapshotManager.js';
 import {
@@ -98,6 +98,7 @@ export class DataCubeExtendManagerState extends DataCubeQuerySnapshotController
     column: T,
     isGroupLevel: boolean,
     columnKind: DataCubeColumnKind | undefined,
+    editor: DataCubeNewColumnState,
   ) {
     (isGroupLevel ? this.groupExtendedColumns : this.leafExtendedColumns).push(
       new DataCubeQueryExtendedColumnState(column),
@@ -111,6 +112,7 @@ export class DataCubeExtendManagerState extends DataCubeQuerySnapshotController
     }
 
     this.columnConfigurations.push(columnConfiguration);
+    deleteEntry(this.newColumnEditors, editor);
     this.applyChanges();
   }
 
@@ -129,6 +131,12 @@ export class DataCubeExtendManagerState extends DataCubeQuerySnapshotController
     this.groupExtendedColumns = snapshot.data.groupExtendedColumns.map(
       (col) => new DataCubeQueryExtendedColumnState(col),
     );
+
+    // trigger re-compile in each existing column editor as the base query has changed
+    this.newColumnEditors.forEach((editor) => {
+      editor.getReturnType().catch(noop());
+    });
+    // TODO: existingColumnEditors
   }
 
   applyChanges() {

package/src/stores/dataCube/grid/DataCubeGridClientEngine.ts

@@ -22,21 +22,37 @@ import type {
 } from '@ag-grid-community/core';
 import type { DataCubeGridState } from './DataCubeGridState.js';
 import {
+  IllegalStateError,
   LogEvent,
   assertErrorThrown,
   guaranteeNonNullable,
+  hashObject,
   isBoolean,
   isNonNullable,
+  pruneObject,
 } from '@finos/legend-shared';
 import { buildExecutableQuery } from '../core/DataCubeQueryBuilder.js';
-import { type TabularDataSet, V1_Lambda } from '@finos/legend-graph';
+import {
+  type TabularDataSet,
+  type V1_AppliedFunction,
+  V1_Lambda,
+} from '@finos/legend-graph';
 import { APPLICATION_EVENT } from '@finos/legend-application';
 import { buildQuerySnapshot } from './DataCubeGridQuerySnapshotBuilder.js';
 import { generateRowGroupingDrilldownExecutableQueryPostProcessor } from './DataCubeGridQueryBuilder.js';
 import { makeObservable, observable, runInAction } from 'mobx';
-import type { DataCubeConfigurationColorKey } from '../core/DataCubeConfiguration.js';
+import type {
+  DataCubeConfiguration,
+  DataCubeConfigurationColorKey,
+} from '../core/DataCubeConfiguration.js';
 import { AlertType } from '../../../components/repl/Alert.js';
 import { DEFAULT_LARGE_ALERT_WINDOW_CONFIG } from '../../LayoutManagerState.js';
+import type { DataCubeQuerySnapshot } from '../core/DataCubeQuerySnapshot.js';
+import type { DataCubeEngine } from '../DataCubeEngine.js';
+import { generateColumnDefs } from './DataCubeGridConfigurationBuilder.js';
+import type { DataCubeQueryFunctionMap } from '../core/DataCubeQueryEngine.js';
+import type { DataCubeQueryFilterOperation } from '../core/filter/DataCubeQueryFilterOperation.js';
+import type { DataCubeQueryAggregateOperation } from '../core/aggregation/DataCubeQueryAggregateOperation.js';
 
 type GridClientCellValue = string | number | boolean | null | undefined;
 type GridClientRowData = {
@@ -69,6 +85,9 @@ export enum INTERNAL__GridClientUtilityCssClassName {
   TEXT_ALIGN_PREFIX = 'data-cube-grid__utility--text-align-',
   TEXT_COLOR_PREFIX = 'data-cube-grid__utility--text-color-',
   BACKGROUND_COLOR_PREFIX = 'data-cube-grid__utility--background-color-',
+
+  PIVOT_COLUMN_GROUP = 'data-cube-grid__utility--pivot-column-group',
+  PIVOT_COLUMN_GROUP_PREFIX = 'data-cube-grid__utility--pivot-column-group-',
 }
 export const generateFontFamilyUtilityClassName = (fontFamily: string) =>
   `${INTERNAL__GridClientUtilityCssClassName.FONT_FAMILY_PREFIX}${fontFamily.replaceAll(' ', '-')}`;
@@ -103,6 +122,7 @@ export const INTERNAL__GRID_CLIENT_DEFAULT_CACHE_BLOCK_SIZE = 100;
 // so the value set must be reasonable, or else it can crash the application!
 export const INTERNAL__GRID_CLIENT_MAX_CACHE_BLOCK_SIZE = 1e5;
 
+export const INTERNAL__GRID_CLIENT_PIVOT_COLUMN_GROUP_COLOR_ROTATION_SIZE = 5;
 export const INTERNAL__GRID_CLIENT_SIDE_BAR_WIDTH = 200;
 export const INTERNAL__GRID_CLIENT_COLUMN_MIN_WIDTH = 50;
 export const INTERNAL__GRID_CLIENT_HEADER_HEIGHT = 24;
@@ -111,8 +131,8 @@ export const INTERNAL__GRID_CLIENT_TOOLTIP_SHOW_DELAY = 1000;
 export const INTERNAL__GRID_CLIENT_AUTO_RESIZE_PADDING = 10;
 export const INTERNAL__GRID_CLIENT_MISSING_VALUE = '__MISSING';
 export const INTERNAL__GRID_CLIENT_TREE_COLUMN_ID = 'INTERNAL__tree';
-export const INTERNAL__GRID_CLIENT_FILTER_TRIGGER_COLUMN_ID =
-  'INTERNAL__filterTrigger';
+export const INTERNAL__GRID_CLIENT_DATA_FETCH_MANUAL_TRIGGER_COLUMN_ID =
+  'INTERNAL__dataFetchManualTrigger';
 export const INTERNAL__GRID_CLIENT_ROW_GROUPING_COUNT_AGG_COLUMN_ID =
   'INTERNAL__count';
 
@@ -150,6 +170,36 @@ export function getDataForAllFilteredNodes<T>(client: GridApi<T>): T[] {
   return rows;
 }
 
+/**
+ * This method computes the hash code for the parts of the snapshot that should trigger data fetching.
+ * This is used to manually trigger server-side row model data source getRows() method.
+ */
+export function computeHashCodeForDataFetchManualTrigger(
+  snapshot: DataCubeQuerySnapshot,
+  configuration: DataCubeConfiguration,
+) {
+  return hashObject(
+    pruneObject({
+      ...snapshot.data,
+      configuration: {
+        initialExpandLevel: configuration.initialExpandLevel,
+        showRootAggregation: configuration.showRootAggregation,
+        showLeafCount: configuration.showLeafCount,
+        pivotTotalColumnPlacement: configuration.pivotTotalColumnPlacement,
+        treeGroupSortFunction: configuration.treeGroupSortFunction,
+        columns: configuration.columns.map((column) => ({
+          name: column.name,
+          type: column.type,
+          aggregateOperator: column.aggregateOperator,
+          aggregationParameters: column.aggregationParameters,
+          excludedFromHorizontalPivot: column.excludedFromHorizontalPivot,
+          horizontalPivotSortFunction: column.horizontalPivotSortFunction,
+        })),
+      },
+    }),
+  );
+}
+
 function TDStoRowData(tds: TabularDataSet): GridClientRowData[] {
   return tds.rows.map((_row, rowIdx) => {
     const row: GridClientRowData = {};
@@ -167,6 +217,58 @@ function TDStoRowData(tds: TabularDataSet): GridClientRowData[] {
   });
 }
 
+async function getCastColumns(
+  currentSnapshot: DataCubeQuerySnapshot,
+  engine: DataCubeEngine,
+) {
+  if (!currentSnapshot.data.pivot) {
+    throw new IllegalStateError(
+      `Can't build cast columns collector query when no pivot is specified`,
+    );
+  }
+  const snapshot = currentSnapshot.clone();
+  guaranteeNonNullable(snapshot.data.pivot).castColumns = [];
+  snapshot.data.groupBy = undefined;
+  snapshot.data.sortColumns = [];
+  snapshot.data.limit = 0;
+  const query = buildExecutableQuery(
+    snapshot,
+    engine.filterOperations,
+    engine.aggregateOperations,
+    {
+      postProcessor: (
+        _snapshot: DataCubeQuerySnapshot,
+        sequence: V1_AppliedFunction[],
+        funcMap: DataCubeQueryFunctionMap,
+        configuration: DataCubeConfiguration,
+        filterOperations: DataCubeQueryFilterOperation[],
+        aggregateOperations: DataCubeQueryAggregateOperation[],
+      ) => {
+        const _unprocess = (funcMapKey: keyof DataCubeQueryFunctionMap) => {
+          const func = funcMap[funcMapKey];
+          if (func) {
+            sequence.splice(sequence.indexOf(func), 1);
+            funcMap[funcMapKey] = undefined;
+          }
+        };
+
+        if (funcMap.groupExtend) {
+          _unprocess('groupExtend');
+        }
+      },
+    },
+  );
+
+  const lambda = new V1_Lambda();
+  lambda.body.push(query);
+  const result = await engine.executeQuery(lambda);
+
+  return result.result.builder.columns.map((column) => ({
+    name: column.name,
+    type: column.type as string,
+  }));
+}
+
 export class DataCubeGridClientServerSideDataSource
   implements IServerSideDatasource
 {
@@ -205,18 +307,56 @@ export class DataCubeGridClientServerSideDataSource
     // ------------------------------ SNAPSHOT ------------------------------
 
     const currentSnapshot = guaranteeNonNullable(this.grid.getLatestSnapshot());
-    // TODO: when we support pivoting, we should make a quick call to check for columns
-    // created by pivots and specify them as cast columns when pivot is activated
-    const syncedSnapshot = buildQuerySnapshot(params.request, currentSnapshot);
-    if (syncedSnapshot.hashCode !== currentSnapshot.hashCode) {
-      this.grid.publishSnapshot(syncedSnapshot);
+    const newSnapshot = buildQuerySnapshot(params.request, currentSnapshot);
+    // NOTE: if h-pivot is enabled
+    // if h-pivot is enabled, update the cast columns
+    // and panels which might be affected by this (e.g. sorts)
+    // TOOD?: this is an expensive operation in certain case, so we might want to
+    // optimize when this gets called
+    if (newSnapshot.data.pivot) {
+      try {
+        const castColumns = await getCastColumns(
+          newSnapshot,
+          this.grid.dataCube.engine,
+        );
+        newSnapshot.data.pivot.castColumns = castColumns;
+        newSnapshot.data.sortColumns = newSnapshot.data.sortColumns.filter(
+          (column) =>
+            [...castColumns, ...newSnapshot.data.groupExtendedColumns].find(
+              (col) => column.name === col.name,
+            ),
+        );
+      } catch (error) {
+        assertErrorThrown(error);
+        this.grid.dataCube.repl.alertError(error, {
+          message: `Query Validation Failure: Can't retrieve pivot results column metadata. ${error.message}`,
+        });
+        // fail early since we can't proceed without the cast columns validated
+        params.fail();
+        this.grid.dataCube.endTask(task);
+        return;
+      }
+    }
+
+    newSnapshot.finalize();
+    if (newSnapshot.hashCode !== currentSnapshot.hashCode) {
+      // update grid column definitions since new columns could have been added
+      // due to changes in pivot
+      this.grid.client.updateGridOptions({
+        columnDefs: generateColumnDefs(
+          newSnapshot,
+          this.grid.queryConfiguration,
+          this.grid.dataCube,
+        ),
+      });
+      this.grid.publishSnapshot(newSnapshot);
     }
 
     // ------------------------------ DATA ------------------------------
 
     try {
       const executableQuery = buildExecutableQuery(
-        syncedSnapshot,
+        newSnapshot,
         this.grid.dataCube.engine.filterOperations,
         this.grid.dataCube.engine.aggregateOperations,
         {