@finos/legend-data-cube 0.1.1 → 0.1.3

package/src/stores/view/grid/DataCubeGridClientEngine.ts

@@ -31,11 +31,7 @@ import {
   pruneObject,
 } from '@finos/legend-shared';
 import { buildExecutableQuery } from '../../core/DataCubeQueryBuilder.js';
-import {
-  PureClientVersion,
-  type TabularDataSet,
-  V1_Lambda,
-} from '@finos/legend-graph';
+import { PureClientVersion, type TabularDataSet } from '@finos/legend-graph';
 import { makeObservable, observable, runInAction } from 'mobx';
 import type {
   DataCubeConfiguration,
@@ -46,7 +42,6 @@ import { _findCol, _sortByColName } from '../../core/model/DataCubeColumn.js';
 import { isPivotResultColumnName } from '../../core/DataCubeQueryEngine.js';
 import { buildQuerySnapshot } from './DataCubeGridQuerySnapshotBuilder.js';
 import { AlertType } from '../../services/DataCubeAlertService.js';
-import { sum } from 'mathjs';
 import type { DataCubeViewState } from '../DataCubeViewState.js';
 import {
   _aggCountCol,
@@ -55,6 +50,8 @@ import {
 import { _colSpecArrayParam } from '../../core/DataCubeQuerySnapshotBuilderUtils.js';
 import { DataCubeSettingKey } from '../../../__lib__/DataCubeSetting.js';
 import { DEFAULT_ALERT_WINDOW_CONFIG } from '../../services/DataCubeLayoutService.js';
+import type { DataCubeExecutionResult } from '../../core/DataCubeEngine.js';
+import { _lambda } from '../../core/DataCubeQueryBuilderUtils.js';
 
 type DataCubeGridClientCellValue = string | number | boolean | null | undefined;
 type DataCubeGridClientRowData = {
@@ -231,19 +228,17 @@ function buildRowData(
           ? value
           : INTERNAL__GRID_CLIENT_MISSING_VALUE;
       if (snapshot.data.pivot && snapshot.data.groupBy) {
-        row[INTERNAL__GRID_CLIENT_ROW_GROUPING_COUNT_AGG_COLUMN_ID] = Number(
-          sum(
-            result.columns
-              .filter(
-                (col) =>
-                  isPivotResultColumnName(col) &&
-                  col.endsWith(
-                    INTERNAL__GRID_CLIENT_ROW_GROUPING_COUNT_AGG_COLUMN_ID,
-                  ),
-              )
-              .map((col) => (row[col] as number | undefined) ?? 0),
-          ).toString(),
-        );
+        row[INTERNAL__GRID_CLIENT_ROW_GROUPING_COUNT_AGG_COLUMN_ID] =
+          result.columns
+            .filter(
+              (col) =>
+                isPivotResultColumnName(col) &&
+                col.endsWith(
+                  INTERNAL__GRID_CLIENT_ROW_GROUPING_COUNT_AGG_COLUMN_ID,
+                ),
+            )
+            .map((col) => (row[col] as number | undefined) ?? 0)
+            .reduce((a, b) => a + b, 0);
       }
     });
     return row;
@@ -275,18 +270,20 @@ async function getCastColumns(
     },
   });
 
-  const lambda = new V1_Lambda();
-  lambda.body.push(query);
-  const result = await view.engine.executeQuery(lambda, view.source, {
-    debug: view.settingService.getBooleanValue(
-      DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
-    ),
-    clientVersion: view.settingService.getBooleanValue(
-      DataCubeSettingKey.DEBUGGER__USE_DEV_CLIENT_PROTOCOL_VERSION,
-    )
-      ? PureClientVersion.VX_X_X
-      : undefined,
-  });
+  const result = await view.engine.executeQuery(
+    _lambda([], [query]),
+    view.source,
+    {
+      debug: view.settingService.getBooleanValue(
+        DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
+      ),
+      clientVersion: view.settingService.getBooleanValue(
+        DataCubeSettingKey.DEBUGGER__USE_DEV_CLIENT_PROTOCOL_VERSION,
+      )
+        ? PureClientVersion.VX_X_X
+        : undefined,
+    },
+  );
 
   return result.result.builder.columns.map((column) => ({
     name: column.name,
@@ -324,7 +321,7 @@ export class DataCubeGridClientServerSideDataSource
   }
 
   async fetchRows(params: IServerSideGetRowsParams<unknown, unknown>) {
-    const task = this._view.taskService.newTask('Fetching data');
+    const task = this._view.taskService.newTask('Fetching data...');
 
     // ------------------------------ SNAPSHOT ------------------------------
 
@@ -394,29 +391,18 @@ export class DataCubeGridClientServerSideDataSource
 
     // ------------------------------ DATA ------------------------------
 
+    const executableQuery = buildGridDataFetchExecutableQuery(
+      request,
+      newSnapshot,
+      this._view.source,
+      this._view.engine,
+      this._grid.isPaginationEnabled,
+    );
+    let result: DataCubeExecutionResult;
+
     try {
-      const executableQuery = buildGridDataFetchExecutableQuery(
-        request,
-        newSnapshot,
-        this._view.source,
-        this._view.engine,
-        this._grid.isPaginationEnabled,
-      );
-      const lambda = new V1_Lambda();
-      lambda.body.push(executableQuery);
-      if (
-        this._view.settingService.getBooleanValue(
-          DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
-        )
-      ) {
-        this._view.engine.debugProcess(
-          `Execution Plan`,
-          ['Query', lambda],
-          ['Config', `pagination=${this._grid.isPaginationEnabled}`],
-        );
-      }
-      const result = await this._view.engine.executeQuery(
-        lambda,
+      result = await this._view.engine.executeQuery(
+        _lambda([], [executableQuery]),
         this._view.source,
         {
           debug: this._view.settingService.getBooleanValue(
@@ -429,136 +415,134 @@ export class DataCubeGridClientServerSideDataSource
             : undefined,
         },
       );
-      const rowData = buildRowData(result.result.result, newSnapshot);
-      if (
-        this._view.settingService.getBooleanValue(
-          DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
-        )
-      ) {
-        this._view.engine.debugProcess(
-          `Execution`,
-          ['Query', result.executedQuery],
-          [
-            'Stats',
-            `${rowData.length} rows, ${result.result.result.columns.length} columns`,
-          ],
-          ['SQL', result.executedSQL],
-          ['SQL Execution Time', result.executionTime],
-        );
-      }
+    } catch (error) {
+      assertErrorThrown(error);
+      this._view.alertService.alertError(error, {
+        message: `Data Fetch Failure: ${error.message}`,
+      });
+      this._view.taskService.endTask(task);
+      params.fail();
+      return;
+    }
 
-      if (this._grid.isPaginationEnabled) {
-        params.success({ rowData });
-        // only update row count when loading the top-level drilldown data
-        if (isTopLevelRequest) {
-          runInAction(() => {
-            this.rowCount = (request.startRow ?? 0) + rowData.length;
-          });
-        }
+    const rowData = buildRowData(result.result.result, newSnapshot);
+    if (
+      this._view.settingService.getBooleanValue(
+        DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
+      )
+    ) {
+      this._view.engine.debugProcess(
+        `Execution`,
+        ['Query', result.executedQuery],
+        [
+          'Stats',
+          `${rowData.length} rows, ${result.result.result.columns.length} columns`,
+        ],
+        ['SQL', result.executedSQL],
+        ['SQL Execution Time', result.executionTime],
+      );
+    }
 
-        // toggle no-rows overlay
+    if (this._grid.isPaginationEnabled) {
+      params.success({ rowData });
+      // only update row count when loading the top-level drilldown data
+      if (isTopLevelRequest) {
+        runInAction(() => {
+          this.rowCount = (request.startRow ?? 0) + rowData.length;
+        });
+      }
+
+      // toggle no-rows overlay
+      if (isTopLevelRequest && request.startRow === 0 && rowData.length === 0) {
+        this._grid.client.showNoRowsOverlay();
+      } else {
+        this._grid.client.hideOverlay();
+      }
+    } else {
+      // NOTE: When pagination is disabled and the user currently scrolls to somewhere in the grid, as data is fetched
+      // and the operation does not force a scroll top (for example, grouping will always force scrolling to the
+      // top, while sorting will leave scroll position as is), the grid ends up showing the wrong data because
+      // the data being displayed does not take into account the scroll position, but by the start and end row
+      // which stay constant as pagination is disabled.
+      //
+      // In order to handle this, when pagination is disabled, we tune the start and end row by setting the cache block size
+      // to a high-enough value (100k-1m). However, ag-grid use cache block size to pre-allocate memory for the rows,
+      // which means we must cap/tune this value reasonably to prevent the app from crashing.
+      //
+      // When there are just too many rows (exceeding the maximum cache block size), we will fallback to a slightly less ideal
+      // behavior by forcing a scroll top for every data fetch and also reset the cache block size to the default value to save memory
+      if (rowData.length > INTERNAL__GRID_CLIENT_MAX_CACHE_BLOCK_SIZE) {
         if (
-          isTopLevelRequest &&
-          request.startRow === 0 &&
-          rowData.length === 0
+          !this._view.settingService.getBooleanValue(
+            DataCubeSettingKey.GRID_CLIENT__SUPPRESS_LARGE_DATASET_WARNING,
+          )
         ) {
-          this._grid.client.showNoRowsOverlay();
-        } else {
-          this._grid.client.hideOverlay();
-        }
-      } else {
-        // NOTE: When pagination is disabled and the user currently scrolls to somewhere in the grid, as data is fetched
-        // and the operation does not force a scroll top (for example, grouping will always force scrolling to the
-        // top, while sorting will leave scroll position as is), the grid ends up showing the wrong data because
-        // the data being displayed does not take into account the scroll position, but by the start and end row
-        // which stay constant as pagination is disabled.
-        //
-        // In order to handle this, when pagination is disabled, we tune the start and end row by setting the cache block size
-        // to a high-enough value (100k-1m). However, ag-grid use cache block size to pre-allocate memory for the rows,
-        // which means we must cap/tune this value reasonably to prevent the app from crashing.
-        //
-        // When there are just too many rows (exceeding the maximum cache block size), we will fallback to a slightly less ideal
-        // behavior by forcing a scroll top for every data fetch and also reset the cache block size to the default value to save memory
-        if (rowData.length > INTERNAL__GRID_CLIENT_MAX_CACHE_BLOCK_SIZE) {
-          if (
-            !this._view.settingService.getBooleanValue(
-              DataCubeSettingKey.GRID_CLIENT__SUPPRESS_LARGE_DATASET_WARNING,
-            )
-          ) {
-            this._view.alertService.alert({
-              message: `Large dataset (>${INTERNAL__GRID_CLIENT_MAX_CACHE_BLOCK_SIZE} rows) detected!`,
-              text: `Overall app performance can be impacted by large dataset due to longer query execution time and increased memory usage. At its limit, the engine can crash!\nTo boost performance, consider enabling pagination while working with large dataset.`,
-              type: AlertType.WARNING,
-              actions: [
-                {
-                  label: 'Enable Pagination',
-                  handler: () => {
-                    this._grid.setPaginationEnabled(true);
-                  },
+          this._view.alertService.alert({
+            message: `Large dataset (>${INTERNAL__GRID_CLIENT_MAX_CACHE_BLOCK_SIZE} rows) detected!`,
+            text: `Overall app performance can be impacted by large dataset due to longer query execution time and increased memory usage. At its limit, the engine can crash!\nTo boost performance, consider enabling pagination while working with large dataset.`,
+            type: AlertType.WARNING,
+            actions: [
+              {
+                label: 'Enable Pagination',
+                handler: () => {
+                  this._grid.setPaginationEnabled(true);
                 },
-                {
-                  label: 'Dismiss Warning',
-                  handler: () => {
-                    this._view.settingService.updateValue(
-                      this._view.dataCube.api,
-                      DataCubeSettingKey.GRID_CLIENT__SUPPRESS_LARGE_DATASET_WARNING,
-                      true,
-                    );
-                  },
+              },
+              {
+                label: 'Dismiss Warning',
+                handler: () => {
+                  this._view.settingService.updateValue(
+                    this._view.dataCube.api,
+                    DataCubeSettingKey.GRID_CLIENT__SUPPRESS_LARGE_DATASET_WARNING,
+                    true,
+                  );
                 },
-              ],
-              windowConfig: {
-                ...DEFAULT_ALERT_WINDOW_CONFIG,
-                width: 600,
-                minWidth: 300,
-                minHeight: 150,
               },
-            });
-          }
-
-          // NOTE: when drilldown occurs, we will scroll top until the drilldown row is reached
-          params.api.ensureIndexVisible(params.parentNode.rowIndex ?? 0, 'top');
-        }
-
-        params.success({
-          rowData,
-          // Setting row count to disable infinite-scrolling when pagination is disabled
-          // See https://www.ag-grid.com/react-data-grid/infinite-scrolling/#setting-last-row-index
-          rowCount: rowData.length,
-        });
-
-        // only update row count when loading the top-level drilldown data
-        if (isTopLevelRequest) {
-          runInAction(() => {
-            this.rowCount = rowData.length;
+            ],
+            windowConfig: {
+              ...DEFAULT_ALERT_WINDOW_CONFIG,
+              width: 600,
+              minWidth: 300,
+              minHeight: 150,
+            },
           });
         }
 
-        // toggle no-rows overlay
-        if (isTopLevelRequest && rowData.length === 0) {
-          this._grid.client.showNoRowsOverlay();
-        } else {
-          this._grid.client.hideOverlay();
-        }
+        // NOTE: when drilldown occurs, we will scroll top until the drilldown row is reached
+        params.api.ensureIndexVisible(params.parentNode.rowIndex ?? 0, 'top');
       }
 
-      // Resize columns to fit content on all actions (drilldown, changing queries, fetching new page, etc.)
-      // NOTE: we might want to restrict this to only happen on certain actions
-      //
-      // `setTimeout()` is need to ensure this runs after the grid has been updated with the new data
-      // since ag-grid does not provide an event hook for this action for server-side row model.
-      setTimeout(() => {
-        this._grid.client.autoSizeAllColumns();
-      }, 0);
-    } catch (error) {
-      assertErrorThrown(error);
-      this._view.alertService.alertError(error, {
-        message: `Data Fetch Failure: ${error.message}`,
+      params.success({
+        rowData,
+        // Setting row count to disable infinite-scrolling when pagination is disabled
+        // See https://www.ag-grid.com/react-data-grid/infinite-scrolling/#setting-last-row-index
+        rowCount: rowData.length,
       });
-      params.fail();
-    } finally {
-      this._view.taskService.endTask(task);
+
+      // only update row count when loading the top-level drilldown data
+      if (isTopLevelRequest) {
+        runInAction(() => {
+          this.rowCount = rowData.length;
+        });
+      }
+
+      // toggle no-rows overlay
+      if (isTopLevelRequest && rowData.length === 0) {
+        this._grid.client.showNoRowsOverlay();
+      } else {
+        this._grid.client.hideOverlay();
+      }
     }
+
+    // Resize columns to fit content on all actions (drilldown, changing queries, fetching new page, etc.)
+    // NOTE: we might want to restrict this to only happen on certain actions
+    //
+    // `setTimeout()` is need to ensure this runs after the grid has been updated with the new data
+    // since ag-grid does not provide an event hook for this action for server-side row model.
+    setTimeout(() => {
+      this._grid.client.autoSizeAllColumns();
+    }, 0);
+    this._view.taskService.endTask(task);
   }
 
   getRows(params: IServerSideGetRowsParams<unknown, unknown>) {

package/src/stores/view/grid/DataCubeGridConfigurationBuilder.tsx

@@ -62,7 +62,6 @@ import {
 } from './DataCubeGridClientEngine.js';
 import {
   at,
-  last,
   getQueryParameters,
   getQueryParameterValue,
   isNonNullable,
@@ -758,15 +757,15 @@ function generateDefinitionForPivotResultColumns(
 
       // sort the leaf level columns based on the order of selected/configuration columns
       (currentCollection as ColDef[]).sort((a, b) => {
-        const colAName = last(
-          a.colId?.split(PIVOT_COLUMN_NAME_VALUE_SEPARATOR) ?? [],
-        );
+        const colAName = (
+          a.colId?.split(PIVOT_COLUMN_NAME_VALUE_SEPARATOR) ?? []
+        ).at(-1);
         const colAConf = colAName
           ? _findCol(configuration.columns, colAName)
           : undefined;
-        const colBName = last(
-          b.colId?.split(PIVOT_COLUMN_NAME_VALUE_SEPARATOR) ?? [],
-        );
+        const colBName = (
+          b.colId?.split(PIVOT_COLUMN_NAME_VALUE_SEPARATOR) ?? []
+        ).at(-1);
         const colBConf = colBName
           ? _findCol(configuration.columns, colBName)
           : undefined;

package/src/stores/view/grid/DataCubeGridState.ts

@@ -38,6 +38,8 @@ import { DataCubeConfiguration } from '../../core/model/DataCubeConfiguration.js
 import { DataCubeGridControllerState } from './DataCubeGridControllerState.js';
 import { DataCubeGridClientExportEngine } from './DataCubeGridClientExportEngine.js';
 import { DataCubeSettingKey } from '../../../__lib__/DataCubeSetting.js';
+import { DEFAULT_ALERT_WINDOW_CONFIG } from '../../services/DataCubeLayoutService.js';
+import { AlertType } from '../../services/DataCubeAlertService.js';
 
 /**
  * This query editor state is responsible for syncing the internal state of ag-grid
@@ -125,32 +127,95 @@ export class DataCubeGridState extends DataCubeQuerySnapshotController {
   }
 
   async setCachingEnabled(val: boolean) {
-    const currentVal = this.isCachingEnabled;
-    if (val === currentVal) {
+    if (val === this.isCachingEnabled) {
       return;
     }
 
-    runInAction(() => {
-      this.isCachingEnabled = val;
-    });
+    // disable caching
+    if (val === false) {
+      await this._view.disposeCache();
+
+      runInAction(() => {
+        this.isCachingEnabled = val;
 
-    if (val) {
+        // hard reset the grid, this will force the grid to fetch data again
+        this.clientDataSource = new DataCubeGridClientServerSideDataSource(
+          this,
+          this._view,
+        );
+      });
+
+      return;
+    }
+
+    // enable caching
+    const run = async () => {
       await this._view.initializeCache();
-      if (this._view.processCacheState.hasFailed) {
-        this.isCachingEnabled = currentVal;
-        return;
+
+      // only update value if cache processing succeeds
+      if (this._view.processCacheState.hasSucceeded) {
+        runInAction(() => {
+          this.isCachingEnabled = val;
+
+          // hard reset the grid, this will force the grid to fetch data again
+          this.clientDataSource = new DataCubeGridClientServerSideDataSource(
+            this,
+            this._view,
+          );
+        });
       }
+    };
+
+    // TODO?: we might want to do a quick check here for the amount of data the cache
+    // will handle, so maybe fire a COUNT query to check for the number of records
+    // and check that against a threshold, we will need to alter the prompt below accordingly.
+    if (
+      this._settingService.getBooleanValue(
+        DataCubeSettingKey.GRID_CLIENT__SHOW_CACHE_PERFORMANCE_WARNING,
+      )
+    ) {
+      this._view.alertService.alert({
+        message: `Confirm you want to proceed with caching`,
+        text: `When enabled, the source dataset will be cached locally in order to boost query performance. But depending on computational resource available to your environment, sometimes, caching can negatively impact the overall performance, and can even lead to crashes.\nDo you still want to proceed?`,
+        type: AlertType.WARNING,
+        actions: [
+          {
+            label: 'Abort',
+            handler: () => {},
+          },
+          {
+            label: 'Proceed',
+            handler: () => {
+              run().catch((error) =>
+                this._view.alertService.alertUnhandledError(error),
+              );
+            },
+          },
+          {
+            label: 'Dismiss Warning and Proceed',
+            handler: () => {
+              this._view.settingService.updateValue(
+                this._view.dataCube.api,
+                DataCubeSettingKey.GRID_CLIENT__SHOW_CACHE_PERFORMANCE_WARNING,
+                false,
+              );
+              run().catch((error) =>
+                this._view.alertService.alertUnhandledError(error),
+              );
+            },
+          },
+        ],
+        windowConfig: {
+          ...DEFAULT_ALERT_WINDOW_CONFIG,
+          width: 600,
+          height: 210,
+          minWidth: 300,
+          minHeight: 150,
+        },
+      });
     } else {
-      await this._view.disposeCache();
+      await run();
     }
-
-    runInAction(() => {
-      // hard reset the grid, this will force the grid to fetch data again
-      this.clientDataSource = new DataCubeGridClientServerSideDataSource(
-        this,
-        this._view,
-      );
-    });
   }
 
   setScrollHintText(val: string | undefined) {