@finos/legend-data-cube 0.1.32 → 0.2.0

package/src/stores/core/DataCubeQueryBuilder.ts

@@ -23,13 +23,21 @@
 
 import {
   PRIMITIVE_TYPE,
+  V1_AppliedProperty,
+  V1_CString,
   V1_Lambda,
+  V1_Variable,
   type V1_AppliedFunction,
 } from '@finos/legend-graph';
-import { type DataCubeSnapshot } from './DataCubeSnapshot.js';
+import {
+  type DataCubeSnapshot,
+  type DataCubeSnapshotExtendedColumn,
+  type DataCubeSnapshotGroupBy,
+} from './DataCubeSnapshot.js';
 import { at, guaranteeType } from '@finos/legend-shared';
 import {
   DataCubeFunction,
+  DataCubeQueryFilterGroupOperator,
   DataCubeQuerySortDirection,
   type DataCubeQueryFunctionMap,
 } from './DataCubeQueryEngine.js';
@@ -49,10 +57,15 @@ import {
   _var,
   _functionCompositionProcessor,
   _extendRootAggregation,
+  _flattenFilterSnapshot,
 } from './DataCubeQueryBuilderUtils.js';
 import type { DataCubeSource } from './model/DataCubeSource.js';
-import { _findCol } from './model/DataCubeColumn.js';
+import { _findCol, _toCol } from './model/DataCubeColumn.js';
 import type { DataCubeEngine } from './DataCubeEngine.js';
+import {
+  DIMENSIONAL_L0_COLUMN,
+  type DataCubeDimensionalNode,
+} from '../view/grid/DataCubeGridDimensionalTree.js';
 
 export function buildExecutableQuery(
   snapshot: DataCubeSnapshot,
@@ -312,3 +325,244 @@ export function buildExecutableQuery(
   }
   return at(sequence, sequence.length - 1);
 }
+
+export function buildDimensionalExecutableQuery(
+  snapshot: DataCubeSnapshot,
+  source: DataCubeSource,
+  engine: DataCubeEngine,
+  nodes: DataCubeDimensionalNode[],
+  options?: {
+    postProcessor?: (
+      snapshot: DataCubeSnapshot,
+      sequence: V1_AppliedFunction[],
+      funcMap: DataCubeQueryFunctionMap,
+      configuration: DataCubeConfiguration,
+      engine: DataCubeEngine,
+    ) => void;
+    rootAggregation?:
+      | {
+          columnName: string;
+        }
+      | undefined;
+    pagination?:
+      | {
+          start: number;
+          end: number;
+        }
+      | undefined;
+    skipExecutionContext?: boolean;
+  },
+) {
+  const data = snapshot.data;
+  const configuration = DataCubeConfiguration.serialization.fromJson(
+    data.configuration,
+  );
+  const sequence: V1_AppliedFunction[] = [];
+  const funcMap: DataCubeQueryFunctionMap = {};
+  const _process = _functionCompositionProcessor(sequence, funcMap);
+
+  // ------------------------------ DIMENSIONS -------------------------------------
+
+  if (data.tree && nodes.length > 0) {
+    const dimensionColNames = configuration.dimensions.dimensions.flatMap(
+      (col) => col.columns,
+    );
+
+    // removing columns which have been added to represent diimensions
+    configuration.columns = configuration.columns.map((col) => {
+      if (dimensionColNames.includes(col.name)) {
+        col.isSelected = false;
+      }
+      return col;
+    });
+
+    // Adding dimensions to the extended column
+    nodes.forEach((node) => {
+      const lambda = new V1_Lambda();
+      const parameter = new V1_Variable();
+      parameter.name = 'temp';
+      if (node.column === DIMENSIONAL_L0_COLUMN) {
+        const defaultValue = new V1_CString();
+        defaultValue.value = node.column;
+        lambda.body = [defaultValue];
+      } else {
+        const defaultValue = new V1_AppliedProperty();
+        defaultValue.parameters = [parameter];
+        defaultValue.property = node.column;
+        lambda.body = [defaultValue];
+      }
+      lambda.parameters = [parameter];
+      data.leafExtendedColumns.unshift({
+        name: node.dimension,
+        type: PRIMITIVE_TYPE.STRING,
+        mapFn: engine.serializeValueSpecification(lambda),
+      } satisfies DataCubeSnapshotExtendedColumn);
+    });
+
+    // Adding dimensions to selected list
+    data.selectColumns = data.selectColumns.filter(
+      (col) => !dimensionColNames.includes(col.name),
+    );
+
+    const dimensionCols = nodes.map((col) =>
+      _toCol({ name: col.dimension, type: PRIMITIVE_TYPE.STRING }),
+    );
+    data.selectColumns.unshift(...dimensionCols);
+
+    // Adding filter based on group by nodes
+    const groupByNodes = nodes.flatMap((x) => x.groupByNodes);
+    if (groupByNodes.length > 0) {
+      const filter = [
+        _flattenFilterSnapshot(nodes.flatMap((x) => x.groupByNodes)),
+      ];
+      if (data.filter) {
+        filter.push(data.filter);
+        data.filter = {
+          groupOperator: DataCubeQueryFilterGroupOperator.AND,
+          conditions: filter,
+        };
+      } else {
+        data.filter = filter.at(0);
+      }
+    }
+
+    // Adding dimensions to groupby
+    if (data.groupBy) {
+      data.groupBy.columns.unshift(...dimensionCols);
+    } else {
+      data.groupBy = {
+        columns: dimensionCols,
+      } satisfies DataCubeSnapshotGroupBy;
+    }
+  }
+
+  // --------------------------------- LEAF-LEVEL EXTEND ---------------------------------
+
+  if (data.leafExtendedColumns.length) {
+    _process(
+      'leafExtend',
+      data.leafExtendedColumns.map((col) =>
+        _function(DataCubeFunction.EXTEND, [
+          _cols([
+            _colSpec(
+              col.name,
+              guaranteeType(
+                engine.deserializeValueSpecification(col.mapFn),
+                V1_Lambda,
+              ),
+              col.reduceFn
+                ? guaranteeType(
+                    engine.deserializeValueSpecification(col.reduceFn),
+                    V1_Lambda,
+                  )
+                : undefined,
+            ),
+          ]),
+        ]),
+      ),
+    );
+  }
+
+  // --------------------------------- FILTER ---------------------------------
+
+  if (data.filter) {
+    _process(
+      'filter',
+      _function(DataCubeFunction.FILTER, [
+        _lambda([_var()], [_filter(data.filter, engine.filterOperations)]),
+      ]),
+    );
+  }
+
+  // --------------------------------- SELECT ---------------------------------
+
+  if (data.selectColumns.length) {
+    _process(
+      'select',
+      _function(DataCubeFunction.SELECT, [
+        _cols(data.selectColumns.map((col) => _colSpec(col.name))),
+      ]),
+    );
+  }
+
+  // --------------------------------- PIVOT ---------------------------------
+
+  // TODO: rethink pivots for dimensional grid mode
+
+  // --------------------------------- GROUP BY ---------------------------------
+
+  if (data.groupBy) {
+    const groupBy = data.groupBy;
+    if (configuration.showRootAggregation && options?.rootAggregation) {
+      sequence.push(_extendRootAggregation(options.rootAggregation.columnName));
+    }
+    _process(
+      'groupBy',
+      _function(DataCubeFunction.GROUP_BY, [
+        _cols(groupBy.columns.map((col) => _colSpec(col.name))),
+        _cols(
+          _groupByAggCols(
+            groupBy.columns,
+            snapshot,
+            configuration,
+            engine.aggregateOperations,
+          ),
+        ),
+      ]),
+    );
+    _process(
+      'groupBySort',
+      _function(DataCubeFunction.SORT, [
+        _collection(
+          groupBy.columns.map((col) =>
+            _function(
+              configuration.treeColumnSortDirection ===
+                DataCubeQuerySortDirection.ASCENDING
+                ? DataCubeFunction.ASCENDING
+                : DataCubeFunction.DESCENDING,
+              [_col(col.name)],
+            ),
+          ),
+        ),
+      ]),
+    );
+  }
+
+  // --------------------------------- GROUP-LEVEL EXTEND ---------------------------------
+
+  // TODO: implement group level extends for dimensional grid mode
+
+  // --------------------------------- SORT ---------------------------------
+
+  // TODO: implement sort for dimensional grid mode
+
+  // --------------------------------- LIMIT ---------------------------------
+
+  // TODO: rethink limits for dimensional grid mode
+
+  // --------------------------------- SLICE ---------------------------------
+
+  // TODO: rethink slice for dimensional grid mode
+  // slice can lead to incorrect view of data for dimensionality
+
+  // --------------------------------- FINALIZE ---------------------------------
+
+  if (!options?.skipExecutionContext) {
+    const executionContext = engine.buildExecutionContext(source);
+    if (executionContext) {
+      sequence.push(executionContext);
+    }
+  }
+
+  options?.postProcessor?.(snapshot, sequence, funcMap, configuration, engine);
+
+  if (sequence.length === 0) {
+    return source.query;
+  }
+  for (let i = 0; i < sequence.length; i++) {
+    at(sequence, i).parameters.unshift(
+      i === 0 ? source.query : at(sequence, i - 1),
+    );
+  }
+  return at(sequence, sequence.length - 1);
+}

package/src/stores/core/DataCubeQueryBuilderUtils.ts

@@ -81,6 +81,7 @@ import {
   isPivotResultColumnName,
   getPivotResultColumnBaseColumnName,
   DEFAULT_ROOT_AGGREGATION_COLUMN_VALUE,
+  DataCubeQueryFilterOperator,
 } from './DataCubeQueryEngine.js';
 import type { DataCubeQueryFilterOperation } from './filter/DataCubeQueryFilterOperation.js';
 import type { DataCubeQueryAggregateOperation } from './aggregation/DataCubeQueryAggregateOperation.js';
@@ -88,6 +89,7 @@ import {
   DataCubeColumnConfiguration,
   type DataCubeConfiguration,
 } from './model/DataCubeConfiguration.js';
+import type { DataCubeDimensionalGroupByNode } from '../view/grid/DataCubeGridDimensionalTree.js';
 
 // --------------------------------- UTILITIES ---------------------------------
 
@@ -645,3 +647,23 @@ export function _filter(
     return filterCondition.not ? _not(condition) : condition;
   }
 }
+
+export function _flattenFilterSnapshot(
+  groupByNodes: DataCubeDimensionalGroupByNode[],
+): DataCubeSnapshotFilter {
+  const filterConditions = groupByNodes.map((node) => {
+    return {
+      name: node.column,
+      type: 'String', // Assuming all filters are strings — update if needed
+      operator: DataCubeQueryFilterOperator.EQUAL,
+      value: {
+        type: 'String',
+        value: node.filter,
+      } satisfies DataCubeOperationValue,
+    } satisfies DataCubeSnapshotFilterCondition;
+  });
+  return {
+    groupOperator: DataCubeQueryFilterGroupOperator.AND,
+    conditions: filterConditions,
+  } satisfies DataCubeSnapshotFilter;
+}

package/src/stores/core/DataCubeQueryEngine.ts

@@ -273,6 +273,11 @@ export enum DataCubeGridMode {
   MULTIDIMENSIONAL = 'Multidimensional',
 }
 
+export enum DataCubeClientModelOption {
+  ROW_DATA = 'rowData',
+  COLUMN_DEFS = 'columnDefs',
+}
+
 export enum DataCubeTitleBarMenuItems {
   VIEW_SOURCE = 'View Source',
   RESET_TO_LATEST_SAVE = 'Reset to Latest Save',
@@ -409,6 +414,10 @@ export function ofDataType(
   return dataTypes.includes(getDataType(type));
 }
 
+export function isDimensionalGridMode(gridMode: DataCubeGridMode) {
+  return gridMode === DataCubeGridMode.MULTIDIMENSIONAL;
+}
+
 export function isPrimitiveType(type: string) {
   return (
     [

package/src/stores/core/DataCubeSnapshot.ts

@@ -29,6 +29,7 @@ import type {
   DataCubeQuerySortDirection,
 } from './DataCubeQueryEngine.js';
 import type { DataCubeColumn } from './model/DataCubeColumn.js';
+import type { DataCubeDimensionalTree } from '../view/grid/DataCubeGridDimensionalTree.js';
 
 export type DataCubeSnapshotFilterCondition = DataCubeColumn & {
   value: DataCubeOperationValue;
@@ -85,6 +86,7 @@ export type DataCubeSnapshotData = {
   groupExtendedColumns: DataCubeSnapshotExtendedColumn[];
   sortColumns: DataCubeSnapshotSortColumn[];
   limit: number | undefined;
+  tree?: DataCubeDimensionalTree | undefined;
 };
 
 export class DataCubeSnapshot {

package/src/stores/view/editor/DataCubeEditorDimensionsPanelState.ts

@@ -226,13 +226,14 @@ export class DataCubeEditorDimensionsPanelState
         name: dimension.name,
         columns: dimension.columns.map((column) => column.name),
       }));
+      newSnapshot.data.configuration = {
+        ...newSnapshot.data.configuration,
+        dimensions: DataCubeDimensionsConfiguration.serialization.toJson(
+          dimensionsConfiguration,
+        ),
+      };
+    } else {
+      newSnapshot = baseSnapshot;
     }
-
-    newSnapshot.data.configuration = {
-      ...newSnapshot.data.configuration,
-      dimensions: DataCubeDimensionsConfiguration.serialization.toJson(
-        dimensionsConfiguration,
-      ),
-    };
   }
 }

package/src/stores/view/grid/DataCubeGridClientEngine.ts

@@ -25,6 +25,7 @@ import {
   IllegalStateError,
   assertErrorThrown,
   guaranteeNonNullable,
+  guaranteeType,
   hashObject,
   isBoolean,
   isNonNullable,
@@ -35,11 +36,13 @@ import {
   PRIMITIVE_TYPE,
   PureClientVersion,
   type TabularDataSet,
+  type TDSRow,
 } from '@finos/legend-graph';
 import { makeObservable, observable, runInAction } from 'mobx';
-import type {
+import {
   DataCubeConfiguration,
-  DataCubeConfigurationColorKey,
+  type DataCubeConfigurationColorKey,
+  type DataCubeDimensionConfiguration,
 } from '../../core/model/DataCubeConfiguration.js';
 import { type DataCubeSnapshot } from '../../core/DataCubeSnapshot.js';
 import { _findCol, _sortByColName } from '../../core/model/DataCubeColumn.js';
@@ -47,7 +50,10 @@ import { isPivotResultColumnName } from '../../core/DataCubeQueryEngine.js';
 import { buildSnapshotFromGridState } from './DataCubeGridSnapshotBuilder.js';
 import { AlertType } from '../../services/DataCubeAlertService.js';
 import type { DataCubeViewState } from '../DataCubeViewState.js';
-import { buildGridDataFetchExecutableQuery } from './DataCubeGridQueryBuilder.js';
+import {
+  buildDimensionalGridDataFetchExecutableQuery,
+  buildGridDataFetchExecutableQuery,
+} from './DataCubeGridQueryBuilder.js';
 import { DataCubeSettingKey } from '../../../__lib__/DataCubeSetting.js';
 import { DEFAULT_ALERT_WINDOW_CONFIG } from '../../services/DataCubeLayoutService.js';
 import {
@@ -57,8 +63,22 @@ import {
 import { _lambda } from '../../core/DataCubeQueryBuilderUtils.js';
 import { sum } from 'mathjs';
 import { DataCubeEvent } from '../../../__lib__/DataCubeEvent.js';
-
-type DataCubeGridClientCellValue = string | number | boolean | null | undefined;
+import { generateColumnDefsForDimensions } from './DataCubeGridConfigurationBuilder.js';
+import {
+  DataCubeDimensionalMetadata,
+  generateDimensionalPaths,
+  type DataCubeDimensionalNode,
+  type DataCubeDimensionalGroupByNode,
+  DIMENSIONAL_L0_COLUMN,
+} from './DataCubeGridDimensionalTree.js';
+
+type DataCubeGridClientCellValue =
+  | string
+  | number
+  | boolean
+  | Map<string, DataCubeDimensionalMetadata>
+  | null
+  | undefined;
 type DataCubeGridClientRowData = {
   [key: string]: DataCubeGridClientCellValue;
 };
@@ -259,6 +279,136 @@ function buildRowData(
   });
 }
 
+function buildDimensionalRowData(
+  result: TabularDataSet,
+  snapshot: DataCubeSnapshot,
+  dimensionalNodes: DataCubeDimensionalNode[],
+  rowData: DataCubeGridClientRowData[],
+  dimensions: DataCubeDimensionConfiguration[],
+): DataCubeGridClientRowData[] {
+  const newRows = result.rows.map((rawRow) =>
+    _createClientRow(rawRow, result.columns, dimensionalNodes, dimensions),
+  );
+
+  const updated = [...rowData];
+  for (const newRow of newRows) {
+    const insertIndex = _findInsertIndex(newRow, updated, dimensionalNodes);
+    updated.splice(insertIndex, 0, newRow);
+  }
+
+  return updated;
+}
+
+function _createClientRow(
+  rawRow: TDSRow,
+  columns: string[],
+  nodes: DataCubeDimensionalNode[],
+  configs: DataCubeDimensionConfiguration[],
+): DataCubeGridClientRowData {
+  const row: DataCubeGridClientRowData = {};
+  rawRow.values.forEach((value, i) => {
+    row[guaranteeNonNullable(columns[i])] = isBoolean(value)
+      ? String(value)
+      : isNonNullable(value)
+        ? value
+        : INTERNAL__GRID_CLIENT_MISSING_VALUE;
+  });
+
+  const metadata = new Map<string, DataCubeDimensionalMetadata>();
+  for (const node of nodes) {
+    const config = guaranteeNonNullable(
+      configs.find((d) => d.name === node.dimension),
+    );
+    metadata.set(
+      node.dimension,
+      new DataCubeDimensionalMetadata(
+        node.column,
+        config.columns.indexOf(node.column),
+        node.groupByNodes,
+      ),
+    );
+  }
+  row.metadata = metadata;
+  return row;
+}
+
+function _findInsertIndex(
+  newRow: DataCubeGridClientRowData,
+  existing: DataCubeGridClientRowData[],
+  dimensionalNodes: DataCubeDimensionalNode[],
+): number {
+  const metadata = guaranteeType(
+    newRow.metadata,
+    Map<string, DataCubeDimensionalMetadata>,
+  );
+  const groupByNodes = [...metadata.values()].flatMap((m) => m.groupByNodes);
+  const dimensions = dimensionalNodes.map((d) => d.dimension);
+  const latest = _latestGroupByFilters(groupByNodes);
+
+  for (let i = existing.length - 1; i >= 0; i--) {
+    const existingRow = guaranteeNonNullable(existing[i]);
+    if (
+      _isParentRow(existingRow, groupByNodes.length, newRow, latest, dimensions)
+    ) {
+      return i + 1;
+    }
+  }
+
+  return existing.length;
+}
+
+function _isParentRow(
+  existing: DataCubeGridClientRowData,
+  groupByCount: number,
+  newRow: DataCubeGridClientRowData,
+  filters: DataCubeDimensionalGroupByNode[],
+  dimensions: string[],
+): boolean {
+  if (groupByCount === 0) {
+    let allCount = 0;
+    const match = dimensions.every((d) => {
+      const e = existing[d],
+        n = newRow[d];
+      if (e === null || n === null) {
+        return false;
+      }
+      if (String(e) === DIMENSIONAL_L0_COLUMN) {
+        allCount++;
+        return true;
+      }
+      return String(e) === String(n);
+    });
+    return match && allCount >= 1 && allCount < dimensions.length;
+  }
+
+  return dimensions.every((d) => {
+    const filter = filters.find((f) => f.dimension === d);
+    const e = existing[d],
+      n = newRow[d];
+    if (e === null || n === null) {
+      return false;
+    }
+    if (filter) {
+      return (
+        String(e) === String(filter.filter) ||
+        String(e) === String(n) ||
+        String(e) === DIMENSIONAL_L0_COLUMN
+      );
+    }
+    return String(e) === String(n);
+  });
+}
+
+function _latestGroupByFilters(
+  nodes: DataCubeDimensionalGroupByNode[],
+): DataCubeDimensionalGroupByNode[] {
+  const map = new Map<string, DataCubeDimensionalGroupByNode>();
+  for (const node of nodes) {
+    map.set(node.dimension, node);
+  }
+  return [...map.values()];
+}
+
 async function getCastColumns(
   currentSnapshot: DataCubeSnapshot,
   view: DataCubeViewState,
@@ -302,6 +452,103 @@ async function getCastColumns(
   });
 }
 
+export async function fetchDimensionalQueryRows(
+  snapshot: DataCubeSnapshot,
+  view: DataCubeViewState,
+) {
+  const task = view.taskService.newTask('Fetching data...');
+
+  let rowData: DataCubeGridClientRowData[] = [];
+
+  const configuration = DataCubeConfiguration.serialization.fromJson(
+    snapshot.data.configuration,
+  );
+
+  // upgrade here by not shooting extra sqls
+  const paths = generateDimensionalPaths(
+    guaranteeNonNullable(snapshot.data.tree),
+  );
+
+  for (const path of paths) {
+    const executableQuery = buildDimensionalGridDataFetchExecutableQuery(
+      snapshot,
+      view.source,
+      view.engine,
+      path,
+    );
+    let result: DataCubeExecutionResult;
+
+    try {
+      result = await view.engine.executeQuery(
+        _lambda([], [executableQuery]),
+        view.source,
+        {
+          debug: view.settingService.getBooleanValue(
+            DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
+          ),
+          clientVersion: view.settingService.getBooleanValue(
+            DataCubeSettingKey.DEBUGGER__USE_DEV_CLIENT_PROTOCOL_VERSION,
+          )
+            ? PureClientVersion.VX_X_X
+            : undefined,
+        },
+      );
+
+      rowData = buildDimensionalRowData(
+        result.result.result,
+        snapshot,
+        path,
+        rowData,
+        configuration.dimensions.dimensions,
+      );
+
+      view.taskService.endTask(task);
+    } catch (error) {
+      assertErrorThrown(error);
+      if (error instanceof DataCubeExecutionError) {
+        view.alertService.alertExecutionError(error, {
+          message: `Data Fetch Failure: Can't execute query.`,
+          text: `Error: ${error.message}`,
+        });
+      } else {
+        view.alertService.alertError(error, {
+          message: `Data Fetch Failure: ${error.message}`,
+        });
+      }
+      view.taskService.endTask(task);
+      return {
+        rowData: [],
+        columnDefs: [],
+      };
+    }
+
+    if (
+      view.settingService.getBooleanValue(
+        DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
+      )
+    ) {
+      view.engine.debugProcess(
+        `Execution`,
+        ['Query', result.executedQuery],
+        [
+          'Stats',
+          `${rowData.length} rows, ${result.result.result.columns.length} columns`,
+        ],
+        [
+          'SQL',
+          result.executedSQL ?? `-- Error: failed to extract executed SQL`,
+        ],
+        ['SQL Execution Time', result.executionTime],
+      );
+    }
+  }
+
+  return {
+    rowData: rowData,
+    columnDefs: generateColumnDefsForDimensions(snapshot, configuration),
+  };
+}
+
 export type DataCubeGridClientDataFetchRequest = {
   startRow: number | undefined;
   endRow: number | undefined;