@finos/legend-data-cube 0.1.32 → 0.2.1

package/src/stores/core/DataCubeQueryBuilderUtils.ts

@@ -81,6 +81,7 @@ import {
   isPivotResultColumnName,
   getPivotResultColumnBaseColumnName,
   DEFAULT_ROOT_AGGREGATION_COLUMN_VALUE,
+  DataCubeQueryFilterOperator,
 } from './DataCubeQueryEngine.js';
 import type { DataCubeQueryFilterOperation } from './filter/DataCubeQueryFilterOperation.js';
 import type { DataCubeQueryAggregateOperation } from './aggregation/DataCubeQueryAggregateOperation.js';
@@ -88,6 +89,7 @@ import {
   DataCubeColumnConfiguration,
   type DataCubeConfiguration,
 } from './model/DataCubeConfiguration.js';
+import type { DataCubeDimensionalGroupByNode } from '../view/grid/DataCubeGridDimensionalTree.js';
 
 // --------------------------------- UTILITIES ---------------------------------
 
@@ -645,3 +647,23 @@ export function _filter(
     return filterCondition.not ? _not(condition) : condition;
   }
 }
+
+export function _flattenFilterSnapshot(
+  groupByNodes: DataCubeDimensionalGroupByNode[],
+): DataCubeSnapshotFilter {
+  const filterConditions = groupByNodes.map((node) => {
+    return {
+      name: node.column,
+      type: 'String', // Assuming all filters are strings — update if needed
+      operator: DataCubeQueryFilterOperator.EQUAL,
+      value: {
+        type: 'String',
+        value: node.filter,
+      } satisfies DataCubeOperationValue,
+    } satisfies DataCubeSnapshotFilterCondition;
+  });
+  return {
+    groupOperator: DataCubeQueryFilterGroupOperator.AND,
+    conditions: filterConditions,
+  } satisfies DataCubeSnapshotFilter;
+}

package/src/stores/core/DataCubeQueryEngine.ts

@@ -273,6 +273,12 @@ export enum DataCubeGridMode {
   MULTIDIMENSIONAL = 'Multidimensional',
 }
 
+export enum DataCubeClientModelOption {
+  ROW_DATA = 'rowData',
+  COLUMN_DEFS = 'columnDefs',
+  LOADING = 'loading',
+}
+
 export enum DataCubeTitleBarMenuItems {
   VIEW_SOURCE = 'View Source',
   RESET_TO_LATEST_SAVE = 'Reset to Latest Save',
@@ -409,6 +415,10 @@ export function ofDataType(
   return dataTypes.includes(getDataType(type));
 }
 
+export function isDimensionalGridMode(gridMode: DataCubeGridMode) {
+  return gridMode === DataCubeGridMode.MULTIDIMENSIONAL;
+}
+
 export function isPrimitiveType(type: string) {
   return (
     [

package/src/stores/core/DataCubeSnapshot.ts

@@ -29,6 +29,7 @@ import type {
   DataCubeQuerySortDirection,
 } from './DataCubeQueryEngine.js';
 import type { DataCubeColumn } from './model/DataCubeColumn.js';
+import type { DataCubeDimensionalTree } from '../view/grid/DataCubeGridDimensionalTree.js';
 
 export type DataCubeSnapshotFilterCondition = DataCubeColumn & {
   value: DataCubeOperationValue;
@@ -85,6 +86,7 @@ export type DataCubeSnapshotData = {
   groupExtendedColumns: DataCubeSnapshotExtendedColumn[];
   sortColumns: DataCubeSnapshotSortColumn[];
   limit: number | undefined;
+  dimensionalTree?: DataCubeDimensionalTree | undefined;
 };
 
 export class DataCubeSnapshot {

package/src/stores/core/DataCubeSnapshotBuilder.ts

@@ -611,7 +611,7 @@ export async function validateAndBuildSnapshot(
     engine,
   );
   data.configuration = configuration.serialize();
-
+  data.dimensionalTree = specification.dimensionalTree;
   return snapshot.finalize();
 }
 

package/src/stores/core/model/DataCubeSpecification.ts

@@ -21,6 +21,7 @@ import {
 } from '@finos/legend-shared';
 import { createModelSchema, optional, primitive, raw } from 'serializr';
 import { DataCubeConfiguration } from './DataCubeConfiguration.js';
+import { DataCubeDimensionalTree } from '../../view/grid/DataCubeGridDimensionalTree.js';
 
 export class DataCubeSpecificationOptions {
   autoEnableCache?: boolean | undefined;
@@ -37,6 +38,7 @@ export class DataCubeSpecification {
   configuration?: DataCubeConfiguration | undefined;
   source!: PlainObject;
   options?: DataCubeSpecificationOptions | undefined;
+  dimensionalTree?: DataCubeDimensionalTree | undefined;
 
   static readonly serialization = new SerializationFactory(
     createModelSchema(DataCubeSpecification, {
@@ -48,6 +50,9 @@ export class DataCubeSpecification {
       ),
       query: primitive(),
       source: raw(),
+      dimensionalTree: optional(
+        usingModelSchema(DataCubeDimensionalTree.serialization.schema),
+      ),
     }),
   );
 }

package/src/stores/view/DataCubeViewState.ts

@@ -104,6 +104,7 @@ export class DataCubeViewState {
       snapshot.data.configuration,
     );
     query.query = await this.engine.getPartialQueryCode(snapshot);
+    query.dimensionalTree = snapshot.data.dimensionalTree;
     return query;
   }
 

package/src/stores/view/editor/DataCubeEditorDimensionsPanelState.ts

@@ -213,6 +213,7 @@ export class DataCubeEditorDimensionsPanelState
         return _dimension;
       }),
     );
+    this.refreshDimensionsTreeData();
   }
 
   buildSnapshot(newSnapshot: DataCubeSnapshot, baseSnapshot: DataCubeSnapshot) {
@@ -226,13 +227,14 @@ export class DataCubeEditorDimensionsPanelState
         name: dimension.name,
         columns: dimension.columns.map((column) => column.name),
       }));
+      newSnapshot.data.configuration = {
+        ...newSnapshot.data.configuration,
+        dimensions: DataCubeDimensionsConfiguration.serialization.toJson(
+          dimensionsConfiguration,
+        ),
+      };
+    } else {
+      newSnapshot = baseSnapshot;
     }
-
-    newSnapshot.data.configuration = {
-      ...newSnapshot.data.configuration,
-      dimensions: DataCubeDimensionsConfiguration.serialization.toJson(
-        dimensionsConfiguration,
-      ),
-    };
   }
 }

package/src/stores/view/editor/DataCubeEditorState.tsx

@@ -147,6 +147,7 @@ export class DataCubeEditorState extends DataCubeSnapshotController {
     return 'editor';
   }
 
+  //TODO: for dimensional grid, we need to reload the grid when we apply changes
   async applyChanges(options?: { closeAfterApply?: boolean | undefined }) {
     this.finalizationState.inProgress();
 

package/src/stores/view/grid/DataCubeGridClientEngine.ts

@@ -25,6 +25,7 @@ import {
   IllegalStateError,
   assertErrorThrown,
   guaranteeNonNullable,
+  guaranteeType,
   hashObject,
   isBoolean,
   isNonNullable,
@@ -35,19 +36,27 @@ import {
   PRIMITIVE_TYPE,
   PureClientVersion,
   type TabularDataSet,
+  type TDSRow,
 } from '@finos/legend-graph';
 import { makeObservable, observable, runInAction } from 'mobx';
-import type {
+import {
   DataCubeConfiguration,
-  DataCubeConfigurationColorKey,
+  type DataCubeConfigurationColorKey,
+  type DataCubeDimensionConfiguration,
 } from '../../core/model/DataCubeConfiguration.js';
 import { type DataCubeSnapshot } from '../../core/DataCubeSnapshot.js';
 import { _findCol, _sortByColName } from '../../core/model/DataCubeColumn.js';
-import { isPivotResultColumnName } from '../../core/DataCubeQueryEngine.js';
+import {
+  DataCubeClientModelOption,
+  isPivotResultColumnName,
+} from '../../core/DataCubeQueryEngine.js';
 import { buildSnapshotFromGridState } from './DataCubeGridSnapshotBuilder.js';
 import { AlertType } from '../../services/DataCubeAlertService.js';
 import type { DataCubeViewState } from '../DataCubeViewState.js';
-import { buildGridDataFetchExecutableQuery } from './DataCubeGridQueryBuilder.js';
+import {
+  buildDimensionalGridDataFetchExecutableQuery,
+  buildGridDataFetchExecutableQuery,
+} from './DataCubeGridQueryBuilder.js';
 import { DataCubeSettingKey } from '../../../__lib__/DataCubeSetting.js';
 import { DEFAULT_ALERT_WINDOW_CONFIG } from '../../services/DataCubeLayoutService.js';
 import {
@@ -57,8 +66,24 @@ import {
 import { _lambda } from '../../core/DataCubeQueryBuilderUtils.js';
 import { sum } from 'mathjs';
 import { DataCubeEvent } from '../../../__lib__/DataCubeEvent.js';
-
-type DataCubeGridClientCellValue = string | number | boolean | null | undefined;
+import { generateColumnDefsForDimensions } from './DataCubeGridConfigurationBuilder.js';
+import {
+  DataCubeDimensionalMetadata,
+  generateDimensionalPaths,
+  type DataCubeDimensionalNode,
+  type DataCubeDimensionalGroupByNode,
+  type DataCubeDimensionalTree,
+  DIMENSIONAL_L0_COLUMN,
+  findExtraPaths,
+} from './DataCubeGridDimensionalTree.js';
+
+type DataCubeGridClientCellValue =
+  | string
+  | number
+  | boolean
+  | Map<string, DataCubeDimensionalMetadata>
+  | null
+  | undefined;
 type DataCubeGridClientRowData = {
   [key: string]: DataCubeGridClientCellValue;
 };
@@ -259,6 +284,136 @@ function buildRowData(
   });
 }
 
+function buildDimensionalRowData(
+  result: TabularDataSet,
+  snapshot: DataCubeSnapshot,
+  dimensionalNodes: DataCubeDimensionalNode[],
+  rowData: DataCubeGridClientRowData[],
+  dimensions: DataCubeDimensionConfiguration[],
+): DataCubeGridClientRowData[] {
+  const newRows = result.rows.map((rawRow) =>
+    _createClientRow(rawRow, result.columns, dimensionalNodes, dimensions),
+  );
+
+  const updated = [...rowData];
+  for (const newRow of newRows) {
+    const insertIndex = _findInsertIndex(newRow, updated, dimensionalNodes);
+    updated.splice(insertIndex, 0, newRow);
+  }
+
+  return updated;
+}
+
+function _createClientRow(
+  rawRow: TDSRow,
+  columns: string[],
+  nodes: DataCubeDimensionalNode[],
+  configs: DataCubeDimensionConfiguration[],
+): DataCubeGridClientRowData {
+  const row: DataCubeGridClientRowData = {};
+  rawRow.values.forEach((value, i) => {
+    row[guaranteeNonNullable(columns[i])] = isBoolean(value)
+      ? String(value)
+      : isNonNullable(value)
+        ? value
+        : INTERNAL__GRID_CLIENT_MISSING_VALUE;
+  });
+
+  const metadata = new Map<string, DataCubeDimensionalMetadata>();
+  for (const node of nodes) {
+    const config = guaranteeNonNullable(
+      configs.find((d) => d.name === node.dimension),
+    );
+    metadata.set(
+      node.dimension,
+      new DataCubeDimensionalMetadata(
+        node.column,
+        config.columns.indexOf(node.column),
+        node.groupByNodes,
+      ),
+    );
+  }
+  row.metadata = metadata;
+  return row;
+}
+
+function _findInsertIndex(
+  newRow: DataCubeGridClientRowData,
+  existing: DataCubeGridClientRowData[],
+  dimensionalNodes: DataCubeDimensionalNode[],
+): number {
+  const metadata = guaranteeType(
+    newRow.metadata,
+    Map<string, DataCubeDimensionalMetadata>,
+  );
+  const groupByNodes = [...metadata.values()].flatMap((m) => m.groupByNodes);
+  const dimensions = dimensionalNodes.map((d) => d.dimension);
+  const latest = _latestGroupByFilters(groupByNodes);
+
+  for (let i = existing.length - 1; i >= 0; i--) {
+    const existingRow = guaranteeNonNullable(existing[i]);
+    if (
+      _isParentRow(existingRow, groupByNodes.length, newRow, latest, dimensions)
+    ) {
+      return i + 1;
+    }
+  }
+
+  return existing.length;
+}
+
+function _isParentRow(
+  existing: DataCubeGridClientRowData,
+  groupByCount: number,
+  newRow: DataCubeGridClientRowData,
+  filters: DataCubeDimensionalGroupByNode[],
+  dimensions: string[],
+): boolean {
+  if (groupByCount === 0) {
+    let allCount = 0;
+    const match = dimensions.every((d) => {
+      const e = existing[d],
+        n = newRow[d];
+      if (e === null || n === null) {
+        return false;
+      }
+      if (String(e) === DIMENSIONAL_L0_COLUMN) {
+        allCount++;
+        return true;
+      }
+      return String(e) === String(n);
+    });
+    return match && allCount >= 1 && allCount <= dimensions.length;
+  }
+
+  return dimensions.every((d) => {
+    const filter = filters.find((f) => f.dimension === d);
+    const e = existing[d],
+      n = newRow[d];
+    if (e === null || n === null) {
+      return false;
+    }
+    if (filter) {
+      return (
+        String(e) === String(filter.filter) ||
+        String(e) === String(n) ||
+        String(e) === DIMENSIONAL_L0_COLUMN
+      );
+    }
+    return String(e) === String(n);
+  });
+}
+
+function _latestGroupByFilters(
+  nodes: DataCubeDimensionalGroupByNode[],
+): DataCubeDimensionalGroupByNode[] {
+  const map = new Map<string, DataCubeDimensionalGroupByNode>();
+  for (const node of nodes) {
+    map.set(node.dimension, node);
+  }
+  return [...map.values()];
+}
+
 async function getCastColumns(
   currentSnapshot: DataCubeSnapshot,
   view: DataCubeViewState,
@@ -302,6 +457,113 @@ async function getCastColumns(
   });
 }
 
+export async function fetchDimensionalQueryRows(
+  snapshot: DataCubeSnapshot,
+  view: DataCubeViewState,
+  previousTree?: DataCubeDimensionalTree,
+) {
+  const task = view.taskService.newTask('Fetching data...');
+
+  let rowData: DataCubeGridClientRowData[] = [];
+
+  const configuration = DataCubeConfiguration.serialization.fromJson(
+    snapshot.data.configuration,
+  );
+
+  // upgrade here by not shooting extra sqls
+  let paths = generateDimensionalPaths(
+    guaranteeNonNullable(snapshot.data.dimensionalTree),
+  );
+
+  if (previousTree) {
+    const oldPaths = generateDimensionalPaths(previousTree);
+    if (oldPaths.length <= paths.length) {
+      rowData = view.grid.client.getGridOption(
+        DataCubeClientModelOption.ROW_DATA,
+      ) as DataCubeGridClientRowData[];
+      paths = findExtraPaths(oldPaths, paths);
+    }
+  }
+
+  for (const path of paths) {
+    const executableQuery = buildDimensionalGridDataFetchExecutableQuery(
+      snapshot,
+      view.source,
+      view.engine,
+      path,
+    );
+    let result: DataCubeExecutionResult;
+
+    try {
+      result = await view.engine.executeQuery(
+        _lambda([], [executableQuery]),
+        view.source,
+        {
+          debug: view.settingService.getBooleanValue(
+            DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
+          ),
+          clientVersion: view.settingService.getBooleanValue(
+            DataCubeSettingKey.DEBUGGER__USE_DEV_CLIENT_PROTOCOL_VERSION,
+          )
+            ? PureClientVersion.VX_X_X
+            : undefined,
+        },
+      );
+
+      rowData = buildDimensionalRowData(
+        result.result.result,
+        snapshot,
+        path,
+        rowData,
+        configuration.dimensions.dimensions,
+      );
+    } catch (error) {
+      assertErrorThrown(error);
+      if (error instanceof DataCubeExecutionError) {
+        view.alertService.alertExecutionError(error, {
+          message: `Data Fetch Failure: Can't execute query.`,
+          text: `Error: ${error.message}`,
+        });
+      } else {
+        view.alertService.alertError(error, {
+          message: `Data Fetch Failure: ${error.message}`,
+        });
+      }
+      view.taskService.endTask(task);
+      return {
+        rowData: [],
+        columnDefs: [],
+      };
+    }
+
+    if (
+      view.settingService.getBooleanValue(
+        DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
+      )
+    ) {
+      view.engine.debugProcess(
+        `Execution`,
+        ['Query', result.executedQuery],
+        [
+          'Stats',
+          `${rowData.length} rows, ${result.result.result.columns.length} columns`,
+        ],
+        [
+          'SQL',
+          result.executedSQL ?? `-- Error: failed to extract executed SQL`,
+        ],
+        ['SQL Execution Time', result.executionTime],
+      );
+    }
+  }
+
+  view.taskService.endTask(task);
+  return {
+    rowData: rowData,
+    columnDefs: generateColumnDefsForDimensions(snapshot, configuration),
+  };
+}
+
 export type DataCubeGridClientDataFetchRequest = {
   startRow: number | undefined;
   endRow: number | undefined;