@finos/legend-data-cube 0.0.41 → 0.0.43

package/src/stores/core/DataCubeQueryBuilderUtils.ts

@@ -221,7 +221,10 @@ export function _collection(values: V1_ValueSpecification[]) {
   return collection;
 }
 
-export function _primitiveValue(type: string, value: unknown) {
+export function _primitiveValue(
+  type: string,
+  value: unknown,
+): V1_PrimitiveValueSpecification {
   const _val = <T extends V1_PrimitiveValueSpecification & { value: unknown }>(
     primitiveValue: T,
     val: unknown,
@@ -249,7 +252,9 @@ export function _primitiveValue(type: string, value: unknown) {
     case PRIMITIVE_TYPE.STRICTTIME:
       return _val(new V1_CStrictTime(), guaranteeIsString(value));
     default:
-      throw new UnsupportedOperationError(`Unsupported value type '${type}'`);
+      throw new UnsupportedOperationError(
+        `Can't build primitive value instance for unsupported type '${type}'`,
+      );
   }
 }
 
@@ -306,7 +311,7 @@ export function _value(
       return _property(guaranteeIsString(value.value), variable);
     default:
       throw new UnsupportedOperationError(
-        `Unsupported value type '${value.type}'`,
+        `Can't build value instance for unsupported type '${value.type}'`,
       );
   }
 }
@@ -315,14 +320,6 @@ export function _not(fn: V1_AppliedFunction) {
   return _function(_functionName(DataCubeFunction.NOT), [fn]);
 }
 
-export function _selectFunction(columns: DataCubeColumn[]) {
-  return _function(DataCubeFunction.SELECT, [
-    _cols(columns.map((col) => _colSpec(col.name))),
-  ]);
-}
-
-// --------------------------------- BUILDING BLOCKS ---------------------------------
-
 export function _col(
   name: string,
   function1?: V1_Lambda | undefined,
@@ -340,6 +337,31 @@ export function _cols(colSpecs: V1_ColSpec[]) {
   return _classInstance(V1_ClassInstanceType.COL_SPEC_ARRAY, colSpecArray);
 }
 
+// --------------------------------- BUILDING BLOCKS ---------------------------------
+
+export function _selectFunction(columns: DataCubeColumn[]) {
+  return _function(DataCubeFunction.SELECT, [
+    _cols(columns.map((col) => _colSpec(col.name))),
+  ]);
+}
+
+export function _extendRootAggregation(columnName: string) {
+  return _function(DataCubeFunction.EXTEND, [
+    _col(
+      columnName,
+      _lambda(
+        [_var()],
+        [
+          _primitiveValue(
+            PRIMITIVE_TYPE.STRING,
+            DEFAULT_ROOT_AGGREGATION_COLUMN_VALUE,
+          ),
+        ],
+      ),
+    ),
+  ]);
+}
+
 // NOTE: this is the column name used for the dummy count() aggregate
 // when no aggregate is specified in groupBy() or pivot()
 const INTERNAL__FILLER_COUNT_AGG_COLUMN_NAME =
@@ -393,7 +415,7 @@ export function _pivotAggCols(
       const aggCol = operation?.buildAggregateColumn(agg);
       if (!aggCol) {
         throw new UnsupportedOperationError(
-          `Unsupported aggregate operation '${agg.aggregateOperator}'`,
+          `Can't build aggregate column for unsupported operator '${agg.aggregateOperator}'`,
         );
       }
       return aggCol;
@@ -443,7 +465,7 @@ export function _groupByAggCols(
         const aggCol = operation?.buildAggregateColumn(agg);
         if (!aggCol) {
           throw new UnsupportedOperationError(
-            `Unsupported aggregate operation '${agg.aggregateOperator}'`,
+            `Can't build aggregate column for unsupported operator '${agg.aggregateOperator}'`,
           );
         }
         return aggCol;
@@ -484,7 +506,7 @@ export function _groupByAggCols(
         const aggCol = operation?.buildAggregateColumn(columnConfiguration);
         if (!aggCol) {
           throw new UnsupportedOperationError(
-            `Unsupported aggregate operation '${columnConfiguration.aggregateOperator}'`,
+            `Can't build aggregate column for unsupported operator '${columnConfiguration.aggregateOperator}'`,
           );
         }
         return aggCol;
@@ -505,7 +527,7 @@ export function _groupByAggCols(
         const aggCol = operation?.buildAggregateColumn(columnConfiguration);
         if (!aggCol) {
           throw new UnsupportedOperationError(
-            `Unsupported aggregate operation '${columnConfiguration.aggregateOperator}'`,
+            `Can't build aggregate column for unsupported operator '${columnConfiguration.aggregateOperator}'`,
           );
         }
         return aggCol;
@@ -513,23 +535,6 @@ export function _groupByAggCols(
   ]);
 }
 
-export function _extendRootAggregation(columnName: string) {
-  return _function(DataCubeFunction.EXTEND, [
-    _col(
-      columnName,
-      _lambda(
-        [_var()],
-        [
-          _primitiveValue(
-            PRIMITIVE_TYPE.STRING,
-            DEFAULT_ROOT_AGGREGATION_COLUMN_VALUE,
-          ),
-        ],
-      ),
-    ),
-  ]);
-}
-
 export function _filter(
   filter: DataCubeQuerySnapshotFilter | DataCubeQuerySnapshotFilterCondition,
   filterOperations: DataCubeQueryFilterOperation[],
@@ -559,7 +564,7 @@ export function _filter(
     const condition = operation?.buildConditionExpression(filterCondition);
     if (!condition) {
       throw new UnsupportedOperationError(
-        `Unsupported filter operation '${filterCondition.operator}'`,
+        `Can't build filter condition expression for unsupported operator '${filterCondition.operator}'`,
       );
     }
     return filterCondition.not ? _not(condition) : condition;

package/src/stores/core/DataCubeQueryEngine.ts

@@ -142,11 +142,27 @@ export enum DataCubeColumnKind {
 
 export enum DataCubeOperationAdvancedValueType {
   COLUMN = 'COLUMN',
+  VOID = 'VOID',
   // PARAMETER
 }
 
+export function isPrimitiveType(type: string) {
+  return (
+    [
+      PRIMITIVE_TYPE.NUMBER,
+      PRIMITIVE_TYPE.INTEGER,
+      PRIMITIVE_TYPE.DECIMAL,
+      PRIMITIVE_TYPE.FLOAT,
+      PRIMITIVE_TYPE.DATE,
+      PRIMITIVE_TYPE.STRICTDATE,
+      PRIMITIVE_TYPE.DATETIME,
+      PRIMITIVE_TYPE.STRING,
+    ] as string[]
+  ).includes(type);
+}
+
 export type DataCubeOperationValue = {
-  value: unknown;
+  value?: unknown;
   type: string;
 };
 

package/src/stores/core/DataCubeQuerySnapshot.ts

@@ -31,7 +31,7 @@ import type {
 import type { DataCubeColumn } from './model/DataCubeColumn.js';
 
 export type DataCubeQuerySnapshotFilterCondition = DataCubeColumn & {
-  value: DataCubeOperationValue | undefined;
+  value: DataCubeOperationValue;
   operator: string;
   not?: boolean | undefined;
 };

package/src/stores/core/DataCubeQuerySnapshotBuilder.ts

@@ -25,20 +25,16 @@
 import {
   V1_AppliedFunction,
   V1_CInteger,
-  type V1_ColSpec,
   V1_Collection,
   extractElementNameFromPath as _name,
   matchFunctionName,
   type V1_ValueSpecification,
+  V1_Lambda,
 } from '@finos/legend-graph';
 import type { DataCubeQuery } from './model/DataCubeQuery.js';
 import { DataCubeQuerySnapshot } from './DataCubeQuerySnapshot.js';
 import { _toCol, type DataCubeColumn } from './model/DataCubeColumn.js';
-import {
-  assertType,
-  getNonNullableEntry,
-  guaranteeNonNullable,
-} from '@finos/legend-shared';
+import { assertTrue, at, guaranteeNonNullable } from '@finos/legend-shared';
 import {
   DataCubeQuerySortDirection,
   DataCubeFunction,
@@ -46,15 +42,19 @@ import {
 } from './DataCubeQueryEngine.js';
 import { buildDefaultConfiguration } from './DataCubeConfigurationBuilder.js';
 import {
-  _buildFilterSnapshot,
   _colSpecArrayParam,
   _colSpecParam,
   _funcMatch,
-  _lambdaParam,
   _param,
+  _extractExtendedColumns,
+  _filter,
+  _relationType,
+  _genericTypeParam,
+  _packageableType,
 } from './DataCubeQuerySnapshotBuilderUtils.js';
 import type { DataCubeSource } from './model/DataCubeSource.js';
 import type { DataCubeQueryFilterOperation } from './filter/DataCubeQueryFilterOperation.js';
+import type { DataCubeQueryAggregateOperation } from './aggregation/DataCubeQueryAggregateOperation.js';
 
 // --------------------------------- BUILDING BLOCKS ---------------------------------
 
@@ -102,13 +102,13 @@ enum _FUNCTION_SEQUENCE_COMPOSITION_PART {
   LIMIT = 'limit',
 }
 
-function isFunctionInValidSequence(
-  value: string,
-): value is _FUNCTION_SEQUENCE_COMPOSITION_PART {
-  return Object.values(_FUNCTION_SEQUENCE_COMPOSITION_PART).includes(
-    value as _FUNCTION_SEQUENCE_COMPOSITION_PART,
-  );
-}
+// function isFunctionInValidSequence(
+//   value: string,
+// ): value is _FUNCTION_SEQUENCE_COMPOSITION_PART {
+//   return Object.values(_FUNCTION_SEQUENCE_COMPOSITION_PART).includes(
+//     value as _FUNCTION_SEQUENCE_COMPOSITION_PART,
+//   );
+// }
 
 // This corresponds to the function sequence that we currently support:
 //
@@ -216,9 +216,7 @@ function extractFunctionMap(
 ): DataCubeQueryFunctionMap {
   // Make sure this is a sequence of function calls
   if (!(query instanceof V1_AppliedFunction)) {
-    throw new Error(
-      `Query must be a sequence of function calls (e.g. x()->y()->z())`,
-    );
+    throw new Error(`Can't process expression: Expected a function expression`);
   }
   const sequence: V1_AppliedFunction[] = [];
   let currentFunc = query;
@@ -230,34 +228,30 @@ function extractFunctionMap(
 
     // Check that all functions in sequence are supported (matching name and number of parameters)
     if (!supportedFunc) {
-      throw new Error(`Found unsupported function ${currentFunc.function}()`);
+      throw new Error(
+        `Can't process expression: Found unsupported function ${currentFunc.function}()`,
+      );
     }
+
+    // recursively unwrap the nested function expression to build the function sequence,
+    // i.e. if we have the expression x(y(z(t(...)))), we need to unwrap them and build the sequence
+    // t(...)->z()->y()->x() and simultaneously, remove the first parameter from each function for
+    // simplicity, except for the innermost function
     if (currentFunc.parameters.length > supportedFunc.parameters) {
-      const valueSpecification = currentFunc.parameters[0];
-      if (
-        !(
-          valueSpecification instanceof V1_AppliedFunction ||
-          isFunctionInValidSequence(currentFunc.function)
-        )
-      ) {
-        throw new Error(
-          `Query must be a sequence of function calls (e.g. x()->y()->z())`,
-        );
-      } else if (
-        currentFunc.function === _FUNCTION_SEQUENCE_COMPOSITION_PART.FILTER
-      ) {
-        currentFunc.parameters = currentFunc.parameters.slice(1);
-        sequence.unshift(currentFunc);
-        if (valueSpecification instanceof V1_AppliedFunction) {
-          currentFunc = valueSpecification;
-        } else {
-          break;
-        }
-      } else {
-        currentFunc.parameters = currentFunc.parameters.slice(1);
-        sequence.unshift(currentFunc);
-        currentFunc = valueSpecification as V1_AppliedFunction;
-      }
+      // assert that the supported function has the expected number of parameters
+      assertTrue(
+        currentFunc.parameters.length === supportedFunc.parameters + 1,
+        `Can't process ${_name(currentFunc.function)}() expression: Expected at most ${supportedFunc.parameters + 1} parameters provided, got ${currentFunc.parameters.length}`,
+      );
+      const func = _param(
+        currentFunc,
+        0,
+        V1_AppliedFunction,
+        `Can't process expression: Expected a sequence of function calls (e.g. x()->y()->z())`,
+      );
+      currentFunc.parameters = currentFunc.parameters.slice(1);
+      sequence.unshift(currentFunc);
+      currentFunc = func;
     } else {
       sequence.unshift(currentFunc);
       break;
@@ -273,7 +267,7 @@ function extractFunctionMap(
   );
   if (!matchResult) {
     throw new Error(
-      `Unsupported function composition ${sequence.map((fn) => `${_name(fn.function)}()`).join('->')} (supported composition: ${_FUNCTION_SEQUENCE_COMPOSITION_PATTERN.map((node) => `${'funcs' in node ? `[${node.funcs.map((childNode) => `${_name(childNode.func)}()`).join('->')}]` : `${_name(node.func)}()`}`).join('->')})`,
+      `Can't process expression: Unsupported function composition ${sequence.map((fn) => `${_name(fn.function)}()`).join('->')} (supported composition: ${_FUNCTION_SEQUENCE_COMPOSITION_PATTERN.map((node) => `${'funcs' in node ? `[${node.funcs.map((childNode) => `${_name(childNode.func)}()`).join('->')}]` : `${_name(node.func)}()`}`).join('->')})`,
     );
   }
 
@@ -286,7 +280,7 @@ function extractFunctionMap(
     if (isNaN(idx) || idx >= sequence.length) {
       return undefined;
     }
-    const func = getNonNullableEntry(sequence, idx);
+    const func = at(sequence, idx);
     return func;
   };
 
@@ -295,7 +289,7 @@ function extractFunctionMap(
     select: _process(_FUNCTION_SEQUENCE_COMPOSITION_PART.SELECT),
     filter: _process(_FUNCTION_SEQUENCE_COMPOSITION_PART.FILTER),
     pivotSort: _process(_FUNCTION_SEQUENCE_COMPOSITION_PART.PIVOT_SORT),
-    pivot: _process(_FUNCTION_SEQUENCE_COMPOSITION_PART.PIVOT_CAST),
+    pivot: _process(_FUNCTION_SEQUENCE_COMPOSITION_PART.PIVOT),
     pivotCast: _process(_FUNCTION_SEQUENCE_COMPOSITION_PART.PIVOT_CAST),
     groupBy: _process(_FUNCTION_SEQUENCE_COMPOSITION_PART.GROUP_BY),
     groupBySort: _process(_FUNCTION_SEQUENCE_COMPOSITION_PART.GROUP_BY_SORT),
@@ -317,7 +311,8 @@ export function validateAndBuildQuerySnapshot(
   partialQuery: V1_ValueSpecification,
   source: DataCubeSource,
   baseQuery: DataCubeQuery,
-  filterOperations?: DataCubeQueryFilterOperation[],
+  filterOperations: DataCubeQueryFilterOperation[],
+  aggregateOperations: DataCubeQueryAggregateOperation[],
 ) {
   // --------------------------------- BASE ---------------------------------
   // Build the function call sequence and the function map to make the
@@ -326,28 +321,74 @@ export function validateAndBuildQuerySnapshot(
   const funcMap = extractFunctionMap(partialQuery);
   const snapshot = DataCubeQuerySnapshot.create({});
   const data = snapshot.data;
+  const columnNames = new Set<string>();
+  /**
+   * We want to make sure all columns, either from source or created, e.g. extended columns,
+   * have unique names. This is to simplify the logic within DataCube so different components
+   * can easily refer to columns by name without having to worry about conflicts.
+   */
+  const _checkColName = (col: DataCubeColumn, message: string) => {
+    if (columnNames.has(col.name)) {
+      throw new Error(message);
+    }
+    columnNames.add(col.name);
+  };
   const colsMap = new Map<string, DataCubeColumn>();
-  const _col = (colSpec: V1_ColSpec) => {
+  const _getCol = (colName: string) => {
     const column = guaranteeNonNullable(
-      colsMap.get(colSpec.name),
-      `Can't find column '${colSpec.name}'`,
+      colsMap.get(colName),
+      `Can't find column '${colName}'`,
     );
     return _toCol(column);
   };
+  const _setCol = (col: DataCubeColumn) => colsMap.set(col.name, col);
+  // const _unsetCol = (col: DataCubeColumn) => colsMap.delete(col.name);
 
   // -------------------------------- SOURCE --------------------------------
 
   data.sourceColumns = source.columns;
-  data.sourceColumns.forEach((col) => colsMap.set(col.name, col));
+  data.sourceColumns.forEach((col) => _setCol(col));
+  data.sourceColumns.forEach((col) =>
+    _checkColName(
+      col,
+      `Can't process source column '${col.name}': another column with the same name is already registered`,
+    ),
+  );
 
   // --------------------------- LEAF-LEVEL EXTEND ---------------------------
-  /** TODO: @datacube roundtrip */
+
+  if (funcMap.leafExtend) {
+    // TODO: get column types (call engine to get the type)
+    data.leafExtendedColumns = _extractExtendedColumns(funcMap.leafExtend);
+    // TODO: populate the column types
+    data.leafExtendedColumns.forEach((col) => _setCol(col));
+    data.leafExtendedColumns.forEach((col) =>
+      _checkColName(
+        col,
+        `Can't process leaf-level extended column '${col.name}': another column with the same name is already registered`,
+      ),
+    );
+  }
 
   // --------------------------------- FILTER ---------------------------------
+
   if (funcMap.filter) {
-    data.filter = _buildFilterSnapshot(
-      _lambdaParam(funcMap.filter, 0).body[0]!,
-      filterOperations!,
+    // TODO: verify column presence
+    // TODO: verify column types
+    const lambda = _param(
+      funcMap.filter,
+      0,
+      V1_Lambda,
+      `Can't process filter() expression: Expected parameter at index 0 to be a lambda expression`,
+    );
+    assertTrue(
+      lambda.body.length === 1,
+      `Can't process filter() expression: Expected lambda body to have exactly 1 expression`,
+    );
+    data.filter = _filter(
+      guaranteeNonNullable(lambda.body[0]),
+      _getCol,
+      filterOperations,
     );
   }
 
@@ -355,28 +396,59 @@ export function validateAndBuildQuerySnapshot(
 
   if (funcMap.select) {
     data.selectColumns = _colSpecArrayParam(funcMap.select, 0).colSpecs.map(
-      (colSpec) => _col(colSpec),
+      (colSpec) => _getCol(colSpec.name),
     );
+    // TODO: remove columns that are not selected from colsMap
   }
 
   // --------------------------------- PIVOT ---------------------------------
-  /** TODO: @datacube roundtrip */
+
+  if (funcMap.pivot && funcMap.pivotCast) {
+    // TODO: verify column presence
+    // TODO: verify column types
+    data.pivot = {
+      columns: _colSpecArrayParam(funcMap.pivot, 0).colSpecs.map((colSpec) =>
+        _getCol(colSpec.name),
+      ),
+      castColumns: _relationType(
+        _genericTypeParam(funcMap.pivotCast, 0).genericType,
+      ).columns.map((column) => ({
+        name: column.name,
+        type: _packageableType(column.genericType).fullPath,
+      })),
+    };
+  }
   // TODO: verify groupBy agg columns, pivot agg columns and configuration agree
 
   // --------------------------------- GROUP BY ---------------------------------
 
   if (funcMap.groupBy) {
+    // TODO: verify column presence
+    // TODO: verify column types
     data.groupBy = {
       columns: _colSpecArrayParam(funcMap.groupBy, 0).colSpecs.map((colSpec) =>
-        _col(colSpec),
+        _getCol(colSpec.name),
       ),
-      // TODO: verify groupBy agg columns, pivot agg columns and configuration agree
-      // TODO: verify groupBy sort columns and configuration agree
     };
+    // TODO: verify groupBy agg columns, pivot agg columns and configuration agree
+    // TODO: verify sort columns
+    // TODO: use configuration information present in the baseQuery configuration?
   }
 
   // --------------------------- GROUP-LEVEL EXTEND ---------------------------
-  /** TODO: @datacube roundtrip */
+
+  if (funcMap.groupExtend) {
+    // TODO: get column types (call engine to get the type)
+    data.groupExtendedColumns = _extractExtendedColumns(funcMap.groupExtend);
+    // TODO: populate the column types
+    data.groupExtendedColumns.forEach((col) => _setCol(col));
+    data.groupExtendedColumns.forEach((col) =>
+      _checkColName(
+        col,
+        `Can't process group-level extended column '${col.name}': another column with the same name is already registered`,
+      ),
+    );
+  }
 
   // --------------------------------- SORT ---------------------------------
 
@@ -388,11 +460,13 @@ export function validateAndBuildQuerySnapshot(
           DataCubeFunction.DESCENDING,
         ]);
         return {
-          ..._col(_colSpecParam(sortColFunc, 0)),
-          direction:
-            _name(sortColFunc.function) === DataCubeFunction.ASCENDING
-              ? DataCubeQuerySortDirection.ASCENDING
-              : DataCubeQuerySortDirection.DESCENDING,
+          ..._getCol(_colSpecParam(sortColFunc, 0).name),
+          direction: matchFunctionName(
+            sortColFunc.function,
+            DataCubeFunction.ASCENDING,
+          )
+            ? DataCubeQuerySortDirection.ASCENDING
+            : DataCubeQuerySortDirection.DESCENDING,
         };
       },
     );
@@ -401,8 +475,12 @@ export function validateAndBuildQuerySnapshot(
   // --------------------------------- LIMIT ---------------------------------
 
   if (funcMap.limit) {
-    const value = funcMap.limit.parameters[0];
-    assertType(value, V1_CInteger);
+    const value = _param(
+      funcMap.limit,
+      0,
+      V1_CInteger,
+      `Can't process limit() expression: Expected parameter at index 0 to be an integer instance value`,
+    );
     data.limit = value.value;
   }
 
@@ -441,13 +519,8 @@ export function validateAndBuildQuerySnapshot(
 
 /**
  * TODO: @datacube roundtrip - implement the logic to reconcile the configuration with the query
- * - [ ] disallow naming an extended column the same as a source column/leaf-level extended column
- *       even if the column is not selected
  * - [ ] columns (missing/extra columns - remove or generate default column configuration)
- * - [ ] column kind
- * - [ ] column type
- * - [ ] base off the type, check the settings
- * - [ ] aggregation
+ * - [ ] base off the type and kind, check the settings to see if it's compatible
  * - [ ] verify groupBy agg columns, pivot agg columns and configuration agree
  * - [ ] verify groupBy sort columns and tree column sort direction configuration agree
  */