@finos/legend-data-cube 0.0.43 → 0.0.44

package/src/stores/core/DataCubeQuerySnapshotBuilderUtils.ts

@@ -29,6 +29,7 @@ import {
   V1_ColSpec,
   V1_ColSpecArray,
   V1_GenericTypeInstance,
+  V1_Variable,
   type V1_ValueSpecification,
   V1_PrimitiveValueSpecification,
   extractElementNameFromPath as _name,
@@ -36,15 +37,18 @@ import {
   V1_RelationType,
   V1_PackageableType,
   type V1_GenericType,
+  V1_Collection,
+  type V1_Lambda,
 } from '@finos/legend-graph';
-import { type DataCubeColumn } from './model/DataCubeColumn.js';
+import { _findCol, type DataCubeColumn } from './model/DataCubeColumn.js';
 import {
+  assertErrorThrown,
   assertTrue,
   assertType,
   at,
+  deepEqual,
   guaranteeNonNullable,
   guaranteeType,
-  IllegalStateError,
   uniq,
   UnsupportedOperationError,
   type Clazz,
@@ -53,19 +57,60 @@ import {
   DataCubeFunction,
   DataCubeOperationAdvancedValueType,
   DataCubeQueryFilterGroupOperator,
+  DataCubeQuerySortDirection,
+  DEFAULT_LAMBDA_VARIABLE_NAME,
   getDataType,
+  getPivotResultColumnBaseColumnName,
+  isPivotResultColumnName,
   TREE_COLUMN_VALUE_SEPARATOR,
   type DataCubeOperationValue,
 } from './DataCubeQueryEngine.js';
 import type {
+  DataCubeQuerySnapshotAggregateColumn,
   DataCubeQuerySnapshotFilter,
   DataCubeQuerySnapshotFilterCondition,
+  DataCubeQuerySnapshotGroupBy,
+  DataCubeQuerySnapshotPivot,
 } from './DataCubeQuerySnapshot.js';
-import { _serializeValueSpecification } from './DataCubeQueryBuilderUtils.js';
 import type { DataCubeQueryFilterOperation } from './filter/DataCubeQueryFilterOperation.js';
+import type { DataCubeEngine } from './DataCubeEngine.js';
+import {
+  _cols,
+  _colSpec,
+  _function,
+  _lambda,
+  _serializeValueSpecification,
+  _synthesizeMinimalSourceQuery,
+} from './DataCubeQueryBuilderUtils.js';
+import { INTERNAL__DataCubeSource } from './model/DataCubeSource.js';
+import type { DataCubeQueryAggregateOperation } from './aggregation/DataCubeQueryAggregateOperation.js';
 
 // --------------------------------- UTILITIES ---------------------------------
 
+export function _var(variable: V1_Variable) {
+  assertTrue(
+    variable.name === DEFAULT_LAMBDA_VARIABLE_NAME,
+    `Can't process variable '${variable.name}': expected variable name to be '${DEFAULT_LAMBDA_VARIABLE_NAME}'`,
+  );
+}
+
+export function _propertyCol(
+  property: V1_AppliedProperty,
+  columnGetter: (name: string) => DataCubeColumn,
+) {
+  assertTrue(
+    property.parameters.length === 1,
+    `Can't process property '${property.property}': expected exactly 1 parameter`,
+  );
+  const variable = guaranteeType(
+    at(property.parameters, 0),
+    V1_Variable,
+    `Can't process property '${property.property}': failed to extract variable`,
+  );
+  _var(variable);
+  return columnGetter(property.property);
+}
+
 export function _param<T extends V1_ValueSpecification>(
   func: V1_AppliedFunction,
   paramIdx: number,
@@ -74,13 +119,13 @@ export function _param<T extends V1_ValueSpecification>(
 ): T {
   assertTrue(
     func.parameters.length >= paramIdx + 1,
-    `Can't process ${_name(func.function)}() expression: Expected at least ${paramIdx + 1} parameter(s)`,
+    `Can't process ${_name(func.function)}() expression: expected at least ${paramIdx + 1} parameter(s)`,
   );
   return guaranteeType(
     func.parameters[paramIdx],
     clazz,
     message ??
-      `Can't process ${_name(func.function)}() expression: Found unexpected type for parameter at index ${paramIdx}`,
+      `Can't process ${_name(func.function)}() expression: found unexpected type for parameter at index ${paramIdx}`,
   );
 }
 
@@ -88,7 +133,7 @@ export function _colSpecParam(func: V1_AppliedFunction, paramIdx: number) {
   return guaranteeType(
     _param(func, paramIdx, V1_ClassInstance).value,
     V1_ColSpec,
-    `Can't process ${_name(func.function)}() expression: Expected parameter at index ${paramIdx} to be a column specification`,
+    `Can't process ${_name(func.function)}() expression: expected parameter at index ${paramIdx} to be a column specification`,
   );
 }
 
@@ -96,7 +141,7 @@ export function _colSpecArrayParam(func: V1_AppliedFunction, paramIdx: number) {
   return guaranteeType(
     _param(func, paramIdx, V1_ClassInstance).value,
     V1_ColSpecArray,
-    `Can't process ${_name(func.function)}() expression: Expected parameter at index ${paramIdx} to be a column specification list`,
+    `Can't process ${_name(func.function)}() expression: expected parameter at index ${paramIdx} to be a column specification list`,
   );
 }
 
@@ -105,10 +150,23 @@ export function _genericTypeParam(func: V1_AppliedFunction, paramIdx: number) {
     func,
     paramIdx,
     V1_GenericTypeInstance,
-    `Can't process ${_name(func.function)}: Expected parameter at index ${paramIdx} to be a generic type instance`,
+    `Can't process ${_name(func.function)}: expected parameter at index ${paramIdx} to be a generic type instance`,
   );
 }
 
+export function _unwrapLambda(lambda: V1_Lambda, message?: string | undefined) {
+  assertTrue(
+    lambda.body.length === 1,
+    `${message ?? `Can't process lambda`}: expected lambda body to have exactly 1 expression`,
+  );
+  assertTrue(
+    lambda.parameters.length === 1,
+    `${message ?? `Can't process lambda`}: expected lambda to have exactly 1 parameter`,
+  );
+  _var(at(lambda.parameters, 0));
+  return at(lambda.body, 0);
+}
+
 export function _funcMatch(
   value: V1_ValueSpecification | undefined,
   functionNames: string | string[],
@@ -116,14 +174,14 @@ export function _funcMatch(
   assertType(
     value,
     V1_AppliedFunction,
-    `Can't process function: Found unexpected value specification type`,
+    `Can't process function: found unexpected value specification type`,
   );
   assertTrue(
     matchFunctionName(
       value.function,
       Array.isArray(functionNames) ? functionNames : [functionNames],
     ),
-    `Can't process function: Expected function to be one of [${uniq((Array.isArray(functionNames) ? functionNames : [functionNames]).map(_name)).join(', ')}]`,
+    `Can't process function: expected function to be one of [${uniq((Array.isArray(functionNames) ? functionNames : [functionNames]).map(_name)).join(', ')}]`,
   );
   return value;
 }
@@ -164,7 +222,50 @@ export function _operationPrimitiveValue(
   } else if (value instanceof V1_CStrictTime) {
     return { value: value.value, type: PRIMITIVE_TYPE.STRICTTIME };
   }
-  throw new UnsupportedOperationError(`Can't process primitive value`);
+  throw new UnsupportedOperationError(
+    `Can't process unsupported operation primitive value`,
+  );
+}
+
+export function _operationValue(
+  value: V1_ValueSpecification | undefined,
+  columnGetter: (name: string) => DataCubeColumn,
+  columnChecker?: ((column: DataCubeColumn) => void) | undefined,
+) {
+  if (value instanceof V1_PrimitiveValueSpecification) {
+    return _operationPrimitiveValue(value);
+  } else if (value instanceof V1_AppliedProperty) {
+    const column = _propertyCol(value, columnGetter);
+    columnChecker?.(column);
+    return {
+      value: column.name,
+      type: DataCubeOperationAdvancedValueType.COLUMN,
+    };
+  } else if (value === undefined) {
+    return {
+      type: DataCubeOperationAdvancedValueType.VOID,
+    };
+  }
+  throw new UnsupportedOperationError(
+    `Can't process unsupported operation value`,
+  );
+}
+
+export function _checkDuplicateColumns(
+  columns: DataCubeColumn[],
+  message?: ((colName: string) => string) | undefined,
+) {
+  const cols = new Set<string>();
+  columns.forEach((col) => {
+    if (cols.has(col.name)) {
+      throw new Error(
+        message?.(col.name) ??
+          `Can't process expression: found duplicate columns '${col.name}'`,
+      );
+    } else {
+      cols.add(col.name);
+    }
+  });
 }
 
 // --------------------------------- BUILDING BLOCKS ---------------------------------
@@ -186,10 +287,8 @@ export function _pruneExpandedPaths(
   let lastCommonIndex = -1;
   for (let i = 0; i < length; i++) {
     if (
-      guaranteeNonNullable(prevGroupByCols[i]).name !==
-        guaranteeNonNullable(currentGroupByCols[i]).name ||
-      guaranteeNonNullable(prevGroupByCols[i]).type !==
-        guaranteeNonNullable(currentGroupByCols[i]).type
+      at(prevGroupByCols, i).name !== at(currentGroupByCols, i).name ||
+      at(prevGroupByCols, i).type !== at(currentGroupByCols, i).type
     ) {
       break;
     }
@@ -203,58 +302,74 @@ export function _pruneExpandedPaths(
     .sort();
 }
 
-export function _extractExtendedColumns(func: V1_AppliedFunction) {
-  const extendFuncs: V1_AppliedFunction[] = [];
-  let currentFunc = func;
-
-  while (currentFunc instanceof V1_AppliedFunction) {
-    // since we are processing a chain of extend(), we can finish processing when
-    // encountering a different function.
-    if (!matchFunctionName(currentFunc.function, DataCubeFunction.EXTEND)) {
-      break;
-    }
-
-    if (currentFunc.parameters.length === 2) {
-      const valueSpecification = currentFunc.parameters[0];
-      if (!(valueSpecification instanceof V1_AppliedFunction)) {
-        throw new IllegalStateError(
-          `Can't process extend() expression: Expected a chain of function calls (e.g. x()->y()->z())`,
-        );
-      } else {
-        currentFunc.parameters = currentFunc.parameters.slice(1);
-        extendFuncs.unshift(currentFunc);
-        currentFunc = valueSpecification;
-      }
-    } else {
-      assertTrue(
-        currentFunc.parameters.length === 1,
-        `Can't process extend() expression: Expected 1 parameter, got ${currentFunc.parameters.length}`,
-      );
-      extendFuncs.unshift(currentFunc);
-      break;
-    }
-  }
-  return extendFuncs.map((extendFunc) => {
-    const colSpecs = _colSpecArrayParam(extendFunc, 0).colSpecs;
+export async function _extractExtendedColumns(
+  funcs: V1_AppliedFunction[],
+  currentColumns: DataCubeColumn[],
+  engine: DataCubeEngine,
+) {
+  const colSpecs = funcs.map((extendFunc) => {
+    // TODO: support extend() with window (OLAP), this assertion will no longer work
+    const _colSpecs = _colSpecArrayParam(extendFunc, 0).colSpecs;
     assertTrue(
-      colSpecs.length === 1,
-      `Can't process extend() expression: Expected 1 column specification, got ${colSpecs.length}`,
+      _colSpecs.length === 1,
+      `Can't process extend() expression: expected 1 column specification, got ${_colSpecs.length}`,
     );
-    const colSpec = at(colSpecs, 0);
-    return {
-      name: colSpec.name,
-      type: '', // NOTE: we don't have type information for extended columns at this point
-      mapFn: _serializeValueSpecification(
-        guaranteeNonNullable(
-          colSpec.function1,
-          `Can't process extend() expression: Expected a transformation function expression`,
+    return at(_colSpecs, 0);
+  });
+
+  // get the types
+  const sourceQuery = _synthesizeMinimalSourceQuery(currentColumns);
+  const sequence = colSpecs.map((colSpec) =>
+    _function(DataCubeFunction.EXTEND, [
+      _cols([
+        _colSpec(
+          colSpec.name,
+          guaranteeNonNullable(
+            colSpec.function1,
+            `Can't process extend() expression: expected a transformation function expression for column '${colSpec.name}'`,
+          ),
+          colSpec.function2,
         ),
+      ]),
+    ]),
+  );
+  for (let i = 0; i < sequence.length; i++) {
+    at(sequence, i).parameters.unshift(
+      i === 0 ? sourceQuery : at(sequence, i - 1),
+    );
+  }
+  const query = at(sequence, sequence.length - 1);
+  let columns: DataCubeColumn[] = [];
+  try {
+    columns = (
+      await engine.getQueryRelationReturnType(
+        _lambda([], [query]),
+        new INTERNAL__DataCubeSource(),
+      )
+    ).columns;
+  } catch (error) {
+    assertErrorThrown(error);
+    throw new Error(
+      `Can't process extend() expression: failed to retrieve type information for columns. Error: ${error.message}`,
+    );
+  }
+
+  return colSpecs.map((colSpec) => ({
+    name: colSpec.name,
+    type: guaranteeNonNullable(
+      _findCol(columns, colSpec.name),
+      `Can't process extend() expression: failed to retrieve type information for column '${colSpec.name}'`,
+    ).type,
+    mapFn: _serializeValueSpecification(
+      guaranteeNonNullable(
+        colSpec.function1,
+        `Can't process extend() expression: expected a transformation function expression for column '${colSpec.name}'`,
       ),
-      reduceFn: colSpec.function2
-        ? _serializeValueSpecification(colSpec.function2)
-        : undefined,
-    };
-  });
+    ),
+    reduceFn: colSpec.function2
+      ? _serializeValueSpecification(colSpec.function2)
+      : undefined,
+  }));
 }
 
 export function _filter(
@@ -264,7 +379,7 @@ export function _filter(
 ): DataCubeQuerySnapshotFilter {
   if (!(value instanceof V1_AppliedFunction)) {
     throw new Error(
-      `Can't process filter() expression: Expected a function expression`,
+      `Can't process filter() expression: expected a function expression`,
     );
   }
 
@@ -303,7 +418,7 @@ export function _unwrapNotFilterCondition(func: V1_AppliedFunction) {
   );
   assertTrue(
     func.parameters.length === 1,
-    `Can't process not() function: Expected 1 parameter`,
+    `Can't process not() function: expected 1 parameter`,
   );
   return _param(func, 0, V1_AppliedFunction);
 }
@@ -315,7 +430,7 @@ function _filterCondition(
 ): DataCubeQuerySnapshotFilterCondition | DataCubeQuerySnapshotFilter {
   if (!(value instanceof V1_AppliedFunction)) {
     throw new UnsupportedOperationError(
-      `Can't process filter condition expression: Expected a function expression`,
+      `Can't process filter condition expression: expected a function expression`,
     );
   }
 
@@ -375,31 +490,6 @@ function _filterCondition(
   throw new Error(`Can't process filter condition: no matching operator found`);
 }
 
-function _filterConditionValue(
-  value: V1_ValueSpecification | undefined,
-  column: DataCubeColumn,
-  columnGetter: (name: string) => DataCubeColumn,
-) {
-  if (value instanceof V1_PrimitiveValueSpecification) {
-    return _operationPrimitiveValue(value);
-  } else if (value instanceof V1_AppliedProperty) {
-    const column2 = columnGetter(value.property);
-    if (getDataType(column.type) !== getDataType(column2.type)) {
-      return undefined;
-    }
-    return {
-      value: column2.name,
-      type: DataCubeOperationAdvancedValueType.COLUMN,
-    };
-  } else if (value === undefined) {
-    return {
-      type: DataCubeOperationAdvancedValueType.VOID,
-    };
-  }
-
-  return undefined;
-}
-
 /**
  * Processes filter conditions of form: column | operator | value, e.g.
  * $x.Age > 5
@@ -408,40 +498,50 @@ function _filterConditionValue(
  * $x.Age > $x.Age2
  * $x.Name == $x.Name2
  */
-export function _baseFilterCondition(
-  expression: V1_AppliedFunction,
-  columnGetter: (name: string) => DataCubeColumn,
+export function _filterCondition_base(
+  expression: V1_AppliedFunction | undefined,
   func: string,
+  columnGetter: (name: string) => DataCubeColumn,
 ) {
-  if (matchFunctionName(expression.function, func)) {
-    if (
-      expression.parameters.length !== 2 &&
-      expression.parameters.length !== 1
-    ) {
-      return undefined;
-    }
+  if (!expression) {
+    return undefined;
+  }
+  try {
+    if (matchFunctionName(expression.function, func)) {
+      if (
+        expression.parameters.length !== 2 &&
+        expression.parameters.length !== 1
+      ) {
+        return undefined;
+      }
 
-    const column =
-      expression.parameters[0] instanceof V1_AppliedProperty
-        ? columnGetter(expression.parameters[0].property)
-        : undefined;
-    if (!column) {
-      return undefined;
-    }
+      let column: DataCubeColumn | undefined;
+      if (expression.parameters[0] instanceof V1_AppliedProperty) {
+        column = _propertyCol(expression.parameters[0], columnGetter);
+      }
+      if (!column) {
+        return undefined;
+      }
 
-    const value = _filterConditionValue(
-      expression.parameters[1],
-      column,
-      columnGetter,
-    );
-    if (!value) {
-      return undefined;
-    }
+      const value = _operationValue(
+        expression.parameters[1],
+        columnGetter,
+        (_column) => {
+          if (getDataType(column.type) !== getDataType(_column.type)) {
+            throw new Error(
+              `Can't process filter condition: found incompatible columns`,
+            );
+          }
+        },
+      );
 
-    return {
-      column,
-      value,
-    };
+      return {
+        column,
+        value,
+      };
+    }
+  } catch {
+    return undefined;
   }
   return undefined;
 }
@@ -451,58 +551,398 @@ export function _baseFilterCondition(
  * $x.Name->toLower() == 'abc'->toLower()
  * $x.Name->toLower() == $x.Name2->toLower()
  */
-export function _caseSensitiveBaseFilterCondition(
-  expression: V1_AppliedFunction,
+export function _filterCondition_caseSensitive(
+  expression: V1_AppliedFunction | undefined,
+  func: string,
   columnGetter: (name: string) => DataCubeColumn,
+) {
+  if (!expression) {
+    return undefined;
+  }
+  try {
+    if (matchFunctionName(expression.function, func)) {
+      if (expression.parameters.length !== 2) {
+        return undefined;
+      }
+
+      const param1 = expression.parameters[0];
+      if (
+        !(param1 instanceof V1_AppliedFunction) ||
+        !matchFunctionName(param1.function, DataCubeFunction.TO_LOWERCASE)
+      ) {
+        return undefined;
+      }
+      if (param1.parameters.length !== 1) {
+        return undefined;
+      }
+      let column: DataCubeColumn | undefined;
+      if (param1.parameters[0] instanceof V1_AppliedProperty) {
+        column = _propertyCol(param1.parameters[0], columnGetter);
+      }
+      if (!column) {
+        return undefined;
+      }
+
+      const param2 = expression.parameters[1];
+      if (
+        !(param2 instanceof V1_AppliedFunction) ||
+        !matchFunctionName(param2.function, DataCubeFunction.TO_LOWERCASE)
+      ) {
+        return undefined;
+      }
+      if (param2.parameters.length !== 1) {
+        return undefined;
+      }
+      const value = _operationValue(
+        param2.parameters[0],
+        columnGetter,
+        (_column) => {
+          if (getDataType(column.type) !== getDataType(_column.type)) {
+            throw new Error(
+              `Can't process filter condition: found incompatible columns`,
+            );
+          }
+        },
+      );
+
+      return {
+        column,
+        value,
+      };
+    }
+  } catch {
+    return undefined;
+  }
+  return undefined;
+}
+
+export function _aggCol(
+  colSpec: V1_ColSpec,
+  columnGetter: (name: string) => DataCubeColumn,
+  aggregateOperations: DataCubeQueryAggregateOperation[],
+) {
+  for (const operation of aggregateOperations) {
+    const col = operation.buildAggregateColumnSnapshot(colSpec, columnGetter);
+    if (col) {
+      return col;
+    }
+  }
+  throw new Error(
+    `Can't process aggregate column '${colSpec.name}': no matching operator found`,
+  );
+}
+
+export function _agg_base(
+  colSpec: V1_ColSpec,
   func: string,
+  columnGetter: (name: string) => DataCubeColumn,
 ) {
-  if (matchFunctionName(expression.function, func)) {
-    if (expression.parameters.length !== 2) {
-      return undefined;
+  try {
+    if (colSpec.function1 && colSpec.function2) {
+      const mapper = _unwrapLambda(colSpec.function1);
+      const reducer = _unwrapLambda(colSpec.function2);
+
+      if (
+        mapper instanceof V1_AppliedProperty &&
+        reducer instanceof V1_AppliedFunction &&
+        reducer.parameters.length >= 1 &&
+        matchFunctionName(reducer.function, func)
+      ) {
+        const column = _propertyCol(mapper, columnGetter);
+        assertTrue(
+          column.name === colSpec.name,
+          `Can't process aggregate column: column name must match mapper column name`,
+        );
+        const variable = _param(reducer, 0, V1_Variable);
+        _var(variable);
+
+        return {
+          column,
+          paramterValues: reducer.parameters
+            .slice(1)
+            .map((value) => _operationValue(value, columnGetter)),
+        };
+      }
     }
+  } catch {
+    return undefined;
+  }
+  return undefined;
+}
 
-    const param1 = expression.parameters[0];
-    if (
-      !(param1 instanceof V1_AppliedFunction) ||
-      !matchFunctionName(param1.function, DataCubeFunction.TO_LOWERCASE)
-    ) {
-      return undefined;
+export function _pivotSort(
+  func: V1_AppliedFunction,
+  pivotColumns: DataCubeColumn[],
+  columnGetter: (name: string) => DataCubeColumn,
+) {
+  const sortColumns = _sort(func, columnGetter);
+  const groupColumns = new Set(pivotColumns.map((col) => col.name));
+  const columnsToSort = new Set(pivotColumns.map((col) => col.name));
+  sortColumns.forEach((col) => {
+    if (groupColumns.has(col.name)) {
+      columnsToSort.delete(col.name);
+    } else {
+      throw new Error(
+        `Can't process pivot() expression: sort column '${col.name}' must be a pivot column`,
+      );
     }
-    if (param1.parameters.length !== 1) {
-      return undefined;
+  });
+  if (columnsToSort.size !== 0) {
+    throw new Error(
+      `Can't process pivot() expression: found unsorted pivot column(s) (${Array.from(
+        columnsToSort.values(),
+      )
+        .sort()
+        .map((col) => `'${col}'`)
+        .join(', ')})`,
+    );
+  }
+
+  _checkDuplicateColumns(
+    sortColumns,
+    (colName) =>
+      `Can't process pivot() expression: found duplicate sort columns '${colName}'`,
+  );
+
+  return sortColumns;
+}
+
+export function _validatePivot(
+  pivot: DataCubeQuerySnapshotPivot,
+  pivotAggColumns: DataCubeQuerySnapshotAggregateColumn[],
+  availableColumns: DataCubeColumn[],
+) {
+  // check for duplicate columns
+  const pivotColumns = pivot.columns;
+  const castColumns = pivot.castColumns;
+  _checkDuplicateColumns(
+    pivotColumns,
+    (colName) =>
+      `Can't process pivot() expression: found duplicate pivot columns '${colName}'`,
+  );
+  _checkDuplicateColumns(
+    pivotAggColumns,
+    (colName) =>
+      `Can't process pivot() expression: found duplicate aggregate columns '${colName}'`,
+  );
+  _checkDuplicateColumns(
+    castColumns,
+    (colName) =>
+      `Can't process pivot() expression: found duplicate cast columns '${colName}'`,
+  );
+
+  // check pivot columns are not aggregated on
+  pivotAggColumns.forEach((col) => {
+    if (_findCol(pivotColumns, col.name)) {
+      throw new Error(
+        `Can't process pivot() expression: pivot column '${col.name}' must not be aggregated on`,
+      );
     }
-    const column =
-      param1.parameters[0] instanceof V1_AppliedProperty
-        ? columnGetter(param1.parameters[0].property)
-        : undefined;
-    if (!column) {
-      return undefined;
+  });
+
+  // check cast columns
+  // NOTE: we cannot and should not do strict checks here as cast columns are dependent on the data
+
+  // check that the columns used by pivot() as group columns are present in cast columns
+  const pivotGroupColumns = availableColumns.filter(
+    (col) =>
+      !(
+        _findCol(pivotColumns, col.name) ?? _findCol(pivotAggColumns, col.name)
+      ),
+  );
+  pivotGroupColumns.forEach((col) => {
+    if (!_findCol(castColumns, col.name)) {
+      throw new Error(
+        `Can't process pivot() expression: expected pivot group column '${col.name}' in cast columns`,
+      );
     }
+  });
 
-    const param2 = expression.parameters[1];
-    if (
-      !(param2 instanceof V1_AppliedFunction) ||
-      !matchFunctionName(param2.function, DataCubeFunction.TO_LOWERCASE)
-    ) {
-      return undefined;
+  // check that columns used in pivot() should not show up in cast columns
+  pivotColumns.forEach((col) => {
+    if (_findCol(castColumns, col.name)) {
+      throw new Error(
+        `Can't process pivot() expression: expected pivot column '${col.name}' to not present in cast columns`,
+      );
     }
-    if (param2.parameters.length !== 1) {
-      return undefined;
+  });
+  pivotAggColumns.forEach((col) => {
+    if (_findCol(castColumns, col.name)) {
+      throw new Error(
+        `Can't process pivot() expression: expected pivot aggregate column '${col.name}' to not present in cast columns`,
+      );
     }
-    const value = _filterConditionValue(
-      param2.parameters[0],
-      column,
-      columnGetter,
+  });
+
+  // check that cast column resulted from an aggregation (usually has name of form VAL1__|__COL1)
+  // has a matching aggregate column (i.e. COL1)
+  castColumns
+    .filter((col) => isPivotResultColumnName(col.name))
+    .forEach((col) => {
+      const aggColName = getPivotResultColumnBaseColumnName(col.name);
+      if (!_findCol(pivotAggColumns, aggColName)) {
+        throw new Error(
+          `Can't process pivot() expression: fail to match cast column '${col.name}' to a specified aggregate column`,
+        );
+      }
+    });
+}
+
+export function _groupBySort(
+  func: V1_AppliedFunction,
+  groupByColumns: DataCubeColumn[],
+  columnGetter: (name: string) => DataCubeColumn,
+) {
+  const sortColumns = _sort(func, columnGetter);
+  const groupColumns = new Set(groupByColumns.map((col) => col.name));
+  const columnsToSort = new Set(groupByColumns.map((col) => col.name));
+  let sortDirection: DataCubeQuerySortDirection | undefined = undefined;
+  sortColumns.forEach((col) => {
+    if (groupColumns.has(col.name)) {
+      columnsToSort.delete(col.name);
+    } else {
+      throw new Error(
+        `Can't process groupBy() expression: sort column '${col.name}' must be a group column`,
+      );
+    }
+
+    if (!sortDirection) {
+      sortDirection = col.direction;
+    } else if (col.direction !== sortDirection) {
+      throw new Error(
+        `Can't process groupBy() expression: all group columns must be sorted in the same direction`,
+      );
+    }
+  });
+  if (columnsToSort.size !== 0) {
+    throw new Error(
+      `Can't process groupBy() expression: found unsorted group column(s) (${Array.from(
+        columnsToSort.values(),
+      )
+        .sort()
+        .map((col) => `'${col}'`)
+        .join(', ')})`,
     );
+  }
+
+  _checkDuplicateColumns(
+    sortColumns,
+    (colName) =>
+      `Can't process groupBy() expression: found duplicate sort columns '${colName}'`,
+  );
+
+  return sortColumns;
+}
+
+export function _validateGroupBy(
+  groupBy: DataCubeQuerySnapshotGroupBy,
+  groupByAggColumns: DataCubeQuerySnapshotAggregateColumn[],
+  pivot: DataCubeQuerySnapshotPivot | undefined,
+  pivotAggColumns: DataCubeQuerySnapshotAggregateColumn[],
+  availableColumns: DataCubeColumn[],
+) {
+  // check for duplicate columns
+  const groupByColumns = groupBy.columns;
+  _checkDuplicateColumns(
+    groupByColumns,
+    (colName) =>
+      `Can't process groupBy() expression: found duplicate group columns '${colName}'`,
+  );
+  _checkDuplicateColumns(
+    groupByAggColumns,
+    (colName) =>
+      `Can't process groupBy() expression: found duplicate aggregate columns '${colName}'`,
+  );
 
-    if (!value) {
-      return undefined;
+  // check group columns are not aggregated on
+  groupByAggColumns.forEach((col) => {
+    if (_findCol(groupByColumns, col.name)) {
+      throw new Error(
+        `Can't process groupBy() expression: group column '${col.name}' must not be aggregated on`,
+      );
+    }
+  });
+
+  // check all available columns are either grouped on or aggregatd on
+  availableColumns.forEach((col) => {
+    if (
+      !(
+        _findCol(groupByColumns, col.name) ??
+        _findCol(groupByAggColumns, col.name)
+      )
+    ) {
+      throw new Error(
+        `Can't process groupBy() expression: column '${col.name}' is neither grouped nor aggregated on`,
+      );
     }
+  });
+
+  // check against pivot if present
+  if (pivot) {
+    const aggCols = new Map<string, DataCubeQuerySnapshotAggregateColumn>();
+
+    // check if aggregation specification is consistent (i.e. same type, operator, parameterValues)
+    // between groupBy aggregate columns
+    groupByAggColumns
+      .filter((col) => isPivotResultColumnName(col.name))
+      .forEach((col) => {
+        const aggColName = getPivotResultColumnBaseColumnName(col.name);
+        const aggCol = {
+          ...col,
+          name: aggColName,
+        };
+
+        const existingAggCol = aggCols.get(aggColName);
+
+        if (!existingAggCol) {
+          aggCols.set(aggColName, aggCol);
+        } else if (!deepEqual(existingAggCol, aggCol)) {
+          throw new Error(
+            `Can't process groupBy() expression: found conflicting aggregation specification for column '${aggColName}'`,
+          );
+        }
+      });
+
+    // check if pivot() aggregate columns are consistent with groupBy() aggregate columns
+    pivotAggColumns.forEach((col) => {
+      const existingAggCol = aggCols.get(col.name);
+      if (!existingAggCol) {
+        throw new Error(
+          `Can't process groupBy() expression: column '${col.name}' is aggregated in pivot() expression but not in groupBy() expression`,
+        );
+      }
+
+      if (!deepEqual(existingAggCol, col)) {
+        throw new Error(
+          `Can't process groupBy() expression: found conflicting aggregation specification for column '${col.name}'`,
+        );
+      }
+    });
+  }
+}
 
+export function _sort(
+  func: V1_AppliedFunction,
+  columnGetter: (name: string) => DataCubeColumn,
+) {
+  return _param(
+    func,
+    0,
+    V1_Collection,
+    `Can't process sort() expression: expected parameter at index 0 to be a collection`,
+  ).values.map((value) => {
+    const sortColFunc = _funcMatch(value, [
+      DataCubeFunction.ASCENDING,
+      DataCubeFunction.DESCENDING,
+    ]);
     return {
-      column,
-      value,
+      ...columnGetter(_colSpecParam(sortColFunc, 0).name),
+      direction: matchFunctionName(
+        sortColFunc.function,
+        DataCubeFunction.ASCENDING,
+      )
+        ? DataCubeQuerySortDirection.ASCENDING
+        : DataCubeQuerySortDirection.DESCENDING,
     };
-  }
-  return undefined;
+  });
 }