@finos/legend-application-query 5.2.6 → 5.2.7

package/src/stores/fetch-structure/projection/QueryBuilderProjectionValueSpecificationBuilder.ts

@@ -20,14 +20,31 @@ import {
   extractElementNameFromPath,
   GenericType,
   GenericTypeExplicitReference,
-  matchFunctionName,
   Multiplicity,
   PrimitiveInstanceValue,
   PRIMITIVE_TYPE,
   SimpleFunctionExpression,
   TYPICAL_MULTIPLICITY_TYPE,
+  type ValueSpecification,
+  INTERNAL__UnknownValueSpecification,
+  V1_serializeRawValueSpecification,
+  V1_transformRawLambda,
+  V1_GraphTransformerContextBuilder,
+  matchFunctionName,
 } from '@finos/legend-graph';
+import {
+  guaranteeNonNullable,
+  UnsupportedOperationError,
+} from '@finos/legend-shared';
 import { QUERY_BUILDER_SUPPORTED_FUNCTIONS } from '../../../QueryBuilder_Const.js';
+import { buildPropertyExpressionChain } from '../../QueryBuilderValueSpecificationBuilderHelper.js';
+import { buildGenericLambdaFunctionInstanceValue } from '../../QueryBuilderValueSpecificationHelper.js';
+import { appendPostFilter } from './post-filter/QueryBuilderPostFilterValueSpecificationBuilder.js';
+import {
+  QueryBuilderDerivationProjectionColumnState,
+  QueryBuilderSimpleProjectionColumnState,
+} from './QueryBuilderProjectionColumnState.js';
+import type { QueryBuilderProjectionState } from './QueryBuilderProjectionState.js';
 import {
   COLUMN_SORT_TYPE,
   type QueryResultSetModifierState,
@@ -64,8 +81,8 @@ const buildSortExpression = (
   return sortColumnFunction;
 };
 
-export const appendResultSetModifiers = (
-  resultModifiersState: QueryResultSetModifierState,
+const appendResultSetModifier = (
+  resultModifierState: QueryResultSetModifierState,
   lambda: LambdaFunction,
   options?:
     | {
@@ -74,30 +91,23 @@ export const appendResultSetModifiers = (
     | undefined,
 ): LambdaFunction => {
   const multiplicityOne =
-    resultModifiersState.projectionState.queryBuilderState.graphManagerState.graph.getTypicalMultiplicity(
+    resultModifierState.projectionState.queryBuilderState.graphManagerState.graph.getTypicalMultiplicity(
       TYPICAL_MULTIPLICITY_TYPE.ONE,
     );
   if (lambda.expressionSequence.length === 1) {
     const func = lambda.expressionSequence[0];
     if (func instanceof SimpleFunctionExpression) {
       if (
-        matchFunctionName(
-          func.functionName,
+        matchFunctionName(func.functionName, [
           QUERY_BUILDER_SUPPORTED_FUNCTIONS.TDS_PROJECT,
-        ) ||
-        matchFunctionName(
-          func.functionName,
           QUERY_BUILDER_SUPPORTED_FUNCTIONS.TDS_GROUP_BY,
-        ) ||
-        matchFunctionName(
-          func.functionName,
           QUERY_BUILDER_SUPPORTED_FUNCTIONS.TDS_FILTER,
-        )
+        ])
       ) {
         let currentExpression = func;
 
         // build distinct()
-        if (resultModifiersState.distinct) {
+        if (resultModifierState.distinct) {
           const distinctFunction = new SimpleFunctionExpression(
             extractElementNameFromPath(
               QUERY_BUILDER_SUPPORTED_FUNCTIONS.TDS_DISTINCT,
@@ -109,7 +119,7 @@ export const appendResultSetModifiers = (
         }
 
         // build sort()
-        if (resultModifiersState.sortColumns.length) {
+        if (resultModifierState.sortColumns.length) {
           const sortFunction = new SimpleFunctionExpression(
             extractElementNameFromPath(
               QUERY_BUILDER_SUPPORTED_FUNCTIONS.TDS_SORT,
@@ -117,26 +127,26 @@ export const appendResultSetModifiers = (
             multiplicityOne,
           );
           const multiplicity = new Multiplicity(
-            resultModifiersState.sortColumns.length,
-            resultModifiersState.sortColumns.length,
+            resultModifierState.sortColumns.length,
+            resultModifierState.sortColumns.length,
           );
           const collection = new CollectionInstanceValue(
             multiplicity,
             undefined,
           );
           collection.values =
-            resultModifiersState.sortColumns.map(buildSortExpression);
+            resultModifierState.sortColumns.map(buildSortExpression);
           sortFunction.parametersValues[0] = currentExpression;
           sortFunction.parametersValues[1] = collection;
           currentExpression = sortFunction;
         }
 
         // build take()
-        if (resultModifiersState.limit || options?.overridingLimit) {
+        if (resultModifierState.limit || options?.overridingLimit) {
           const limit = new PrimitiveInstanceValue(
             GenericTypeExplicitReference.create(
               new GenericType(
-                resultModifiersState.projectionState.queryBuilderState.graphManagerState.graph.getPrimitiveType(
+                resultModifierState.projectionState.queryBuilderState.graphManagerState.graph.getPrimitiveType(
                   PRIMITIVE_TYPE.INTEGER,
                 ),
               ),
@@ -145,7 +155,7 @@ export const appendResultSetModifiers = (
           );
           limit.values = [
             Math.min(
-              resultModifiersState.limit ?? Number.MAX_SAFE_INTEGER,
+              resultModifierState.limit ?? Number.MAX_SAFE_INTEGER,
               options?.overridingLimit ?? Number.MAX_SAFE_INTEGER,
             ),
           ];
@@ -168,3 +178,238 @@ export const appendResultSetModifiers = (
   }
   return lambda;
 };
+
+export const appendProjection = (
+  projectionState: QueryBuilderProjectionState,
+  lambdaFunction: LambdaFunction,
+  options?: {
+    /**
+     * Set queryBuilderState to `true` when we construct query for execution within the app.
+     * queryBuilderState will make the lambda function building process overrides several query values, such as the row limit.
+     */
+    isBuildingExecutionQuery?: boolean | undefined;
+    keepSourceInformation?: boolean | undefined;
+  },
+): void => {
+  const queryBuilderState = projectionState.queryBuilderState;
+  const precedingExpression = guaranteeNonNullable(
+    lambdaFunction.expressionSequence[0],
+    `Can't build projection expression: preceding expression is not defined`,
+  );
+  const multiplicityOne =
+    queryBuilderState.graphManagerState.graph.getTypicalMultiplicity(
+      TYPICAL_MULTIPLICITY_TYPE.ONE,
+    );
+  const typeString = queryBuilderState.graphManagerState.graph.getPrimitiveType(
+    PRIMITIVE_TYPE.STRING,
+  );
+
+  // build projection
+  if (projectionState.aggregationState.columns.length) {
+    // aggregation
+    const groupByFunction = new SimpleFunctionExpression(
+      extractElementNameFromPath(
+        QUERY_BUILDER_SUPPORTED_FUNCTIONS.TDS_GROUP_BY,
+      ),
+      multiplicityOne,
+    );
+
+    const colLambdas = new CollectionInstanceValue(
+      new Multiplicity(
+        projectionState.columns.length -
+          projectionState.aggregationState.columns.length,
+        projectionState.columns.length -
+          projectionState.aggregationState.columns.length,
+      ),
+    );
+    const aggregateLambdas = new CollectionInstanceValue(
+      new Multiplicity(
+        projectionState.aggregationState.columns.length,
+        projectionState.aggregationState.columns.length,
+      ),
+    );
+    const colAliases = new CollectionInstanceValue(
+      new Multiplicity(
+        projectionState.columns.length,
+        projectionState.columns.length,
+      ),
+    );
+    projectionState.columns.forEach((projectionColumnState) => {
+      // column alias
+      const colAlias = new PrimitiveInstanceValue(
+        GenericTypeExplicitReference.create(new GenericType(typeString)),
+        multiplicityOne,
+      );
+      colAlias.values.push(projectionColumnState.columnName);
+      colAliases.values.push(colAlias);
+
+      const aggregateColumnState =
+        projectionState.aggregationState.columns.find(
+          (column) => column.projectionColumnState === projectionColumnState,
+        );
+
+      // column projection
+      let columnLambda: ValueSpecification;
+      if (
+        projectionColumnState instanceof QueryBuilderSimpleProjectionColumnState
+      ) {
+        columnLambda = buildGenericLambdaFunctionInstanceValue(
+          projectionColumnState.lambdaParameterName,
+          [
+            buildPropertyExpressionChain(
+              projectionColumnState.propertyExpressionState.propertyExpression,
+              projectionColumnState.propertyExpressionState.queryBuilderState,
+            ),
+          ],
+          queryBuilderState.graphManagerState.graph,
+        );
+      } else if (
+        projectionColumnState instanceof
+        QueryBuilderDerivationProjectionColumnState
+      ) {
+        columnLambda = new INTERNAL__UnknownValueSpecification(
+          V1_serializeRawValueSpecification(
+            V1_transformRawLambda(
+              projectionColumnState.lambda,
+              new V1_GraphTransformerContextBuilder(
+                // TODO?: do we need to include the plugins here?
+                [],
+              )
+                .withKeepSourceInformationFlag(
+                  Boolean(options?.keepSourceInformation),
+                )
+                .build(),
+            ),
+          ),
+        );
+      } else {
+        throw new UnsupportedOperationError(
+          `Can't build project() column expression: unsupported projection column state`,
+          projectionColumnState,
+        );
+      }
+
+      // column aggregation
+      if (aggregateColumnState) {
+        const aggregateFunctionExpression = new SimpleFunctionExpression(
+          extractElementNameFromPath(QUERY_BUILDER_SUPPORTED_FUNCTIONS.TDS_AGG),
+          multiplicityOne,
+        );
+        const aggregateLambda = buildGenericLambdaFunctionInstanceValue(
+          aggregateColumnState.lambdaParameterName,
+          [
+            aggregateColumnState.operator.buildAggregateExpressionFromState(
+              aggregateColumnState,
+            ),
+          ],
+          aggregateColumnState.aggregationState.projectionState
+            .queryBuilderState.graphManagerState.graph,
+        );
+        aggregateFunctionExpression.parametersValues = [
+          columnLambda,
+          aggregateLambda,
+        ];
+
+        aggregateLambdas.values.push(aggregateFunctionExpression);
+      } else {
+        colLambdas.values.push(columnLambda);
+      }
+    });
+    groupByFunction.parametersValues = [
+      precedingExpression,
+      colLambdas,
+      aggregateLambdas,
+      colAliases,
+    ];
+    lambdaFunction.expressionSequence[0] = groupByFunction;
+  } else if (projectionState.columns.length) {
+    // projection
+    const projectFunction = new SimpleFunctionExpression(
+      extractElementNameFromPath(QUERY_BUILDER_SUPPORTED_FUNCTIONS.TDS_PROJECT),
+      multiplicityOne,
+    );
+    const colLambdas = new CollectionInstanceValue(
+      new Multiplicity(
+        projectionState.columns.length,
+        projectionState.columns.length,
+      ),
+    );
+    const colAliases = new CollectionInstanceValue(
+      new Multiplicity(
+        projectionState.columns.length,
+        projectionState.columns.length,
+      ),
+    );
+    projectionState.columns.forEach((projectionColumnState) => {
+      // column alias
+      const colAlias = new PrimitiveInstanceValue(
+        GenericTypeExplicitReference.create(new GenericType(typeString)),
+        multiplicityOne,
+      );
+      colAlias.values.push(projectionColumnState.columnName);
+      colAliases.values.push(colAlias);
+
+      // column projection
+      let columnLambda: ValueSpecification;
+      if (
+        projectionColumnState instanceof QueryBuilderSimpleProjectionColumnState
+      ) {
+        columnLambda = buildGenericLambdaFunctionInstanceValue(
+          projectionColumnState.lambdaParameterName,
+          [
+            buildPropertyExpressionChain(
+              projectionColumnState.propertyExpressionState.propertyExpression,
+              projectionColumnState.propertyExpressionState.queryBuilderState,
+            ),
+          ],
+          queryBuilderState.graphManagerState.graph,
+        );
+      } else if (
+        projectionColumnState instanceof
+        QueryBuilderDerivationProjectionColumnState
+      ) {
+        columnLambda = new INTERNAL__UnknownValueSpecification(
+          V1_serializeRawValueSpecification(
+            V1_transformRawLambda(
+              projectionColumnState.lambda,
+              new V1_GraphTransformerContextBuilder(
+                // TODO?: do we need to include the plugins here?
+                [],
+              )
+                .withKeepSourceInformationFlag(
+                  Boolean(options?.keepSourceInformation),
+                )
+                .build(),
+            ),
+          ),
+        );
+      } else {
+        throw new UnsupportedOperationError(
+          `Can't build project() column expression: unsupported projection column state`,
+          projectionColumnState,
+        );
+      }
+      colLambdas.values.push(columnLambda);
+    });
+    projectFunction.parametersValues = [
+      precedingExpression,
+      colLambdas,
+      colAliases,
+    ];
+    lambdaFunction.expressionSequence[0] = projectFunction;
+  }
+
+  // build post-filter
+  appendPostFilter(projectionState.postFilterState, lambdaFunction);
+
+  // build result set modifiers
+  appendResultSetModifier(
+    projectionState.resultSetModifierState,
+    lambdaFunction,
+    {
+      overridingLimit: options?.isBuildingExecutionQuery
+        ? queryBuilderState.resultState.previewLimit
+        : undefined,
+    },
+  );
+};

package/src/stores/fetch-structure/projection/aggregation/QueryBuilderAggregationStateBuilder.ts

@@ -0,0 +1,228 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  CollectionInstanceValue,
+  LambdaFunctionInstanceValue,
+  matchFunctionName,
+  SimpleFunctionExpression,
+  VariableExpression,
+} from '@finos/legend-graph';
+import {
+  assertTrue,
+  assertType,
+  guaranteeNonNullable,
+  guaranteeType,
+  isNonNullable,
+  returnUndefOnError,
+  UnsupportedOperationError,
+} from '@finos/legend-shared';
+import { QUERY_BUILDER_SUPPORTED_FUNCTIONS } from '../../../../QueryBuilder_Const.js';
+import type { QueryBuilderState } from '../../../QueryBuilderState.js';
+import { QueryBuilderValueSpecificationProcessor } from '../../../QueryBuilderStateBuilder.js';
+import { extractNullableStringFromInstanceValue } from '../../../QueryBuilderValueSpecificationHelper.js';
+import { FETCH_STRUCTURE_IMPLEMENTATION } from '../../QueryBuilderFetchStructureImplementationState.js';
+import { QueryBuilderProjectionState } from '../QueryBuilderProjectionState.js';
+
+export const processTDSAggregateExpression = (
+  expression: SimpleFunctionExpression,
+  parentExpression: SimpleFunctionExpression | undefined,
+  queryBuilderState: QueryBuilderState,
+): void => {
+  // check parent expression
+  assertTrue(
+    Boolean(
+      parentExpression &&
+        matchFunctionName(
+          parentExpression.functionName,
+          QUERY_BUILDER_SUPPORTED_FUNCTIONS.TDS_GROUP_BY,
+        ),
+    ),
+    `Can't process agg() expression: only support agg() used within a groupBy() expression`,
+  );
+
+  // check parameters
+  assertTrue(
+    expression.parametersValues.length === 2,
+    `Can't process agg() expression: agg() expects 2 arguments`,
+  );
+
+  // process column lambda
+  QueryBuilderValueSpecificationProcessor.processChild(
+    // TODO?: do we want to do more validation here for the shape of the column lambda?
+    guaranteeNonNullable(expression.parametersValues[0]),
+    expression,
+    queryBuilderState,
+  );
+
+  // build state
+  if (
+    queryBuilderState.fetchStructureState.implementation instanceof
+    QueryBuilderProjectionState
+  ) {
+    const projectionState =
+      queryBuilderState.fetchStructureState.implementation;
+    const aggregationState = projectionState.aggregationState;
+    // NOTE: since we process agg() expressions one by one, we know that the current agg()
+    // always correspond to the last column projection state, based on our processing procedure
+    const projectionColumnState = guaranteeNonNullable(
+      projectionState.columns[projectionState.columns.length - 1],
+    );
+    const aggregateLambda = expression.parametersValues[1];
+    assertType(
+      aggregateLambda,
+      LambdaFunctionInstanceValue,
+      `Can't process agg() expression: agg() expects argument #1 to be a lambda function`,
+    );
+
+    const lambdaFunc = guaranteeNonNullable(
+      aggregateLambda.values[0],
+      `Can't process agg() lambda: agg() lambda function is missing`,
+    );
+    assertTrue(
+      lambdaFunc.expressionSequence.length === 1,
+      `Can't process agg() lambda: only support agg() lambda body with 1 expression`,
+    );
+    const aggregateColumnExpression = guaranteeType(
+      lambdaFunc.expressionSequence[0],
+      SimpleFunctionExpression,
+      `Can't process agg() lambda: only support agg() lambda body with 1 expression`,
+    );
+
+    assertTrue(
+      lambdaFunc.functionType.parameters.length === 1,
+      `Can't process agg() lambda: only support agg() lambda with 1 parameter`,
+    );
+
+    const lambdaParam = guaranteeType(
+      lambdaFunc.functionType.parameters[0],
+      VariableExpression,
+      `Can't process agg() lambda: only support agg() lambda with 1 parameter`,
+    );
+
+    for (const operator of aggregationState.operators) {
+      // NOTE: this allow plugin author to either return `undefined` or throw error
+      // if there is a problem with building the lambda. Either case, the plugin is
+      // considered as not supporting the lambda.
+      const aggregateColumnState = returnUndefOnError(() =>
+        operator.buildAggregateColumnState(
+          aggregateColumnExpression,
+          lambdaParam,
+          projectionColumnState,
+        ),
+      );
+      if (aggregateColumnState) {
+        aggregationState.addColumn(aggregateColumnState);
+        return;
+      }
+    }
+    throw new UnsupportedOperationError(
+      `Can't process aggregate expression function: no compatible aggregate operator processer available from plugins`,
+    );
+  }
+};
+
+export const processTDSGroupByExpression = (
+  expression: SimpleFunctionExpression,
+  queryBuilderState: QueryBuilderState,
+): void => {
+  // update fetch-structure
+  queryBuilderState.fetchStructureState.changeImplementation(
+    FETCH_STRUCTURE_IMPLEMENTATION.PROJECTION,
+  );
+
+  // check parameters
+  assertTrue(
+    expression.parametersValues.length === 4,
+    `Can't process groupBy() expression: groupBy() expects 3 arguments`,
+  );
+
+  // check preceding expression
+  const precedingExpression = guaranteeType(
+    expression.parametersValues[0],
+    SimpleFunctionExpression,
+    `Can't process groupBy() expression: only support groupBy() immediately following an expression`,
+  );
+  assertTrue(
+    matchFunctionName(precedingExpression.functionName, [
+      QUERY_BUILDER_SUPPORTED_FUNCTIONS.GET_ALL,
+      QUERY_BUILDER_SUPPORTED_FUNCTIONS.FILTER,
+    ]),
+    `Can't process groupBy() expression: only support groupBy() immediately following either getAll() or filter()`,
+  );
+  QueryBuilderValueSpecificationProcessor.process(
+    precedingExpression,
+    queryBuilderState,
+  );
+
+  // process columns
+  const columnExpressions = expression.parametersValues[1];
+  assertType(
+    columnExpressions,
+    CollectionInstanceValue,
+    `Can't process groupBy() expression: groupBy() expects argument #1 to be a collection`,
+  );
+  columnExpressions.values.map((value) =>
+    QueryBuilderValueSpecificationProcessor.processChild(
+      value,
+      expression,
+      queryBuilderState,
+    ),
+  );
+
+  // process aggregations
+  const aggregateLambdas = expression.parametersValues[2];
+  assertType(
+    aggregateLambdas,
+    CollectionInstanceValue,
+    `Can't process groupBy() expression: groupBy() expects argument #2 to be a collection`,
+  );
+  aggregateLambdas.values.map((value) =>
+    QueryBuilderValueSpecificationProcessor.processChild(
+      value,
+      expression,
+      queryBuilderState,
+    ),
+  );
+
+  // process column aliases
+  const columnAliases = expression.parametersValues[3];
+  assertType(
+    columnAliases,
+    CollectionInstanceValue,
+    `Can't process groupBy() expression: groupBy() expects argument #3 to be a collection`,
+  );
+  assertTrue(
+    columnAliases.values.length ===
+      columnExpressions.values.length + aggregateLambdas.values.length,
+    `Can't process groupBy() expression: number of aliases does not match the number of columns`,
+  );
+  const aliases = columnAliases.values
+    .map(extractNullableStringFromInstanceValue)
+    .filter(isNonNullable);
+
+  // build state
+  if (
+    queryBuilderState.fetchStructureState.implementation instanceof
+    QueryBuilderProjectionState
+  ) {
+    const projectionState =
+      queryBuilderState.fetchStructureState.implementation;
+    projectionState.columns.forEach((column, idx) =>
+      column.setColumnName(aliases[idx] as string),
+    );
+  }
+};

package/src/stores/fetch-structure/projection/aggregation/operators/QueryBuilderAggregateOperator_DistinctCount.ts

@@ -110,7 +110,7 @@ export class QueryBuilderAggregateOperator_DistinctCount extends QueryBuilderAgg
       );
       aggregateColumnState.setLambdaParameterName(lambdaParam.name);
 
-      // count expression
+      // process count expression
       assertTrue(
         expression.parametersValues.length === 1,
         `Can't process count() expression: count() expects no argument`,