@finos/legend-application-repl 0.0.60 → 1.0.1

package/src/stores/dataCube/core/DataCubeQuerySnapshotBuilder.ts

@@ -1,411 +0,0 @@
-/**
- * Copyright (c) 2020-present, Goldman Sachs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/***************************************************************************************
- * [CORE]
- *
- * This and its corresponding utilitites are used to build the query snapshot from the
- * executable query. This is needed when we initialize the application by loading a
- * persisted query.
- ***************************************************************************************/
-
-import {
-  V1_AppliedFunction,
-  V1_CInteger,
-  type V1_ColSpec,
-  V1_Collection,
-  V1_serializeValueSpecification,
-  extractElementNameFromPath as _name,
-  matchFunctionName,
-  type V1_ValueSpecification,
-} from '@finos/legend-graph';
-import type { DataCubeQuery } from '../../../server/DataCubeQuery.js';
-import {
-  _toCol,
-  DataCubeQuerySnapshot,
-  type DataCubeQuerySnapshotColumn,
-} from './DataCubeQuerySnapshot.js';
-import { assertType, guaranteeNonNullable } from '@finos/legend-shared';
-import {
-  DataCubeQuerySortOperator,
-  DataCubeFunction,
-  type DataCubeQueryFunctionMap,
-} from './DataCubeQueryEngine.js';
-import { DataCubeConfiguration } from './DataCubeConfiguration.js';
-import { buildDefaultConfiguration } from './DataCubeConfigurationBuilder.js';
-import {
-  _colSpecArrayParam,
-  _colSpecParam,
-  _funcMatch,
-  _param,
-} from './DataCubeQuerySnapshotBuilderUtils.js';
-
-// --------------------------------- BUILDING BLOCKS ---------------------------------
-
-const _SUPPORTED_TOP_LEVEL_FUNCTIONS: {
-  func: string;
-  parameters: number;
-}[] = [
-  { func: DataCubeFunction.EXTEND, parameters: 1 },
-  { func: DataCubeFunction.FILTER, parameters: 1 },
-  { func: DataCubeFunction.SELECT, parameters: 1 },
-  { func: DataCubeFunction.GROUP_BY, parameters: 3 },
-  { func: DataCubeFunction.PIVOT, parameters: 3 },
-  { func: DataCubeFunction.CAST, parameters: 1 },
-  { func: DataCubeFunction.SORT, parameters: 1 },
-  { func: DataCubeFunction.LIMIT, parameters: 1 },
-];
-
-type FunctionSequenceCompositionNodePattern = {
-  name: string;
-  repeat?: boolean | undefined;
-  required?: boolean | undefined;
-};
-type FunctionSequenceCompositionSimpleNodePattern =
-  FunctionSequenceCompositionNodePattern & {
-    func: string;
-  };
-type FunctionSequenceCompositionGroupNodePattern =
-  FunctionSequenceCompositionNodePattern & {
-    funcs: FunctionSequenceCompositionSimpleNodePattern[];
-  };
-
-// NOTE: this corresponds to the sequence:
-// extend()->filter()->select()->sort()->pivot()->cast()->groupBy()->extend()->sort()->limit()
-// which represents the ONLY query shape that we currently support
-const _FUNCTION_SEQUENCE_COMPOSITION_PATTERN: (
-  | FunctionSequenceCompositionSimpleNodePattern
-  | FunctionSequenceCompositionGroupNodePattern
-)[] = [
-  { name: 'leaf_extend', func: DataCubeFunction.EXTEND }, // leaf-level extend
-  { name: 'filter', func: DataCubeFunction.FILTER },
-  { name: 'select', func: DataCubeFunction.SELECT },
-  {
-    name: 'pivot_group',
-    funcs: [
-      { name: 'pivot_sort', func: DataCubeFunction.SORT, required: true }, // pre-pivot sort
-      { name: 'pivot', func: DataCubeFunction.PIVOT, required: true },
-      { name: 'pivot_cast', func: DataCubeFunction.CAST, required: true }, // cast to a relation type post pivot() to enable type-checking
-    ],
-  },
-  { name: 'group_by', func: DataCubeFunction.GROUP_BY },
-  { name: 'group_extend', func: DataCubeFunction.EXTEND }, // group-level extend
-  { name: 'sort', func: DataCubeFunction.SORT },
-  { name: 'limit', func: DataCubeFunction.LIMIT },
-];
-const _FUNCTION_SEQUENCE_COMPOSITION_PATTERN_REGEXP = new RegExp(
-  `^${_FUNCTION_SEQUENCE_COMPOSITION_PATTERN
-    .map((node) =>
-      'funcs' in node
-        ? `(${node.funcs.map(
-            (childNode) =>
-              `(?<${childNode.name}><${_name(childNode.func)}>)${childNode.repeat ? '*' : !childNode.required ? '?' : ''}`,
-          )})${node.repeat ? '*' : !node.required ? '?' : ''}`
-        : `(?<${node.name}><${_name(node.func)}>)${node.repeat ? '*' : !node.required ? '?' : ''}`,
-    )
-    .join('')}$`,
-);
-
-type FunctionSequenceMatchingResult = {
-  sequence: V1_AppliedFunction[];
-  leafExtend: boolean;
-  pivot: boolean;
-  groupExtend: boolean;
-  sort: boolean;
-};
-/**
- * Since the query created by DataCube will be a chain call of various functions,
- * this utility function will extract that sequence of function calls as well as
- * do various basic checks for the validity, composition, and order of those functions.
- */
-function extractQueryFunctionSequence(
-  query: V1_ValueSpecification,
-): FunctionSequenceMatchingResult {
-  // Make sure this is a sequence of function calls
-  if (!(query instanceof V1_AppliedFunction)) {
-    throw new Error(
-      `Query must be a sequence of function calls (e.g. x()->y()->z())`,
-    );
-  }
-  const sequence: V1_AppliedFunction[] = [];
-  let currentFunc = query;
-  while (currentFunc instanceof V1_AppliedFunction) {
-    const supportedFunc = _SUPPORTED_TOP_LEVEL_FUNCTIONS.find((spec) =>
-      matchFunctionName(currentFunc.function, spec.func),
-    );
-
-    // Check that all functions in sequence are supported
-    if (!supportedFunc) {
-      throw new Error(`Found unsupported function ${currentFunc.function}()`);
-    }
-    if (currentFunc.parameters.length > supportedFunc.parameters) {
-      const vs = currentFunc.parameters[0];
-      if (!(vs instanceof V1_AppliedFunction)) {
-        throw new Error(
-          `Query must be a sequence of function calls (e.g. x()->y()->z())`,
-        );
-      }
-      currentFunc.parameters = currentFunc.parameters.slice(1);
-      sequence.unshift(currentFunc);
-      currentFunc = vs;
-    } else {
-      sequence.unshift(currentFunc);
-      break;
-    }
-  }
-
-  // Check that sequence follows the supported pattern
-  const sequenceFormText = sequence
-    .map((func) => `<${_name(func.function)}>`)
-    .join('');
-  const matchResult = sequenceFormText.match(
-    _FUNCTION_SEQUENCE_COMPOSITION_PATTERN_REGEXP,
-  );
-  if (!matchResult) {
-    throw new Error(
-      `Unsupported function composition ${sequence.map((fn) => `${_name(fn.function)}()`).join('->')} (supported composition: ${_FUNCTION_SEQUENCE_COMPOSITION_PATTERN.map((node) => `${'funcs' in node ? `[${node.funcs.map((childNode) => `${_name(childNode.func)}()`).join('->')}]` : `${_name(node.func)}()`}`).join('->')})`,
-    );
-  }
-
-  return {
-    sequence,
-    leafExtend: Boolean(matchResult.groups?.leaf_extend),
-    pivot: Boolean(matchResult.groups?.pivot),
-    groupExtend: Boolean(matchResult.groups?.group_extend),
-    sort: Boolean(matchResult.groups?.sort),
-  };
-}
-
-/**
- * Turn the function sequence into a map of available functions
- * for easier construction of the snapshot
- */
-function extractFunctionMap(
-  sequenceMatchingResult: FunctionSequenceMatchingResult,
-): DataCubeQueryFunctionMap {
-  const {
-    sequence,
-    leafExtend: hasLeafExtend,
-    groupExtend: hasGroupExtend,
-    pivot: hasPivot,
-    sort: hasSort,
-  } = sequenceMatchingResult;
-  const leafExtend = hasLeafExtend
-    ? sequence.find((func) =>
-        matchFunctionName(func.function, DataCubeFunction.EXTEND),
-      )
-    : undefined;
-  const filter = sequence.find((func) =>
-    matchFunctionName(func.function, DataCubeFunction.FILTER),
-  );
-  const select = sequence.find((func) =>
-    matchFunctionName(func.function, DataCubeFunction.SELECT),
-  );
-  const pivotSort = hasPivot
-    ? sequence.find((func) =>
-        matchFunctionName(func.function, DataCubeFunction.SORT),
-      )
-    : undefined;
-  const pivot = hasPivot
-    ? sequence.find((func) =>
-        matchFunctionName(func.function, DataCubeFunction.PIVOT),
-      )
-    : undefined;
-  const pivotCast = hasPivot
-    ? sequence.find((func) =>
-        matchFunctionName(func.function, DataCubeFunction.CAST),
-      )
-    : undefined;
-  const groupBy = sequence.find((func) =>
-    matchFunctionName(func.function, DataCubeFunction.GROUP_BY),
-  );
-  const groupExtend = hasGroupExtend
-    ? sequence.find((func) =>
-        matchFunctionName(func.function, DataCubeFunction.EXTEND),
-      )
-    : undefined;
-  const sort = hasSort
-    ? sequence.find((func) =>
-        matchFunctionName(func.function, DataCubeFunction.SORT),
-      )
-    : undefined;
-  const limit = sequence.find((func) =>
-    matchFunctionName(func.function, DataCubeFunction.LIMIT),
-  );
-
-  return {
-    leafExtend: leafExtend,
-    select,
-    filter,
-    groupBy,
-    pivotSort,
-    pivot,
-    pivotCast,
-    groupExtend,
-    sort,
-    limit,
-  };
-}
-
-// --------------------------------- MAIN ---------------------------------
-
-/**
- * Analyze the partial query to build a query snapshot.
- *
- * Implementation-wise, this extracts the function call sequence, then walk the
- * sequence in order to fill in the information for the snapshot.
- */
-export function validateAndBuildQuerySnapshot(
-  partialQuery: V1_ValueSpecification,
-  sourceQuery: V1_ValueSpecification,
-  baseQuery: DataCubeQuery,
-) {
-  // --------------------------------- BASE ---------------------------------
-  // Build the function call sequence and the function map to make the
-  // analysis more ergonomic
-
-  const sequenceMatchingResult = extractQueryFunctionSequence(partialQuery);
-  const funcMap = extractFunctionMap(sequenceMatchingResult);
-  const snapshot = DataCubeQuerySnapshot.create(
-    baseQuery.name,
-    baseQuery.source.runtime,
-    baseQuery.source.mapping,
-    V1_serializeValueSpecification(sourceQuery, []),
-    {},
-  );
-  const data = snapshot.data;
-  const colsMap = new Map<string, DataCubeQuerySnapshotColumn>();
-  const _col = (colSpec: V1_ColSpec) => {
-    const column = guaranteeNonNullable(
-      colsMap.get(colSpec.name),
-      `Can't find column '${colSpec.name}'`,
-    );
-    return _toCol(column);
-  };
-
-  // --------------------------------- SOURCE ---------------------------------
-
-  data.sourceColumns = baseQuery.source.columns.map((col) => _toCol(col));
-  data.sourceColumns.map((col) => colsMap.set(col.name, col));
-
-  // --------------------------------- LEAF-LEVEL EXTEND ---------------------------------
-  /** TODO: @datacube roundtrip */
-
-  // --------------------------------- FILTER ---------------------------------
-
-  if (funcMap.filter) {
-    /** TODO: @datacube roundtrip */
-    data.filter = undefined;
-    // data.selectColumns = _colSpecArrayParam(funcMap.select, 0).colSpecs.map(
-    //   (colSpec) => ({
-    //     _col(colSpec),
-    //   }),
-    // );
-  }
-
-  // --------------------------------- SELECT ---------------------------------
-
-  if (funcMap.select) {
-    data.selectColumns = _colSpecArrayParam(funcMap.select, 0).colSpecs.map(
-      (colSpec) => _col(colSpec),
-    );
-  }
-
-  // --------------------------------- PIVOT ---------------------------------
-  /** TODO: @datacube roundtrip */
-  // TODO: verify group-by agg columns, pivot agg columns and configuration agree
-
-  // --------------------------------- GROUP BY ---------------------------------
-
-  if (funcMap.groupBy) {
-    data.groupBy = {
-      columns: _colSpecArrayParam(funcMap.groupBy, 0).colSpecs.map((colSpec) =>
-        _col(colSpec),
-      ),
-      // TODO: verify group-by agg columns, pivot agg columns and configuration agree
-    };
-  }
-
-  // --------------------------------- GROUP-LEVEL EXTEND ---------------------------------
-  /** TODO: @datacube roundtrip */
-
-  // --------------------------------- SORT ---------------------------------
-
-  if (funcMap.sort) {
-    data.sortColumns = _param(funcMap.sort, 0, V1_Collection).values.map(
-      (value) => {
-        const sortColFunc = _funcMatch(value, [
-          DataCubeFunction.ASC,
-          DataCubeFunction.DESC,
-        ]);
-        return {
-          ..._col(_colSpecParam(sortColFunc, 0)),
-          operation:
-            _name(sortColFunc.function) === DataCubeFunction.ASC
-              ? DataCubeQuerySortOperator.ASCENDING
-              : DataCubeQuerySortOperator.DESCENDING,
-        };
-      },
-    );
-  }
-
-  // --------------------------------- LIMIT ---------------------------------
-
-  if (funcMap.limit) {
-    const value = funcMap.limit.parameters[0];
-    assertType(value, V1_CInteger);
-    data.limit = value.value;
-  }
-
-  // --------------------------------- CONFIGURATION ---------------------------------
-  // NOTE: we aim to keep the data query a Pure query instead of a specification object
-  // so this configuration holds mostly layout and styling customization.
-  // But there are overlaps, i.e. certain data query configuration are stored in the
-  // configuration, e.g. column aggregation type, this is because a column
-  // aggregation's preference can be populated even when there's no aggregation specified
-  // in the data query.
-  //
-  // Arguably, the query should be the single source for these information, but when
-  // the configuration for a particular data query function coming from multiple sources
-  // conflict, we need to do some reconciliation (or throw error). Some examples include:
-  // - missing/extra columns present in the configuration
-  // - column kind and type conflict with aggregation
-  // - column kind and type conflict with the column configuration
-  //
-  // In certain cases, configuration needs to be generated from default presets. This
-  // helps with use cases where the query might comes from a different source, such as
-  // Studio or Query, or another part of Engine.
-  const configuration = baseQuery.configuration
-    ? DataCubeConfiguration.serialization.fromJson(baseQuery.configuration)
-    : buildDefaultConfiguration([
-        ...baseQuery.source.columns,
-        ...data.leafExtendedColumns,
-        ...data.groupExtendedColumns,
-      ]);
-  data.configuration =
-    DataCubeConfiguration.serialization.toJson(configuration);
-  /**
-   * TODO: @datacube roundtrip - implement the logic to reconcile the configuration with the query
-   * - columns (missing/extra columns - remove or generate default column configuration)
-   * - column kind
-   * - column type
-   * - base off the type, check the settings
-   * - aggregation
-   * - verify group-by agg columns, pivot agg columns and configuration agree
-   */
-
-  return snapshot.finalize();
-}

package/src/stores/dataCube/core/DataCubeQuerySnapshotBuilderUtils.ts

@@ -1,197 +0,0 @@
-/**
- * Copyright (c) 2020-present, Goldman Sachs
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import {
-  V1_AppliedFunction,
-  V1_ClassInstance,
-  V1_ColSpec,
-  V1_ColSpecArray,
-  V1_Lambda,
-  extractElementNameFromPath as _name,
-  matchFunctionName,
-  type V1_ValueSpecification,
-} from '@finos/legend-graph';
-import { type DataCubeQuerySnapshotColumn } from './DataCubeQuerySnapshot.js';
-import {
-  assertTrue,
-  assertType,
-  guaranteeType,
-  UnsupportedOperationError,
-  type Clazz,
-} from '@finos/legend-shared';
-import {
-  DataCubeFunction,
-  DataCubeAggregateOperator,
-} from './DataCubeQueryEngine.js';
-
-// --------------------------------- UTILITIES ---------------------------------
-
-export function _param<T extends V1_ValueSpecification>(
-  func: V1_AppliedFunction,
-  paramIdx: number,
-  clazz: Clazz<T>,
-): T {
-  assertTrue(
-    func.parameters.length >= paramIdx + 1,
-    `Can't process ${_name(func.function)}: Expected at least ${paramIdx + 1} parameter(s)`,
-  );
-  return guaranteeType(
-    func.parameters[paramIdx],
-    clazz,
-    `Can't process ${_name(func.function)}: Found unexpected type for parameter at index ${paramIdx}`,
-  );
-}
-
-export function _colSpecParam(func: V1_AppliedFunction, paramIdx: number) {
-  return guaranteeType(
-    _param(func, paramIdx, V1_ClassInstance).value,
-    V1_ColSpec,
-    `Can't process ${_name(func.function)}: Expected parameter at index ${paramIdx} to be a column specification`,
-  );
-}
-
-export function _colSpecArrayParam(func: V1_AppliedFunction, paramIdx: number) {
-  return guaranteeType(
-    _param(func, paramIdx, V1_ClassInstance).value,
-    V1_ColSpecArray,
-    `Can't process ${_name(func.function)}: Expected parameter at index ${paramIdx} to be a column specification list`,
-  );
-}
-
-export function _funcMatch(
-  value: V1_ValueSpecification | undefined,
-  functionNames: string | string[],
-) {
-  assertType(
-    value,
-    V1_AppliedFunction,
-    `Can't process function: Found unexpected value specification type`,
-  );
-  assertTrue(
-    matchFunctionName(
-      value.function,
-      Array.isArray(functionNames) ? functionNames : [functionNames],
-    ),
-    `Can't process function: Expected function name to be one of [${Array.isArray(functionNames) ? functionNames.join(', ') : functionNames}]`,
-  );
-  return value;
-}
-
-// TODO: move these functions out into aggregator utils when we make agg operator handling systematic
-// similar to what we did with filter
-function _aggFuncMatch(
-  value: V1_ValueSpecification | undefined,
-  functionNames: string | string[],
-) {
-  assertType(
-    value,
-    V1_Lambda,
-    `Can't process aggregation: Found unexpected value specification type`,
-  );
-  return _funcMatch(value.body[0], functionNames);
-}
-
-export function _aggCol(
-  colSpec: V1_ColSpec,
-  column: DataCubeQuerySnapshotColumn,
-) {
-  const func = _aggFuncMatch(
-    colSpec.function2,
-    Object.values(DataCubeFunction),
-  );
-  if (matchFunctionName(func.function, DataCubeFunction.COUNT)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperator.COUNT,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.SUM)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperator.SUM,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.AVERAGE)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperator.AVERAGE,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.MIN)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperator.MIN,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.MAX)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperator.MAX,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.FIRST)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperator.FIRST,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.LAST)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperator.LAST,
-      parameters: [],
-    };
-  } else if (
-    matchFunctionName(func.function, DataCubeFunction.VARIANCE_POPULATION)
-  ) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperator.VARIANCE_POPULATION,
-      parameters: [],
-    };
-  } else if (
-    matchFunctionName(func.function, DataCubeFunction.VARIANCE_SAMPLE)
-  ) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperator.VARIANCE_SAMPLE,
-      parameters: [],
-    };
-  } else if (
-    matchFunctionName(
-      func.function,
-      DataCubeFunction.STANDARD_DEVIATION_POPULATION,
-    )
-  ) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperator.STANDARD_DEVIATION_POPULATION,
-      parameters: [],
-    };
-  } else if (
-    matchFunctionName(func.function, DataCubeFunction.STANDARD_DEVIATION_SAMPLE)
-  ) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperator.STANDARD_DEVIATION_SAMPLE,
-      parameters: [],
-    };
-  } else {
-    throw new UnsupportedOperationError(
-      `Unsupported aggregate function '${func.function}'`,
-    );
-  }
-}