@finos/legend-application-repl 0.0.48 → 0.0.49

package/src/stores/dataCube/core/DataCubeQuerySnapshotBuilder.ts

@@ -17,19 +17,16 @@
 /***************************************************************************************
  * [CORE]
  *
- * These are utilities used to build the query snapshot from the executable query.
- * This is needed when we initialize the application by loading a persisted query.
+ * This and its corresponding utilitites are used to build the query snapshot from the
+ * executable query. This is needed when we initialize the application by loading a
+ * persisted query.
  ***************************************************************************************/
 
 import {
-  PRIMITIVE_TYPE,
   V1_AppliedFunction,
   V1_CInteger,
-  V1_ClassInstance,
-  V1_ColSpec,
-  V1_ColSpecArray,
+  type V1_ColSpec,
   V1_Collection,
-  V1_Lambda,
   V1_serializeValueSpecification,
   extractElementNameFromPath as _name,
   matchFunctionName,
@@ -38,189 +35,23 @@ import {
 import type { DataCubeQuery } from '../../../server/DataCubeQuery.js';
 import {
   DataCubeQuerySnapshot,
-  type DataCubeQuerySnapshotAggregateColumn,
   type DataCubeQuerySnapshotColumn,
 } from './DataCubeQuerySnapshot.js';
+import { assertType, guaranteeNonNullable } from '@finos/legend-shared';
 import {
-  assertTrue,
-  assertType,
-  guaranteeNonNullable,
-  guaranteeType,
-  UnsupportedOperationError,
-  type Clazz,
-} from '@finos/legend-shared';
-import {
-  DataCubeQuerySortOperation,
+  DataCubeQuerySortOperator,
   DataCubeFunction,
   type DataCubeQueryFunctionMap,
-  DataCubeAggregateOperation,
 } from './DataCubeQueryEngine.js';
 import { DataCubeConfiguration } from './DataCubeConfiguration.js';
 import { buildDefaultConfiguration } from './DataCubeConfigurationBuilder.js';
-
-// --------------------------------- UTILITIES ---------------------------------
-
-function _param<T extends V1_ValueSpecification>(
-  func: V1_AppliedFunction,
-  paramIdx: number,
-  clazz: Clazz<T>,
-): T {
-  assertTrue(
-    func.parameters.length >= paramIdx + 1,
-    `Can't process ${_name(func.function)}: Expected at least ${paramIdx + 1} parameter(s)`,
-  );
-  return guaranteeType(
-    func.parameters[paramIdx],
-    clazz,
-    `Can't process ${_name(func.function)}: Found unexpected type for parameter at index ${paramIdx}`,
-  );
-}
-
-function _colSpecParam(func: V1_AppliedFunction, paramIdx: number) {
-  return guaranteeType(
-    _param(func, paramIdx, V1_ClassInstance).value,
-    V1_ColSpec,
-    `Can't process ${_name(func.function)}: Expected parameter at index ${paramIdx} to be a column specification`,
-  );
-}
-
-function _colSpecArrayParam(func: V1_AppliedFunction, paramIdx: number) {
-  return guaranteeType(
-    _param(func, paramIdx, V1_ClassInstance).value,
-    V1_ColSpecArray,
-    `Can't process ${_name(func.function)}: Expected parameter at index ${paramIdx} to be a column specification list`,
-  );
-}
-
-function _funcMatch(
-  value: V1_ValueSpecification | undefined,
-  functionNames: string | string[],
-) {
-  assertType(
-    value,
-    V1_AppliedFunction,
-    `Can't process function: Found unexpected value specification type`,
-  );
-  assertTrue(
-    matchFunctionName(
-      value.function,
-      Array.isArray(functionNames) ? functionNames : [functionNames],
-    ),
-    `Can't process function: Expected function name to be one of [${Array.isArray(functionNames) ? functionNames.join(', ') : functionNames}]`,
-  );
-  return value;
-}
-
-function _aggFuncMatch(
-  value: V1_ValueSpecification | undefined,
-  functionNames: string | string[],
-) {
-  assertType(
-    value,
-    V1_Lambda,
-    `Can't process aggregation: Found unexpected value specification type`,
-  );
-  return _funcMatch(value.body[0], functionNames);
-}
-
-export function _defaultAggCol(
-  name: string,
-  type: string,
-): DataCubeQuerySnapshotAggregateColumn | undefined {
-  switch (type) {
-    case PRIMITIVE_TYPE.NUMBER:
-    case PRIMITIVE_TYPE.INTEGER:
-    case PRIMITIVE_TYPE.DECIMAL:
-    case PRIMITIVE_TYPE.FLOAT: {
-      return {
-        name,
-        type,
-        operation: DataCubeAggregateOperation.SUM,
-        parameters: [],
-      };
-    }
-    default:
-      return undefined;
-  }
-}
-
-function _aggCol(colSpec: V1_ColSpec, column: DataCubeQuerySnapshotColumn) {
-  const func = _aggFuncMatch(
-    colSpec.function2,
-    Object.values(DataCubeFunction),
-  );
-  if (matchFunctionName(func.function, DataCubeFunction.COUNT)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperation.COUNT,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.SUM)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperation.SUM,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.AVERAGE)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperation.AVERAGE,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.MIN)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperation.MIN,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.MAX)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperation.MAX,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.FIRST)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperation.FIRST,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.LAST)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperation.LAST,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.VAR_POP)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperation.VAR_POP,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.VAR_SAMP)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperation.VAR_SAMP,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.STDDEV_POP)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperation.STDDEV_POP,
-      parameters: [],
-    };
-  } else if (matchFunctionName(func.function, DataCubeFunction.STDDEV_SAMP)) {
-    return {
-      ...column,
-      operation: DataCubeAggregateOperation.STDDEV_SAMP,
-      parameters: [],
-    };
-  } else {
-    throw new UnsupportedOperationError(
-      `Unsupported aggregate function '${func.function}'`,
-    );
-  }
-}
+import {
+  _aggCol,
+  _colSpecArrayParam,
+  _colSpecParam,
+  _funcMatch,
+  _param,
+} from './DataCubeQuerySnapshotBuilderUtils.js';
 
 // --------------------------------- BUILDING BLOCKS ---------------------------------
 
@@ -228,27 +59,26 @@ const _SUPPORTED_TOP_LEVEL_FUNCTIONS: {
   func: string;
   parameters: number;
 }[] = [
-  { func: DataCubeFunction.FILTER, parameters: 1 },
   { func: DataCubeFunction.EXTEND, parameters: 1 },
+  { func: DataCubeFunction.FILTER, parameters: 1 },
   { func: DataCubeFunction.SELECT, parameters: 1 },
   { func: DataCubeFunction.GROUP_BY, parameters: 3 },
-  { func: DataCubeFunction.LIMIT, parameters: 1 },
   { func: DataCubeFunction.PIVOT, parameters: 3 },
-  { func: DataCubeFunction.SORT, parameters: 1 },
-
   { func: DataCubeFunction.CAST, parameters: 1 },
+  { func: DataCubeFunction.SORT, parameters: 1 },
+  { func: DataCubeFunction.LIMIT, parameters: 1 },
 ];
 
 // NOTE: this corresponds to the sequence:
-// filter()->extend()->groupBy()->select()->pivot()->cast()->extend()->sort()->limit()
+// extend()->filter()->groupBy()->select()->pivot()->cast()->extend()->sort()->limit()
 // which represents the ONLY query shape that we currently support
 const _FUNCTION_SEQUENCE_COMPOSITION_PATTERN: {
   func: string;
   repeat?: boolean | undefined;
   required?: boolean | undefined;
 }[] = [
-  { func: DataCubeFunction.FILTER },
   { func: DataCubeFunction.EXTEND },
+  { func: DataCubeFunction.FILTER },
   { func: DataCubeFunction.SELECT },
   { func: DataCubeFunction.GROUP_BY },
   { func: DataCubeFunction.PIVOT },
@@ -315,7 +145,7 @@ function extractQueryFunctionSequence(query: V1_ValueSpecification) {
     );
   }
 
-  // Special checks
+  // TODO: we need to improve this logic since there can be more than just 2 extend() functions
   // Check that the first and second extend() must be separated by either groupBy() or pivot() (i.e. aggregation)
   const firstExtendIndex = sequence.findIndex((func) =>
     matchFunctionName(func.function, DataCubeFunction.EXTEND),
@@ -464,9 +294,13 @@ export function validateAndBuildQuerySnapshot(
   }));
   data.sourceColumns.map((col) => colsMap.set(col.name, col));
 
+  // --------------------------------- LEAF EXTEND ---------------------------------
+  // TODO: @akphi - implement this
+
   // --------------------------------- FILTER ---------------------------------
 
   if (funcMap.filter) {
+    data.filter = undefined; // TODO: @akphi - implement this
     // data.selectColumns = _colSpecArrayParam(funcMap.select, 0).colSpecs.map(
     //   (colSpec) => ({
     //     _col(colSpec),
@@ -474,9 +308,6 @@ export function validateAndBuildQuerySnapshot(
     // );
   }
 
-  // --------------------------------- LEAF EXTEND ---------------------------------
-  // TODO: @akphi - implement this
-
   // --------------------------------- SELECT ---------------------------------
 
   if (funcMap.select) {
@@ -522,8 +353,8 @@ export function validateAndBuildQuerySnapshot(
           ..._col(_colSpecParam(sortColFunc, 0)),
           operation:
             _name(sortColFunc.function) === DataCubeFunction.ASC
-              ? DataCubeQuerySortOperation.ASCENDING
-              : DataCubeQuerySortOperation.DESCENDING,
+              ? DataCubeQuerySortOperator.ASCENDING
+              : DataCubeQuerySortOperator.DESCENDING,
         };
       },
     );

package/src/stores/dataCube/core/DataCubeQuerySnapshotBuilderUtils.ts

@@ -0,0 +1,211 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  PRIMITIVE_TYPE,
+  V1_AppliedFunction,
+  V1_ClassInstance,
+  V1_ColSpec,
+  V1_ColSpecArray,
+  V1_Lambda,
+  extractElementNameFromPath as _name,
+  matchFunctionName,
+  type V1_ValueSpecification,
+} from '@finos/legend-graph';
+import {
+  type DataCubeQuerySnapshotAggregateColumn,
+  type DataCubeQuerySnapshotColumn,
+} from './DataCubeQuerySnapshot.js';
+import {
+  assertTrue,
+  assertType,
+  guaranteeType,
+  UnsupportedOperationError,
+  type Clazz,
+} from '@finos/legend-shared';
+import {
+  DataCubeFunction,
+  DataCubeAggregateOperator,
+} from './DataCubeQueryEngine.js';
+
+// --------------------------------- UTILITIES ---------------------------------
+
+export function _param<T extends V1_ValueSpecification>(
+  func: V1_AppliedFunction,
+  paramIdx: number,
+  clazz: Clazz<T>,
+): T {
+  assertTrue(
+    func.parameters.length >= paramIdx + 1,
+    `Can't process ${_name(func.function)}: Expected at least ${paramIdx + 1} parameter(s)`,
+  );
+  return guaranteeType(
+    func.parameters[paramIdx],
+    clazz,
+    `Can't process ${_name(func.function)}: Found unexpected type for parameter at index ${paramIdx}`,
+  );
+}
+
+export function _colSpecParam(func: V1_AppliedFunction, paramIdx: number) {
+  return guaranteeType(
+    _param(func, paramIdx, V1_ClassInstance).value,
+    V1_ColSpec,
+    `Can't process ${_name(func.function)}: Expected parameter at index ${paramIdx} to be a column specification`,
+  );
+}
+
+export function _colSpecArrayParam(func: V1_AppliedFunction, paramIdx: number) {
+  return guaranteeType(
+    _param(func, paramIdx, V1_ClassInstance).value,
+    V1_ColSpecArray,
+    `Can't process ${_name(func.function)}: Expected parameter at index ${paramIdx} to be a column specification list`,
+  );
+}
+
+export function _funcMatch(
+  value: V1_ValueSpecification | undefined,
+  functionNames: string | string[],
+) {
+  assertType(
+    value,
+    V1_AppliedFunction,
+    `Can't process function: Found unexpected value specification type`,
+  );
+  assertTrue(
+    matchFunctionName(
+      value.function,
+      Array.isArray(functionNames) ? functionNames : [functionNames],
+    ),
+    `Can't process function: Expected function name to be one of [${Array.isArray(functionNames) ? functionNames.join(', ') : functionNames}]`,
+  );
+  return value;
+}
+
+// TODO: move these functions out into aggregator utils when we make agg operator handling systematic
+// similar to what we did with filter
+function _aggFuncMatch(
+  value: V1_ValueSpecification | undefined,
+  functionNames: string | string[],
+) {
+  assertType(
+    value,
+    V1_Lambda,
+    `Can't process aggregation: Found unexpected value specification type`,
+  );
+  return _funcMatch(value.body[0], functionNames);
+}
+
+export function _defaultAggCol(
+  name: string,
+  type: string,
+): DataCubeQuerySnapshotAggregateColumn | undefined {
+  switch (type) {
+    case PRIMITIVE_TYPE.NUMBER:
+    case PRIMITIVE_TYPE.INTEGER:
+    case PRIMITIVE_TYPE.DECIMAL:
+    case PRIMITIVE_TYPE.FLOAT: {
+      return {
+        name,
+        type,
+        operation: DataCubeAggregateOperator.SUM,
+        parameters: [],
+      };
+    }
+    default:
+      return undefined;
+  }
+}
+
+export function _aggCol(
+  colSpec: V1_ColSpec,
+  column: DataCubeQuerySnapshotColumn,
+) {
+  const func = _aggFuncMatch(
+    colSpec.function2,
+    Object.values(DataCubeFunction),
+  );
+  if (matchFunctionName(func.function, DataCubeFunction.COUNT)) {
+    return {
+      ...column,
+      operation: DataCubeAggregateOperator.COUNT,
+      parameters: [],
+    };
+  } else if (matchFunctionName(func.function, DataCubeFunction.SUM)) {
+    return {
+      ...column,
+      operation: DataCubeAggregateOperator.SUM,
+      parameters: [],
+    };
+  } else if (matchFunctionName(func.function, DataCubeFunction.AVERAGE)) {
+    return {
+      ...column,
+      operation: DataCubeAggregateOperator.AVERAGE,
+      parameters: [],
+    };
+  } else if (matchFunctionName(func.function, DataCubeFunction.MIN)) {
+    return {
+      ...column,
+      operation: DataCubeAggregateOperator.MIN,
+      parameters: [],
+    };
+  } else if (matchFunctionName(func.function, DataCubeFunction.MAX)) {
+    return {
+      ...column,
+      operation: DataCubeAggregateOperator.MAX,
+      parameters: [],
+    };
+  } else if (matchFunctionName(func.function, DataCubeFunction.FIRST)) {
+    return {
+      ...column,
+      operation: DataCubeAggregateOperator.FIRST,
+      parameters: [],
+    };
+  } else if (matchFunctionName(func.function, DataCubeFunction.LAST)) {
+    return {
+      ...column,
+      operation: DataCubeAggregateOperator.LAST,
+      parameters: [],
+    };
+  } else if (matchFunctionName(func.function, DataCubeFunction.VAR_POP)) {
+    return {
+      ...column,
+      operation: DataCubeAggregateOperator.VAR_POP,
+      parameters: [],
+    };
+  } else if (matchFunctionName(func.function, DataCubeFunction.VAR_SAMP)) {
+    return {
+      ...column,
+      operation: DataCubeAggregateOperator.VAR_SAMP,
+      parameters: [],
+    };
+  } else if (matchFunctionName(func.function, DataCubeFunction.STDDEV_POP)) {
+    return {
+      ...column,
+      operation: DataCubeAggregateOperator.STDDEV_POP,
+      parameters: [],
+    };
+  } else if (matchFunctionName(func.function, DataCubeFunction.STDDEV_SAMP)) {
+    return {
+      ...column,
+      operation: DataCubeAggregateOperator.STDDEV_SAMP,
+      parameters: [],
+    };
+  } else {
+    throw new UnsupportedOperationError(
+      `Unsupported aggregate function '${func.function}'`,
+    );
+  }
+}

package/src/stores/dataCube/core/DataCubeQuerySnapshotManager.ts

@@ -17,18 +17,67 @@
 import { APPLICATION_EVENT } from '@finos/legend-application';
 import type { DataCubeState } from '../DataCubeState.js';
 import type { DataCubeQuerySnapshot } from './DataCubeQuerySnapshot.js';
-import type { DataCubeQuerySnapshotSubscriber } from './DataCubeQuerySnapshotSubscriber.js';
 import {
   IllegalStateError,
   LogEvent,
   assertErrorThrown,
+  deepDiff,
   guaranteeNonNullable,
 } from '@finos/legend-shared';
 import type { DataCubeQuery } from '../../../server/DataCubeQuery.js';
+import type { LegendREPLApplicationStore } from '../../LegendREPLBaseStore.js';
 
 // TODO: set a stack depth when we implement undo/redo
 // const DATA_CUBE_MAX_SNAPSHOT_COUNT = 100;
 
+interface DataCubeQuerySnapshotSubscriber {
+  getLatestSnapshot(): DataCubeQuerySnapshot | undefined;
+  receiveSnapshot(snapshot: DataCubeQuerySnapshot): Promise<void>;
+}
+
+export abstract class DataCubeQuerySnapshotController
+  implements DataCubeQuerySnapshotSubscriber
+{
+  readonly dataCube!: DataCubeState;
+  readonly application!: LegendREPLApplicationStore;
+
+  private latestSnapshot: DataCubeQuerySnapshot | undefined;
+
+  constructor(dataCube: DataCubeState) {
+    this.dataCube = dataCube;
+    this.application = dataCube.application;
+  }
+
+  abstract applySnapshot(
+    snapshot: DataCubeQuerySnapshot,
+    previousSnapshot: DataCubeQuerySnapshot | undefined,
+  ): Promise<void>;
+
+  async receiveSnapshot(snapshot: DataCubeQuerySnapshot) {
+    const previousSnapshot = this.latestSnapshot;
+    this.latestSnapshot = snapshot;
+    await this.applySnapshot(snapshot, previousSnapshot);
+  }
+
+  publishSnapshot(snapshot: DataCubeQuerySnapshot) {
+    if (this.dataCube.engine.enableDebugMode) {
+      this.application.debugProcess(
+        `New Snapshot`,
+        '\nSnapshot',
+        snapshot,
+        '\nDiff',
+        deepDiff(snapshot, this.latestSnapshot ?? {}),
+      );
+    }
+    this.latestSnapshot = snapshot;
+    this.dataCube.snapshotManager.broadcastSnapshot(snapshot);
+  }
+
+  getLatestSnapshot() {
+    return this.latestSnapshot;
+  }
+}
+
 export class DataCubeQuerySnapshotManager {
   private readonly dataCube: DataCubeState;
   private readonly subscribers: DataCubeQuerySnapshotSubscriber[] = [];

package/src/stores/dataCube/core/{DataCubeCoreState.ts → DataCubeStaticContentDisplayState.ts}

@@ -16,13 +16,13 @@
 
 import { action, makeObservable, observable } from 'mobx';
 import type { DataCubeState } from '../DataCubeState.js';
-import { DataCubeQuerySnapshotSubscriber } from './DataCubeQuerySnapshotSubscriber.js';
+import { DataCubeQuerySnapshotController } from './DataCubeQuerySnapshotManager.js';
 import type { DataCubeQuerySnapshot } from './DataCubeQuerySnapshot.js';
 import type { DataCubeQuery } from '../../../server/DataCubeQuery.js';
 import { formatDate } from '@finos/legend-shared';
 import { DEFAULT_REPORT_NAME } from './DataCubeQueryEngine.js';
 
-export class DataCubeCoreState extends DataCubeQuerySnapshotSubscriber {
+export class DataCubeStaticContentDisplayState extends DataCubeQuerySnapshotController {
   baseQuery!: DataCubeQuery;
   name = DEFAULT_REPORT_NAME;
   private startTime?: number | undefined;
@@ -53,8 +53,4 @@ export class DataCubeCoreState extends DataCubeQuerySnapshotSubscriber {
       `\u229E ${data.name}${this.startTime ? ` - ${formatDate(new Date(this.startTime), 'HH:mm:ss EEE MMM dd yyyy')}` : ''}`,
     );
   }
-
-  override async initialize() {
-    // do nothing
-  }
 }