@finos/legend-data-cube 0.0.48 → 0.1.1

package/src/stores/core/DataCubeQueryBuilderUtils.ts

@@ -43,7 +43,6 @@ import {
   V1_PackageableElementPtr,
   type V1_PrimitiveValueSpecification,
   V1_Variable,
-  V1_deserializeValueSpecification,
   extractElementNameFromPath,
   type V1_ValueSpecification,
   extractPackagePathFromPath,
@@ -53,7 +52,6 @@ import {
   V1_createGenericTypeWithRawType,
   V1_createRelationType,
   V1_createRelationTypeColumn,
-  V1_serializeValueSpecification,
 } from '@finos/legend-graph';
 import {
   type DataCubeQuerySnapshotFilterCondition,
@@ -67,8 +65,6 @@ import {
   guaranteeIsBoolean,
   guaranteeIsNumber,
   UnsupportedOperationError,
-  guaranteeType,
-  type PlainObject,
 } from '@finos/legend-shared';
 import {
   DataCubeFunction,
@@ -91,69 +87,6 @@ import {
 
 // --------------------------------- UTILITIES ---------------------------------
 
-export function _functionCompositionProcessor(
-  sequence: V1_AppliedFunction[],
-  funcMap: DataCubeQueryFunctionMap,
-) {
-  return (
-    key: keyof DataCubeQueryFunctionMap,
-    data: V1_AppliedFunction | V1_AppliedFunction[],
-  ) => {
-    switch (key) {
-      case 'leafExtend':
-      case 'groupExtend': {
-        if (Array.isArray(data)) {
-          data.forEach((func) => sequence.push(func));
-          funcMap[key] = data;
-        }
-        break;
-      }
-      default: {
-        funcMap[key] = data as V1_AppliedFunction;
-        sequence.push(data as V1_AppliedFunction);
-      }
-    }
-  };
-}
-
-export function _functionCompositionUnProcessor(
-  sequence: V1_AppliedFunction[],
-  funcMap: DataCubeQueryFunctionMap,
-) {
-  return (key: keyof DataCubeQueryFunctionMap) => {
-    const data = funcMap[key];
-    if (data) {
-      if (Array.isArray(data)) {
-        data.forEach((func) => sequence.splice(sequence.indexOf(func), 1));
-      } else {
-        sequence.splice(sequence.indexOf(data), 1);
-      }
-      funcMap[key] = undefined;
-    }
-  };
-}
-
-export function _deserializeLambda(json: PlainObject<V1_Lambda>) {
-  return guaranteeType(V1_deserializeValueSpecification(json, []), V1_Lambda);
-}
-
-export function _deserializeValueSpecification(
-  json: PlainObject<V1_ValueSpecification>,
-) {
-  return V1_deserializeValueSpecification(json, []);
-}
-
-export function _serializeValueSpecification(protocol: V1_ValueSpecification) {
-  return V1_serializeValueSpecification(protocol, []);
-}
-
-export function _deserializeFunction(json: PlainObject<V1_Lambda>) {
-  return guaranteeType(
-    V1_deserializeValueSpecification(json, []),
-    V1_AppliedFunction,
-  );
-}
-
 export function _var() {
   const variable = new V1_Variable();
   // NOTE: we simplify processing logic by forcing all variable names to default value, i.e. x
@@ -356,6 +289,48 @@ export function _cols(colSpecs: V1_ColSpec[]) {
 
 // --------------------------------- BUILDING BLOCKS ---------------------------------
 
+export function _functionCompositionProcessor(
+  sequence: V1_AppliedFunction[],
+  funcMap: DataCubeQueryFunctionMap,
+) {
+  return (
+    key: keyof DataCubeQueryFunctionMap,
+    data: V1_AppliedFunction | V1_AppliedFunction[],
+  ) => {
+    switch (key) {
+      case 'leafExtend':
+      case 'groupExtend': {
+        if (Array.isArray(data)) {
+          data.forEach((func) => sequence.push(func));
+          funcMap[key] = data;
+        }
+        break;
+      }
+      default: {
+        funcMap[key] = data as V1_AppliedFunction;
+        sequence.push(data as V1_AppliedFunction);
+      }
+    }
+  };
+}
+
+export function _functionCompositionUnProcessor(
+  sequence: V1_AppliedFunction[],
+  funcMap: DataCubeQueryFunctionMap,
+) {
+  return (key: keyof DataCubeQueryFunctionMap) => {
+    const data = funcMap[key];
+    if (data) {
+      if (Array.isArray(data)) {
+        data.forEach((func) => sequence.splice(sequence.indexOf(func), 1));
+      } else {
+        sequence.splice(sequence.indexOf(data), 1);
+      }
+      funcMap[key] = undefined;
+    }
+  };
+}
+
 export function _selectFunction(columns: DataCubeColumn[]) {
   return _function(DataCubeFunction.SELECT, [
     _cols(columns.map((col) => _colSpec(col.name))),

package/src/stores/core/DataCubeQuerySnapshot.ts

@@ -71,10 +71,10 @@ export type DataCubeQuerySnapshotPivot = {
 
 export type DataCubeQuerySnapshotProcessingContext = {
   snapshot: DataCubeQuerySnapshot;
-  pivotAggColumns: DataCubeQuerySnapshotAggregateColumn[];
-  pivotSortColumns: DataCubeQuerySnapshotSortColumn[];
-  groupByAggColumns: DataCubeQuerySnapshotAggregateColumn[];
-  groupBySortColumns: DataCubeQuerySnapshotSortColumn[];
+  pivotAggColumns?: DataCubeQuerySnapshotAggregateColumn[] | undefined;
+  pivotSortColumns?: DataCubeQuerySnapshotSortColumn[] | undefined;
+  groupByAggColumns?: DataCubeQuerySnapshotAggregateColumn[] | undefined;
+  groupBySortColumns?: DataCubeQuerySnapshotSortColumn[] | undefined;
 };
 
 export type DataCubeQuerySnapshotData = {
@@ -185,4 +185,15 @@ export class DataCubeQuerySnapshot {
     clone._hashCode = this._hashCode;
     return clone;
   }
+
+  serialize() {
+    const clone = this.clone();
+    return {
+      uuid: clone.uuid,
+      data: clone.data,
+      _isPatchChange: clone._isPatchChange,
+      _finalized: clone._finalized,
+      _hashCode: clone._hashCode,
+    };
+  }
 }

package/src/stores/core/DataCubeQuerySnapshotBuilder.ts

@@ -73,10 +73,6 @@ import {
 } from './DataCubeQuerySnapshotBuilderUtils.js';
 import type { DataCubeSource } from './model/DataCubeSource.js';
 import type { DataCubeEngine } from './DataCubeEngine.js';
-import {
-  _deserializeValueSpecification,
-  _serializeValueSpecification,
-} from './DataCubeQueryBuilderUtils.js';
 
 // --------------------------------- BUILDING BLOCKS ---------------------------------
 
@@ -365,8 +361,8 @@ export async function validateAndBuildQuerySnapshot(
   // analysis more ergonomic
 
   // Clone the query since we will mutate it during the process
-  const query = _deserializeValueSpecification(
-    _serializeValueSpecification(partialQuery),
+  const query = engine.deserializeValueSpecification(
+    engine.serializeValueSpecification(partialQuery),
   );
   const funcMap = extractFunctionMap(query);
   const snapshot = DataCubeQuerySnapshot.create({});

package/src/stores/core/DataCubeQuerySnapshotBuilderUtils.ts

@@ -79,7 +79,6 @@ import {
   _colSpec,
   _function,
   _lambda,
-  _serializeValueSpecification,
   _synthesizeMinimalSourceQuery,
 } from './DataCubeQueryBuilderUtils.js';
 import { INTERNAL__DataCubeSource } from './model/DataCubeSource.js';
@@ -360,14 +359,14 @@ export async function _extractExtendedColumns(
       _findCol(columns, colSpec.name),
       `Can't process extend() expression: failed to retrieve type information for column '${colSpec.name}'`,
     ).type,
-    mapFn: _serializeValueSpecification(
+    mapFn: engine.serializeValueSpecification(
       guaranteeNonNullable(
         colSpec.function1,
         `Can't process extend() expression: expected a transformation function expression for column '${colSpec.name}'`,
       ),
     ),
     reduceFn: colSpec.function2
-      ? _serializeValueSpecification(colSpec.function2)
+      ? engine.serializeValueSpecification(colSpec.function2)
       : undefined,
   }));
 }

package/src/stores/core/model/CachedDataCubeSource.ts

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { V1_PureModelContextData } from '@finos/legend-graph';
+import { DataCubeSource } from './DataCubeSource.js';
+
+export const CACHED_DATA_CUBE_SOURCE_TYPE = 'cached';
+
+export class CachedDataCubeSource extends DataCubeSource {
+  model!: V1_PureModelContextData;
+  runtime!: string;
+  db!: string;
+  schema!: string;
+  table!: string;
+  count!: number;
+}

package/src/stores/services/DataCubeQuerySnapshotService.ts

@@ -88,8 +88,8 @@ export abstract class DataCubeQuerySnapshotController
       this._engine.debugProcess(
         `New Snapshot`,
         ['Publisher', this.getSnapshotSubscriberName()],
-        ['Snapshot', snapshot],
-        ['Previous Snapshot', previousSnapshot],
+        ['Snapshot', snapshot.serialize()],
+        ['Previous Snapshot', previousSnapshot?.serialize()],
         ['Diff', deepDiff(previousSnapshot ?? {}, snapshot)],
       );
     }

package/src/stores/services/DataCubeTaskService.ts

@@ -20,6 +20,8 @@ import { action, computed, makeObservable, observable } from 'mobx';
 export class Task {
   readonly uuid = uuid();
   readonly name: string;
+  // use Date.now() vs. performance.now() because need to record the moment
+  // the task starts/ends for logging purposes.
   readonly startTime = Date.now();
   readonly ownerId?: string | undefined;
 

package/src/stores/view/DataCubeViewState.ts

@@ -24,13 +24,6 @@ import {
 import { DataCubeQuerySnapshotService } from '../services/DataCubeQuerySnapshotService.js';
 import { DataCubeInfoState } from './DataCubeInfoState.js';
 import { validateAndBuildQuerySnapshot } from '../core/DataCubeQuerySnapshotBuilder.js';
-import {
-  action,
-  computed,
-  makeObservable,
-  observable,
-  runInAction,
-} from 'mobx';
 import { DataCubeFilterEditorState } from './filter/DataCubeFilterEditorState.js';
 import { DataCubeExtendManagerState } from './extend/DataCubeExtendManagerState.js';
 import type { DataCubeState } from '../DataCubeState.js';
@@ -43,6 +36,7 @@ import { DataCubeConfiguration } from '../core/model/DataCubeConfiguration.js';
 import type { DataCubeLayoutService } from '../services/DataCubeLayoutService.js';
 import type { DataCubeAlertService } from '../services/DataCubeAlertService.js';
 import type { DataCubeSettingService } from '../services/DataCubeSettingService.js';
+import { CachedDataCubeSource } from '../core/model/CachedDataCubeSource.js';
 
 export class DataCubeViewState {
   readonly dataCube: DataCubeState;
@@ -61,17 +55,12 @@ export class DataCubeViewState {
   readonly extend: DataCubeExtendManagerState;
 
   readonly initializeState = ActionState.create();
+  readonly processCacheState = ActionState.create();
 
   private _source?: DataCubeSource | undefined;
+  private _originalSource?: DataCubeSource | undefined;
 
   constructor(dataCube: DataCubeState) {
-    makeObservable<DataCubeViewState, '_source'>(this, {
-      _source: observable,
-      source: computed,
-
-      initialize: action,
-    });
-
     this.dataCube = dataCube;
     this.engine = dataCube.engine;
     this.logService = dataCube.logService;
@@ -92,6 +81,10 @@ export class DataCubeViewState {
     this.extend = new DataCubeExtendManagerState(this);
   }
 
+  getOriginalSource() {
+    return this._originalSource;
+  }
+
   async generateDataCubeQuery() {
     const snapshot = this.snapshotService.currentSnapshot;
     const query = new DataCubeQuery();
@@ -110,6 +103,51 @@ export class DataCubeViewState {
     return this._source;
   }
 
+  async initializeCache() {
+    this.processCacheState.inProgress();
+    const task = this.taskService.newTask('Initializing Cache');
+
+    try {
+      const cachedSource = await this.engine.initializeCache(this.source);
+      if (cachedSource) {
+        this._source = cachedSource;
+      }
+      this.processCacheState.pass();
+    } catch (error) {
+      assertErrorThrown(error);
+      this.alertService.alertError(error, {
+        message: `Cache Processing Failure: ${error.message}`,
+      });
+      this._source = this._originalSource;
+      this.processCacheState.fail();
+    } finally {
+      this.taskService.endTask(task);
+    }
+  }
+
+  async disposeCache() {
+    if (!(this._source instanceof CachedDataCubeSource)) {
+      return;
+    }
+    const cachedSource = this._source;
+    this._source = this._originalSource;
+
+    this.processCacheState.inProgress();
+    const task = this.taskService.newTask('Disposing Cache');
+
+    try {
+      await this.engine.disposeCache(cachedSource);
+    } catch (error) {
+      assertErrorThrown(error);
+      this.alertService.alertError(error, {
+        message: `Cache Processing Failure: ${error.message}`,
+      });
+    } finally {
+      this.processCacheState.complete();
+      this.taskService.endTask(task);
+    }
+  }
+
   async initialize(query: DataCubeQuery) {
     this.initializeState.inProgress();
     const task = this.taskService.newTask('Initializing');
@@ -128,9 +166,8 @@ export class DataCubeViewState {
         }),
       );
       const source = await this.engine.processQuerySource(query.source);
-      runInAction(() => {
-        this._source = source;
-      });
+      this._source = source;
+      this._originalSource = source;
       const partialQuery = await this.engine.parseValueSpecification(
         query.query,
       );

package/src/stores/view/extend/DataCubeColumnEditorState.tsx

@@ -47,11 +47,7 @@ import {
 } from '@finos/legend-graph';
 import type { DataCubeColumnConfiguration } from '../../core/model/DataCubeConfiguration.js';
 import type { DataCubeQuerySnapshotExtendedColumn } from '../../core/DataCubeQuerySnapshot.js';
-import {
-  _lambda,
-  _serializeValueSpecification,
-  _deserializeValueSpecification,
-} from '../../core/DataCubeQueryBuilderUtils.js';
+import { _lambda } from '../../core/DataCubeQueryBuilderUtils.js';
 import { _findCol } from '../../core/model/DataCubeColumn.js';
 
 export abstract class DataCubeColumnBaseEditorState {
@@ -220,13 +216,9 @@ export abstract class DataCubeColumnBaseEditorState {
     return _lambda(
       [],
       [
-        buildExecutableQuery(
-          snapshot,
-          this.view.source,
-          () => undefined,
-          this.view.engine.filterOperations,
-          this.view.engine.aggregateOperations,
-        ),
+        buildExecutableQuery(snapshot, this.view.source, this.view.engine, {
+          skipExecutionContext: true,
+        }),
       ],
     );
   }
@@ -390,7 +382,7 @@ export class DataCubeNewColumnState extends DataCubeColumnBaseEditorState {
       {
         name: this.name,
         type: returnType,
-        mapFn: _serializeValueSpecification(query),
+        mapFn: this.view.engine.serializeValueSpecification(query),
       },
       this.isGroupLevel,
       this.columnKind,
@@ -440,7 +432,7 @@ export class DataCubeExistingColumnEditorState extends DataCubeColumnBaseEditorS
 
   override async getInitialCode(): Promise<string> {
     return this.view.engine.getValueSpecificationCode(
-      _deserializeValueSpecification(this.initialData.mapFn),
+      this.view.engine.deserializeValueSpecification(this.initialData.mapFn),
       true,
     );
   }
@@ -516,7 +508,7 @@ export class DataCubeExistingColumnEditorState extends DataCubeColumnBaseEditorS
       {
         name: this.name,
         type: returnType,
-        mapFn: _serializeValueSpecification(query),
+        mapFn: this.view.engine.serializeValueSpecification(query),
       },
       this.isGroupLevel,
       this.columnKind,

package/src/stores/view/grid/DataCubeGridClientEngine.ts

@@ -121,6 +121,7 @@ export const generateBackgroundColorUtilityClassName = (
 // See https://www.ag-grid.com/react-data-grid/server-side-model-configuration/#server-side-cache
 export const INTERNAL__GRID_CLIENT_DEFAULT_CACHE_BLOCK_SIZE = 500;
 export const INTERNAL__GRID_CLIENT_DEFAULT_ENABLE_PAGINATION = true;
+export const INTERNAL__GRID_CLIENT_DEFAULT_ENABLE_CACHING = false;
 // NOTE: The cache block size is used by ag-grid to pre-allocate memory for the grid
 // so the value set must be reasonable, or else it can crash the engine!
 export const INTERNAL__GRID_CLIENT_MAX_CACHE_BLOCK_SIZE = 1e3;
@@ -264,29 +265,15 @@ async function getCastColumns(
   snapshot.data.groupExtendedColumns = [];
   snapshot.data.sortColumns = [];
   snapshot.data.limit = 0;
-  const query = buildExecutableQuery(
-    snapshot,
-    view.source,
-    (source) => view.engine.buildExecutionContext(source),
-    view.engine.filterOperations,
-    view.engine.aggregateOperations,
-    {
-      postProcessor: (
-        _snapshot,
-        sequence,
-        funcMap,
-        configuration,
-        filterOperations,
-        aggregateOperations,
-      ) => {
-        // when both pivot and groupBy present, we need to account for the count column
-        // in the cast expression
-        if (funcMap.pivot && currentSnapshot.data.groupBy) {
-          _colSpecArrayParam(funcMap.pivot, 1).colSpecs.push(_aggCountCol());
-        }
-      },
+  const query = buildExecutableQuery(snapshot, view.source, view.engine, {
+    postProcessor: (_snapshot, sequence, funcMap, configuration, engine) => {
+      // when both pivot and groupBy present, we need to account for the count column
+      // in the cast expression
+      if (funcMap.pivot && currentSnapshot.data.groupBy) {
+        _colSpecArrayParam(funcMap.pivot, 1).colSpecs.push(_aggCountCol());
+      }
     },
-  );
+  });
 
   const lambda = new V1_Lambda();
   lambda.body.push(query);
@@ -412,13 +399,22 @@ export class DataCubeGridClientServerSideDataSource
         request,
         newSnapshot,
         this._view.source,
-        (source) => this._view.engine.buildExecutionContext(source),
-        this._view.engine.filterOperations,
-        this._view.engine.aggregateOperations,
+        this._view.engine,
         this._grid.isPaginationEnabled,
       );
       const lambda = new V1_Lambda();
       lambda.body.push(executableQuery);
+      if (
+        this._view.settingService.getBooleanValue(
+          DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
+        )
+      ) {
+        this._view.engine.debugProcess(
+          `Execution Plan`,
+          ['Query', lambda],
+          ['Config', `pagination=${this._grid.isPaginationEnabled}`],
+        );
+      }
       const result = await this._view.engine.executeQuery(
         lambda,
         this._view.source,
@@ -442,12 +438,12 @@ export class DataCubeGridClientServerSideDataSource
         this._view.engine.debugProcess(
           `Execution`,
           ['Query', result.executedQuery],
-          ['Config', `pagination=${this._grid.isPaginationEnabled}`],
           [
             'Stats',
             `${rowData.length} rows, ${result.result.result.columns.length} columns`,
           ],
           ['SQL', result.executedSQL],
+          ['SQL Execution Time', result.executionTime],
         );
       }
 

package/src/stores/view/grid/DataCubeGridQueryBuilder.ts

@@ -60,11 +60,10 @@ import {
   type DataCubeGridClientDataFetchRequest,
 } from './DataCubeGridClientEngine.js';
 import { DataCubeConfiguration } from '../../core/model/DataCubeConfiguration.js';
-import type { DataCubeQueryFilterOperation } from '../../core/filter/DataCubeQueryFilterOperation.js';
-import type { DataCubeQueryAggregateOperation } from '../../core/aggregation/DataCubeQueryAggregateOperation.js';
 import { _colSpecArrayParam } from '../../core/DataCubeQuerySnapshotBuilderUtils.js';
 import { buildExecutableQuery } from '../../core/DataCubeQueryBuilder.js';
 import type { DataCubeSource } from '../../core/model/DataCubeSource.js';
+import type { DataCubeEngine } from '../../core/DataCubeEngine.js';
 
 /*****************************************************************************
  * [GRID]
@@ -148,11 +147,7 @@ export function buildGridDataFetchExecutableQuery(
   request: DataCubeGridClientDataFetchRequest,
   snapshot: DataCubeQuerySnapshot,
   source: DataCubeSource,
-  executionContextBuilder: (
-    source: DataCubeSource,
-  ) => V1_AppliedFunction | undefined,
-  filterOperations: DataCubeQueryFilterOperation[],
-  aggregateOperations: DataCubeQueryAggregateOperation[],
+  engine: DataCubeEngine,
   enablePagination: boolean,
 ) {
   const processedSnapshot = snapshot.clone();
@@ -170,28 +165,21 @@ export function buildGridDataFetchExecutableQuery(
       ],
     };
   }
-  return buildExecutableQuery(
-    processedSnapshot,
-    source,
-    executionContextBuilder,
-    filterOperations,
-    aggregateOperations,
-    {
-      postProcessor: generateGridDataFetchExecutableQueryPostProcessor(request),
-      rootAggregation: {
-        columnName: INTERNAL__GRID_CLIENT_ROOT_AGGREGATION_COLUMN_ID,
-      },
-      pagination:
-        enablePagination &&
-        request.startRow !== undefined &&
-        request.endRow !== undefined
-          ? {
-              start: request.startRow,
-              end: request.endRow,
-            }
-          : undefined,
+  return buildExecutableQuery(processedSnapshot, source, engine, {
+    postProcessor: generateGridDataFetchExecutableQueryPostProcessor(request),
+    rootAggregation: {
+      columnName: INTERNAL__GRID_CLIENT_ROOT_AGGREGATION_COLUMN_ID,
     },
-  );
+    pagination:
+      enablePagination &&
+      request.startRow !== undefined &&
+      request.endRow !== undefined
+        ? {
+            start: request.startRow,
+            end: request.endRow,
+          }
+        : undefined,
+  });
 }
 
 function generateGridDataFetchExecutableQueryPostProcessor(
@@ -202,8 +190,7 @@ function generateGridDataFetchExecutableQueryPostProcessor(
     sequence: V1_AppliedFunction[],
     funcMap: DataCubeQueryFunctionMap,
     configuration: DataCubeConfiguration,
-    filterOperations: DataCubeQueryFilterOperation[],
-    aggregateOperations: DataCubeQueryAggregateOperation[],
+    engine: DataCubeEngine,
   ) => {
     const _unprocess = _functionCompositionUnProcessor(sequence, funcMap);
     const data = snapshot.data;
@@ -285,7 +272,7 @@ function generateGridDataFetchExecutableQueryPostProcessor(
                     }),
                     groupOperator: DataCubeQueryFilterGroupOperator.AND,
                   },
-                  filterOperations,
+                  engine.filterOperations,
                 ),
               ],
             ),
@@ -312,7 +299,7 @@ function generateGridDataFetchExecutableQueryPostProcessor(
               groupByColumns,
               snapshot,
               configuration,
-              aggregateOperations,
+              engine.aggregateOperations,
             ),
 
             // if pivot is present, add sum aggregation columns for each