@finos/legend-data-cube 0.1.0 → 0.1.1

package/src/stores/core/DataCubeQuerySnapshotBuilder.ts

@@ -73,10 +73,6 @@ import {
 } from './DataCubeQuerySnapshotBuilderUtils.js';
 import type { DataCubeSource } from './model/DataCubeSource.js';
 import type { DataCubeEngine } from './DataCubeEngine.js';
-import {
-  _deserializeValueSpecification,
-  _serializeValueSpecification,
-} from './DataCubeQueryBuilderUtils.js';
 
 // --------------------------------- BUILDING BLOCKS ---------------------------------
 
@@ -365,8 +361,8 @@ export async function validateAndBuildQuerySnapshot(
   // analysis more ergonomic
 
   // Clone the query since we will mutate it during the process
-  const query = _deserializeValueSpecification(
-    _serializeValueSpecification(partialQuery),
+  const query = engine.deserializeValueSpecification(
+    engine.serializeValueSpecification(partialQuery),
   );
   const funcMap = extractFunctionMap(query);
   const snapshot = DataCubeQuerySnapshot.create({});

package/src/stores/core/DataCubeQuerySnapshotBuilderUtils.ts

@@ -79,7 +79,6 @@ import {
   _colSpec,
   _function,
   _lambda,
-  _serializeValueSpecification,
   _synthesizeMinimalSourceQuery,
 } from './DataCubeQueryBuilderUtils.js';
 import { INTERNAL__DataCubeSource } from './model/DataCubeSource.js';
@@ -360,14 +359,14 @@ export async function _extractExtendedColumns(
       _findCol(columns, colSpec.name),
       `Can't process extend() expression: failed to retrieve type information for column '${colSpec.name}'`,
     ).type,
-    mapFn: _serializeValueSpecification(
+    mapFn: engine.serializeValueSpecification(
       guaranteeNonNullable(
         colSpec.function1,
         `Can't process extend() expression: expected a transformation function expression for column '${colSpec.name}'`,
       ),
     ),
     reduceFn: colSpec.function2
-      ? _serializeValueSpecification(colSpec.function2)
+      ? engine.serializeValueSpecification(colSpec.function2)
       : undefined,
   }));
 }

package/src/stores/core/model/CachedDataCubeSource.ts

@@ -22,4 +22,8 @@ export const CACHED_DATA_CUBE_SOURCE_TYPE = 'cached';
 export class CachedDataCubeSource extends DataCubeSource {
   model!: V1_PureModelContextData;
   runtime!: string;
+  db!: string;
+  schema!: string;
+  table!: string;
+  count!: number;
 }

package/src/stores/services/DataCubeQuerySnapshotService.ts

@@ -88,8 +88,8 @@ export abstract class DataCubeQuerySnapshotController
       this._engine.debugProcess(
         `New Snapshot`,
         ['Publisher', this.getSnapshotSubscriberName()],
-        ['Snapshot', snapshot],
-        ['Previous Snapshot', previousSnapshot],
+        ['Snapshot', snapshot.serialize()],
+        ['Previous Snapshot', previousSnapshot?.serialize()],
         ['Diff', deepDiff(previousSnapshot ?? {}, snapshot)],
       );
     }

package/src/stores/services/DataCubeTaskService.ts

@@ -20,6 +20,8 @@ import { action, computed, makeObservable, observable } from 'mobx';
 export class Task {
   readonly uuid = uuid();
   readonly name: string;
+  // use Date.now() vs. performance.now() because need to record the moment
+  // the task starts/ends for logging purposes.
   readonly startTime = Date.now();
   readonly ownerId?: string | undefined;
 

package/src/stores/view/DataCubeViewState.ts

@@ -24,13 +24,6 @@ import {
 import { DataCubeQuerySnapshotService } from '../services/DataCubeQuerySnapshotService.js';
 import { DataCubeInfoState } from './DataCubeInfoState.js';
 import { validateAndBuildQuerySnapshot } from '../core/DataCubeQuerySnapshotBuilder.js';
-import {
-  action,
-  computed,
-  makeObservable,
-  observable,
-  runInAction,
-} from 'mobx';
 import { DataCubeFilterEditorState } from './filter/DataCubeFilterEditorState.js';
 import { DataCubeExtendManagerState } from './extend/DataCubeExtendManagerState.js';
 import type { DataCubeState } from '../DataCubeState.js';
@@ -43,6 +36,7 @@ import { DataCubeConfiguration } from '../core/model/DataCubeConfiguration.js';
 import type { DataCubeLayoutService } from '../services/DataCubeLayoutService.js';
 import type { DataCubeAlertService } from '../services/DataCubeAlertService.js';
 import type { DataCubeSettingService } from '../services/DataCubeSettingService.js';
+import { CachedDataCubeSource } from '../core/model/CachedDataCubeSource.js';
 
 export class DataCubeViewState {
   readonly dataCube: DataCubeState;
@@ -61,18 +55,12 @@ export class DataCubeViewState {
   readonly extend: DataCubeExtendManagerState;
 
   readonly initializeState = ActionState.create();
+  readonly processCacheState = ActionState.create();
 
   private _source?: DataCubeSource | undefined;
   private _originalSource?: DataCubeSource | undefined;
 
   constructor(dataCube: DataCubeState) {
-    makeObservable<DataCubeViewState, '_source'>(this, {
-      _source: observable,
-      source: computed,
-
-      initialize: action,
-    });
-
     this.dataCube = dataCube;
     this.engine = dataCube.engine;
     this.logService = dataCube.logService;
@@ -93,6 +81,10 @@ export class DataCubeViewState {
     this.extend = new DataCubeExtendManagerState(this);
   }
 
+  getOriginalSource() {
+    return this._originalSource;
+  }
+
   async generateDataCubeQuery() {
     const snapshot = this.snapshotService.currentSnapshot;
     const query = new DataCubeQuery();
@@ -112,18 +104,48 @@ export class DataCubeViewState {
   }
 
   async initializeCache() {
-    this.initializeState.inProgress();
-    const cachedSource = await this.engine.initializeCache(this.source);
-    if (cachedSource !== undefined) {
-      this._originalSource = this._source;
-      this._source = cachedSource;
+    this.processCacheState.inProgress();
+    const task = this.taskService.newTask('Initializing Cache');
+
+    try {
+      const cachedSource = await this.engine.initializeCache(this.source);
+      if (cachedSource) {
+        this._source = cachedSource;
+      }
+      this.processCacheState.pass();
+    } catch (error) {
+      assertErrorThrown(error);
+      this.alertService.alertError(error, {
+        message: `Cache Processing Failure: ${error.message}`,
+      });
+      this._source = this._originalSource;
+      this.processCacheState.fail();
+    } finally {
+      this.taskService.endTask(task);
     }
-    this.initializeState.pass();
   }
 
-  async clearCache() {
-    await this.engine.clearCache();
+  async disposeCache() {
+    if (!(this._source instanceof CachedDataCubeSource)) {
+      return;
+    }
+    const cachedSource = this._source;
     this._source = this._originalSource;
+
+    this.processCacheState.inProgress();
+    const task = this.taskService.newTask('Disposing Cache');
+
+    try {
+      await this.engine.disposeCache(cachedSource);
+    } catch (error) {
+      assertErrorThrown(error);
+      this.alertService.alertError(error, {
+        message: `Cache Processing Failure: ${error.message}`,
+      });
+    } finally {
+      this.processCacheState.complete();
+      this.taskService.endTask(task);
+    }
   }
 
   async initialize(query: DataCubeQuery) {
@@ -144,9 +166,8 @@ export class DataCubeViewState {
         }),
       );
       const source = await this.engine.processQuerySource(query.source);
-      runInAction(() => {
-        this._source = source;
-      });
+      this._source = source;
+      this._originalSource = source;
       const partialQuery = await this.engine.parseValueSpecification(
         query.query,
       );
@@ -156,7 +177,6 @@ export class DataCubeViewState {
         query,
         this.engine,
       );
-      this.engine.processInitialSnapshot?.(source, initialSnapshot);
       this.snapshotService.broadcastSnapshot(initialSnapshot);
       this.dataCube.options?.onViewInitialized?.({
         api: this.dataCube.api,

package/src/stores/view/extend/DataCubeColumnEditorState.tsx

@@ -47,11 +47,7 @@ import {
 } from '@finos/legend-graph';
 import type { DataCubeColumnConfiguration } from '../../core/model/DataCubeConfiguration.js';
 import type { DataCubeQuerySnapshotExtendedColumn } from '../../core/DataCubeQuerySnapshot.js';
-import {
-  _lambda,
-  _serializeValueSpecification,
-  _deserializeValueSpecification,
-} from '../../core/DataCubeQueryBuilderUtils.js';
+import { _lambda } from '../../core/DataCubeQueryBuilderUtils.js';
 import { _findCol } from '../../core/model/DataCubeColumn.js';
 
 export abstract class DataCubeColumnBaseEditorState {
@@ -220,13 +216,9 @@ export abstract class DataCubeColumnBaseEditorState {
     return _lambda(
       [],
       [
-        buildExecutableQuery(
-          snapshot,
-          this.view.source,
-          () => undefined,
-          this.view.engine.filterOperations,
-          this.view.engine.aggregateOperations,
-        ),
+        buildExecutableQuery(snapshot, this.view.source, this.view.engine, {
+          skipExecutionContext: true,
+        }),
       ],
     );
   }
@@ -390,7 +382,7 @@ export class DataCubeNewColumnState extends DataCubeColumnBaseEditorState {
       {
         name: this.name,
         type: returnType,
-        mapFn: _serializeValueSpecification(query),
+        mapFn: this.view.engine.serializeValueSpecification(query),
       },
       this.isGroupLevel,
       this.columnKind,
@@ -440,7 +432,7 @@ export class DataCubeExistingColumnEditorState extends DataCubeColumnBaseEditorS
 
   override async getInitialCode(): Promise<string> {
     return this.view.engine.getValueSpecificationCode(
-      _deserializeValueSpecification(this.initialData.mapFn),
+      this.view.engine.deserializeValueSpecification(this.initialData.mapFn),
       true,
     );
   }
@@ -516,7 +508,7 @@ export class DataCubeExistingColumnEditorState extends DataCubeColumnBaseEditorS
       {
         name: this.name,
         type: returnType,
-        mapFn: _serializeValueSpecification(query),
+        mapFn: this.view.engine.serializeValueSpecification(query),
       },
       this.isGroupLevel,
       this.columnKind,

package/src/stores/view/grid/DataCubeGridClientEngine.ts

@@ -265,29 +265,15 @@ async function getCastColumns(
   snapshot.data.groupExtendedColumns = [];
   snapshot.data.sortColumns = [];
   snapshot.data.limit = 0;
-  const query = buildExecutableQuery(
-    snapshot,
-    view.source,
-    (source) => view.engine.buildExecutionContext(source),
-    view.engine.filterOperations,
-    view.engine.aggregateOperations,
-    {
-      postProcessor: (
-        _snapshot,
-        sequence,
-        funcMap,
-        configuration,
-        filterOperations,
-        aggregateOperations,
-      ) => {
-        // when both pivot and groupBy present, we need to account for the count column
-        // in the cast expression
-        if (funcMap.pivot && currentSnapshot.data.groupBy) {
-          _colSpecArrayParam(funcMap.pivot, 1).colSpecs.push(_aggCountCol());
-        }
-      },
+  const query = buildExecutableQuery(snapshot, view.source, view.engine, {
+    postProcessor: (_snapshot, sequence, funcMap, configuration, engine) => {
+      // when both pivot and groupBy present, we need to account for the count column
+      // in the cast expression
+      if (funcMap.pivot && currentSnapshot.data.groupBy) {
+        _colSpecArrayParam(funcMap.pivot, 1).colSpecs.push(_aggCountCol());
+      }
     },
-  );
+  });
 
   const lambda = new V1_Lambda();
   lambda.body.push(query);
@@ -413,13 +399,22 @@ export class DataCubeGridClientServerSideDataSource
         request,
         newSnapshot,
         this._view.source,
-        (source) => this._view.engine.buildExecutionContext(source),
-        this._view.engine.filterOperations,
-        this._view.engine.aggregateOperations,
+        this._view.engine,
         this._grid.isPaginationEnabled,
       );
       const lambda = new V1_Lambda();
       lambda.body.push(executableQuery);
+      if (
+        this._view.settingService.getBooleanValue(
+          DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
+        )
+      ) {
+        this._view.engine.debugProcess(
+          `Execution Plan`,
+          ['Query', lambda],
+          ['Config', `pagination=${this._grid.isPaginationEnabled}`],
+        );
+      }
       const result = await this._view.engine.executeQuery(
         lambda,
         this._view.source,
@@ -443,7 +438,6 @@ export class DataCubeGridClientServerSideDataSource
         this._view.engine.debugProcess(
           `Execution`,
           ['Query', result.executedQuery],
-          ['Config', `pagination=${this._grid.isPaginationEnabled}`],
           [
             'Stats',
             `${rowData.length} rows, ${result.result.result.columns.length} columns`,

package/src/stores/view/grid/DataCubeGridQueryBuilder.ts

@@ -60,11 +60,10 @@ import {
   type DataCubeGridClientDataFetchRequest,
 } from './DataCubeGridClientEngine.js';
 import { DataCubeConfiguration } from '../../core/model/DataCubeConfiguration.js';
-import type { DataCubeQueryFilterOperation } from '../../core/filter/DataCubeQueryFilterOperation.js';
-import type { DataCubeQueryAggregateOperation } from '../../core/aggregation/DataCubeQueryAggregateOperation.js';
 import { _colSpecArrayParam } from '../../core/DataCubeQuerySnapshotBuilderUtils.js';
 import { buildExecutableQuery } from '../../core/DataCubeQueryBuilder.js';
 import type { DataCubeSource } from '../../core/model/DataCubeSource.js';
+import type { DataCubeEngine } from '../../core/DataCubeEngine.js';
 
 /*****************************************************************************
  * [GRID]
@@ -148,11 +147,7 @@ export function buildGridDataFetchExecutableQuery(
   request: DataCubeGridClientDataFetchRequest,
   snapshot: DataCubeQuerySnapshot,
   source: DataCubeSource,
-  executionContextBuilder: (
-    source: DataCubeSource,
-  ) => V1_AppliedFunction | undefined,
-  filterOperations: DataCubeQueryFilterOperation[],
-  aggregateOperations: DataCubeQueryAggregateOperation[],
+  engine: DataCubeEngine,
   enablePagination: boolean,
 ) {
   const processedSnapshot = snapshot.clone();
@@ -170,28 +165,21 @@ export function buildGridDataFetchExecutableQuery(
       ],
     };
   }
-  return buildExecutableQuery(
-    processedSnapshot,
-    source,
-    executionContextBuilder,
-    filterOperations,
-    aggregateOperations,
-    {
-      postProcessor: generateGridDataFetchExecutableQueryPostProcessor(request),
-      rootAggregation: {
-        columnName: INTERNAL__GRID_CLIENT_ROOT_AGGREGATION_COLUMN_ID,
-      },
-      pagination:
-        enablePagination &&
-        request.startRow !== undefined &&
-        request.endRow !== undefined
-          ? {
-              start: request.startRow,
-              end: request.endRow,
-            }
-          : undefined,
+  return buildExecutableQuery(processedSnapshot, source, engine, {
+    postProcessor: generateGridDataFetchExecutableQueryPostProcessor(request),
+    rootAggregation: {
+      columnName: INTERNAL__GRID_CLIENT_ROOT_AGGREGATION_COLUMN_ID,
     },
-  );
+    pagination:
+      enablePagination &&
+      request.startRow !== undefined &&
+      request.endRow !== undefined
+        ? {
+            start: request.startRow,
+            end: request.endRow,
+          }
+        : undefined,
+  });
 }
 
 function generateGridDataFetchExecutableQueryPostProcessor(
@@ -202,8 +190,7 @@ function generateGridDataFetchExecutableQueryPostProcessor(
     sequence: V1_AppliedFunction[],
     funcMap: DataCubeQueryFunctionMap,
     configuration: DataCubeConfiguration,
-    filterOperations: DataCubeQueryFilterOperation[],
-    aggregateOperations: DataCubeQueryAggregateOperation[],
+    engine: DataCubeEngine,
   ) => {
     const _unprocess = _functionCompositionUnProcessor(sequence, funcMap);
     const data = snapshot.data;
@@ -285,7 +272,7 @@ function generateGridDataFetchExecutableQueryPostProcessor(
                     }),
                     groupOperator: DataCubeQueryFilterGroupOperator.AND,
                   },
-                  filterOperations,
+                  engine.filterOperations,
                 ),
               ],
             ),
@@ -312,7 +299,7 @@ function generateGridDataFetchExecutableQueryPostProcessor(
               groupByColumns,
               snapshot,
               configuration,
-              aggregateOperations,
+              engine.aggregateOperations,
             ),
 
             // if pivot is present, add sum aggregation columns for each

package/src/stores/view/grid/DataCubeGridState.ts

@@ -19,7 +19,7 @@ import {
   guaranteeNonNullable,
   type DebouncedFunc,
 } from '@finos/legend-shared';
-import { action, makeObservable, observable } from 'mobx';
+import { action, makeObservable, observable, runInAction } from 'mobx';
 import type { GridApi } from 'ag-grid-community';
 import type { DataCubeViewState } from '../DataCubeViewState.js';
 import {
@@ -125,16 +125,32 @@ export class DataCubeGridState extends DataCubeQuerySnapshotController {
   }
 
   async setCachingEnabled(val: boolean) {
-    this.isCachingEnabled = val;
-    if (this.isCachingEnabled) {
-      this._client?.setGridOption('loading', true);
+    const currentVal = this.isCachingEnabled;
+    if (val === currentVal) {
+      return;
+    }
+
+    runInAction(() => {
+      this.isCachingEnabled = val;
+    });
+
+    if (val) {
       await this._view.initializeCache();
-      this._client?.setGridOption('loading', false);
+      if (this._view.processCacheState.hasFailed) {
+        this.isCachingEnabled = currentVal;
+        return;
+      }
     } else {
-      this._client?.setGridOption('loading', true);
-      await this._view.clearCache();
-      this._client?.setGridOption('loading', false);
+      await this._view.disposeCache();
     }
+
+    runInAction(() => {
+      // hard reset the grid, this will force the grid to fetch data again
+      this.clientDataSource = new DataCubeGridClientServerSideDataSource(
+        this,
+        this._view,
+      );
+    });
   }
 
   setScrollHintText(val: string | undefined) {