@finos/legend-data-cube 0.1.0 → 0.1.2

package/src/stores/core/DataCubeConfigurationBuilder.ts

@@ -133,7 +133,7 @@ export function newConfiguration(
   //
   // since we have made sure all groupBy sort columns must be of the same direction
   // we simply retrieve the direction from the one of the column provided
-  if (groupBySortColumns.length) {
+  if (groupBySortColumns?.length) {
     configuration.treeColumnSortDirection = at(groupBySortColumns, 0).direction;
   }
 

package/src/stores/core/DataCubeEngine.tsx

@@ -20,6 +20,8 @@ import {
   type TDSExecutionResult,
   type V1_AppliedFunction,
   PRIMITIVE_TYPE,
+  V1_deserializeValueSpecification,
+  V1_serializeValueSpecification,
 } from '@finos/legend-graph';
 import {
   getFilterOperation,
@@ -65,20 +67,25 @@ import { DataCubeQueryFilterOperation__EndWithCaseInsensitive } from './filter/D
 import { DataCubeQueryFilterOperation__NotEndWith } from './filter/DataCubeQueryFilterOperation__NotEndWith.js';
 import { DataCubeQueryFilterOperation__IsNull } from './filter/DataCubeQueryFilterOperation__IsNull.js';
 import { DataCubeQueryFilterOperation__IsNotNull } from './filter/DataCubeQueryFilterOperation__IsNotNull.js';
-import type { DataCubeQuerySnapshot } from './DataCubeQuerySnapshot.js';
+import { DataCubeQuerySnapshot } from './DataCubeQuerySnapshot.js';
 import { buildExecutableQuery } from './DataCubeQueryBuilder.js';
-import type { DataCubeColumn } from './model/DataCubeColumn.js';
+import { _toCol, type DataCubeColumn } from './model/DataCubeColumn.js';
 import {
   type DataCubeSource,
   INTERNAL__DataCubeSource,
 } from './model/DataCubeSource.js';
-import { _primitiveValue } from './DataCubeQueryBuilderUtils.js';
+import {
+  _primitiveValue,
+  _selectFunction,
+} from './DataCubeQueryBuilderUtils.js';
 import {
   type DocumentationEntry,
   type LogEvent,
   type PlainObject,
 } from '@finos/legend-shared';
 import type { CachedDataCubeSource } from './model/CachedDataCubeSource.js';
+import { DataCubeQuery } from './model/DataCubeQuery.js';
+import { newConfiguration } from './DataCubeConfigurationBuilder.js';
 
 export type CompletionItem = {
   completion: string;
@@ -94,6 +101,11 @@ export type DataCubeExecutionOptions = {
   clientVersion?: string | undefined;
 };
 
+export type DataCubeCacheInitializationOptions = {
+  debug?: boolean | undefined;
+  clientVersion?: string | undefined;
+};
+
 export type DataCubeExecutionResult = {
   result: TDSExecutionResult;
   executedQuery: string;
@@ -168,6 +180,14 @@ export abstract class DataCubeEngine {
     return getAggregateOperation(value, this.aggregateOperations);
   }
 
+  deserializeValueSpecification(json: PlainObject<V1_ValueSpecification>) {
+    return V1_deserializeValueSpecification(json, []);
+  }
+
+  serializeValueSpecification(protocol: V1_ValueSpecification) {
+    return V1_serializeValueSpecification(protocol, []);
+  }
+
   /**
    * By default, for a function chain, Pure grammar composer will extract the first parameter of the first function
    * and render it as the caller of that function rather than a parameter
@@ -184,21 +204,34 @@ export abstract class DataCubeEngine {
     source.query = _primitiveValue(PRIMITIVE_TYPE.STRING, '');
     return (
       await this.getValueSpecificationCode(
-        buildExecutableQuery(
-          snapshot,
-          source,
-          () => undefined,
-          this.filterOperations,
-          this.aggregateOperations,
-        ),
+        buildExecutableQuery(snapshot, source, this, {
+          skipExecutionContext: true,
+        }),
         pretty,
       )
     ).substring(`''->`.length);
   }
 
+  async generateBaseQuery(sourceData: PlainObject) {
+    const source = await this.processQuerySource(sourceData);
+    const query = new DataCubeQuery();
+    query.source = sourceData;
+    query.query = await this.getValueSpecificationCode(
+      _selectFunction(source.columns),
+    );
+    const snapshot = DataCubeQuerySnapshot.create({});
+    snapshot.data.sourceColumns = source.columns.map(_toCol);
+    snapshot.data.selectColumns = source.columns.map(_toCol);
+    const configuration = newConfiguration({
+      snapshot,
+    });
+    query.configuration = configuration;
+    return query;
+  }
+
   // ---------------------------------- PROCESSOR ----------------------------------
 
-  abstract processQuerySource(value: PlainObject): Promise<DataCubeSource>;
+  abstract processQuerySource(sourceData: PlainObject): Promise<DataCubeSource>;
 
   abstract parseValueSpecification(
     code: string,
@@ -237,22 +270,16 @@ export abstract class DataCubeEngine {
     source: DataCubeSource,
   ): V1_AppliedFunction | undefined;
 
-  processInitialSnapshot?(
-    source: DataCubeSource,
-    snapshot: DataCubeQuerySnapshot,
-  ): DataCubeQuerySnapshot;
-
-  // ---------------------------------- CACHING --------------------------------------
+  // ---------------------------------- CACHING ----------------------------------
 
   async initializeCache(
     source: DataCubeSource,
+    options?: DataCubeCacheInitializationOptions | undefined,
   ): Promise<CachedDataCubeSource | undefined> {
     return undefined;
   }
 
-  async clearCache() {
-    // do nothing
-  }
+  async disposeCache(source: CachedDataCubeSource) {}
 
   // ---------------------------------- APPLICATION ----------------------------------
 

package/src/stores/core/DataCubeQueryBuilder.ts

@@ -21,22 +21,24 @@
  * the query snapshot. The executable query is then used to fetch data.
  ***************************************************************************************/
 
-import { PRIMITIVE_TYPE, type V1_AppliedFunction } from '@finos/legend-graph';
+import {
+  PRIMITIVE_TYPE,
+  V1_Lambda,
+  type V1_AppliedFunction,
+} from '@finos/legend-graph';
 import { type DataCubeQuerySnapshot } from './DataCubeQuerySnapshot.js';
-import { at } from '@finos/legend-shared';
+import { at, guaranteeType } from '@finos/legend-shared';
 import {
   DataCubeFunction,
   DataCubeQuerySortDirection,
   type DataCubeQueryFunctionMap,
 } from './DataCubeQueryEngine.js';
 import { DataCubeConfiguration } from './model/DataCubeConfiguration.js';
-import type { DataCubeQueryFilterOperation } from './filter/DataCubeQueryFilterOperation.js';
 import {
   _col,
   _collection,
   _cols,
   _colSpec,
-  _deserializeLambda,
   _filter,
   _function,
   _groupByAggCols,
@@ -48,26 +50,21 @@ import {
   _functionCompositionProcessor,
   _extendRootAggregation,
 } from './DataCubeQueryBuilderUtils.js';
-import type { DataCubeQueryAggregateOperation } from './aggregation/DataCubeQueryAggregateOperation.js';
 import type { DataCubeSource } from './model/DataCubeSource.js';
 import { _findCol } from './model/DataCubeColumn.js';
+import type { DataCubeEngine } from './DataCubeEngine.js';
 
 export function buildExecutableQuery(
   snapshot: DataCubeQuerySnapshot,
   source: DataCubeSource,
-  executionContextBuilder: (
-    source: DataCubeSource,
-  ) => V1_AppliedFunction | undefined,
-  filterOperations: DataCubeQueryFilterOperation[],
-  aggregateOperations: DataCubeQueryAggregateOperation[],
+  engine: DataCubeEngine,
   options?: {
     postProcessor?: (
       snapshot: DataCubeQuerySnapshot,
       sequence: V1_AppliedFunction[],
       funcMap: DataCubeQueryFunctionMap,
       configuration: DataCubeConfiguration,
-      filterOperations: DataCubeQueryFilterOperation[],
-      aggregateOperations: DataCubeQueryAggregateOperation[],
+      engine: DataCubeEngine,
     ) => void;
     rootAggregation?:
       | {
@@ -101,8 +98,16 @@ export function buildExecutableQuery(
           _cols([
             _colSpec(
               col.name,
-              _deserializeLambda(col.mapFn),
-              col.reduceFn ? _deserializeLambda(col.reduceFn) : undefined,
+              guaranteeType(
+                engine.deserializeValueSpecification(col.mapFn),
+                V1_Lambda,
+              ),
+              col.reduceFn
+                ? guaranteeType(
+                    engine.deserializeValueSpecification(col.reduceFn),
+                    V1_Lambda,
+                  )
+                : undefined,
             ),
           ]),
         ]),
@@ -116,7 +121,7 @@ export function buildExecutableQuery(
     _process(
       'filter',
       _function(DataCubeFunction.FILTER, [
-        _lambda([_var()], [_filter(data.filter, filterOperations)]),
+        _lambda([_var()], [_filter(data.filter, engine.filterOperations)]),
       ]),
     );
   }
@@ -164,7 +169,7 @@ export function buildExecutableQuery(
             pivot.columns,
             snapshot,
             configuration,
-            aggregateOperations,
+            engine.aggregateOperations,
           ),
         ),
       ]),
@@ -194,7 +199,7 @@ export function buildExecutableQuery(
             groupBy.columns,
             snapshot,
             configuration,
-            aggregateOperations,
+            engine.aggregateOperations,
           ),
         ),
       ]),
@@ -227,8 +232,16 @@ export function buildExecutableQuery(
           _cols([
             _colSpec(
               col.name,
-              _deserializeLambda(col.mapFn),
-              col.reduceFn ? _deserializeLambda(col.reduceFn) : undefined,
+              guaranteeType(
+                engine.deserializeValueSpecification(col.mapFn),
+                V1_Lambda,
+              ),
+              col.reduceFn
+                ? guaranteeType(
+                    engine.deserializeValueSpecification(col.reduceFn),
+                    V1_Lambda,
+                  )
+                : undefined,
             ),
           ]),
         ]),
@@ -280,19 +293,14 @@ export function buildExecutableQuery(
 
   // --------------------------------- FINALIZE ---------------------------------
 
-  const executionContext = executionContextBuilder(source);
-  if (executionContext) {
-    sequence.push(executionContext);
+  if (!options?.skipExecutionContext) {
+    const executionContext = engine.buildExecutionContext(source);
+    if (executionContext) {
+      sequence.push(executionContext);
+    }
   }
 
-  options?.postProcessor?.(
-    snapshot,
-    sequence,
-    funcMap,
-    configuration,
-    filterOperations,
-    aggregateOperations,
-  );
+  options?.postProcessor?.(snapshot, sequence, funcMap, configuration, engine);
 
   if (sequence.length === 0) {
     return source.query;

package/src/stores/core/DataCubeQueryBuilderUtils.ts

@@ -43,7 +43,6 @@ import {
   V1_PackageableElementPtr,
   type V1_PrimitiveValueSpecification,
   V1_Variable,
-  V1_deserializeValueSpecification,
   extractElementNameFromPath,
   type V1_ValueSpecification,
   extractPackagePathFromPath,
@@ -53,7 +52,6 @@ import {
   V1_createGenericTypeWithRawType,
   V1_createRelationType,
   V1_createRelationTypeColumn,
-  V1_serializeValueSpecification,
 } from '@finos/legend-graph';
 import {
   type DataCubeQuerySnapshotFilterCondition,
@@ -67,8 +65,6 @@ import {
   guaranteeIsBoolean,
   guaranteeIsNumber,
   UnsupportedOperationError,
-  guaranteeType,
-  type PlainObject,
 } from '@finos/legend-shared';
 import {
   DataCubeFunction,
@@ -91,69 +87,6 @@ import {
 
 // --------------------------------- UTILITIES ---------------------------------
 
-export function _functionCompositionProcessor(
-  sequence: V1_AppliedFunction[],
-  funcMap: DataCubeQueryFunctionMap,
-) {
-  return (
-    key: keyof DataCubeQueryFunctionMap,
-    data: V1_AppliedFunction | V1_AppliedFunction[],
-  ) => {
-    switch (key) {
-      case 'leafExtend':
-      case 'groupExtend': {
-        if (Array.isArray(data)) {
-          data.forEach((func) => sequence.push(func));
-          funcMap[key] = data;
-        }
-        break;
-      }
-      default: {
-        funcMap[key] = data as V1_AppliedFunction;
-        sequence.push(data as V1_AppliedFunction);
-      }
-    }
-  };
-}
-
-export function _functionCompositionUnProcessor(
-  sequence: V1_AppliedFunction[],
-  funcMap: DataCubeQueryFunctionMap,
-) {
-  return (key: keyof DataCubeQueryFunctionMap) => {
-    const data = funcMap[key];
-    if (data) {
-      if (Array.isArray(data)) {
-        data.forEach((func) => sequence.splice(sequence.indexOf(func), 1));
-      } else {
-        sequence.splice(sequence.indexOf(data), 1);
-      }
-      funcMap[key] = undefined;
-    }
-  };
-}
-
-export function _deserializeLambda(json: PlainObject<V1_Lambda>) {
-  return guaranteeType(V1_deserializeValueSpecification(json, []), V1_Lambda);
-}
-
-export function _deserializeValueSpecification(
-  json: PlainObject<V1_ValueSpecification>,
-) {
-  return V1_deserializeValueSpecification(json, []);
-}
-
-export function _serializeValueSpecification(protocol: V1_ValueSpecification) {
-  return V1_serializeValueSpecification(protocol, []);
-}
-
-export function _deserializeFunction(json: PlainObject<V1_Lambda>) {
-  return guaranteeType(
-    V1_deserializeValueSpecification(json, []),
-    V1_AppliedFunction,
-  );
-}
-
 export function _var() {
   const variable = new V1_Variable();
   // NOTE: we simplify processing logic by forcing all variable names to default value, i.e. x
@@ -356,6 +289,48 @@ export function _cols(colSpecs: V1_ColSpec[]) {
 
 // --------------------------------- BUILDING BLOCKS ---------------------------------
 
+export function _functionCompositionProcessor(
+  sequence: V1_AppliedFunction[],
+  funcMap: DataCubeQueryFunctionMap,
+) {
+  return (
+    key: keyof DataCubeQueryFunctionMap,
+    data: V1_AppliedFunction | V1_AppliedFunction[],
+  ) => {
+    switch (key) {
+      case 'leafExtend':
+      case 'groupExtend': {
+        if (Array.isArray(data)) {
+          data.forEach((func) => sequence.push(func));
+          funcMap[key] = data;
+        }
+        break;
+      }
+      default: {
+        funcMap[key] = data as V1_AppliedFunction;
+        sequence.push(data as V1_AppliedFunction);
+      }
+    }
+  };
+}
+
+export function _functionCompositionUnProcessor(
+  sequence: V1_AppliedFunction[],
+  funcMap: DataCubeQueryFunctionMap,
+) {
+  return (key: keyof DataCubeQueryFunctionMap) => {
+    const data = funcMap[key];
+    if (data) {
+      if (Array.isArray(data)) {
+        data.forEach((func) => sequence.splice(sequence.indexOf(func), 1));
+      } else {
+        sequence.splice(sequence.indexOf(data), 1);
+      }
+      funcMap[key] = undefined;
+    }
+  };
+}
+
 export function _selectFunction(columns: DataCubeColumn[]) {
   return _function(DataCubeFunction.SELECT, [
     _cols(columns.map((col) => _colSpec(col.name))),

package/src/stores/core/DataCubeQuerySnapshot.ts

@@ -71,10 +71,10 @@ export type DataCubeQuerySnapshotPivot = {
 
 export type DataCubeQuerySnapshotProcessingContext = {
   snapshot: DataCubeQuerySnapshot;
-  pivotAggColumns: DataCubeQuerySnapshotAggregateColumn[];
-  pivotSortColumns: DataCubeQuerySnapshotSortColumn[];
-  groupByAggColumns: DataCubeQuerySnapshotAggregateColumn[];
-  groupBySortColumns: DataCubeQuerySnapshotSortColumn[];
+  pivotAggColumns?: DataCubeQuerySnapshotAggregateColumn[] | undefined;
+  pivotSortColumns?: DataCubeQuerySnapshotSortColumn[] | undefined;
+  groupByAggColumns?: DataCubeQuerySnapshotAggregateColumn[] | undefined;
+  groupBySortColumns?: DataCubeQuerySnapshotSortColumn[] | undefined;
 };
 
 export type DataCubeQuerySnapshotData = {
@@ -185,4 +185,15 @@ export class DataCubeQuerySnapshot {
     clone._hashCode = this._hashCode;
     return clone;
   }
+
+  serialize() {
+    const clone = this.clone();
+    return {
+      uuid: clone.uuid,
+      data: clone.data,
+      _isPatchChange: clone._isPatchChange,
+      _finalized: clone._finalized,
+      _hashCode: clone._hashCode,
+    };
+  }
 }

package/src/stores/core/DataCubeQuerySnapshotBuilder.ts

@@ -73,10 +73,6 @@ import {
 } from './DataCubeQuerySnapshotBuilderUtils.js';
 import type { DataCubeSource } from './model/DataCubeSource.js';
 import type { DataCubeEngine } from './DataCubeEngine.js';
-import {
-  _deserializeValueSpecification,
-  _serializeValueSpecification,
-} from './DataCubeQueryBuilderUtils.js';
 
 // --------------------------------- BUILDING BLOCKS ---------------------------------
 
@@ -365,8 +361,8 @@ export async function validateAndBuildQuerySnapshot(
   // analysis more ergonomic
 
   // Clone the query since we will mutate it during the process
-  const query = _deserializeValueSpecification(
-    _serializeValueSpecification(partialQuery),
+  const query = engine.deserializeValueSpecification(
+    engine.serializeValueSpecification(partialQuery),
   );
   const funcMap = extractFunctionMap(query);
   const snapshot = DataCubeQuerySnapshot.create({});

package/src/stores/core/DataCubeQuerySnapshotBuilderUtils.ts

@@ -79,7 +79,6 @@ import {
   _colSpec,
   _function,
   _lambda,
-  _serializeValueSpecification,
   _synthesizeMinimalSourceQuery,
 } from './DataCubeQueryBuilderUtils.js';
 import { INTERNAL__DataCubeSource } from './model/DataCubeSource.js';
@@ -360,14 +359,14 @@ export async function _extractExtendedColumns(
       _findCol(columns, colSpec.name),
       `Can't process extend() expression: failed to retrieve type information for column '${colSpec.name}'`,
     ).type,
-    mapFn: _serializeValueSpecification(
+    mapFn: engine.serializeValueSpecification(
       guaranteeNonNullable(
         colSpec.function1,
         `Can't process extend() expression: expected a transformation function expression for column '${colSpec.name}'`,
       ),
     ),
     reduceFn: colSpec.function2
-      ? _serializeValueSpecification(colSpec.function2)
+      ? engine.serializeValueSpecification(colSpec.function2)
       : undefined,
   }));
 }

package/src/stores/core/model/CachedDataCubeSource.ts

@@ -22,4 +22,8 @@ export const CACHED_DATA_CUBE_SOURCE_TYPE = 'cached';
 export class CachedDataCubeSource extends DataCubeSource {
   model!: V1_PureModelContextData;
   runtime!: string;
+  db!: string;
+  schema!: string;
+  table!: string;
+  count!: number;
 }

package/src/stores/services/DataCubeQuerySnapshotService.ts

@@ -88,8 +88,8 @@ export abstract class DataCubeQuerySnapshotController
       this._engine.debugProcess(
         `New Snapshot`,
         ['Publisher', this.getSnapshotSubscriberName()],
-        ['Snapshot', snapshot],
-        ['Previous Snapshot', previousSnapshot],
+        ['Snapshot', snapshot.serialize()],
+        ['Previous Snapshot', previousSnapshot?.serialize()],
         ['Diff', deepDiff(previousSnapshot ?? {}, snapshot)],
       );
     }

package/src/stores/services/DataCubeSettingService.tsx

@@ -161,6 +161,14 @@ export class DataCubeSettingService {
         action: (api) => api.reload(),
         defaultValue: undefined,
       } satisfies DataCubeSetting,
+      {
+        key: DataCubeSettingKey.GRID_CLIENT__SHOW_CACHE_PERFORMANCE_WARNING,
+        title: `Caching Performance Warning: Enabled`,
+        description: `Alerts user of potential performance and resource-related issues that caching can cause when enabled.`,
+        group: DataCubeSettingGroup.GRID,
+        type: DataCubeSettingType.BOOLEAN,
+        defaultValue: true,
+      } satisfies DataCubeSetting<boolean>,
       {
         key: DataCubeSettingKey.GRID_CLIENT__SUPPRESS_LARGE_DATASET_WARNING,
         title: `Large Dataset Warning: Disabled`,

package/src/stores/services/DataCubeTaskService.ts

@@ -19,14 +19,16 @@ import { action, computed, makeObservable, observable } from 'mobx';
 
 export class Task {
   readonly uuid = uuid();
-  readonly name: string;
+  readonly description: string;
+  // use Date.now() vs. performance.now() because need to record the moment
+  // the task starts/ends for logging purposes.
   readonly startTime = Date.now();
   readonly ownerId?: string | undefined;
 
   endTime?: number | undefined;
 
-  constructor(name: string, ownerId?: string | undefined) {
-    this.name = name;
+  constructor(description: string, ownerId?: string | undefined) {
+    this.description = description;
     this.ownerId = ownerId;
   }
 
@@ -48,8 +50,8 @@ export class TaskManager {
     });
   }
 
-  newTask(name: string, ownerId?: string | undefined) {
-    const task = new Task(name, ownerId);
+  newTask(description: string, ownerId?: string | undefined) {
+    const task = new Task(description, ownerId);
     this.tasks.push(task);
     return task;
   }
@@ -87,8 +89,8 @@ export class DataCubeTaskService {
     return this.manager.tasks.filter((task) => task.ownerId === this.uuid);
   }
 
-  newTask(name: string) {
-    return this.manager.newTask(name, this.uuid);
+  newTask(description: string) {
+    return this.manager.newTask(description, this.uuid);
   }
 
   endTask(task: Task) {