@finos/legend-data-cube 0.0.35 → 0.0.37

package/src/stores/core/DataCubeEngine.tsx

@@ -69,22 +69,16 @@ import {
   configureCodeEditor,
   setupPureLanguageService,
 } from '@finos/legend-code-editor';
-import { DataCubeFont, DataCubeFunction } from './DataCubeQueryEngine.js';
+import { DataCubeFont } from './DataCubeQueryEngine.js';
 import type { DataCubeQuerySnapshot } from './DataCubeQuerySnapshot.js';
 import { buildExecutableQuery } from './DataCubeQueryBuilder.js';
 import type { DataCubeColumn } from './models/DataCubeColumn.js';
 import { LicenseManager } from 'ag-grid-enterprise';
-import type { DataCubeQuery } from './models/DataCubeQuery.js';
 import {
   type DataCubeSource,
   INTERNAL__DataCubeSource,
 } from './models/DataCubeSource.js';
-import {
-  _cols,
-  _colSpec,
-  _function,
-  _primitiveValue,
-} from './DataCubeQueryBuilderUtils.js';
+import { _primitiveValue } from './DataCubeQueryBuilderUtils.js';
 import {
   uuid,
   type DocumentationEntry,
@@ -105,27 +99,26 @@ import {
 } from './DataCubeLayoutManagerState.js';
 import { editor as monacoEditorAPI, Uri } from 'monaco-editor';
 import { DataCubeCodeCheckErrorAlert } from '../../components/core/DataCubeCodeCheckErrorAlert.js';
-import type { DataCubeAPI } from '../DataCubeAPI.js';
 
 export type CompletionItem = {
   completion: string;
   display: string;
 };
 
-export type RelationType = {
+export type DataCubeRelationType = {
   columns: DataCubeColumn[];
 };
 
+export type DataCubeExecutionOptions = {
+  debug?: boolean | undefined;
+};
+
 export type DataCubeExecutionResult = {
   result: TDSExecutionResult;
   executedQuery: string;
   executedSQL: string;
 };
 
-export type DataCubeEngineConfiguration = {
-  gridClientLicense?: string | undefined;
-};
-
 export abstract class DataCubeEngine {
   readonly layout = new LayoutManagerState();
   readonly filterOperations = [
@@ -176,16 +169,11 @@ export abstract class DataCubeEngine {
     new DataCubeQueryAggregateOperation__JoinStrings(),
   ];
 
-  protected async fetchConfiguration(): Promise<DataCubeEngineConfiguration> {
-    return {
-      gridClientLicense: undefined,
-    };
-  }
-
-  async initialize(): Promise<void> {
-    const config = await this.fetchConfiguration();
-    if (config.gridClientLicense) {
-      LicenseManager.setLicenseKey(config.gridClientLicense);
+  async initialize(options?: {
+    gridClientLicense?: string | undefined;
+  }): Promise<void> {
+    if (options?.gridClientLicense) {
+      LicenseManager.setLicenseKey(options.gridClientLicense);
     }
     await configureCodeEditor(DataCubeFont.ROBOTO_MONO, (error) => {
       throw error;
@@ -203,7 +191,36 @@ export abstract class DataCubeEngine {
     return getAggregateOperation(value, this.aggregateOperations);
   }
 
-  abstract getBaseQuery(): Promise<DataCubeQuery | undefined>;
+  /**
+   * By default, for a function chain, Pure grammar composer will extract the first parameter of the first function
+   * and render it as the caller of that function rather than a parameter
+   * e.g. fx(fy(p1, p2), p3) will be rendered as p1->fy(p2)->fx(p3) instead of fy(p1, p2)-> fx(p3)
+   *
+   * We do a hack to get around this by setting a dummy value as the first parameter of the first function in the chain.
+   * Then remove this dummy value from the final code.
+   */
+  async getPartialQueryCode(
+    snapshot: DataCubeQuerySnapshot,
+    pretty?: boolean | undefined,
+  ) {
+    const source = new INTERNAL__DataCubeSource();
+    source.query = _primitiveValue(PRIMITIVE_TYPE.STRING, '');
+    return (
+      await this.getValueSpecificationCode(
+        buildExecutableQuery(
+          snapshot,
+          source,
+          () => undefined,
+          this.filterOperations,
+          this.aggregateOperations,
+        ),
+        pretty,
+      )
+    ).substring(`''->`.length);
+  }
+
+  // ---------------------------------- INTERFACE ----------------------------------
+
   abstract processQuerySource(value: PlainObject): Promise<DataCubeSource>;
 
   abstract parseValueSpecification(
@@ -225,58 +242,24 @@ export abstract class DataCubeEngine {
   abstract getQueryRelationType(
     query: V1_Lambda,
     source: DataCubeSource,
-  ): Promise<RelationType>;
+  ): Promise<DataCubeRelationType>;
 
   abstract getQueryCodeRelationReturnType(
     code: string,
     baseQuery: V1_ValueSpecification,
     source: DataCubeSource,
-  ): Promise<RelationType>;
+  ): Promise<DataCubeRelationType>;
 
   abstract executeQuery(
     query: V1_Lambda,
     source: DataCubeSource,
-    api: DataCubeAPI,
+    options?: DataCubeExecutionOptions | undefined,
   ): Promise<DataCubeExecutionResult>;
 
   abstract buildExecutionContext(
     source: DataCubeSource,
   ): V1_AppliedFunction | undefined;
 
-  /**
-   * By default, for a function chain, Pure grammar composer will extract the first parameter of the first function
-   * and render it as the caller of that function rather than a parameter
-   * e.g. fx(fy(p1, p2), p3) will be rendered as p1->fy(p2)->fx(p3) instead of fy(p1, p2)-> fx(p3)
-   *
-   * We do a hack to get around this by setting a dummy value as the first parameter of the first function in the chain.
-   * Then remove this dummy value from the final code.
-   */
-  async getPartialQueryCode(
-    snapshot: DataCubeQuerySnapshot,
-    pretty?: boolean | undefined,
-  ) {
-    const source = new INTERNAL__DataCubeSource();
-    source.query = _primitiveValue(PRIMITIVE_TYPE.STRING, '');
-    return (
-      await this.getValueSpecificationCode(
-        buildExecutableQuery(
-          snapshot,
-          source,
-          () => undefined,
-          this.filterOperations,
-          this.aggregateOperations,
-        ),
-        pretty,
-      )
-    ).substring(`''->`.length);
-  }
-
-  generateInitialQuery(snapshot: DataCubeQuerySnapshot): V1_AppliedFunction {
-    return _function(DataCubeFunction.SELECT, [
-      _cols(snapshot.data.sourceColumns.map((col) => _colSpec(col.name))),
-    ]);
-  }
-
   // ---------------------------------- DOCUMENTATION ----------------------------------
 
   getDocumentationURL(): string | undefined {

package/src/stores/core/DataCubeQueryBuilder.ts

@@ -79,6 +79,7 @@ export function buildExecutableQuery(
           end: number;
         }
       | undefined;
+    skipExecutionContext?: boolean;
   },
 ) {
   const data = snapshot.data;

package/src/stores/core/DataCubeQueryBuilderUtils.ts

@@ -53,6 +53,7 @@ import {
   V1_createGenericTypeWithRawType,
   V1_createRelationType,
   V1_createRelationTypeColumn,
+  V1_serializeValueSpecification,
 } from '@finos/legend-graph';
 import {
   type DataCubeQuerySnapshotFilterCondition,
@@ -117,6 +118,16 @@ export function _deserializeLambda(json: PlainObject<V1_Lambda>) {
   return guaranteeType(V1_deserializeValueSpecification(json, []), V1_Lambda);
 }
 
+export function _deserializeValueSpecification(
+  json: PlainObject<V1_ValueSpecification>,
+) {
+  return V1_deserializeValueSpecification(json, []);
+}
+
+export function _serializeValueSpecification(protocol: V1_ValueSpecification) {
+  return V1_serializeValueSpecification(protocol, []);
+}
+
 export function _deserializeFunction(json: PlainObject<V1_Lambda>) {
   return guaranteeType(
     V1_deserializeValueSpecification(json, []),
@@ -304,6 +315,12 @@ export function _not(fn: V1_AppliedFunction) {
   return _function(_functionName(DataCubeFunction.NOT), [fn]);
 }
 
+export function _selectFunction(columns: DataCubeColumn[]) {
+  return _function(DataCubeFunction.SELECT, [
+    _cols(columns.map((col) => _colSpec(col.name))),
+  ]);
+}
+
 // --------------------------------- BUILDING BLOCKS ---------------------------------
 
 export function _col(

package/src/stores/core/DataCubeQuerySnapshotBuilder.ts

@@ -301,12 +301,12 @@ export function validateAndBuildQuerySnapshot(
     return _toCol(column);
   };
 
-  // --------------------------------- SOURCE ---------------------------------
+  // -------------------------------- SOURCE --------------------------------
 
-  data.sourceColumns = source.sourceColumns;
+  data.sourceColumns = source.columns;
   data.sourceColumns.forEach((col) => colsMap.set(col.name, col));
 
-  // --------------------------------- LEAF-LEVEL EXTEND ---------------------------------
+  // --------------------------- LEAF-LEVEL EXTEND ---------------------------
   /** TODO: @datacube roundtrip */
 
   // --------------------------------- FILTER ---------------------------------
@@ -336,7 +336,7 @@ export function validateAndBuildQuerySnapshot(
     };
   }
 
-  // --------------------------------- GROUP-LEVEL EXTEND ---------------------------------
+  // --------------------------- GROUP-LEVEL EXTEND ---------------------------
   /** TODO: @datacube roundtrip */
 
   // --------------------------------- SORT ---------------------------------
@@ -368,41 +368,70 @@ export function validateAndBuildQuerySnapshot(
   }
 
   // --------------------------------- CONFIGURATION ---------------------------------
-  // NOTE: we aim to keep the data query a Pure query instead of a specification object
-  // so this configuration holds mostly layout and styling customization.
-  // But there are overlaps, i.e. certain data query configuration are stored in the
-  // configuration, e.g. column aggregation type, this is because a column
-  // aggregation's preference can be populated even when there's no aggregation specified
-  // in the data query.
   //
-  // Arguably, the query should be the single source for these information, but when
-  // the configuration for a particular data query function coming from multiple sources
-  // conflict, we need to do some reconciliation (or throw error). Some examples include:
-  // - missing/extra columns present in the configuration
+  // A data-cube conceptually consists of a data query, in form of a Pure query, instead
+  // of a particular specification object format, and this configuration, that holds mostly
+  // layout and styling customization. But there are overlaps, i.e. certain "meta" query
+  // configuration are stored in this configuration, e.g. column aggregation type, because
+  // a column aggregation's preference needs to be specified even when there's no aggregation
+  // specified over that column in the data query.
+  //
+  // But in the example above, if the column is part of an aggregation, we have to ensure
+  // the configuration is consistent with the query. Conflicts can happen, for example:
   // - column kind and type conflict with aggregation
   // - column kind and type conflict with the column configuration
   //
-  // In certain cases, configuration needs to be generated from default presets. This
-  // helps with use cases where the query might comes from a different source, such as
-  // Studio or Query, or another part of Engine.
+  // In those cases, we need to reconcile to make sure the query and the configuration to agree.
+  // The query will take precedence when conflicts happen, and if the conflict cannot be resolved
+  // somehow, we will throw a validation error. We decide so because in certain cases, configuration
+  // needs to be generated from default presets, such as for use cases where the query comes from a
+  // different source, such as Studio or Query, or another part of Engine, where the layout
+  // configuration is not specified.
+  //
+  // ----------------------------------------------------------------------------------
+
+  const configuration = validateAndBuildConfiguration(
+    snapshot,
+    funcMap,
+    baseQuery,
+  );
+  data.configuration = configuration.serialize();
+
+  return snapshot.finalize();
+}
+
+/**
+ * TODO: @datacube roundtrip - implement the logic to reconcile the configuration with the query
+ * - [ ] disallow naming an extended column the same as a source column/leaf-level extended column
+ *       even if the column is not selected
+ * - [ ] columns (missing/extra columns - remove or generate default column configuration)
+ * - [ ] column kind
+ * - [ ] column type
+ * - [ ] base off the type, check the settings
+ * - [ ] aggregation
+ * - [ ] verify groupBy agg columns, pivot agg columns and configuration agree
+ * - [ ] verify groupBy sort columns and tree column sort direction configuration agree
+ */
+function validateAndBuildConfiguration(
+  snapshot: DataCubeQuerySnapshot,
+  funcMap: DataCubeQueryFunctionMap,
+  baseQuery: DataCubeQuery,
+) {
+  const data = snapshot.data;
   const configuration =
     baseQuery.configuration?.clone() ??
     buildDefaultConfiguration([
-      ...source.sourceColumns,
+      ...data.sourceColumns,
       ...data.leafExtendedColumns,
       ...data.groupExtendedColumns,
     ]);
-  data.configuration = configuration.serialize();
-  /**
-   * TODO: @datacube roundtrip - implement the logic to reconcile the configuration with the query
-   * - [ ] columns (missing/extra columns - remove or generate default column configuration)
-   * - [ ] column kind
-   * - [ ] column type
-   * - [ ] base off the type, check the settings
-   * - [ ] aggregation
-   * - [ ] verify groupBy agg columns, pivot agg columns and configuration agree
-   * - [ ] verify groupBy sort columns and tree column sort direction configuration agree
-   */
 
-  return snapshot.finalize();
+  // column selection
+  configuration.columns.forEach((column) => {
+    column.isSelected = Boolean(
+      data.groupExtendedColumns.find((col) => col.name === column.name) ??
+        data.selectColumns.find((col) => col.name === column.name),
+    );
+  });
+  return configuration;
 }

package/src/stores/core/DataCubeSetting.ts

@@ -0,0 +1,63 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type DataCubeSettingValue =
+  | string
+  | number
+  | boolean
+  | object
+  | undefined;
+
+export type DataCubeSettingValues = {
+  [key: string]: DataCubeSettingValue;
+};
+
+export enum DataCubeSettingGroup {
+  DEBUG = 'Debug',
+  GRID = 'Grid',
+}
+
+export enum DataCubeSettingType {
+  BOOLEAN = 'boolean',
+  NUMERIC = 'numeric',
+  STRING = 'string',
+  ACTION = 'action',
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type DataCubeSetting<T extends DataCubeSettingValue = any> = {
+  key: string;
+  title: string;
+  group: string;
+  type: DataCubeSettingType;
+  defaultValue: T;
+  action?: ((newValue: T) => void) | undefined;
+  description?: string | undefined;
+  valueOptional?: boolean | undefined;
+  requiresReload?: boolean | undefined;
+  // numeric value specifics - TODO: we might want to break these downw to separate types
+  numericValueMin?: number | undefined;
+  numericValueMax?: number | undefined;
+  numericValueStep?: number | undefined;
+};
+
+export enum DataCubeSettingKey {
+  DEBUGGER__ENABLE_DEBUG_MODE = 'dataCube.debugger.enableDebugMode',
+  GRID_CLIENT__ROW_BUFFER = 'dataCube.grid-client.rowBuffer',
+  GRID_CLIENT__PURGE_CLOSED_ROW_NODES = 'dataCube.grid-client.purgeClosedRowNodes',
+  GRID_CLIENT__SUPPRESS_LARGE_DATASET_WARNING = 'dataCube.grid-client.suppressLargeDatasetWarning',
+  GRID_CLIENT__ACTION__REFRESH_FAILED_DATA_FETCHES = 'dataCube.grid-client.action.refreshFailedDataFetches',
+}

package/src/stores/core/models/AdhocQueryDataCubeSource.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  SerializationFactory,
+  usingConstantValueSchema,
+  type PlainObject,
+} from '@finos/legend-shared';
+import { DataCubeSource } from './DataCubeSource.js';
+import { createModelSchema, primitive, raw } from 'serializr';
+
+export class AdhocQueryDataCubeSource extends DataCubeSource {
+  runtime!: string;
+  model!: PlainObject;
+}
+
+export const ADHOC_QUERY_DATA_CUBE_SOURCE_TYPE = 'adhocQuery';
+
+export class RawAdhocQueryDataCubeSource {
+  query!: string;
+  runtime!: string;
+  model!: PlainObject;
+
+  static readonly serialization = new SerializationFactory(
+    createModelSchema(RawAdhocQueryDataCubeSource, {
+      _type: usingConstantValueSchema(ADHOC_QUERY_DATA_CUBE_SOURCE_TYPE),
+      model: raw(),
+      query: primitive(),
+      runtime: primitive(),
+    }),
+  );
+}

package/src/stores/core/models/DataCubeSource.ts

@@ -18,7 +18,7 @@ import type { V1_ValueSpecification } from '@finos/legend-graph';
 import type { DataCubeColumn } from './DataCubeColumn.js';
 
 export abstract class DataCubeSource {
-  sourceColumns: DataCubeColumn[] = [];
+  columns: DataCubeColumn[] = [];
   query!: V1_ValueSpecification;
 }
 

package/src/stores/view/DataCubeQuerySnapshotManager.ts

@@ -23,6 +23,7 @@ import {
   guaranteeNonNullable,
 } from '@finos/legend-shared';
 import type { DataCubeQuery } from '../core/models/DataCubeQuery.js';
+import { DataCubeSettingKey } from '../core/DataCubeSetting.js';
 
 // TODO: set a stack depth when we implement undo/redo
 // const DATA_CUBE_MAX_SNAPSHOT_COUNT = 100;
@@ -64,7 +65,11 @@ export abstract class DataCubeQuerySnapshotController
     const previousSnapshot = this.latestSnapshot;
     this.latestSnapshot = snapshot;
 
-    if (this.view.dataCube.settings.enableDebugMode) {
+    if (
+      this.view.dataCube.settings.getBooleanValue(
+        DataCubeSettingKey.DEBUGGER__ENABLE_DEBUG_MODE,
+      )
+    ) {
       this.view.engine.debugProcess(
         `New Snapshot`,
         ['Publisher', this.getSnapshotSubscriberName()],
@@ -105,9 +110,16 @@ export class DataCubeQuerySnapshotManager {
         subscriber.getSnapshotSubscriberName(),
     );
     if (existingSubscriber) {
-      throw new IllegalStateError(
-        `Subscriber with name '${subscriber.getSnapshotSubscriberName()}' already exists`,
-      );
+      // eslint-disable-next-line no-process-env
+      if (process.env.NODE_ENV === 'development') {
+        this.view.engine.logDebug(
+          `Subscriber with name '${subscriber.getSnapshotSubscriberName()}' already exists`,
+        );
+      } else {
+        throw new IllegalStateError(
+          `Subscriber with name '${subscriber.getSnapshotSubscriberName()}' already exists`,
+        );
+      }
     }
     this.subscribers.push(subscriber);
   }

package/src/stores/view/DataCubeViewState.ts

@@ -57,7 +57,6 @@ class DataCubeTask {
 export class DataCubeViewState {
   readonly dataCube: DataCubeState;
   readonly engine: DataCubeEngine;
-
   readonly snapshotManager: DataCubeQuerySnapshotManager;
 
   readonly info: DataCubeInfoState;
@@ -79,6 +78,8 @@ export class DataCubeViewState {
       runningTasks: observable,
       newTask: action,
       endTask: action,
+
+      initialize: action,
     });
 
     this.dataCube = dataCube;
@@ -118,7 +119,7 @@ export class DataCubeViewState {
     return task;
   }
 
-  async initialize(initialQuery?: DataCubeQuery | undefined) {
+  async initialize(query: DataCubeQuery) {
     const task = this.newTask('Initializing');
     try {
       await Promise.all(
@@ -133,30 +134,20 @@ export class DataCubeViewState {
           this.snapshotManager.registerSubscriber(state);
         }),
       );
-      const baseQuery = initialQuery ?? (await this.engine.getBaseQuery());
-      if (!baseQuery) {
-        this.dataCube.alertAction({
-          message: `Initialization Failure: No initial input provided`,
-          prompt: `Make sure to either specify the initial input when setting up DataCube or the initial input getter in engine.`,
-          type: AlertType.ERROR,
-          actions: [],
-        });
-        return;
-      }
-      const source = await this.engine.processQuerySource(baseQuery.source);
+      const source = await this.engine.processQuerySource(query.source);
       runInAction(() => {
         this._source = source;
       });
       const partialQuery = await this.engine.parseValueSpecification(
-        baseQuery.query,
+        query.query,
       );
       const initialSnapshot = validateAndBuildQuerySnapshot(
         partialQuery,
-        this.source,
-        baseQuery,
+        source,
+        query,
       );
       this.snapshotManager.broadcastSnapshot(initialSnapshot);
-    } catch (error: unknown) {
+    } catch (error) {
       assertErrorThrown(error);
       this.dataCube.alertAction({
         message: `Initialization Failure: ${error.message}`,

package/src/stores/view/extend/DataCubeColumnEditorState.tsx

@@ -42,14 +42,16 @@ import type { DataCubeExtendManagerState } from './DataCubeExtendManagerState.js
 import {
   EngineError,
   PRIMITIVE_TYPE,
-  V1_deserializeValueSpecification,
   V1_Lambda,
-  V1_serializeValueSpecification,
   type V1_ValueSpecification,
 } from '@finos/legend-graph';
 import type { DataCubeColumnConfiguration } from '../../core/models/DataCubeConfiguration.js';
 import type { DataCubeQuerySnapshotExtendedColumn } from '../../core/DataCubeQuerySnapshot.js';
-import { _lambda } from '../../core/DataCubeQueryBuilderUtils.js';
+import {
+  _lambda,
+  _serializeValueSpecification,
+  _deserializeValueSpecification,
+} from '../../core/DataCubeQueryBuilderUtils.js';
 
 export abstract class DataCubeColumnBaseEditorState {
   readonly uuid = uuid();
@@ -392,7 +394,7 @@ export class DataCubeNewColumnState extends DataCubeColumnBaseEditorState {
       {
         name: this.name,
         type: returnType,
-        mapFn: V1_serializeValueSpecification(query, []),
+        mapFn: _serializeValueSpecification(query),
       },
       this.isGroupLevel,
       this.columnKind,
@@ -442,7 +444,7 @@ export class DataCubeExistingColumnEditorState extends DataCubeColumnBaseEditorS
 
   override async getInitialCode(): Promise<string> {
     return this.view.engine.getValueSpecificationCode(
-      V1_deserializeValueSpecification(this.initialData.mapFn, []),
+      _deserializeValueSpecification(this.initialData.mapFn),
       true,
     );
   }
@@ -518,7 +520,7 @@ export class DataCubeExistingColumnEditorState extends DataCubeColumnBaseEditorS
       {
         name: this.name,
         type: returnType,
-        mapFn: V1_serializeValueSpecification(query, []),
+        mapFn: _serializeValueSpecification(query),
       },
       this.isGroupLevel,
       this.columnKind,