@finos/legend-extension-dsl-data-quality 1.0.0

package/src/components/states/DataQualityClassValidationState.ts

@@ -0,0 +1,308 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { action, flow, makeObservable, observable } from 'mobx';
+import {
+  type Class,
+  type Mapping,
+  type PackageableElement,
+  type Runtime,
+  getMappingCompatibleClasses,
+  PackageableElementExplicitReference,
+  RuntimePointer,
+} from '@finos/legend-graph';
+import type {
+  EditorStore,
+  ElementEditorState,
+} from '@finos/legend-application-studio';
+import {
+  DataQualityClassValidationsConfiguration,
+  DataSpaceDataQualityExecutionContext,
+  MappingAndRuntimeDataQualityExecutionContext,
+} from '../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
+import { DataQualityState } from './DataQualityState.js';
+import {
+  type DataSpaceInfo,
+  type DataSpaceExecutionContext,
+  DataSpace,
+  resolveUsableDataSpaceClasses,
+} from '@finos/legend-extension-dsl-data-space/graph';
+import { ELEMENT_CREATION_BASIS } from '../DSL_DataQuality_ClassElementDriver.js';
+import {
+  type GeneratorFn,
+  assertType,
+  filterByType,
+  getNullableFirstEntry,
+  guaranteeNonNullable,
+  guaranteeType,
+} from '@finos/legend-shared';
+import {
+  dataQualityClassValidation_setContextDataQualityContext,
+  dataQualityClassValidation_setDataQualityContext,
+  dataQualityClassValidation_setDataQualityGraphFetchTree,
+  dataQualityClassValidation_setFilter,
+  dataQualityClassValidation_setRuntimeDataQualityContext,
+} from '../../graph-manager/DSL_DataQuality_GraphModifierHelper.js';
+import { DataQualityGraphFetchTreeState } from './DataQualityGraphFetchTreeState.js';
+import { DataQualityResultState } from './DataQualityResultState.js';
+
+export class DataQualityClassValidationState extends DataQualityState {
+  dataSpace?: DataSpace | undefined;
+  dataSpaces: DataSpaceInfo[] = [];
+  validationElementCreationBasis: ELEMENT_CREATION_BASIS =
+    ELEMENT_CREATION_BASIS.MAPPING_RUNTIME_BASED;
+
+  constructor(
+    editorStore: EditorStore,
+    element: DataQualityClassValidationsConfiguration,
+  ) {
+    super(editorStore, element);
+    makeObservable(this, {
+      dataSpaces: observable,
+      dataSpace: observable,
+      onDataSpaceChange: action,
+      initializeState: action,
+      updateElementOnMappingChange: action,
+      updateElementOnRuntimeChange: action,
+      updateElementOnExecutionContextChange: action,
+      updateElementOnDataSpaceChange: action,
+      loadDataSpaces: flow,
+    });
+    this.initializeState();
+  }
+
+  reprocess(
+    newElement: PackageableElement,
+    editorStore: EditorStore,
+  ): ElementEditorState {
+    const element = guaranteeType(
+      newElement,
+      DataQualityClassValidationsConfiguration,
+      'Element inside data quality class validation editor state must be a class validation element',
+    );
+
+    return new DataQualityClassValidationState(editorStore, element);
+  }
+
+  override get constraintsConfigurationElement(): DataQualityClassValidationsConfiguration {
+    return guaranteeType(
+      this.element,
+      DataQualityClassValidationsConfiguration,
+      'Element inside data quality class validation state must be a data quality class validation configuration element',
+    );
+  }
+
+  initializeState() {
+    let classOptions: Class[] = [];
+    if (
+      this.constraintsConfigurationElement.context instanceof
+      MappingAndRuntimeDataQualityExecutionContext
+    ) {
+      this.validationElementCreationBasis =
+        ELEMENT_CREATION_BASIS.MAPPING_RUNTIME_BASED;
+      this.dataQualityQueryBuilderState.executionContextState.setMapping(
+        this.constraintsConfigurationElement.context.mapping.value,
+      );
+      this.dataQualityQueryBuilderState.executionContextState.setRuntimeValue(
+        new RuntimePointer(
+          this.constraintsConfigurationElement.context.runtime,
+        ),
+      );
+      classOptions = getMappingCompatibleClasses(
+        this.dataQualityQueryBuilderState.executionContextState.mapping!,
+        this.graphManagerState.usableClasses,
+      );
+    } else if (
+      this.constraintsConfigurationElement.context instanceof
+      DataSpaceDataQualityExecutionContext
+    ) {
+      this.validationElementCreationBasis =
+        ELEMENT_CREATION_BASIS.DATASPACE_BASED;
+      this.dataSpace =
+        this.constraintsConfigurationElement.context.dataSpace.value;
+      this.executionContext = this.dataSpace.defaultExecutionContext;
+      this.dataQualityQueryBuilderState.executionContextState.setMapping(
+        this.dataSpace.defaultExecutionContext.mapping.value,
+      );
+      this.dataQualityQueryBuilderState.executionContextState.setRuntimeValue(
+        new RuntimePointer(
+          this.dataSpace.defaultExecutionContext.defaultRuntime,
+        ),
+      );
+      classOptions = resolveUsableDataSpaceClasses(
+        this.dataSpace,
+        this.dataQualityQueryBuilderState.executionContextState.mapping!,
+        this.graphManagerState,
+      );
+    }
+    this.initializeGraphFetchTreeState(
+      this.constraintsConfigurationElement.dataQualityRootGraphFetchTree,
+    );
+    this.initializeFilterState(this.constraintsConfigurationElement.filter);
+
+    if (this.constraintsConfigurationElement.dataQualityRootGraphFetchTree) {
+      this.dataQualityQueryBuilderState.setClass(
+        this.constraintsConfigurationElement.dataQualityRootGraphFetchTree.class
+          .value,
+      );
+    } else {
+      this.dataQualityQueryBuilderState.setClass(
+        getNullableFirstEntry(classOptions),
+      );
+      this.dataQualityGraphFetchTreeState = new DataQualityGraphFetchTreeState(
+        this,
+      );
+      dataQualityClassValidation_setDataQualityGraphFetchTree(
+        this.constraintsConfigurationElement,
+        this.dataQualityGraphFetchTreeState.treeData
+          ? this.dataQualityGraphFetchTreeState.treeData.tree
+          : undefined,
+      );
+    }
+  }
+
+  changeMapping(val: Mapping, options?: { keepQueryContent?: boolean }): void {
+    //reset result
+    this.dataQualityQueryBuilderState.executionContextState.setMapping(val);
+    this.dataQualityQueryBuilderState.executionContextState.setRuntimeValue(
+      undefined,
+    );
+    this.dataQualityQueryBuilderState.explorerState.setTreeData(undefined);
+    this.dataQualityQueryBuilderState.setClass(undefined);
+    this.dataQualityGraphFetchTreeState = new DataQualityGraphFetchTreeState(
+      this,
+    );
+    this.structuralValidationsGraphFetchTreeState =
+      new DataQualityGraphFetchTreeState(this);
+    this.resultState = new DataQualityResultState(this);
+  }
+
+  changeRuntime(val: Runtime) {
+    this.dataQualityQueryBuilderState.changeRuntime(val);
+    this.resultState = new DataQualityResultState(this);
+  }
+
+  updateElementOnMappingChange() {
+    const mappingAndRuntimeExecutionContext =
+      new MappingAndRuntimeDataQualityExecutionContext();
+    mappingAndRuntimeExecutionContext.mapping =
+      PackageableElementExplicitReference.create(
+        this.dataQualityQueryBuilderState.executionContextState.mapping!,
+      );
+    guaranteeNonNullable(
+      this.dataQualityQueryBuilderState.executionContextState.runtimeValue,
+    );
+    mappingAndRuntimeExecutionContext.runtime = (
+      this.dataQualityQueryBuilderState.executionContextState
+        .runtimeValue as RuntimePointer
+    ).packageableRuntime;
+    dataQualityClassValidation_setDataQualityContext(
+      this.constraintsConfigurationElement,
+      mappingAndRuntimeExecutionContext,
+    );
+    dataQualityClassValidation_setDataQualityGraphFetchTree(
+      this.constraintsConfigurationElement,
+      undefined,
+    );
+    dataQualityClassValidation_setFilter(
+      this.constraintsConfigurationElement,
+      undefined,
+    );
+  }
+
+  updateElementOnRuntimeChange(runtime: Runtime) {
+    assertType(runtime, RuntimePointer);
+    dataQualityClassValidation_setRuntimeDataQualityContext(
+      this.constraintsConfigurationElement,
+      runtime.packageableRuntime,
+      this.editorStore.changeDetectionState.observerContext,
+    );
+  }
+
+  updateElementOnExecutionContextChange(context: string) {
+    dataQualityClassValidation_setContextDataQualityContext(
+      this.constraintsConfigurationElement,
+      context,
+    );
+  }
+
+  propagateExecutionContextChange(
+    executionContext: DataSpaceExecutionContext,
+  ): void {
+    const mapping = executionContext.mapping.value;
+    this.changeMapping(mapping);
+    this.dataQualityQueryBuilderState.changeRuntime(
+      new RuntimePointer(executionContext.defaultRuntime),
+    );
+    const compatibleClasses = resolveUsableDataSpaceClasses(
+      this.dataSpace!,
+      mapping,
+      this.graphManagerState,
+    );
+    // if there is no chosen class or the chosen one is not compatible
+    // with the mapping then pick a compatible class if possible
+    if (
+      !this.dataQualityQueryBuilderState.class ||
+      !compatibleClasses.includes(this.dataQualityQueryBuilderState.class)
+    ) {
+      const possibleNewClass = getNullableFirstEntry(compatibleClasses);
+      if (possibleNewClass) {
+        this.changeClass(possibleNewClass);
+      }
+    }
+  }
+
+  onDataSpaceChange(dataSpaceInfo: DataSpaceInfo) {
+    this.dataSpace = guaranteeType(
+      this.graphManagerState.graph.getElement(dataSpaceInfo.path),
+      DataSpace,
+    );
+    this.setExecutionContext(this.dataSpace.defaultExecutionContext);
+    this.propagateExecutionContextChange(
+      this.dataSpace.defaultExecutionContext,
+    );
+  }
+
+  updateElementOnDataSpaceChange() {
+    const dataSpaceExecutionContext =
+      new DataSpaceDataQualityExecutionContext();
+    dataSpaceExecutionContext.context = this.executionContext.name;
+    dataSpaceExecutionContext.dataSpace =
+      PackageableElementExplicitReference.create(this.dataSpace!);
+    dataQualityClassValidation_setDataQualityContext(
+      this.constraintsConfigurationElement,
+      dataSpaceExecutionContext,
+    );
+    this.updateElementOnClassChange();
+  }
+
+  *loadDataSpaces(): GeneratorFn<void> {
+    this.dataSpaces = this.graphManagerState.graph.allOwnElements
+      .filter(filterByType(DataSpace))
+      .map(
+        (e) =>
+          ({
+            groupId: undefined,
+            artifactId: undefined,
+            versionId: undefined,
+            path: e.path,
+            name: e.name,
+            title: e.title,
+            defaultExecutionContext: e.defaultExecutionContext.name,
+          }) as DataSpaceInfo,
+      );
+  }
+}

package/src/components/states/DataQualityGraphFetchTreeState.ts

@@ -0,0 +1,125 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { type Hashable, hashArray } from '@finos/legend-shared';
+import {
+  type DataQualityGraphFetchTreeData,
+  type DataQualityGraphFetchTreeNodeData,
+  updateNodeConstraints,
+  addQueryBuilderPropertyNode,
+  buildGraphFetchTreeData,
+} from '../utils/DataQualityGraphFetchTreeUtil.js';
+import type { DataQualityState } from './DataQualityState.js';
+import { action, makeObservable, observable } from 'mobx';
+import {
+  type Class,
+  type Constraint,
+  PackageableElementExplicitReference,
+} from '@finos/legend-graph';
+import type {
+  QueryBuilderExplorerTreePropertyNodeData,
+  QueryBuilderExplorerTreeRootNodeData,
+} from '@finos/legend-query-builder';
+import { DataQualityRootGraphFetchTree } from '../../graph/metamodel/pure/packageableElements/data-quality/DataQualityGraphFetchTree.js';
+import { DATA_QUALITY_HASH_STRUCTURE } from '../../graph/metamodel/DSL_DataQuality_HashUtils.js';
+
+export class DataQualityGraphFetchTreeState implements Hashable {
+  readonly dataQualityState: DataQualityState;
+  treeData?: DataQualityGraphFetchTreeData | undefined;
+
+  constructor(dataQualityState: DataQualityState) {
+    makeObservable(this, {
+      treeData: observable.ref,
+      setGraphFetchTree: action,
+    });
+
+    this.dataQualityState = dataQualityState;
+    this.updateTreeData(
+      this.dataQualityState.dataQualityQueryBuilderState.class,
+    );
+  }
+  setGraphFetchTree(val: DataQualityGraphFetchTreeData | undefined): void {
+    this.treeData = val;
+  }
+
+  private updateTreeData(_class: Class | undefined): void {
+    this.setGraphFetchTree(
+      _class
+        ? buildGraphFetchTreeData(
+            this.dataQualityState.editorStore,
+            new DataQualityRootGraphFetchTree(
+              PackageableElementExplicitReference.create(_class),
+            ),
+            false,
+            false,
+            false,
+          )
+        : undefined,
+    );
+  }
+
+  onClassChange(_class: Class | undefined): void {
+    this.updateTreeData(_class);
+  }
+
+  updateNode(
+    node: DataQualityGraphFetchTreeNodeData,
+    constraint: Constraint,
+    addConstraint: boolean,
+  ): void {
+    if (!this.treeData) {
+      this.dataQualityState.applicationStore.notificationService.notifyWarning(
+        `Can't add property: data quality graph-fetch tree has not been properly initialized`,
+      );
+      return;
+    }
+    updateNodeConstraints(this.treeData, node, constraint, addConstraint);
+    this.setGraphFetchTree({ ...this.treeData });
+  }
+
+  addProperty(
+    node:
+      | QueryBuilderExplorerTreePropertyNodeData
+      | QueryBuilderExplorerTreeRootNodeData,
+    options?: {
+      refreshTreeData?: boolean;
+    },
+  ): void {
+    if (!this.treeData) {
+      this.dataQualityState.applicationStore.notificationService.notifyWarning(
+        `Can't add property: data quality graph-fetch tree has not been properly initialized`,
+      );
+      return;
+    }
+    addQueryBuilderPropertyNode(
+      this.treeData,
+      this.dataQualityState.dataQualityQueryBuilderState.explorerState
+        .nonNullableTreeData,
+      node,
+      this.dataQualityState,
+    );
+    if (options?.refreshTreeData) {
+      this.setGraphFetchTree({ ...this.treeData });
+    }
+  }
+
+  get hashCode(): string {
+    return hashArray([
+      DATA_QUALITY_HASH_STRUCTURE.DATA_QUALITY_GRAPH_FETCH_STATE,
+      this.treeData?.tree ?? '',
+    ]);
+  }
+}

package/src/components/states/DataQualityQueryBuilderState.ts

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { QueryBuilderState } from '@finos/legend-query-builder';
+
+export class DataQualityQueryBuilderState extends QueryBuilderState {}

package/src/components/states/DataQualityResultState.ts

@@ -0,0 +1,217 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { DataQualityState } from './DataQualityState.js';
+import {
+  ExecutionPlanState,
+  QUERY_BUILDER_EVENT,
+  QueryBuilderTelemetryHelper,
+} from '@finos/legend-query-builder';
+import { action, flow, makeObservable, observable } from 'mobx';
+import {
+  type GeneratorFn,
+  ActionState,
+  assertErrorThrown,
+  LogEvent,
+  StopWatch,
+} from '@finos/legend-shared';
+import {
+  type RawExecutionPlan,
+  type ExecutionResult,
+  GRAPH_MANAGER_EVENT,
+} from '@finos/legend-graph';
+import { getDataQualityPureGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataQuality_PureGraphManagerExtension.js';
+
+export const DEFAULT_LIMIT = 1000;
+export class DataQualityResultState {
+  readonly dataQualityState: DataQualityState;
+  readonly executionPlanState: ExecutionPlanState;
+
+  previewLimit = DEFAULT_LIMIT;
+  pressedRunQuery = ActionState.create();
+  isRunningQuery = false;
+  isGeneratingPlan = false;
+  executionResult?: ExecutionResult | undefined;
+  executionDuration?: number | undefined;
+  latestRunHashCode?: string | undefined;
+  queryRunPromise: Promise<ExecutionResult> | undefined = undefined;
+  isQueryUsageViewerOpened = false;
+
+  constructor(dataQualityState: DataQualityState) {
+    makeObservable(this, {
+      executionResult: observable,
+      previewLimit: observable,
+      executionDuration: observable,
+      latestRunHashCode: observable,
+      queryRunPromise: observable,
+      isGeneratingPlan: observable,
+      isRunningQuery: observable,
+      setIsRunningQuery: action,
+      setExecutionResult: action,
+      setExecutionDuration: action,
+      setPreviewLimit: action,
+      setQueryRunPromise: action,
+      runQuery: flow,
+      cancelQuery: flow,
+      generatePlan: flow,
+    });
+
+    this.dataQualityState = dataQualityState;
+    this.executionPlanState = new ExecutionPlanState(
+      this.dataQualityState.applicationStore,
+      this.dataQualityState.graphManagerState,
+    );
+  }
+
+  setIsRunningQuery(val: boolean): void {
+    this.isRunningQuery = val;
+  }
+
+  setExecutionResult(val: ExecutionResult | undefined): void {
+    this.executionResult = val;
+  }
+
+  setExecutionDuration(val: number | undefined): void {
+    this.executionDuration = val;
+  }
+
+  setPreviewLimit(val: number): void {
+    this.previewLimit = Math.max(1, val);
+  }
+
+  setQueryRunPromise(promise: Promise<ExecutionResult> | undefined): void {
+    this.queryRunPromise = promise;
+  }
+
+  get checkForStaleResults(): boolean {
+    if (this.latestRunHashCode !== this.dataQualityState.hashCode) {
+      return true;
+    }
+    return false;
+  }
+
+  *runQuery(): GeneratorFn<void> {
+    let promise;
+    try {
+      this.setIsRunningQuery(true);
+      const currentHashCode = this.dataQualityState.hashCode;
+      const packagePath =
+        this.dataQualityState.constraintsConfigurationElement.path;
+      const model = this.dataQualityState.graphManagerState.graph;
+
+      const stopWatch = new StopWatch();
+
+      promise = getDataQualityPureGraphManagerExtension(
+        this.dataQualityState.graphManagerState.graphManager,
+      ).execute(model, packagePath, this.previewLimit);
+
+      this.setQueryRunPromise(promise);
+      const result = (yield promise) as ExecutionResult;
+
+      if (this.queryRunPromise === promise) {
+        this.setExecutionResult(result);
+        this.latestRunHashCode = currentHashCode;
+        this.setExecutionDuration(stopWatch.elapsed);
+      }
+    } catch (error) {
+      if (this.queryRunPromise === promise) {
+        assertErrorThrown(error);
+        this.dataQualityState.applicationStore.logService.error(
+          LogEvent.create(GRAPH_MANAGER_EVENT.EXECUTION_FAILURE),
+          error,
+        );
+        this.dataQualityState.applicationStore.notificationService.notifyError(
+          error,
+        );
+      }
+    } finally {
+      this.setIsRunningQuery(false);
+      this.pressedRunQuery.complete();
+    }
+  }
+
+  *cancelQuery(): GeneratorFn<void> {
+    this.pressedRunQuery.complete();
+    this.setIsRunningQuery(false);
+    this.setQueryRunPromise(undefined);
+    try {
+      yield this.dataQualityState.graphManagerState.graphManager.cancelUserExecutions(
+        true,
+      );
+    } catch (error) {
+      // Don't notify users about success or failure
+      this.dataQualityState.applicationStore.logService.error(
+        LogEvent.create(GRAPH_MANAGER_EVENT.EXECUTION_FAILURE),
+        error,
+      );
+    }
+  }
+
+  *generatePlan(debug: boolean): GeneratorFn<void> {
+    const packagePath =
+      this.dataQualityState.constraintsConfigurationElement.path;
+    const model = this.dataQualityState.graphManagerState.graph;
+    try {
+      this.isGeneratingPlan = true;
+      let rawPlan: RawExecutionPlan;
+
+      const stopWatch = new StopWatch();
+
+      if (debug) {
+        const debugResult = (yield getDataQualityPureGraphManagerExtension(
+          this.dataQualityState.graphManagerState.graphManager,
+        ).debugExecutionPlanGeneration(model, packagePath)) as {
+          plan: RawExecutionPlan;
+          debug: string;
+        };
+        rawPlan = debugResult.plan;
+        this.executionPlanState.setDebugText(debugResult.debug);
+      } else {
+        QueryBuilderTelemetryHelper.logEvent_ExecutionPlanGenerationLaunched(
+          this.dataQualityState.applicationStore.telemetryService,
+        );
+        rawPlan = (yield getDataQualityPureGraphManagerExtension(
+          this.dataQualityState.graphManagerState.graphManager,
+        ).generatePlan(model, packagePath)) as RawExecutionPlan;
+      }
+
+      stopWatch.record();
+      try {
+        this.executionPlanState.setRawPlan(rawPlan);
+        const plan =
+          this.dataQualityState.graphManagerState.graphManager.buildExecutionPlan(
+            rawPlan,
+            this.dataQualityState.graphManagerState.graph,
+          );
+        this.executionPlanState.initialize(plan);
+      } catch {
+        //do nothing
+      }
+      stopWatch.record(QUERY_BUILDER_EVENT.BUILD_EXECUTION_PLAN__SUCCESS);
+    } catch (error) {
+      assertErrorThrown(error);
+      this.dataQualityState.applicationStore.logService.error(
+        LogEvent.create(GRAPH_MANAGER_EVENT.EXECUTION_FAILURE),
+        error,
+      );
+      this.dataQualityState.applicationStore.notificationService.notifyError(
+        error,
+      );
+    } finally {
+      this.isGeneratingPlan = false;
+    }
+  }
+}