@finos/legend-extension-dsl-data-quality 2.0.173 → 2.1.0

package/src/components/DataQualityRelationValidationConfigurationEditor.tsx

@@ -24,6 +24,7 @@ import {
 import {
   DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB,
   DataQualityRelationValidationConfigurationState,
+  EXECUTION_TYPE,
 } from './states/DataQualityRelationValidationConfigurationState.js';
 import React, { useCallback, useEffect, useRef } from 'react';
 import {
@@ -112,11 +113,13 @@ const RelationDefinitionEditor = observer(
       dataQualityRelationValidationConfigurationState.isReadOnly;
 
     const isExportDisabled =
-      dataQualityRelationValidationConfigurationState.isRunningValidation ||
+      dataQualityRelationValidationConfigurationState.lastExecutionType ===
+        undefined ||
+      dataQualityRelationValidationConfigurationState.isRunning ||
       dataQualityRelationValidationConfigurationState.isGeneratingPlan;
 
-    const exportValidationResults = async (format: string): Promise<void> => {
-      dataQualityRelationValidationConfigurationState.handleExport(format);
+    const exportResults = async (format: string): Promise<void> => {
+      dataQualityRelationValidationConfigurationState.exportData(format);
     };
 
     const getResultSetDescription = (
@@ -134,10 +137,18 @@ const RelationDefinitionEditor = observer(
       if (!queryDuration) {
         return undefined;
       }
-      return `lambda ran in ${queryDuration}`;
+      const executionName =
+        dataQualityRelationValidationConfigurationState.lastExecutionType ===
+        EXECUTION_TYPE.EXECUTION
+          ? 'Query'
+          : dataQualityRelationValidationConfigurationState.lastExecutionType ===
+              EXECUTION_TYPE.PROFILING
+            ? 'Data profile'
+            : 'Lambda';
+      return `${executionName} ran in ${queryDuration}`;
     };
     const resultDescription =
-      !dataQualityRelationValidationConfigurationState.isRunningValidation &&
+      !dataQualityRelationValidationConfigurationState.isRunning &&
       lambdaExecutionResult
         ? getResultSetDescription(lambdaExecutionResult)
         : undefined;
@@ -151,7 +162,7 @@ const RelationDefinitionEditor = observer(
             label: 'Accept',
             type: ActionAlertActionType.PROCEED_WITH_CAUTION,
             handler: applicationStore.guardUnhandledError(() =>
-              exportValidationResults(format),
+              exportResults(format),
             ),
           },
           {
@@ -258,7 +269,7 @@ const RelationDefinitionEditor = observer(
         <PanelLoadingIndicator
           isLoading={
             resultState.isGeneratingPlan ||
-            dataQualityRelationValidationConfigurationState.isRunningValidation
+            dataQualityRelationValidationConfigurationState.isRunning
           }
         />
         <div className="relation-validation-config-editor__definition">
@@ -343,11 +354,20 @@ const RelationDefinitionEditor = observer(
                 <div className="relation-validation-config-editor__definition__item__header__title">
                   RESULT
                 </div>
-                {dataQualityRelationValidationConfigurationState.isRunningValidation && (
-                  <div className="panel__header__title__label__status">
-                    Running Validation...
-                  </div>
-                )}
+                {dataQualityRelationValidationConfigurationState.isRunning &&
+                  dataQualityRelationValidationConfigurationState.currentExecutionType ===
+                    EXECUTION_TYPE.EXECUTION && (
+                    <div className="panel__header__title__label__status">
+                      Running Validation...
+                    </div>
+                  )}
+                {dataQualityRelationValidationConfigurationState.isRunning &&
+                  dataQualityRelationValidationConfigurationState.currentExecutionType ===
+                    EXECUTION_TYPE.PROFILING && (
+                    <div className="panel__header__title__label__status">
+                      Running Profiling...
+                    </div>
+                  )}
                 <div
                   data-testid={
                     DATA_QUALITY_VALIDATION_TEST_ID.DATA_QUALITY_VALIDATION_RESULT_ANALYTICS
@@ -435,18 +455,26 @@ export const DataQualityRelationValidationConfigurationEditor = observer(() => {
       dataQualityRelationValidationConfigurationState.setSelectedTab(tab);
 
   const executionIsRunning =
-    dataQualityRelationValidationConfigurationState.isRunningValidation ||
+    dataQualityRelationValidationConfigurationState.isRunning ||
     dataQualityRelationValidationConfigurationState.isGeneratingPlan;
 
-  const cancelValidation = applicationStore.guardUnhandledError(() =>
+  const cancelRun = applicationStore.guardUnhandledError(() =>
+    flowResult(dataQualityRelationValidationConfigurationState.cancelRun()),
+  );
+
+  const runValidation = applicationStore.guardUnhandledError(() =>
     flowResult(
-      dataQualityRelationValidationConfigurationState.cancelValidationRun(),
+      dataQualityRelationValidationConfigurationState.handleRun(
+        EXECUTION_TYPE.EXECUTION,
+      ),
     ),
   );
 
-  const runValidation = applicationStore.guardUnhandledError(() =>
+  const runDataProfiling = applicationStore.guardUnhandledError(() =>
     flowResult(
-      dataQualityRelationValidationConfigurationState.handleRunValidation(),
+      dataQualityRelationValidationConfigurationState.handleRun(
+        EXECUTION_TYPE.PROFILING,
+      ),
     ),
   );
 
@@ -513,10 +541,10 @@ export const DataQualityRelationValidationConfigurationEditor = observer(() => {
               DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB.DEFINITION && (
               <>
                 <div className="btn__dropdown-combo btn__dropdown-combo--primary">
-                  {dataQualityRelationValidationConfigurationState.isRunningValidation ? (
+                  {dataQualityRelationValidationConfigurationState.isRunning ? (
                     <button
                       className="btn__dropdown-combo__canceler"
-                      onClick={cancelValidation}
+                      onClick={cancelRun}
                       tabIndex={-1}
                     >
                       <div className="btn--dark btn--caution btn__dropdown-combo__canceler__label">
@@ -557,6 +585,12 @@ export const DataQualityRelationValidationConfigurationEditor = observer(() => {
                             >
                               Debug
                             </MenuContentItem>
+                            <MenuContentItem
+                              className="btn__dropdown-combo__option"
+                              onClick={runDataProfiling}
+                            >
+                              Data Profile
+                            </MenuContentItem>
                           </MenuContent>
                         }
                         menuProps={{

package/src/components/states/DataQualityRelationResultState.ts

@@ -159,7 +159,7 @@ export class DataQualityRelationResultState {
     if (parameters.length) {
       this.dataQualityRelationValidationConfigurationState.parametersState.openModal(
         queryLambda,
-        true,
+        () => this.runValidation(),
       );
     } else {
       flowResult(this.runValidation()).catch(

package/src/components/states/DataQualityRelationValidationConfigurationState.ts

@@ -79,6 +79,11 @@ export enum DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB {
   TRIAL_RUN = 'Trial Run',
 }
 
+export enum EXECUTION_TYPE {
+  EXECUTION = 'EXECUTION',
+  PROFILING = 'PROFILING',
+}
+
 export class RelationFunctionDefinitionEditorState extends LambdaEditorState {
   readonly editorStore: EditorStore;
   readonly relationValidationElement: DataQualityRelationValidationConfiguration;
@@ -211,15 +216,11 @@ export class RelationDefinitionParameterState extends LambdaParametersState {
       relationValidationConfigurationState;
   }
 
-  openModal(lambda: RawLambda, runLambdaWithConstraints: boolean): void {
+  openModal(lambda: RawLambda, onSubmit: () => GeneratorFn<void>): void {
     this.parameterStates = this.build(lambda);
     this.parameterValuesEditorState.open(
       (): Promise<void> =>
-        flowResult(
-          runLambdaWithConstraints
-            ? this.relationValidationConfigurationState.resultState.runValidation()
-            : this.relationValidationConfigurationState.runValidation(),
-        ).catch(
+        flowResult(onSubmit()).catch(
           this.relationValidationConfigurationState.editorStore.applicationStore
             .alertUnhandledError,
         ),
@@ -258,10 +259,11 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
   readonly exportState = ActionState.create();
   selectedTab: DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB;
 
-  isRunningValidation = false;
+  lastExecutionType: EXECUTION_TYPE | undefined = undefined;
+  currentExecutionType: EXECUTION_TYPE | undefined = undefined;
   isGeneratingPlan = false;
-  validationRunPromise: Promise<ExecutionResult> | undefined = undefined;
-  executionResult?: ExecutionResult | undefined; // NOTE: stored as lossless JSON string
+  runPromise: Promise<ExecutionResult> | undefined = undefined;
+  executionResult?: ExecutionResult | undefined;
   executionPlanState: ExecutionPlanState;
   validationStates: DataQualityRelationValidationState[] = [];
   parametersState: RelationDefinitionParameterState;
@@ -278,14 +280,15 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
 
     makeObservable(this, {
       selectedTab: observable,
-      isRunningValidation: observable,
-      validationRunPromise: observable,
+      currentExecutionType: observable,
+      runPromise: observable,
       executionResult: observable,
       resultState: observable,
       executionDuration: observable,
       latestRunHashCode: observable,
+      lastExecutionType: observable,
       setSelectedTab: action,
-      setValidationRunPromise: action,
+      setRunPromise: action,
       setExecutionResult: action,
       addValidationState: action,
       resetResultState: action,
@@ -293,13 +296,15 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
       validationElement: computed,
       relationValidationOptions: computed,
       checkForStaleResults: computed,
-      runValidation: flow,
-      handleRunValidation: flow,
+      isRunning: computed,
+      run: flow,
+      handleRun: flow,
+      exportData: flow,
       convertValidationLambdaObjects: flow,
-      cancelValidationRun: flow,
+      cancelRun: flow,
       generatePlan: flow,
-      exportData: flow,
     });
+
     assertType(
       element,
       DataQualityRelationValidationConfiguration,
@@ -378,6 +383,10 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
     return false;
   }
 
+  get isRunning(): boolean {
+    return this.currentExecutionType !== undefined;
+  }
+
   setExecutionDuration(val: number | undefined): void {
     this.executionDuration = val;
   }
@@ -407,30 +416,32 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
     }
   }
 
-  setValidationRunPromise = (
-    promise: Promise<ExecutionResult> | undefined,
-  ): void => {
-    this.validationRunPromise = promise;
+  setRunPromise = (promise: Promise<ExecutionResult> | undefined): void => {
+    this.runPromise = promise;
   };
 
   setSelectedTab(tab: DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB): void {
     this.selectedTab = tab;
   }
 
-  setExecutionResult = (executionResult: ExecutionResult | undefined): void => {
+  setExecutionResult = (
+    executionResult: ExecutionResult | undefined,
+    type: EXECUTION_TYPE,
+  ): void => {
+    this.lastExecutionType = type;
     this.executionResult = executionResult;
   };
 
-  *handleRunValidation(): GeneratorFn<void> {
-    if (this.isRunningValidation) {
+  *handleRun(type: EXECUTION_TYPE): GeneratorFn<void> {
+    if (this.isRunning) {
       return;
     }
     const queryLambda = this.bodyExpressionSequence;
     const parameters = (queryLambda.parameters ?? []) as object[];
     if (parameters.length) {
-      this.parametersState.openModal(queryLambda, false);
+      this.parametersState.openModal(queryLambda, () => this.run(type));
     } else {
-      flowResult(this.runValidation()).catch(
+      flowResult(this.run(type)).catch(
         this.editorStore.applicationStore.alertUnhandledError,
       );
     }
@@ -484,41 +495,45 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
     );
   }
 
-  setIsRunningValidation(val: boolean): void {
-    this.isRunningValidation = val;
-  }
-
-  *runValidation(): GeneratorFn<void> {
-    let promise;
+  *run(type: EXECUTION_TYPE): GeneratorFn<void> {
+    let promise: Promise<ExecutionResult> | undefined = undefined;
+    const stopWatch = new StopWatch();
     try {
-      this.setIsRunningValidation(true);
+      this.currentExecutionType = type;
       const currentHashCode = this.hashCode;
       const packagePath = this.validationElement.path;
       const model = this.editorStore.graphManagerState.graph;
-      const stopWatch = new StopWatch();
-
-      promise = getDataQualityPureGraphManagerExtension(
+      const extension = getDataQualityPureGraphManagerExtension(
         this.editorStore.graphManagerState.graphManager,
-      ).execute(model, packagePath, {
-        runQuery: true,
+      );
+
+      const options = {
         lambdaParameterValues: buildExecutionParameterValues(
           this.parametersState.parameterStates,
           this.editorStore.graphManagerState,
         ),
-      });
+      };
+
+      promise =
+        type === EXECUTION_TYPE.PROFILING
+          ? extension.runDataProfiling(model, packagePath, options)
+          : extension.execute(model, packagePath, {
+              ...options,
+              runQuery: true,
+            });
 
-      this.setValidationRunPromise(promise);
+      this.setRunPromise(promise);
       const result = (yield promise) as ExecutionResult;
 
-      if (this.validationRunPromise === promise) {
-        this.setExecutionResult(result);
+      if (this.runPromise === promise) {
+        this.setExecutionResult(result, type);
         this.latestRunHashCode = currentHashCode;
         this.setExecutionDuration(stopWatch.elapsed);
       }
     } catch (error) {
-      if (this.validationRunPromise === promise) {
+      if (this.runPromise === promise) {
         assertErrorThrown(error);
-        this.setExecutionResult(undefined);
+        this.setExecutionResult(undefined, type);
         this.editorStore.applicationStore.logService.error(
           LogEvent.create(GRAPH_MANAGER_EVENT.EXECUTION_FAILURE),
           error,
@@ -528,13 +543,13 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
         );
       }
     } finally {
-      this.setIsRunningValidation(false);
+      this.currentExecutionType = undefined;
     }
   }
 
-  *cancelValidationRun(): GeneratorFn<void> {
-    this.setIsRunningValidation(false);
-    this.setValidationRunPromise(undefined);
+  *cancelRun(): GeneratorFn<void> {
+    this.currentExecutionType = undefined;
+    this.setRunPromise(undefined);
     try {
       yield this.editorStore.graphManagerState.graphManager.cancelUserExecutions(
         true,
@@ -600,6 +615,7 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
   *exportData(format: string): GeneratorFn<void> {
     try {
       this.exportState.inProgress();
+      const type = this.lastExecutionType;
       const packagePath = this.validationElement.path;
       const model = this.editorStore.graphManagerState.graph;
       this.editorStore.applicationStore.notificationService.notifySuccess(
@@ -608,16 +624,30 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
       const exportData = this.resultState.getExportDataInfo(format);
       const contentType = exportData.contentType;
       const serializationFormat = exportData.serializationFormat;
-      const result = (yield getDataQualityPureGraphManagerExtension(
+      const extension = getDataQualityPureGraphManagerExtension(
         this.editorStore.graphManagerState.graphManager,
-      ).exportData(model, packagePath, {
+      );
+
+      const options = {
         serializationFormat,
-        runQuery: true,
         lambdaParameterValues: buildExecutionParameterValues(
           this.parametersState.parameterStates,
           this.editorStore.graphManagerState,
         ),
-      })) as Response;
+      };
+
+      const result =
+        type === EXECUTION_TYPE.PROFILING
+          ? ((yield extension.exportDataProfiling(
+              model,
+              packagePath,
+              options,
+            )) as Response)
+          : ((yield extension.exportData(model, packagePath, {
+              ...options,
+              runQuery: true,
+            })) as Response);
+
       if (result.headers.get(V1_DELEGATED_EXPORT_HEADER) === 'true') {
         if (result.status === 200) {
           this.exportState.pass();
@@ -645,26 +675,6 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
     }
   }
 
-  handleExport(format: string) {
-    const queryLambda = this.bodyExpressionSequence;
-    const parameters = (queryLambda.parameters ?? []) as object[];
-    if (parameters.length) {
-      this.parametersState.parameterStates =
-        this.parametersState.build(queryLambda);
-      this.parametersState.parameterValuesEditorState.open(
-        (): Promise<void> =>
-          flowResult(this.exportData(format)).catch(
-            this.editorStore.applicationStore.alertUnhandledError,
-          ),
-        PARAMETER_SUBMIT_ACTION.EXPORT,
-      );
-    } else {
-      flowResult(this.exportData(format)).catch(
-        this.editorStore.applicationStore.alertUnhandledError,
-      );
-    }
-  }
-
   get hashCode(): string {
     return hashArray([
       DATA_QUALITY_HASH_STRUCTURE.DATA_QUALITY_RELATION_VALIDATION,

package/src/graph-manager/protocol/pure/DSL_DataQuality_PureGraphManagerExtension.ts

@@ -32,6 +32,18 @@ export abstract class DSL_DataQuality_PureGraphManagerExtension extends Abstract
     options: DQExecuteInputOptions,
   ): Promise<RawExecutionPlan>;
 
+  abstract runDataProfiling(
+    graph: PureModel,
+    packagePath: string,
+    options: DQExecuteInputOptions,
+  ): Promise<ExecutionResult>;
+
+  abstract exportDataProfiling(
+    graph: PureModel,
+    packagePath: string,
+    options: DQExecuteInputOptions,
+  ): Promise<Response>;
+
   abstract execute(
     graph: PureModel,
     packagePath: string,

package/src/graph-manager/protocol/pure/v1/V1_DSL_Data_Quality_PureGraphManagerExtension.ts

@@ -388,4 +388,115 @@ export class V1_DSL_Data_Quality_PureGraphManagerExtension extends DSL_DataQuali
       true,
     ) as DataQualityRootGraphFetchTree;
   };
+
+  private executeDataProfiling = (
+    input: PlainObject<V1_DQExecuteInput>,
+    options?: {
+      returnAsResponse?: boolean;
+      serializationFormat?: EXECUTION_SERIALIZATION_FORMAT | undefined;
+    },
+  ): Promise<PlainObject<V1_ExecutionResult> | Response> => {
+    const engineServerClient = guaranteeType(
+      this.graphManager.engine,
+      V1_RemoteEngine,
+      'executeDataProfiling is only supported by remote engine',
+    ).getEngineServerClient();
+    return engineServerClient.postWithTracing(
+      engineServerClient.getTraceData(DQ_GENERATE_EXECUTION_PLAN),
+      `${engineServerClient._pure()}/dataquality/profile`,
+      input,
+      {},
+      undefined,
+      {
+        serializationFormat: options?.serializationFormat
+          ? V1_getEngineSerializationFormat(options.serializationFormat)
+          : undefined,
+      },
+      { enableCompression: true },
+      {
+        skipProcessing: Boolean(options?.returnAsResponse),
+      },
+    );
+  };
+
+  runDataProfiling = async (
+    graph: PureModel,
+    packagePath: string,
+    options: DQExecuteInputOptions,
+  ): Promise<ExecutionResult> => {
+    const input = this.createExecutionInput(
+      graph,
+      packagePath,
+      new V1_DQExecuteInput(),
+      options,
+    );
+
+    try {
+      const profilingResult = (await this.executeDataProfiling(
+        V1_DQExecuteInput.serialization.toJson(input),
+        {
+          returnAsResponse: true,
+        },
+      )) as Response;
+
+      const profilingResultInText = await profilingResult.text();
+
+      const rawExecutionResult =
+        returnUndefOnError(() =>
+          this.graphManager.engine.parseExecutionResults(
+            profilingResultInText,
+            undefined,
+          ),
+        ) ?? profilingResultInText;
+
+      const v1_executionResult =
+        V1_deserializeExecutionResult(rawExecutionResult);
+      return V1_buildExecutionResult(v1_executionResult);
+    } catch (error) {
+      assertErrorThrown(error);
+      if (error instanceof NetworkClientError) {
+        throw V1_buildExecutionError(
+          V1_ExecutionError.serialization.fromJson(
+            error.payload as PlainObject<V1_ExecutionError>,
+          ),
+        );
+      }
+      throw error;
+    }
+  };
+
+  exportDataProfiling = async (
+    graph: PureModel,
+    packagePath: string,
+    options: DQExecuteInputOptions,
+  ): Promise<Response> => {
+    const input = this.createExecutionInput(
+      graph,
+      packagePath,
+      new V1_DQExecuteInput(),
+      options,
+    );
+
+    try {
+      return guaranteeNonNullable(
+        (await this.executeDataProfiling(
+          V1_DQExecuteInput.serialization.toJson(input),
+          {
+            serializationFormat: options.serializationFormat,
+            returnAsResponse: true,
+          },
+        )) as Response,
+      );
+    } catch (error) {
+      assertErrorThrown(error);
+      if (error instanceof NetworkClientError) {
+        throw V1_buildExecutionError(
+          V1_ExecutionError.serialization.fromJson(
+            error.payload as PlainObject<V1_ExecutionError>,
+          ),
+        );
+      }
+      throw error;
+    }
+  };
 }