@finos/legend-extension-dsl-data-quality 2.0.16 → 2.0.18

package/src/components/DataQualityRelationValidationConfigurationEditor.tsx

@@ -31,9 +31,13 @@ import {
   CaretDownIcon,
   clsx,
   ControlledDropdownMenu,
+  CsvIcon,
   DragPreviewLayer,
+  ExclamationTriangleIcon,
   MenuContent,
   MenuContentItem,
+  MenuContentItemIcon,
+  MenuContentItemLabel,
   Panel,
   PanelContent,
   PanelLoadingIndicator,
@@ -41,7 +45,11 @@ import {
   PlayIcon,
   PlusIcon,
 } from '@finos/legend-art';
-import { prettyCONSTName, returnUndefOnError } from '@finos/legend-shared';
+import {
+  prettyCONSTName,
+  prettyDuration,
+  returnUndefOnError,
+} from '@finos/legend-shared';
 import {
   ExecutionPlanViewer,
   LambdaEditor,
@@ -49,6 +57,7 @@ import {
 } from '@finos/legend-query-builder';
 import {
   type RawVariableExpression,
+  type ExecutionResult,
   PrimitiveType,
   stub_RawVariableExpression,
   Type,
@@ -57,6 +66,8 @@ import {
   TDSExecutionResult,
 } from '@finos/legend-graph';
 import {
+  ActionAlertActionType,
+  ActionAlertType,
   DEFAULT_TAB_SIZE,
   useApplicationNavigationContext,
   useApplicationStore,
@@ -77,6 +88,10 @@ import { DataQualityRelationValidationsEditor } from './DataQualityRelationValid
 import { CodeEditor } from '@finos/legend-lego/code-editor';
 import { CODE_EDITOR_LANGUAGE } from '@finos/legend-code-editor';
 import { DataQualityRelationGridResult } from './DataQualityRelationGridResult.js';
+import {
+  DATA_QUALITY_VALIDATION_TEST_ID,
+  USER_ATTESTATION_MESSAGE,
+} from './constants/DataQualityConstants.js';
 
 const RelationDefinitionEditor = observer(
   (props: {
@@ -85,6 +100,9 @@ const RelationDefinitionEditor = observer(
     const { dataQualityRelationValidationConfigurationState } = props;
     const { relationFunctionDefinitionEditorState, resultState } =
       dataQualityRelationValidationConfigurationState;
+    const {
+      editorStore: { applicationStore },
+    } = dataQualityRelationValidationConfigurationState;
     const validationElement =
       dataQualityRelationValidationConfigurationState.validationElement;
     const lambdaExecutionResult =
@@ -93,6 +111,58 @@ const RelationDefinitionEditor = observer(
     const isReadOnly =
       dataQualityRelationValidationConfigurationState.isReadOnly;
 
+    const isExportDisabled =
+      dataQualityRelationValidationConfigurationState.isRunningValidation ||
+      dataQualityRelationValidationConfigurationState.isGeneratingPlan;
+
+    const exportValidationResults = async (format: string): Promise<void> => {
+      dataQualityRelationValidationConfigurationState.handleExport(format);
+    };
+
+    const getResultSetDescription = (
+      _executionResult: ExecutionResult,
+    ): string | undefined => {
+      const queryDuration =
+        dataQualityRelationValidationConfigurationState.executionDuration
+          ? prettyDuration(
+              dataQualityRelationValidationConfigurationState.executionDuration,
+              {
+                ms: true,
+              },
+            )
+          : undefined;
+      if (!queryDuration) {
+        return undefined;
+      }
+      return `lambda ran in ${queryDuration}`;
+    };
+    const resultDescription =
+      !dataQualityRelationValidationConfigurationState.isRunningValidation &&
+      lambdaExecutionResult
+        ? getResultSetDescription(lambdaExecutionResult)
+        : undefined;
+
+    const confirmExport = (format: string): void => {
+      applicationStore.alertService.setActionAlertInfo({
+        message: USER_ATTESTATION_MESSAGE,
+        type: ActionAlertType.CAUTION,
+        actions: [
+          {
+            label: 'Accept',
+            type: ActionAlertActionType.PROCEED_WITH_CAUTION,
+            handler: applicationStore.guardUnhandledError(() =>
+              exportValidationResults(format),
+            ),
+          },
+          {
+            label: 'Decline',
+            type: ActionAlertActionType.PROCEED,
+            default: true,
+          },
+        ],
+      });
+    };
+
     const addParameter = (): void => {
       dataQualityRelationValidation_addParameter(
         validationElement.query.parameters,
@@ -269,9 +339,71 @@ const RelationDefinitionEditor = observer(
           </div>
           <div className="relation-validation-config-editor__item">
             <div className="relation-validation-config-editor__definition__item__header">
-              <div className="relation-validation-config-editor__definition__item__header__title">
-                RESULT
+              <div className="relation-validation-config-editor__definition__item__header-group">
+                <div className="relation-validation-config-editor__definition__item__header__title">
+                  RESULT
+                </div>
+                {dataQualityRelationValidationConfigurationState.isRunningValidation && (
+                  <div className="panel__header__title__label__status">
+                    Running Validation...
+                  </div>
+                )}
+                <div
+                  data-testid={
+                    DATA_QUALITY_VALIDATION_TEST_ID.DATA_QUALITY_VALIDATION_RESULT_ANALYTICS
+                  }
+                  className="data-quality-validation__result__analytics"
+                >
+                  {resultDescription ?? ''}
+                </div>
+                {lambdaExecutionResult &&
+                  dataQualityRelationValidationConfigurationState.checkForStaleResults && (
+                    <div className="data-quality-validation__result__stale-status">
+                      <div className="data-quality-validation__result__stale-status__icon">
+                        <ExclamationTriangleIcon />
+                      </div>
+                      <div className="data-quality-validation__result__stale-status__label">
+                        Preview data might be stale
+                      </div>
+                    </div>
+                  )}
               </div>
+              <ControlledDropdownMenu
+                className="data-quality-validation__result__export__dropdown"
+                title="Export"
+                disabled={isExportDisabled}
+                content={
+                  <MenuContent>
+                    {Object.values(resultState.exportDataFormatOptions).map(
+                      (format) => (
+                        <MenuContentItem
+                          key={format}
+                          onClick={(): void => confirmExport(format)}
+                        >
+                          <MenuContentItemIcon>
+                            <CsvIcon />
+                          </MenuContentItemIcon>
+                          <MenuContentItemLabel>{format}</MenuContentItemLabel>
+                        </MenuContentItem>
+                      ),
+                    )}
+                  </MenuContent>
+                }
+                menuProps={{
+                  anchorOrigin: { vertical: 'bottom', horizontal: 'right' },
+                  transformOrigin: { vertical: 'top', horizontal: 'right' },
+                  elevation: 7,
+                }}
+              >
+                <div className="relation-validation-config-editor__definition__item__header__action">
+                  <div className="data-quality-validation__result__export__dropdown__label">
+                    Export
+                  </div>
+                  <div className="data-quality-validation__result__export__dropdown__trigger">
+                    <CaretDownIcon />
+                  </div>
+                </div>
+              </ControlledDropdownMenu>
             </div>
             <div className="relation-validation-config-editor__definition__item__content">
               <div className="relation-validation-config-editor__definition__result-viewer">

package/src/components/constants/DataQualityConstants.ts

@@ -22,3 +22,6 @@ export enum DATA_QUALITY_VALIDATION_TEST_ID {
   DATA_QUALITY_VALIDATION_RESULT_PANEL = 'data-quality-result-panel',
   DATA_QUALITY_VALIDATION_RESULT_ANALYTICS = 'data-quality-result-analytics',
 }
+
+export const USER_ATTESTATION_MESSAGE =
+  'I attest that I am aware of the sensitive data leakage risk when exporting queried data. The data I export will only be used by me.';

package/src/components/states/DataQualityRelationResultState.ts

@@ -15,10 +15,12 @@
  */
 
 import {
+  type ExportDataInfo,
   buildExecutionParameterValues,
   ExecutionPlanState,
   QUERY_BUILDER_EVENT,
   QueryBuilderTelemetryHelper,
+  PARAMETER_SUBMIT_ACTION,
 } from '@finos/legend-query-builder';
 import { action, flow, flowResult, makeObservable, observable } from 'mobx';
 import {
@@ -26,15 +28,22 @@ import {
   assertErrorThrown,
   LogEvent,
   StopWatch,
+  getContentTypeFileExtension,
+  ActionState,
+  ContentType,
+  UnsupportedOperationError,
 } from '@finos/legend-shared';
 import {
   type ExecutionResult,
   type RawExecutionPlan,
   GRAPH_MANAGER_EVENT,
+  V1_DELEGATED_EXPORT_HEADER,
+  EXECUTION_SERIALIZATION_FORMAT,
 } from '@finos/legend-graph';
 import { getDataQualityPureGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataQuality_PureGraphManagerExtension.js';
 import type { DataQualityRelationValidationConfigurationState } from './DataQualityRelationValidationConfigurationState.js';
 import type { DataQualityRelationValidation } from '../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
+import { downloadStream } from '@finos/legend-application';
 
 export type DataQualityRelationResultCellDataType =
   | string
@@ -56,6 +65,7 @@ export const DEFAULT_LIMIT = 1000;
 export class DataQualityRelationResultState {
   readonly dataQualityRelationValidationConfigurationState: DataQualityRelationValidationConfigurationState;
   readonly executionPlanState: ExecutionPlanState;
+  readonly exportState = ActionState.create();
 
   previewLimit = DEFAULT_LIMIT;
   isRunningValidation = false;
@@ -85,9 +95,11 @@ export class DataQualityRelationResultState {
       setValidationRunPromise: action,
       setValidationToRun: action,
       handleRunValidation: action,
+      handleExport: action,
       runValidation: flow,
       cancelValidation: flow,
       generatePlan: flow,
+      exportData: flow,
     });
 
     this.dataQualityRelationValidationConfigurationState =
@@ -150,6 +162,32 @@ export class DataQualityRelationResultState {
     }
   }
 
+  handleExport(format: string) {
+    const queryLambda =
+      this.dataQualityRelationValidationConfigurationState
+        .bodyExpressionSequence;
+    const parameters = (queryLambda.parameters ?? []) as object[];
+    if (parameters.length) {
+      this.dataQualityRelationValidationConfigurationState.parametersState.parameterStates =
+        this.dataQualityRelationValidationConfigurationState.parametersState.build(
+          queryLambda,
+        );
+      this.dataQualityRelationValidationConfigurationState.parametersState.parameterValuesEditorState.open(
+        (): Promise<void> =>
+          flowResult(this.exportData(format)).catch(
+            this.dataQualityRelationValidationConfigurationState.editorStore
+              .applicationStore.alertUnhandledError,
+          ),
+        PARAMETER_SUBMIT_ACTION.EXPORT,
+      );
+    } else {
+      flowResult(this.exportData(format)).catch(
+        this.dataQualityRelationValidationConfigurationState.editorStore
+          .applicationStore.alertUnhandledError,
+      );
+    }
+  }
+
   *runValidation(): GeneratorFn<void> {
     let promise;
     try {
@@ -287,4 +325,81 @@ export class DataQualityRelationResultState {
       this.isGeneratingPlan = false;
     }
   }
+
+  getExportDataInfo(format: string): ExportDataInfo {
+    switch (format) {
+      case EXECUTION_SERIALIZATION_FORMAT.CSV:
+        return {
+          contentType: ContentType.TEXT_CSV,
+          serializationFormat: EXECUTION_SERIALIZATION_FORMAT.CSV,
+        };
+
+      default:
+        throw new UnsupportedOperationError(
+          `Unsupported TDS export type ${format}`,
+        );
+    }
+  }
+
+  get exportDataFormatOptions(): string[] {
+    return [EXECUTION_SERIALIZATION_FORMAT.CSV];
+  }
+
+  *exportData(format: string): GeneratorFn<void> {
+    try {
+      this.exportState.inProgress();
+      const packagePath =
+        this.dataQualityRelationValidationConfigurationState.validationElement
+          .path;
+      const model =
+        this.dataQualityRelationValidationConfigurationState.editorStore
+          .graphManagerState.graph;
+      this.dataQualityRelationValidationConfigurationState.editorStore.applicationStore.notificationService.notifySuccess(
+        `Export ${format} will run in background`,
+      );
+      const exportData = this.getExportDataInfo(format);
+      const contentType = exportData.contentType;
+      const serializationFormat = exportData.serializationFormat;
+      const result = (yield getDataQualityPureGraphManagerExtension(
+        this.dataQualityRelationValidationConfigurationState.editorStore
+          .graphManagerState.graphManager,
+      ).exportData(model, packagePath, {
+        serializationFormat,
+        previewLimit: this.previewLimit,
+        validationName: this.validationToRun?.name,
+        lambdaParameterValues: buildExecutionParameterValues(
+          this.dataQualityRelationValidationConfigurationState.parametersState
+            .parameterStates,
+          this.dataQualityRelationValidationConfigurationState.editorStore
+            .graphManagerState,
+        ),
+      })) as Response;
+      if (result.headers.get(V1_DELEGATED_EXPORT_HEADER) === 'true') {
+        if (result.status === 200) {
+          this.exportState.pass();
+        } else {
+          this.exportState.fail();
+        }
+        return;
+      }
+      downloadStream(
+        result,
+        `result.${getContentTypeFileExtension(contentType)}`,
+        exportData.contentType,
+      )
+        .then(() => {
+          this.exportState.pass();
+        })
+        .catch((error) => {
+          assertErrorThrown(error);
+        });
+    } catch (error) {
+      this.exportState.fail();
+      assertErrorThrown(error);
+      this.dataQualityRelationValidationConfigurationState.editorStore.applicationStore.notificationService.notifyError(
+        error,
+      );
+      this.exportState.complete();
+    }
+  }
 }

package/src/components/states/DataQualityRelationValidationConfigurationState.ts

@@ -32,6 +32,9 @@ import {
   hashArray,
   LogEvent,
   filterByType,
+  getContentTypeFileExtension,
+  ActionState,
+  StopWatch,
 } from '@finos/legend-shared';
 import {
   type ExecutionResult,
@@ -45,6 +48,7 @@ import {
   buildLambdaVariableExpressions,
   observe_ValueSpecification,
   VariableExpression,
+  V1_DELEGATED_EXPORT_HEADER,
 } from '@finos/legend-graph';
 import {
   action,
@@ -67,6 +71,7 @@ import { DataQualityRelationResultState } from './DataQualityRelationResultState
 import { DATA_QUALITY_HASH_STRUCTURE } from '../../graph/metamodel/DSL_DataQuality_HashUtils.js';
 import type { SelectOption } from '@finos/legend-art';
 import { getDataQualityPureGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataQuality_PureGraphManagerExtension.js';
+import { downloadStream } from '@finos/legend-application';
 
 export enum DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB {
   DEFINITION = 'Definition',
@@ -250,6 +255,7 @@ export class RelationDefinitionParameterState extends LambdaParametersState {
 
 export class DataQualityRelationValidationConfigurationState extends ElementEditorState {
   readonly relationFunctionDefinitionEditorState: RelationFunctionDefinitionEditorState;
+  readonly exportState = ActionState.create();
   selectedTab: DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB;
 
   isRunningValidation = false;
@@ -261,6 +267,8 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
   parametersState: RelationDefinitionParameterState;
   isConvertingValidationLambdaObjects = false;
   resultState: DataQualityRelationResultState;
+  executionDuration?: number | undefined;
+  latestRunHashCode?: string | undefined;
 
   constructor(
     editorStore: EditorStore,
@@ -274,18 +282,23 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
       validationRunPromise: observable,
       executionResult: observable,
       resultState: observable,
+      executionDuration: observable,
+      latestRunHashCode: observable,
       setSelectedTab: action,
       setValidationRunPromise: action,
       setExecutionResult: action,
       addValidationState: action,
       resetResultState: action,
+      setExecutionDuration: action,
       validationElement: computed,
       relationValidationOptions: computed,
+      checkForStaleResults: computed,
       runValidation: flow,
       handleRunValidation: flow,
       convertValidationLambdaObjects: flow,
       cancelValidationRun: flow,
       generatePlan: flow,
+      exportData: flow,
     });
     assertType(
       element,
@@ -358,6 +371,17 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
     }));
   }
 
+  get checkForStaleResults(): boolean {
+    if (this.latestRunHashCode !== this.hashCode) {
+      return true;
+    }
+    return false;
+  }
+
+  setExecutionDuration(val: number | undefined): void {
+    this.executionDuration = val;
+  }
+
   resetResultState(): void {
     this.resultState = new DataQualityRelationResultState(this);
   }
@@ -468,8 +492,10 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
     let promise;
     try {
       this.setIsRunningValidation(true);
+      const currentHashCode = this.hashCode;
       const packagePath = this.validationElement.path;
       const model = this.editorStore.graphManagerState.graph;
+      const stopWatch = new StopWatch();
 
       promise = getDataQualityPureGraphManagerExtension(
         this.editorStore.graphManagerState.graphManager,
@@ -486,6 +512,8 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
 
       if (this.validationRunPromise === promise) {
         this.setExecutionResult(result);
+        this.latestRunHashCode = currentHashCode;
+        this.setExecutionDuration(stopWatch.elapsed);
       }
     } catch (error) {
       if (this.validationRunPromise === promise) {
@@ -569,6 +597,74 @@ export class DataQualityRelationValidationConfigurationState extends ElementEdit
     }
   }
 
+  *exportData(format: string): GeneratorFn<void> {
+    try {
+      this.exportState.inProgress();
+      const packagePath = this.validationElement.path;
+      const model = this.editorStore.graphManagerState.graph;
+      this.editorStore.applicationStore.notificationService.notifySuccess(
+        `Export ${format} will run in background`,
+      );
+      const exportData = this.resultState.getExportDataInfo(format);
+      const contentType = exportData.contentType;
+      const serializationFormat = exportData.serializationFormat;
+      const result = (yield getDataQualityPureGraphManagerExtension(
+        this.editorStore.graphManagerState.graphManager,
+      ).exportData(model, packagePath, {
+        serializationFormat,
+        runQuery: true,
+        lambdaParameterValues: buildExecutionParameterValues(
+          this.parametersState.parameterStates,
+          this.editorStore.graphManagerState,
+        ),
+      })) as Response;
+      if (result.headers.get(V1_DELEGATED_EXPORT_HEADER) === 'true') {
+        if (result.status === 200) {
+          this.exportState.pass();
+        } else {
+          this.exportState.fail();
+        }
+        return;
+      }
+      downloadStream(
+        result,
+        `result.${getContentTypeFileExtension(contentType)}`,
+        exportData.contentType,
+      )
+        .then(() => {
+          this.exportState.pass();
+        })
+        .catch((error) => {
+          assertErrorThrown(error);
+        });
+    } catch (error) {
+      this.exportState.fail();
+      assertErrorThrown(error);
+      this.editorStore.applicationStore.notificationService.notifyError(error);
+      this.exportState.complete();
+    }
+  }
+
+  handleExport(format: string) {
+    const queryLambda = this.bodyExpressionSequence;
+    const parameters = (queryLambda.parameters ?? []) as object[];
+    if (parameters.length) {
+      this.parametersState.parameterStates =
+        this.parametersState.build(queryLambda);
+      this.parametersState.parameterValuesEditorState.open(
+        (): Promise<void> =>
+          flowResult(this.exportData(format)).catch(
+            this.editorStore.applicationStore.alertUnhandledError,
+          ),
+        PARAMETER_SUBMIT_ACTION.EXPORT,
+      );
+    } else {
+      flowResult(this.exportData(format)).catch(
+        this.editorStore.applicationStore.alertUnhandledError,
+      );
+    }
+  }
+
   get hashCode(): string {
     return hashArray([
       DATA_QUALITY_HASH_STRUCTURE.DATA_QUALITY_RELATION_VALIDATION,

package/src/graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.ts

@@ -23,6 +23,7 @@ import {
   type RawLambda,
   type ParameterValue,
   type RawVariableExpression,
+  type EXECUTION_SERIALIZATION_FORMAT,
   PackageableElement,
   hashRawLambda,
 } from '@finos/legend-graph';
@@ -41,6 +42,7 @@ export interface DQExecuteInputOptions {
   validationName?: string | undefined;
   previewLimit?: number | undefined;
   runQuery?: boolean | undefined;
+  serializationFormat?: EXECUTION_SERIALIZATION_FORMAT | undefined;
 }
 
 export abstract class DataQualityExecutionContext implements Hashable {

package/src/graph-manager/protocol/pure/DSL_DataQuality_PureGraphManagerExtension.ts

@@ -38,6 +38,12 @@ export abstract class DSL_DataQuality_PureGraphManagerExtension extends Abstract
     options: DQExecuteInputOptions,
   ): Promise<ExecutionResult>;
 
+  abstract exportData(
+    graph: PureModel,
+    packagePath: string,
+    options: DQExecuteInputOptions,
+  ): Promise<Response>;
+
   abstract debugExecutionPlanGeneration(
     graph: PureModel,
     packagePath: string,