@finos/legend-extension-dsl-data-quality 2.1.33 → 2.1.35

package/src/components/states/DataQualityState.ts

@@ -158,7 +158,7 @@ export abstract class DataQualityState extends ElementEditorState {
       setShowDateSelection: action,
       updateElementOnClassChange: action,
       checkConstraintsSelectedAtNode: action,
-      changeClass: action,
+      changeSourceElement: action,
       setProcessingDate: action,
       setBusinessDate: action,
       fetchStructuralValidations: flow,
@@ -220,7 +220,7 @@ export abstract class DataQualityState extends ElementEditorState {
   }
 
   get currentClassMilestoningStrategy(): MILESTONING_STEREOTYPE | undefined {
-    const currentclass = this.dataQualityQueryBuilderState.class;
+    const currentclass = this.dataQualityQueryBuilderState.sourceClass;
     if (currentclass !== undefined) {
       return getMilestoneTemporalStereotype(
         currentclass,
@@ -360,7 +360,7 @@ export abstract class DataQualityState extends ElementEditorState {
       return;
     }
     const genericType = new GenericType(
-      guaranteeNonNullable(this.dataQualityQueryBuilderState.class),
+      guaranteeNonNullable(this.dataQualityQueryBuilderState.sourceClass),
     );
     const genericTypeReference =
       GenericTypeExplicitReference.create(genericType);
@@ -409,14 +409,14 @@ export abstract class DataQualityState extends ElementEditorState {
       : 'data-quality-validation__setup__data-space';
   }
 
-  changeClass(val: Class): void {
-    this.dataQualityQueryBuilderState.changeClass(val);
+  changeSourceElement(val: Class): void {
+    this.dataQualityQueryBuilderState.changeSourceElement(val);
     this.structuralValidationsGraphFetchTreeState =
       new DataQualityGraphFetchTreeState(this);
     this.resultState = new DataQualityResultState(this);
     this.initializeGraphFetchTreeState(
       buildDefaultDataQualityRootGraphFetchTree(
-        guaranteeNonNullable(this.dataQualityQueryBuilderState.class),
+        guaranteeNonNullable(this.dataQualityQueryBuilderState.sourceClass),
       ),
     );
     this.updateElementOnClassChange();

package/src/graph/metamodel/DSL_DataQuality_HashUtils.ts

@@ -30,4 +30,6 @@ export enum DATA_QUALITY_HASH_STRUCTURE {
   DATA_QUALITY_RELATION_VALIDATION = 'DATA_QUALITY_RELATION_VALIDATION',
   DATA_QUALITY_RELATION_FUNCTION_DEFINITION = 'DATA_QUALITY_RELATION_FUNCTION_DEFINITION',
   DATA_QUALITY_RELATION_VALIDATION_QUERY = 'DATA_QUALITY_RELATION_VALIDATION_QUERY',
+  DATA_QUALITY_RELATION_COMPARISON_HASH_STRUCTURE = 'DATA_QUALITY_RELATION_COMPARISON',
+  DATA_QUALITY_MD5_HASH_STRATEGY = 'DATA_QUALITY_MD5_HASH_STRATEGY',
 }

package/src/graph/metamodel/DSL_DataQuality_PureGraphPlugin.ts

@@ -21,6 +21,7 @@ import {
   DataQualityClassValidationsConfiguration,
   DataQualityServiceValidationConfiguration,
   DataQualityRelationValidationConfiguration,
+  DataQualityRelationComparisonConfiguration,
 } from './pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
 
 export class DSL_DataQuality_PureGraphPlugin extends PureGraphPlugin {
@@ -33,6 +34,7 @@ export class DSL_DataQuality_PureGraphPlugin extends PureGraphPlugin {
       DataQualityClassValidationsConfiguration,
       DataQualityServiceValidationConfiguration,
       DataQualityRelationValidationConfiguration,
+      DataQualityRelationComparisonConfiguration,
     ];
   }
 }

package/src/graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.ts

@@ -51,6 +51,23 @@ export interface DQValidationSuggestionInputOptions {
   lambdaParameterValues?: ParameterValue[];
 }
 
+export interface DQReconciliationInputOptions {
+  clientVersion?: string | undefined;
+  source: RawLambda;
+  target: RawLambda;
+  keys: string[];
+  colsForHash: string[];
+  limit?: number | undefined;
+  aggregatedHash?: boolean | undefined;
+  sourceHashCol?: string | undefined;
+  targetHashCol?: string | undefined;
+  includeColumnValues?: boolean | undefined;
+  runSourceQuery?: boolean | undefined;
+  runTargetQuery?: boolean | undefined;
+  sourceLambdaParameterValues?: ParameterValue[];
+  targetLambdaParameterValues?: ParameterValue[];
+}
+
 export abstract class DataQualityExecutionContext implements Hashable {
   abstract get hashCode(): string;
 }
@@ -189,3 +206,52 @@ export class DataQualityRelationValidationConfiguration
     return visitor.visit_PackageableElement(this);
   }
 }
+
+export abstract class ReconStrategy implements Hashable {
+  abstract get hashCode(): string;
+}
+
+export class MD5HashStrategy extends ReconStrategy {
+  sourceHashColumn?: string | undefined;
+  targetHashColumn?: string | undefined;
+  aggregatedHash?: boolean | undefined;
+
+  override get hashCode(): string {
+    return hashArray([
+      DATA_QUALITY_HASH_STRUCTURE.DATA_QUALITY_MD5_HASH_STRATEGY,
+      this.sourceHashColumn ?? '',
+      this.targetHashColumn ?? '',
+      String(this.aggregatedHash ?? ''),
+    ]);
+  }
+}
+
+export class DataQualityRelationComparisonConfiguration
+  extends PackageableElement
+  implements Hashable
+{
+  source!: DataQualityRelationQueryLambda;
+  target!: DataQualityRelationQueryLambda;
+  keys: string[] = [];
+  columnsToCompare: string[] = [];
+  strategy!: ReconStrategy;
+  expectedMatch?: number | undefined;
+
+  protected override get _elementHashCode(): string {
+    return hashArray([
+      DATA_QUALITY_HASH_STRUCTURE.DATA_QUALITY_RELATION_COMPARISON_HASH_STRUCTURE,
+      this.source,
+      this.target,
+      hashArray(this.keys),
+      hashArray(this.columnsToCompare),
+      this.expectedMatch ?? '',
+      this.strategy,
+    ]);
+  }
+
+  accept_PackageableElementVisitor<T>(
+    visitor: PackageableElementVisitor<T>,
+  ): T {
+    return visitor.visit_PackageableElement(this);
+  }
+}

package/src/graph-manager/DSL_DataQuality_GraphManagerHelper.ts

@@ -20,6 +20,7 @@ import {
   DataQualityClassValidationsConfiguration,
   DataQualityServiceValidationConfiguration,
   DataQualityRelationValidationConfiguration,
+  DataQualityRelationComparisonConfiguration,
 } from '../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
 
 export const getOwnDataQualityClassValidationsConfiguration = (
@@ -57,3 +58,15 @@ export const getOwnDataQualityRelationValidationsConfiguration = (
     ),
     `Can't find data quality tds validation element on '${path}'`,
   );
+
+export const getOwnDataQualityRelationComparisonConfiguration = (
+  path: string,
+  graph: BasicModel,
+): DataQualityRelationComparisonConfiguration =>
+  guaranteeNonNullable(
+    graph.getOwnNullableExtensionElement(
+      path,
+      DataQualityRelationComparisonConfiguration,
+    ),
+    `Can't find data quality relation comparison element on '${path}'`,
+  );

package/src/graph-manager/DSL_DataQuality_GraphModifierHelper.ts

@@ -20,6 +20,9 @@ import {
   type DataQualityExecutionContext,
   type DataQualityRelationValidation,
   type DataQualityRelationValidationConfiguration,
+  type DataQualityRelationComparisonConfiguration,
+  type DataQualityRelationQueryLambda,
+  type ReconStrategy,
   type RelationValidationType,
   DataSpaceDataQualityExecutionContext,
   MappingAndRuntimeDataQualityExecutionContext,
@@ -209,3 +212,57 @@ export const dataQualityRelationValidation_swapParameters = action(
     );
   },
 );
+
+export const dataQualityRelationComparison_setSource = action(
+  (
+    element: DataQualityRelationComparisonConfiguration,
+    val: DataQualityRelationQueryLambda,
+  ): void => {
+    element.source = val;
+  },
+);
+
+export const dataQualityRelationComparison_setTarget = action(
+  (
+    element: DataQualityRelationComparisonConfiguration,
+    val: DataQualityRelationQueryLambda,
+  ): void => {
+    element.target = val;
+  },
+);
+
+export const dataQualityRelationComparison_setKeys = action(
+  (
+    element: DataQualityRelationComparisonConfiguration,
+    val: string[],
+  ): void => {
+    element.keys = val;
+  },
+);
+
+export const dataQualityRelationComparison_setColumnsToCompare = action(
+  (
+    element: DataQualityRelationComparisonConfiguration,
+    val: string[],
+  ): void => {
+    element.columnsToCompare = val;
+  },
+);
+
+export const dataQualityRelationComparison_setStrategy = action(
+  (
+    element: DataQualityRelationComparisonConfiguration,
+    val: ReconStrategy,
+  ): void => {
+    element.strategy = val;
+  },
+);
+
+export const dataQualityRelationComparison_setExpectedMatch = action(
+  (
+    element: DataQualityRelationComparisonConfiguration,
+    val: number | undefined,
+  ): void => {
+    element.expectedMatch = val;
+  },
+);

package/src/graph-manager/DSL_DataQuality_PureGraphManagerPlugin.ts

@@ -25,11 +25,13 @@ import {
   DataQualityClassValidationsConfiguration,
   DataQualityServiceValidationConfiguration,
   DataQualityRelationValidationConfiguration,
+  DataQualityRelationComparisonConfiguration,
 } from '../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
 import {
   observe_DataQualityConstraintsConfiguration,
   observe_DataQualityServiceValidationConfiguration,
   observe_DataQualityRelationValidationConfiguration,
+  observe_DataQualityRelationComparisonConfiguration,
 } from './action/changeDetection/DSL_DataQuality_ObserverHelper.js';
 import { DSL_DataQuality_buildGraphManagerExtension } from './protocol/pure/DSL_DataQuality_buildGraphManagerExtension.js';
 
@@ -56,6 +58,12 @@ export class DSL_DataQuality_PureGraphManagerPlugin extends PureGraphManagerPlug
             context,
           );
         }
+        if (element instanceof DataQualityRelationComparisonConfiguration) {
+          return observe_DataQualityRelationComparisonConfiguration(
+            element,
+            context,
+          );
+        }
         return undefined;
       },
     ];

package/src/graph-manager/action/changeDetection/DSL_DataQuality_ObserverHelper.ts

@@ -31,6 +31,9 @@ import {
   type DataQualityRelationValidation,
   type DataQualityRelationValidationConfiguration,
   type DataQualityRelationQueryLambda,
+  type DataQualityRelationComparisonConfiguration,
+  type ReconStrategy,
+  MD5HashStrategy,
   DataSpaceDataQualityExecutionContext,
   MappingAndRuntimeDataQualityExecutionContext,
 } from '../../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
@@ -203,3 +206,42 @@ export const observe_DataQualityRelationValidationConfiguration =
       return metamodel;
     },
   );
+
+export const observe_ReconStrategy = skipObserved(
+  (metamodel: ReconStrategy): ReconStrategy => {
+    if (metamodel instanceof MD5HashStrategy) {
+      return makeObservable(metamodel, {
+        sourceHashColumn: observable,
+        targetHashColumn: observable,
+        aggregatedHash: observable,
+        hashCode: computed,
+      });
+    }
+    return metamodel;
+  },
+);
+
+export const observe_DataQualityRelationComparisonConfiguration =
+  skipObservedWithContext(
+    (
+      metamodel: DataQualityRelationComparisonConfiguration,
+    ): DataQualityRelationComparisonConfiguration => {
+      observe_Abstract_PackageableElement(metamodel);
+      makeObservable<
+        DataQualityRelationComparisonConfiguration,
+        '_elementHashCode'
+      >(metamodel, {
+        _elementHashCode: override,
+        source: observable,
+        target: observable,
+        keys: observable,
+        columnsToCompare: observable,
+        strategy: observable,
+        expectedMatch: observable,
+      });
+      observe_DataQualityRelationQueryLambda(metamodel.source);
+      observe_DataQualityRelationQueryLambda(metamodel.target);
+      observe_ReconStrategy(metamodel.strategy);
+      return metamodel;
+    },
+  );

package/src/graph-manager/protocol/pure/DSL_DataQuality_PureGraphManagerExtension.ts

@@ -26,6 +26,7 @@ import { guaranteeNonNullable } from '@finos/legend-shared';
 import type {
   DataQualityRelationValidation,
   DQExecuteInputOptions,
+  DQReconciliationInputOptions,
 } from '../../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
 
 export abstract class DSL_DataQuality_PureGraphManagerExtension extends AbstractPureGraphManagerExtension {
@@ -76,6 +77,21 @@ export abstract class DSL_DataQuality_PureGraphManagerExtension extends Abstract
     packagePath: string,
     options: DQExecuteInputOptions,
   ): Promise<DataQualityRelationValidation>;
+
+  abstract runReconciliation(
+    graph: PureModel,
+    options: DQReconciliationInputOptions,
+  ): Promise<ExecutionResult>;
+
+  abstract runReconciliationSourceQuery(
+    graph: PureModel,
+    options: DQReconciliationInputOptions,
+  ): Promise<ExecutionResult>;
+
+  abstract runReconciliationTargetQuery(
+    graph: PureModel,
+    options: DQReconciliationInputOptions,
+  ): Promise<ExecutionResult>;
 }
 
 export const getDataQualityPureGraphManagerExtension = (

package/src/graph-manager/protocol/pure/DSL_DataQuality_PureProtocolProcessorPlugin.ts

@@ -21,31 +21,38 @@ import {
   V1_DataQualityServiceValidationsConfiguration,
   V1_DataQualityRelationValidationsConfiguration,
   V1_DataQualityValidationsConfiguration,
+  V1_DataQualityRelationComparisonConfiguration,
 } from './v1/V1_DataQualityValidationConfiguration.js';
 import {
   DataQualityClassValidationsConfiguration,
   DataQualityServiceValidationConfiguration,
   DataQualityRelationValidationConfiguration,
+  DataQualityRelationComparisonConfiguration,
 } from '../../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
 import {
   V1_DATA_QUALITY_PROTOCOL_TYPE,
   V1_DATA_QUALITY_SERVICE_PROTOCOL_TYPE,
   V1_DATA_QUALITY_RELATION_PROTOCOL_TYPE,
+  V1_DATA_QUALITY_RELATION_COMPARISON_PROTOCOL_TYPE,
   V1_deserializeDataQualityClassValidation,
   V1_deserializeDataQualityServiceValidation,
   V1_deserializeDataQualityRelationValidation,
   V1_serializeDataQualityClassValidation,
   V1_serializeDataQualityServiceValidation,
   V1_serializeDataQualityRelationValidation,
+  V1_serializeDataQualityRelationComparison,
+  V1_deserializeDataQualityRelationComparison,
 } from './v1/transformation/pureProtocol/V1_DSL_DataQuality_ProtocolHelper.js';
 import {
   V1_transformDataQualityClassValidationConfiguration,
   V1_transformDataQualityRelationValidationConfiguration,
+  V1_transformDataQualityRelationComparisonConfiguration,
 } from './v1/transformation/V1_DSL_DataQuality_ValueSpecificationTransformer.js';
 import {
   V1_buildDataQualityClassValidationConfiguration,
   V1_buildDataQualityRelationValidationConfiguration,
   V1_buildDataQualityServiceValidationConfiguration,
+  V1_buildDataQualityRelationComparisonConfiguration,
 } from './v1/transformation/V1_DSL_DataQuality_ValueSpecificationBuilderHelper.js';
 import { V1_DataQualityRootGraphFetchTree } from './v1/model/graphFetch/V1_DataQualityRootGraphFetchTree.js';
 import {
@@ -81,6 +88,8 @@ import { V1_buildValidationFunctionExpression } from './v1/V1_DataQualityValidat
 const DATA_QUALITY_CLASSIFIER_PATH = 'meta::external::dataquality::DataQuality';
 const DATA_QUALITY_RELATION_VALIDATION_CLASSIFIER_PATH =
   'meta::external::dataquality::DataQualityRelationValidation';
+const DATA_QUALITY_RELATION_COMPARISON_CLASSIFIER_PATH =
+  'meta::external::dataquality::DataQualityRelationComparison';
 
 export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolProcessorPlugin {
   constructor() {
@@ -210,6 +219,46 @@ export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolPro
           );
         },
       }),
+
+      new V1_ElementBuilder<V1_DataQualityRelationComparisonConfiguration>({
+        elementClassName: V1_DATA_QUALITY_RELATION_COMPARISON_PROTOCOL_TYPE,
+        _class: V1_DataQualityRelationComparisonConfiguration,
+        firstPass: (
+          elementProtocol: V1_PackageableElement,
+          context: V1_GraphBuilderContext,
+        ): PackageableElement => {
+          assertType(
+            elementProtocol,
+            V1_DataQualityRelationComparisonConfiguration,
+          );
+          const element = new DataQualityRelationComparisonConfiguration(
+            elementProtocol.name,
+          );
+          const path = V1_buildFullPath(
+            elementProtocol.package,
+            elementProtocol.name,
+          );
+          context.currentSubGraph.setOwnElementInExtension(
+            path,
+            element,
+            DataQualityRelationComparisonConfiguration,
+          );
+          return element;
+        },
+        secondPass: (
+          elementProtocol: V1_PackageableElement,
+          context: V1_GraphBuilderContext,
+        ): void => {
+          assertType(
+            elementProtocol,
+            V1_DataQualityRelationComparisonConfiguration,
+          );
+          V1_buildDataQualityRelationComparisonConfiguration(
+            elementProtocol,
+            context,
+          );
+        },
+      }),
     ];
   }
 
@@ -220,6 +269,10 @@ export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolPro
           protocol instanceof V1_DataQualityRelationValidationsConfiguration
         ) {
           return DATA_QUALITY_RELATION_VALIDATION_CLASSIFIER_PATH;
+        } else if (
+          protocol instanceof V1_DataQualityRelationComparisonConfiguration
+        ) {
+          return DATA_QUALITY_RELATION_COMPARISON_CLASSIFIER_PATH;
         } else if (protocol instanceof V1_DataQualityValidationsConfiguration) {
           return DATA_QUALITY_CLASSIFIER_PATH;
         }
@@ -245,6 +298,9 @@ export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolPro
         ) {
           return V1_serializeDataQualityRelationValidation(protocol, plugins);
         }
+        if (protocol instanceof V1_DataQualityRelationComparisonConfiguration) {
+          return V1_serializeDataQualityRelationComparison(protocol, plugins);
+        }
         return undefined;
       },
     ];
@@ -304,6 +360,9 @@ export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolPro
         if (json._type === V1_DATA_QUALITY_RELATION_PROTOCOL_TYPE) {
           return V1_deserializeDataQualityRelationValidation(json, plugins);
         }
+        if (json._type === V1_DATA_QUALITY_RELATION_COMPARISON_PROTOCOL_TYPE) {
+          return V1_deserializeDataQualityRelationComparison(json, plugins);
+        }
         return undefined;
       },
     ];
@@ -327,6 +386,12 @@ export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolPro
             context,
           );
         }
+        if (metamodel instanceof DataQualityRelationComparisonConfiguration) {
+          return V1_transformDataQualityRelationComparisonConfiguration(
+            metamodel,
+            context,
+          );
+        }
         return undefined;
       },
     ];