@finos/legend-extension-dsl-data-quality 2.1.34 → 2.1.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/lib/components/DSL_DataQuality_LegendStudioApplicationPlugin.d.ts.map +1 -1
  2. package/lib/components/DSL_DataQuality_LegendStudioApplicationPlugin.js +45 -13
  3. package/lib/components/DSL_DataQuality_LegendStudioApplicationPlugin.js.map +1 -1
  4. package/lib/components/DataQualityCustomSelector.d.ts +16 -1
  5. package/lib/components/DataQualityCustomSelector.d.ts.map +1 -1
  6. package/lib/components/DataQualityCustomSelector.js +29 -4
  7. package/lib/components/DataQualityCustomSelector.js.map +1 -1
  8. package/lib/components/DataQualityRelationComparisonEditor.d.ts +19 -0
  9. package/lib/components/DataQualityRelationComparisonEditor.d.ts.map +1 -0
  10. package/lib/components/DataQualityRelationComparisonEditor.js +239 -0
  11. package/lib/components/DataQualityRelationComparisonEditor.js.map +1 -0
  12. package/lib/components/DataQualityRelationGridResult.d.ts +3 -0
  13. package/lib/components/DataQualityRelationGridResult.d.ts.map +1 -1
  14. package/lib/components/DataQualityRelationGridResult.js +1 -1
  15. package/lib/components/DataQualityRelationGridResult.js.map +1 -1
  16. package/lib/components/DataQualityRelationLambdaGUIDataTypeHandlers.d.ts +0 -10
  17. package/lib/components/DataQualityRelationLambdaGUIDataTypeHandlers.d.ts.map +1 -1
  18. package/lib/components/DataQualityRelationLambdaGUIDataTypeHandlers.js +11 -24
  19. package/lib/components/DataQualityRelationLambdaGUIDataTypeHandlers.js.map +1 -1
  20. package/lib/components/DataQualityRelationLambdaGUIValidationEditor.d.ts.map +1 -1
  21. package/lib/components/DataQualityRelationLambdaGUIValidationEditor.js +6 -4
  22. package/lib/components/DataQualityRelationLambdaGUIValidationEditor.js.map +1 -1
  23. package/lib/components/states/DataQualityRelationComparisonConfigurationState.d.ts +111 -0
  24. package/lib/components/states/DataQualityRelationComparisonConfigurationState.d.ts.map +1 -0
  25. package/lib/components/states/DataQualityRelationComparisonConfigurationState.js +495 -0
  26. package/lib/components/states/DataQualityRelationComparisonConfigurationState.js.map +1 -0
  27. package/lib/data-quality-custom-selector.css +2 -2
  28. package/lib/data-quality-custom-selector.css.map +1 -1
  29. package/lib/graph/metamodel/DSL_DataQuality_HashUtils.d.ts +3 -1
  30. package/lib/graph/metamodel/DSL_DataQuality_HashUtils.d.ts.map +1 -1
  31. package/lib/graph/metamodel/DSL_DataQuality_HashUtils.js +2 -0
  32. package/lib/graph/metamodel/DSL_DataQuality_HashUtils.js.map +1 -1
  33. package/lib/graph/metamodel/DSL_DataQuality_PureGraphPlugin.d.ts.map +1 -1
  34. package/lib/graph/metamodel/DSL_DataQuality_PureGraphPlugin.js +2 -1
  35. package/lib/graph/metamodel/DSL_DataQuality_PureGraphPlugin.js.map +1 -1
  36. package/lib/graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.d.ts +35 -0
  37. package/lib/graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.d.ts.map +1 -1
  38. package/lib/graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js +37 -0
  39. package/lib/graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js.map +1 -1
  40. package/lib/graph-manager/DSL_DataQuality_GraphManagerHelper.d.ts +2 -1
  41. package/lib/graph-manager/DSL_DataQuality_GraphManagerHelper.d.ts.map +1 -1
  42. package/lib/graph-manager/DSL_DataQuality_GraphManagerHelper.js +2 -1
  43. package/lib/graph-manager/DSL_DataQuality_GraphManagerHelper.js.map +1 -1
  44. package/lib/graph-manager/DSL_DataQuality_GraphModifierHelper.d.ts +7 -1
  45. package/lib/graph-manager/DSL_DataQuality_GraphModifierHelper.d.ts.map +1 -1
  46. package/lib/graph-manager/DSL_DataQuality_GraphModifierHelper.js +18 -0
  47. package/lib/graph-manager/DSL_DataQuality_GraphModifierHelper.js.map +1 -1
  48. package/lib/graph-manager/DSL_DataQuality_PureGraphManagerPlugin.d.ts.map +1 -1
  49. package/lib/graph-manager/DSL_DataQuality_PureGraphManagerPlugin.js +5 -2
  50. package/lib/graph-manager/DSL_DataQuality_PureGraphManagerPlugin.js.map +1 -1
  51. package/lib/graph-manager/action/changeDetection/DSL_DataQuality_ObserverHelper.d.ts +3 -1
  52. package/lib/graph-manager/action/changeDetection/DSL_DataQuality_ObserverHelper.d.ts.map +1 -1
  53. package/lib/graph-manager/action/changeDetection/DSL_DataQuality_ObserverHelper.js +28 -1
  54. package/lib/graph-manager/action/changeDetection/DSL_DataQuality_ObserverHelper.js.map +1 -1
  55. package/lib/graph-manager/protocol/pure/DSL_DataQuality_PureGraphManagerExtension.d.ts +4 -1
  56. package/lib/graph-manager/protocol/pure/DSL_DataQuality_PureGraphManagerExtension.d.ts.map +1 -1
  57. package/lib/graph-manager/protocol/pure/DSL_DataQuality_PureGraphManagerExtension.js.map +1 -1
  58. package/lib/graph-manager/protocol/pure/DSL_DataQuality_PureProtocolProcessorPlugin.d.ts.map +1 -1
  59. package/lib/graph-manager/protocol/pure/DSL_DataQuality_PureProtocolProcessorPlugin.js +33 -5
  60. package/lib/graph-manager/protocol/pure/DSL_DataQuality_PureProtocolProcessorPlugin.js.map +1 -1
  61. package/lib/graph-manager/protocol/pure/v1/V1_DSL_Data_Quality_PureGraphManagerExtension.d.ts +26 -2
  62. package/lib/graph-manager/protocol/pure/v1/V1_DSL_Data_Quality_PureGraphManagerExtension.d.ts.map +1 -1
  63. package/lib/graph-manager/protocol/pure/v1/V1_DSL_Data_Quality_PureGraphManagerExtension.js +107 -3
  64. package/lib/graph-manager/protocol/pure/v1/V1_DSL_Data_Quality_PureGraphManagerExtension.js.map +1 -1
  65. package/lib/graph-manager/protocol/pure/v1/V1_DataQualityValidationConfiguration.d.ts +19 -0
  66. package/lib/graph-manager/protocol/pure/v1/V1_DataQualityValidationConfiguration.d.ts.map +1 -1
  67. package/lib/graph-manager/protocol/pure/v1/V1_DataQualityValidationConfiguration.js +37 -0
  68. package/lib/graph-manager/protocol/pure/v1/V1_DataQualityValidationConfiguration.js.map +1 -1
  69. package/lib/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationBuilderHelper.d.ts +2 -1
  70. package/lib/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationBuilderHelper.d.ts.map +1 -1
  71. package/lib/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationBuilderHelper.js +26 -3
  72. package/lib/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationBuilderHelper.js.map +1 -1
  73. package/lib/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationTransformer.d.ts +3 -2
  74. package/lib/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationTransformer.d.ts.map +1 -1
  75. package/lib/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationTransformer.js +29 -2
  76. package/lib/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationTransformer.js.map +1 -1
  77. package/lib/graph-manager/protocol/pure/v1/transformation/pureProtocol/V1_DSL_DataQuality_ProtocolHelper.d.ts +4 -1
  78. package/lib/graph-manager/protocol/pure/v1/transformation/pureProtocol/V1_DSL_DataQuality_ProtocolHelper.d.ts.map +1 -1
  79. package/lib/graph-manager/protocol/pure/v1/transformation/pureProtocol/V1_DSL_DataQuality_ProtocolHelper.js +37 -2
  80. package/lib/graph-manager/protocol/pure/v1/transformation/pureProtocol/V1_DSL_DataQuality_ProtocolHelper.js.map +1 -1
  81. package/lib/index.css +2 -2
  82. package/lib/index.css.map +1 -1
  83. package/lib/package.json +1 -1
  84. package/package.json +7 -7
  85. package/src/components/DSL_DataQuality_LegendStudioApplicationPlugin.tsx +64 -12
  86. package/src/components/DataQualityCustomSelector.tsx +111 -6
  87. package/src/components/DataQualityRelationComparisonEditor.tsx +795 -0
  88. package/src/components/DataQualityRelationGridResult.tsx +1 -1
  89. package/src/components/DataQualityRelationLambdaGUIDataTypeHandlers.tsx +9 -87
  90. package/src/components/DataQualityRelationLambdaGUIValidationEditor.tsx +16 -6
  91. package/src/components/states/DataQualityRelationComparisonConfigurationState.ts +747 -0
  92. package/src/graph/metamodel/DSL_DataQuality_HashUtils.ts +2 -0
  93. package/src/graph/metamodel/DSL_DataQuality_PureGraphPlugin.ts +2 -0
  94. package/src/graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.ts +66 -0
  95. package/src/graph-manager/DSL_DataQuality_GraphManagerHelper.ts +13 -0
  96. package/src/graph-manager/DSL_DataQuality_GraphModifierHelper.ts +57 -0
  97. package/src/graph-manager/DSL_DataQuality_PureGraphManagerPlugin.ts +8 -0
  98. package/src/graph-manager/action/changeDetection/DSL_DataQuality_ObserverHelper.ts +42 -0
  99. package/src/graph-manager/protocol/pure/DSL_DataQuality_PureGraphManagerExtension.ts +16 -0
  100. package/src/graph-manager/protocol/pure/DSL_DataQuality_PureProtocolProcessorPlugin.ts +65 -0
  101. package/src/graph-manager/protocol/pure/v1/V1_DSL_Data_Quality_PureGraphManagerExtension.ts +171 -1
  102. package/src/graph-manager/protocol/pure/v1/V1_DataQualityValidationConfiguration.ts +49 -0
  103. package/src/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationBuilderHelper.ts +39 -0
  104. package/src/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationTransformer.ts +50 -0
  105. package/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/V1_DSL_DataQuality_ProtocolHelper.ts +71 -0
  106. package/style/_data-quality-relation-comparison-editor.scss +361 -0
  107. package/style/data-quality-custom-selector.scss +23 -0
  108. package/style/index.scss +74 -0
  109. package/tsconfig.json +2 -0
@@ -30,4 +30,6 @@ export enum DATA_QUALITY_HASH_STRUCTURE {
30
30
  DATA_QUALITY_RELATION_VALIDATION = 'DATA_QUALITY_RELATION_VALIDATION',
31
31
  DATA_QUALITY_RELATION_FUNCTION_DEFINITION = 'DATA_QUALITY_RELATION_FUNCTION_DEFINITION',
32
32
  DATA_QUALITY_RELATION_VALIDATION_QUERY = 'DATA_QUALITY_RELATION_VALIDATION_QUERY',
33
+ DATA_QUALITY_RELATION_COMPARISON_HASH_STRUCTURE = 'DATA_QUALITY_RELATION_COMPARISON',
34
+ DATA_QUALITY_MD5_HASH_STRATEGY = 'DATA_QUALITY_MD5_HASH_STRATEGY',
33
35
  }
@@ -21,6 +21,7 @@ import {
21
21
  DataQualityClassValidationsConfiguration,
22
22
  DataQualityServiceValidationConfiguration,
23
23
  DataQualityRelationValidationConfiguration,
24
+ DataQualityRelationComparisonConfiguration,
24
25
  } from './pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
25
26
 
26
27
  export class DSL_DataQuality_PureGraphPlugin extends PureGraphPlugin {
@@ -33,6 +34,7 @@ export class DSL_DataQuality_PureGraphPlugin extends PureGraphPlugin {
33
34
  DataQualityClassValidationsConfiguration,
34
35
  DataQualityServiceValidationConfiguration,
35
36
  DataQualityRelationValidationConfiguration,
37
+ DataQualityRelationComparisonConfiguration,
36
38
  ];
37
39
  }
38
40
  }
@@ -51,6 +51,23 @@ export interface DQValidationSuggestionInputOptions {
51
51
  lambdaParameterValues?: ParameterValue[];
52
52
  }
53
53
 
54
+ export interface DQReconciliationInputOptions {
55
+ clientVersion?: string | undefined;
56
+ source: RawLambda;
57
+ target: RawLambda;
58
+ keys: string[];
59
+ colsForHash: string[];
60
+ limit?: number | undefined;
61
+ aggregatedHash?: boolean | undefined;
62
+ sourceHashCol?: string | undefined;
63
+ targetHashCol?: string | undefined;
64
+ includeColumnValues?: boolean | undefined;
65
+ runSourceQuery?: boolean | undefined;
66
+ runTargetQuery?: boolean | undefined;
67
+ sourceLambdaParameterValues?: ParameterValue[];
68
+ targetLambdaParameterValues?: ParameterValue[];
69
+ }
70
+
54
71
  export abstract class DataQualityExecutionContext implements Hashable {
55
72
  abstract get hashCode(): string;
56
73
  }
@@ -189,3 +206,52 @@ export class DataQualityRelationValidationConfiguration
189
206
  return visitor.visit_PackageableElement(this);
190
207
  }
191
208
  }
209
+
210
+ export abstract class ReconStrategy implements Hashable {
211
+ abstract get hashCode(): string;
212
+ }
213
+
214
+ export class MD5HashStrategy extends ReconStrategy {
215
+ sourceHashColumn?: string | undefined;
216
+ targetHashColumn?: string | undefined;
217
+ aggregatedHash?: boolean | undefined;
218
+
219
+ override get hashCode(): string {
220
+ return hashArray([
221
+ DATA_QUALITY_HASH_STRUCTURE.DATA_QUALITY_MD5_HASH_STRATEGY,
222
+ this.sourceHashColumn ?? '',
223
+ this.targetHashColumn ?? '',
224
+ String(this.aggregatedHash ?? ''),
225
+ ]);
226
+ }
227
+ }
228
+
229
+ export class DataQualityRelationComparisonConfiguration
230
+ extends PackageableElement
231
+ implements Hashable
232
+ {
233
+ source!: DataQualityRelationQueryLambda;
234
+ target!: DataQualityRelationQueryLambda;
235
+ keys: string[] = [];
236
+ columnsToCompare: string[] = [];
237
+ strategy!: ReconStrategy;
238
+ expectedMatch?: number | undefined;
239
+
240
+ protected override get _elementHashCode(): string {
241
+ return hashArray([
242
+ DATA_QUALITY_HASH_STRUCTURE.DATA_QUALITY_RELATION_COMPARISON_HASH_STRUCTURE,
243
+ this.source,
244
+ this.target,
245
+ hashArray(this.keys),
246
+ hashArray(this.columnsToCompare),
247
+ this.expectedMatch ?? '',
248
+ this.strategy,
249
+ ]);
250
+ }
251
+
252
+ accept_PackageableElementVisitor<T>(
253
+ visitor: PackageableElementVisitor<T>,
254
+ ): T {
255
+ return visitor.visit_PackageableElement(this);
256
+ }
257
+ }
@@ -20,6 +20,7 @@ import {
20
20
  DataQualityClassValidationsConfiguration,
21
21
  DataQualityServiceValidationConfiguration,
22
22
  DataQualityRelationValidationConfiguration,
23
+ DataQualityRelationComparisonConfiguration,
23
24
  } from '../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
24
25
 
25
26
  export const getOwnDataQualityClassValidationsConfiguration = (
@@ -57,3 +58,15 @@ export const getOwnDataQualityRelationValidationsConfiguration = (
57
58
  ),
58
59
  `Can't find data quality tds validation element on '${path}'`,
59
60
  );
61
+
62
+ export const getOwnDataQualityRelationComparisonConfiguration = (
63
+ path: string,
64
+ graph: BasicModel,
65
+ ): DataQualityRelationComparisonConfiguration =>
66
+ guaranteeNonNullable(
67
+ graph.getOwnNullableExtensionElement(
68
+ path,
69
+ DataQualityRelationComparisonConfiguration,
70
+ ),
71
+ `Can't find data quality relation comparison element on '${path}'`,
72
+ );
@@ -20,6 +20,9 @@ import {
20
20
  type DataQualityExecutionContext,
21
21
  type DataQualityRelationValidation,
22
22
  type DataQualityRelationValidationConfiguration,
23
+ type DataQualityRelationComparisonConfiguration,
24
+ type DataQualityRelationQueryLambda,
25
+ type ReconStrategy,
23
26
  type RelationValidationType,
24
27
  DataSpaceDataQualityExecutionContext,
25
28
  MappingAndRuntimeDataQualityExecutionContext,
@@ -209,3 +212,57 @@ export const dataQualityRelationValidation_swapParameters = action(
209
212
  );
210
213
  },
211
214
  );
215
+
216
+ export const dataQualityRelationComparison_setSource = action(
217
+ (
218
+ element: DataQualityRelationComparisonConfiguration,
219
+ val: DataQualityRelationQueryLambda,
220
+ ): void => {
221
+ element.source = val;
222
+ },
223
+ );
224
+
225
+ export const dataQualityRelationComparison_setTarget = action(
226
+ (
227
+ element: DataQualityRelationComparisonConfiguration,
228
+ val: DataQualityRelationQueryLambda,
229
+ ): void => {
230
+ element.target = val;
231
+ },
232
+ );
233
+
234
+ export const dataQualityRelationComparison_setKeys = action(
235
+ (
236
+ element: DataQualityRelationComparisonConfiguration,
237
+ val: string[],
238
+ ): void => {
239
+ element.keys = val;
240
+ },
241
+ );
242
+
243
+ export const dataQualityRelationComparison_setColumnsToCompare = action(
244
+ (
245
+ element: DataQualityRelationComparisonConfiguration,
246
+ val: string[],
247
+ ): void => {
248
+ element.columnsToCompare = val;
249
+ },
250
+ );
251
+
252
+ export const dataQualityRelationComparison_setStrategy = action(
253
+ (
254
+ element: DataQualityRelationComparisonConfiguration,
255
+ val: ReconStrategy,
256
+ ): void => {
257
+ element.strategy = val;
258
+ },
259
+ );
260
+
261
+ export const dataQualityRelationComparison_setExpectedMatch = action(
262
+ (
263
+ element: DataQualityRelationComparisonConfiguration,
264
+ val: number | undefined,
265
+ ): void => {
266
+ element.expectedMatch = val;
267
+ },
268
+ );
@@ -25,11 +25,13 @@ import {
25
25
  DataQualityClassValidationsConfiguration,
26
26
  DataQualityServiceValidationConfiguration,
27
27
  DataQualityRelationValidationConfiguration,
28
+ DataQualityRelationComparisonConfiguration,
28
29
  } from '../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
29
30
  import {
30
31
  observe_DataQualityConstraintsConfiguration,
31
32
  observe_DataQualityServiceValidationConfiguration,
32
33
  observe_DataQualityRelationValidationConfiguration,
34
+ observe_DataQualityRelationComparisonConfiguration,
33
35
  } from './action/changeDetection/DSL_DataQuality_ObserverHelper.js';
34
36
  import { DSL_DataQuality_buildGraphManagerExtension } from './protocol/pure/DSL_DataQuality_buildGraphManagerExtension.js';
35
37
 
@@ -56,6 +58,12 @@ export class DSL_DataQuality_PureGraphManagerPlugin extends PureGraphManagerPlug
56
58
  context,
57
59
  );
58
60
  }
61
+ if (element instanceof DataQualityRelationComparisonConfiguration) {
62
+ return observe_DataQualityRelationComparisonConfiguration(
63
+ element,
64
+ context,
65
+ );
66
+ }
59
67
  return undefined;
60
68
  },
61
69
  ];
@@ -31,6 +31,9 @@ import {
31
31
  type DataQualityRelationValidation,
32
32
  type DataQualityRelationValidationConfiguration,
33
33
  type DataQualityRelationQueryLambda,
34
+ type DataQualityRelationComparisonConfiguration,
35
+ type ReconStrategy,
36
+ MD5HashStrategy,
34
37
  DataSpaceDataQualityExecutionContext,
35
38
  MappingAndRuntimeDataQualityExecutionContext,
36
39
  } from '../../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
@@ -203,3 +206,42 @@ export const observe_DataQualityRelationValidationConfiguration =
203
206
  return metamodel;
204
207
  },
205
208
  );
209
+
210
+ export const observe_ReconStrategy = skipObserved(
211
+ (metamodel: ReconStrategy): ReconStrategy => {
212
+ if (metamodel instanceof MD5HashStrategy) {
213
+ return makeObservable(metamodel, {
214
+ sourceHashColumn: observable,
215
+ targetHashColumn: observable,
216
+ aggregatedHash: observable,
217
+ hashCode: computed,
218
+ });
219
+ }
220
+ return metamodel;
221
+ },
222
+ );
223
+
224
+ export const observe_DataQualityRelationComparisonConfiguration =
225
+ skipObservedWithContext(
226
+ (
227
+ metamodel: DataQualityRelationComparisonConfiguration,
228
+ ): DataQualityRelationComparisonConfiguration => {
229
+ observe_Abstract_PackageableElement(metamodel);
230
+ makeObservable<
231
+ DataQualityRelationComparisonConfiguration,
232
+ '_elementHashCode'
233
+ >(metamodel, {
234
+ _elementHashCode: override,
235
+ source: observable,
236
+ target: observable,
237
+ keys: observable,
238
+ columnsToCompare: observable,
239
+ strategy: observable,
240
+ expectedMatch: observable,
241
+ });
242
+ observe_DataQualityRelationQueryLambda(metamodel.source);
243
+ observe_DataQualityRelationQueryLambda(metamodel.target);
244
+ observe_ReconStrategy(metamodel.strategy);
245
+ return metamodel;
246
+ },
247
+ );
@@ -26,6 +26,7 @@ import { guaranteeNonNullable } from '@finos/legend-shared';
26
26
  import type {
27
27
  DataQualityRelationValidation,
28
28
  DQExecuteInputOptions,
29
+ DQReconciliationInputOptions,
29
30
  } from '../../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
30
31
 
31
32
  export abstract class DSL_DataQuality_PureGraphManagerExtension extends AbstractPureGraphManagerExtension {
@@ -76,6 +77,21 @@ export abstract class DSL_DataQuality_PureGraphManagerExtension extends Abstract
76
77
  packagePath: string,
77
78
  options: DQExecuteInputOptions,
78
79
  ): Promise<DataQualityRelationValidation>;
80
+
81
+ abstract runReconciliation(
82
+ graph: PureModel,
83
+ options: DQReconciliationInputOptions,
84
+ ): Promise<ExecutionResult>;
85
+
86
+ abstract runReconciliationSourceQuery(
87
+ graph: PureModel,
88
+ options: DQReconciliationInputOptions,
89
+ ): Promise<ExecutionResult>;
90
+
91
+ abstract runReconciliationTargetQuery(
92
+ graph: PureModel,
93
+ options: DQReconciliationInputOptions,
94
+ ): Promise<ExecutionResult>;
79
95
  }
80
96
 
81
97
  export const getDataQualityPureGraphManagerExtension = (
@@ -21,31 +21,38 @@ import {
21
21
  V1_DataQualityServiceValidationsConfiguration,
22
22
  V1_DataQualityRelationValidationsConfiguration,
23
23
  V1_DataQualityValidationsConfiguration,
24
+ V1_DataQualityRelationComparisonConfiguration,
24
25
  } from './v1/V1_DataQualityValidationConfiguration.js';
25
26
  import {
26
27
  DataQualityClassValidationsConfiguration,
27
28
  DataQualityServiceValidationConfiguration,
28
29
  DataQualityRelationValidationConfiguration,
30
+ DataQualityRelationComparisonConfiguration,
29
31
  } from '../../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
30
32
  import {
31
33
  V1_DATA_QUALITY_PROTOCOL_TYPE,
32
34
  V1_DATA_QUALITY_SERVICE_PROTOCOL_TYPE,
33
35
  V1_DATA_QUALITY_RELATION_PROTOCOL_TYPE,
36
+ V1_DATA_QUALITY_RELATION_COMPARISON_PROTOCOL_TYPE,
34
37
  V1_deserializeDataQualityClassValidation,
35
38
  V1_deserializeDataQualityServiceValidation,
36
39
  V1_deserializeDataQualityRelationValidation,
37
40
  V1_serializeDataQualityClassValidation,
38
41
  V1_serializeDataQualityServiceValidation,
39
42
  V1_serializeDataQualityRelationValidation,
43
+ V1_serializeDataQualityRelationComparison,
44
+ V1_deserializeDataQualityRelationComparison,
40
45
  } from './v1/transformation/pureProtocol/V1_DSL_DataQuality_ProtocolHelper.js';
41
46
  import {
42
47
  V1_transformDataQualityClassValidationConfiguration,
43
48
  V1_transformDataQualityRelationValidationConfiguration,
49
+ V1_transformDataQualityRelationComparisonConfiguration,
44
50
  } from './v1/transformation/V1_DSL_DataQuality_ValueSpecificationTransformer.js';
45
51
  import {
46
52
  V1_buildDataQualityClassValidationConfiguration,
47
53
  V1_buildDataQualityRelationValidationConfiguration,
48
54
  V1_buildDataQualityServiceValidationConfiguration,
55
+ V1_buildDataQualityRelationComparisonConfiguration,
49
56
  } from './v1/transformation/V1_DSL_DataQuality_ValueSpecificationBuilderHelper.js';
50
57
  import { V1_DataQualityRootGraphFetchTree } from './v1/model/graphFetch/V1_DataQualityRootGraphFetchTree.js';
51
58
  import {
@@ -81,6 +88,8 @@ import { V1_buildValidationFunctionExpression } from './v1/V1_DataQualityValidat
81
88
  const DATA_QUALITY_CLASSIFIER_PATH = 'meta::external::dataquality::DataQuality';
82
89
  const DATA_QUALITY_RELATION_VALIDATION_CLASSIFIER_PATH =
83
90
  'meta::external::dataquality::DataQualityRelationValidation';
91
+ const DATA_QUALITY_RELATION_COMPARISON_CLASSIFIER_PATH =
92
+ 'meta::external::dataquality::DataQualityRelationComparison';
84
93
 
85
94
  export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolProcessorPlugin {
86
95
  constructor() {
@@ -210,6 +219,46 @@ export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolPro
210
219
  );
211
220
  },
212
221
  }),
222
+
223
+ new V1_ElementBuilder<V1_DataQualityRelationComparisonConfiguration>({
224
+ elementClassName: V1_DATA_QUALITY_RELATION_COMPARISON_PROTOCOL_TYPE,
225
+ _class: V1_DataQualityRelationComparisonConfiguration,
226
+ firstPass: (
227
+ elementProtocol: V1_PackageableElement,
228
+ context: V1_GraphBuilderContext,
229
+ ): PackageableElement => {
230
+ assertType(
231
+ elementProtocol,
232
+ V1_DataQualityRelationComparisonConfiguration,
233
+ );
234
+ const element = new DataQualityRelationComparisonConfiguration(
235
+ elementProtocol.name,
236
+ );
237
+ const path = V1_buildFullPath(
238
+ elementProtocol.package,
239
+ elementProtocol.name,
240
+ );
241
+ context.currentSubGraph.setOwnElementInExtension(
242
+ path,
243
+ element,
244
+ DataQualityRelationComparisonConfiguration,
245
+ );
246
+ return element;
247
+ },
248
+ secondPass: (
249
+ elementProtocol: V1_PackageableElement,
250
+ context: V1_GraphBuilderContext,
251
+ ): void => {
252
+ assertType(
253
+ elementProtocol,
254
+ V1_DataQualityRelationComparisonConfiguration,
255
+ );
256
+ V1_buildDataQualityRelationComparisonConfiguration(
257
+ elementProtocol,
258
+ context,
259
+ );
260
+ },
261
+ }),
213
262
  ];
214
263
  }
215
264
 
@@ -220,6 +269,10 @@ export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolPro
220
269
  protocol instanceof V1_DataQualityRelationValidationsConfiguration
221
270
  ) {
222
271
  return DATA_QUALITY_RELATION_VALIDATION_CLASSIFIER_PATH;
272
+ } else if (
273
+ protocol instanceof V1_DataQualityRelationComparisonConfiguration
274
+ ) {
275
+ return DATA_QUALITY_RELATION_COMPARISON_CLASSIFIER_PATH;
223
276
  } else if (protocol instanceof V1_DataQualityValidationsConfiguration) {
224
277
  return DATA_QUALITY_CLASSIFIER_PATH;
225
278
  }
@@ -245,6 +298,9 @@ export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolPro
245
298
  ) {
246
299
  return V1_serializeDataQualityRelationValidation(protocol, plugins);
247
300
  }
301
+ if (protocol instanceof V1_DataQualityRelationComparisonConfiguration) {
302
+ return V1_serializeDataQualityRelationComparison(protocol, plugins);
303
+ }
248
304
  return undefined;
249
305
  },
250
306
  ];
@@ -304,6 +360,9 @@ export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolPro
304
360
  if (json._type === V1_DATA_QUALITY_RELATION_PROTOCOL_TYPE) {
305
361
  return V1_deserializeDataQualityRelationValidation(json, plugins);
306
362
  }
363
+ if (json._type === V1_DATA_QUALITY_RELATION_COMPARISON_PROTOCOL_TYPE) {
364
+ return V1_deserializeDataQualityRelationComparison(json, plugins);
365
+ }
307
366
  return undefined;
308
367
  },
309
368
  ];
@@ -327,6 +386,12 @@ export class DSL_DataQuality_PureProtocolProcessorPlugin extends PureProtocolPro
327
386
  context,
328
387
  );
329
388
  }
389
+ if (metamodel instanceof DataQualityRelationComparisonConfiguration) {
390
+ return V1_transformDataQualityRelationComparisonConfiguration(
391
+ metamodel,
392
+ context,
393
+ );
394
+ }
330
395
  return undefined;
331
396
  },
332
397
  ];
@@ -27,6 +27,8 @@ import {
27
27
  type V1_ParameterValue,
28
28
  type V1_PureModelContext,
29
29
  type V1_RootGraphFetchTree,
30
+ type RawLambda,
31
+ type V1_RawLambda,
30
32
  V1_getEngineSerializationFormat,
31
33
  LegendSDLC,
32
34
  PureClientVersion,
@@ -34,6 +36,7 @@ import {
34
36
  V1_buildExecutionResult,
35
37
  V1_ExecutionError,
36
38
  V1_GraphBuilderContextBuilder,
39
+ V1_GraphTransformerContextBuilder,
37
40
  V1_LegendSDLC,
38
41
  V1_ProcessingContext,
39
42
  V1_Protocol,
@@ -43,9 +46,11 @@ import {
43
46
  V1_deserializeExecutionResult,
44
47
  V1_parameterValueModelSchema,
45
48
  V1_transformParameterValue,
49
+ V1_transformRawLambda,
46
50
  V1_RemoteEngine,
51
+ V1_rawLambdaModelSchema,
47
52
  } from '@finos/legend-graph';
48
- import { createModelSchema, optional, primitive } from 'serializr';
53
+ import { createModelSchema, list, optional, primitive } from 'serializr';
49
54
  import {
50
55
  type PlainObject,
51
56
  assertErrorThrown,
@@ -56,6 +61,7 @@ import {
56
61
  returnUndefOnError,
57
62
  SerializationFactory,
58
63
  UnsupportedOperationError,
64
+ usingModelSchema,
59
65
  } from '@finos/legend-shared';
60
66
  import { DSL_DataQuality_PureGraphManagerExtension } from '../DSL_DataQuality_PureGraphManagerExtension.js';
61
67
  import {
@@ -66,6 +72,7 @@ import type { DataQualityRootGraphFetchTree } from '../../../../graph/metamodel/
66
72
  import type {
67
73
  DataQualityRelationValidation,
68
74
  DQExecuteInputOptions,
75
+ DQReconciliationInputOptions,
69
76
  DQValidationSuggestionInputOptions,
70
77
  } from '../../../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
71
78
 
@@ -75,6 +82,7 @@ const DQ_EXECUTE_DATA_PROFILING = 'execute data profiling';
75
82
  const DQ_FETCH_RULE_SUGGESTIONS = 'fetch rule suggestions';
76
83
  const DQ_DEBUG_EXECUTION_PLAN = 'debug execution plan';
77
84
  const DQ_FETCH_PROPERTY_PATH_TREE = 'dq fetch property path tree';
85
+ const DQ_EXECUTE_RECONCILIATION = 'execute reconciliation';
78
86
 
79
87
  export class V1_DQExecuteInput {
80
88
  clientVersion: string | undefined;
@@ -115,6 +123,48 @@ export class V1_DQRuleSuggestionInput {
115
123
  );
116
124
  }
117
125
 
126
+ export class V1_DQReconciliationInput {
127
+ clientVersion: string | undefined;
128
+ model!: V1_PureModelContext;
129
+ source!: V1_RawLambda;
130
+ target!: V1_RawLambda;
131
+ keys: string[] = [];
132
+ colsForHash: string[] = [];
133
+ defectLimit: number | undefined;
134
+ aggregatedHash: boolean | undefined;
135
+ sourceHashCol: string | undefined;
136
+ targetHashCol: string | undefined;
137
+ includeColumnValues: boolean | undefined;
138
+ runSourceQuery: boolean | undefined;
139
+ runTargetQuery: boolean | undefined;
140
+ sourceLambdaParameterValues: V1_ParameterValue[] = [];
141
+ targetLambdaParameterValues: V1_ParameterValue[] = [];
142
+
143
+ static readonly serialization = new SerializationFactory(
144
+ createModelSchema(V1_DQReconciliationInput, {
145
+ clientVersion: optional(primitive()),
146
+ model: V1_pureModelContextPropSchema,
147
+ source: usingModelSchema(V1_rawLambdaModelSchema),
148
+ target: usingModelSchema(V1_rawLambdaModelSchema),
149
+ keys: list(primitive()),
150
+ colsForHash: list(primitive()),
151
+ defectLimit: optional(primitive()),
152
+ aggregatedHash: optional(primitive()),
153
+ sourceHashCol: optional(primitive()),
154
+ targetHashCol: optional(primitive()),
155
+ includeColumnValues: optional(primitive()),
156
+ runSourceQuery: optional(primitive()),
157
+ runTargetQuery: optional(primitive()),
158
+ sourceLambdaParameterValues: customListWithSchema(
159
+ V1_parameterValueModelSchema,
160
+ ),
161
+ targetLambdaParameterValues: customListWithSchema(
162
+ V1_parameterValueModelSchema,
163
+ ),
164
+ }),
165
+ );
166
+ }
167
+
118
168
  export class V1_DSL_Data_Quality_PureGraphManagerExtension extends DSL_DataQuality_PureGraphManagerExtension {
119
169
  declare graphManager: V1_PureGraphManager;
120
170
  static readonly DEV_PROTOCOL_VERSION = PureClientVersion.VX_X_X;
@@ -560,4 +610,124 @@ export class V1_DSL_Data_Quality_PureGraphManagerExtension extends DSL_DataQuali
560
610
  {},
561
611
  );
562
612
  };
613
+
614
+ private rawLambdaToV1(lambda: RawLambda): V1_RawLambda {
615
+ return V1_transformRawLambda(
616
+ lambda,
617
+ new V1_GraphTransformerContextBuilder(
618
+ this.graphManager.pluginManager.getPureProtocolProcessorPlugins(),
619
+ ).build(),
620
+ );
621
+ }
622
+
623
+ private createReconciliationInput(
624
+ graph: PureModel,
625
+ options: DQReconciliationInputOptions,
626
+ ): V1_DQReconciliationInput {
627
+ const input = new V1_DQReconciliationInput();
628
+ input.clientVersion =
629
+ options.clientVersion ??
630
+ V1_DSL_Data_Quality_PureGraphManagerExtension.DEV_PROTOCOL_VERSION;
631
+ input.model = graph.origin
632
+ ? this.buildPureModelSDLCPointer(graph.origin, undefined)
633
+ : this.graphManager.getFullGraphModelData(graph);
634
+ input.source = this.rawLambdaToV1(options.source);
635
+ input.target = this.rawLambdaToV1(options.target);
636
+ input.keys = options.keys;
637
+ input.colsForHash = options.colsForHash;
638
+ input.defectLimit = options.limit;
639
+ input.aggregatedHash = options.aggregatedHash;
640
+ input.sourceHashCol = options.sourceHashCol;
641
+ input.targetHashCol = options.targetHashCol;
642
+ input.includeColumnValues = options.includeColumnValues;
643
+ input.runSourceQuery = options.runSourceQuery;
644
+ input.runTargetQuery = options.runTargetQuery;
645
+ if (options.sourceLambdaParameterValues) {
646
+ input.sourceLambdaParameterValues =
647
+ options.sourceLambdaParameterValues.map(V1_transformParameterValue);
648
+ }
649
+ if (options.targetLambdaParameterValues) {
650
+ input.targetLambdaParameterValues =
651
+ options.targetLambdaParameterValues.map(V1_transformParameterValue);
652
+ }
653
+ return input;
654
+ }
655
+
656
+ runReconciliation = async (
657
+ graph: PureModel,
658
+ options: DQReconciliationInputOptions,
659
+ ): Promise<ExecutionResult> => {
660
+ const input = this.createReconciliationInput(graph, options);
661
+
662
+ return this.runReconciliationWithInput(input);
663
+ };
664
+
665
+ runReconciliationSourceQuery = async (
666
+ graph: PureModel,
667
+ options: DQReconciliationInputOptions,
668
+ ): Promise<ExecutionResult> => {
669
+ const input = this.createReconciliationInput(graph, {
670
+ ...options,
671
+ runSourceQuery: true,
672
+ runTargetQuery: undefined,
673
+ });
674
+ return this.runReconciliationWithInput(input);
675
+ };
676
+
677
+ runReconciliationTargetQuery = async (
678
+ graph: PureModel,
679
+ options: DQReconciliationInputOptions,
680
+ ): Promise<ExecutionResult> => {
681
+ const input = this.createReconciliationInput(graph, {
682
+ ...options,
683
+ runSourceQuery: undefined,
684
+ runTargetQuery: true,
685
+ });
686
+ return this.runReconciliationWithInput(input);
687
+ };
688
+
689
+ private async runReconciliationWithInput(
690
+ input: V1_DQReconciliationInput,
691
+ ): Promise<ExecutionResult> {
692
+ try {
693
+ const engineServerClient = guaranteeType(
694
+ this.graphManager.engine,
695
+ V1_RemoteEngine,
696
+ 'runReconciliation is only supported by remote engine',
697
+ ).getEngineServerClient();
698
+
699
+ const result = await engineServerClient.postWithTracing(
700
+ engineServerClient.getTraceData(DQ_EXECUTE_RECONCILIATION),
701
+ `${engineServerClient._pure()}/dataquality/reconciliation`,
702
+ V1_DQReconciliationInput.serialization.toJson(input),
703
+ {},
704
+ undefined,
705
+ undefined,
706
+ { enableCompression: true },
707
+ { skipProcessing: true },
708
+ );
709
+
710
+ const resultInText = await (result as Response).text();
711
+ const rawExecutionResult =
712
+ returnUndefOnError(() =>
713
+ this.graphManager.engine.parseExecutionResults(
714
+ resultInText,
715
+ undefined,
716
+ ),
717
+ ) ?? resultInText;
718
+ const v1_executionResult =
719
+ V1_deserializeExecutionResult(rawExecutionResult);
720
+ return V1_buildExecutionResult(v1_executionResult);
721
+ } catch (error) {
722
+ assertErrorThrown(error);
723
+ if (error instanceof NetworkClientError) {
724
+ throw V1_buildExecutionError(
725
+ V1_ExecutionError.serialization.fromJson(
726
+ error.payload as PlainObject<V1_ExecutionError>,
727
+ ),
728
+ );
729
+ }
730
+ throw error;
731
+ }
732
+ }
563
733
  }