@finos/legend-extension-dsl-data-quality 2.1.33 → 2.1.35

package/src/graph-manager/protocol/pure/v1/V1_DSL_Data_Quality_PureGraphManagerExtension.ts

@@ -27,6 +27,8 @@ import {
   type V1_ParameterValue,
   type V1_PureModelContext,
   type V1_RootGraphFetchTree,
+  type RawLambda,
+  type V1_RawLambda,
   V1_getEngineSerializationFormat,
   LegendSDLC,
   PureClientVersion,
@@ -34,6 +36,7 @@ import {
   V1_buildExecutionResult,
   V1_ExecutionError,
   V1_GraphBuilderContextBuilder,
+  V1_GraphTransformerContextBuilder,
   V1_LegendSDLC,
   V1_ProcessingContext,
   V1_Protocol,
@@ -43,9 +46,11 @@ import {
   V1_deserializeExecutionResult,
   V1_parameterValueModelSchema,
   V1_transformParameterValue,
+  V1_transformRawLambda,
   V1_RemoteEngine,
+  V1_rawLambdaModelSchema,
 } from '@finos/legend-graph';
-import { createModelSchema, optional, primitive } from 'serializr';
+import { createModelSchema, list, optional, primitive } from 'serializr';
 import {
   type PlainObject,
   assertErrorThrown,
@@ -56,6 +61,7 @@ import {
   returnUndefOnError,
   SerializationFactory,
   UnsupportedOperationError,
+  usingModelSchema,
 } from '@finos/legend-shared';
 import { DSL_DataQuality_PureGraphManagerExtension } from '../DSL_DataQuality_PureGraphManagerExtension.js';
 import {
@@ -66,6 +72,7 @@ import type { DataQualityRootGraphFetchTree } from '../../../../graph/metamodel/
 import type {
   DataQualityRelationValidation,
   DQExecuteInputOptions,
+  DQReconciliationInputOptions,
   DQValidationSuggestionInputOptions,
 } from '../../../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
 
@@ -75,6 +82,7 @@ const DQ_EXECUTE_DATA_PROFILING = 'execute data profiling';
 const DQ_FETCH_RULE_SUGGESTIONS = 'fetch rule suggestions';
 const DQ_DEBUG_EXECUTION_PLAN = 'debug execution plan';
 const DQ_FETCH_PROPERTY_PATH_TREE = 'dq fetch property path tree';
+const DQ_EXECUTE_RECONCILIATION = 'execute reconciliation';
 
 export class V1_DQExecuteInput {
   clientVersion: string | undefined;
@@ -115,6 +123,48 @@ export class V1_DQRuleSuggestionInput {
   );
 }
 
+export class V1_DQReconciliationInput {
+  clientVersion: string | undefined;
+  model!: V1_PureModelContext;
+  source!: V1_RawLambda;
+  target!: V1_RawLambda;
+  keys: string[] = [];
+  colsForHash: string[] = [];
+  defectLimit: number | undefined;
+  aggregatedHash: boolean | undefined;
+  sourceHashCol: string | undefined;
+  targetHashCol: string | undefined;
+  includeColumnValues: boolean | undefined;
+  runSourceQuery: boolean | undefined;
+  runTargetQuery: boolean | undefined;
+  sourceLambdaParameterValues: V1_ParameterValue[] = [];
+  targetLambdaParameterValues: V1_ParameterValue[] = [];
+
+  static readonly serialization = new SerializationFactory(
+    createModelSchema(V1_DQReconciliationInput, {
+      clientVersion: optional(primitive()),
+      model: V1_pureModelContextPropSchema,
+      source: usingModelSchema(V1_rawLambdaModelSchema),
+      target: usingModelSchema(V1_rawLambdaModelSchema),
+      keys: list(primitive()),
+      colsForHash: list(primitive()),
+      defectLimit: optional(primitive()),
+      aggregatedHash: optional(primitive()),
+      sourceHashCol: optional(primitive()),
+      targetHashCol: optional(primitive()),
+      includeColumnValues: optional(primitive()),
+      runSourceQuery: optional(primitive()),
+      runTargetQuery: optional(primitive()),
+      sourceLambdaParameterValues: customListWithSchema(
+        V1_parameterValueModelSchema,
+      ),
+      targetLambdaParameterValues: customListWithSchema(
+        V1_parameterValueModelSchema,
+      ),
+    }),
+  );
+}
+
 export class V1_DSL_Data_Quality_PureGraphManagerExtension extends DSL_DataQuality_PureGraphManagerExtension {
   declare graphManager: V1_PureGraphManager;
   static readonly DEV_PROTOCOL_VERSION = PureClientVersion.VX_X_X;
@@ -560,4 +610,124 @@ export class V1_DSL_Data_Quality_PureGraphManagerExtension extends DSL_DataQuali
       {},
     );
   };
+
+  private rawLambdaToV1(lambda: RawLambda): V1_RawLambda {
+    return V1_transformRawLambda(
+      lambda,
+      new V1_GraphTransformerContextBuilder(
+        this.graphManager.pluginManager.getPureProtocolProcessorPlugins(),
+      ).build(),
+    );
+  }
+
+  private createReconciliationInput(
+    graph: PureModel,
+    options: DQReconciliationInputOptions,
+  ): V1_DQReconciliationInput {
+    const input = new V1_DQReconciliationInput();
+    input.clientVersion =
+      options.clientVersion ??
+      V1_DSL_Data_Quality_PureGraphManagerExtension.DEV_PROTOCOL_VERSION;
+    input.model = graph.origin
+      ? this.buildPureModelSDLCPointer(graph.origin, undefined)
+      : this.graphManager.getFullGraphModelData(graph);
+    input.source = this.rawLambdaToV1(options.source);
+    input.target = this.rawLambdaToV1(options.target);
+    input.keys = options.keys;
+    input.colsForHash = options.colsForHash;
+    input.defectLimit = options.limit;
+    input.aggregatedHash = options.aggregatedHash;
+    input.sourceHashCol = options.sourceHashCol;
+    input.targetHashCol = options.targetHashCol;
+    input.includeColumnValues = options.includeColumnValues;
+    input.runSourceQuery = options.runSourceQuery;
+    input.runTargetQuery = options.runTargetQuery;
+    if (options.sourceLambdaParameterValues) {
+      input.sourceLambdaParameterValues =
+        options.sourceLambdaParameterValues.map(V1_transformParameterValue);
+    }
+    if (options.targetLambdaParameterValues) {
+      input.targetLambdaParameterValues =
+        options.targetLambdaParameterValues.map(V1_transformParameterValue);
+    }
+    return input;
+  }
+
+  runReconciliation = async (
+    graph: PureModel,
+    options: DQReconciliationInputOptions,
+  ): Promise<ExecutionResult> => {
+    const input = this.createReconciliationInput(graph, options);
+
+    return this.runReconciliationWithInput(input);
+  };
+
+  runReconciliationSourceQuery = async (
+    graph: PureModel,
+    options: DQReconciliationInputOptions,
+  ): Promise<ExecutionResult> => {
+    const input = this.createReconciliationInput(graph, {
+      ...options,
+      runSourceQuery: true,
+      runTargetQuery: undefined,
+    });
+    return this.runReconciliationWithInput(input);
+  };
+
+  runReconciliationTargetQuery = async (
+    graph: PureModel,
+    options: DQReconciliationInputOptions,
+  ): Promise<ExecutionResult> => {
+    const input = this.createReconciliationInput(graph, {
+      ...options,
+      runSourceQuery: undefined,
+      runTargetQuery: true,
+    });
+    return this.runReconciliationWithInput(input);
+  };
+
+  private async runReconciliationWithInput(
+    input: V1_DQReconciliationInput,
+  ): Promise<ExecutionResult> {
+    try {
+      const engineServerClient = guaranteeType(
+        this.graphManager.engine,
+        V1_RemoteEngine,
+        'runReconciliation is only supported by remote engine',
+      ).getEngineServerClient();
+
+      const result = await engineServerClient.postWithTracing(
+        engineServerClient.getTraceData(DQ_EXECUTE_RECONCILIATION),
+        `${engineServerClient._pure()}/dataquality/reconciliation`,
+        V1_DQReconciliationInput.serialization.toJson(input),
+        {},
+        undefined,
+        undefined,
+        { enableCompression: true },
+        { skipProcessing: true },
+      );
+
+      const resultInText = await (result as Response).text();
+      const rawExecutionResult =
+        returnUndefOnError(() =>
+          this.graphManager.engine.parseExecutionResults(
+            resultInText,
+            undefined,
+          ),
+        ) ?? resultInText;
+      const v1_executionResult =
+        V1_deserializeExecutionResult(rawExecutionResult);
+      return V1_buildExecutionResult(v1_executionResult);
+    } catch (error) {
+      assertErrorThrown(error);
+      if (error instanceof NetworkClientError) {
+        throw V1_buildExecutionError(
+          V1_ExecutionError.serialization.fromJson(
+            error.payload as PlainObject<V1_ExecutionError>,
+          ),
+        );
+      }
+      throw error;
+    }
+  }
 }

package/src/graph-manager/protocol/pure/v1/V1_DataQualityValidationConfiguration.ts

@@ -165,3 +165,52 @@ export class V1_DataQualityRelationValidationsConfiguration
     return visitor.visit_PackageableElement(this);
   }
 }
+
+export abstract class V1_ReconStrategy implements Hashable {
+  abstract get hashCode(): string;
+}
+
+export class V1_MD5HashStrategy extends V1_ReconStrategy {
+  sourceHashColumn?: string | undefined;
+  targetHashColumn?: string | undefined;
+  aggregatedHash?: boolean | undefined;
+
+  override get hashCode(): string {
+    return hashArray([
+      DATA_QUALITY_HASH_STRUCTURE.DATA_QUALITY_MD5_HASH_STRATEGY,
+      this.sourceHashColumn ?? '',
+      this.targetHashColumn ?? '',
+      this.aggregatedHash ?? false,
+    ]);
+  }
+}
+
+export class V1_DataQualityRelationComparisonConfiguration
+  extends V1_DataQualityValidationsConfiguration
+  implements Hashable
+{
+  source!: V1_DataQualityRelationQueryLambda;
+  target!: V1_DataQualityRelationQueryLambda;
+  keys: string[] = [];
+  columnsToCompare: string[] = [];
+  strategy!: V1_ReconStrategy;
+  expectedMatch?: number | undefined;
+
+  override get hashCode(): string {
+    return hashArray([
+      DATA_QUALITY_HASH_STRUCTURE.DATA_QUALITY_RELATION_COMPARISON_HASH_STRUCTURE,
+      this.source,
+      this.target,
+      hashArray(this.keys),
+      hashArray(this.columnsToCompare),
+      String(this.expectedMatch ?? ''),
+      this.strategy,
+    ]);
+  }
+
+  accept_PackageableElementVisitor<T>(
+    visitor: V1_PackageableElementVisitor<T>,
+  ): T {
+    return visitor.visit_PackageableElement(this);
+  }
+}

package/src/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationBuilderHelper.ts

@@ -31,8 +31,10 @@ import {
   type V1_DataQualityRelationValidation,
   type V1_DataQualityRelationValidationsConfiguration,
   type V1_DataQualityServiceValidationsConfiguration,
+  type V1_DataQualityRelationComparisonConfiguration,
   V1_DataSpaceDataQualityExecutionContext,
   V1_MappingAndRuntimeDataQualityExecutionContext,
+  V1_MD5HashStrategy,
 } from '../V1_DataQualityValidationConfiguration.js';
 import {
   type DataQualityExecutionContext,
@@ -40,6 +42,7 @@ import {
   DataQualityRelationValidation,
   DataSpaceDataQualityExecutionContext,
   MappingAndRuntimeDataQualityExecutionContext,
+  MD5HashStrategy,
 } from '../../../../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
 import {
   type V1_GraphBuilderContext,
@@ -65,6 +68,7 @@ import {
   getOwnDataQualityClassValidationsConfiguration,
   getOwnDataQualityServiceValidationsConfiguration,
   getOwnDataQualityRelationValidationsConfiguration,
+  getOwnDataQualityRelationComparisonConfiguration,
 } from '../../../../DSL_DataQuality_GraphManagerHelper.js';
 
 export function V1_buildDataQualityExecutionContext(
@@ -355,3 +359,38 @@ export function V1_buildDataQualityServiceValidationConfiguration(
         ) as DataQualityRootGraphFetchTree)
       : undefined;
 }
+
+export function V1_buildDataQualityRelationComparisonConfiguration(
+  elementProtocol: V1_DataQualityRelationComparisonConfiguration,
+  context: V1_GraphBuilderContext,
+): void {
+  const path = V1_buildFullPath(elementProtocol.package, elementProtocol.name);
+  const element = getOwnDataQualityRelationComparisonConfiguration(
+    path,
+    context.currentSubGraph,
+  );
+  element.source = new DataQualityRelationQueryLambda();
+  element.source.body = elementProtocol.source.body;
+  element.source.parameters = elementProtocol.source.parameters.map((param) =>
+    V1_buildVariable(param, context),
+  );
+  element.target = new DataQualityRelationQueryLambda();
+  element.target.body = elementProtocol.target.body;
+  element.target.parameters = elementProtocol.target.parameters.map((param) =>
+    V1_buildVariable(param, context),
+  );
+  element.keys = elementProtocol.keys;
+  element.columnsToCompare = elementProtocol.columnsToCompare;
+  if (elementProtocol.strategy instanceof V1_MD5HashStrategy) {
+    const strategy = new MD5HashStrategy();
+    strategy.sourceHashColumn = elementProtocol.strategy.sourceHashColumn;
+    strategy.targetHashColumn = elementProtocol.strategy.targetHashColumn;
+    strategy.aggregatedHash = elementProtocol.strategy.aggregatedHash;
+    element.strategy = strategy;
+  } else {
+    throw new UnsupportedOperationError(
+      `Can't build recon strategy: unsupported type`,
+    );
+  }
+  element.expectedMatch = elementProtocol.expectedMatch;
+}

package/src/graph-manager/protocol/pure/v1/transformation/V1_DSL_DataQuality_ValueSpecificationTransformer.ts

@@ -45,17 +45,23 @@ import {
   type DataQualityRelationValidation,
   type DataQualityRelationValidationConfiguration,
   type DataQualityClassValidationsConfiguration,
+  type DataQualityRelationComparisonConfiguration,
+  type ReconStrategy,
+  MD5HashStrategy,
   DataSpaceDataQualityExecutionContext,
   MappingAndRuntimeDataQualityExecutionContext,
 } from '../../../../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
 import {
   type V1_DataQualityExecutionContext,
+  type V1_ReconStrategy,
   V1_DataQualityClassValidationsConfiguration,
   V1_DataQualityRelationValidation,
   V1_DataQualityRelationValidationsConfiguration,
   V1_DataSpaceDataQualityExecutionContext,
   V1_MappingAndRuntimeDataQualityExecutionContext,
   V1_DataQualityRelationQueryLambda,
+  V1_DataQualityRelationComparisonConfiguration,
+  V1_MD5HashStrategy,
 } from '../V1_DataQualityValidationConfiguration.js';
 import { DATA_SPACE_ELEMENT_POINTER } from '@finos/legend-extension-dsl-data-space/graph';
 
@@ -224,3 +230,47 @@ export function V1_transformDataQualityRelationValidationConfiguration(
   );
   return protocol;
 }
+
+function V1_transformReconStrategy(value: ReconStrategy): V1_ReconStrategy {
+  if (value instanceof MD5HashStrategy) {
+    const protocol = new V1_MD5HashStrategy();
+    protocol.sourceHashColumn = value.sourceHashColumn;
+    protocol.targetHashColumn = value.targetHashColumn;
+    protocol.aggregatedHash = value.aggregatedHash;
+    return protocol;
+  }
+  throw new UnsupportedOperationError(
+    `Can't transform recon strategy: unsupported type`,
+  );
+}
+
+export function V1_transformDataQualityRelationComparisonConfiguration(
+  metamodel: DataQualityRelationComparisonConfiguration,
+  context: V1_GraphTransformerContext,
+): V1_DataQualityRelationComparisonConfiguration {
+  const protocol = new V1_DataQualityRelationComparisonConfiguration();
+  V1_initPackageableElement(protocol, metamodel);
+  protocol.name = metamodel.name;
+  protocol.package = metamodel.package?.path ?? '';
+  protocol.source = new V1_DataQualityRelationQueryLambda();
+  protocol.source.body = metamodel.source.body;
+  protocol.source.parameters = metamodel.source.parameters.map(
+    (v) =>
+      v.accept_RawValueSpecificationVisitor(
+        new V1_RawValueSpecificationTransformer(context),
+      ) as V1_RawVariable,
+  );
+  protocol.target = new V1_DataQualityRelationQueryLambda();
+  protocol.target.body = metamodel.target.body;
+  protocol.target.parameters = metamodel.target.parameters.map(
+    (v) =>
+      v.accept_RawValueSpecificationVisitor(
+        new V1_RawValueSpecificationTransformer(context),
+      ) as V1_RawVariable,
+  );
+  protocol.keys = metamodel.keys;
+  protocol.columnsToCompare = metamodel.columnsToCompare;
+  protocol.strategy = V1_transformReconStrategy(metamodel.strategy);
+  protocol.expectedMatch = metamodel.expectedMatch;
+  return protocol;
+}

package/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/V1_DSL_DataQuality_ProtocolHelper.ts

@@ -24,9 +24,11 @@ import {
   list,
   optional,
   raw,
+  SKIP,
 } from 'serializr';
 import {
   type V1_DataQualityExecutionContext,
+  type V1_ReconStrategy,
   V1_DataQualityClassValidationsConfiguration,
   V1_DataQualityServiceValidationsConfiguration,
   V1_DataSpaceDataQualityExecutionContext,
@@ -34,6 +36,8 @@ import {
   V1_DataQualityRelationValidationsConfiguration,
   V1_DataQualityRelationValidation,
   V1_DataQualityRelationQueryLambda,
+  V1_DataQualityRelationComparisonConfiguration,
+  V1_MD5HashStrategy,
 } from '../../V1_DataQualityValidationConfiguration.js';
 import {
   type PlainObject,
@@ -60,6 +64,8 @@ export const V1_DATA_QUALITY_RELATION_PROTOCOL_TYPE =
   'dataqualityRelationValidation';
 export const V1_DATA_QUALITY_SERVICE_PROTOCOL_TYPE =
   'dataQualityServiceValidations';
+export const V1_DATA_QUALITY_RELATION_COMPARISON_PROTOCOL_TYPE =
+  'dataQualityRelationComparison';
 const V1_DATA_QUALITY_DATASPACE_EXECUTION_CONTEXT =
   'dataSpaceDataQualityExecutionContext';
 const V1_DATA_QUALITY_MAPPING_AND_RUNTIME_EXECUTION_CONTEXT =
@@ -251,3 +257,68 @@ export const V1_deserializeDataQualityRelationValidation = (
   plugins: PureProtocolProcessorPlugin[],
 ): V1_DataQualityRelationValidationsConfiguration =>
   deserialize(V1_dataQualityRelationValidationModelSchema(plugins), json);
+
+const V1_MD5_HASH_STRATEGY_TYPE = 'md5Hash';
+
+const V1_md5HashStrategyModelSchema = createModelSchema(V1_MD5HashStrategy, {
+  _type: usingConstantValueSchema(V1_MD5_HASH_STRATEGY_TYPE),
+  sourceHashColumn: optional(primitive()),
+  targetHashColumn: optional(primitive()),
+  aggregatedHash: optional(primitive()),
+});
+
+function V1_serializeReconStrategy(
+  value: V1_ReconStrategy,
+): PlainObject<V1_ReconStrategy> {
+  if (value instanceof V1_MD5HashStrategy) {
+    return serialize(V1_md5HashStrategyModelSchema, value);
+  }
+  throw new UnsupportedOperationError(
+    `Can't serialize recon strategy: unsupported type`,
+  );
+}
+
+function V1_deserializeReconStrategy(
+  json: PlainObject<V1_ReconStrategy>,
+): V1_ReconStrategy {
+  switch (json._type) {
+    case V1_MD5_HASH_STRATEGY_TYPE:
+      return deserialize(V1_md5HashStrategyModelSchema, json);
+    default:
+      throw new UnsupportedOperationError(
+        `Can't deserialize recon strategy of type '${json._type}'`,
+      );
+  }
+}
+
+const V1_dataQualityRelationComparisonModelSchema = (
+  plugins: PureProtocolProcessorPlugin[],
+): ModelSchema<V1_DataQualityRelationComparisonConfiguration> =>
+  createModelSchema(V1_DataQualityRelationComparisonConfiguration, {
+    _type: usingConstantValueSchema(
+      V1_DATA_QUALITY_RELATION_COMPARISON_PROTOCOL_TYPE,
+    ),
+    name: primitive(),
+    package: primitive(),
+    source: usingModelSchema(V1_rawLambdaModelSchemaParameters),
+    target: usingModelSchema(V1_rawLambdaModelSchemaParameters),
+    keys: list(primitive()),
+    columnsToCompare: list(primitive()),
+    strategy: optionalCustom(
+      (val) => (val ? V1_serializeReconStrategy(val) : SKIP),
+      (val) => (val ? V1_deserializeReconStrategy(val) : SKIP),
+    ),
+    expectedMatch: optional(primitive()),
+  });
+
+export const V1_serializeDataQualityRelationComparison = (
+  protocol: V1_DataQualityRelationComparisonConfiguration,
+  plugins: PureProtocolProcessorPlugin[],
+): PlainObject<V1_DataQualityRelationComparisonConfiguration> =>
+  serialize(V1_dataQualityRelationComparisonModelSchema(plugins), protocol);
+
+export const V1_deserializeDataQualityRelationComparison = (
+  json: PlainObject<V1_DataQualityRelationComparisonConfiguration>,
+  plugins: PureProtocolProcessorPlugin[],
+): V1_DataQualityRelationComparisonConfiguration =>
+  deserialize(V1_dataQualityRelationComparisonModelSchema(plugins), json);