@finos/legend-extension-dsl-data-quality 2.1.31 → 2.1.32

package/src/components/utils/DataQualityValidateFunctionToPureLambdaVisitor.ts

@@ -19,13 +19,14 @@ import {
   SUPPORTED_TYPES,
 } from '../constants/DataQualityConstants.js';
 import type { LambdaBody } from './DataQualityLambdaParameterParser.js';
-import type {
-  DataQualityValidationFilterFunction,
-  DataQualityValidationCustomHelperFunction,
-  DataQualityValidationAssertionFunction,
-  DataQualityValidationFunctionVisitor,
-  DataQualityValidationFilterCondition,
-  DataQualityValidationLogicalGroupFunction,
+import {
+  type DataQualityValidationFilterFunction,
+  type DataQualityValidationCustomHelperFunction,
+  type DataQualityValidationAssertionFunction,
+  type DataQualityValidationFunctionVisitor,
+  type DataQualityValidationFilterCondition,
+  type DataQualityValidationLogicalGroupFunction,
+  DataQualityValidationPropertyGuarantee,
 } from './DataQualityValidationFunction.js';
 import type {
   GraphManagerState,
@@ -141,7 +142,11 @@ export class DataQualityValidateFunctionToPureLambdaVisitor
     const { property, otherParams } = func.parameters;
     const functionParameters: LambdaBody[] = [];
 
-    functionParameters.push(this.serializeValueSpecification(property));
+    if (property instanceof DataQualityValidationPropertyGuarantee) {
+      functionParameters.push(property.accept(this));
+    } else {
+      functionParameters.push(this.serializeValueSpecification(property));
+    }
 
     if (otherParams.length > 0) {
       otherParams.forEach((param) => {
@@ -168,4 +173,16 @@ export class DataQualityValidateFunctionToPureLambdaVisitor
       parameters: [leftLambdaBody, rightLambdaBody],
     } as LambdaBody;
   }
+
+  visitPropertyGuarantee(
+    func: DataQualityValidationPropertyGuarantee,
+  ): LambdaBody {
+    const { property } = func.parameters;
+
+    return {
+      _type: SUPPORTED_TYPES.FUNCTION,
+      function: func.name,
+      parameters: [this.serializeValueSpecification(property)],
+    } as LambdaBody;
+  }
 }

package/src/components/utils/DataQualityValidationDescriptionGeneratorVisitor.ts

@@ -15,12 +15,13 @@
  */
 
 import { DataQualityFunctionDefaults } from './DataQualityFunctionDefaults.js';
-import type {
-  DataQualityValidationFunctionVisitor,
-  DataQualityValidationFilterFunction,
-  DataQualityValidationCustomHelperFunction,
-  DataQualityValidationFilterCondition,
-  DataQualityValidationLogicalGroupFunction,
+import {
+  type DataQualityValidationFunctionVisitor,
+  type DataQualityValidationFilterFunction,
+  type DataQualityValidationCustomHelperFunction,
+  type DataQualityValidationFilterCondition,
+  type DataQualityValidationLogicalGroupFunction,
+  DataQualityValidationPropertyGuarantee,
 } from './DataQualityValidationFunction.js';
 import {
   CollectionInstanceValue,
@@ -34,6 +35,10 @@ export class DataQualityValidationDescriptionGeneratorVisitor
     return '';
   }
 
+  visitPropertyGuarantee(func: DataQualityValidationPropertyGuarantee): string {
+    return '';
+  }
+
   visitFilter(func: DataQualityValidationFilterFunction): string {
     const { parameters } = func;
     const { lambda } = parameters;
@@ -78,7 +83,10 @@ export class DataQualityValidationDescriptionGeneratorVisitor
     const values: string[] = [];
     const description =
       DataQualityFunctionDefaults.getFunctionDescriptionTemplate(func.name);
-    const columnName = property.func.value.name;
+    const columnName =
+      property instanceof DataQualityValidationPropertyGuarantee
+        ? property.parameters.property.func.value.name
+        : property.func.value.name;
 
     if (!columnName || !description) {
       return '';

package/src/components/utils/DataQualityValidationFunction.ts

@@ -61,10 +61,14 @@ export interface FilterValidationParameters extends BaseValidationParameters {
 }
 
 export interface FilterConditionParameters {
-  property: AbstractPropertyExpression;
+  property: AbstractPropertyExpression | DataQualityValidationPropertyGuarantee;
   otherParams: (PrimitiveInstanceValue | CollectionInstanceValue)[];
 }
 
+export interface PropertyGuaranteeParameters {
+  property: AbstractPropertyExpression;
+}
+
 export interface LogicalGroupValidationParameters {
   left:
     | DataQualityValidationFilterCondition
@@ -103,6 +107,10 @@ export interface DataQualityValidationFunctionVisitor<T> {
   visitCustomHelper(func: DataQualityValidationCustomHelperFunction): T;
   visitFilterCondition(func: DataQualityValidationFilterCondition): T;
   visitLogicalGroup(func: DataQualityValidationLogicalGroupFunction): T;
+  visitPropertyGuarantee(
+    func: DataQualityValidationPropertyGuarantee,
+    context?: unknown,
+  ): T;
 }
 
 export class DataQualityValidationFilterFunction extends DataQualityValidationFunction<FilterValidationParameters> {
@@ -120,21 +128,38 @@ export class DataQualityValidationFilterFunction extends DataQualityValidationFu
   }
 }
 
-export class DataQualityValidationFilterCondition extends DataQualityValidationFunction<FilterConditionParameters> {
+abstract class AbstractDataQualityValidationFilterCondition<
+  T,
+> extends DataQualityValidationFunction<T> {
   name: string;
-  parameters: FilterConditionParameters;
+  parameters: T;
 
-  constructor(name: string, parameters: FilterConditionParameters) {
+  constructor(name: string, parameters: T) {
     super();
     this.name = name;
     this.parameters = parameters;
   }
 
+  abstract override accept<R>(
+    visitor: DataQualityValidationFunctionVisitor<R>,
+  ): R;
+}
+
+export class DataQualityValidationFilterCondition extends AbstractDataQualityValidationFilterCondition<FilterConditionParameters> {
   accept<R>(visitor: DataQualityValidationFunctionVisitor<R>): R {
     return visitor.visitFilterCondition(this);
   }
 }
 
+export class DataQualityValidationPropertyGuarantee extends AbstractDataQualityValidationFilterCondition<PropertyGuaranteeParameters> {
+  override accept<R>(
+    visitor: DataQualityValidationFunctionVisitor<R>,
+    context?: unknown,
+  ): R {
+    return visitor.visitPropertyGuarantee(this, context);
+  }
+}
+
 export class DataQualityValidationLogicalGroupFunction extends DataQualityValidationFunction<LogicalGroupValidationParameters> {
   name: DATA_QUALITY_VALIDATION_LOGICAL_FUNCTIONS;
   parameters: LogicalGroupValidationParameters;

package/src/components/utils/DataQualityValidationFunctionCloningVisitor.ts

@@ -19,6 +19,7 @@ import {
   type DataQualityValidationFunctionVisitor,
   DataQualityValidationFilterCondition,
   type DataQualityValidationLogicalGroupFunction,
+  DataQualityValidationPropertyGuarantee,
 } from '../utils/DataQualityValidationFunction.js';
 import { assertType, UnsupportedOperationError } from '@finos/legend-shared';
 import type { DataQualityValidationFunctionFactory } from './DataQualityValidationFunctionFactory.js';
@@ -34,25 +35,112 @@ import {
   type AbstractProperty,
   Multiplicity,
 } from '@finos/legend-graph';
-import { DATA_QUALITY_FILTER_VALIDATION_HELPER_FUNCTIONS } from '../constants/DataQualityConstants.js';
 import {
-  cloneAbstractPropertyExpression,
+  DATA_QUALITY_FILTER_VALIDATION_HELPER_FUNCTIONS,
+  DATA_QUALITY_VALIDATION_PROPERTY_GUARANTEE_FUNCTIONS,
+} from '../constants/DataQualityConstants.js';
+import {
   cloneValueSpecification,
   instanceValue_setValues,
-  propertyExpression_setFunc,
 } from '@finos/legend-query-builder';
 import {
   observe_DataQualityValidationCustomHelperFunction,
   observe_DataQualityValidationFilterCondition,
   observe_DataQualityValidationFilterFunction,
+  observe_DataQualityValidationPropertyGuarantee,
 } from './DataQualityValidationFunctionObserver.js';
+import { dataQualityValidationLogicalGroupFunction_setParametersValues } from './DataQualityValidationFunctionModifier.js';
+import { DataQualityFunctionDefaults } from './DataQualityFunctionDefaults.js';
+
+const VISITOR_ERROR = new UnsupportedOperationError('Visitor not implemented');
+
+export type DataQualityValidationFilterFunctions =
+  | DataQualityValidationFilterCondition
+  | DataQualityValidationLogicalGroupFunction;
+
+export class DataQualityValidationFilterFunctionsCloningVisitor
+  implements
+    DataQualityValidationFunctionVisitor<DataQualityValidationFilterFunctions>
+{
+  private newName: string;
+  private cloneFactory: DataQualityValidationFunctionFactory;
+  private observerContext: ObserverContext;
+  private readonly isCurrentColOptional: boolean;
+
+  constructor(
+    newName: string,
+    cloneFactory: DataQualityValidationFunctionFactory,
+    observerContext: ObserverContext,
+    isCurrentColOptional: boolean,
+  ) {
+    this.newName = newName;
+    this.cloneFactory = cloneFactory;
+    this.observerContext = observerContext;
+    this.isCurrentColOptional = isCurrentColOptional;
+  }
+
+  visitAssertion(): DataQualityValidationFilterFunctions {
+    throw VISITOR_ERROR;
+  }
+
+  visitCustomHelper(): DataQualityValidationFilterFunctions {
+    throw VISITOR_ERROR;
+  }
+
+  visitFilter(): DataQualityValidationFilterFunctions {
+    throw VISITOR_ERROR;
+  }
+
+  visitFilterCondition(func: DataQualityValidationFilterCondition) {
+    const currentColumn =
+      func.parameters.property instanceof AbstractPropertyExpression
+        ? func.parameters.property
+        : func.parameters.property.parameters.property;
+    const clone = this.cloneFactory.createFilterConditionFunction(
+      this.newName,
+      currentColumn.func.value.name,
+    );
+
+    const isPureFunctionColumnRequired =
+      DataQualityFunctionDefaults.getIsPureFunctionColumnRequired(this.newName);
+
+    if (isPureFunctionColumnRequired && this.isCurrentColOptional) {
+      clone.parameters.property =
+        this.cloneFactory.createPropertyGuaranteeFunction(
+          DATA_QUALITY_VALIDATION_PROPERTY_GUARANTEE_FUNCTIONS.TO_ONE,
+          currentColumn.func.value.name,
+        );
+      dataQualityValidationLogicalGroupFunction_setParametersValues(
+        clone.parameters.property.parameters.property,
+        currentColumn.parametersValues.map((param) =>
+          cloneValueSpecification(param, this.observerContext),
+        ),
+      );
+    } else {
+      dataQualityValidationLogicalGroupFunction_setParametersValues(
+        clone.parameters.property as AbstractPropertyExpression,
+        currentColumn.parametersValues.map((param) =>
+          cloneValueSpecification(param, this.observerContext),
+        ),
+      );
+    }
+
+    return observe_DataQualityValidationFilterCondition(clone);
+  }
+
+  visitPropertyGuarantee(): DataQualityValidationFilterCondition {
+    throw VISITOR_ERROR;
+  }
+
+  visitLogicalGroup(func: DataQualityValidationLogicalGroupFunction) {
+    return func;
+  }
+}
 
 type AnyDataQualityValidationFunction =
   | DataQualityValidationFilterFunction
   | DataQualityValidationCustomHelperFunction;
 
-const VISITOR_ERROR = new UnsupportedOperationError('Visitor not implemented');
-
 export class DataQualityValidationFunctionCloningVisitor
   implements
     DataQualityValidationFunctionVisitor<AnyDataQualityValidationFunction>
@@ -60,15 +148,18 @@ export class DataQualityValidationFunctionCloningVisitor
   private newName: string;
   private cloneFactory: DataQualityValidationFunctionFactory;
   private observerContext: ObserverContext;
+  private readonly isCurrentColOptional: boolean;
 
   constructor(
     newName: string,
     cloneFactory: DataQualityValidationFunctionFactory,
     observerContext: ObserverContext,
+    isCurrentColOptional: boolean,
   ) {
     this.newName = newName;
     this.cloneFactory = cloneFactory;
     this.observerContext = observerContext;
+    this.isCurrentColOptional = isCurrentColOptional;
   }
 
   visitAssertion(): AnyDataQualityValidationFunction {
@@ -83,24 +174,32 @@ export class DataQualityValidationFunctionCloningVisitor
     throw VISITOR_ERROR;
   }
 
+  visitPropertyGuarantee(): AnyDataQualityValidationFunction {
+    throw VISITOR_ERROR;
+  }
+
   visitFilter(func: DataQualityValidationFilterFunction) {
-    const clone = this.createFunctionForCloning();
     const body = func.parameters.lambda.body;
     assertType(body, DataQualityValidationFilterCondition);
-    const property = body.parameters.property;
+    let property = body.parameters.property;
+    if (property instanceof DataQualityValidationPropertyGuarantee) {
+      property = property.parameters.property;
+    }
+    const clone = this.createFunctionForCloning();
     clone.id = func.id;
 
     if (clone instanceof DataQualityValidationCustomHelperFunction) {
       return this.updateCustomHelpFunctionCol(clone, property.func.value.name);
     }
 
-    (
-      clone.parameters.lambda.body as DataQualityValidationFilterCondition
-    ).parameters.property = cloneAbstractPropertyExpression(
-      property,
+    const visitor = new DataQualityValidationFilterFunctionsCloningVisitor(
+      this.newName,
+      this.cloneFactory,
       this.observerContext,
+      this.isCurrentColOptional,
     );
 
+    clone.parameters.lambda.body = body.accept(visitor);
     return clone;
   }
 
@@ -145,12 +244,9 @@ export class DataQualityValidationFunctionCloningVisitor
       );
     }
 
-    const clone = this.cloneFactory.createFilterFunction();
-    clone.parameters.lambda.body = observe_DataQualityValidationFilterCondition(
-      this.cloneFactory.createFilterChildFunction(this.newName),
+    return observe_DataQualityValidationFilterFunction(
+      this.cloneFactory.createFilterFunction(),
     );
-
-    return observe_DataQualityValidationFilterFunction(clone);
   }
 
   private updateCustomHelpFunctionCol(
@@ -173,82 +269,40 @@ export class DataQualityValidationFunctionCloningVisitor
     col: string,
     variableRef?: VariableExpression,
   ) {
-    assertType(
-      clone.parameters.lambda.body,
-      DataQualityValidationFilterCondition,
-    );
-
     const property = new AbstractPropertyExpression('');
     property.func = PropertyExplicitReference.create({
       name: col,
     } as AbstractProperty);
-
     property.parametersValues = [
       cloneValueSpecification(
         variableRef ?? new VariableExpression('row', Multiplicity.ZERO),
         this.observerContext,
       ),
     ];
-
     observe_AbstractPropertyExpression(property, this.observerContext);
-    clone.parameters.lambda.body.parameters.property = property;
-    return clone;
-  }
-}
-
-export type DataQualityValidationFilterFunctions =
-  | DataQualityValidationFilterCondition
-  | DataQualityValidationLogicalGroupFunction;
-
-export class DataQualityValidationFilterFunctionsCloningVisitor
-  implements
-    DataQualityValidationFunctionVisitor<DataQualityValidationFilterFunctions>
-{
-  private newName: string;
-  private cloneFactory: DataQualityValidationFunctionFactory;
-  private observerContext: ObserverContext;
-
-  constructor(
-    newName: string,
-    cloneFactory: DataQualityValidationFunctionFactory,
-    observerContext: ObserverContext,
-  ) {
-    this.newName = newName;
-    this.cloneFactory = cloneFactory;
-    this.observerContext = observerContext;
-  }
 
-  visitAssertion(): DataQualityValidationFilterFunctions {
-    throw VISITOR_ERROR;
-  }
+    const body = this.cloneFactory.createFilterChildFunction(this.newName);
 
-  visitCustomHelper(): DataQualityValidationFilterFunctions {
-    throw VISITOR_ERROR;
-  }
-
-  visitFilter(): DataQualityValidationFilterFunctions {
-    throw VISITOR_ERROR;
-  }
-
-  visitFilterCondition(func: DataQualityValidationFilterCondition) {
-    const clone = this.cloneFactory.createFilterConditionFunction(this.newName);
-    const currentColumn = func.parameters.property;
-
-    propertyExpression_setFunc(
-      clone.parameters.property,
-      PropertyExplicitReference.create({
-        name: currentColumn.func.value.name,
-      } as AbstractProperty),
-    );
-    clone.parameters.property.parametersValues =
-      currentColumn.parametersValues.map((param) =>
-        cloneValueSpecification(param, this.observerContext),
+    if (
+      DataQualityFunctionDefaults.getIsPureFunctionColumnRequired(
+        this.newName,
+      ) &&
+      this.isCurrentColOptional
+    ) {
+      const guarantee = this.cloneFactory.createPropertyGuaranteeFunction(
+        DATA_QUALITY_VALIDATION_PROPERTY_GUARANTEE_FUNCTIONS.TO_ONE,
       );
 
-    return observe_DataQualityValidationFilterCondition(clone);
-  }
+      guarantee.parameters.property = property;
+      body.parameters.property =
+        observe_DataQualityValidationPropertyGuarantee(guarantee);
+    } else {
+      body.parameters.property = property;
+    }
 
-  visitLogicalGroup(func: DataQualityValidationLogicalGroupFunction) {
-    return func;
+    clone.parameters.lambda.body =
+      observe_DataQualityValidationFilterCondition(body);
+
+    return clone;
   }
 }

package/src/components/utils/DataQualityValidationFunctionFactory.ts

@@ -33,6 +33,7 @@ import {
   DataQualityValidationFilterCondition,
   DataQualityValidationFilterFunction,
   DataQualityValidationLogicalGroupFunction,
+  DataQualityValidationPropertyGuarantee,
 } from './DataQualityValidationFunction.js';
 import { UnsupportedOperationError } from '@finos/legend-shared';
 import { DataQualityFunctionDefaults } from './DataQualityFunctionDefaults.js';
@@ -82,6 +83,7 @@ export class DataQualityValidationFunctionFactory {
 
   createFilterConditionFunction(
     name: string,
+    propertyName?: string,
   ): DataQualityValidationFilterCondition {
     const otherParams = DataQualityFunctionDefaults.getPureFunctionDefaults(
       name,
@@ -90,7 +92,7 @@ export class DataQualityValidationFunctionFactory {
     );
     const property = DataQualityLambdaParameterParser.processPropertyParameter(
       {
-        property: '',
+        property: propertyName ?? '',
         parameters: [
           {
             name: 'row',
@@ -106,6 +108,27 @@ export class DataQualityValidationFunctionFactory {
     });
   }
 
+  createPropertyGuaranteeFunction(
+    name: string,
+    propertyName?: string,
+  ): DataQualityValidationPropertyGuarantee {
+    const property = DataQualityLambdaParameterParser.processPropertyParameter(
+      {
+        property: propertyName ?? '',
+        parameters: [
+          {
+            name: 'row',
+            _type: SUPPORTED_TYPES.VAR,
+          },
+        ],
+      } as LambdaBody,
+      this.observerContext,
+    );
+    return new DataQualityValidationPropertyGuarantee(name, {
+      property,
+    });
+  }
+
   createLogicalFunction(
     name: DATA_QUALITY_VALIDATION_LOGICAL_FUNCTIONS,
   ): DataQualityValidationLogicalGroupFunction {

package/src/components/utils/DataQualityValidationFunctionModifier.ts

@@ -18,10 +18,15 @@ import {
   type DataQualityValidationFilterCondition,
   type DataQualityValidationFilterFunction,
   DataQualityValidationLogicalGroupFunction,
+  type DataQualityValidationPropertyGuarantee,
 } from '../utils/DataQualityValidationFunction.js';
 import type { DATA_QUALITY_VALIDATION_LOGICAL_FUNCTIONS } from '../constants/DataQualityConstants.js';
 import type { DataQualityValidationFunctionFactory } from './DataQualityValidationFunctionFactory.js';
-import { type ObserverContext } from '@finos/legend-graph';
+import type {
+  AbstractPropertyExpression,
+  ValueSpecification,
+  ObserverContext,
+} from '@finos/legend-graph';
 import { observe_DataQualityValidationLogicalGroupFunction } from './DataQualityValidationFunctionObserver.js';
 import { DataQualityValidationFilterFunctionsCloningVisitor } from './DataQualityValidationFunctionCloningVisitor.js';
 import { action } from 'mobx';
@@ -33,11 +38,13 @@ export const dataQualityValidationFilterFunction_addLogicalOperation = action(
     operator: DATA_QUALITY_VALIDATION_LOGICAL_FUNCTIONS,
     factory: DataQualityValidationFunctionFactory,
     observerContext: ObserverContext,
+    isCurrentColOptional: boolean,
   ): void => {
     const visitor = new DataQualityValidationFilterFunctionsCloningVisitor(
       currentCondition.name,
       factory,
       observerContext,
+      isCurrentColOptional,
     );
     const newCondition = currentCondition.accept(visitor);
 
@@ -59,12 +66,14 @@ export const dataQualityValidationFilterFunction_transformConditionToLogicalGrou
       operator: DATA_QUALITY_VALIDATION_LOGICAL_FUNCTIONS,
       factory: DataQualityValidationFunctionFactory,
       observerContext: ObserverContext,
+      isCurrentColOptional: boolean,
     ): void => {
       const createGroup = () => {
         const visitor = new DataQualityValidationFilterFunctionsCloningVisitor(
           conditionToTransform.name,
           factory,
           observerContext,
+          isCurrentColOptional,
         );
         const newCondition = conditionToTransform.accept(visitor);
         const logicalGroup = factory.createLogicalFunction(operator);
@@ -174,6 +183,7 @@ export const dataQualityValidationLogicalGroupFunction_changeGroupFunction =
       childToChange: 'left' | 'right',
       factory: DataQualityValidationFunctionFactory,
       observerContext: ObserverContext,
+      isCurrentColOptional: boolean,
     ) => {
       const currentChild =
         childToChange === 'left'
@@ -184,6 +194,7 @@ export const dataQualityValidationLogicalGroupFunction_changeGroupFunction =
         newFunctionName,
         factory,
         observerContext,
+        isCurrentColOptional,
       );
 
       if (childToChange === 'left') {
@@ -193,3 +204,23 @@ export const dataQualityValidationLogicalGroupFunction_changeGroupFunction =
       }
     },
   );
+
+export const dataQualityValidationLogicalGroupFunction_setPropertyGuaranteeFunction =
+  action(
+    (
+      target: DataQualityValidationFilterCondition['parameters'],
+      property: DataQualityValidationPropertyGuarantee,
+    ) => {
+      target.property = property;
+    },
+  );
+
+export const dataQualityValidationLogicalGroupFunction_setParametersValues =
+  action(
+    (
+      target: AbstractPropertyExpression,
+      parametersValues: ValueSpecification[],
+    ) => {
+      target.parametersValues = parametersValues;
+    },
+  );