@truedat/dq 5.2.8 → 5.3.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [5.3.0] 2023-03-08
+
+### Added
+
+- [TD-4438] Referenced in/not in reference dataset field validation operator
+
 ## [5.2.8] 2023-02-28
 
 ### Fixed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@truedat/dq",
-  "version": "5.2.8",
+  "version": "5.3.0",
   "description": "Truedat Web Data Quality Module",
   "sideEffects": false,
   "jsnext:main": "src/index.js",
@@ -92,8 +92,8 @@
   },
   "dependencies": {
     "@apollo/client": "^3.7.1",
-    "@truedat/core": "5.2.4",
-    "@truedat/df": "5.2.4",
+    "@truedat/core": "5.3.0",
+    "@truedat/df": "5.3.0",
     "decode-uri-component": "^0.2.2",
     "graphql": "^15.5.3",
     "moment": "^2.29.4",
@@ -118,5 +118,5 @@
     "react-dom": ">= 16.8.6 < 17",
     "semantic-ui-react": ">= 2.0.3 < 2.2"
   },
-  "gitHead": "562bc4667fa44eeb0e224a886c41849adbd69a18"
+  "gitHead": "4893c314084a4ea7ca48a4e251d4e47565f3f030"
 }

package/src/components/ConditionCell.js

@@ -23,6 +23,17 @@ QualifyAndLinkStructure.propTypes = {
   aliasArray: PropTypes.array,
 };
 
+const referenceDatasetFieldToLink = ({ name, referenceDataset }) => {
+  return (
+    <>
+      <Link to={linkTo.REFERENCE_DATASET({ id: referenceDataset.id })}>
+        <div className="highlighted">{`(${referenceDataset.name})`}</div>{" "}
+      </Link>
+      <span>{`.${name}`}</span>
+    </>
+  );
+};
+
 const FormattedLink = ({ value, aliasArray, operator = {} }) => {
   switch (operator?.value_type) {
     case "field":
@@ -39,6 +50,8 @@ const FormattedLink = ({ value, aliasArray, operator = {} }) => {
           ))(value)}
         </>
       );
+    case "reference_dataset_field":
+      return referenceDatasetFieldToLink(value);
     default:
       return (
         <span className="highlighted">
@@ -109,6 +122,13 @@ export const getValues = ({ value = [], operator = {}, value_modifier }) => {
         ["path", "metadata", "id", "type", "parent_index"],
         value_modifier
       )
+    : operator?.value_type === "reference_dataset_field"
+    ? valuesFromKeys(
+        defaultOrValue,
+        ["referenceDataset", "name"],
+        [],
+        value_modifier
+      )
     : valuesFromKeys(defaultOrValue, ["raw"], [], value_modifier);
 };
 

package/src/components/NewRuleImplementation.js

@@ -30,9 +30,16 @@ const updateDatasetKey = (data) =>
 
 // object either field, raw value, list of fields, list of raw values
 const updateValueKeyContent = (value) => {
-  if (value?.id || _.negate(_.isUndefined)(value?.raw) || value?.fields) {
+  if (
+    value?.id ||
+    value?.referenceDataset ||
+    _.negate(_.isUndefined)(value?.raw) ||
+    value?.fields
+  ) {
     return value?.id || value?.type === "reference_dataset_field"
-      ? _.pick(["id", "parent_index", "name", "type"])(value)
+      ? _.pick(["id", "parent_index", "referenceDataset", "name", "type"])(
+          value
+        )
       : _.pick(["raw", "fields"])(value);
   }
 

package/src/components/__tests__/RemediationForm.spec.js

@@ -18,5 +18,5 @@ describe("<RemediationForm />", () => {
       expect(queryByText(/loading/i)).not.toBeInTheDocument()
     );
     expect(container).toMatchSnapshot();
-  });
+  }, 70000);
 });

package/src/components/ruleImplementationForm/FiltersField.js

@@ -2,6 +2,7 @@ import _ from "lodash/fp";
 import React, { useState } from "react";
 import PropTypes from "prop-types";
 import { getStructureFieldOptionValue } from "@truedat/dd/selectors/getStructureFieldOptionValue";
+import ReferenceDatasetFieldSelector from "@truedat/dd/components/ReferenceDatasetFieldSelector";
 import { getStructureFields } from "../../selectors/getStructureFields";
 import DateField from "./DateField";
 import DateTimeField from "./DateTimeField";
@@ -167,6 +168,19 @@ export const FiltersField = ({
           )}
         </>
       );
+
+    case "reference_dataset_field":
+      return (
+        <ReferenceDatasetFieldSelector
+          onChange={onChange}
+          /* value could be null if the reference dataset dropdown is
+           * selected but not the one for the its field.
+           */
+          referenceDataset={value?.referenceDataset}
+          header={value?.name}
+        />
+      );
+
     case "string_list":
       return (
         <StringListField

package/src/components/ruleImplementationForm/FiltersFormGroup.js

@@ -129,7 +129,13 @@ export const FiltersFormGroup = ({
       {!_.isEmpty(clause?.operator) &&
         _.range(0, clause?.operator?.arity || 1).map((nth_value) => (
           <Form.Field
-            width={clause?.operator?.arity === 2 ? 4 : 9}
+            width={
+              clause?.operator?.arity != null
+                ? clause?.operator?.arity === 2
+                  ? 4
+                  : 9
+                : 4
+            }
             key={`value_${nth_value}`}
           >
             <FiltersField

package/src/components/ruleImplementationForm/FiltersGrid.js

@@ -84,6 +84,8 @@ export const FiltersGrid = ({
           return composeFieldValue(v);
         })(value),
       };
+    } else if (value_type === "reference_dataset_field") {
+      return value;
     } else {
       return { raw: value };
     }

package/src/components/ruleImplementationForm/NumberField.js

@@ -3,7 +3,7 @@ import PropTypes from "prop-types";
 import React from "react";
 import { Form } from "semantic-ui-react";
 
-const targetValue = e => {
+const targetValue = (e) => {
   const target = _.path("target.value")(e);
   return _.isNil(target) || target === "" ? target : parseFloat(target);
 };
@@ -17,7 +17,7 @@ export const NumberField = ({ name, label, value, onChange }) => {
       control="input"
       value={raw}
       name={name}
-      onChange={e => onChange(e, targetValue(e))}
+      onChange={(e) => onChange(e, targetValue(e))}
     />
   );
 };
@@ -26,7 +26,7 @@ NumberField.propTypes = {
   name: PropTypes.string,
   label: PropTypes.string,
   value: PropTypes.number,
-  onChange: PropTypes.func
+  onChange: PropTypes.func,
 };
 
 export default NumberField;

package/src/components/ruleImplementationForm/RuleImplementationForm.js

@@ -216,11 +216,33 @@ export const RuleImplementationForm = ({
         !emptyAttr(attr, item) &&
         validArity(attr, item) &&
         nestedValid(attr, item)
-    )(attrs);
+    )(attrs) && isValidValue(attrs, item?.value);
 
   const mandatoryAttr = (attr, item) =>
     attr === "population" || !_.isNil(_.path(attr)(item));
 
+  /* Some operators have no value (for example, "UNIQUE")
+   * In this case, value will be null and that is fine.
+   * Other operators will fill value as an array of one or
+   * two elements (two in case of "between values", for example)
+   * If value is an array it has to contain something
+   * other than null. Also check for incomplete reference dataset field
+   * (reference dataset selected but not its field).
+   */
+  const isValidValue = (attrs, value) =>
+    attrs.includes("value") && Array.isArray(value)
+      ? _.every((val) => val != null && isValidReferenceDatasetField(val))(
+          value
+        )
+      : true;
+
+  const isValidReferenceDatasetField = (value) =>
+    value?.type === "reference_dataset_field"
+      ? value?.name != null &&
+        value?.referenceDataset?.id != null &&
+        value?.referenceDataset?.name != null
+      : true;
+
   const emptyAttr = (attr, item) =>
     attr === "operator" ? _.isEmpty(_.prop(attr)(item)) : false;
 

package/src/components/ruleImplementationForm/__tests__/__snapshots__/FiltersFormGroup.spec.js.snap

@@ -60,7 +60,7 @@ exports[`<FiltersFormGroup /> matches the latest snapshot 1`] = `
       </div>
     </div>
     <div
-      class="nine wide field"
+      class="four wide field"
     />
   </div>
 </div>

package/src/components/ruleImplementationForm/__tests__/__snapshots__/FiltersGroup.spec.js.snap

@@ -67,7 +67,7 @@ exports[`<FiltersGroup /> matches the latest snapshot when siblings provided 1`]
           </div>
         </div>
         <div
-          class="nine wide field"
+          class="four wide field"
         />
       </div>
     </div>

package/src/components/ruleImplementationForm/__tests__/__snapshots__/ValueConditions.spec.js.snap

@@ -66,7 +66,7 @@ exports[`<ValueConditions /> matches the latest snapshot when siblings provided
           </div>
         </div>
         <div
-          class="nine wide field"
+          class="four wide field"
         />
       </div>
     </div>

package/src/messages/en.js

@@ -164,6 +164,8 @@ export default {
   "quality_result.under_goal": "Under goal",
   "quality_result.under_minimum": "Under threshold",
   "quality_result.under_minumum": "Under threshold",
+  "referenceDataset.dropdown.placeholder": "Reference dataset",
+  "referenceDatasetFields.dropdown.placeholder": "Reference dataset fields",
   "remediation": "Remediation plan",
   "remediation.actions.create": "Create Remediation Plan",
   "remediation.actions.delete.confirmation.content":
@@ -432,11 +434,13 @@ export default {
   "ruleImplementation.operator.not_in_list.string_list": "is not in the list",
   "ruleImplementation.operator.not_references": "is not referenced",
   "ruleImplementation.operator.not_references.field": "is not referenced",
+  "ruleImplementation.operator.not_references.reference_dataset_field": "not in Reference dataset",
   "ruleImplementation.operator.number_of_decimals": "has a number of decimals",
   "ruleImplementation.operator.number_of_decimals.number":
     "has a number of decimals",
   "ruleImplementation.operator.references": "referenced in",
   "ruleImplementation.operator.references.field": "referenced in field",
+  "ruleImplementation.operator.references.reference_dataset_field": "in Reference dataset",
   "ruleImplementation.operator.regex_format": "matches regular expression",
   "ruleImplementation.operator.regex_format.string":
     "matches regular expression",

package/src/messages/es.js

@@ -168,6 +168,8 @@ export default {
   "quality_result.under_goal": "Por debajo del objetivo",
   "quality_result.under_minimum": "Por debajo del umbral",
   "quality_result.under_minumum": "Por debajo del umbral",
+  "referenceDataset.dropdown.placeholder": "Dato de referencia",
+  "referenceDatasetFields.dropdown.placeholder": "Campo de dato de referencia",
   "remediation": "Plan de remediación",
   "remediation.actions.create": "Crear plan de remediación",
   "remediation.actions.delete.confirmation.content":
@@ -442,10 +444,12 @@ export default {
   "ruleImplementation.operator.not_in_list.string_list": "no está en la lista",
   "ruleImplementation.operator.not_references": "no está referenciado en",
   "ruleImplementation.operator.not_references.field": "no referenciado en",
+  "ruleImplementation.operator.not_references.reference_dataset_field": "no en Datos de Referencia",
   "ruleImplementation.operator.number_of_decimals": "tiene nº decimales",
   "ruleImplementation.operator.number_of_decimals.number": "tiene nº decimales",
   "ruleImplementation.operator.references": "referenciado en",
   "ruleImplementation.operator.references.field": "referenciado en",
+  "ruleImplementation.operator.references.reference_dataset_field": "en Datos de Referencia",
   "ruleImplementation.operator.regex_format": "cumple la expresión regular",
   "ruleImplementation.operator.regex_format.string":
     "cumple la expresión regular",