@truedat/dq 4.28.5 → 4.29.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [4.28.8] 2021-09-20
+
+### Added
+
+- [TD-3970] Rule implementation support for casting fields as other types
+
 ## [4.28.3] 2021-09-14
 
 ### Fixed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@truedat/dq",
-  "version": "4.28.5",
+  "version": "4.29.0",
   "description": "Truedat Web Data Quality Module",
   "sideEffects": false,
   "jsnext:main": "src/index.js",
@@ -31,7 +31,7 @@
     "@babel/plugin-transform-modules-commonjs": "^7.15.0",
     "@babel/preset-env": "^7.15.0",
     "@babel/preset-react": "^7.14.5",
-    "@truedat/test": "4.28.5",
+    "@truedat/test": "4.29.0",
     "babel-jest": "^27.0.6",
     "babel-plugin-dynamic-import-node": "^2.3.3",
     "babel-plugin-lodash": "^3.3.4",
@@ -82,8 +82,8 @@
   },
   "dependencies": {
     "@apollo/client": "^3.4.10",
-    "@truedat/core": "4.28.5",
-    "@truedat/df": "4.28.5",
+    "@truedat/core": "4.29.0",
+    "@truedat/df": "4.29.0",
     "axios": "^0.19.2",
     "graphql": "^15.5.3",
     "path-to-regexp": "^1.7.0",
@@ -103,5 +103,5 @@
     "react-dom": ">= 16.8.6 < 17",
     "semantic-ui-react": ">= 0.88.2 < 2.1"
   },
-  "gitHead": "e004c6a673d6824a4807e4378b6e5c83e947c992"
+  "gitHead": "42028609748cd7f1686ea783264a9cdfa7b0ca8f"
 }

package/src/components/ImplementationSummary.js

@@ -11,7 +11,7 @@ const defaults = [
   "implementationKey",
   "dataset",
   "population",
-  "validations"
+  "validations",
 ];
 
 const concatValues = (values, link) => _.join(link)(values);
@@ -36,34 +36,35 @@ const FormattedLink = ({ value, operator = {} }) =>
 
 FormattedLink.propTypes = {
   value: PropTypes.object,
-  operator: PropTypes.object
+  operator: PropTypes.object,
 };
 
 const path = ({ name, path = [] }) => concatValues([...path, name], " > ");
 
-const empty = rows => rows && _.every(r => _.isEmpty(r) || _.isNil(r))(rows);
+const empty = (rows) =>
+  rows && _.every((r) => _.isEmpty(r) || _.isNil(r))(rows);
 
-const nilOrEmpty = v => _.isNil(v) || v === "";
+const nilOrEmpty = (v) => _.isNil(v) || v === "";
 
 const filterNilOrEmpties = (values, keys) =>
-  _.filter(v => _.every(k => !nilOrEmpty(_.prop(k)(v.kv)))(keys))(values);
+  _.filter((v) => _.every((k) => !nilOrEmpty(_.prop(k)(v.kv)))(keys))(values);
 
-const valuesFromKeys = (values, keys, optionalKeys) =>
+const valuesFromKeys = (values, keys, optionalKeys, value_modifier) =>
   _.flow(
-    _.map(v => {
+    _.map((v) => {
       return { kv: _.pick(keys)(v), optv: _.pick(optionalKeys)(v) };
     }),
-    v => filterNilOrEmpties(v, keys),
-    _.map(v => {
-      return { ...v.kv, ...v.optv };
+    (v) => filterNilOrEmpties(v, keys),
+    _.map.convert({ cap: false })((v, i) => {
+      return { ...v.kv, ...v.optv, modifier: _.nth(i)(value_modifier) };
     })
   )(values);
 
-export const getValues = ({ value = [], operator = {} }) => {
+export const getValues = ({ value = [], operator = {}, value_modifier }) => {
   const defaultOrValue = value || [];
   return _.propEq("value_type", "field")(operator)
-    ? valuesFromKeys(defaultOrValue, ["id", "name"], ["path"])
-    : valuesFromKeys(defaultOrValue, ["raw"], []);
+    ? valuesFromKeys(defaultOrValue, ["id", "name"], ["path"], value_modifier)
+    : valuesFromKeys(defaultOrValue, ["raw"], [], value_modifier);
 };
 
 const GeneralSummary = ({ executable, implementationKey }) => (
@@ -102,7 +103,7 @@ const GeneralSummary = ({ executable, implementationKey }) => (
 
 GeneralSummary.propTypes = {
   executable: PropTypes.bool,
-  implementationKey: PropTypes.string
+  implementationKey: PropTypes.string,
 };
 
 const DatasetSummary = ({ rows }) =>
@@ -129,7 +130,7 @@ const DatasetSummary = ({ rows }) =>
                 <Table.Cell key={i}>
                   <Link
                     to={linkTo.STRUCTURE({
-                      id: _.pathOr("", "structure.id")(row)
+                      id: _.pathOr("", "structure.id")(row),
                     })}
                   >
                     <span key={i} className="highlighted">{`"${path(
@@ -146,7 +147,7 @@ const DatasetSummary = ({ rows }) =>
   );
 
 DatasetSummary.propTypes = {
-  rows: PropTypes.array
+  rows: PropTypes.array,
 };
 
 const OperatorMessage = ({ operator }) =>
@@ -155,12 +156,12 @@ const OperatorMessage = ({ operator }) =>
       id={`ruleImplementation.operator.${_.prop("name")(operator)}`}
       defaultMessage={_.prop("name")(operator)}
     >
-      {message => <span className="highlighted">{`"${message}"`}</span>}
+      {(message) => <span className="highlighted">{`"${message}"`}</span>}
     </FormattedMessage>
   ) : null;
 
 OperatorMessage.propTypes = {
-  operator: PropTypes.object
+  operator: PropTypes.object,
 };
 
 const ConditionCell = ({ row }) => {
@@ -170,16 +171,37 @@ const ConditionCell = ({ row }) => {
     <Table.Row>
       <Table.Cell width={5}>
         {_.has("structure")(row) && (
-          <Link
-            to={linkTo.STRUCTURE({
-              id: _.path("structure.id")(row)
-            })}
-          >
-            <span className="highlighted">{`"${_.pathOr(
-              "",
-              "structure.name"
-            )(row)}"`}</span>
-          </Link>
+          <>
+            <Link
+              to={linkTo.STRUCTURE({
+                id: _.path("structure.id")(row),
+              })}
+            >
+              <span className="highlighted">{`"${_.pathOr(
+                "",
+                "structure.name"
+              )(row)}"`}</span>
+            </Link>
+            {row?.modifier && (
+              <>
+                <div className="smaller">
+                  <FormattedMessage
+                    id={`filtersGrid.field.modifier.${row.modifier.name}`}
+                    defaultMessage={row.modifier.name}
+                  />
+                </div>
+                {_.flow(
+                  _.prop("modifier.params"),
+                  _.toPairs,
+                  _.map(([key, value]) => (
+                    <div key={key} className="smaller">
+                      {value}
+                    </div>
+                  ))
+                )(row)}
+              </>
+            )}
+          </>
         )}
       </Table.Cell>
       <Table.Cell width={5}>
@@ -195,6 +217,25 @@ const ConditionCell = ({ row }) => {
                 <FormattedMessage id="summary.link.and" />{" "}
               </span>
             )}
+            {value?.modifier && (
+              <>
+                <div className="smaller">
+                  <FormattedMessage
+                    id={`filtersGrid.field.modifier.${value.modifier.name}`}
+                    defaultMessage={value.modifier.name}
+                  />
+                </div>
+                {_.flow(
+                  _.prop("modifier.params"),
+                  _.toPairs,
+                  _.map(([key, value]) => (
+                    <div key={key} className="smaller">
+                      {value}
+                    </div>
+                  ))
+                )(value)}
+              </>
+            )}
           </Fragment>
         ))(values)}
       </Table.Cell>
@@ -203,7 +244,7 @@ const ConditionCell = ({ row }) => {
 };
 
 ConditionCell.propTypes = {
-  row: PropTypes.object
+  row: PropTypes.object,
 };
 
 const ConditionSummary = ({ rows, type, icon }) =>
@@ -243,18 +284,13 @@ const ConditionSummary = ({ rows, type, icon }) =>
 ConditionSummary.propTypes = {
   icon: PropTypes.string,
   rows: PropTypes.array,
-  type: PropTypes.string
+  type: PropTypes.string,
 };
 
 export const ImplementationSummary = ({ ruleImplementation, activeSteps }) => {
   const steps = _.isEmpty(activeSteps) ? defaults : activeSteps;
-  const {
-    executable,
-    implementationKey,
-    dataset,
-    population,
-    validations
-  } = _.pick(steps)(ruleImplementation);
+  const { executable, implementationKey, dataset, population, validations } =
+    _.pick(steps)(ruleImplementation);
   return (
     <>
       {(_.includes("implementationKey")(steps) ||
@@ -283,7 +319,7 @@ export const ImplementationSummary = ({ ruleImplementation, activeSteps }) => {
 
 ImplementationSummary.propTypes = {
   activeSteps: PropTypes.array,
-  ruleImplementation: PropTypes.object
+  ruleImplementation: PropTypes.object,
 };
 
 export default ImplementationSummary;

package/src/components/NewRuleImplementation.js

@@ -30,6 +30,18 @@ const updateDatasetKey = (data) =>
 
 const updateConditionValue = (acc, value, key) => {
   if (key === "structure") return _.set(key, _.pick(["id"])(value))(acc);
+  if (key === "modifier")
+    return _.set(
+      key,
+      _.isNil(value) ? null : _.pick(["name", "params"])(value)
+    )(acc);
+  if (key === "value_modifier")
+    return _.set(
+      key,
+      _.every(_.isNil)(value)
+        ? []
+        : _.map((v) => _.pick(["name", "params"])(v))(value)
+    )(acc);
   if (key === "operator")
     return _.set(
       key,
@@ -63,7 +75,16 @@ const datasetAttributes = (dataset) =>
 
 const conditionAttributes = (condition) =>
   _.flow(
-    _.map(_.pickAll(["structure", "operator", "value", "population"])),
+    _.map(
+      _.pickAll([
+        "structure",
+        "modifier",
+        "operator",
+        "value",
+        "value_modifier",
+        "population",
+      ])
+    ),
     _.map(updateConditionKey)
   )(condition);
 
@@ -72,6 +93,8 @@ const fieldTypeFromStructure = (row, structures, operators, scope) => {
   const field = _.find(_.propEq("data_structure_id", structure_id))(structures);
   const field_type = getFieldType(field);
   const structure = _.prop("structure")(row);
+  const modifier = _.prop("modifier")(row);
+  const value_modifier = _.prop("value_modifier")(row);
   const operator = enrichOperator(
     operators,
     scope,
@@ -80,7 +103,9 @@ const fieldTypeFromStructure = (row, structures, operators, scope) => {
   );
   const updatedRow = {
     ...row,
-    operator: operator,
+    operator,
+    modifier,
+    value_modifier,
     structure: { ...structure, field_type },
   };
   return updatedRow;
@@ -116,7 +141,7 @@ const addFieldType = (
   const rows = _.map((row) =>
     rowFieldType(row, fields, siblings, operators, scope)
   )(_.prop(implementationPropertyName)(ruleImplementationProps));
-  return _.isEmpty(rows) ? [{}] : rows;
+  return _.isEmpty(rows) ? [] : rows;
 };
 
 const enrichOperator = (operators, scope, field_type, operator) => {
@@ -201,7 +226,7 @@ export const NewRuleImplementation = ({
           implementationKey: "",
           implementationType: "default",
           dataset: [{}],
-          population: [{}],
+          population: [],
           validations: [{}],
           rawContent: {
             database: "",
@@ -253,7 +278,7 @@ export const NewRuleImplementation = ({
       ...ruleImplementation,
       dataset,
       population: _.isEmpty(population_within_dataset)
-        ? [{}]
+        ? []
         : population_within_dataset,
       validations: _.isEmpty(validations_within_dataset)
         ? [{}]
@@ -383,11 +408,8 @@ export const NewRuleImplementation = ({
             {ruleImplementation.implementationType != "raw" && (
               <RuleImplementationForm
                 ruleImplementation={ruleImplementation}
-                structures={_.prop("dataset")(ruleImplementation)}
                 setStructures={setDataset}
-                population={_.prop("population")(ruleImplementation)}
                 setPopulation={setPopulation}
-                validations={_.prop("validations")(ruleImplementation)}
                 setValidations={setValidations}
                 operators={operators}
                 setImplementationKey={setImplementationKey}

package/src/components/__tests__/__snapshots__/NewRuleImplementation.spec.js.snap

@@ -31,11 +31,6 @@ exports[`<NewRuleImplementation /> matches the latest snapshot 1`] = `
         <Connect(RuleImplementationForm)
           handleSubmit={[Function]}
           onChange={[Function]}
-          population={
-            Array [
-              Object {},
-            ]
-          }
           ruleImplementation={
             Object {
               "dataset": Array [
@@ -46,9 +41,7 @@ exports[`<NewRuleImplementation /> matches the latest snapshot 1`] = `
               "executable": true,
               "implementationKey": "",
               "implementationType": "default",
-              "population": Array [
-                Object {},
-              ],
+              "population": Array [],
               "rawContent": Object {
                 "database": "",
                 "dataset": "",
@@ -65,16 +58,6 @@ exports[`<NewRuleImplementation /> matches the latest snapshot 1`] = `
           setPopulation={[Function]}
           setStructures={[Function]}
           setValidations={[Function]}
-          structures={
-            Array [
-              Object {},
-            ]
-          }
-          validations={
-            Array [
-              Object {},
-            ]
-          }
         />
       </GridRow>
     </GridColumn>
@@ -92,9 +75,7 @@ exports[`<NewRuleImplementation /> matches the latest snapshot 1`] = `
             "executable": true,
             "implementationKey": "",
             "implementationType": "default",
-            "population": Array [
-              Object {},
-            ],
+            "population": Array [],
             "rawContent": Object {
               "database": "",
               "dataset": "",

package/src/components/ruleImplementationForm/FieldModifier.js

@@ -0,0 +1,51 @@
+import { getOr } from "lodash/fp";
+import React from "react";
+import { useIntl } from "react-intl";
+import { Form, Icon } from "semantic-ui-react";
+
+export default function FieldModifier({
+  modifier,
+  value,
+  onChange,
+  removable = false,
+}) {
+  const { formatMessage } = useIntl();
+
+  return (
+    <div className={"flex-row-align-end"}>
+      <Form.Field>
+        <label>
+          {formatMessage({ id: `filtersGrid.field.modifier.${modifier.name}` })}
+        </label>
+        {getOr(
+          [],
+          "params"
+        )(modifier).map((param, key) => (
+          <input
+            key={key}
+            placeholder={formatMessage({
+              id: `filtersGrid.field.modifier.${modifier.name}.${param.name}`,
+            })}
+            value={getOr("", `params.${param.name}`)(value)}
+            onChange={({ target }) => {
+              onChange({
+                ...value,
+                params: {
+                  ...getOr({}, "params")(value),
+                  [param.name]: target.value,
+                },
+              });
+            }}
+          />
+        ))}
+      </Form.Field>
+      {removable && (
+        <Icon
+          name="x"
+          className="selectable force-margin-bottom"
+          onClick={() => onChange(null)}
+        />
+      )}
+    </div>
+  );
+}

package/src/components/ruleImplementationForm/FiltersField.js

@@ -10,6 +10,7 @@ import FixedListField from "./FixedListField";
 import NumberField from "./NumberField";
 import StringField from "./StringField";
 import StringListField from "./StringListField";
+import FieldModifier from "./FieldModifier";
 
 const StructureFieldsDropdown = React.lazy(() =>
   import("@truedat/dd/components/StructureFieldsDropdown")
@@ -41,12 +42,16 @@ export const FiltersField = ({
   operator,
   fieldType,
   value,
+  modifier,
   name,
   onChange,
+  typeCastModifiers,
 }) => {
   const [active, setActive] = useState(false);
   const { value_type, value_type_filter, fixed_values } = operator;
 
+  const modifierDef = _.find({ name: modifier?.name })(typeCastModifiers);
+
   switch (value_type) {
     case "string":
       return !fixed_values ? (
@@ -109,17 +114,31 @@ export const FiltersField = ({
           fieldWidth={15}
         />
       ) : (
-        <StructureFieldsDropdown
-          label={label}
-          inline={false}
-          parentStructures={parentStructures}
-          structureFields={structureFields}
-          filters={{ field_type: [fieldType] }}
-          onSelectField={(value) => {
-            onChange(null, _.pick(["data_structure_id", "name"])(value));
-          }}
-          value={_.prop("id")(value)}
-        />
+        <>
+          <StructureFieldsDropdown
+            label={label}
+            inline={false}
+            parentStructures={parentStructures}
+            structureFields={structureFields}
+            typeCastModifiers={typeCastModifiers}
+            filters={{ field_type: [fieldType] }}
+            onSelectField={(value, modifier) =>
+              onChange(
+                null,
+                _.pick(["data_structure_id", "name"])(value),
+                modifier ? { name: modifier.name } : null
+              )
+            }
+            value={_.prop("id")(value)}
+          />
+          {modifier && (
+            <FieldModifier
+              modifier={modifierDef}
+              value={modifier}
+              onChange={(modifier) => onChange(null, value, modifier)}
+            />
+          )}
+        </>
       );
     case "string_list":
       return (
@@ -144,6 +163,8 @@ FiltersField.propTypes = {
   value: PropTypes.string,
   name: PropTypes.string,
   onChange: PropTypes.func,
+  modifier: PropTypes.object,
+  typeCastModifiers: PropTypes.array,
 };
 
 const mapStateToProps = (state) => ({