@truedat/dq 4.41.3 → 4.42.1

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## [4.42.1] 2022-04-12
+
+# Changed
+
+- [TD-4536] Support rule implementation with multiple populations
+
+## [4.41.4] 2022-04-04
+
+### Changed
+
+- [TD-4450] Sources in implementation form are now fetched using GraphQL API
+  instead of from global state
+
 ## [4.40.13] 2022-03-21
 
 ### Fixed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@truedat/dq",
-  "version": "4.41.3",
+  "version": "4.42.1",
   "description": "Truedat Web Data Quality Module",
   "sideEffects": false,
   "jsnext:main": "src/index.js",
@@ -31,7 +31,7 @@
     "@babel/plugin-transform-modules-commonjs": "^7.15.0",
     "@babel/preset-env": "^7.15.0",
     "@babel/preset-react": "^7.14.5",
-    "@truedat/test": "4.41.2",
+    "@truedat/test": "4.42.0",
     "babel-jest": "^27.0.6",
     "babel-plugin-dynamic-import-node": "^2.3.3",
     "babel-plugin-lodash": "^3.3.4",
@@ -82,8 +82,8 @@
   },
   "dependencies": {
     "@apollo/client": "^3.4.10",
-    "@truedat/core": "4.41.3",
-    "@truedat/df": "4.41.3",
+    "@truedat/core": "4.42.0",
+    "@truedat/df": "4.42.0",
     "axios": "^0.19.2",
     "graphql": "^15.5.3",
     "path-to-regexp": "^1.7.0",
@@ -103,5 +103,5 @@
     "react-dom": ">= 16.8.6 < 17",
     "semantic-ui-react": ">= 0.88.2 < 2.1"
   },
-  "gitHead": "946829496288ea4ff05b3029b37d8c8d16393127"
+  "gitHead": "0661831301c7357a78ff0d5f72a7bb2da8d8dd72"
 }

package/src/components/ConditionSummary.js

@@ -163,41 +163,55 @@ ConditionCell.propTypes = {
   alias: PropTypes.array,
 };
 
-export const ConditionSummary = ({ rows, type, icon, alias }) =>
-  empty(rows) ? null : (
+export const ConditionSummary = ({ rows, type, icon, alias }) => {
+  const areMultidimensionalRows =
+    rows.filter(Array.isArray).length == rows.length;
+  const wrappedRows = areMultidimensionalRows ? rows : [rows];
+  return empty(rows) ? null : (
     <>
       <Header as="h3">
         <Icon name={icon} size="small" />
         <Header.Content>
-          <FormattedMessage
-            id={`ruleImplementations.summary.headers.${type}`}
-          />
+          {areMultidimensionalRows ? (
+            <FormattedMessage
+              id={`ruleImplementations.summary.headers.${type}s`}
+            />
+          ) : (
+            <FormattedMessage
+              id={`ruleImplementations.summary.headers.${type}`}
+            />
+          )}
         </Header.Content>
       </Header>
-      <Segment>
-        <Table basic="very">
-          <Table.Header>
-            <Table.Row>
-              <Table.HeaderCell>
-                <FormattedMessage id={`ruleImplementation.summary.field`} />
-              </Table.HeaderCell>
-              <Table.HeaderCell>
-                <FormattedMessage id={`ruleImplementation.summary.operator`} />
-              </Table.HeaderCell>
-              <Table.HeaderCell>
-                <FormattedMessage id={`ruleImplementation.summary.values`} />
-              </Table.HeaderCell>
-            </Table.Row>
-          </Table.Header>
-          <Table.Body>
-            {rows.map((row, i) => (
-              <ConditionCell key={i} row={row} alias={alias} />
-            ))}
-          </Table.Body>
-        </Table>
-      </Segment>
+      {wrappedRows.map((rows, i) => (
+        <Segment key={i}>
+          <Table basic="very">
+            <Table.Header>
+              <Table.Row>
+                <Table.HeaderCell>
+                  <FormattedMessage id={`ruleImplementation.summary.field`} />
+                </Table.HeaderCell>
+                <Table.HeaderCell>
+                  <FormattedMessage
+                    id={`ruleImplementation.summary.operator`}
+                  />
+                </Table.HeaderCell>
+                <Table.HeaderCell>
+                  <FormattedMessage id={`ruleImplementation.summary.values`} />
+                </Table.HeaderCell>
+              </Table.Row>
+            </Table.Header>
+            <Table.Body>
+              {rows.map((row, i) => (
+                <ConditionCell key={i} row={row} alias={alias} />
+              ))}
+            </Table.Body>
+          </Table>
+        </Segment>
+      ))}
     </>
   );
+};
 
 ConditionSummary.propTypes = {
   icon: PropTypes.string,

package/src/components/ImplementationSummary.js

@@ -13,7 +13,7 @@ const defaults = [
   "executable",
   "implementationKey",
   "dataset",
-  "population",
+  "populations",
   "validations",
 ];
 
@@ -130,7 +130,7 @@ DatasetSummary.propTypes = {
 
 export const ImplementationSummary = ({ ruleImplementation, activeSteps }) => {
   const steps = _.isEmpty(activeSteps) ? defaults : activeSteps;
-  const { dataset, population, validations } =
+  const { dataset, populations, validations } =
     _.pick(steps)(ruleImplementation);
   const alias = _.map(_.propOr({}, "alias"))(dataset);
   return (
@@ -142,14 +142,14 @@ export const ImplementationSummary = ({ ruleImplementation, activeSteps }) => {
       {dataset && _.includes("dataset")(steps) && (
         <DatasetSummary rows={dataset} alias={alias} />
       )}
-      {population && _.includes("population")(steps) && (
+      {populations && _.includes("populations")(steps) ? (
         <ConditionSummary
           type="population"
           icon="user"
-          rows={population}
+          rows={populations}
           alias={alias}
         />
-      )}
+      ) : null}
       {validations && _.includes("validations")(steps) && (
         <ConditionSummary
           type="validations"

package/src/components/NewRuleImplementation.js

@@ -4,6 +4,8 @@ import PropTypes from "prop-types";
 import { connect } from "react-redux";
 import { Button, Divider, Header, Grid } from "semantic-ui-react";
 import { FormattedMessage } from "react-intl";
+import { gql, useQuery } from "@apollo/client";
+import { Loading } from "@truedat/core/components";
 import { getFieldType } from "@truedat/core/services/fieldType";
 import { applyTemplate } from "@truedat/df/utils";
 import {
@@ -90,6 +92,9 @@ const conditionAttributes = (condition) =>
     _.map(updateConditionKey)
   )(condition);
 
+const populationsAttributes = (populations) =>
+  _.flow(_.map(conditionAttributes))(populations);
+
 const fieldTypeFromStructure = (row, structures, operators, scope) => {
   const structure_id = _.path("structure.id")(row);
   const field = _.find(_.propEq("data_structure_id", structure_id))(structures);
@@ -240,6 +245,7 @@ export const NewRuleImplementation = ({
   rule,
   ruleImplementationProps,
   sources,
+  sourcesLoading,
   structuresFields,
   structuresSiblings,
   operators,
@@ -250,6 +256,7 @@ export const NewRuleImplementation = ({
     _.propOr([{}], "dataset"),
     withDefaultAlias
   )(ruleImplementationProps);
+
   const [ruleImplementation, setRuleImplementation] = useState(
     edition
       ? {
@@ -262,17 +269,22 @@ export const NewRuleImplementation = ({
             ruleImplementationProps
           ),
           dataset: precalculatedDataset,
-          population: withConditionDefaultAlias(
-            addFieldType(
-              ruleImplementationProps,
-              structuresFields,
-              structuresSiblings,
-              "population",
-              "population",
-              operators
-            ),
-            precalculatedDataset
-          ),
+          populations: _.flow(
+            _.pathOr([], "populations"),
+            _.map((population) =>
+              withConditionDefaultAlias(
+                addFieldType(
+                  { population: population },
+                  structuresFields,
+                  structuresSiblings,
+                  "population",
+                  "population",
+                  operators
+                ),
+                precalculatedDataset
+              )
+            )
+          )(ruleImplementationProps),
           validations: withConditionDefaultAlias(
             addFieldType(
               ruleImplementationProps,
@@ -299,6 +311,7 @@ export const NewRuleImplementation = ({
           implementationType: "default",
           dataset: [{}],
           population: [],
+          populations: [],
           validations: [{}],
           rawContent: {
             database: "",
@@ -392,6 +405,23 @@ export const NewRuleImplementation = ({
     setRuleImplementation({ ...ruleImplementation, population });
   };
 
+  const addPopulation = () => {
+    const populations = _.concat(
+      _.pathOr([], "populations")(ruleImplementation)
+    )([[{}]]);
+    setRuleImplementation({ ...ruleImplementation, populations: populations });
+  };
+
+  const setPopulations = (population, index) => {
+    const populations = _.flow(
+      _.pathOr([], "populations"),
+      _.update(`[${index}]`, (_previousPopulation) => population),
+      _.reject(_.isEmpty)
+    )(ruleImplementation);
+
+    setRuleImplementation({ ...ruleImplementation, populations: populations });
+  };
+
   const setValidations = (validations) => {
     setRuleImplementation({ ...ruleImplementation, validations });
   };
@@ -413,8 +443,8 @@ export const NewRuleImplementation = ({
   }, [template]);
 
   const doSubmit = () => {
-    const population = _.filter(_.negate(_.isEmpty))(
-      ruleImplementation.population
+    const populations = _.filter(_.negate(_.isEmpty))(
+      ruleImplementation.populations
     );
     const validations = _.filter(_.negate(_.isEmpty))(
       ruleImplementation.validations
@@ -449,7 +479,7 @@ export const NewRuleImplementation = ({
         : {
             executable: ruleImplementation.executable,
             dataset: datasetAttributes(_.prop("dataset")(ruleImplementation)),
-            population: conditionAttributes(population),
+            populations: populationsAttributes(populations),
             validations: conditionAttributes(validations),
             implementation_key: _.prop("implementationKey")(ruleImplementation),
             implementation_type:
@@ -522,12 +552,15 @@ export const NewRuleImplementation = ({
                 ruleImplementation={ruleImplementation}
                 onChange={onChange}
                 handleSubmit={doSubmit}
+                sources={sources}
+                sourcesLoading={sourcesLoading}
               />
             ) : (
               <RuleImplementationForm
                 ruleImplementation={ruleImplementation}
                 setStructures={setDataset}
-                setPopulation={setPopulation}
+                addPopulation={addPopulation}
+                setPopulations={setPopulations}
                 setValidations={setValidations}
                 operators={operators}
                 setImplementationKey={setImplementationKey}
@@ -561,6 +594,33 @@ NewRuleImplementation.propTypes = {
   template: PropTypes.object,
   applyTemplate: PropTypes.func,
   sources: PropTypes.array,
+  sourcesLoading: PropTypes.bool,
+};
+
+export const SOURCES_WITH_CONFIG = gql`
+  query SOURCES_WITH_CONFIG($jobTypes: String) {
+    sources(jobTypes: $jobTypes) {
+      id
+      externalId
+      config
+    }
+  }
+`;
+
+export const NewRuleImplementationLoader = (props) => {
+  const { data, error, loading } = useQuery(SOURCES_WITH_CONFIG, {
+    variables: { jobTypes: "quality" },
+  });
+  if (loading) return <Loading />;
+  if (error) return null;
+  const sources = data?.sources || [];
+  return (
+    <NewRuleImplementation
+      sources={sources}
+      sourcesLoading={loading}
+      {...props}
+    />
+  );
 };
 
 const mapStateToProps = (state) => ({
@@ -570,7 +630,6 @@ const mapStateToProps = (state) => ({
     raw_content: state.ruleImplementationRaw,
   },
   ruleImplementationRaw: state.ruleImplementationRaw,
-  sources: state.sources,
   structuresFields: state.structuresFields,
   structuresSiblings: state.structuresSiblings,
   canManageRaw: _.prop("manage_raw_quality_rule_implementations")(
@@ -584,4 +643,4 @@ const mapStateToProps = (state) => ({
 export default connect(mapStateToProps, {
   createRuleImplementation,
   updateRuleImplementation,
-})(NewRuleImplementation);
+})(NewRuleImplementationLoader);

package/src/components/RuleImplementationProperties.js

@@ -12,7 +12,7 @@ const DynamicFormViewer = React.lazy(() =>
   import("@truedat/df/components/DynamicFormViewer")
 );
 
-const summarySteps = ["dataset", "population", "validations"];
+const summarySteps = ["dataset", "populations", "validations"];
 
 export const RuleImplementationProperties = ({
   ruleImplementation,
@@ -74,6 +74,7 @@ const mapStateToProps = ({
 }) => ({
   ruleImplementation: _.pick([
     ...summarySteps,
+    "populations",
     "executable",
     "event_type",
     "event_message",

package/src/components/__test_samples__/NewRuleImplementationProps.js

@@ -130,56 +130,58 @@ export const newRuleImplementationProps = {
     id: 852,
     implementation_key: "impl_example_1",
     implementation_type: "default",
-    population: [
-      {
-        operator: {
-          name: "eq",
-          value_type: "field",
-        },
-        structure: {
-          external_id:
-            "https://glue.eu-west-1.amazonaws.com/#/576759405678/cepsa-demo-s3/02_es_provincias/descripcion",
-          id: 4817544,
-          metadata: {
-            account: "576759405678",
-            data_type_class: "string",
-            database: "cepsa-demo-s3",
-            order: "2.0",
-            region: "eu-west-1",
-            tablename: "02_es_provincias",
-            type: "string",
-          },
-          name: "descripcion",
-          path: ["cepsa-demo-s3", "02_es_provincias"],
-          system: {
-            external_id: "glue",
-            id: 94,
-            name: "Glue",
+    populations: [
+      [
+        {
+          operator: {
+            name: "eq",
+            value_type: "field",
           },
-          type: "Column",
-        },
-        value: [
-          {
+          structure: {
             external_id:
-              "/home/alberto/projects/connectors/td-connector-txt-files/data/########_D_PELAYO_POLIZAS_20201202033016_I.txt[Aux. Externo]",
-            id: 4814767,
+              "https://glue.eu-west-1.amazonaws.com/#/576759405678/cepsa-demo-s3/02_es_provincias/descripcion",
+            id: 4817544,
             metadata: {
-              order: "8",
+              account: "576759405678",
+              data_type_class: "string",
+              database: "cepsa-demo-s3",
+              order: "2.0",
+              region: "eu-west-1",
+              tablename: "02_es_provincias",
+              type: "string",
             },
-            name: "Aux. Externo",
-            path: [
-              "Area_Vida",
-              "########_D_PELAYO_POLIZAS_20201202033016_I.txt",
-            ],
+            name: "descripcion",
+            path: ["cepsa-demo-s3", "02_es_provincias"],
             system: {
-              external_id: "file_system",
-              id: 33,
-              name: "File System",
+              external_id: "glue",
+              id: 94,
+              name: "Glue",
             },
             type: "Column",
           },
-        ],
-      },
+          value: [
+            {
+              external_id:
+                "/home/alberto/projects/connectors/td-connector-txt-files/data/########_D_PELAYO_POLIZAS_20201202033016_I.txt[Aux. Externo]",
+              id: 4814767,
+              metadata: {
+                order: "8",
+              },
+              name: "Aux. Externo",
+              path: [
+                "Area_Vida",
+                "########_D_PELAYO_POLIZAS_20201202033016_I.txt",
+              ],
+              system: {
+                external_id: "file_system",
+                id: 33,
+                name: "File System",
+              },
+              type: "Column",
+            },
+          ],
+        },
+      ],
     ],
     results: [
       {

package/src/components/__tests__/ImplementationSummary.spec.js

@@ -12,7 +12,7 @@ const renderOpts = {
       "quality.threshold": "threshold",
       "quality.goal": "goal",
       "ruleImplementations.props.result_type.percentage": "percentage",
-
+      "ruleImplementations.summary.headers.populations": "populations",
       "ruleImplementation.summary.field": "Field",
       "ruleImplementation.summary.operator": "Operator",
       "ruleImplementation.summary.structure": "Structure",
@@ -46,7 +46,7 @@ const ruleImplementation = {
       },
     },
   ],
-  population: [{}],
+  populations: [[{}]],
   validations: [
     {
       structure: { field_type: "string", name: "Mes", id: 2598 },
@@ -67,7 +67,7 @@ describe("<ImplementationSummary />", () => {
       "implementationKey",
       "executable",
       "dataset",
-      "population",
+      "populations",
       "validations",
     ];
     const props = {
@@ -79,7 +79,7 @@ describe("<ImplementationSummary />", () => {
   });
 
   it("displays correctly validations values of list type", () => {
-    const steps = ["dataset", "population", "validations"];
+    const steps = ["dataset", "populations", "validations"];
     const props = {
       steps,
       ruleImplementation,

package/src/components/__tests__/__snapshots__/ImplementationSummary.spec.js.snap

@@ -144,6 +144,67 @@ exports[`<ImplementationSummary /> displays correctly validations values of list
       </tbody>
     </table>
   </div>
+  <h3
+    class="ui header"
+  >
+    <i
+      aria-hidden="true"
+      class="user small icon"
+    />
+    <div
+      class="content"
+    >
+      populations
+    </div>
+  </h3>
+  <div
+    class="ui segment"
+  >
+    <table
+      class="ui very basic table"
+    >
+      <thead
+        class=""
+      >
+        <tr
+          class=""
+        >
+          <th
+            class=""
+          >
+            Field
+          </th>
+          <th
+            class=""
+          >
+            Operator
+          </th>
+          <th
+            class=""
+          >
+            Values
+          </th>
+        </tr>
+      </thead>
+      <tbody
+        class=""
+      >
+        <tr
+          class=""
+        >
+          <td
+            class="five wide"
+          />
+          <td
+            class="five wide"
+          />
+          <td
+            class="five wide"
+          />
+        </tr>
+      </tbody>
+    </table>
+  </div>
   <h3
     class="ui header"
   >
@@ -264,8 +325,10 @@ exports[`<ImplementationSummary /> matches the latest snapshot 1`] = `
         "implementationKey": "ImplKey",
         "implementationType": "",
         "minimum": 10,
-        "population": Array [
-          Object {},
+        "populations": Array [
+          Array [
+            Object {},
+          ],
         ],
         "rawContent": Object {
           "dataset": "",
@@ -340,7 +403,9 @@ exports[`<ImplementationSummary /> matches the latest snapshot 1`] = `
     icon="user"
     rows={
       Array [
-        Object {},
+        Array [
+          Object {},
+        ],
       ]
     }
     type="population"