@truedat/dq 4.33.10 → 4.35.2

package/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## [4.35.2] 2021-12-20
+
+### Fixed
+
+- [TD-4380] Allow modifier in operator filter
+
+## [4.34.3] 2021-12-15
+
+### Added
+
+- [TD-4273] Upload `RuleResults`
+
+## [4.34.2] 2021-12-14
+
+### Added
+
+- [TD-4314] Add BulkLoad for `rules`
+- [TD-4301] Add BulkLoad for `implementations`
+
+## [4.34.0] 2021-12-02
+
+### Changed
+
+- [TD-4270] Move goal and threshold from Rule to Implementation
+- [TD-4295] Changed how implementation Information section is displayed
+
 ## [4.33.10] 2021-11-30
 
 - [TD-4306] Fix popups inside execution popup

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@truedat/dq",
-  "version": "4.33.10",
+  "version": "4.35.2",
   "description": "Truedat Web Data Quality Module",
   "sideEffects": false,
   "jsnext:main": "src/index.js",
@@ -31,7 +31,7 @@
     "@babel/plugin-transform-modules-commonjs": "^7.15.0",
     "@babel/preset-env": "^7.15.0",
     "@babel/preset-react": "^7.14.5",
-    "@truedat/test": "4.33.10",
+    "@truedat/test": "4.34.0",
     "babel-jest": "^27.0.6",
     "babel-plugin-dynamic-import-node": "^2.3.3",
     "babel-plugin-lodash": "^3.3.4",
@@ -82,8 +82,8 @@
   },
   "dependencies": {
     "@apollo/client": "^3.4.10",
-    "@truedat/core": "4.33.10",
-    "@truedat/df": "4.33.10",
+    "@truedat/core": "4.35.2",
+    "@truedat/df": "4.35.2",
     "axios": "^0.19.2",
     "graphql": "^15.5.3",
     "path-to-regexp": "^1.7.0",
@@ -103,5 +103,5 @@
     "react-dom": ">= 16.8.6 < 17",
     "semantic-ui-react": ">= 0.88.2 < 2.1"
   },
-  "gitHead": "b316406207a0b4621b8fffb33e6348712416b0c2"
+  "gitHead": "2d5ffbb16aaf62db662165ccd8088c65fcb6ec74"
 }

package/src/api.js

@@ -4,6 +4,7 @@ const API_EXECUTION_GROUPS = "/api/execution_groups";
 const API_RULE = "/api/rules/:id";
 const API_RULES = "/api/rules";
 const API_RULES_SEARCH = "/api/rules/search";
+const API_RULES_UPLOAD = "/api/rules/upload";
 const API_RULE_FILTERS_SEARCH = "/api/rule_filters/search";
 const API_RULE_IMPLEMENTATION = "/api/rule_implementations/:id";
 const API_RULE_IMPLEMENTATIONS = "/api/rule_implementations";
@@ -13,7 +14,9 @@ const API_RULE_IMPLEMENTATIONS_FROM_RULE =
 const API_RULE_IMPLEMENTATIONS_SEARCH = "/api/rule_implementations/search";
 const API_RULE_IMPLEMENTATION_FILTERS_SEARCH =
   "/api/rule_implementation_filters/search";
+const API_RULE_IMPLEMENTATIONS_UPLOAD = "/api/rule_implementations/upload";
 const API_RULE_RESULT = "/api/rule_results/:id";
+const API_RULE_RESULTS = "/api/rule_results";
 const API_SUBSCRIPTIONS_SEARCH = "/api/subscriptions/user/me/search";
 const API_SUBSCRIPTION = "/api/subscriptions/:id";
 const API_SUBSCRIPTIONS = "/api/subscriptions";
@@ -25,6 +28,7 @@ export {
   API_RULE,
   API_RULES,
   API_RULES_SEARCH,
+  API_RULES_UPLOAD,
   API_RULE_FILTERS_SEARCH,
   API_RULE_IMPLEMENTATION,
   API_RULE_IMPLEMENTATIONS,
@@ -32,8 +36,10 @@ export {
   API_RULE_IMPLEMENTATIONS_FROM_RULE,
   API_RULE_IMPLEMENTATIONS_SEARCH,
   API_RULE_IMPLEMENTATION_FILTERS_SEARCH,
+  API_RULE_IMPLEMENTATIONS_UPLOAD,
   API_RULE_RESULT,
+  API_RULE_RESULTS,
   API_SUBSCRIPTIONS_SEARCH,
   API_SUBSCRIPTION,
-  API_SUBSCRIPTIONS
+  API_SUBSCRIPTIONS,
 };

package/src/components/ConditionSummary.js

@@ -1,7 +1,7 @@
 import _ from "lodash/fp";
 import React, { Fragment } from "react";
 import PropTypes from "prop-types";
-import { Header, Icon, Table } from "semantic-ui-react";
+import { Header, Icon, Segment, Table } from "semantic-ui-react";
 import { Link } from "react-router-dom";
 import { FormattedMessage } from "react-intl";
 import { linkTo } from "@truedat/core/routes";
@@ -168,26 +168,28 @@ export const ConditionSummary = ({ rows, type, icon }) =>
           />
         </Header.Content>
       </Header>
-      <Table basic="very">
-        <Table.Header>
-          <Table.Row>
-            <Table.HeaderCell>
-              <FormattedMessage id={`ruleImplementation.summary.field`} />
-            </Table.HeaderCell>
-            <Table.HeaderCell>
-              <FormattedMessage id={`ruleImplementation.summary.operator`} />
-            </Table.HeaderCell>
-            <Table.HeaderCell>
-              <FormattedMessage id={`ruleImplementation.summary.values`} />
-            </Table.HeaderCell>
-          </Table.Row>
-        </Table.Header>
-        <Table.Body>
-          {rows.map((row, i) => (
-            <ConditionCell key={i} row={row} />
-          ))}
-        </Table.Body>
-      </Table>
+      <Segment>
+        <Table basic="very">
+          <Table.Header>
+            <Table.Row>
+              <Table.HeaderCell>
+                <FormattedMessage id={`ruleImplementation.summary.field`} />
+              </Table.HeaderCell>
+              <Table.HeaderCell>
+                <FormattedMessage id={`ruleImplementation.summary.operator`} />
+              </Table.HeaderCell>
+              <Table.HeaderCell>
+                <FormattedMessage id={`ruleImplementation.summary.values`} />
+              </Table.HeaderCell>
+            </Table.Row>
+          </Table.Header>
+          <Table.Body>
+            {rows.map((row, i) => (
+              <ConditionCell key={i} row={row} />
+            ))}
+          </Table.Body>
+        </Table>
+      </Segment>
     </>
   );
 

package/src/components/ExecutionDetails.js

@@ -8,7 +8,7 @@ import { Table, Message, Icon } from "semantic-ui-react";
 import { DateTime } from "@truedat/core/components";
 import { selectColor } from "../functions/selectors";
 
-const GeneralInformation = ({ rule, ruleResult }) => {
+const GeneralInformation = ({ ruleImplementation, ruleResult }) => {
   const { formatMessage, formatNumber: _formatNumber } = useIntl();
   const formatNumber = (num) => (_.isNil(num) ? num : _formatNumber(num));
   return (
@@ -18,7 +18,10 @@ const GeneralInformation = ({ rule, ruleResult }) => {
           {formatMessage({ id: "ruleResult.props.quality" })}
         </Table.Cell>
         <Table.Cell>
-          <Icon name="circle" color={selectColor({ ...rule, ...ruleResult })} />
+          <Icon
+            name="circle"
+            color={selectColor({ ...ruleImplementation, ...ruleResult })}
+          />
           {`${parseFloat(ruleResult.result)} %`}
         </Table.Cell>
       </Table.Row>
@@ -50,7 +53,7 @@ const GeneralInformation = ({ rule, ruleResult }) => {
 };
 
 GeneralInformation.propTypes = {
-  rule: PropTypes.object,
+  ruleImplementation: PropTypes.object,
   ruleResult: PropTypes.object,
 };
 
@@ -70,11 +73,7 @@ const DetailRow = ({ details }) => {
   )(details);
 };
 
-export const ExecutionDetails = ({
-  rule,
-  ruleImplementation,
-  rule_result_id,
-}) => {
+export const ExecutionDetails = ({ ruleImplementation, rule_result_id }) => {
   const { formatMessage } = useIntl();
   const { rule_result_id: id } = useParams();
 
@@ -104,7 +103,10 @@ export const ExecutionDetails = ({
         </Table.Row>
       </Table.Header>
       <Table.Body>
-        <GeneralInformation rule={rule} ruleResult={ruleResult} />
+        <GeneralInformation
+          ruleImplementation={ruleImplementation}
+          ruleResult={ruleResult}
+        />
       </Table.Body>
 
       <Table.Header>
@@ -126,16 +128,11 @@ export const ExecutionDetails = ({
 
 ExecutionDetails.propTypes = {
   ruleImplementation: PropTypes.object,
-  rule: PropTypes.object,
-  customColumns: PropTypes.array,
-  isAdmin: PropTypes.bool,
   rule_result_id: PropTypes.number,
 };
 
 const mapStateToProps = (state) => ({
-  rule: state.rule,
   ruleImplementation: state.ruleImplementation,
-  isAdmin: state.authentication.role === "admin",
 });
 
 export default connect(mapStateToProps)(ExecutionDetails);

package/src/components/ExecutionGroup.js

@@ -1,4 +1,4 @@
-import { flow, pick, reject, size, sortBy } from "lodash/fp";
+import { flow, pick, filter, reject, size, prop, sortBy } from "lodash/fp";
 import React from "react";
 import PropTypes from "prop-types";
 import { connect } from "react-redux";
@@ -15,41 +15,41 @@ const columns = [
   {
     name: "ruleImplementations.props.status",
     fieldDecorator: RuleEventDecorator,
-    fieldSelector: pick(["_embedded.quality_events"])
+    fieldSelector: pick(["_embedded.quality_events"]),
   },
   {
     name: "ruleImplementations.props.implementation_key",
     fieldSelector: "_embedded.implementation",
-    fieldDecorator: RuleImplementationResultsLink
+    fieldDecorator: RuleImplementationResultsLink,
   },
   {
     name: "rule.props.name",
-    fieldSelector: "_embedded.rule.name"
+    fieldSelector: "_embedded.rule.name",
   },
   {
     name: "ruleResult.props.date",
     fieldDecorator: DateTime,
     textAlign: "center",
-    fieldSelector: ({ _embedded }) => ({ value: _embedded?.result?.date })
+    fieldSelector: ({ _embedded }) => ({ value: _embedded?.result?.date }),
   },
   {
     name: "ruleResult.props.records",
     textAlign: "right",
-    fieldSelector: "_embedded.result.records"
+    fieldSelector: "_embedded.result.records",
   },
   {
     name: "ruleResult.props.errors",
     textAlign: "right",
-    fieldSelector: "_embedded.result.errors"
+    fieldSelector: "_embedded.result.errors",
   },
   {
     name: "ruleResult.props.quality",
     fieldSelector: ({ _embedded }) => ({
       ruleResult: _embedded?.result,
-      rule: _embedded?.rule
+      ruleImplementation: _embedded?.implementation,
     }),
-    fieldDecorator: RuleResultDecorator
-  }
+    fieldDecorator: RuleResultDecorator,
+  },
 ];
 
 export const ExecutionRow = ({ execution }) => (
@@ -65,7 +65,7 @@ export const ExecutionRow = ({ execution }) => (
 );
 
 ExecutionRow.propTypes = {
-  execution: PropTypes.object
+  execution: PropTypes.object,
 };
 
 export const ExecutionGroup = ({ executionGroup, executionGroupLoading }) => {
@@ -74,10 +74,19 @@ export const ExecutionGroup = ({ executionGroup, executionGroupLoading }) => {
     executionGroup?._embedded?.executions
   );
   const count = size(executions);
-  const pendingCount = flow(reject("_embedded.result"), size)(executions);
+  const missingResultCount = flow(reject("_embedded.result"), size)(executions);
+  const errorCount = flow(
+    filter(flow(prop("_embedded.quality_events"), size)),
+    size
+  )(executions);
+
   return executionGroupLoading ? null : (
     <>
-      <ExecutionGroupMessage count={count} pending={pendingCount} />
+      <ExecutionGroupMessage
+        count={count}
+        pending={missingResultCount - errorCount}
+        error={errorCount}
+      />
       <Table>
         <Table.Header>
           <Table.Row>
@@ -102,12 +111,12 @@ export const ExecutionGroup = ({ executionGroup, executionGroupLoading }) => {
 
 ExecutionGroup.propTypes = {
   executionGroup: PropTypes.object,
-  executionGroupLoading: PropTypes.bool
+  executionGroupLoading: PropTypes.bool,
 };
 
 export const mapStateToProps = ({ executionGroup, executionGroupLoading }) => ({
   executionGroup,
-  executionGroupLoading
+  executionGroupLoading,
 });
 
 export default connect(mapStateToProps)(ExecutionGroup);

package/src/components/ImplementationResultBar.js

@@ -0,0 +1,80 @@
+import _ from "lodash/fp";
+import React from "react";
+import PropTypes from "prop-types";
+import { Popup } from "semantic-ui-react";
+import { Link } from "react-router-dom";
+import { FormattedMessage, useIntl } from "react-intl";
+import { linkTo } from "@truedat/core/routes";
+import { calculateResultDecoration } from "./RuleResultDecorator";
+import "../styles/ruleSummary.less";
+
+export default function ImplementationResultBar({ implementation }) {
+  const intl = useIntl();
+  const ruleResult = _.head(implementation?.results);
+  const { color, resultText } =
+    implementation.event_type === "FAILED"
+      ? {
+          color: "failed",
+          resultText: intl.formatMessage({ id: "quality.error" }),
+        }
+      : calculateResultDecoration({
+          ruleResult,
+          ruleImplementation: implementation,
+          date: ruleResult?.date,
+          intl,
+        });
+
+  const propsByColor = {
+    red: {
+      className: "under-minimum-color",
+      tooltip: "under_minimum",
+    },
+    yellow: {
+      className: "under-goal-color",
+      tooltip: "under_goal",
+    },
+    green: {
+      className: "over-goal-color",
+      tooltip: "over_goal",
+    },
+    grey: {
+      className: "not-executed-color",
+      tooltip: "no_execution",
+    },
+    failed: {
+      className: "failed-color",
+      tooltip: "failed",
+    },
+  };
+  const summaryProps = propsByColor[color];
+  return (
+    <div className="rule-summary-wrapper">
+      <Link
+        as="div"
+        className="rule-summary pointer"
+        to={linkTo.RULE_IMPLEMENTATION_RESULTS({
+          id: implementation.rule_id,
+          implementation_id: implementation.id,
+        })}
+      >
+        <Popup
+          position="bottom center"
+          basic
+          on="hover"
+          content={
+            <FormattedMessage id={`quality_result.${summaryProps.tooltip}`} />
+          }
+          trigger={
+            <span className={summaryProps.className} style={{ width: "100%" }}>
+              <span className="value">{resultText}</span>
+            </span>
+          }
+        />
+      </Link>
+    </div>
+  );
+}
+
+ImplementationResultBar.propTypes = {
+  implementation: PropTypes.object.isRequired,
+};

package/src/components/ImplementationSummary.js

@@ -1,11 +1,12 @@
 import _ from "lodash/fp";
 import React from "react";
 import PropTypes from "prop-types";
-import { Header, Icon, Label, Grid, Table } from "semantic-ui-react";
+import { Header, Icon, Table, Segment } from "semantic-ui-react";
 import { Link } from "react-router-dom";
 import { FormattedMessage } from "react-intl";
 import { linkTo } from "@truedat/core/routes";
 import ConditionSummary, { empty, path } from "./ConditionSummary";
+import InformationSummary from "./InformationSummary";
 
 const defaults = [
   "executable",
@@ -38,45 +39,6 @@ FormattedLink.propTypes = {
   operator: PropTypes.object,
 };
 
-const GeneralSummary = ({ executable, implementationKey }) => (
-  <>
-    <Header as="h3">
-      <Icon name="database" size="small" />
-      <Header.Content>
-        <FormattedMessage id="ruleImplementations.summary.headers.implementation" />
-      </Header.Content>
-    </Header>
-    <Grid>
-      <Grid.Row>
-        <Grid.Column>
-          <Label color={executable ? "olive" : "teal"}>
-            <FormattedMessage
-              id={`ruleImplementation.props.executable.${executable}`}
-            />
-          </Label>
-        </Grid.Column>
-      </Grid.Row>
-      {implementationKey && (
-        <Grid.Row columns={2}>
-          <Grid.Column width={4}>
-            <FormattedMessage
-              id={`ruleImplementation.summary.implementation_key`}
-            />
-          </Grid.Column>
-          <Grid.Column width={12}>
-            <span className="highlighted">{`"${implementationKey}"`}</span>
-          </Grid.Column>
-        </Grid.Row>
-      )}
-    </Grid>
-  </>
-);
-
-GeneralSummary.propTypes = {
-  executable: PropTypes.bool,
-  implementationKey: PropTypes.string,
-};
-
 const DatasetSummary = ({ rows }) =>
   empty(rows) ? null : (
     <>
@@ -86,34 +48,36 @@ const DatasetSummary = ({ rows }) =>
           <FormattedMessage id="ruleImplementations.summary.headers.dataset" />
         </Header.Content>
       </Header>
-      <Table basic="very">
-        <Table.Header>
-          <Table.Row>
-            <Table.HeaderCell>
-              <FormattedMessage id={`ruleImplementation.summary.structure`} />
-            </Table.HeaderCell>
-          </Table.Row>
-        </Table.Header>
-        <Table.Body>
-          {rows.map((row, i) => (
-            <Table.Row key={i}>
-              {_.has("structure")(row) && (
-                <Table.Cell key={i}>
-                  <Link
-                    to={linkTo.STRUCTURE({
-                      id: _.pathOr("", "structure.id")(row),
-                    })}
-                  >
-                    <span key={i} className="highlighted">{`"${path(
-                      _.propOr({}, "structure")(row)
-                    )}"`}</span>
-                  </Link>
-                </Table.Cell>
-              )}
+      <Segment>
+        <Table basic="very">
+          <Table.Header>
+            <Table.Row>
+              <Table.HeaderCell>
+                <FormattedMessage id={`ruleImplementation.summary.structure`} />
+              </Table.HeaderCell>
             </Table.Row>
-          ))}
-        </Table.Body>
-      </Table>
+          </Table.Header>
+          <Table.Body>
+            {rows.map((row, i) => (
+              <Table.Row key={i}>
+                {_.has("structure")(row) && (
+                  <Table.Cell key={i}>
+                    <Link
+                      to={linkTo.STRUCTURE({
+                        id: _.pathOr("", "structure.id")(row),
+                      })}
+                    >
+                      <span key={i} className="highlighted">{`"${path(
+                        _.propOr({}, "structure")(row)
+                      )}"`}</span>
+                    </Link>
+                  </Table.Cell>
+                )}
+              </Table.Row>
+            ))}
+          </Table.Body>
+        </Table>
+      </Segment>
     </>
   );
 
@@ -123,16 +87,13 @@ DatasetSummary.propTypes = {
 
 export const ImplementationSummary = ({ ruleImplementation, activeSteps }) => {
   const steps = _.isEmpty(activeSteps) ? defaults : activeSteps;
-  const { executable, implementationKey, dataset, population, validations } =
+  const { dataset, population, validations } =
     _.pick(steps)(ruleImplementation);
   return (
     <>
       {(_.includes("implementationKey")(steps) ||
         _.includes("executable")(steps)) && (
-        <GeneralSummary
-          executable={executable}
-          implementationKey={implementationKey}
-        />
+        <InformationSummary ruleImplementation={ruleImplementation} />
       )}
       {dataset && _.includes("dataset")(steps) && (
         <DatasetSummary rows={dataset} />

package/src/components/ImplementationsUploadButton.js

@@ -0,0 +1,61 @@
+import React from "react";
+import PropTypes from "prop-types";
+import { connect } from "react-redux";
+import { Dropdown } from "semantic-ui-react";
+import { FormattedMessage, useIntl } from "react-intl";
+import { UploadModal } from "@truedat/core/components";
+import { API_RULE_IMPLEMENTATIONS_UPLOAD } from "../api";
+import { uploadImplementations } from "../routines";
+
+const uploadAction = {
+  method: "POST",
+  href: API_RULE_IMPLEMENTATIONS_UPLOAD,
+};
+
+export const ImplementationsUploadButton = ({
+  uploadImplementations,
+  loading,
+}) => {
+  const { formatMessage } = useIntl();
+  return (
+    <UploadModal
+      icon="upload"
+      trigger={
+        <Dropdown.Item
+          icon="upload"
+          text={formatMessage({
+            id: "ruleImplementations.actions.upload.tooltip",
+          })}
+          disabled={loading}
+        />
+      }
+      header={
+        <FormattedMessage id="ruleImplementations.actions.upload.confirmation.header" />
+      }
+      content={
+        <FormattedMessage id="ruleImplementations.actions.upload.confirmation.content" />
+      }
+      param={"implementations"}
+      handleSubmit={(data) =>
+        uploadImplementations({
+          action: "upload",
+          data,
+          ...uploadAction,
+        })
+      }
+    />
+  );
+};
+
+ImplementationsUploadButton.propTypes = {
+  uploadImplementations: PropTypes.func,
+  loading: PropTypes.bool,
+};
+
+const mapStateToProps = ({ uploadImplementationsFile: { loading } }) => ({
+  loading,
+});
+
+export default connect(mapStateToProps, { uploadImplementations })(
+  ImplementationsUploadButton
+);