@truedat/dq 4.56.1 → 4.56.3

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## [4.56.3] 2022-11-24
+
+### Added
+
+- [TD-5286] Implementation results pagination
+
+## [4.56.2] 2022-11-24
+
+### Changed
+
+- [TD-5247] Display the name of the table together with field in implementations
+  details
+
 ## [4.54.12] 2022-11-14
 
 ### Added

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@truedat/dq",
-  "version": "4.56.1",
+  "version": "4.56.3",
   "description": "Truedat Web Data Quality Module",
   "sideEffects": false,
   "jsnext:main": "src/index.js",
@@ -34,7 +34,7 @@
     "@testing-library/jest-dom": "^5.16.4",
     "@testing-library/react": "^12.0.0",
     "@testing-library/user-event": "^13.2.1",
-    "@truedat/test": "4.56.0",
+    "@truedat/test": "4.56.3",
     "babel-jest": "^28.1.0",
     "babel-plugin-dynamic-import-node": "^2.3.3",
     "babel-plugin-lodash": "^3.3.4",
@@ -93,8 +93,8 @@
   },
   "dependencies": {
     "@apollo/client": "^3.7.0",
-    "@truedat/core": "4.56.1",
-    "@truedat/df": "4.56.1",
+    "@truedat/core": "4.56.3",
+    "@truedat/df": "4.56.3",
     "graphql": "^15.5.3",
     "path-to-regexp": "^1.7.0",
     "prop-types": "^15.8.1",
@@ -114,5 +114,5 @@
     "react-dom": ">= 16.8.6 < 17",
     "semantic-ui-react": ">= 0.88.2 < 2.1"
   },
-  "gitHead": "d2b3dcb36b7d81889ee5642699595439750c0801"
+  "gitHead": "7940a71d7e8ed3eb6c1fc088cf631e37e87fd093"
 }

package/src/api/queries.js

@@ -15,16 +15,17 @@ export const IMPLEMENTATION_VERSIONS_QUERY = gql`
   }
 `;
 
-export const IMPLEMENTATION_RESULTS_QUERY = gql`
-  query Implementation($id: ID!) {
+export const IMPLEMENTATION_RESULTS_CONNECTION_QUERY = gql`
+  query Implementation(
+    $id: ID!
+    $after: Cursor
+    $before: Cursor
+    $last: Int
+    $first: Int
+  ) {
     implementation(id: $id) {
       id
-      implementationKey
-      lastQualityEvent {
-        insertedAt
-        message
-        type
-      }
+      version
       versions {
         id
         implementationKey
@@ -32,7 +33,20 @@ export const IMPLEMENTATION_RESULTS_QUERY = gql`
         status
         minimum
         goal
-        results {
+      }
+      lastQualityEvent {
+        insertedAt
+        message
+        type
+      }
+      resultsConnection(
+        first: $first
+        last: $last
+        before: $before
+        after: $after
+      ) {
+        totalCount
+        page {
           id
           date
           details
@@ -43,8 +57,22 @@ export const IMPLEMENTATION_RESULTS_QUERY = gql`
           records
           result
           resultType
+          implementation {
+            id
+            version
+          }
+          __typename
+        }
+        pageInfo {
+          startCursor
+          endCursor
+          hasNextPage
+          hasPreviousPage
+          __typename
         }
+        __typename
       }
+      __typename
     }
   }
 `;

package/src/components/ConditionSummary.js

@@ -125,11 +125,11 @@ OperatorMessage.propTypes = {
 };
 
 const ConditionCell = ({ row, alias, validationPopulationCondition }) => {
-  const alias_text = _.flow(
-    _.find({ index: row.structure?.parent_index }),
-    _.propOr(null, "text")
-  )(alias);
-
+  const alias_text =
+    _.flow(
+      _.find({ index: row.structure?.parent_index }),
+      _.propOr(null, "text")
+    )(alias) || (row.structure ? _.last(row.structure.path) : null);
   const values = getValues(row);
   const operator = _.prop("operator")(row);
   const { formatMessage } = useIntl();

package/src/components/ImplementationExecutions.js

@@ -58,33 +58,36 @@ ExecutionRow.propTypes = {
   implementation: PropTypes.object,
 };
 
-export const ImplementationExecutions = ({ page }) => (
-  <Table basic="very">
-    <Table.Header>
-      <Table.Row>
-        <Table.HeaderCell width="1" textAlign="right">
-          <FormattedMessage id="ruleImplementations.props.version" />
-        </Table.HeaderCell>
-        <Table.HeaderCell width="2">
-          <FormattedMessage id="execution" />
-        </Table.HeaderCell>
-        <Table.HeaderCell width="2">
-          <FormattedMessage id="execution.latestEvent.insertedAt" />
-        </Table.HeaderCell>
-        <Table.HeaderCell width="2">
-          <FormattedMessage id="execution.latestEvent.type" />
-        </Table.HeaderCell>
-        <Table.HeaderCell>
-          <FormattedMessage id="execution.latestEvent.message" />
-        </Table.HeaderCell>
-      </Table.Row>
-    </Table.Header>
-    <Table.Body>
-      {page.map((props, key) => (
-        <ExecutionRow key={key} {...props} />
-      ))}
-    </Table.Body>
-  </Table>
+export const ImplementationExecutions = (connection) => (
+  <>
+    <Table basic="very">
+      <Table.Header>
+        <Table.Row>
+          <Table.HeaderCell width="1" textAlign="right">
+            <FormattedMessage id="ruleImplementations.props.version" />
+          </Table.HeaderCell>
+          <Table.HeaderCell width="2">
+            <FormattedMessage id="execution" />
+          </Table.HeaderCell>
+          <Table.HeaderCell width="2">
+            <FormattedMessage id="execution.latestEvent.insertedAt" />
+          </Table.HeaderCell>
+          <Table.HeaderCell width="2">
+            <FormattedMessage id="execution.latestEvent.type" />
+          </Table.HeaderCell>
+          <Table.HeaderCell>
+            <FormattedMessage id="execution.latestEvent.message" />
+          </Table.HeaderCell>
+        </Table.Row>
+      </Table.Header>
+      <Table.Body>
+        {connection.page.map((props, key) => (
+          <ExecutionRow key={key} {...props} />
+        ))}
+      </Table.Body>
+    </Table>
+    <CursorPagination {...connection} />
+  </>
 );
 
 ImplementationExecutions.propTypes = {
@@ -111,9 +114,6 @@ export const ImplementationExecutionsLoader = () => {
           <Grid.Column>
             <ImplementationExecutionFilters />
           </Grid.Column>
-          <Grid.Column textAlign="right">
-            <CursorPagination {...connection} />
-          </Grid.Column>
         </Grid.Row>
       </Grid>
       {loading ? <Loading /> : null}

package/src/components/NewRuleImplementation.js

@@ -358,7 +358,7 @@ export const NewRuleImplementation = ({
         }
   );
 
-  const withoutUnusedAliasText = (dataset) => {
+  const withAliasText = (dataset) => {
     const duplicatedStructures = _.flow(
       _.groupBy("structure.id"),
       _.pickBy((x) => x.length > 1),
@@ -366,23 +366,24 @@ export const NewRuleImplementation = ({
       _.map(Number)
     )(dataset);
     return _.map((ds) =>
-      _.has("structure.id")(ds) &&
-      _.includes(ds.structure.id)(duplicatedStructures)
-        ? _.path("alias.text")(ds)
-          ? ds
-          : {
-              ...ds,
-              alias: {
-                ...ds.alias,
-                text: `${ds.structure.name} (${ds.alias.index})`,
-              },
-            }
-        : { ...ds, alias: { ...ds.alias, text: null } }
+      _.path("alias.text")(ds)
+        ? ds
+        : {
+            ...ds,
+            alias: {
+              ...ds.alias,
+              text:
+                _.has("structure.id")(ds) &&
+                _.includes(ds.structure.id)(duplicatedStructures)
+                  ? `${ds.structure?.name} (${ds.alias?.index})`
+                  : `${ds.structure?.name}`,
+            },
+          }
     )(dataset);
   };
 
   const setDataset = (dataset) => {
-    const cleanDataset = withoutUnusedAliasText(dataset);
+    const cleanDataset = withAliasText(dataset);
     //remove invalid references of fields in validations and populations after dataset edition
     const datasetStructures = _.map(_.path("structure.id"))(cleanDataset);
     const all_fields = _.flow(

package/src/components/RuleImplementationResults.js

@@ -4,29 +4,28 @@ import { useParams } from "react-router-dom";
 import { useQuery } from "@apollo/client";
 import PropTypes from "prop-types";
 import { useIntl } from "react-intl";
-import { Message, Divider } from "semantic-ui-react";
+import { Segment, Message, Divider } from "semantic-ui-react";
 import { columnDecorator } from "@truedat/core/services";
 import { DateTime, Loading } from "@truedat/core/components";
-import { IMPLEMENTATION_RESULTS_QUERY } from "../api/queries";
+import { CursorPagination } from "@truedat/core/components";
+import { useLocation } from "react-router-dom";
+import queryString from "query-string";
+import { IMPLEMENTATION_RESULTS_CONNECTION_QUERY } from "../api/queries";
 import RuleResultsTable from "./RuleResultsTable";
 
+export const PAGE_SIZE = 20;
+
 export const getCustomColumnsWithData = (ruleResults, columns) =>
   _.filter((column) =>
     _.any(_.flow(columnDecorator(column), _.negate(_.isEmpty)))(ruleResults)
   )(columns);
 
-export const RuleImplementationResults = ({ ruleImplementation }) => {
+export const RuleImplementationResults = ({ implementation, results }) => {
   const { formatMessage, locale } = useIntl();
 
-  if (_.isEmpty(ruleImplementation)) return null;
-
-  const implementationVersions = _.propOr([], "versions")(ruleImplementation);
-  const currentImplementationVersion = _.flow(
-    _.filter((version) => version.id === ruleImplementation.id),
-    _.first
-  )(implementationVersions);
+  if (_.isEmpty(implementation)) return null;
 
-  const { lastQualityEvent } = ruleImplementation;
+  const { lastQualityEvent } = implementation;
 
   return (
     <>
@@ -46,31 +45,45 @@ export const RuleImplementationResults = ({ ruleImplementation }) => {
           <p style={{ whiteSpace: "pre-wrap" }}>{lastQualityEvent.message}</p>
         </Message>
       ) : null}
-      <RuleResultsTable
-        currentImplementationVersion={currentImplementationVersion}
-        implementationVersions={implementationVersions}
-      />
+      <RuleResultsTable implementation={implementation} results={results} />
     </>
   );
 };
 
 RuleImplementationResults.propTypes = {
-  ruleImplementation: PropTypes.object,
+  results: PropTypes.array,
+  implementation: PropTypes.object,
 };
 
 export const RuleImplementationResultsLoader = (props) => {
   const { implementation_id: id } = useParams();
-  const { loading, error, data } = useQuery(IMPLEMENTATION_RESULTS_QUERY, {
-    variables: { id },
-  });
+
+  const { search } = useLocation();
+  const { before, after } = queryString.parse(search);
+  const variables = after
+    ? { id, after, before, first: PAGE_SIZE }
+    : { id, after, before, last: PAGE_SIZE };
+
+  const { loading, error, data } = useQuery(
+    IMPLEMENTATION_RESULTS_CONNECTION_QUERY,
+    {
+      variables,
+    }
+  );
   if (error) return null;
   if (loading) return <Loading />;
-  const ruleImplementation = data?.implementation;
+  const implementation = data?.implementation;
+  const connection = data?.implementation?.resultsConnection;
+  const results = implementation?.resultsConnection?.page;
   return (
-    <RuleImplementationResults
-      ruleImplementation={ruleImplementation}
-      {...props}
-    />
+    <Segment attached="bottom">
+      <RuleImplementationResults
+        implementation={implementation}
+        results={results}
+        {...props}
+      />
+      <CursorPagination {...connection} />
+    </Segment>
   );
 };
 

package/src/components/RuleResultRow.js

@@ -79,7 +79,10 @@ export const RuleResultRow = ({
             <>
               <Icon
                 name="circle"
-                color={selectColor({ ...ruleImplementation, ...ruleResult })}
+                color={selectColor({
+                  ...ruleImplementation,
+                  ...ruleResult,
+                })}
               />
               {`${parseFloat(ruleResult.result)} %`}
             </>

package/src/components/RuleResultsTable.js

@@ -21,22 +21,14 @@ export const getCustomColumnsWithData = (ruleResults, columns) =>
     _.any(_.flow(columnDecorator(column), _.negate(_.isEmpty)))(ruleResults)
   )(columns);
 
-export const RuleResultsTable = ({
-  columns,
-  implementationVersions,
-  currentImplementationVersion,
-}) => {
+export const RuleResultsTable = ({ columns, implementation, results }) => {
   const isAdmin = useAuthorized();
   const { formatMessage } = useIntl();
 
-  const ruleResults = _.flow(
-    _.map((iv) => _.pathOr([], "results")(iv)),
-    _.flatten
-  )(implementationVersions);
-  const optionalColumns = getOptionalColumnsWithData(ruleResults);
-  const customColumns = getCustomColumnsWithData(ruleResults, columns);
+  const optionalColumns = getOptionalColumnsWithData(results);
+  const customColumns = getCustomColumnsWithData(results, columns);
 
-  return _.isEmpty(ruleResults) ? (
+  return _.isEmpty(results) ? (
     <Message
       style={{ marginTop: "14px" }}
       header={formatMessage({
@@ -77,28 +69,43 @@ export const RuleResultsTable = ({
         </Table.Row>
       </Table.Header>
       <Table.Body>
-        {implementationVersions.map((version) =>
-          _.pathOr(
-            [],
-            "results"
-          )(version).map((result, key) => (
+        {_.flow(
+          _.groupBy("implementation.version"),
+          _.toPairs, // first element: version number; second element: version results.
+          // order by descending version number
+          _.orderBy(
+            (versionAndResultsTuple) => versionAndResultsTuple[0],
+            "desc"
+          ),
+          _.mapValues((versionAndResultsTuple) =>
+            versionAndResultsTuple[1].map(
+              (result, indexWithinVersionGroup) => ({
+                indexWithinVersionGroup,
+                ...result,
+              })
+            )
+          ),
+          _.reduce((acc, versionResults) => [...acc, ...versionResults], []),
+          _.map((result) => (
             <RuleResultRow
-              key={key}
+              key={result.id}
               optionalColumns={optionalColumns}
               ruleResult={result}
               customColumns={customColumns}
-              ruleImplementation={version}
-              active={currentImplementationVersion.id === version.id}
+              ruleImplementation={_.find(
+                (version) => version?.id === result?.implementation?.id
+              )(implementation?.versions)}
+              active={implementation.version === result.implementation.version}
               tagLabel={
-                key === 0
+                result.indexWithinVersionGroup === 0
                   ? `${formatMessage({
                       id: "ruleImplementations.props.version",
-                    })}: ${version.version}`
+                    })}: ${result.implementation.version}`
                   : null
               }
             />
           ))
-        )}
+        )(results)}
       </Table.Body>
     </Table>
   );
@@ -106,8 +113,8 @@ export const RuleResultsTable = ({
 
 RuleResultsTable.propTypes = {
   columns: PropTypes.array,
-  implementationVersions: PropTypes.array,
-  currentImplementationVersion: PropTypes.object,
+  implementation: PropTypes.object,
+  results: PropTypes.array,
 };
 
 const mapStateToProps = (state) => ({

package/src/components/__tests__/ConditionSummary.spec.js

@@ -5,7 +5,12 @@ import { ConditionSummary } from "../ConditionSummary";
 describe("<ConditionSummary />", () => {
   const rows = [
     {
-      structure: { field_type: "string", name: "Mes", id: 2598 },
+      structure: {
+        field_type: "string",
+        name: "Mes",
+        id: 2598,
+        parent_index: 1,
+      },
       operator: { name: "in_list", value_type: "string_list" },
       value: [{ raw: ["4", "5", "6"] }],
     },
@@ -14,6 +19,12 @@ describe("<ConditionSummary />", () => {
     icon: "setting",
     type: "filters",
     rows,
+    alias: [
+      {
+        index: 1,
+        text: "foo_alias",
+      },
+    ],
   };
   const renderOpts = {
     messages: {

package/src/components/__tests__/RuleImplementationResults.spec.js

@@ -1,78 +1,31 @@
 import _ from "lodash/fp";
 import React from "react";
 import { render } from "@truedat/test/render";
-import {
-  RuleImplementationResults,
+import RuleImplementationResults, {
   getCustomColumnsWithData,
+  PAGE_SIZE,
 } from "../RuleImplementationResults";
 
+import { implementationResultsMock } from "./__fixtures__/implementationResultsMocks";
+
+jest.mock("react-router-dom", () => ({
+  ...jest.requireActual("react-router-dom"),
+  useParams: () => ({ implementation_id: "2" }),
+}));
+
 describe("<RuleImplementationResults />", () => {
-  const ruleImplementation = {
-    id: 1,
-    implementation_key: "default_key",
-    versions: [
-      {
-        implementation_key: "default_key",
-        id: 1,
-        goal: "20.0",
-        minimum: "10.0",
-        results: [
-          {
-            errors: 2,
-            hasRemediation: false,
-            hasSegments: true,
-            id: "66577",
-            records: 31,
-            result: "93.54",
-            resultType: "percentage",
-          },
-          {
-            errors: 2,
-            hasRemediation: false,
-            hasSegments: true,
-            id: "66578",
-            records: 31,
-            result: "93.54",
-            resultType: "percentage",
-          },
-        ],
-      },
-      {
-        implementation_key: "key_2",
-        id: 2,
-        goal: "20.0",
-        minimum: "10.0",
-        results: [
-          {
-            errors: 2,
-            hasRemediation: false,
-            hasSegments: true,
-            id: "66579",
-            records: 31,
-            result: "93.54",
-            resultType: "percentage",
-          },
-          {
-            errors: 2,
-            hasRemediation: false,
-            hasSegments: true,
-            id: "66580",
-            records: 31,
-            result: "93.54",
-            resultType: "percentage",
-          },
-        ],
-      },
-    ],
+  const renderOpts = {
+    mocks: [implementationResultsMock({ id: "2", last: PAGE_SIZE })],
   };
+
   const ruleResultsColumns = [{ foo: "bar", name: "foo" }];
-  const props = {
-    ruleImplementation,
-    ruleResultsColumns,
-  };
 
-  it("matches the latest snapshot", () => {
-    const { container } = render(<RuleImplementationResults {...props} />);
+  it("matches the latest snapshot", async () => {
+    const { container, findByRole } = render(
+      <RuleImplementationResults />,
+      renderOpts
+    );
+    expect(await findByRole("table")).toBeInTheDocument();
     expect(container).toMatchSnapshot();
   });
 });

package/src/components/__tests__/__fixtures__/NewRuleImplementationProps.js

@@ -37,6 +37,10 @@ export const newRuleImplementationProps = {
   ruleImplementationProps: {
     dataset: [
       {
+        alias: {
+          index: 1,
+          text: "FS",
+        },
         clauses: [],
         structure: {
           external_id: "/data/example.txt",
@@ -53,6 +57,10 @@ export const newRuleImplementationProps = {
         },
       },
       {
+        alias: {
+          index: 2,
+          text: "table2",
+        },
         clauses: [
           {
             left: {
@@ -67,6 +75,7 @@ export const newRuleImplementationProps = {
                 tablename: "table2",
                 type: "string",
               },
+              parent_index: 2,
               name: "capital",
               path: ["bucket", "table2"],
               system: {
@@ -82,6 +91,7 @@ export const newRuleImplementationProps = {
               metadata: {
                 metadata: { order: "8" },
               },
+              parent_index: 1,
               name: "external",
               path: ["Main", "example.txt"],
               system: {
@@ -112,6 +122,7 @@ export const newRuleImplementationProps = {
           },
           type: "Table",
         },
+        join_type: "inner"
       },
     ],
     deleted_at: null,