@truedat/dd 4.33.3 → 4.33.8

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## [4.33.8] 2021-11-29
+
+### Fixed
+
+- [TD-4353] Fix Bulk update
+
+## [4.33.5] 2021-11-28
+
+### Added
+
+- [TD-4262] Timeout messages
+
+## [4.33.4] 2021-11-26
+
+### Added
+
+- [TD-4262] Lineage graph polling
+
 ## [4.33.3] 2021-11-25
 
 ### Added

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@truedat/dd",
-  "version": "4.33.3",
+  "version": "4.33.8",
   "description": "Truedat Web Data Dictionary",
   "sideEffects": false,
   "jsnext:main": "src/index.js",
@@ -34,7 +34,7 @@
     "@testing-library/jest-dom": "^5.14.1",
     "@testing-library/react": "^12.0.0",
     "@testing-library/user-event": "^13.2.1",
-    "@truedat/test": "4.33.3",
+    "@truedat/test": "4.33.8",
     "babel-jest": "^27.0.6",
     "babel-plugin-dynamic-import-node": "^2.3.3",
     "babel-plugin-lodash": "^3.3.4",
@@ -84,9 +84,9 @@
     ]
   },
   "dependencies": {
-    "@truedat/auth": "4.33.3",
-    "@truedat/core": "4.33.3",
-    "@truedat/df": "4.33.3",
+    "@truedat/auth": "4.33.8",
+    "@truedat/core": "4.33.8",
+    "@truedat/df": "4.33.8",
     "lodash": "^4.17.21",
     "path-to-regexp": "^1.7.0",
     "prop-types": "^15.7.2",
@@ -107,5 +107,5 @@
     "react-dom": ">= 16.8.6 < 17",
     "semantic-ui-react": ">= 0.88.2 < 2.1"
   },
-  "gitHead": "4423c4077cf22e7d21d2679670287ed3d86ec8d7"
+  "gitHead": "14e8164b15d17c5d25ebd0296b6a9899b32dff1d"
 }

package/src/api.js

@@ -14,8 +14,10 @@ const API_GRANT_REQUESTS = "/api/grant_requests";
 const API_GRANTS_CSV = "/api/grants/csv";
 const API_GRANTS_SEARCH = "/api/grants/search";
 const API_GRAPH = "/api/graphs/:id";
+const API_GRAPH_HASH = "/api/graphs/hash/:hash";
 const API_GRAPHS = "/api/graphs";
 const API_GRAPH_CSV = "/api/graphs/csv";
+const API_LINEAGE_EVENTS = "/api/lineage_events";
 const API_MY_GRANT_FILTERS_SEARCH = "/api/grant_filters/search/mine";
 const API_MY_GRANTS_SEARCH = "/api/grants/search/mine";
 const API_NODE = "/api/nodes/:id";
@@ -58,9 +60,11 @@ export {
   API_GRANT_REQUESTS,
   API_GRANTS_SEARCH,
   API_GRAPH,
+  API_GRAPH_HASH,
   API_GRAPHS,
   API_GRAPH_CSV,
   API_GRANTS_CSV,
+  API_LINEAGE_EVENTS,
   API_MY_GRANTS_SEARCH,
   API_MY_GRANT_FILTERS_SEARCH,
   API_NODE,

package/src/components/GraphLoader.js

@@ -2,27 +2,41 @@ import _ from "lodash/fp";
 import { useEffect } from "react";
 import PropTypes from "prop-types";
 import { connect } from "react-redux";
-import { useRouteMatch } from "react-router-dom";
+import { useRouteMatch, useHistory } from "react-router-dom";
 import { GRAPH } from "@truedat/core/routes";
-import { fetchGraph, clearGraph } from "../routines";
+import { fetchGraph, clearGraph, cancelPoll } from "../routines";
 
-export const GraphLoader = ({ clearGraph, fetchGraph, graph }) => {
+export const GraphLoader = ({ clearGraph, fetchGraph, graph, cancelPoll }) => {
   const match = useRouteMatch(GRAPH);
   const id = _.path("params.id")(match);
+
+  const history = useHistory();
+
   useEffect(() => {
     return () => {
       clearGraph();
     };
   }, [clearGraph]);
+
   useEffect(() => {
     if (id) {
       if (_.isEmpty(graph)) {
         fetchGraph({ id });
-      } else if (!_.propEq("id", id)(graph)) {
+      } else if (id != graph.id) {
         fetchGraph({ id });
       }
     }
-  }, [graph, fetchGraph, id]);
+  }, [/*graph, */ /*fetchGraph,*/ id]);
+
+  useEffect(() => {
+    return () => {
+      // && history.location.pathname === "any specific path")
+      if (history.action === "POP") {
+        cancelPoll();
+      }
+    };
+  }, [history]);
+
   return null;
 };
 
@@ -30,10 +44,11 @@ GraphLoader.propTypes = {
   clearGraph: PropTypes.func.isRequired,
   fetchGraph: PropTypes.func.isRequired,
   graph: PropTypes.object,
+  cancelPoll: PropTypes.func,
 };
 
 const mapStateToProps = ({ graph }) => ({ graph });
 
-export default connect(mapStateToProps, { clearGraph, fetchGraph })(
+export default connect(mapStateToProps, { clearGraph, fetchGraph, cancelPoll })(
   GraphLoader
 );

package/src/components/LineageEvents.js

@@ -0,0 +1,23 @@
+import React from "react";
+import { Header, Icon, Segment, Dimmer, Loader } from "semantic-ui-react";
+import { FormattedMessage } from "react-intl";
+import LineageEventsTable from "./LineageEventsTable";
+
+export const LineageEvents = () => (
+  <Segment>
+    <Header as="h2">
+      <Icon circular name="cogs" />
+      <Header.Content>
+        <FormattedMessage id="lineage.events.header" />
+        <Header.Subheader>
+          <FormattedMessage id="lineage.events.subheader" />
+        </Header.Subheader>
+      </Header.Content>
+    </Header>
+    <Segment attached="bottom">
+      <LineageEventsTable />
+    </Segment>
+  </Segment>
+);
+
+export default LineageEvents;

package/src/components/LineageEventsTable.js

@@ -0,0 +1,94 @@
+import _ from "lodash/fp";
+import React, { useEffect } from "react";
+import PropTypes from "prop-types";
+import { FormattedMessage } from "react-intl";
+import { Header, Icon, Table } from "semantic-ui-react";
+import { useHistory } from "react-router-dom";
+import { connect } from "react-redux";
+import { useIntl } from "react-intl";
+import { columnDecorator } from "@truedat/core/services";
+import { linkTo } from "@truedat/core/routes";
+import { lineageEventColumns as columns } from "../selectors/getLineageEventColumns";
+import { fetchLineageEvents } from "../routines";
+
+export const LineageEventsTable = ({
+  fetchLineageEvents,
+  lineageEvents,
+  lineageEventsLoading,
+}) => {
+  const { formatMessage } = useIntl();
+  const history = useHistory();
+
+  useEffect(() => {
+    fetchLineageEvents();
+  }, [fetchLineageEvents]);
+
+  return (
+    <>
+      {!_.isEmpty(lineageEvents) && (
+        <Table sortable selectable>
+          <Table.Header>
+            <Table.Row>
+              {columns.map((column, i) => {
+                return (
+                  <Table.HeaderCell
+                    key={i}
+                    width={column.width}
+                    content={formatMessage({
+                      id: `lineage.events.props.${column.name}`,
+                      defaultMessage: column.name,
+                    })}
+                  />
+                );
+              })}
+            </Table.Row>
+          </Table.Header>
+          <Table.Body>
+            {lineageEvents.map((event, i) => (
+              <Table.Row
+                key={i}
+                onClick={
+                  event.graph_id
+                    ? () => history.push(linkTo.GRAPH({ id: event?.graph_id }))
+                    : null
+                }
+              >
+                {columns &&
+                  columns.map((column, key) => (
+                    <Table.Cell
+                      key={key}
+                      textAlign={column.textAlign}
+                      content={columnDecorator(column)(event)}
+                    />
+                  ))}
+              </Table.Row>
+            ))}
+          </Table.Body>
+        </Table>
+      )}
+      {_.isEmpty(lineageEvents) && !lineageEventsLoading && (
+        <Header as="h4">
+          <Icon name="search" />
+          <Header.Content>
+            <FormattedMessage id="lineage.events.search.results.empty" />
+          </Header.Content>
+        </Header>
+      )}
+    </>
+  );
+};
+
+LineageEventsTable.propTypes = {
+  lineageEvents: PropTypes.array,
+  fetchLineageEvents: PropTypes.func,
+  lineageEventsLoading: PropTypes.bool,
+};
+
+const mapStateToProps = (state) => ({
+  lineageEvents: state.lineageEvents,
+  lineageEventsLoading: state.lineageEventsLoading,
+});
+
+export default connect(mapStateToProps, { fetchLineageEvents })(
+  LineageEventsTable
+);

package/src/components/LineageForm.js

@@ -54,10 +54,11 @@ export const LineageForm = ({
   const [open, setOpen] = useState(true);
   const [selectLevels, setSelectLevels] = useState(levelsSelection);
   const [levels, setLevels] = useState(lineageLevels);
-  const handleSubmit = (type) => () =>
+  const handleSubmit = (type) => () => {
     _.isNil(levels)
       ? createGraph({ ids: selectedResources, type })
       : createGraph({ ids: selectedResources, type, levels });
+  };
   const disabled = _.isEmpty(selectedResources);
   useEffect(() => {
     setOpen(!graphLoading && _.isEmpty(graph));

package/src/components/LineageGraph.js

@@ -4,12 +4,19 @@ import PropTypes from "prop-types";
 import { connect } from "react-redux";
 import { Dimmer, Loader } from "semantic-ui-react";
 import svgPanZoom from "svg-pan-zoom";
-import { FormattedMessage } from "react-intl";
-import { clearNode, selectNode, hoverNode, initSvg } from "../routines";
+import { FormattedMessage, useIntl } from "react-intl";
+import {
+  clearNode,
+  selectNode,
+  hoverNode,
+  initSvg,
+  cancelPoll,
+} from "../routines";
 import LineageLink from "./LineageLink";
 import LineageDefs from "./LineageDefs";
 import LineageGroup from "./LineageGroup";
 import LineageResource from "./LineageResource";
+import "../styles/lineageGraph.less";
 
 const svgOptions = { minZoom: 0.2, maxZoom: 2.0, fit: false, center: false };
 const disablePanZoom = (svg) => {
@@ -50,11 +57,18 @@ export const LineageGraph = ({
   groups = [],
   resources = [],
   loading,
+  loadingInfo,
+  cancelPoll,
 }) => {
   const ref = useRef(null);
   const [state, dispatch] = useReducer(reducer, initialState);
   const { selectedNode, hover, svg } = state;
 
+  useEffect(() => {
+    // Anything in here is fired on component mount.
+    return () => cancelPoll();
+  }, []);
+
   useEffect(() => {
     if (!_.isEmpty(resources)) {
       const svg = svgPanZoom(ref.current, svgOptions);
@@ -67,6 +81,8 @@ export const LineageGraph = ({
   const onSelect = (g) => () => dispatch(selectNode.trigger(g));
   const onHover = (g) =>
     selectedNode ? undefined : () => dispatch(hoverNode.trigger(g));
+  const { formatMessage } = useIntl();
+
   return (
     <>
       <Dimmer.Dimmable dimmed={loading} className="dl">
@@ -74,8 +90,35 @@ export const LineageGraph = ({
           <Loader size="large">
             <FormattedMessage
               id="lineage.graph.loading"
-              defaultMessage="Analyzing..."
+              defaultMessage={"Analyzing..."}
             />
+            {loadingInfo?.data?.graph_hash && loadingInfo?.data?.status && (
+              <p>
+                {formatMessage({
+                  id:
+                    _.toLower(loadingInfo?.data?.status) === "already_started"
+                      ? "lineage.graph.loading.background.existing"
+                      : _.toLower(loadingInfo?.data?.status) ===
+                          "just_started" &&
+                        "lineage.graph.loading.background.launched",
+                })}
+              </p>
+            )}
+            {loadingInfo?.data?.graph_hash && (
+              <p>
+                Graph hash:{" "}
+                <span className="graph-hash">
+                  {loadingInfo?.data?.graph_hash}
+                </span>
+              </p>
+            )}
+            <p>
+              {loadingInfo?.data?.retry &&
+                formatMessage(
+                  { id: "lineage.graph.loading.polling" },
+                  { retry: loadingInfo?.data?.retry }
+                )}
+            </p>
           </Loader>
         </Dimmer>
         <svg
@@ -121,17 +164,25 @@ LineageGraph.propTypes = {
   paths: PropTypes.array,
   resources: PropTypes.array,
   height: PropTypes.string,
+  loading: PropTypes.bool,
+  loadingInfo: PropTypes.object,
+  cancelPoll: PropTypes.func,
 };
 
 const mapStateToProps = (
   { graph = {}, graphLoading, selectedNode },
   ownProps
-) => ({
-  paths: _.prop("paths")(_.propOr(graph, "graph")(ownProps)),
-  groups: _.prop("groups")(_.propOr(graph, "graph")(ownProps)),
-  resources: _.prop("resources")(_.propOr(graph, "graph")(ownProps)),
-  loading: graphLoading,
-  selectedNode,
-});
+) => {
+  return {
+    paths: _.prop("paths")(_.propOr(graph, "graph")(ownProps)),
+    groups: _.prop("groups")(_.propOr(graph, "graph")(ownProps)),
+    resources: _.prop("resources")(_.propOr(graph, "graph")(ownProps)),
+    loading: graphLoading !== false,
+    loadingInfo: graphLoading,
+    selectedNode,
+  };
+};
 
-export default connect(mapStateToProps, { clearNode })(LineageGraph);
+export default connect(mapStateToProps, { clearNode, cancelPoll })(
+  LineageGraph
+);

package/src/components/LineageRoutes.js

@@ -2,7 +2,8 @@ import React from "react";
 import { Route } from "react-router-dom";
 import { Unauthorized } from "@truedat/core/components";
 import { useAuthorized } from "@truedat/core/hooks";
-import { GRAPH, GRAPHS, SAMPLE } from "@truedat/core/routes";
+import { GRAPH, GRAPHS, LINEAGE_EVENTS, SAMPLE } from "@truedat/core/routes";
+import LineageEvents from "./LineageEvents";
 import LineageGraph from "./LineageGraph";
 import LineageSample from "./LineageSample";
 import LineageForm from "./LineageForm";
@@ -13,16 +14,13 @@ const Graphs = () => (
   <>
     <LineageForm />
     <GraphRedirector />
-  </>
-);
-
-const Graph = () => (
-  <>
-    <GraphLoader />
+    {/* Load LineageGraph here so that its <Loader> is already available when clicking the LinageForm */}
     <LineageGraph />
   </>
 );
 
+const Graph = () => <GraphLoader />;
+
 export const LineageRoutes = () => {
   const authorized = useAuthorized("lineage");
   return (
@@ -35,6 +33,10 @@ export const LineageRoutes = () => {
         path={GRAPH}
         render={() => (authorized ? <Graph /> : <Unauthorized />)}
       />
+      <Route
+        path={LINEAGE_EVENTS}
+        render={() => (authorized ? <LineageEvents /> : <Unauthorized />)}
+      />
       <Route
         path={SAMPLE}
         render={() => (authorized ? <LineageSample /> : <Unauthorized />)}

package/src/components/StructuresBulkUpdate.js

@@ -15,7 +15,6 @@ import { FormattedMessage } from "react-intl";
 import { ConfirmModal, HistoryBackButton } from "@truedat/core/components";
 import { selectTemplate } from "@truedat/df/routines";
 import { applyTemplateWithoutDefaults } from "@truedat/df/utils";
-import { composeProviders } from "redux-saga-test-plan/providers";
 import { compose } from "redux";
 import { withRouter } from "react-router-dom";
 import { bulkUpdateStructures } from "../routines";
@@ -82,7 +81,7 @@ export class StructuresBulkUpdate extends React.Component {
     template: PropTypes.object,
     templateId: PropTypes.number,
     templatesLoaded: PropTypes.bool,
-    location: PropTypes.object,
+    canAutoPublish: PropTypes.bool,
   };
 
   state = initialState;
@@ -123,15 +122,24 @@ export class StructuresBulkUpdate extends React.Component {
   };
 
   render() {
-    const { loading, template, templateId, structureCount, templatesLoaded } =
-      this.props;
+    const {
+      loading,
+      template,
+      templateId,
+      structureCount,
+      templatesLoaded,
+      canAutoPublish,
+      structureActiveFilters,
+    } = this.props;
     const { content } = this.state;
     const disableAction = !template || _.isEmpty(template);
 
     return templatesLoaded ? (
       <Container as={Segment} text>
         <TemplateLoader />
-        {!_.isEmpty(template) && structureCount != 0 ? (
+        {!_.isEmpty(template) &&
+        structureCount != 0 &&
+        _.has("type.raw")(structureActiveFilters) ? (
           <>
             <Header as="h2">
               <Icon name="book" />
@@ -161,10 +169,7 @@ export class StructuresBulkUpdate extends React.Component {
                 />
               )}
               <ConfirmModal
-                actions={actions(
-                  this.handleSubmit,
-                  this.props.location?.state.canAutoPublish
-                )}
+                actions={actions(this.handleSubmit, canAutoPublish)}
                 trigger={
                   <Button
                     primary
@@ -231,6 +236,8 @@ const mapStateToProps = ({
   structures,
   structureTypes,
   structureTypesLoading,
+  userDomains,
+  structureActiveFilters,
 }) => {
   const structure = _.head(structures);
   const templateId = findTemplate(structureTypes, templates, structure);
@@ -242,6 +249,13 @@ const mapStateToProps = ({
     structureCount,
     structureTypesLoaded: !structureTypesLoading,
     templatesLoaded: !templatesLoading && !_.isEmpty(templates),
+    userDomains,
+    canAutoPublish: _.any(
+      ({ domains, permission }) =>
+        !_.isEmpty(domains) &&
+        ["publish_structure_note_from_draft"].includes(permission)
+    )(userDomains),
+    structureActiveFilters,
   };
 };
 

package/src/components/StructuresRoutes.js

@@ -88,6 +88,13 @@ const AuthorizedRoutes = () => {
         path={STRUCTURES_BULK_UPDATE}
         render={() => (
           <>
+            <UserDomainsLoader
+              permissions={[
+                "publish_structure_note_from_draft",
+                "edit_structure_note",
+                "create_structure_note",
+              ]}
+            />
             <StructureCrumbs
               name={formatMessage({ id: "structures.bulkUpdate.crumbs.top" })}
             />

package/src/components/__tests__/LineageEventsTable.spec.js

@@ -0,0 +1,39 @@
+import React from "react";
+import { render } from "@truedat/test/render";
+import { LineageEventsTable } from "../LineageEventsTable";
+import en from "../../messages/en";
+
+describe("<LineageEventsTable />", () => {
+  const lineageEvents = [
+    {
+      graph_id: 1234,
+      graph_data: "graph_data",
+      status: "COMPLETED",
+      message: "message",
+      inserted_at: "2021-11-26T10:09:27.168999Z",
+    },
+  ];
+
+  const lineageEventsLoading = false;
+  const fetchLineageEvents = jest.fn();
+
+  it("matches the latest snapshot", () => {
+    const props = { fetchLineageEvents, lineageEvents, lineageEventsLoading };
+    const { container } = render(<LineageEventsTable {...props} />, {
+      messages: { en },
+    });
+    expect(container).toMatchSnapshot();
+  });
+
+  it("matches the latest snapshot (no jobs)", () => {
+    const props = {
+      fetchLineageEvents,
+      lineageEvents: [],
+      lineageEventsLoading,
+    };
+    const { container } = render(<LineageEventsTable {...props} />, {
+      messages: { en },
+    });
+    expect(container).toMatchSnapshot();
+  });
+});

package/src/components/__tests__/StructuresBulkUpdate.spec.js

@@ -17,6 +17,7 @@ describe("<StructuresBulkUpdate />", () => {
     templatesLoaded: true,
     structureTypes: [{ name: "doc", template_id: 1 }],
     structureTypesLoaded: true,
+    structureActiveFilters: { "type.raw": "booo" },
   };
 
   it("matches the latest snapshot", () => {
@@ -136,6 +137,7 @@ describe("<StructuresBulkUpdate />", () => {
       selectedDomain: { id: 1 },
       templatesLoaded: true,
       bulkUpdateStructures: jest.fn(),
+      structureActiveFilters: { "type.raw": "booo" },
     };
     const wrapper = shallow(<StructuresBulkUpdate {...props} />);
     wrapper.setState({

package/src/components/__tests__/__snapshots__/LineageEventsTable.spec.js.snap

@@ -0,0 +1,90 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`<LineageEventsTable /> matches the latest snapshot (no jobs) 1`] = `
+<div>
+  <h4
+    class="ui header"
+  >
+    <i
+      aria-hidden="true"
+      class="search icon"
+    />
+    <div
+      class="content"
+    >
+      No graphs found
+    </div>
+  </h4>
+</div>
+`;
+
+exports[`<LineageEventsTable /> matches the latest snapshot 1`] = `
+<div>
+  <table
+    class="ui selectable sortable table"
+  >
+    <thead
+      class=""
+    >
+      <tr
+        class=""
+      >
+        <th
+          class="two wide"
+        >
+          Graph
+        </th>
+        <th
+          class="two wide"
+        >
+          Status
+        </th>
+        <th
+          class="two wide"
+        >
+          Message
+        </th>
+        <th
+          class="two wide"
+        >
+          Date
+        </th>
+      </tr>
+    </thead>
+    <tbody
+      class=""
+    >
+      <tr
+        class=""
+      >
+        <td
+          class=""
+        >
+          <span>
+            graph_data
+          </span>
+        </td>
+        <td
+          class=""
+        >
+          Completed
+        </td>
+        <td
+          class=""
+        >
+          message
+        </td>
+        <td
+          class="center aligned"
+        >
+          <time
+            datetime="1637921367168"
+          >
+            2021-11-26 10:09
+          </time>
+        </td>
+      </tr>
+    </tbody>
+  </table>
+</div>
+`;