@truedat/dd 4.33.0 → 4.33.4

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [4.33.4] 2021-11-26
+
+### Added
+
+- [TD-4262] Lineage graph polling
+
+## [4.33.3] 2021-11-25
+
+### Added
+
+- [TD-4344] `structure_types` improve API request
+
 ## [4.33.0] 2021-11-17
 
 ### Added

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@truedat/dd",
-  "version": "4.33.0",
+  "version": "4.33.4",
   "description": "Truedat Web Data Dictionary",
   "sideEffects": false,
   "jsnext:main": "src/index.js",
@@ -34,7 +34,7 @@
     "@testing-library/jest-dom": "^5.14.1",
     "@testing-library/react": "^12.0.0",
     "@testing-library/user-event": "^13.2.1",
-    "@truedat/test": "4.33.0",
+    "@truedat/test": "4.33.4",
     "babel-jest": "^27.0.6",
     "babel-plugin-dynamic-import-node": "^2.3.3",
     "babel-plugin-lodash": "^3.3.4",
@@ -84,9 +84,9 @@
     ]
   },
   "dependencies": {
-    "@truedat/auth": "4.33.0",
-    "@truedat/core": "4.33.0",
-    "@truedat/df": "4.33.0",
+    "@truedat/auth": "4.33.4",
+    "@truedat/core": "4.33.4",
+    "@truedat/df": "4.33.4",
     "lodash": "^4.17.21",
     "path-to-regexp": "^1.7.0",
     "prop-types": "^15.7.2",
@@ -107,5 +107,5 @@
     "react-dom": ">= 16.8.6 < 17",
     "semantic-ui-react": ">= 0.88.2 < 2.1"
   },
-  "gitHead": "b257cb534058698ce0509a9db1750cce1e9e81c3"
+  "gitHead": "08160df210692628d37599956bc78c587c019afc"
 }

package/src/api.js

@@ -14,8 +14,10 @@ const API_GRANT_REQUESTS = "/api/grant_requests";
 const API_GRANTS_CSV = "/api/grants/csv";
 const API_GRANTS_SEARCH = "/api/grants/search";
 const API_GRAPH = "/api/graphs/:id";
+const API_GRAPH_HASH = "/api/graphs/hash/:hash";
 const API_GRAPHS = "/api/graphs";
 const API_GRAPH_CSV = "/api/graphs/csv";
+const API_LINEAGE_EVENTS = "/api/lineage_events";
 const API_MY_GRANT_FILTERS_SEARCH = "/api/grant_filters/search/mine";
 const API_MY_GRANTS_SEARCH = "/api/grants/search/mine";
 const API_NODE = "/api/nodes/:id";
@@ -29,6 +31,7 @@ const API_STRUCTURE_TAG = "/api/data_structure_tags/:id";
 const API_STRUCTURE_TAGS = "/api/data_structure_tags";
 const API_STRUCTURE_TYPE = "/api/data_structure_types/:id";
 const API_STRUCTURE_TYPES = "/api/data_structure_types";
+const API_STRUCTURE_TYPES_LITE = "/api/data_structure_types/lite";
 const API_STRUCTURES_TAG = "/api/data_structures/:data_structure_id/tags/:id";
 const API_STRUCTURES_TAGS = "/api/data_structures/:data_structure_id/tags";
 const API_STRUCTURE_USER_FILTERS = "/api/data_structure_user_filters";
@@ -57,9 +60,11 @@ export {
   API_GRANT_REQUESTS,
   API_GRANTS_SEARCH,
   API_GRAPH,
+  API_GRAPH_HASH,
   API_GRAPHS,
   API_GRAPH_CSV,
   API_GRANTS_CSV,
+  API_LINEAGE_EVENTS,
   API_MY_GRANTS_SEARCH,
   API_MY_GRANT_FILTERS_SEARCH,
   API_NODE,
@@ -73,6 +78,7 @@ export {
   API_STRUCTURE_TAGS,
   API_STRUCTURE_TYPE,
   API_STRUCTURE_TYPES,
+  API_STRUCTURE_TYPES_LITE,
   API_STRUCTURE_USER_FILTERS,
   API_STRUCTURE_USER_FILTER,
   API_STRUCTURES_TAG,

package/src/components/DictionaryRoutes.js

@@ -30,7 +30,6 @@ const TemplatesLoader = React.lazy(() =>
 const Loaders = () => (
   <>
     <StructuresLoader />
-    <StructureTypesLoader />
     <StructureFiltersLoader />
     <TemplatesLoader scope="dd" />
   </>
@@ -49,6 +48,7 @@ export const DictionaryRoutes = () => {
         path={STRUCTURES}
         render={() => (
           <>
+            <StructureTypesLoader liteRequest={true} />
             <SystemsLoader />
             <StructuresRoutes />
           </>
@@ -58,17 +58,35 @@ export const DictionaryRoutes = () => {
         path={SYSTEMS}
         render={() => (
           <>
+            <StructureTypesLoader liteRequest={true} />
             <SystemsLoader />
             <SystemsRoutes />
           </>
         )}
       />
-      <Route path={STRUCTURE_TAGS} render={() => <StructureTagsRoutes />} />
-      <Route path={STRUCTURE_TYPES} render={() => <StructureTypesRoutes />} />
+      <Route
+        path={STRUCTURE_TAGS}
+        render={() => (
+          <>
+            <StructureTypesLoader liteRequest={true} />
+            <StructureTagsRoutes />
+          </>
+        )}
+      />
+      <Route
+        path={STRUCTURE_TYPES}
+        render={() => (
+          <>
+            <StructureTypesLoader />
+            <StructureTypesRoutes />
+          </>
+        )}
+      />
       <Route
         path={PROFILE_GROUP}
         render={() => (
           <>
+            <StructureTypesLoader liteRequest={true} />
             <Route render={() => <ProfileGroupLoader />} />
             <Switch>
               <Route

package/src/components/GraphLoader.js

@@ -2,27 +2,41 @@ import _ from "lodash/fp";
 import { useEffect } from "react";
 import PropTypes from "prop-types";
 import { connect } from "react-redux";
-import { useRouteMatch } from "react-router-dom";
+import { useRouteMatch, useHistory } from "react-router-dom";
 import { GRAPH } from "@truedat/core/routes";
-import { fetchGraph, clearGraph } from "../routines";
+import { fetchGraph, clearGraph, cancelPoll } from "../routines";
 
-export const GraphLoader = ({ clearGraph, fetchGraph, graph }) => {
+export const GraphLoader = ({ clearGraph, fetchGraph, graph, cancelPoll }) => {
   const match = useRouteMatch(GRAPH);
   const id = _.path("params.id")(match);
+  
+  const history = useHistory();
+
   useEffect(() => {
     return () => {
       clearGraph();
     };
   }, [clearGraph]);
+
   useEffect(() => {
     if (id) {
       if (_.isEmpty(graph)) {
         fetchGraph({ id });
-      } else if (!_.propEq("id", id)(graph)) {
+      } else if (id != graph.id) {
         fetchGraph({ id });
       }
     }
-  }, [graph, fetchGraph, id]);
+  }, [/*graph, *//*fetchGraph,*/ id]);
+
+  useEffect(() => {
+    return () => {
+      // && history.location.pathname === "any specific path")
+      if (history.action === "POP") {
+        cancelPoll();
+      }
+    };
+  }, [history]);
+
   return null;
 };
 
@@ -30,10 +44,12 @@ GraphLoader.propTypes = {
   clearGraph: PropTypes.func.isRequired,
   fetchGraph: PropTypes.func.isRequired,
   graph: PropTypes.object,
+  cancelPoll: PropTypes.func,
 };
 
 const mapStateToProps = ({ graph }) => ({ graph });
 
-export default connect(mapStateToProps, { clearGraph, fetchGraph })(
+export default connect(mapStateToProps, { clearGraph, fetchGraph, cancelPoll })(
   GraphLoader
 );
+

package/src/components/LineageEvents.js

@@ -0,0 +1,23 @@
+import React from "react";
+import { Header, Icon, Segment, Dimmer, Loader } from "semantic-ui-react";
+import { FormattedMessage } from "react-intl";
+import LineageEventsTable from "./LineageEventsTable"
+
+export const LineageEvents = () => (
+  <Segment>
+    <Header as="h2">
+      <Icon circular name="cogs" />
+      <Header.Content>
+        <FormattedMessage id="lineage.events.header" />
+        <Header.Subheader>
+          <FormattedMessage id="lineage.events.subheader" />
+        </Header.Subheader>
+      </Header.Content>
+    </Header>
+    <Segment attached="bottom">
+      <LineageEventsTable />
+    </Segment>
+  </Segment>
+);
+
+export default LineageEvents;

package/src/components/LineageEventsTable.js

@@ -0,0 +1,91 @@
+import _ from "lodash/fp";
+import React, { useEffect } from "react";
+import PropTypes from "prop-types";
+import { FormattedMessage } from "react-intl";
+import { Header, Icon, Table } from "semantic-ui-react";
+import { useHistory, } from "react-router-dom";
+import { connect } from "react-redux";
+import { useIntl } from "react-intl";
+import { columnDecorator } from "@truedat/core/services";
+import { linkTo } from "@truedat/core/routes";
+import { lineageEventColumns as columns } from "../selectors/getLineageEventColumns";
+import { fetchLineageEvents } from "../routines";
+
+
+export const LineageEventsTable = ({ fetchLineageEvents, lineageEvents, lineageEventsLoading }) => {
+  const { formatMessage } = useIntl();
+  const history = useHistory();
+
+  useEffect(() => {
+    fetchLineageEvents();
+  }, [fetchLineageEvents]);
+
+  return (
+    <>
+      {!_.isEmpty(lineageEvents) && (
+        <Table sortable selectable>
+          <Table.Header>
+            <Table.Row>
+              {columns.map((column, i) => {
+                return (
+                  <Table.HeaderCell
+                    key={i}
+                    width={column.width}
+                    content={formatMessage({
+                      id: `lineage.events.props.${column.name}`,
+                      defaultMessage: column.name,
+                    })}
+                  />
+                );
+              })}
+            </Table.Row>
+          </Table.Header>
+          <Table.Body>
+            {lineageEvents.map((event, i) => (
+              <Table.Row
+                key={i}
+                onClick={
+                  event.graph_id
+                    ? () => history.push(linkTo.GRAPH({ id: event?.graph_id }))
+                    : null
+                }
+              >
+                {columns &&
+                  columns.map((column, key) => (
+                    <Table.Cell
+                      key={key}
+                      textAlign={column.textAlign}
+                      content={columnDecorator(column)(event)}
+                    />
+                  ))}
+              </Table.Row>
+            ))}
+          </Table.Body>
+        </Table>
+      )}
+      {_.isEmpty(lineageEvents) && !lineageEventsLoading && (
+        <Header as="h4">
+          <Icon name="search" />
+          <Header.Content>
+            <FormattedMessage id="lineage.events.search.results.empty" />
+          </Header.Content>
+        </Header>
+      )}
+    </>
+  );
+};
+
+LineageEventsTable.propTypes = {
+  lineageEvents: PropTypes.array,
+  fetchLineageEvents: PropTypes.func,
+  lineageEventsLoading: PropTypes.bool,
+};
+
+const mapStateToProps = (state) => ({
+  lineageEvents: state.lineageEvents,
+  lineageEventsLoading: state.lineageEventsLoading,
+});
+
+export default connect(mapStateToProps, { fetchLineageEvents })(
+  LineageEventsTable
+);

package/src/components/LineageForm.js

@@ -54,10 +54,11 @@ export const LineageForm = ({
   const [open, setOpen] = useState(true);
   const [selectLevels, setSelectLevels] = useState(levelsSelection);
   const [levels, setLevels] = useState(lineageLevels);
-  const handleSubmit = (type) => () =>
+  const handleSubmit = (type) => () => {
     _.isNil(levels)
       ? createGraph({ ids: selectedResources, type })
       : createGraph({ ids: selectedResources, type, levels });
+  }
   const disabled = _.isEmpty(selectedResources);
   useEffect(() => {
     setOpen(!graphLoading && _.isEmpty(graph));

package/src/components/LineageGraph.js

@@ -4,12 +4,19 @@ import PropTypes from "prop-types";
 import { connect } from "react-redux";
 import { Dimmer, Loader } from "semantic-ui-react";
 import svgPanZoom from "svg-pan-zoom";
-import { FormattedMessage } from "react-intl";
-import { clearNode, selectNode, hoverNode, initSvg } from "../routines";
+import { FormattedMessage, useIntl } from "react-intl";
+import {
+  clearNode,
+  selectNode,
+  hoverNode,
+  initSvg,
+  cancelPoll,
+} from "../routines";
 import LineageLink from "./LineageLink";
 import LineageDefs from "./LineageDefs";
 import LineageGroup from "./LineageGroup";
 import LineageResource from "./LineageResource";
+import "../styles/lineageGraph.less";
 
 const svgOptions = { minZoom: 0.2, maxZoom: 2.0, fit: false, center: false };
 const disablePanZoom = (svg) => {
@@ -50,11 +57,18 @@ export const LineageGraph = ({
   groups = [],
   resources = [],
   loading,
+  loadingInfo,
+  cancelPoll,
 }) => {
   const ref = useRef(null);
   const [state, dispatch] = useReducer(reducer, initialState);
   const { selectedNode, hover, svg } = state;
 
+  useEffect(() => {
+    // Anything in here is fired on component mount.
+    return () => cancelPoll();
+  }, []);
+
   useEffect(() => {
     if (!_.isEmpty(resources)) {
       const svg = svgPanZoom(ref.current, svgOptions);
@@ -67,6 +81,8 @@ export const LineageGraph = ({
   const onSelect = (g) => () => dispatch(selectNode.trigger(g));
   const onHover = (g) =>
     selectedNode ? undefined : () => dispatch(hoverNode.trigger(g));
+  const { formatMessage } = useIntl();
+
   return (
     <>
       <Dimmer.Dimmable dimmed={loading} className="dl">
@@ -74,8 +90,34 @@ export const LineageGraph = ({
           <Loader size="large">
             <FormattedMessage
               id="lineage.graph.loading"
-              defaultMessage="Analyzing..."
+              defaultMessage={"Analyzing..."}
             />
+            {loadingInfo?.data?.graph_hash && loadingInfo?.data?.status && (
+              <p>
+                {formatMessage({
+                  id:
+                    loadingInfo?.data?.status === "already_started"
+                      ? "lineage.graph.loading.background.existing"
+                      : loadingInfo?.data?.status === "just_started" &&
+                        "lineage.graph.loading.background.launched",
+                })}
+              </p>
+            )}
+            {loadingInfo?.data?.graph_hash && (
+              <p>
+                Graph hash:{" "}
+                <span className="graph-hash">
+                  {loadingInfo?.data?.graph_hash}
+                </span>
+              </p>
+            )}
+            <p>
+              {loadingInfo?.data?.retry &&
+                formatMessage(
+                  { id: "lineage.graph.loading.polling" },
+                  { retry: loadingInfo?.data?.retry }
+                )}
+            </p>
           </Loader>
         </Dimmer>
         <svg
@@ -121,17 +163,23 @@ LineageGraph.propTypes = {
   paths: PropTypes.array,
   resources: PropTypes.array,
   height: PropTypes.string,
+  loading: PropTypes.bool,
+  loadingInfo: PropTypes.object,
+  cancelPoll: PropTypes.func,
 };
 
 const mapStateToProps = (
   { graph = {}, graphLoading, selectedNode },
   ownProps
-) => ({
-  paths: _.prop("paths")(_.propOr(graph, "graph")(ownProps)),
-  groups: _.prop("groups")(_.propOr(graph, "graph")(ownProps)),
-  resources: _.prop("resources")(_.propOr(graph, "graph")(ownProps)),
-  loading: graphLoading,
-  selectedNode,
-});
+) => {
+  return {
+    paths: _.prop("paths")(_.propOr(graph, "graph")(ownProps)),
+    groups: _.prop("groups")(_.propOr(graph, "graph")(ownProps)),
+    resources: _.prop("resources")(_.propOr(graph, "graph")(ownProps)),
+    loading: graphLoading !== false,
+    loadingInfo: graphLoading,
+    selectedNode,
+  };
+};
 
-export default connect(mapStateToProps, { clearNode })(LineageGraph);
+export default connect(mapStateToProps, { clearNode, cancelPoll })(LineageGraph);

package/src/components/LineageRoutes.js

@@ -2,7 +2,8 @@ import React from "react";
 import { Route } from "react-router-dom";
 import { Unauthorized } from "@truedat/core/components";
 import { useAuthorized } from "@truedat/core/hooks";
-import { GRAPH, GRAPHS, SAMPLE } from "@truedat/core/routes";
+import { GRAPH, GRAPHS, LINEAGE_EVENTS, SAMPLE } from "@truedat/core/routes";
+import LineageEvents from "./LineageEvents"
 import LineageGraph from "./LineageGraph";
 import LineageSample from "./LineageSample";
 import LineageForm from "./LineageForm";
@@ -13,16 +14,13 @@ const Graphs = () => (
   <>
     <LineageForm />
     <GraphRedirector />
-  </>
-);
-
-const Graph = () => (
-  <>
-    <GraphLoader />
+    {/* Load LineageGraph here so that its <Loader> is already available when clicking the LinageForm */}
     <LineageGraph />
   </>
 );
 
+const Graph = () => <GraphLoader />;
+
 export const LineageRoutes = () => {
   const authorized = useAuthorized("lineage");
   return (
@@ -35,6 +33,10 @@ export const LineageRoutes = () => {
         path={GRAPH}
         render={() => (authorized ? <Graph /> : <Unauthorized />)}
       />
+      <Route
+        path={LINEAGE_EVENTS}
+        render={() => (authorized ? <LineageEvents /> : <Unauthorized />)}
+      />
       <Route
         path={SAMPLE}
         render={() => (authorized ? <LineageSample /> : <Unauthorized />)}

package/src/components/StructureTypesLoader.js

@@ -6,19 +6,21 @@ import { clearStructureTypes, fetchStructureTypes } from "../routines";
 export const StructureTypesLoader = ({
   clearStructureTypes,
   fetchStructureTypes,
+  liteRequest = false,
 }) => {
   useEffect(() => {
-    fetchStructureTypes();
+    fetchStructureTypes({ liteRequest });
     return () => {
       clearStructureTypes();
     };
-  }, [clearStructureTypes, fetchStructureTypes]);
+  }, [clearStructureTypes, fetchStructureTypes, liteRequest]);
   return null;
 };
 
 StructureTypesLoader.propTypes = {
   clearStructureTypes: PropTypes.func,
   fetchStructureTypes: PropTypes.func,
+  liteRequest: PropTypes.bool,
 };
 
 export default connect(null, { clearStructureTypes, fetchStructureTypes })(

package/src/components/__tests__/LineageEventsTable.spec.js

@@ -0,0 +1,40 @@
+import React from "react";
+import { render } from "@truedat/test/render";
+import { LineageEventsTable } from "../LineageEventsTable";
+import en from "../../messages/en";
+
+
+describe("<LineageEventsTable />", () => {
+  const lineageEvents = [
+    {
+      graph_id: 1234,
+      graph_data: "graph_data",
+      status: "COMPLETED",
+      message: "message",
+      inserted_at: "2021-11-26T10:09:27.168999Z",
+    },
+  ];
+
+  const lineageEventsLoading = false;
+  const fetchLineageEvents = jest.fn();
+
+  it("matches the latest snapshot", () => {
+    const props = { fetchLineageEvents, lineageEvents, lineageEventsLoading };
+    const { container } = render(<LineageEventsTable {...props} />, {
+      messages: { en },
+    });
+    expect(container).toMatchSnapshot();
+  });
+
+  it("matches the latest snapshot (no jobs)", () => {
+    const props = {
+      fetchLineageEvents,
+      lineageEvents: [],
+      lineageEventsLoading,
+    };
+    const { container } = render(<LineageEventsTable {...props} />, {
+      messages: { en },
+    });
+    expect(container).toMatchSnapshot();
+  });
+});

package/src/components/__tests__/__snapshots__/LineageEventsTable.spec.js.snap

@@ -0,0 +1,88 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`<LineageEventsTable /> matches the latest snapshot (no jobs) 1`] = `
+<div>
+  <h4
+    class="ui header"
+  >
+    <i
+      aria-hidden="true"
+      class="search icon"
+    />
+    <div
+      class="content"
+    >
+      No graphs found
+    </div>
+  </h4>
+</div>
+`;
+
+exports[`<LineageEventsTable /> matches the latest snapshot 1`] = `
+<div>
+  <table
+    class="ui selectable sortable table"
+  >
+    <thead
+      class=""
+    >
+      <tr
+        class=""
+      >
+        <th
+          class="two wide"
+        >
+          Graph
+        </th>
+        <th
+          class="two wide"
+        >
+          Status
+        </th>
+        <th
+          class="two wide"
+        >
+          Message
+        </th>
+        <th
+          class="two wide"
+        >
+          Date
+        </th>
+      </tr>
+    </thead>
+    <tbody
+      class=""
+    >
+      <tr
+        class=""
+      >
+        <td
+          class=""
+        >
+          graph_data
+        </td>
+        <td
+          class=""
+        >
+          Completed
+        </td>
+        <td
+          class=""
+        >
+          message
+        </td>
+        <td
+          class="center aligned"
+        >
+          <time
+            datetime="1637921367168"
+          >
+            2021-11-26 10:09
+          </time>
+        </td>
+      </tr>
+    </tbody>
+  </table>
+</div>
+`;

package/src/components/__tests__/__snapshots__/LineageRoutes.spec.js.snap

@@ -10,6 +10,10 @@ exports[`<LineageRoutes /> matches the latest snapshot 1`] = `
     path="/graphs/:id"
     render={[Function]}
   />
+  <Route
+    path="/lineage_events"
+    render={[Function]}
+  />
   <Route
     path="/sample"
     render={[Function]}