@truedat/dd 5.2.8 → 5.3.0

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [5.3.0] 2023-03-08
+
+### Added
+
+- [TD-4438] Reference dataset field selector
+
+## [5.2.9] 2023-02-28
+
+### Added
+
+- [TD-5599] Support for Tasks
+
 ## [5.2.8] 2023-02-28
 
 ### Fixed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@truedat/dd",
-  "version": "5.2.8",
+  "version": "5.3.0",
   "description": "Truedat Web Data Dictionary",
   "sideEffects": false,
   "jsnext:main": "src/index.js",
@@ -87,9 +87,9 @@
   },
   "dependencies": {
     "@apollo/client": "^3.7.1",
-    "@truedat/auth": "5.2.4",
-    "@truedat/core": "5.2.4",
-    "@truedat/df": "5.2.4",
+    "@truedat/auth": "5.3.0",
+    "@truedat/core": "5.3.0",
+    "@truedat/df": "5.3.0",
     "lodash": "^4.17.21",
     "moment": "^2.29.4",
     "path-to-regexp": "^1.7.0",
@@ -114,5 +114,5 @@
     "react-dom": ">= 16.8.6 < 17",
     "semantic-ui-react": ">= 2.0.3 < 2.2"
   },
-  "gitHead": "562bc4667fa44eeb0e224a886c41849adbd69a18"
+  "gitHead": "4893c314084a4ea7ca48a4e251d4e47565f3f030"
 }

package/src/api/queries.js

@@ -446,3 +446,34 @@ export const DATA_STRUCTURE_VERSION_QUERY = gql`
     }
   }
 `;
+
+export const TASKS_QUERY = gql`
+  query Tasks {
+    tasks {
+      id
+      index
+      status
+      started_at
+      last_message_at
+      count
+      processed
+      elapsed
+    }
+  }
+`;
+
+export const TASK_QUERY = gql`
+  query Task($id: String) {
+    task(id: $id) {
+      id
+      index
+      status
+      started_at
+      last_message_at
+      count
+      processed
+      memory_trace
+      elapsed
+    }
+  }
+`;

package/src/components/DictionaryRoutes.js

@@ -11,6 +11,7 @@ import {
   STRUCTURE_TAGS,
   STRUCTURE_TYPES,
   SYSTEMS,
+  TASKS,
 } from "@truedat/core/routes";
 import PendingStructureNotes from "./PendingStructureNotes";
 import ProfileExecution from "./ProfileExecution";
@@ -28,6 +29,7 @@ import StructureTypesRoutes from "./StructureTypesRoutes";
 import SystemsRoutes from "./SystemsRoutes";
 import SystemsLoader from "./SystemsLoader";
 import TagsRoutes from "./TagsRoutes";
+import TasksRoutes from "./TasksRoutes";
 
 const TemplatesLoader = React.lazy(() =>
   import("@truedat/df/templates/components/TemplatesLoader")
@@ -124,6 +126,7 @@ export const DictionaryRoutes = () => {
           </>
         )}
       />
+      <Route path={TASKS} render={() => <TasksRoutes />} />
     </>
   );
 };

package/src/components/ReferenceDatasetFieldSelector.js

@@ -0,0 +1,146 @@
+import _ from "lodash/fp";
+import PropTypes from "prop-types";
+import React, { useEffect, useState } from "react";
+import { useIntl } from "react-intl";
+import { useQuery } from "@apollo/client";
+import { Loading } from "@truedat/core/components";
+import { Dropdown } from "semantic-ui-react";
+import { REFERENCE_DATASETS_QUERY } from "@truedat/dd/api/queries";
+
+const refDatasetsToOptions = (referenceDatasets) => {
+  return _.map(({ id, name }) => {
+    return { key: id, text: name, value: id };
+  })(referenceDatasets);
+};
+
+const headersToOptions = (refDatasetId) => (headers) => {
+  return _.map((header) => {
+    return {
+      key: `${refDatasetId}_${header}`,
+      text: header,
+      value: header,
+    };
+  })(headers);
+};
+
+const onReferenceDatasetChange =
+  (referenceDatasets, setHeaders, setSelectedRefDataset, onChange) =>
+  (_e, { value: refDatasetId }) => {
+    /* Only reference dataset selected, its field is still yet to be chosen.
+     * Therefore:
+     * Clear any previously selected field still showing in ConditionSummary
+     * Make validation fail using field nulled name so that "Next"
+     * RuleImplementationForm button is disabled.
+     */
+    onChange(
+      null,
+      {
+        type: "reference_dataset_field",
+        name: null,
+        referenceDataset: { id: refDatasetId },
+      },
+      null
+    );
+    setSelectedRefDataset(_.find({ id: refDatasetId })(referenceDatasets));
+    _.flow(
+      _.find({ id: refDatasetId }),
+      _.prop("headers"),
+      setHeaders
+    )(referenceDatasets);
+  };
+
+export const ReferenceDatasetFieldSelector = ({
+  referenceDatasets,
+  referenceDataset,
+  header,
+  onChange,
+}) => {
+  const { formatMessage } = useIntl();
+  const [headers, setHeaders] = useState(
+    _.find({ id: referenceDataset?.id })(referenceDatasets)?.headers || []
+  );
+  const [selectedRefDataset, setSelectedRefDataset] =
+    useState(referenceDataset);
+
+  const [selectedHeader, setSelectedHeader] = useState(header);
+
+  // if the operator dropdown changes, referenceDataset will come as undefined
+  useEffect(() => {
+    if (referenceDataset == null) {
+      setSelectedRefDataset(null);
+      setSelectedHeader(null);
+      setHeaders([]);
+    }
+  }, [referenceDataset]);
+
+  return (
+    <>
+      <label>
+        {formatMessage({
+          id: "referenceDatasetFields.dropdown.placeholder",
+        })}
+      </label>
+      <Dropdown
+        className="reference-dataset"
+        selection
+        placeholder={formatMessage({
+          id: "referenceDataset.dropdown.placeholder",
+        })}
+        options={refDatasetsToOptions(referenceDatasets)}
+        value={selectedRefDataset?.id || ""}
+        onChange={onReferenceDatasetChange(
+          referenceDatasets,
+          setHeaders,
+          setSelectedRefDataset,
+          onChange
+        )}
+      />
+      <Dropdown
+        selection
+        placeholder={formatMessage({
+          id: "referenceDatasetFields.dropdown.placeholder",
+        })}
+        disabled={!selectedRefDataset}
+        options={headersToOptions(selectedRefDataset?.id)(headers)}
+        value={selectedHeader || ""}
+        onChange={(e, { value: newHeader }) => {
+          setSelectedHeader(newHeader);
+          onChange(
+            null,
+            {
+              referenceDataset: _.pick(["id", "name"], selectedRefDataset),
+              name: newHeader,
+              type: "reference_dataset_field",
+            },
+            null
+          );
+        }}
+      />
+    </>
+  );
+};
+
+ReferenceDatasetFieldSelector.propTypes = {
+  header: PropTypes.string,
+  onChange: PropTypes.func,
+  referenceDatasets: PropTypes.array,
+  referenceDataset: PropTypes.object,
+};
+
+export const ReferenceDatasetFieldSelectorLoader = (props) => {
+  const { loading, error, data } = useQuery(REFERENCE_DATASETS_QUERY, {
+    fetchPolicy: "cache-and-network",
+  });
+  if (error) return null;
+  if (loading) return <Loading />;
+  const referenceDatasets = data?.referenceDatasets || [];
+  return (
+    <ReferenceDatasetFieldSelector
+      loading={loading}
+      referenceDatasets={referenceDatasets}
+      {...props}
+    />
+  );
+};
+
+export default ReferenceDatasetFieldSelectorLoader;

package/src/components/Task.js

@@ -0,0 +1,149 @@
+import _ from "lodash/fp";
+import React from "react";
+import { useParams, Link } from "react-router-dom";
+import { useQuery } from "@apollo/client";
+import { FormattedMessage } from "react-intl";
+import {
+  Breadcrumb,
+  Button,
+  Header,
+  Icon,
+  Grid,
+  Item,
+  Segment,
+  Progress,
+  Divider,
+} from "semantic-ui-react";
+import { Dimmer, Loader } from "semantic-ui-react";
+import Moment from "react-moment";
+import { TASKS } from "@truedat/core/routes";
+import { TASK_QUERY } from "../api/queries";
+
+function Task() {
+  const { id } = useParams();
+  const { loading, data, refetch } = useQuery(TASK_QUERY, {
+    fetchPolicy: "cache-and-network",
+    variables: { id },
+  });
+
+  const task = _.get("task")(data);
+
+  const memoryStats = _.flow(
+    _.get("memory_trace"),
+    _.reduce(
+      (stats, [_ts, memory]) => ({
+        max: Math.max(stats.max || -Infinity, memory),
+        min: Math.min(stats.min || Infinity, memory),
+        last: memory,
+      }),
+      {}
+    )
+  )(task);
+
+  const parseMemory = (value) => `${Math.round(value / 1024 / 1024)}MB`;
+
+  const properties = [
+    { name: "task.index", content: task?.index },
+    { name: "task.status", content: task?.status },
+
+    {
+      name: "task.started_at",
+      content: <Moment date={task?.started_at} format="YYYY-MM-DD HH:mm" />,
+    },
+    {
+      name: "task.last_message_at",
+      content: (
+        <Moment date={task?.last_message_at} format="YYYY-MM-DD HH:mm" />
+      ),
+    },
+    { name: "task.elapsed", content: `${task?.elapsed / 1000}s` },
+  ];
+
+  const propertyElement = (name, content) => (
+    <Grid.Column key={name} width={3}>
+      <Item>
+        <Item.Content>
+          <Item.Header as="h5">
+            <FormattedMessage id={name} />
+          </Item.Header>
+          <Item.Description>{content}</Item.Description>
+        </Item.Content>
+      </Item>
+    </Grid.Column>
+  );
+
+  return (
+    <>
+      <Breadcrumb>
+        <Breadcrumb.Section as={Link} to={TASKS} active={false}>
+          <FormattedMessage id="tasks.header" />
+        </Breadcrumb.Section>
+        <>
+          <Breadcrumb.Divider icon="right angle" />
+          <Breadcrumb.Section active>{task?.index}</Breadcrumb.Section>
+        </>
+      </Breadcrumb>
+      <Segment>
+        <Header as="h2">
+          <Icon circular name="check" />
+          <Header.Content>
+            <FormattedMessage id="task.header" />
+            <Header.Subheader>
+              <FormattedMessage id="task.subheader" />
+            </Header.Subheader>
+          </Header.Content>
+        </Header>
+        <Segment basic>
+          <Button
+            primary
+            onClick={() => refetch()}
+            content={<FormattedMessage id="task.refresh" />}
+          />
+        </Segment>
+        <Dimmer.Dimmable dimmed={loading}>
+          <Dimmer active={loading} inverted>
+            <Loader />
+          </Dimmer>
+          {task ? (
+            <>
+              <Grid columns={4} divided>
+                <Grid.Row>
+                  {properties.map(({ name, content }) =>
+                    propertyElement(name, content)
+                  )}
+                </Grid.Row>
+              </Grid>
+              <Divider />
+              <Progress
+                percent={Math.round((task.processed / task.count) * 100)}
+                progress
+              >
+                <FormattedMessage id="task.progress" />
+              </Progress>
+              <Divider />
+
+              <Grid columns={4} divided>
+                <Grid.Row>
+                  {propertyElement(
+                    "task.memory.max",
+                    parseMemory(memoryStats?.max)
+                  )}
+                  {propertyElement(
+                    "task.memory.min",
+                    parseMemory(memoryStats?.min)
+                  )}
+                  {propertyElement(
+                    "task.memory.last",
+                    parseMemory(memoryStats?.last)
+                  )}
+                </Grid.Row>
+              </Grid>
+            </>
+          ) : null}
+        </Dimmer.Dimmable>
+      </Segment>
+    </>
+  );
+}
+
+export default Task;

package/src/components/Tasks.js

@@ -0,0 +1,138 @@
+import _ from "lodash/fp";
+import React from "react";
+import PropTypes from "prop-types";
+import { useHistory } from "react-router-dom";
+import { useQuery } from "@apollo/client";
+import { connect } from "react-redux";
+import { FormattedMessage } from "react-intl";
+import { Button, Header, Icon, Table, Segment } from "semantic-ui-react";
+import { Dimmer, Loader } from "semantic-ui-react";
+import Moment from "react-moment";
+import { reindexAllGrants, reindexAllStructures } from "@truedat/core/routines";
+import { linkTo } from "@truedat/core/routes";
+import { TASKS_QUERY } from "../api/queries";
+
+export const Tasks = ({ reindexAllGrants, reindexAllStructures }) => {
+  const { loading, data, refetch } = useQuery(TASKS_QUERY, {
+    fetchPolicy: "cache-and-network",
+  });
+  const history = useHistory();
+
+  const tasks = _.flow(
+    _.get("tasks"),
+    _.orderBy(["started_at"], ["desc"])
+  )(data);
+
+  return (
+    <Segment>
+      <Header as="h2">
+        <Icon circular name="check" />
+        <Header.Content>
+          <FormattedMessage id="tasks.header" />
+          <Header.Subheader>
+            <FormattedMessage id="tasks.subheader" />
+          </Header.Subheader>
+        </Header.Content>
+      </Header>
+      <Segment basic>
+        <Button
+          primary
+          onClick={() => refetch()}
+          content={<FormattedMessage id="task.refresh" />}
+        />
+        <Button
+          primary
+          floated="right"
+          onClick={() => {
+            reindexAllStructures();
+            refetch();
+          }}
+          content={<FormattedMessage id="task.reindex_all_structures" />}
+        />
+        <Button
+          primary
+          floated="right"
+          onClick={() => {
+            reindexAllGrants();
+            refetch();
+          }}
+          content={<FormattedMessage id="task.reindex_all_grants" />}
+        />
+      </Segment>
+      <Dimmer.Dimmable dimmed={loading}>
+        <Dimmer active={loading} inverted>
+          <Loader />
+        </Dimmer>
+        {data ? (
+          <Table selectable>
+            <Table.Header>
+              <Table.Row>
+                <Table.HeaderCell
+                  content={<FormattedMessage id={`task.index`} />}
+                />
+                <Table.HeaderCell
+                  content={<FormattedMessage id={`task.status`} />}
+                />
+                <Table.HeaderCell
+                  content={<FormattedMessage id={`task.started_at`} />}
+                />
+                <Table.HeaderCell
+                  content={<FormattedMessage id={`task.last_message_at`} />}
+                />
+                <Table.HeaderCell
+                  content={<FormattedMessage id={`task.progress`} />}
+                />
+                <Table.HeaderCell
+                  content={<FormattedMessage id={`task.elapsed`} />}
+                />
+              </Table.Row>
+            </Table.Header>
+            <Table.Body>
+              {tasks.map((task) => (
+                <Table.Row
+                  key={task.started_at}
+                  onClick={() => history.push(linkTo.TASK(task))}
+                >
+                  <Table.Cell content={task.index} />
+                  <Table.Cell content={task.status} />
+
+                  <Table.Cell
+                    content={
+                      <Moment
+                        date={task.started_at}
+                        format="YYYY-MM-DD HH:mm"
+                      />
+                    }
+                  />
+                  <Table.Cell
+                    content={
+                      <Moment
+                        date={task.last_message_at}
+                        format="YYYY-MM-DD HH:mm"
+                      />
+                    }
+                  />
+                  <Table.Cell
+                    content={
+                      task.processed ? `${task.processed}/${task.count}` : "-"
+                    }
+                  />
+                  <Table.Cell
+                    content={task.elapsed ? `${task.elapsed / 1000}s` : "-"}
+                  />
+                </Table.Row>
+              ))}
+            </Table.Body>
+          </Table>
+        ) : null}
+      </Dimmer.Dimmable>
+    </Segment>
+  );
+};
+
+Tasks.propTypes = {
+  reindexAllGrants: PropTypes.func,
+  reindexAllStructures: PropTypes.func,
+};
+
+export default connect(null, { reindexAllGrants, reindexAllStructures })(Tasks);

package/src/components/TasksRoutes.js

@@ -0,0 +1,27 @@
+import { Route } from "react-router-dom";
+import React from "react";
+import { Unauthorized } from "@truedat/core/components";
+import { useAuthorized } from "@truedat/core/hooks";
+import { TASKS, TASK } from "@truedat/core/routes";
+import Tasks from "./Tasks";
+import Task from "./Task";
+
+export const TasksRoutes = () => {
+  const authorized = useAuthorized();
+  return (
+    <>
+      <Route
+        exact
+        path={TASK}
+        render={() => (authorized ? <Task /> : <Unauthorized />)}
+      />
+      <Route
+        exact
+        path={TASKS}
+        render={() => (authorized ? <Tasks /> : <Unauthorized />)}
+      />
+    </>
+  );
+};
+
+export default TasksRoutes;

package/src/components/__tests__/ReferenceDatasetFieldSelector.spec.js

@@ -0,0 +1,86 @@
+import React from "react";
+import { waitFor } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { render } from "@truedat/test/render";
+import { REFERENCE_DATASETS_QUERY } from "../../api/queries";
+import ReferenceDatasetFieldSelector from "../ReferenceDatasetFieldSelector";
+
+const referenceDatasets = [
+  {
+    __typename: "ReferenceDataset",
+    id: "1",
+    name: "countries",
+    headers: ["CODE", "DESC_ES", "DESC_EN"],
+    rowCount: 9,
+    updatedAt: "2023-03-04T12:10:53.245332Z",
+  },
+  {
+    __typename: "ReferenceDataset",
+    id: "2",
+    name: "currencies",
+    headers: ["ENTITY", "Currency", "Alphabetic Code", "Numeric Code"],
+    rowCount: 5,
+    updatedAt: "2023-03-05T09:57:59.765822Z",
+  },
+];
+
+const referenceDatasetsMock = {
+  request: {
+    query: REFERENCE_DATASETS_QUERY,
+  },
+  result: { data: { referenceDatasets } },
+};
+
+describe("<ReferenceDatasetFieldSelector />", () => {
+  it("selecting reference dataset fills fields dropdown; clicking a field calls onChange", async () => {
+    const onChange = jest.fn();
+    const renderOpts = {
+      mocks: [referenceDatasetsMock],
+    };
+    const { queryAllByText, getByRole } = render(
+      <ReferenceDatasetFieldSelector onChange={onChange} />,
+      renderOpts
+    );
+    const countriesOptionQuery = () =>
+      getByRole("option", { name: /countries/i });
+    await waitFor(() => {
+      // also matches "reference dataset field", get first
+      expect(queryAllByText(/Reference dataset/i)[0]).toBeInTheDocument();
+      // same text for label and dropdown, match the second one, the dropdown
+      expect(queryAllByText(/Reference dataset field/i)[1]).toBeInTheDocument();
+      expect(countriesOptionQuery()).toBeInTheDocument();
+      expect(getByRole("option", { name: /currencies/i })).toBeInTheDocument();
+    });
+
+    userEvent.click(countriesOptionQuery());
+
+    const optionQuery = () => getByRole("option", { name: /CODE/i });
+    await waitFor(() => {
+      expect(optionQuery()).toBeInTheDocument();
+    });
+    userEvent.click(optionQuery());
+    expect(onChange).toHaveBeenNthCalledWith(
+      1,
+      null,
+      {
+        name: null,
+        referenceDataset: { id: "1" },
+        type: "reference_dataset_field",
+      },
+      null
+    );
+    expect(onChange).toHaveBeenNthCalledWith(
+      2,
+      null,
+      {
+        name: "CODE",
+        referenceDataset: {
+          id: "1",
+          name: "countries",
+        },
+        type: "reference_dataset_field",
+      },
+      null
+    );
+  });
+});

package/src/components/__tests__/Task.spec.js

@@ -0,0 +1,44 @@
+import React from "react";
+import { render } from "@truedat/test/render";
+import Task from "../Task";
+import { TASK_QUERY } from "../../api/queries";
+import en from "../../messages/en";
+
+jest.mock("@truedat/core/hooks", () => ({
+  useAuthorized: jest.fn(() => true),
+}));
+
+jest.mock("react-router-dom", () => ({
+  ...jest.requireActual("react-router-dom"),
+  useParams: () => ({ id: "123" }),
+}));
+
+const task = {
+  __typename: "Task",
+  id: "123",
+  started_at: "2020-01-01T00:00:00.000Z",
+  last_message_at: "2020-01-01T00:01:00.000Z",
+  index: "grants",
+  status: "processing",
+  count: 10000,
+  processed: 1000,
+  memory_trace: [["2020-01-01T00:01:00.000Z", 100000]],
+  elapsed: 10,
+};
+
+const taskMock = {
+  request: { query: TASK_QUERY, variables: { id: "123" } },
+  result: { data: { task } },
+};
+
+describe("<Task />", () => {
+  it("matches the latest snapshot", async () => {
+    const renderOpts = {
+      messages: { en },
+      mocks: [taskMock],
+    };
+    const { container, findByText } = render(<Task />, renderOpts);
+    await findByText("processing");
+    expect(container).toMatchSnapshot();
+  });
+});