@truedat/dq 7.11.0 → 7.11.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@truedat/dq",
-  "version": "7.11.0",
+  "version": "7.11.2",
   "description": "Truedat Web Data Quality Module",
   "sideEffects": false,
   "module": "src/index.js",
@@ -53,7 +53,7 @@
     "@testing-library/jest-dom": "^6.6.3",
     "@testing-library/react": "^16.3.0",
     "@testing-library/user-event": "^14.6.1",
-    "@truedat/test": "7.11.0",
+    "@truedat/test": "7.11.2",
     "identity-obj-proxy": "^3.0.0",
     "jest": "^29.7.0",
     "redux-saga-test-plan": "^4.0.6"
@@ -86,5 +86,5 @@
     "semantic-ui-react": "^3.0.0-beta.2",
     "swr": "^2.3.3"
   },
-  "gitHead": "c8c0d3e62835df86440869c7b797bbe2fc12f3a6"
+  "gitHead": "c7c9de44c34d3105c78711fa63f6ee8cf7730eaf"
 }

package/src/api.js

@@ -14,6 +14,14 @@ const API_RULE_FILTERS_SEARCH = "/api/rule_filters/search";
 const API_RULE_IMPLEMENTATION = "/api/rule_implementations/:id";
 const API_RULE_IMPLEMENTATIONS = "/api/rule_implementations";
 const API_RULE_IMPLEMENTATIONS_CSV = "/api/rule_implementations/csv";
+const API_RULE_IMPLEMENTATIONS_XLSX_DOWNLOAD =
+  "/api/rule_implementations/xlsx/download";
+const API_RULE_IMPLEMENTATIONS_XLSX_UPLOAD =
+  "/api/rule_implementations/xlsx/upload";
+const API_RULE_IMPLEMENTATIONS_XLSX_UPLOAD_JOBS =
+  "/api/rule_implementations/xlsx/upload_jobs";
+const API_RULE_IMPLEMENTATIONS_XLSX_UPLOAD_JOB =
+  "/api/rule_implementations/xlsx/upload_jobs/:id";
 const API_RULE_IMPLEMENTATIONS_FROM_RULE =
   "/api/rules/:id/rule_implementations";
 const API_RULE_IMPLEMENTATIONS_SEARCH = "/api/rule_implementations/search";
@@ -42,6 +50,10 @@ export {
   API_RULE_IMPLEMENTATION,
   API_RULE_IMPLEMENTATIONS,
   API_RULE_IMPLEMENTATIONS_CSV,
+  API_RULE_IMPLEMENTATIONS_XLSX_DOWNLOAD,
+  API_RULE_IMPLEMENTATIONS_XLSX_UPLOAD,
+  API_RULE_IMPLEMENTATIONS_XLSX_UPLOAD_JOBS,
+  API_RULE_IMPLEMENTATIONS_XLSX_UPLOAD_JOB,
   API_RULE_IMPLEMENTATIONS_FROM_RULE,
   API_RULE_IMPLEMENTATIONS_SEARCH,
   API_RULE_IMPLEMENTATION_FILTERS_SEARCH,

package/src/components/ImplementationSearchResults.js

@@ -5,40 +5,33 @@ import { connect } from "react-redux";
 import { Dimmer, Loader, Segment } from "semantic-ui-react";
 import { useActiveRoute } from "@truedat/core/hooks";
 import { IMPLEMENTATIONS_PENDING } from "@truedat/core/routes";
-import { getExecutionQuery, getRuleImplementationColumns } from "../selectors";
+import SearchWidget from "@truedat/core/search/SearchWidget";
+import { useSearchContext } from "@truedat/core/search/SearchContext";
+import Pagination from "@truedat/core/search/Pagination";
+import { getRuleImplementationColumns } from "../selectors";
 import RuleImplementationsActions from "./RuleImplementationsActions";
 import RuleImplementationsLabelResults from "./RuleImplementationsLabelResults";
-import RuleImplementationsPagination from "./RuleImplementationsPagination";
-import RuleImplementationsSearch from "./RuleImplementationsSearch";
 import RuleImplementationsTable from "./RuleImplementationsTable";
-import RuleImplementationSelectedFilters from "./RuleImplementationSelectedFilters";
 
-const usePrevious = (value) => {
-  const ref = useRef();
-  useEffect(() => {
-    ref.current = value;
-  });
-  return ref.current;
-};
+export const ImplementationSearchResults = ({ embedded, role, columns }) => {
+  const { searchData, loading, toggleHiddenFilterValue, setOnSearchChange } =
+    useSearchContext();
 
-export const ImplementationSearchResults = ({
-  embedded,
-  implementationQuery,
-  role,
-  loading,
-  ruleImplementations,
-  columns,
-}) => {
-  const [executeImplementationsOn, setMode] = useState(
-    _.matches({ filters: { executable: [true] } })(implementationQuery)
-  );
-  const [selectedImplementations, setSelectedImplementations] = useState([]);
-  const previousQuery = usePrevious(implementationQuery);
-  const clean = !_.isEqual(implementationQuery)(previousQuery);
+  const ruleImplementations = searchData?.data;
 
+  const [executeImplementationsOn, setExecuteImplementationsOn] =
+    useState(false);
+  const setMode = (mode) => {
+    setExecuteImplementationsOn(mode);
+    toggleHiddenFilterValue({
+      filter: "executable",
+      value: mode ? [true] : [],
+    });
+  };
+  const [selectedImplementations, setSelectedImplementations] = useState([]);
   useEffect(() => {
-    setSelectedImplementations([]);
-  }, [clean]);
+    setOnSearchChange(() => () => setSelectedImplementations([]));
+  }, []);
 
   const allChecked = () => {
     const ids = _.map(_.prop("id"))(ruleImplementations);
@@ -86,18 +79,16 @@ export const ImplementationSearchResults = ({
     : _.reject(({ name }) => name === "status")(columns);
 
   return (
-    <Segment attached="bottom">
+    <Segment border="1px red" attached="bottom">
       {embedded ? null : (
         <RuleImplementationsActions
           executeImplementationsOn={executeImplementationsOn}
-          implementationQuery={implementationQuery}
           role={role}
           setMode={setMode}
           selectedImplementations={selectedImplementations}
         />
       )}
-      <RuleImplementationsSearch />
-      <RuleImplementationSelectedFilters />
+      <SearchWidget />
       <Dimmer.Dimmable dimmed={loading}>
         {loading ? (
           <Dimmer active inverted>
@@ -117,30 +108,21 @@ export const ImplementationSearchResults = ({
           selectedImplementations={selectedImplementations}
           columns={filteredColumns}
         />
-        <RuleImplementationsPagination />
+        <Pagination />
       </Dimmer.Dimmable>
     </Segment>
   );
 };
 
 ImplementationSearchResults.propTypes = {
-  implementationQuery: PropTypes.object,
-  role: PropTypes.string,
-  loading: PropTypes.bool,
-  ruleImplementations: PropTypes.array,
   embedded: PropTypes.bool,
+  role: PropTypes.string,
   columns: PropTypes.array,
 };
 
 const mapStateToProps = (state, props) => ({
   columns: getRuleImplementationColumns(state),
-  implementationQuery: getExecutionQuery({
-    ...state,
-    ruleImplementationDefaultFilters: _.propOr({}, "defaultFilters")(props),
-  }),
   role: state.authentication?.role,
-  loading: state.ruleImplementationsLoading,
-  ruleImplementations: state.ruleImplementations,
 });
 
 export default connect(mapStateToProps)(ImplementationSearchResults);

package/src/components/ImplementationUploadJobBreadcrumbs.js

@@ -0,0 +1,25 @@
+import PropTypes from "prop-types";
+import { Breadcrumb } from "semantic-ui-react";
+import { Link } from "react-router";
+import { FormattedMessage } from "react-intl";
+import { IMPLEMENTATIONS_UPLOAD_JOBS } from "@truedat/core/routes";
+
+export const ImplementationUploadJobBreadcrumbs = ({ filename }) => (
+  <Breadcrumb>
+    <Breadcrumb.Section as={Link} to={IMPLEMENTATIONS_UPLOAD_JOBS}>
+      <FormattedMessage id="sidemenu.implementations_upload_jobs" />
+    </Breadcrumb.Section>
+    {filename ? (
+      <>
+        <Breadcrumb.Divider icon="right angle" />
+        <Breadcrumb.Section active>{filename}</Breadcrumb.Section>
+      </>
+    ) : null}
+  </Breadcrumb>
+);
+
+ImplementationUploadJobBreadcrumbs.propTypes = {
+  filename: PropTypes.string,
+};
+
+export default ImplementationUploadJobBreadcrumbs;

package/src/components/Implementations.js

@@ -1,28 +1,42 @@
 import { lazy } from "react";
 import PropTypes from "prop-types";
 import { Segment } from "semantic-ui-react";
+import { SearchContextProvider } from "@truedat/core/search/SearchContext";
+import {
+  useRuleImplementationFilters,
+  useRuleImplementationSearch,
+} from "../hooks/useImplementations";
+
 import ImplementationsHeader from "./ImplementationsHeader";
 import ImplementationFiltersLoader from "./ImplementationFiltersLoader";
 import ImplementationSearchResults from "./ImplementationSearchResults";
-import RuleImplementationsLoader from "./RuleImplementationsLoader";
 
-const UserSearchFiltersLoader = lazy(
-  () => import("@truedat/dd/components/UserSearchFiltersLoader")
-);
+export const Implementations = ({ defaultFilters }) => {
+  const searchProps = {
+    initialSortColumn: "implementation_key.raw",
+    initialSortDirection: "ascending",
+    useSearch: useRuleImplementationSearch,
+    useFilters: useRuleImplementationFilters,
+    pageSize: 20,
+    userFiltersType: "user_search_filters",
+    userFilterScope: "rule_implementation",
+    defaultFilters,
+  };
 
-export const Implementations = ({ defaultFilters }) => (
-  <>
-    <ImplementationFiltersLoader defaultFilters={defaultFilters} />
-    <UserSearchFiltersLoader scope="rule_implementation" />
-    <RuleImplementationsLoader defaultFilters={defaultFilters} />
-    <Segment>
-      <ImplementationsHeader />
-      <Segment attached="bottom">
-        <ImplementationSearchResults defaultFilters={defaultFilters} />
-      </Segment>
-    </Segment>
-  </>
-);
+  return (
+    <>
+      <SearchContextProvider {...searchProps}>
+        <ImplementationFiltersLoader defaultFilters={defaultFilters} />
+        <Segment>
+          <ImplementationsHeader />
+          <Segment attached="bottom">
+            <ImplementationSearchResults defaultFilters={defaultFilters} />
+          </Segment>
+        </Segment>
+      </SearchContextProvider>
+    </>
+  );
+};
 
 Implementations.propTypes = {
   defaultFilters: PropTypes.object,

package/src/components/ImplementationsRoutes.js

@@ -51,6 +51,8 @@ import RuleResultDetails from "./RuleResultDetails";
 import RuleResultRemediationLoader from "./RuleResultRemediationLoader";
 import RuleResultsRoutes from "./RuleResultsRoutes";
 import RuleSubscriptionLoader from "./RuleSubscriptionLoader";
+import ImplementationsUploadJobs from "./ImplementationsUploadJobs";
+import ImplementationsUploadJob from "./ImplementationsUploadJob";
 
 const TemplatesLoader = React.lazy(
   () => import("@truedat/core/components/TemplatesLoader")
@@ -104,6 +106,13 @@ export const ImplementationsRoutes = ({
         index
         element={<Implementations defaultFilters={{ status: ["published"] }} />}
       />
+      <Route
+        // IMPLEMENTATIONS_UPLOAD_JOBS = "/implementations/uploadJobs";
+        path="/uploadJobs"
+      >
+        <Route index element={<ImplementationsUploadJobs />} />
+        <Route path=":id" element={<ImplementationsUploadJob />} />
+      </Route>
 
       <Route
         // IMPLEMENTATIONS_PENDING = "/implementations/pending";

package/src/components/ImplementationsUploadButton.js

@@ -1,42 +1,52 @@
-import _ from "lodash/fp";
-import PropTypes from "prop-types";
-import { connect } from "react-redux";
+import { useWebContext } from "@truedat/core/webContext";
 import { Dropdown } from "semantic-ui-react";
 import { FormattedMessage, useIntl } from "react-intl";
 import { UploadModal } from "@truedat/core/components";
-import { API_RULE_IMPLEMENTATIONS_UPLOAD } from "../api";
-import { uploadImplementations } from "../routines";
+import { linkTo } from "@truedat/core/routes";
+import { useSearchContext } from "@truedat/core/search/SearchContext";
+import { useImplementationsUpload } from "../hooks/useImplementations";
 
-const uploadAction = {
-  method: "POST",
-  href: API_RULE_IMPLEMENTATIONS_UPLOAD,
-};
+const buildAlertResult =
+  (setAlertMessage) =>
+  ({ data }) => {
+    setAlertMessage({
+      error: false,
+      header: "implementations.bulkUpload.alert.success.header",
+      icon: "check",
+      color: "blue",
+      text: "",
+      anchor: {
+        reference: linkTo.IMPLEMENTATIONS_UPLOAD_JOB({ id: data.job_id }),
+        label: "implementations.bulkUpload.job.header",
+      },
+    });
+  };
 
-export const ImplementationsUploadButton = ({
-  canAutoPublish,
-  uploadImplementations,
-  loading,
-}) => {
+export const ImplementationsUploadButton = () => {
   const { formatMessage, locale } = useIntl();
+
+  const { trigger: triggerUpload, isMutating: loading } =
+    useImplementationsUpload();
+  const { setAlertMessage } = useWebContext();
+  const { searchData } = useSearchContext();
+
+  const alertResult = buildAlertResult(setAlertMessage);
+  const canAutoPublish = !!searchData?._actions?.autoPublish;
+
   const extraAction = {
     key: "yesWithAutoPublish",
     primary: true,
     content: formatMessage({ id: "uploadModal.accept.publish" }),
-    onClick: (formData) => {
-      // UploadModal is a Modal, not a Form, but sends FormData
-      formData.append("auto_publish", true);
-      return uploadImplementations({
-        action: "upload",
-        formData,
-        lang: locale,
-        ...uploadAction,
-      });
+    onClick: (data) => {
+      data.append("auto_publish", canAutoPublish);
+      data.append("lang", locale);
+      triggerUpload(data).then(alertResult);
     },
   };
   return (
     <UploadModal
       icon="upload"
-      extraAction={canAutoPublish ? extraAction : undefined}
+      extraAction={canAutoPublish && extraAction}
       trigger={
         <Dropdown.Item
           icon="upload"
@@ -53,34 +63,12 @@ export const ImplementationsUploadButton = ({
         <FormattedMessage id="ruleImplementations.actions.upload.confirmation.content" />
       }
       param="implementations"
-      // handleSubmit is the onClick for the "yes" action
-      handleSubmit={(formData) => {
-        return uploadImplementations({
-          action: "upload",
-          formData,
-          lang: locale,
-          ...uploadAction,
-        });
+      handleSubmit={(data) => {
+        data.append("lang", locale);
+        triggerUpload(data).then(alertResult);
       }}
     />
   );
 };
 
-ImplementationsUploadButton.propTypes = {
-  canAutoPublish: PropTypes.bool,
-  uploadImplementations: PropTypes.func,
-  loading: PropTypes.bool,
-};
-
-const mapStateToProps = ({
-  uploadingImplementationsFile,
-  implementationsActions,
-}) => ({
-  canAutoPublish: _.has("autoPublish")(implementationsActions),
-  loading: uploadingImplementationsFile,
-  implementationsActions,
-});
-
-export default connect(mapStateToProps, { uploadImplementations })(
-  ImplementationsUploadButton
-);
+export default ImplementationsUploadButton;

package/src/components/ImplementationsUploadJob.js

@@ -0,0 +1,217 @@
+import _ from "lodash/fp";
+import { useState } from "react";
+import {
+  Header,
+  Icon,
+  Segment,
+  Dimmer,
+  Loader,
+  Table,
+  Label,
+  Button,
+} from "semantic-ui-react";
+import { FormattedMessage, useIntl } from "react-intl";
+import { useParams } from "react-router";
+import { DateTime } from "@truedat/core/components";
+import { useImplementationsUploadJob } from "../hooks/useImplementations";
+import { StatusPill, ResponseCell } from "./implementationsUploadJobParser";
+import ImplementationUploadJobBreadcrumbs from "./ImplementationUploadJobBreadcrumbs";
+
+export default function ImplementationsUploadJob() {
+  const { id } = useParams();
+  const { data, loading } = useImplementationsUploadJob(id);
+  const job = data?.data;
+
+  const { formatMessage } = useIntl();
+  const [expandedGroups, setExpandedGroups] = useState({});
+
+  // Group events by status
+  const groupedEvents = job?.events ? _.groupBy("status", job.events) : {};
+
+  const toggleGroup = (status) => {
+    setExpandedGroups((prev) => ({
+      ...prev,
+      [status]: !prev[status],
+    }));
+  };
+
+  const renderEventGroup = (status, events) => {
+    const isExpanded = expandedGroups[status];
+    const shouldCollapse = status === "INFO" || status === "ERROR";
+
+    if (!shouldCollapse) {
+      // Render non-collapsible events normally
+      return events.map((event, idx) => (
+        <Table.Row key={`${status}-${idx}`}>
+          <Table.Cell>
+            <StatusPill status={event.status} />
+          </Table.Cell>
+          <Table.Cell>
+            <DateTime value={event.inserted_at} />
+          </Table.Cell>
+          <Table.Cell>
+            <ResponseCell {...event} />
+          </Table.Cell>
+        </Table.Row>
+      ));
+    }
+
+    // Render collapsible groups for INFO and ERROR
+    const groupRows = [];
+
+    // Add group header row
+    groupRows.push(
+      <Table.Row
+        key={`${status}-header`}
+        style={{ backgroundColor: "#f8f9fa" }}
+      >
+        <Table.Cell>
+          <Button
+            basic
+            size="mini"
+            icon={isExpanded ? "chevron down" : "chevron right"}
+            content={` ${formatMessage({ id: `implementations.bulkUpload.event.status.${status}` })} (${events.length})`}
+            onClick={() => toggleGroup(status)}
+            style={{ padding: "0.5em" }}
+          />
+        </Table.Cell>
+        <Table.Cell>
+          <DateTime value={events[0]?.inserted_at} />
+        </Table.Cell>
+        <Table.Cell></Table.Cell>
+      </Table.Row>
+    );
+
+    // Add expanded events if group is expanded
+    if (isExpanded) {
+      events.forEach((event, idx) => {
+        groupRows.push(
+          <Table.Row key={`${status}-${idx}`} style={{ paddingLeft: "2em" }}>
+            <Table.Cell style={{ paddingLeft: "2em" }}>
+              <StatusPill status={event.status} />
+            </Table.Cell>
+            <Table.Cell>
+              <DateTime value={event.inserted_at} />
+            </Table.Cell>
+            <Table.Cell>
+              <ResponseCell {...event} />
+            </Table.Cell>
+          </Table.Row>
+        );
+      });
+    }
+
+    return groupRows;
+  };
+
+  return (
+    <>
+      <ImplementationUploadJobBreadcrumbs filename={job?.filename} />
+      <Segment>
+        <Header as="h2">
+          <Icon
+            circular
+            name={formatMessage({
+              id: "implementations.bulkUpload.job.header.icon",
+              defaultMessage: "cogs",
+            })}
+          />
+          <Header.Content>
+            <FormattedMessage id="implementations.bulkUpload.job.header" />
+          </Header.Content>
+        </Header>
+        <Dimmer.Dimmable dimmed={loading}>
+          <Segment attached="bottom">
+            {job ? (
+              <>
+                <div
+                  style={{
+                    display: "flex",
+                    alignItems: "center",
+                    gap: "1.5em",
+                    marginBottom: "1.5em",
+                    flexWrap: "wrap",
+                  }}
+                >
+                  <div style={{ display: "flex", flexDirection: "column" }}>
+                    <span style={{ fontWeight: 600, fontSize: "1.3em" }}>
+                      {job.filename}
+                    </span>
+                    <span style={{ color: "gray", fontSize: "0.7em" }}>
+                      {job.hash}
+                    </span>
+                  </div>
+                  <span>
+                    <Label basic size="small" style={{ marginRight: 4 }}>
+                      <FormattedMessage
+                        id="implementations.bulkUpload.job.table.status"
+                        defaultMessage="Status"
+                      />
+                    </Label>
+                    <StatusPill status={job.latest_status} />
+                  </span>
+                  <span>
+                    <Label basic size="small" style={{ marginRight: 4 }}>
+                      <FormattedMessage
+                        id="implementations.bulkUpload.job.table.inserted_at"
+                        defaultMessage="Inserted At"
+                      />
+                    </Label>
+                    <DateTime value={job.latest_event_at} />
+                  </span>
+                </div>
+                <Table>
+                  <Table.Header>
+                    <Table.Row>
+                      <Table.HeaderCell>
+                        <FormattedMessage
+                          id="implementations.bulkUpload.job.table.status"
+                          defaultMessage="Status"
+                        />
+                      </Table.HeaderCell>
+                      <Table.HeaderCell>
+                        <FormattedMessage
+                          id="implementations.bulkUpload.job.table.inserted_at"
+                          defaultMessage="Inserted At"
+                        />
+                      </Table.HeaderCell>
+                      <Table.HeaderCell>
+                        <FormattedMessage
+                          id="implementations.bulkUpload.job.table.detail"
+                          defaultMessage="Detail"
+                        />
+                      </Table.HeaderCell>
+                    </Table.Row>
+                  </Table.Header>
+                  <Table.Body>
+                    {_.isArray(job.events) && !_.isEmpty(job.events) ? (
+                      Object.entries(groupedEvents)
+                        .map(([status, events]) =>
+                          renderEventGroup(status, events)
+                        )
+                        .flat()
+                    ) : (
+                      <Table.Row>
+                        <Table.Cell colSpan={3}>
+                          <FormattedMessage
+                            id="implementations.bulkUpload.job.table.no_events"
+                            defaultMessage="No events found."
+                          />
+                        </Table.Cell>
+                      </Table.Row>
+                    )}
+                  </Table.Body>
+                </Table>
+              </>
+            ) : null}
+            {loading ? (
+              <Dimmer active inverted>
+                <Loader />
+              </Dimmer>
+            ) : null}
+          </Segment>
+        </Dimmer.Dimmable>
+      </Segment>
+    </>
+  );
+}