datajunction-ui 0.0.1-rc.24 → 0.0.1-rc.26

package/src/app/pages/NodePage/NodeMaterializationTab.jsx

@@ -1,6 +1,9 @@
 import { useEffect, useState } from 'react';
 import ClientCodePopover from './ClientCodePopover';
 import TableIcon from '../../icons/TableIcon';
+import AddMaterializationPopover from './AddMaterializationPopover';
+import * as React from 'react';
+import AddBackfillPopover from './AddBackfillPopover';
 
 const cronstrue = require('cronstrue');
 
@@ -15,18 +18,25 @@ export default function NodeMaterializationTab({ node, djClient }) {
     };
     fetchData().catch(console.error);
   }, [djClient, node]);
+  //
+  // const rangePartition = partition => {
+  //   return (
+  //     <div>
+  //       <span className="badge partition_value">
+  //         <span className="badge partition_value">{partition.range[0]}</span>to
+  //         <span className="badge partition_value">{partition.range[1]}</span>
+  //       </span>
+  //     </div>
+  //   );
+  // };
 
-  const rangePartition = partition => {
-    return (
-      <div>
-        <span className="badge partition_value">
-          <span className="badge partition_value">{partition.range[0]}</span>to
-          <span className="badge partition_value">{partition.range[1]}</span>
-        </span>
-      </div>
-    );
-  };
-
+  const partitionColumnsMap = node
+    ? Object.fromEntries(
+        node?.columns
+          .filter(col => col.partition !== null)
+          .map(col => [col.name, col.display_name]),
+      )
+    : {};
   const cron = materialization => {
     var parsedCron = '';
     try {
@@ -39,10 +49,6 @@ export default function NodeMaterializationTab({ node, djClient }) {
     return materializations.map(materialization => (
       <tr key={materialization.name}>
         <td className="text-start node_name">
-          <a href={materialization.urls[0]}>{materialization.name}</a>
-          <ClientCodePopover code={materialization.clientCode} />
-        </td>
-        <td>
           <span className={`badge cron`}>{materialization.schedule}</span>
           <div className={`cron-description`}>{cron(materialization)} </div>
         </td>
@@ -50,41 +56,24 @@ export default function NodeMaterializationTab({ node, djClient }) {
           {materialization.engine.name}
           <br />
           {materialization.engine.version}
+          <ClientCodePopover code={materialization.clientCode} />
         </td>
         <td>
-          {materialization.config.partitions ? (
-            materialization.config.partitions.map(partition =>
-              partition.type_ === 'categorical' ? (
-                <div className="partition__full" key={partition.name}>
-                  <div className="partition__header">{partition.name}</div>
+          {node.columns
+            .filter(col => col.partition !== null)
+            .map(column => {
+              return (
+                <div className="partition__full" key={column.name}>
+                  <div className="partition__header">{column.display_name}</div>
                   <div className="partition__body">
-                    {partition.values !== null && partition.values.length > 0
-                      ? partition.values.map(val => (
-                          <span
-                            className="badge partition_value"
-                            key={`partition-value-${val}`}
-                          >
-                            {val}
-                          </span>
-                        ))
-                      : null}
-                    {partition.range !== null && partition.range.length > 0
-                      ? rangePartition(partition)
-                      : null}
-                    {(partition.range === null && partition.values === null) ||
-                    (partition.range?.length === 0 &&
-                      partition.values?.length === 0) ? (
-                      <span className={`badge partition_value_highlight`}>
-                        ALL
-                      </span>
-                    ) : null}
+                    <code>{column.name}</code>
+                    <span className="badge partition_value">
+                      {column.partition.type_}
+                    </span>
                   </div>
                 </div>
-              ) : null,
-            )
-          ) : (
-            <br />
-          )}
+              );
+            })}
         </td>
         <td>
           {materialization.output_tables.map(table => (
@@ -101,30 +90,26 @@ export default function NodeMaterializationTab({ node, djClient }) {
             </div>
           ))}
         </td>
-        {/*<td>{Object.keys(materialization.config.spark).map(key => <li className={`list-group-item`}>{key}: {materialization.config.spark[key]}</li>)}</td>*/}
-
         <td>
-          {materialization.config.partitions ? (
-            materialization.config.partitions.map(partition =>
-              partition.type_ === 'temporal' ? (
-                <div className="partition__full" key={partition.name}>
-                  <div className="partition__header">{partition.name}</div>
-                  <div className="partition__body">
-                    {partition.values !== null && partition.values.length > 0
-                      ? partition.values.map(val => (
-                          <span className="badge partition_value">{val}</span>
-                        ))
-                      : null}
-                    {partition.range !== null && partition.range.length > 0
-                      ? rangePartition(partition)
-                      : null}
-                  </div>
+          {materialization.backfills.map(backfill => (
+            <a href={backfill.urls[0]} className="partitionLink">
+              <div className="partition__full" key={backfill.spec.column_name}>
+                <div className="partition__header">
+                  {partitionColumnsMap[backfill.spec.column_name]}
                 </div>
-              ) : null,
-            )
-          ) : (
-            <br />
-          )}
+                <div className="partition__body">
+                  <span className="badge partition_value">
+                    {backfill.spec.range[0]}
+                  </span>
+                  to
+                  <span className="badge partition_value">
+                    {backfill.spec.range[1]}
+                  </span>
+                </div>
+              </div>
+            </a>
+          ))}
+          <AddBackfillPopover node={node} materialization={materialization} />
         </td>
         <td>
           {materialization.urls.map((url, idx) => (
@@ -137,38 +122,112 @@ export default function NodeMaterializationTab({ node, djClient }) {
     ));
   };
   return (
-    <div className="table-responsive">
-      {materializations.length > 0 ? (
-        <table
-          className="card-inner-table table"
-          aria-label="Materializations"
-          aria-hidden="false"
-        >
-          <thead className="fs-7 fw-bold text-gray-400 border-bottom-0">
-            <tr>
-              <th className="text-start">Name</th>
-              <th>Schedule</th>
-              <th>Engine</th>
-              <th>Partitions</th>
-              <th>Output Tables</th>
-              <th>Backfills</th>
-              <th>URLs</th>
-            </tr>
-          </thead>
-          <tbody>
-            {materializationRows(
-              materializations.filter(
-                materialization =>
-                  !(materialization.name === 'default' && node.type === 'cube'),
-              ),
-            )}
-          </tbody>
-        </table>
-      ) : (
-        <div className="message alert" style={{ marginTop: '10px' }}>
-          No materializations available for this node
+    <>
+      <div className="table-vertical">
+        <div>
+          <h2>Materializations</h2>
+          <AddMaterializationPopover node={node} />
+          {materializations.length > 0 ? (
+            <table
+              className="card-inner-table table"
+              aria-label="Materializations"
+              aria-hidden="false"
+            >
+              <thead className="fs-7 fw-bold text-gray-400 border-bottom-0">
+                <tr>
+                  <th className="text-start">Schedule</th>
+                  <th>Engine</th>
+                  <th>Partitions</th>
+                  <th>Output Tables</th>
+                  <th>Backfills</th>
+                  <th>URLs</th>
+                </tr>
+              </thead>
+              <tbody>
+                {materializationRows(
+                  materializations.filter(
+                    materialization =>
+                      !(
+                        materialization.name === 'default' &&
+                        node.type === 'cube'
+                      ),
+                  ),
+                )}
+              </tbody>
+            </table>
+          ) : (
+            <div className="message alert" style={{ marginTop: '10px' }}>
+              No materialization workflows configured for this node.
+            </div>
+          )}
         </div>
-      )}
-    </div>
+        <div>
+          <h2>Materialized Datasets</h2>
+          {node && node.availability !== null ? (
+            <table
+              className="card-inner-table table"
+              aria-label="Availability"
+              aria-hidden="false"
+            >
+              <thead className="fs-7 fw-bold text-gray-400 border-bottom-0">
+                <tr>
+                  <th className="text-start">Catalog</th>
+                  <th>Schema</th>
+                  <th>Table</th>
+                  <th>Valid Through</th>
+                  <th>Partitions</th>
+                </tr>
+              </thead>
+              <tbody>
+                <tr>
+                  <td>{node.availability.schema_}</td>
+                  <td>
+                    {
+                      <div
+                        className={`table__full`}
+                        key={node.availability.table}
+                      >
+                        <div className="table__header">
+                          <TableIcon />{' '}
+                          <span className={`entity-info`}>
+                            {node.availability.catalog +
+                              '.' +
+                              node.availability.schema_}
+                          </span>
+                        </div>
+                        <div className={`table__body upstream_tables`}>
+                          <a href={node.availability.url}>
+                            {node.availability.table}
+                          </a>
+                        </div>
+                      </div>
+                    }
+                  </td>
+                  <td>{node.availability.valid_through_ts}</td>
+                  <td>
+                    <span
+                      className={`badge partition_value`}
+                      style={{ fontSize: '100%' }}
+                    >
+                      <span className={`badge partition_value_highlight`}>
+                        {node.availability.min_temporal_partition}
+                      </span>
+                      to
+                      <span className={`badge partition_value_highlight`}>
+                        {node.availability.max_temporal_partition}
+                      </span>
+                    </span>
+                  </td>
+                </tr>
+              </tbody>
+            </table>
+          ) : (
+            <div className="message alert" style={{ marginTop: '10px' }}>
+              No materialized datasets available for this node.
+            </div>
+          )}
+        </div>
+      </div>
+    </>
   );
 }

package/src/app/pages/NodePage/PartitionColumnPopover.jsx

@@ -0,0 +1,153 @@
+import { useContext, useEffect, useRef, useState } from 'react';
+import * as React from 'react';
+import DJClientContext from '../../providers/djclient';
+import { Field, Form, Formik } from 'formik';
+import { FormikSelect } from '../AddEditNodePage/FormikSelect';
+import EditIcon from '../../icons/EditIcon';
+import { displayMessageAfterSubmit, labelize } from '../../../utils/form';
+
+export default function PartitionColumnPopover({ column, node, onSubmit }) {
+  const djClient = useContext(DJClientContext).DataJunctionAPI;
+  const [popoverAnchor, setPopoverAnchor] = useState(false);
+  const ref = useRef(null);
+
+  useEffect(() => {
+    const handleClickOutside = event => {
+      if (ref.current && !ref.current.contains(event.target)) {
+        setPopoverAnchor(false);
+      }
+    };
+    document.addEventListener('click', handleClickOutside, true);
+    return () => {
+      document.removeEventListener('click', handleClickOutside, true);
+    };
+  }, [setPopoverAnchor]);
+
+  const savePartition = async (
+    { node, column, partition_type, format, granularity },
+    { setSubmitting, setStatus },
+  ) => {
+    setSubmitting(false);
+    const response = await djClient.setPartition(
+      node,
+      column,
+      partition_type,
+      format,
+      granularity,
+    );
+    if (response.status === 200 || response.status === 201) {
+      setStatus({ success: 'Saved!' });
+    } else {
+      setStatus({
+        failure: `${response.json.message}`,
+      });
+    }
+    onSubmit();
+    // window.location.reload();
+  };
+
+  return (
+    <>
+      <button
+        className="edit_button"
+        aria-label="PartitionColumn"
+        tabIndex="0"
+        onClick={() => {
+          setPopoverAnchor(!popoverAnchor);
+        }}
+      >
+        <EditIcon />
+      </button>
+      <div
+        className="popover"
+        role="dialog"
+        aria-label="client-code"
+        style={{ display: popoverAnchor === false ? 'none' : 'block' }}
+        ref={ref}
+      >
+        <Formik
+          initialValues={{
+            column: column.name,
+            node: node.name,
+            partition_type: '',
+            format: 'yyyyMMdd',
+            granularity: 'day',
+          }}
+          onSubmit={savePartition}
+        >
+          {function Render({ values, isSubmitting, status, setFieldValue }) {
+            return (
+              <Form>
+                {displayMessageAfterSubmit(status)}
+                <span data-testid="edit-partition">
+                  <label htmlFor="react-select-3-input">Partition Type</label>
+                  <Field
+                    as="select"
+                    name="partition_type"
+                    id="partitionType"
+                    placeholder="Partition Type"
+                  >
+                    <option value=""></option>
+                    <option value="temporal">Temporal</option>
+                    <option value="categorical">Categorical</option>
+                  </Field>
+                </span>
+                <input
+                  hidden={true}
+                  name="column"
+                  value={column.name}
+                  readOnly={true}
+                />
+                <input
+                  hidden={true}
+                  name="node"
+                  value={node.name}
+                  readOnly={true}
+                />
+                <br />
+                <br />
+                {values.partition_type === 'temporal' ? (
+                  <>
+                    <label htmlFor="react-select-3-input">
+                      Partition Format
+                    </label>
+                    <Field
+                      type="text"
+                      name="format"
+                      id="partitionFormat"
+                      placeholder="Optional temporal partition format (ex: yyyyMMdd)"
+                    />
+                    <br />
+                    <br />
+                    <label htmlFor="react-select-3-input">
+                      Partition Granularity
+                    </label>
+                    <Field
+                      as="select"
+                      name="granularity"
+                      id="partitionGranularity"
+                      placeholder="Granularity"
+                    >
+                      <option value="day">Day</option>
+                      <option value="hour">Hour</option>
+                    </Field>
+                  </>
+                ) : (
+                  ''
+                )}
+                <button
+                  className="add_node"
+                  type="submit"
+                  aria-label="SaveEditColumn"
+                  aria-hidden="false"
+                >
+                  Save
+                </button>
+              </Form>
+            );
+          }}
+        </Formik>
+      </div>
+    </>
+  );
+}

package/src/app/pages/NodePage/__tests__/AddBackfillPopover.test.jsx

@@ -0,0 +1,47 @@
+import React from 'react';
+import { render, fireEvent, waitFor, screen } from '@testing-library/react';
+import EditColumnPopover from '../EditColumnPopover';
+import DJClientContext from '../../../providers/djclient';
+import AddBackfillPopover from '../AddBackfillPopover';
+import { mocks } from '../../../../mocks/mockNodes';
+
+const mockDjClient = {
+  DataJunctionAPI: {
+    runBackfill: jest.fn(),
+  },
+};
+
+describe('<AddBackfillPopover />', () => {
+  it('renders correctly and handles form submission', async () => {
+    // Mock onSubmit function
+    const onSubmitMock = jest.fn();
+
+    mockDjClient.DataJunctionAPI.runBackfill.mockReturnValue({
+      status: 201,
+      json: { message: '' },
+    });
+
+    // Render the component
+    const { getByLabelText, getByText } = render(
+      <DJClientContext.Provider value={mockDjClient}>
+        <AddBackfillPopover
+          node={mocks.mockMetricNode}
+          materialization={mocks.nodeMaterializations}
+          onSubmit={onSubmitMock}
+        />
+      </DJClientContext.Provider>,
+    );
+
+    // Open the popover
+    fireEvent.click(getByLabelText('AddBackfill'));
+
+    fireEvent.click(getByText('Save'));
+    getByText('Save').click();
+
+    // Expect setAttributes to be called
+    await waitFor(() => {
+      expect(mockDjClient.DataJunctionAPI.runBackfill).toHaveBeenCalled();
+      expect(getByText('Saved!')).toBeInTheDocument();
+    });
+  });
+});

package/src/app/pages/NodePage/__tests__/NodePage.test.jsx

@@ -36,6 +36,7 @@ describe('<NodePage />', () => {
         nodesWithDimension: jest.fn(),
         attributes: jest.fn(),
         dimensions: jest.fn(),
+        setPartition: jest.fn(),
       },
     };
   };
@@ -70,6 +71,7 @@ describe('<NodePage />', () => {
         {
           name: 'default_DOT_avg_repair_price',
           type: 'double',
+          display_name: 'Default DOT avg repair price',
           attributes: [],
           dimension: null,
         },
@@ -82,7 +84,7 @@ describe('<NodePage />', () => {
         },
       ],
       created_at: '2023-08-21T16:48:56.932162+00:00',
-      tags: [],
+      tags: [{ name: 'purpose', display_name: 'Purpose' }],
       primary_key: [],
       createNodeClientCode:
         'dj = DJBuilder(DJ_URL)\n\navg_repair_price = dj.create_metric(\n    description="Average repair price",\n    display_name="Default: Avg Repair Price",\n    name="default.avg_repair_price",\n    primary_key=[],\n    query="""SELECT  avg(price) default_DOT_avg_repair_price \n FROM default.repair_order_details\n\n"""\n)',
@@ -303,13 +305,12 @@ describe('<NodePage />', () => {
         'v1.0',
       );
 
-      // expect(screen.getByRole('dialog', { name: 'Table' })).not.toBeInTheDocument();
       expect(
         screen.getByRole('dialog', { name: 'NodeStatus' }),
       ).toBeInTheDocument();
 
       expect(screen.getByRole('dialog', { name: 'Tags' })).toHaveTextContent(
-        '',
+        'Purpose',
       );
 
       expect(
@@ -396,6 +397,11 @@ describe('<NodePage />', () => {
     djClient.DataJunctionAPI.columns.mockReturnValue(mocks.metricNodeColumns);
     djClient.DataJunctionAPI.attributes.mockReturnValue(mocks.attributes);
     djClient.DataJunctionAPI.dimensions.mockReturnValue(mocks.dimensions);
+    djClient.DataJunctionAPI.setPartition.mockReturnValue({
+      status: 200,
+      json: { message: '' },
+    });
+
     const element = (
       <DJClientContext.Provider value={djClient}>
         <NodePage />
@@ -416,6 +422,9 @@ describe('<NodePage />', () => {
       expect(
         screen.getByRole('columnheader', { name: 'ColumnName' }),
       ).toHaveTextContent('default_DOT_avg_repair_price');
+      expect(
+        screen.getByRole('columnheader', { name: 'ColumnDisplayName' }),
+      ).toHaveTextContent('Default DOT avg repair price');
       expect(
         screen.getByRole('columnheader', { name: 'ColumnType' }),
       ).toHaveTextContent('double');
@@ -439,6 +448,19 @@ describe('<NodePage />', () => {
       expect(
         screen.getByRole('button', { name: 'SaveLinkDimension' }),
       ).toBeInTheDocument();
+
+      // check that the set column partition popover can be clicked
+      const partitionColumnPopover = screen.getByRole('button', {
+        name: 'PartitionColumn',
+      });
+      expect(partitionColumnPopover).toBeInTheDocument();
+      fireEvent.click(partitionColumnPopover);
+      const savePartition = screen.getByRole('button', {
+        name: 'SaveEditColumn',
+      });
+      expect(savePartition).toBeInTheDocument();
+      fireEvent.click(savePartition);
+      expect(screen.getByText('Saved!'));
     });
   });
   // check compiled SQL on nodeInfo page
@@ -573,7 +595,10 @@ describe('<NodePage />', () => {
       expect(djClient.DataJunctionAPI.materializations).toHaveBeenCalledWith(
         mocks.mockMetricNode.name,
       );
-      screen.getByText('No materializations available for this node');
+      screen.getByText(
+        'No materialization workflows configured for this node.',
+      );
+      screen.getByText('No materialized datasets available for this node.');
     });
   });
 
@@ -598,20 +623,25 @@ describe('<NodePage />', () => {
         </Routes>
       </MemoryRouter>,
     );
-    await waitFor(() => {
-      fireEvent.click(screen.getByRole('button', { name: 'Materializations' }));
-      expect(djClient.DataJunctionAPI.node).toHaveBeenCalledWith(
-        mocks.mockMetricNode.name,
-      );
-      expect(djClient.DataJunctionAPI.materializations).toHaveBeenCalledWith(
-        mocks.mockMetricNode.name,
-      );
+    await waitFor(
+      () => {
+        fireEvent.click(
+          screen.getByRole('button', { name: 'Materializations' }),
+        );
+        expect(djClient.DataJunctionAPI.node).toHaveBeenCalledWith(
+          mocks.mockMetricNode.name,
+        );
+        expect(djClient.DataJunctionAPI.materializations).toHaveBeenCalledWith(
+          mocks.mockMetricNode.name,
+        );
 
-      expect(
-        screen.getByRole('table', { name: 'Materializations' }),
-      ).toMatchSnapshot();
-    });
-  });
+        expect(
+          screen.getByRole('table', { name: 'Materializations' }),
+        ).toMatchSnapshot();
+      },
+      { timeout: 3000 },
+    );
+  }, 60000);
 
   it('renders the NodeSQL tab', async () => {
     const djClient = mockDJClient();