datajunction-ui 0.0.1-a35.dev0 → 0.0.1-a37

package/src/app/pages/NodePage/AddMaterializationPopover.jsx

@@ -2,27 +2,24 @@ import { useContext, useEffect, useRef, useState } from 'react';
 import * as React from 'react';
 import DJClientContext from '../../providers/djclient';
 import { ErrorMessage, Field, Form, Formik } from 'formik';
-import { FormikSelect } from '../AddEditNodePage/FormikSelect';
-import EditIcon from '../../icons/EditIcon';
 import { displayMessageAfterSubmit, labelize } from '../../../utils/form';
 
 export default function AddMaterializationPopover({ node, onSubmit }) {
   const djClient = useContext(DJClientContext).DataJunctionAPI;
   const [popoverAnchor, setPopoverAnchor] = useState(false);
-  const [engines, setEngines] = useState([]);
-  const [defaultEngine, setDefaultEngine] = useState('');
+  const [options, setOptions] = useState([]);
+  const [jobs, setJobs] = useState([]);
 
   const ref = useRef(null);
 
   useEffect(() => {
     const fetchData = async () => {
-      const engines = await djClient.engines();
-      setEngines(engines);
-      setDefaultEngine(
-        engines && engines.length > 0
-          ? engines[0].name + '__' + engines[0].version
-          : '',
+      const options = await djClient.materializationInfo();
+      setOptions(options);
+      const allowedJobs = options.job_types?.filter(job =>
+        job.allowed_node_types.includes(node.type),
       );
+      setJobs(allowedJobs);
     };
     fetchData().catch(console.error);
     const handleClickOutside = event => {
@@ -41,14 +38,15 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
     { setSubmitting, setStatus },
   ) => {
     setSubmitting(false);
-    const engineVersion = values.engine.split('__').slice(-1).join('');
-    const engineName = values.engine.split('__').slice(0, -1).join('');
+    const config = JSON.parse(values.config);
+    config.lookback_window = values.lookback_window;
+    console.log('values', values);
     const response = await djClient.materialize(
       values.node,
-      engineName,
-      engineVersion,
+      values.job_type,
+      values.strategy,
       values.schedule,
-      values.config,
+      config,
     );
     if (response.status === 200 || response.status === 201) {
       setStatus({ success: 'Saved!' });
@@ -65,7 +63,7 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
     <>
       <button
         className="edit_button"
-        aria-label="PartitionColumn"
+        aria-label="AddMaterialization"
         tabIndex="0"
         onClick={() => {
           setPopoverAnchor(!popoverAnchor);
@@ -90,9 +88,11 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
         <Formik
           initialValues={{
             node: node?.name,
-            engine: defaultEngine,
+            job_type: 'spark_sql',
+            strategy: 'full',
             config: '{"spark": {"spark.executor.memory": "6g"}}',
             schedule: '@daily',
+            lookback_window: '1 DAY',
           }}
           onSubmit={configureMaterialization}
         >
@@ -101,16 +101,15 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
               <Form>
                 <h2>Configure Materialization</h2>
                 {displayMessageAfterSubmit(status)}
-                <span data-testid="edit-partition">
-                  <label htmlFor="engine">Engine</label>
-                  <Field as="select" name="engine">
+                <span data-testid="job-type">
+                  <label htmlFor="job_type">Job Type</label>
+                  <Field as="select" name="job_type">
                     <>
-                      {engines?.map(engine => (
-                        <option value={engine.name + '__' + engine.version}>
-                          {engine.name} {engine.version}
+                      {jobs?.map(job => (
+                        <option key={job.name} value={job.name}>
+                          {job.label}
                         </option>
                       ))}
-                      <option value=""></option>
                     </>
                   </Field>
                 </span>
@@ -122,6 +121,18 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
                 />
                 <br />
                 <br />
+                <span data-testid="edit-partition">
+                  <label htmlFor="strategy">Strategy</label>
+                  <Field as="select" name="strategy">
+                    <>
+                      {options.strategies?.map(strategy => (
+                        <option value={strategy.name}>{strategy.label}</option>
+                      ))}
+                    </>
+                  </Field>
+                </span>
+                <br />
+                <br />
                 <label htmlFor="schedule">Schedule</label>
                 <Field
                   type="text"
@@ -132,6 +143,18 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
                 />
                 <br />
                 <br />
+                <div className="DescriptionInput">
+                  <ErrorMessage name="description" component="span" />
+                  <label htmlFor="Config">Lookback Window</label>
+                  <Field
+                    type="text"
+                    name="lookback_window"
+                    id="lookback_window"
+                    placeholder="1 DAY"
+                    default="1 DAY"
+                  />
+                </div>
+                <br />
                 <div className="DescriptionInput">
                   <ErrorMessage name="description" component="span" />
                   <label htmlFor="Config">Config</label>

package/src/app/pages/NodePage/NodeInfoTab.jsx

@@ -76,7 +76,12 @@ export default function NodeInfoTab({ node }) {
         aria-label="CubeElement"
         aria-hidden="false"
       >
-        <a href={`/nodes/${cubeElem.node_name}`}>{cubeElem.display_name}</a>
+        <a href={`/nodes/${cubeElem.node_name}`}>
+          {cubeElem.type === 'dimension'
+            ? labelize(cubeElem.node_name.split('.').slice(-1)[0]) + ' → '
+            : ''}
+          {cubeElem.display_name}
+        </a>
         <span
           className={`badge node_type__${
             cubeElem.type === 'metric' ? cubeElem.type : 'dimension'

package/src/app/pages/NodePage/NodeMaterializationTab.jsx

@@ -53,11 +53,9 @@ export default function NodeMaterializationTab({ node, djClient }) {
           <div className={`cron-description`}>{cron(materialization)} </div>
         </td>
         <td>
-          {materialization.engine.name}
-          <br />
-          {materialization.engine.version}
-          <ClientCodePopover code={materialization.clientCode} />
+          {materialization.job?.replace('MaterializationJob', '').toUpperCase()}
         </td>
+        <td>{materialization.strategy?.toUpperCase()}</td>
         <td>
           {node.columns
             .filter(col => col.partition !== null)
@@ -90,27 +88,34 @@ export default function NodeMaterializationTab({ node, djClient }) {
             </div>
           ))}
         </td>
-        <td>
-          {materialization.backfills.map(backfill => (
-            <a href={backfill.urls[0]} className="partitionLink">
-              <div className="partition__full" key={backfill.spec.column_name}>
-                <div className="partition__header">
-                  {partitionColumnsMap[backfill.spec.column_name]}
-                </div>
-                <div className="partition__body">
-                  <span className="badge partition_value">
-                    {backfill.spec.range[0]}
-                  </span>
-                  to
-                  <span className="badge partition_value">
-                    {backfill.spec.range[1]}
-                  </span>
+        {materializations[0].strategy === 'incremental_time' ? (
+          <td>
+            {materialization.backfills.map(backfill => (
+              <a href={backfill.urls[0]} className="partitionLink">
+                <div
+                  className="partition__full"
+                  key={backfill.spec.column_name}
+                >
+                  <div className="partition__header">
+                    {partitionColumnsMap[backfill.spec.column_name]}
+                  </div>
+                  <div className="partition__body">
+                    <span className="badge partition_value">
+                      {backfill.spec.range[0]}
+                    </span>
+                    to
+                    <span className="badge partition_value">
+                      {backfill.spec.range[1]}
+                    </span>
+                  </div>
                 </div>
-              </div>
-            </a>
-          ))}
-          <AddBackfillPopover node={node} materialization={materialization} />
-        </td>
+              </a>
+            ))}
+            <AddBackfillPopover node={node} materialization={materialization} />
+          </td>
+        ) : (
+          <></>
+        )}
         <td>
           {materialization.urls.map((url, idx) => (
             <a href={url} key={`url-${idx}`}>
@@ -118,6 +123,9 @@ export default function NodeMaterializationTab({ node, djClient }) {
             </a>
           ))}
         </td>
+        <td>
+          <ClientCodePopover code={materialization.clientCode} />
+        </td>
       </tr>
     ));
   };
@@ -136,10 +144,15 @@ export default function NodeMaterializationTab({ node, djClient }) {
               <thead className="fs-7 fw-bold text-gray-400 border-bottom-0">
                 <tr>
                   <th className="text-start">Schedule</th>
-                  <th>Engine</th>
+                  <th>Job Type</th>
+                  <th>Strategy</th>
                   <th>Partitions</th>
-                  <th>Output Tables</th>
-                  <th>Backfills</th>
+                  <th>Intended Output Tables</th>
+                  {materializations[0].strategy === 'incremental_time' ? (
+                    <th>Backfills</th>
+                  ) : (
+                    <></>
+                  )}
                   <th>URLs</th>
                 </tr>
               </thead>
@@ -171,16 +184,13 @@ export default function NodeMaterializationTab({ node, djClient }) {
             >
               <thead className="fs-7 fw-bold text-gray-400 border-bottom-0">
                 <tr>
-                  <th className="text-start">Catalog</th>
-                  <th>Schema</th>
-                  <th>Table</th>
+                  <th className="text-start">Output Dataset</th>
                   <th>Valid Through</th>
                   <th>Partitions</th>
                 </tr>
               </thead>
               <tbody>
                 <tr>
-                  <td>{node.availability.schema_}</td>
                   <td>
                     {
                       <div
@@ -203,7 +213,9 @@ export default function NodeMaterializationTab({ node, djClient }) {
                       </div>
                     }
                   </td>
-                  <td>{node.availability.valid_through_ts}</td>
+                  <td>
+                    {new Date(node.availability.valid_through_ts).toISOString()}
+                  </td>
                   <td>
                     <span
                       className={`badge partition_value`}

package/src/app/pages/NodePage/__tests__/AddBackfillPopover.test.jsx

@@ -36,7 +36,6 @@ describe('<AddBackfillPopover />', () => {
     fireEvent.click(getByLabelText('AddBackfill'));
 
     fireEvent.click(getByText('Save'));
-    getByText('Save').click();
 
     // Expect setAttributes to be called
     await waitFor(() => {

package/src/app/pages/NodePage/__tests__/AddMaterializationPopover.test.jsx

@@ -0,0 +1,84 @@
+import React from 'react';
+import { render, fireEvent, waitFor, screen } from '@testing-library/react';
+import DJClientContext from '../../../providers/djclient';
+import AddMaterializationPopover from '../AddMaterializationPopover';
+import { mocks } from '../../../../mocks/mockNodes';
+
+const mockDjClient = {
+  DataJunctionAPI: {
+    materialize: jest.fn(),
+    materializationInfo: jest.fn(),
+  },
+};
+
+describe('<AddMaterializationPopover />', () => {
+  it('renders correctly and handles form submission', async () => {
+    // Mock onSubmit function
+    const onSubmitMock = jest.fn();
+    mockDjClient.DataJunctionAPI.materialize.mockReturnValue({
+      status: 201,
+    });
+    mockDjClient.DataJunctionAPI.materializationInfo.mockReturnValue({
+      status: 200,
+      json: {
+        job_types: [
+          {
+            name: 'spark_sql',
+            label: 'Spark SQL',
+            description: 'Spark SQL materialization job',
+            allowed_node_types: ['transform', 'dimension', 'cube'],
+            job_class: 'SparkSqlMaterializationJob',
+          },
+          {
+            name: 'druid_cube',
+            label: 'Druid Cube',
+            description:
+              'Used to materialize a cube to Druid for low-latency access to a set of metrics and dimensions. While the logical cube definition is at the level of metrics and dimensions, a materialized Druid cube will reference measures and dimensions, with rollup configured on the measures where appropriate.',
+            allowed_node_types: ['cube'],
+            job_class: 'DruidCubeMaterializationJob',
+          },
+        ],
+        strategies: [
+          {
+            name: 'full',
+            label: 'Full',
+          },
+          {
+            name: 'snapshot',
+            label: 'Snapshot',
+          },
+          {
+            name: 'incremental_time',
+            label: 'Incremental Time',
+          },
+          {
+            name: 'view',
+            label: 'View',
+          },
+        ],
+      },
+    });
+
+    // Render the component
+    const { getByText } = render(
+      <DJClientContext.Provider value={mockDjClient}>
+        <AddMaterializationPopover
+          node={mocks.mockMetricNode}
+          onSubmit={onSubmitMock}
+        />
+      </DJClientContext.Provider>,
+    );
+
+    // Open the popover
+    fireEvent.click(getByText('+ Add Materialization'));
+
+    // Save the materialization
+    fireEvent.click(getByText('Save'));
+
+    // Expect setAttributes to be called
+    await waitFor(() => {
+      expect(mockDjClient.DataJunctionAPI.materialize).toHaveBeenCalled();
+      expect(getByText('Saved!')).toBeInTheDocument();
+    });
+  });
+});

package/src/app/pages/NodePage/__tests__/__snapshots__/NodePage.test.jsx.snap

@@ -107,16 +107,16 @@ exports[`<NodePage /> renders the NodeMaterialization tab with materializations
         Schedule
       </th>
       <th>
-        Engine
+        Job Type
       </th>
       <th>
-        Partitions
+        Strategy
       </th>
       <th>
-        Output Tables
+        Partitions
       </th>
       <th>
-        Backfills
+        Intended Output Tables
       </th>
       <th>
         URLs
@@ -141,9 +141,80 @@ exports[`<NodePage /> renders the NodeMaterialization tab with materializations
         </div>
       </td>
       <td>
-        spark
-        <br />
-        2.4.4
+        SPARKSQL
+      </td>
+      <td />
+      <td />
+      <td>
+        <div
+          class="table__full"
+        >
+          <div
+            class="table__header"
+          >
+            <svg
+              class="bi bi-table"
+              fill="currentColor"
+              height="16"
+              viewBox="0 0 16 16"
+              width="16"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <path
+                d="M0 2a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v12a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2V2zm15 2h-4v3h4V4zm0 4h-4v3h4V8zm0 4h-4v3h3a1 1 0 0 0 1-1v-2zm-5 3v-3H6v3h4zm-5 0v-3H1v2a1 1 0 0 0 1 1h3zm-4-4h4V8H1v3zm0-4h4V4H1v3zm5-3v3h4V4H6zm4 4H6v3h4V8z"
+              />
+            </svg>
+             
+            <span
+              class="entity-info"
+            >
+              common.a
+            </span>
+          </div>
+          <div
+            class="table__body upstream_tables"
+          />
+        </div>
+        <div
+          class="table__full"
+        >
+          <div
+            class="table__header"
+          >
+            <svg
+              class="bi bi-table"
+              fill="currentColor"
+              height="16"
+              viewBox="0 0 16 16"
+              width="16"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <path
+                d="M0 2a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v12a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2V2zm15 2h-4v3h4V4zm0 4h-4v3h4V8zm0 4h-4v3h3a1 1 0 0 0 1-1v-2zm-5 3v-3H6v3h4zm-5 0v-3H1v2a1 1 0 0 0 1 1h3zm-4-4h4V8H1v3zm0-4h4V4H1v3zm5-3v3h4V4H6zm4 4H6v3h4V8z"
+              />
+            </svg>
+             
+            <span
+              class="entity-info"
+            >
+              common.b
+            </span>
+          </div>
+          <div
+            class="table__body upstream_tables"
+          />
+        </div>
+      </td>
+      <td>
+        <a
+          href="http://fake.url/job"
+        >
+          [
+          1
+          ]
+        </a>
+      </td>
+      <td>
         <button
           aria-label="code-button"
           class="code-button"
@@ -230,173 +301,6 @@ exports[`<NodePage /> renders the NodeMaterialization tab with materializations
           </pre>
         </div>
       </td>
-      <td />
-      <td>
-        <div
-          class="table__full"
-        >
-          <div
-            class="table__header"
-          >
-            <svg
-              class="bi bi-table"
-              fill="currentColor"
-              height="16"
-              viewBox="0 0 16 16"
-              width="16"
-              xmlns="http://www.w3.org/2000/svg"
-            >
-              <path
-                d="M0 2a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v12a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2V2zm15 2h-4v3h4V4zm0 4h-4v3h4V8zm0 4h-4v3h3a1 1 0 0 0 1-1v-2zm-5 3v-3H6v3h4zm-5 0v-3H1v2a1 1 0 0 0 1 1h3zm-4-4h4V8H1v3zm0-4h4V4H1v3zm5-3v3h4V4H6zm4 4H6v3h4V8z"
-              />
-            </svg>
-             
-            <span
-              class="entity-info"
-            >
-              common.a
-            </span>
-          </div>
-          <div
-            class="table__body upstream_tables"
-          />
-        </div>
-        <div
-          class="table__full"
-        >
-          <div
-            class="table__header"
-          >
-            <svg
-              class="bi bi-table"
-              fill="currentColor"
-              height="16"
-              viewBox="0 0 16 16"
-              width="16"
-              xmlns="http://www.w3.org/2000/svg"
-            >
-              <path
-                d="M0 2a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v12a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2V2zm15 2h-4v3h4V4zm0 4h-4v3h4V8zm0 4h-4v3h3a1 1 0 0 0 1-1v-2zm-5 3v-3H6v3h4zm-5 0v-3H1v2a1 1 0 0 0 1 1h3zm-4-4h4V8H1v3zm0-4h4V4H1v3zm5-3v3h4V4H6zm4 4H6v3h4V8z"
-              />
-            </svg>
-             
-            <span
-              class="entity-info"
-            >
-              common.b
-            </span>
-          </div>
-          <div
-            class="table__body upstream_tables"
-          />
-        </div>
-      </td>
-      <td>
-        <a
-          class="partitionLink"
-        >
-          <div
-            class="partition__full"
-          >
-            <div
-              class="partition__header"
-            />
-            <div
-              class="partition__body"
-            >
-              <span
-                class="badge partition_value"
-              >
-                20230101
-              </span>
-              to
-              <span
-                class="badge partition_value"
-              >
-                20230102
-              </span>
-            </div>
-          </div>
-        </a>
-        <button
-          aria-label="AddBackfill"
-          class="edit_button"
-          tabindex="0"
-        >
-          <span
-            class="add_node"
-          >
-            + Add Backfill
-          </span>
-        </button>
-        <div
-          class="fade modal-backdrop in"
-          style="display: none;"
-        />
-        <div
-          aria-label="client-code"
-          class="centerPopover"
-          role="dialog"
-          style="display: none; width: 50%;"
-        >
-          <form
-            action="#"
-          >
-            <h2>
-              Run Backfill
-            </h2>
-            <span
-              data-testid="edit-partition"
-            >
-              <label
-                for="engine"
-                style="padding-bottom: 1rem;"
-              >
-                Engine
-              </label>
-              <select
-                disabled=""
-                id="engine"
-                name="engine"
-              >
-                <option
-                  value="spark"
-                >
-                  spark
-                   
-                  2.4.4
-                </option>
-              </select>
-            </span>
-            <br />
-            <br />
-            <label
-              for="partition"
-              style="padding-bottom: 1rem;"
-            >
-              Partition Range
-            </label>
-            <br />
-            <button
-              aria-hidden="false"
-              aria-label="SaveEditColumn"
-              class="add_node"
-              type="submit"
-            >
-              Save
-            </button>
-          </form>
-        </div>
-      </td>
-      <td>
-        <a
-          href="http://fake.url/job"
-        >
-          [
-          1
-          ]
-        </a>
-      </td>
     </tr>
   </tbody>
 </table>