datajunction-ui 0.0.1-a45.dev5 → 0.0.1-a46

package/src/app/services/DJService.js

@@ -12,7 +12,7 @@ export const DataJunctionAPI = {
   },
 
   logout: async function () {
-    await await fetch(`${DJ_URL}/logout/`, {
+    return await fetch(`${DJ_URL}/logout/`, {
       credentials: 'include',
       method: 'POST',
     });
@@ -388,19 +388,7 @@ export const DataJunctionAPI = {
       })
     ).json();
 
-    return await Promise.all(
-      data.map(async materialization => {
-        materialization.clientCode = await (
-          await fetch(
-            `${DJ_URL}/datajunction-clients/python/add_materialization/${node}/${materialization.name}`,
-            {
-              credentials: 'include',
-            },
-          )
-        ).json();
-        return materialization;
-      }),
-    );
+    return data;
   },
 
   columns: async function (node) {
@@ -804,13 +792,7 @@ export const DataJunctionAPI = {
     );
     return { status: response.status, json: await response.json() };
   },
-  runBackfill: async function (
-    nodeName,
-    materializationName,
-    partitionColumn,
-    from,
-    to,
-  ) {
+  runBackfill: async function (nodeName, materializationName, partitionValues) {
     const response = await fetch(
       `${DJ_URL}/nodes/${nodeName}/materializations/${materializationName}/backfill`,
       {
@@ -818,10 +800,29 @@ export const DataJunctionAPI = {
         headers: {
           'Content-Type': 'application/json',
         },
-        body: JSON.stringify({
-          column_name: partitionColumn,
-          range: [from, to],
-        }),
+        body: JSON.stringify(
+          partitionValues.map(partitionValue => {
+            return {
+              column_name: partitionValue.columnName,
+              range: partitionValue.range,
+              values: partitionValue.values,
+            };
+          }),
+        ),
+        credentials: 'include',
+      },
+    );
+    return { status: response.status, json: await response.json() };
+  },
+  deleteMaterialization: async function (nodeName, materializationName) {
+    console.log('deleting materialization', nodeName, materializationName);
+    const response = await fetch(
+      `${DJ_URL}/nodes/${nodeName}/materializations?materialization_name=${materializationName}`,
+      {
+        method: 'DELETE',
+        headers: {
+          'Content-Type': 'application/json',
+        },
         credentials: 'include',
       },
     );
@@ -844,4 +845,15 @@ export const DataJunctionAPI = {
       })
     ).json();
   },
+  revalidate: async function (node) {
+    return await (
+      await fetch(`${DJ_URL}/nodes/${node}/validate`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        credentials: 'include',
+      })
+    ).json();
+  },
 };

package/src/app/services/__tests__/DJService.test.jsx

@@ -34,6 +34,22 @@ describe('DataJunctionAPI', () => {
     });
   });
 
+  it('calls catalogs correctly', async () => {
+    fetch.mockResponseOnce(JSON.stringify({}));
+    await DataJunctionAPI.catalogs();
+    expect(fetch).toHaveBeenCalledWith(`${DJ_URL}/catalogs`, {
+      credentials: 'include',
+    });
+  });
+
+  it('calls engines correctly', async () => {
+    fetch.mockResponseOnce(JSON.stringify({}));
+    await DataJunctionAPI.engines();
+    expect(fetch).toHaveBeenCalledWith(`${DJ_URL}/engines`, {
+      credentials: 'include',
+    });
+  });
+
   it('calls node correctly', async () => {
     fetch.mockResponseOnce(JSON.stringify(mocks.mockMetricNode));
     const nodeData = await DataJunctionAPI.node(mocks.mockMetricNode.name);
@@ -441,8 +457,8 @@ describe('DataJunctionAPI', () => {
   it('calls materializations correctly', async () => {
     const nodeName = 'default.sample_node';
     const mockMaterializations = [
-      { name: 'materialization1', clientCode: 'from dj import DJClient' },
-      { name: 'materialization2', clientCode: 'from dj import DJClient' },
+      { name: 'materialization1' },
+      { name: 'materialization2' },
     ];
 
     // Mock the first fetch call to return the list of materializations
@@ -465,14 +481,6 @@ describe('DataJunctionAPI', () => {
       },
     );
 
-    // Check the subsequent fetch calls for clientCode
-    mockMaterializations.forEach(mat => {
-      expect(fetch).toHaveBeenCalledWith(
-        `${DJ_URL}/datajunction-clients/python/add_materialization/${nodeName}/${mat.name}`,
-        { credentials: 'include' },
-      );
-    });
-
     // Ensure the result contains the clientCode for each materialization
     expect(result).toEqual(mockMaterializations);
   });
@@ -965,13 +973,12 @@ describe('DataJunctionAPI', () => {
 
   it('calls runBackfill correctly', async () => {
     fetch.mockResponseOnce(JSON.stringify({}));
-    await DataJunctionAPI.runBackfill(
-      'default.hard_hat',
-      'spark',
-      'hire_date',
-      '20230101',
-      '20230202',
-    );
+    await DataJunctionAPI.runBackfill('default.hard_hat', 'spark', [
+      {
+        columnName: 'hire_date',
+        range: ['20230101', '20230202'],
+      },
+    ]);
     expect(fetch).toHaveBeenCalledWith(
       `${DJ_URL}/nodes/default.hard_hat/materializations/spark/backfill`,
       {
@@ -979,12 +986,37 @@ describe('DataJunctionAPI', () => {
         headers: {
           'Content-Type': 'application/json',
         },
-        body: JSON.stringify({
-          column_name: 'hire_date',
-          range: ['20230101', '20230202'],
-        }),
+        body: JSON.stringify([
+          {
+            column_name: 'hire_date',
+            range: ['20230101', '20230202'],
+          },
+        ]),
         method: 'POST',
       },
     );
   });
+
+  it('calls materializationInfo correctly', async () => {
+    fetch.mockResponseOnce(JSON.stringify({}));
+    await DataJunctionAPI.materializationInfo();
+    expect(fetch).toHaveBeenCalledWith(`${DJ_URL}/materialization/info`, {
+      credentials: 'include',
+    });
+  });
+
+  it('calls revalidate correctly', async () => {
+    fetch.mockResponseOnce(JSON.stringify({}));
+    await DataJunctionAPI.revalidate('default.hard_hat');
+    expect(fetch).toHaveBeenCalledWith(
+      `${DJ_URL}/nodes/default.hard_hat/validate`,
+      {
+        credentials: 'include',
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+  });
 });

package/src/mocks/mockNodes.jsx

@@ -41,7 +41,9 @@ export const mocks = {
         type: 'string',
         attributes: [],
         dimension: null,
-        partition: null,
+        partition: {
+          type_: 'categorical',
+        },
       },
     ],
     updated_at: '2024-01-24T16:39:14.029366+00:00',
@@ -103,6 +105,7 @@ export const mocks = {
     created_at: '2023-08-21T16:48:56.841631+00:00',
     tags: [{ name: 'purpose', display_name: 'Purpose' }],
     dimension_links: [],
+    incompatible_druid_functions: ['IF'],
     dimensions: [
       {
         value: 'default.date_dim.dateint',
@@ -460,12 +463,14 @@ export const mocks = {
     {
       backfills: [
         {
-          spec: {
-            column_name: 'default_DOT_hard_hat_DOT_hire_date',
-            values: null,
-            range: ['20230101', '20230102'],
-          },
-          urls: [],
+          spec: [
+            {
+              column_name: 'default_DOT_hard_hat_DOT_hire_date',
+              values: null,
+              range: ['20230101', '20230102'],
+            },
+          ],
+          urls: ['a', 'b'],
         },
       ],
       name: 'country_birth_date_contractor_id_379232101',
@@ -476,6 +481,7 @@ export const mocks = {
           "SELECT  default_DOT_hard_hats.address,\n\tdefault_DOT_hard_hats.birth_date,\n\tdefault_DOT_hard_hats.city,\n\tdefault_DOT_hard_hats.contractor_id,\n\tdefault_DOT_hard_hats.country,\n\tdefault_DOT_hard_hats.first_name,\n\tdefault_DOT_hard_hats.hard_hat_id,\n\tdefault_DOT_hard_hats.hire_date,\n\tdefault_DOT_hard_hats.last_name,\n\tdefault_DOT_hard_hats.manager,\n\tdefault_DOT_hard_hats.postal_code,\n\tdefault_DOT_hard_hats.state,\n\tdefault_DOT_hard_hats.title \n FROM roads.hard_hats AS default_DOT_hard_hats \n WHERE  default_DOT_hard_hats.country IN ('DE', 'MY') AND default_DOT_hard_hats.contractor_id BETWEEN 1 AND 10\n\n",
         upstream_tables: ['default.roads.hard_hats'],
       },
+      strategy: 'incremental_time',
       schedule: '0 * * * *',
       job: 'SparkSqlMaterializationJob',
       output_tables: ['common.a', 'common.b'],
@@ -1552,4 +1558,53 @@ export const mocks = {
       tag_type: 'reports',
     },
   ],
+  materializationInfo: {
+    job_types: [
+      {
+        name: 'spark_sql',
+        label: 'Spark SQL',
+        description: 'Spark SQL materialization job',
+        allowed_node_types: ['transform', 'dimension', 'cube'],
+        job_class: 'SparkSqlMaterializationJob',
+      },
+      {
+        name: 'druid_measures_cube',
+        label: 'Druid Measures Cube (Pre-Agg Cube)',
+        description:
+          "Used to materialize a cube's measures to Druid for low-latency access to a set of metrics and dimensions. While the logical cube definition is at the level of metrics and dimensions, this materialized Druid cube will contain measures and dimensions, with rollup configured on the measures where appropriate.",
+        allowed_node_types: ['cube'],
+        job_class: 'DruidMeasuresCubeMaterializationJob',
+      },
+      {
+        name: 'druid_metrics_cube',
+        label: 'Druid Metrics Cube (Post-Agg Cube)',
+        description:
+          "Used to materialize a cube of metrics and dimensions to Druid for low-latency access. The materialized cube is at the metric level, meaning that all metrics will be aggregated to the level of the cube's dimensions.",
+        allowed_node_types: ['cube'],
+        job_class: 'DruidMetricsCubeMaterializationJob',
+      },
+    ],
+    strategies: [
+      {
+        name: 'full',
+        label: 'Full',
+      },
+      {
+        name: 'snapshot',
+        label: 'Snapshot',
+      },
+      {
+        name: 'snapshot_partition',
+        label: 'Snapshot Partition',
+      },
+      {
+        name: 'incremental_time',
+        label: 'Incremental Time',
+      },
+      {
+        name: 'view',
+        label: 'View',
+      },
+    ],
+  },
 };

package/src/styles/index.css

@@ -593,6 +593,12 @@ tbody th {
   color: #777777;
 }
 
+.strategy {
+  background-color: rgb(255, 239, 215) !important;
+  color: #6c3b21;
+  font-size: 1rem;
+}
+
 .status__valid {
   color: #00b368;
 }
@@ -1018,17 +1024,19 @@ pre {
 }
 
 .add_button {
-  background-color: #f0f8ff !important;
-  color: #24518f;
-  text-transform: uppercase;
+  background-color: #2f7986 !important;
+  color: #fff;
+  text-transform: none;
   vertical-align: middle;
-  padding: 0.2rem 0.5rem 0.1rem 0.5rem;
-  border: 1px solid #819bc0;
-  font-size: 1.2rem;
+  padding-right: 1rem;
+  padding-left: 1rem;
+  padding-top: 0.5rem;
+  padding-bottom: 0.5rem;
+  margin-bottom: 0.5rem !important;
+  font-size: 1rem;
   border-radius: 0.5rem;
   word-wrap: break-word;
   white-space: break-spaces;
-  margin-left: 1rem;
 }
 
 .edit_button {
@@ -1090,10 +1098,6 @@ pre {
   transition: opacity 0.15s linear;
 }
 
-.partitionLink:hover {
-  text-decoration: none;
-}
-
 .dimensionsList {
   padding: 12px;
   opacity: 1;
@@ -1202,3 +1206,116 @@ pre {
   text-transform: uppercase;
   padding-left: 10px;
 }
+
+.validation_error {
+  border: #b34b0025 1px solid;
+  border-left: #b34b00 5px solid;
+  padding-left: 20px;
+  padding-top: 5px;
+  padding-bottom: 5px;
+  font-size: small;
+  width: 600px;
+  word-wrap: break-word;
+  margin-top: 3px;
+  background-color: #ffffff;
+  margin-bottom: 3px;
+  margin-left: -20px;
+}
+
+.partitionLink {
+  /*border: 1px solid #ccc4d5;*/
+  padding: 12px;
+  margin: 5px;
+  border-radius: 0.375rem;
+  background-color: #ccc4d525;
+}
+.partitionLink:hover {
+  background-color: rgba(157, 147, 168, 0.15);
+}
+.partitionLink a:hover {
+  text-decoration: none;
+}
+
+.backfills {
+  margin-left: -4rem;
+  --spacing : 1.5rem;
+  --radius  : 10px;
+}
+
+.backfills li{
+  display      : block;
+  position     : relative;
+  padding-left : calc(2 * var(--spacing) - var(--radius) - 2px);
+}
+
+.backfills ul{
+  margin-left  : calc(var(--radius) - var(--spacing));
+  padding-left : 2rem;
+}
+
+.backfills ul li{
+  border-left : 2px solid #ddd;
+}
+
+.backfills ul li:last-child{
+  border-color : transparent;
+}
+
+.backfills ul li::before{
+  content      : '';
+  display      : block;
+  position     : absolute;
+  top          : calc(var(--spacing) / -2);
+  left         : -2px;
+  width        : calc(var(--spacing) + 2px);
+  height       : calc(var(--spacing) + 1px);
+  border       : solid #ddd;
+  border-width : 0 0 2px 2px;
+}
+
+.backfills summary{
+  display : block;
+  cursor  : pointer;
+  margin-bottom: 10px;
+}
+
+.backfills summary::marker,
+.backfills summary::-webkit-details-marker{
+  display : none;
+}
+
+.backfills summary:focus{
+  outline : none;
+}
+
+.backfills summary:focus-visible{
+  outline : 1px dotted #000;
+}
+
+.backfills summary::before{
+  z-index    : 1;
+  /*background : #696 url('expand-collapse.svg') 0 0;*/
+}
+
+.backfills details[open] > summary::before{
+  background-position : calc(-2 * var(--radius)) 0;
+}
+
+.backfills_header {
+  font-size: 16px;
+  margin-left: 18px;
+  border-left: 2px solid #ddd;
+  padding-left: 40px;
+  margin-bottom: 10px;
+}
+
+.tr {
+  display: inline-flex;
+}
+.tr li {
+  padding-right: 10px;
+}
+.td {
+  display: table-cell;
+  padding: 8px;
+}

package/src/styles/node-creation.scss

@@ -254,6 +254,15 @@ form {
   }
 }
 
+.warning {
+  background-color: rgb(253, 248, 242);
+  color: rgb(190, 105, 37);
+  svg {
+    filter: invert(16%) sepia(68%) saturate(2827%) hue-rotate(344deg)
+      brightness(96%) contrast(100%);
+  }
+}
+
 .SourceCreationInput {
   margin: 0.5rem 0;
   display: inline-grid;

package/src/styles/node-list.css

@@ -0,0 +1,4 @@
+table {
+  width: 100%;
+  height: min-content;
+}

package/src/app/components/forms/NodeTagsInput.jsx

@@ -1,61 +0,0 @@
-import { ErrorMessage } from 'formik';
-import { FormikSelect } from '../../pages/AddEditNodePage/FormikSelect';
-import { Action } from './Action';
-import { useContext, useEffect, useState } from 'react';
-import DJClientContext from '../../providers/djclient';
-
-export default function NodeTagsInput({ action, node }) {
-  const djClient = useContext(DJClientContext).DataJunctionAPI;
-  const [tags, setTags] = useState([]);
-
-  // Get list of tags
-  useEffect(() => {
-    const fetchData = async () => {
-      const tags = await djClient.listTags();
-      setTags(
-        tags.map(tag => ({
-          value: tag.name,
-          label: tag.display_name,
-        })),
-      );
-    };
-    fetchData().catch(console.error);
-  }, [djClient, djClient.listTags]);
-
-  return (
-    <div
-      className="TagsInput"
-      style={{ width: '25%', margin: '1rem 0 1rem 1.2rem' }}
-    >
-      <ErrorMessage name="tags" component="span" />
-      <label htmlFor="react-select-3-input">Tags</label>
-      <span data-testid="select-tags">
-        {action === Action.Edit && node?.tags?.length >= 0 ? (
-          <FormikSelect
-            className=""
-            isMulti={true}
-            selectOptions={tags}
-            formikFieldName="tags"
-            placeholder="Choose Tags"
-            defaultValue={node?.tags?.map(t => {
-              return { value: t.name, label: t.display_name };
-            })}
-          />
-        ) : (
-          ''
-        )}
-        {action === Action.Add ? (
-          <FormikSelect
-            className=""
-            isMulti={true}
-            selectOptions={tags}
-            formikFieldName="tags"
-            placeholder="Choose Tags"
-          />
-        ) : (
-          ''
-        )}
-      </span>
-    </div>
-  );
-}