datajunction-ui 0.0.1-a44.dev1 → 0.0.1-a44.dev4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datajunction-ui",
-  "version": "0.0.1-a44.dev1",
+  "version": "0.0.1-a44.dev4",
   "description": "DataJunction Metrics Platform UI",
   "module": "src/index.tsx",
   "repository": {
@@ -162,7 +162,7 @@
     ],
     "coverageThreshold": {
       "global": {
-        "statements": 90,
+        "statements": 89,
         "branches": 75,
         "lines": 90,
         "functions": 85

package/src/app/components/djgraph/DJNode.jsx

@@ -68,7 +68,7 @@ export function DJNode({ id, data }) {
             {data.type === 'source' ? data.table : data.display_name}
           </a>
           <Collapse
-            collapsed={true}
+            collapsed={data.is_current && data.type != 'metric' ? false : true}
             text={data.type !== 'metric' ? 'columns' : 'dimensions'}
             data={data}
           />

package/src/app/pages/AddEditNodePage/__tests__/AddEditNodePageFormFailed.test.jsx

@@ -101,9 +101,7 @@ describe('AddEditNodePage submission failed', () => {
         status: 404,
       });
 
-      expect(
-        await screen.getByText('Update failed, Some tags were not found'),
-      ).toBeInTheDocument();
+      expect(await screen.getByText('Update failed')).toBeInTheDocument();
     });
   }, 60000);
 });

package/src/app/pages/AddEditNodePage/index.jsx

@@ -24,6 +24,7 @@ import { DisplayNameField } from './DisplayNameField';
 import { DescriptionField } from './DescriptionField';
 import { NodeModeField } from './NodeModeField';
 import { RequiredDimensionsSelect } from './RequiredDimensionsSelect';
+import LoadingIcon from '../../icons/LoadingIcon';
 
 class Action {
   static Add = new Action('add');
@@ -63,19 +64,18 @@ export function AddEditNodePage() {
     return errors;
   };
 
-  const handleSubmit = (values, { setSubmitting, setStatus }) => {
+  const handleSubmit = async (values, { setSubmitting, setStatus }) => {
     if (action === Action.Add) {
-      setTimeout(() => {
-        createNode(values, setStatus);
+      await createNode(values, setStatus).then(_ => {
+        window.scrollTo({ top: 0, left: 0, behavior: 'smooth' });
         setSubmitting(false);
-      }, 400);
+      });
     } else {
-      setTimeout(() => {
-        patchNode(values, setStatus);
+      await patchNode(values, setStatus).then(_ => {
+        window.scrollTo({ top: 0, left: 0, behavior: 'smooth' });
         setSubmitting(false);
-      }, 400);
+      });
     }
-    window.scrollTo({ top: 0, left: 0, behavior: 'smooth' });
   };
 
   const pageTitle =
@@ -169,7 +169,7 @@ export function AddEditNodePage() {
       });
     } else {
       setStatus({
-        failure: `${json.message}, ${tagsResponse.json.message}`,
+        failure: `${json.message}`,
       });
     }
   };
@@ -410,7 +410,12 @@ export function AddEditNodePage() {
                         <NodeModeField />
 
                         <button type="submit" disabled={isSubmitting}>
-                          {action === Action.Add ? 'Create' : 'Save'} {nodeType}
+                          {isSubmitting ? (
+                            <LoadingIcon />
+                          ) : (
+                            (action === Action.Add ? 'Create ' : 'Save ') +
+                            (nodeType ? nodeType : '')
+                          )}
                         </button>
                       </>
                     )}

package/src/app/pages/NodePage/AddMaterializationPopover.jsx

@@ -3,6 +3,8 @@ import * as React from 'react';
 import DJClientContext from '../../providers/djclient';
 import { ErrorMessage, Field, Form, Formik } from 'formik';
 import { displayMessageAfterSubmit, labelize } from '../../../utils/form';
+import { ConfigField } from './MaterializationConfigField';
+import LoadingIcon from '../../icons/LoadingIcon';
 
 export default function AddMaterializationPopover({ node, onSubmit }) {
   const djClient = useContext(DJClientContext).DataJunctionAPI;
@@ -33,29 +35,34 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
     };
   }, [djClient, setPopoverAnchor]);
 
-  const configureMaterialization = async (
-    values,
-    { setSubmitting, setStatus },
-  ) => {
-    setSubmitting(false);
-    const config = JSON.parse(values.config);
+  const materialize = async (values, setStatus) => {
+    const config = {};
+    config.spark = values.spark_config;
     config.lookback_window = values.lookback_window;
-    const response = await djClient.materialize(
+    const { status, json } = await djClient.materialize(
       values.node,
       values.job_type,
       values.strategy,
       values.schedule,
       config,
     );
-    if (response.status === 200 || response.status === 201) {
-      setStatus({ success: 'Saved!' });
+    if (status === 200 || status === 201) {
+      setStatus({ success: json.message });
     } else {
       setStatus({
-        failure: `${response.json.message}`,
+        failure: `${json.message}`,
       });
     }
-    onSubmit();
-    // window.location.reload();
+  };
+
+  const configureMaterialization = async (
+    values,
+    { setSubmitting, setStatus },
+  ) => {
+    await materialize(values, setStatus).then(_ => {
+      window.scrollTo({ top: 0, left: 0, behavior: 'smooth' });
+      setSubmitting(false);
+    });
   };
 
   return (
@@ -87,11 +94,15 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
         <Formik
           initialValues={{
             node: node?.name,
-            job_type: 'spark_sql',
+            job_type:
+              node?.type === 'cube' ? 'druid_metrics_cube' : 'spark_sql',
             strategy: 'full',
-            config: '{"spark": {"spark.executor.memory": "6g"}}',
             schedule: '@daily',
             lookback_window: '1 DAY',
+            spark_config: {
+              'spark.executor.memory': '16g',
+              'spark.memory.fraction': '0.3',
+            },
           }}
           onSubmit={configureMaterialization}
         >
@@ -104,11 +115,21 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
                   <label htmlFor="job_type">Job Type</label>
                   <Field as="select" name="job_type">
                     <>
-                      {jobs?.map(job => (
-                        <option key={job.name} value={job.name}>
-                          {job.label}
-                        </option>
-                      ))}
+                      <option
+                        key={'druid_measures_cube'}
+                        value={'druid_measures_cube'}
+                      >
+                        Druid Measures Cube (Pre-Agg Cube)
+                      </option>
+                      <option
+                        key={'druid_metrics_cube'}
+                        value={'druid_metrics_cube'}
+                      >
+                        Druid Metrics Cube (Post-Agg Cube)
+                      </option>
+                      <option key={'spark_sql'} value={'spark_sql'}>
+                        Iceberg Table
+                      </option>
                     </>
                   </Field>
                 </span>
@@ -124,9 +145,15 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
                   <label htmlFor="strategy">Strategy</label>
                   <Field as="select" name="strategy">
                     <>
-                      {options.strategies?.map(strategy => (
-                        <option value={strategy.name}>{strategy.label}</option>
-                      ))}
+                      <option key={'full'} value={'full'}>
+                        Full
+                      </option>
+                      <option
+                        key={'incremental_time'}
+                        value={'incremental_time'}
+                      >
+                        Incremental Time
+                      </option>
                     </>
                   </Field>
                 </span>
@@ -154,24 +181,20 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
                   />
                 </div>
                 <br />
-                <div className="DescriptionInput">
-                  <ErrorMessage name="description" component="span" />
-                  <label htmlFor="Config">Config</label>
-                  <Field
-                    type="textarea"
-                    as="textarea"
-                    name="config"
-                    id="Config"
-                    placeholder="Optional engine-specific configuration (i.e., Spark conf etc)"
-                  />
-                </div>
+                <ConfigField
+                  value={{
+                    'spark.executor.memory': '16g',
+                    'spark.memory.fraction': '0.3',
+                  }}
+                />
                 <button
                   className="add_node"
                   type="submit"
                   aria-label="SaveEditColumn"
                   aria-hidden="false"
+                  disabled={isSubmitting}
                 >
-                  Save
+                  {isSubmitting ? <LoadingIcon /> : 'Save'}
                 </button>
               </Form>
             );

package/src/app/pages/NodePage/MaterializationConfigField.jsx

@@ -0,0 +1,53 @@
+/**
+ * Materialization configuration field.
+ */
+import React from 'react';
+import { ErrorMessage, Field, useFormikContext } from 'formik';
+import CodeMirror from '@uiw/react-codemirror';
+import { langs } from '@uiw/codemirror-extensions-langs';
+
+export const ConfigField = ({ djClient, value }) => {
+  const formik = useFormikContext();
+  const jsonExt = langs.json();
+
+  const updateFormik = val => {
+    formik.setFieldValue('spark_config', val);
+  };
+
+  return (
+    <div className="DescriptionInput">
+      <ErrorMessage name="spark_config" component="span" />
+      <label htmlFor="SparkConfig">Spark Config</label>
+      <Field
+        type="textarea"
+        style={{ display: 'none' }}
+        as="textarea"
+        name="spark_config"
+        id="SparkConfig"
+      />
+      <div role="button" tabIndex={0} className="relative flex bg-[#282a36]">
+        <CodeMirror
+          id={'spark_config'}
+          name={'spark_config'}
+          extensions={[jsonExt]}
+          value={JSON.stringify(value, null, '    ')}
+          options={{
+            theme: 'default',
+            lineNumbers: true,
+          }}
+          width="100%"
+          height="170px"
+          style={{
+            margin: '0 0 23px 0',
+            flex: 1,
+            fontSize: '150%',
+            textAlign: 'left',
+          }}
+          onChange={(value, viewUpdate) => {
+            updateFormik(value);
+          }}
+        />
+      </div>
+    </div>
+  );
+};

package/src/app/pages/NodePage/NodeGraphTab.jsx

@@ -15,6 +15,11 @@ const NodeLineage = djNode => {
         col.attributes.some(attr => attr.attribute_type.name === 'primary_key'),
       )
       .map(col => col.name);
+    const dimensionLinkForeignKeys = node.dimension_links
+      ? node.dimension_links.flatMap(link =>
+          Object.keys(link.foreign_keys).map(key => key.split('.').slice(-1)),
+        )
+      : [];
     const column_names = node.columns
       .map(col => {
         return {
@@ -23,7 +28,7 @@ const NodeLineage = djNode => {
           dimension: col.dimension !== null ? col.dimension.name : null,
           order: primary_key.includes(col.name)
             ? -1
-            : col.dimension !== null
+            : dimensionLinkForeignKeys.includes(col.name)
             ? 0
             : 1,
         };
@@ -49,28 +54,37 @@ const NodeLineage = djNode => {
   };
 
   const dimensionEdges = node => {
-    return node.columns
-      .filter(col => col.dimension)
-      .map(col => {
-        return {
-          id: col.dimension.name + '->' + node.name + '.' + col.name,
-          source: col.dimension.name,
-          sourceHandle: col.dimension.name,
-          target: node.name,
-          targetHandle: node.name + '.' + col.name,
-          draggable: true,
-          markerStart: {
-            type: MarkerType.Arrow,
-            width: 20,
-            height: 20,
-            color: '#b0b9c2',
-          },
-          style: {
-            strokeWidth: 3,
-            stroke: '#b0b9c2',
-          },
-        };
-      });
+    return node.dimension_links === undefined
+      ? []
+      : node.dimension_links.flatMap(link => {
+          return Object.keys(link.foreign_keys).map(fk => {
+            return {
+              id:
+                link.dimension.name +
+                '->' +
+                node.name +
+                '=' +
+                link.foreign_keys[fk] +
+                '->' +
+                fk,
+              source: link.dimension.name,
+              sourceHandle: link.foreign_keys[fk],
+              target: node.name,
+              targetHandle: fk,
+              draggable: true,
+              markerStart: {
+                type: MarkerType.Arrow,
+                width: 20,
+                height: 20,
+                color: '#b0b9c2',
+              },
+              style: {
+                strokeWidth: 3,
+                stroke: '#b0b9c2',
+              },
+            };
+          });
+        });
   };
 
   const parentEdges = node => {

package/src/app/pages/NodePage/NodeMaterializationTab.jsx

@@ -134,7 +134,7 @@ export default function NodeMaterializationTab({ node, djClient }) {
       <div className="table-vertical">
         <div>
           <h2>Materializations</h2>
-          <AddMaterializationPopover node={node} />
+          {node ? <AddMaterializationPopover node={node} /> : <></>}
           {materializations.length > 0 ? (
             <table
               className="card-inner-table table"

package/src/app/pages/NodePage/__tests__/AddMaterializationPopover.test.jsx

@@ -17,6 +17,9 @@ describe('<AddMaterializationPopover />', () => {
     const onSubmitMock = jest.fn();
     mockDjClient.DataJunctionAPI.materialize.mockReturnValue({
       status: 201,
+      json: {
+        message: 'Saved!',
+      },
     });
     mockDjClient.DataJunctionAPI.materializationInfo.mockReturnValue({
       status: 200,

package/src/app/pages/NodePage/__tests__/NodeGraphTab.test.jsx

@@ -73,6 +73,7 @@ describe('<NodeLineage />', () => {
       parents: [],
       created_at: '2023-08-21T16:48:52.970554+00:00',
       tags: [],
+      dimension_links: [],
     },
     {
       namespace: 'default',
@@ -98,6 +99,7 @@ describe('<NodeLineage />', () => {
       query:
         '\n            SELECT\n              dateint,\n              month,\n              year,\n              day\n            FROM default.date\n        ',
       availability: null,
+      dimension_links: [],
       columns: [
         {
           name: 'dateint',
@@ -165,6 +167,7 @@ describe('<NodeLineage />', () => {
       query:
         '\n            SELECT\n            hard_hat_id,\n            last_name,\n            first_name,\n            title,\n            birth_date,\n            hire_date,\n            address,\n            city,\n            state,\n            postal_code,\n            country,\n            manager,\n            contractor_id\n            FROM default.hard_hats\n        ',
       availability: null,
+      dimension_links: [],
       columns: [
         {
           name: 'hard_hat_id',
@@ -292,6 +295,7 @@ describe('<NodeLineage />', () => {
       query:
         'SELECT  avg(price) default_DOT_avg_repair_price \n FROM default.repair_order_details\n\n',
       availability: null,
+      dimension_links: [],
       columns: [
         {
           name: 'default_DOT_avg_repair_price',

package/src/styles/index.css

@@ -241,7 +241,7 @@ table {
 }
 tr {
   display: table-row;
-  vertical-align: inherit;
+  vertical-align: top;
   border-color: inherit;
 }
 .card-table {
@@ -339,7 +339,7 @@ tr {
 .table thead th,
 td,
 tbody th {
-  vertical-align: middle;
+  vertical-align: top;
   text-align: left;
 }
 .table [data-sort],
@@ -371,6 +371,7 @@ tbody th {
   border-bottom: 0;
   padding: 1rem;
   max-width: 25rem;
+  vertical-align: top;
 }
 .card-inner-table td,
 .card-inner-table th {