datajunction-ui 0.0.1-a44.dev0 → 0.0.1-a44.dev2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datajunction-ui",
-  "version": "0.0.1-a44.dev0",
+  "version": "0.0.1-a44.dev2",
   "description": "DataJunction Metrics Platform UI",
   "module": "src/index.tsx",
   "repository": {
@@ -162,7 +162,7 @@
     ],
     "coverageThreshold": {
       "global": {
-        "statements": 90,
+        "statements": 89,
         "branches": 75,
         "lines": 90,
         "functions": 85

package/src/app/components/djgraph/DJNode.jsx

@@ -68,7 +68,7 @@ export function DJNode({ id, data }) {
             {data.type === 'source' ? data.table : data.display_name}
           </a>
           <Collapse
-            collapsed={true}
+            collapsed={data.is_current && data.type != 'metric' ? false : true}
             text={data.type !== 'metric' ? 'columns' : 'dimensions'}
             data={data}
           />

package/src/app/pages/NodePage/AddMaterializationPopover.jsx

@@ -3,6 +3,7 @@ import * as React from 'react';
 import DJClientContext from '../../providers/djclient';
 import { ErrorMessage, Field, Form, Formik } from 'formik';
 import { displayMessageAfterSubmit, labelize } from '../../../utils/form';
+import {ConfigField} from "./MaterializationConfigField";
 
 export default function AddMaterializationPopover({ node, onSubmit }) {
   const djClient = useContext(DJClientContext).DataJunctionAPI;
@@ -38,7 +39,8 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
     { setSubmitting, setStatus },
   ) => {
     setSubmitting(false);
-    const config = JSON.parse(values.config);
+    const config = {};
+    config.spark = values.spark_config;
     config.lookback_window = values.lookback_window;
     const response = await djClient.materialize(
       values.node,
@@ -48,13 +50,12 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
       config,
     );
     if (response.status === 200 || response.status === 201) {
-      setStatus({ success: 'Saved!' });
+      setStatus({ success: response.json.message });
     } else {
       setStatus({
         failure: `${response.json.message}`,
       });
     }
-    onSubmit();
     // window.location.reload();
   };
 
@@ -87,11 +88,14 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
         <Formik
           initialValues={{
             node: node?.name,
-            job_type: 'spark_sql',
+            job_type: node?.type === 'cube' ? 'druid_cube' : 'spark_sql',
             strategy: 'full',
-            config: '{"spark": {"spark.executor.memory": "6g"}}',
             schedule: '@daily',
             lookback_window: '1 DAY',
+            spark_config: {
+                "spark.executor.memory": "16g",
+                "spark.memory.fraction": "0.3"
+            },
           }}
           onSubmit={configureMaterialization}
         >
@@ -104,11 +108,9 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
                   <label htmlFor="job_type">Job Type</label>
                   <Field as="select" name="job_type">
                     <>
-                      {jobs?.map(job => (
-                        <option key={job.name} value={job.name}>
-                          {job.label}
-                        </option>
-                      ))}
+                      <option key={'druid_measures_cube'} value={'druid_measures_cube'}>Druid Measures Cube (Pre-Agg Cube)</option>
+                      <option key={'druid_metrics_cube'} value={'druid_metrics_cube'}>Druid Metrics Cube (Post-Agg Cube)</option>
+                      <option key={'spark_sql'} value={'spark_sql'}>Iceberg Table</option>
                     </>
                   </Field>
                 </span>
@@ -124,9 +126,12 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
                   <label htmlFor="strategy">Strategy</label>
                   <Field as="select" name="strategy">
                     <>
-                      {options.strategies?.map(strategy => (
-                        <option value={strategy.name}>{strategy.label}</option>
-                      ))}
+                      <option key={'full'} value={'full'}>
+                        Full
+                      </option>
+                      <option key={'incremental_time'} value={'incremental_time'}>
+                        Incremental Time
+                      </option>
                     </>
                   </Field>
                 </span>
@@ -154,17 +159,10 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
                   />
                 </div>
                 <br />
-                <div className="DescriptionInput">
-                  <ErrorMessage name="description" component="span" />
-                  <label htmlFor="Config">Config</label>
-                  <Field
-                    type="textarea"
-                    as="textarea"
-                    name="config"
-                    id="Config"
-                    placeholder="Optional engine-specific configuration (i.e., Spark conf etc)"
-                  />
-                </div>
+                <ConfigField value={{
+                    "spark.executor.memory": "16g",
+                    "spark.memory.fraction": "0.3"
+                }}/>
                 <button
                   className="add_node"
                   type="submit"

package/src/app/pages/NodePage/LinkDimensionPopover.jsx

@@ -8,6 +8,7 @@ import { displayMessageAfterSubmit } from '../../../utils/form';
 
 export default function LinkDimensionPopover({
   column,
+  referencedDimensionNode,
   node,
   options,
   onSubmit,
@@ -28,15 +29,15 @@ export default function LinkDimensionPopover({
     };
   }, [setPopoverAnchor]);
 
-  const columnDimension = column.dimension;
+  const columnDimension = referencedDimensionNode;
 
   const handleSubmit = async (
     { node, column, dimension },
     { setSubmitting, setStatus },
   ) => {
     setSubmitting(false);
-    if (columnDimension?.name && dimension === 'Remove') {
-      await unlinkDimension(node, column, columnDimension?.name, setStatus);
+    if (referencedDimensionNode && dimension === 'Remove') {
+      await unlinkDimension(node, column, referencedDimensionNode, setStatus);
     } else {
       await linkDimension(node, column, dimension, setStatus);
     }
@@ -93,7 +94,7 @@ export default function LinkDimensionPopover({
             column: column.name,
             node: node.name,
             dimension: '',
-            currentDimension: column.dimension?.name,
+            currentDimension: referencedDimensionNode,
           }}
           onSubmit={handleSubmit}
         >
@@ -110,10 +111,10 @@ export default function LinkDimensionPopover({
                     placeholder="Select dimension to link"
                     className=""
                     defaultValue={
-                      column.dimension
+                      referencedDimensionNode
                         ? {
-                            value: column.dimension.name,
-                            label: column.dimension.name,
+                            value: referencedDimensionNode,
+                            label: referencedDimensionNode,
                           }
                         : ''
                     }

package/src/app/pages/NodePage/MaterializationConfigField.jsx

@@ -0,0 +1,53 @@
+/**
+ * Materialization configuration field.
+ */
+import React from 'react';
+import { ErrorMessage, Field, useFormikContext } from 'formik';
+import CodeMirror from '@uiw/react-codemirror';
+import { langs } from '@uiw/codemirror-extensions-langs';
+
+export const ConfigField = ({ djClient, value }) => {
+  const formik = useFormikContext();
+  const jsonExt = langs.json();
+
+  const updateFormik = val => {
+    formik.setFieldValue('spark_config', val);
+  };
+
+  return (
+    <div className="DescriptionInput">
+      <ErrorMessage name="spark_config" component="span" />
+      <label htmlFor="SparkConfig">Spark Config</label>
+      <Field
+        type="textarea"
+        style={{ display: 'none' }}
+        as="textarea"
+        name="spark_config"
+        id="SparkConfig"
+      />
+      <div role="button" tabIndex={0} className="relative flex bg-[#282a36]">
+        <CodeMirror
+          id={'spark_config'}
+          name={'spark_config'}
+          extensions={[jsonExt]}
+          value={JSON.stringify(value, null, "    ")}
+          options={{
+            theme: 'default',
+            lineNumbers: true,
+          }}
+          width="100%"
+          height="170px"
+          style={{
+            margin: '0 0 23px 0',
+            flex: 1,
+            fontSize: '150%',
+            textAlign: 'left',
+          }}
+          onChange={(value, viewUpdate) => {
+            updateFormik(value);
+          }}
+        />
+      </div>
+    </div>
+  );
+};

package/src/app/pages/NodePage/NodeColumnTab.jsx

@@ -12,6 +12,8 @@ export default function NodeColumnTab({ node, djClient }) {
   const [attributes, setAttributes] = useState([]);
   const [dimensions, setDimensions] = useState([]);
   const [columns, setColumns] = useState([]);
+  const [links, setLinks] = useState([]);
+
   useEffect(() => {
     const fetchData = async () => {
       setColumns(await djClient.columns(node));
@@ -88,92 +90,103 @@ export default function NodeColumnTab({ node, djClient }) {
   };
 
   const columnList = columns => {
-    return columns.map(col => (
-      <tr key={col.name}>
-        <td
-          className="text-start"
-          role="columnheader"
-          aria-label="ColumnName"
-          aria-hidden="false"
-        >
-          {col.name}
-        </td>
-        <td>
-          <span
-            className=""
-            role="columnheader"
-            aria-label="ColumnDisplayName"
-            aria-hidden="false"
-          >
-            {col.display_name}
-          </span>
-        </td>
-        <td>
-          <span
-            className={`node_type__${
-              node.type === 'cube' ? col.type : 'transform'
-            } badge node_type`}
+    return columns.map(col => {
+      const dimensionLinks = (links.length > 0 ? links : node?.dimension_links)
+        .map(link => [
+          link.dimension.name,
+          Object.entries(link.foreign_keys).filter(
+            entry => entry[0] === node.name + '.' + col.name,
+          ),
+        ])
+        .filter(keys => keys[1].length >= 1);
+      const referencedDimensionNode =
+        dimensionLinks.length > 0 ? dimensionLinks[0][0] : null;
+      return (
+        <tr key={col.name}>
+          <td
+            className="text-start"
             role="columnheader"
-            aria-label="ColumnType"
+            aria-label="ColumnName"
             aria-hidden="false"
           >
-            {col.type}
-          </span>
-        </td>
-        {node.type !== 'cube' ? (
+            {col.name}
+          </td>
           <td>
-            {col.dimension !== undefined && col.dimension !== null ? (
-              <>
-                <a href={`/nodes/${col.dimension.name}`}>
-                  {col.dimension.name}
-                </a>
-                <ClientCodePopover code={col.clientCode} />
-              </>
-            ) : (
-              ''
-            )}{' '}
-            <LinkDimensionPopover
-              column={col}
-              node={node}
-              options={dimensions}
-              onSubmit={async () => {
-                const res = await djClient.node(node.name);
-                setColumns(res.columns);
-              }}
-            />
+            <span
+              className=""
+              role="columnheader"
+              aria-label="ColumnDisplayName"
+              aria-hidden="false"
+            >
+              {col.display_name}
+            </span>
+          </td>
+          <td>
+            <span
+              className={`node_type__${
+                node.type === 'cube' ? col.type : 'transform'
+              } badge node_type`}
+              role="columnheader"
+              aria-label="ColumnType"
+              aria-hidden="false"
+            >
+              {col.type}
+            </span>
           </td>
-        ) : (
-          ''
-        )}
-        {node.type !== 'cube' ? (
+          {node.type !== 'cube' ? (
+            <td>
+              {referencedDimensionNode !== null ? (
+                <a href={`/nodes/${referencedDimensionNode}`}>
+                  {referencedDimensionNode}
+                </a>
+              ) : (
+                ''
+              )}
+              <LinkDimensionPopover
+                column={col}
+                referencedDimensionNode={referencedDimensionNode}
+                node={node}
+                options={dimensions}
+                onSubmit={async () => {
+                  const res = await djClient.node(node.name);
+                  setColumns(res.columns);
+                  setLinks(res.dimension_links);
+                }}
+              />
+            </td>
+          ) : (
+            ''
+          )}
+          {node.type !== 'cube' ? (
+            <td>
+              {showColumnAttributes(col)}
+              <EditColumnPopover
+                column={col}
+                node={node}
+                options={attributes}
+                onSubmit={async () => {
+                  const res = await djClient.node(node.name);
+                  setColumns(res.columns);
+                }}
+              />
+            </td>
+          ) : (
+            ''
+          )}
           <td>
-            {showColumnAttributes(col)}
-            <EditColumnPopover
+            {showColumnPartition(col)}
+            <PartitionColumnPopover
               column={col}
               node={node}
-              options={attributes}
               onSubmit={async () => {
                 const res = await djClient.node(node.name);
                 setColumns(res.columns);
               }}
             />
           </td>
-        ) : (
-          ''
-        )}
-        <td>
-          {showColumnPartition(col)}
-          <PartitionColumnPopover
-            column={col}
-            node={node}
-            onSubmit={async () => {
-              const res = await djClient.node(node.name);
-              setColumns(res.columns);
-            }}
-          />
-        </td>
-      </tr>
-    ));
+        </tr>
+      );
+    });
   };
 
   return (

package/src/app/pages/NodePage/NodeGraphTab.jsx

@@ -15,6 +15,8 @@ const NodeLineage = djNode => {
         col.attributes.some(attr => attr.attribute_type.name === 'primary_key'),
       )
       .map(col => col.name);
+    const dimensionLinkForeignKeys = node.dimension_links ? node.dimension_links
+      .flatMap(link => Object.keys(link.foreign_keys).map(key => key.split('.').slice(-1))) : [];
     const column_names = node.columns
       .map(col => {
         return {
@@ -23,7 +25,7 @@ const NodeLineage = djNode => {
           dimension: col.dimension !== null ? col.dimension.name : null,
           order: primary_key.includes(col.name)
             ? -1
-            : col.dimension !== null
+            : dimensionLinkForeignKeys.includes(col.name)
             ? 0
             : 1,
         };
@@ -49,27 +51,29 @@ const NodeLineage = djNode => {
   };
 
   const dimensionEdges = node => {
-    return node.columns
-      .filter(col => col.dimension)
-      .map(col => {
-        return {
-          id: col.dimension.name + '->' + node.name + '.' + col.name,
-          source: col.dimension.name,
-          sourceHandle: col.dimension.name,
-          target: node.name,
-          targetHandle: node.name + '.' + col.name,
-          draggable: true,
-          markerStart: {
-            type: MarkerType.Arrow,
-            width: 20,
-            height: 20,
-            color: '#b0b9c2',
-          },
-          style: {
-            strokeWidth: 3,
-            stroke: '#b0b9c2',
-          },
-        };
+    return node.dimension_links === undefined ? [] : node.dimension_links
+      .flatMap(link => {
+        return Object.keys(link.foreign_keys).map(fk => {
+            return {
+              id: link.dimension.name + '->' + node.name + '=' + link.foreign_keys[fk] + '->' + fk,
+              source: link.dimension.name,
+              sourceHandle: link.foreign_keys[fk],
+              target: node.name,
+              targetHandle: fk,
+              draggable: true,
+              markerStart: {
+                type: MarkerType.Arrow,
+                width: 20,
+                height: 20,
+                color: '#b0b9c2',
+              },
+              style: {
+                strokeWidth: 3,
+                stroke: '#b0b9c2',
+              },
+            };
+          }
+        )
       });
   };
 

package/src/app/pages/NodePage/NodeMaterializationTab.jsx

@@ -134,7 +134,8 @@ export default function NodeMaterializationTab({ node, djClient }) {
       <div className="table-vertical">
         <div>
           <h2>Materializations</h2>
-          <AddMaterializationPopover node={node} />
+          {node ?
+          <AddMaterializationPopover node={node} /> : <></>}
           {materializations.length > 0 ? (
             <table
               className="card-inner-table table"

package/src/app/pages/NodePage/__tests__/AddMaterializationPopover.test.jsx

@@ -17,6 +17,9 @@ describe('<AddMaterializationPopover />', () => {
     const onSubmitMock = jest.fn();
     mockDjClient.DataJunctionAPI.materialize.mockReturnValue({
       status: 201,
+      json: {
+        message: 'Saved!',
+      },
     });
     mockDjClient.DataJunctionAPI.materializationInfo.mockReturnValue({
       status: 200,

package/src/app/pages/NodePage/__tests__/LinkDimensionPopover.test.jsx

@@ -42,6 +42,7 @@ describe('<LinkDimensionPopover />', () => {
       <DJClientContext.Provider value={mockDjClient}>
         <LinkDimensionPopover
           column={column}
+          referencedDimensionNode={'default.dimension1'}
           node={node}
           options={options}
           onSubmit={onSubmitMock}
@@ -94,7 +95,6 @@ describe('<LinkDimensionPopover />', () => {
     };
     const node = { name: 'default.node1' };
     const options = [
-      { value: 'Remove', label: '[Remove dimension link]' },
       { value: 'default.dimension1', label: 'Dimension 1' },
       { value: 'default.dimension2', label: 'Dimension 2' },
     ];
@@ -117,6 +117,7 @@ describe('<LinkDimensionPopover />', () => {
       <DJClientContext.Provider value={mockDjClient}>
         <LinkDimensionPopover
           column={column}
+          referencedDimensionNode={'default.dimension1'}
           node={node}
           options={options}
           onSubmit={onSubmitMock}
@@ -148,7 +149,7 @@ describe('<LinkDimensionPopover />', () => {
 
     // Click on the 'Remove' option and save
     fireEvent.keyDown(linkDimension.firstChild, { key: 'ArrowDown' });
-    fireEvent.click(screen.getByText('[Remove dimension link]'));
+    fireEvent.click(screen.getByText('[Remove Dimension]'));
     fireEvent.click(getByText('Save'));
     getByText('Save').click();
 

package/src/app/pages/NodePage/__tests__/NodeColumnTab.test.jsx

@@ -98,6 +98,10 @@ describe('<NodeColumnTab />', () => {
           'default.contractor.contractor_id = default.repair_orders.contractor_id',
         join_cardinality: 'one_to_one',
         role: 'contractor',
+        foreign_keys: {
+          'default.repair_orders.contractor_id':
+            'default.contractor.contractor_id',
+        },
       },
     ],
   };

package/src/app/pages/NodePage/__tests__/NodeGraphTab.test.jsx

@@ -73,6 +73,7 @@ describe('<NodeLineage />', () => {
       parents: [],
       created_at: '2023-08-21T16:48:52.970554+00:00',
       tags: [],
+      dimension_links: [],
     },
     {
       namespace: 'default',
@@ -98,6 +99,7 @@ describe('<NodeLineage />', () => {
       query:
         '\n            SELECT\n              dateint,\n              month,\n              year,\n              day\n            FROM default.date\n        ',
       availability: null,
+      dimension_links: [],
       columns: [
         {
           name: 'dateint',
@@ -165,6 +167,7 @@ describe('<NodeLineage />', () => {
       query:
         '\n            SELECT\n            hard_hat_id,\n            last_name,\n            first_name,\n            title,\n            birth_date,\n            hire_date,\n            address,\n            city,\n            state,\n            postal_code,\n            country,\n            manager,\n            contractor_id\n            FROM default.hard_hats\n        ',
       availability: null,
+      dimension_links: [],
       columns: [
         {
           name: 'hard_hat_id',
@@ -292,6 +295,7 @@ describe('<NodeLineage />', () => {
       query:
         'SELECT  avg(price) default_DOT_avg_repair_price \n FROM default.repair_order_details\n\n',
       availability: null,
+      dimension_links: [],
       columns: [
         {
           name: 'default_DOT_avg_repair_price',

package/src/app/pages/Root/index.tsx

@@ -24,11 +24,19 @@ export function Root() {
       <div className="container d-flex align-items-center justify-content-between">
         <div className="header">
           <div className="logo">
-            <a href={'/'} style={{textTransform: 'none', textDecoration: 'none', color: '#000'}}>
+            <a
+              href={'/'}
+              style={{
+                textTransform: 'none',
+                textDecoration: 'none',
+                color: '#000',
+              }}
+            >
               <h2>
-              <DJLogo />
-              Data<b>Junction</b>
-            </h2></a>
+                <DJLogo />
+                Data<b>Junction</b>
+              </h2>
+            </a>
           </div>
           <Search />
           <div className="menu">

package/src/styles/index.css

@@ -241,7 +241,7 @@ table {
 }
 tr {
   display: table-row;
-  vertical-align: inherit;
+  vertical-align: top;
   border-color: inherit;
 }
 .card-table {
@@ -339,7 +339,7 @@ tr {
 .table thead th,
 td,
 tbody th {
-  vertical-align: middle;
+  vertical-align: top;
   text-align: left;
 }
 .table [data-sort],
@@ -371,6 +371,7 @@ tbody th {
   border-bottom: 0;
   padding: 1rem;
   max-width: 25rem;
+  vertical-align: top;
 }
 .card-inner-table td,
 .card-inner-table th {