datajunction-ui 0.0.1-a44.dev1 → 0.0.1-a44.dev2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datajunction-ui",
-  "version": "0.0.1-a44.dev1",
+  "version": "0.0.1-a44.dev2",
   "description": "DataJunction Metrics Platform UI",
   "module": "src/index.tsx",
   "repository": {
@@ -162,7 +162,7 @@
     ],
     "coverageThreshold": {
       "global": {
-        "statements": 90,
+        "statements": 89,
         "branches": 75,
         "lines": 90,
         "functions": 85

package/src/app/components/djgraph/DJNode.jsx

@@ -68,7 +68,7 @@ export function DJNode({ id, data }) {
             {data.type === 'source' ? data.table : data.display_name}
           </a>
           <Collapse
-            collapsed={true}
+            collapsed={data.is_current && data.type != 'metric' ? false : true}
             text={data.type !== 'metric' ? 'columns' : 'dimensions'}
             data={data}
           />

package/src/app/pages/NodePage/AddMaterializationPopover.jsx

@@ -3,6 +3,7 @@ import * as React from 'react';
 import DJClientContext from '../../providers/djclient';
 import { ErrorMessage, Field, Form, Formik } from 'formik';
 import { displayMessageAfterSubmit, labelize } from '../../../utils/form';
+import {ConfigField} from "./MaterializationConfigField";
 
 export default function AddMaterializationPopover({ node, onSubmit }) {
   const djClient = useContext(DJClientContext).DataJunctionAPI;
@@ -38,7 +39,8 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
     { setSubmitting, setStatus },
   ) => {
     setSubmitting(false);
-    const config = JSON.parse(values.config);
+    const config = {};
+    config.spark = values.spark_config;
     config.lookback_window = values.lookback_window;
     const response = await djClient.materialize(
       values.node,
@@ -48,13 +50,12 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
       config,
     );
     if (response.status === 200 || response.status === 201) {
-      setStatus({ success: 'Saved!' });
+      setStatus({ success: response.json.message });
     } else {
       setStatus({
         failure: `${response.json.message}`,
       });
     }
-    onSubmit();
     // window.location.reload();
   };
 
@@ -87,11 +88,14 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
         <Formik
           initialValues={{
             node: node?.name,
-            job_type: 'spark_sql',
+            job_type: node?.type === 'cube' ? 'druid_cube' : 'spark_sql',
             strategy: 'full',
-            config: '{"spark": {"spark.executor.memory": "6g"}}',
             schedule: '@daily',
             lookback_window: '1 DAY',
+            spark_config: {
+                "spark.executor.memory": "16g",
+                "spark.memory.fraction": "0.3"
+            },
           }}
           onSubmit={configureMaterialization}
         >
@@ -104,11 +108,9 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
                   <label htmlFor="job_type">Job Type</label>
                   <Field as="select" name="job_type">
                     <>
-                      {jobs?.map(job => (
-                        <option key={job.name} value={job.name}>
-                          {job.label}
-                        </option>
-                      ))}
+                      <option key={'druid_measures_cube'} value={'druid_measures_cube'}>Druid Measures Cube (Pre-Agg Cube)</option>
+                      <option key={'druid_metrics_cube'} value={'druid_metrics_cube'}>Druid Metrics Cube (Post-Agg Cube)</option>
+                      <option key={'spark_sql'} value={'spark_sql'}>Iceberg Table</option>
                     </>
                   </Field>
                 </span>
@@ -124,9 +126,12 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
                   <label htmlFor="strategy">Strategy</label>
                   <Field as="select" name="strategy">
                     <>
-                      {options.strategies?.map(strategy => (
-                        <option value={strategy.name}>{strategy.label}</option>
-                      ))}
+                      <option key={'full'} value={'full'}>
+                        Full
+                      </option>
+                      <option key={'incremental_time'} value={'incremental_time'}>
+                        Incremental Time
+                      </option>
                     </>
                   </Field>
                 </span>
@@ -154,17 +159,10 @@ export default function AddMaterializationPopover({ node, onSubmit }) {
                   />
                 </div>
                 <br />
-                <div className="DescriptionInput">
-                  <ErrorMessage name="description" component="span" />
-                  <label htmlFor="Config">Config</label>
-                  <Field
-                    type="textarea"
-                    as="textarea"
-                    name="config"
-                    id="Config"
-                    placeholder="Optional engine-specific configuration (i.e., Spark conf etc)"
-                  />
-                </div>
+                <ConfigField value={{
+                    "spark.executor.memory": "16g",
+                    "spark.memory.fraction": "0.3"
+                }}/>
                 <button
                   className="add_node"
                   type="submit"

package/src/app/pages/NodePage/MaterializationConfigField.jsx

@@ -0,0 +1,53 @@
+/**
+ * Materialization configuration field.
+ */
+import React from 'react';
+import { ErrorMessage, Field, useFormikContext } from 'formik';
+import CodeMirror from '@uiw/react-codemirror';
+import { langs } from '@uiw/codemirror-extensions-langs';
+
+export const ConfigField = ({ djClient, value }) => {
+  const formik = useFormikContext();
+  const jsonExt = langs.json();
+
+  const updateFormik = val => {
+    formik.setFieldValue('spark_config', val);
+  };
+
+  return (
+    <div className="DescriptionInput">
+      <ErrorMessage name="spark_config" component="span" />
+      <label htmlFor="SparkConfig">Spark Config</label>
+      <Field
+        type="textarea"
+        style={{ display: 'none' }}
+        as="textarea"
+        name="spark_config"
+        id="SparkConfig"
+      />
+      <div role="button" tabIndex={0} className="relative flex bg-[#282a36]">
+        <CodeMirror
+          id={'spark_config'}
+          name={'spark_config'}
+          extensions={[jsonExt]}
+          value={JSON.stringify(value, null, "    ")}
+          options={{
+            theme: 'default',
+            lineNumbers: true,
+          }}
+          width="100%"
+          height="170px"
+          style={{
+            margin: '0 0 23px 0',
+            flex: 1,
+            fontSize: '150%',
+            textAlign: 'left',
+          }}
+          onChange={(value, viewUpdate) => {
+            updateFormik(value);
+          }}
+        />
+      </div>
+    </div>
+  );
+};

package/src/app/pages/NodePage/NodeGraphTab.jsx

@@ -15,6 +15,8 @@ const NodeLineage = djNode => {
         col.attributes.some(attr => attr.attribute_type.name === 'primary_key'),
       )
       .map(col => col.name);
+    const dimensionLinkForeignKeys = node.dimension_links ? node.dimension_links
+      .flatMap(link => Object.keys(link.foreign_keys).map(key => key.split('.').slice(-1))) : [];
     const column_names = node.columns
       .map(col => {
         return {
@@ -23,7 +25,7 @@ const NodeLineage = djNode => {
           dimension: col.dimension !== null ? col.dimension.name : null,
           order: primary_key.includes(col.name)
             ? -1
-            : col.dimension !== null
+            : dimensionLinkForeignKeys.includes(col.name)
             ? 0
             : 1,
         };
@@ -49,27 +51,29 @@ const NodeLineage = djNode => {
   };
 
   const dimensionEdges = node => {
-    return node.columns
-      .filter(col => col.dimension)
-      .map(col => {
-        return {
-          id: col.dimension.name + '->' + node.name + '.' + col.name,
-          source: col.dimension.name,
-          sourceHandle: col.dimension.name,
-          target: node.name,
-          targetHandle: node.name + '.' + col.name,
-          draggable: true,
-          markerStart: {
-            type: MarkerType.Arrow,
-            width: 20,
-            height: 20,
-            color: '#b0b9c2',
-          },
-          style: {
-            strokeWidth: 3,
-            stroke: '#b0b9c2',
-          },
-        };
+    return node.dimension_links === undefined ? [] : node.dimension_links
+      .flatMap(link => {
+        return Object.keys(link.foreign_keys).map(fk => {
+            return {
+              id: link.dimension.name + '->' + node.name + '=' + link.foreign_keys[fk] + '->' + fk,
+              source: link.dimension.name,
+              sourceHandle: link.foreign_keys[fk],
+              target: node.name,
+              targetHandle: fk,
+              draggable: true,
+              markerStart: {
+                type: MarkerType.Arrow,
+                width: 20,
+                height: 20,
+                color: '#b0b9c2',
+              },
+              style: {
+                strokeWidth: 3,
+                stroke: '#b0b9c2',
+              },
+            };
+          }
+        )
       });
   };
 

package/src/app/pages/NodePage/NodeMaterializationTab.jsx

@@ -134,7 +134,8 @@ export default function NodeMaterializationTab({ node, djClient }) {
       <div className="table-vertical">
         <div>
           <h2>Materializations</h2>
-          <AddMaterializationPopover node={node} />
+          {node ?
+          <AddMaterializationPopover node={node} /> : <></>}
           {materializations.length > 0 ? (
             <table
               className="card-inner-table table"

package/src/app/pages/NodePage/__tests__/AddMaterializationPopover.test.jsx

@@ -17,6 +17,9 @@ describe('<AddMaterializationPopover />', () => {
     const onSubmitMock = jest.fn();
     mockDjClient.DataJunctionAPI.materialize.mockReturnValue({
       status: 201,
+      json: {
+        message: 'Saved!',
+      },
     });
     mockDjClient.DataJunctionAPI.materializationInfo.mockReturnValue({
       status: 200,

package/src/app/pages/NodePage/__tests__/NodeGraphTab.test.jsx

@@ -73,6 +73,7 @@ describe('<NodeLineage />', () => {
       parents: [],
       created_at: '2023-08-21T16:48:52.970554+00:00',
       tags: [],
+      dimension_links: [],
     },
     {
       namespace: 'default',
@@ -98,6 +99,7 @@ describe('<NodeLineage />', () => {
       query:
         '\n            SELECT\n              dateint,\n              month,\n              year,\n              day\n            FROM default.date\n        ',
       availability: null,
+      dimension_links: [],
       columns: [
         {
           name: 'dateint',
@@ -165,6 +167,7 @@ describe('<NodeLineage />', () => {
       query:
         '\n            SELECT\n            hard_hat_id,\n            last_name,\n            first_name,\n            title,\n            birth_date,\n            hire_date,\n            address,\n            city,\n            state,\n            postal_code,\n            country,\n            manager,\n            contractor_id\n            FROM default.hard_hats\n        ',
       availability: null,
+      dimension_links: [],
       columns: [
         {
           name: 'hard_hat_id',
@@ -292,6 +295,7 @@ describe('<NodeLineage />', () => {
       query:
         'SELECT  avg(price) default_DOT_avg_repair_price \n FROM default.repair_order_details\n\n',
       availability: null,
+      dimension_links: [],
       columns: [
         {
           name: 'default_DOT_avg_repair_price',

package/src/styles/index.css

@@ -241,7 +241,7 @@ table {
 }
 tr {
   display: table-row;
-  vertical-align: inherit;
+  vertical-align: top;
   border-color: inherit;
 }
 .card-table {
@@ -339,7 +339,7 @@ tr {
 .table thead th,
 td,
 tbody th {
-  vertical-align: middle;
+  vertical-align: top;
   text-align: left;
 }
 .table [data-sort],
@@ -371,6 +371,7 @@ tbody th {
   border-bottom: 0;
   padding: 1rem;
   max-width: 25rem;
+  vertical-align: top;
 }
 .card-inner-table td,
 .card-inner-table th {