datajunction-ui 0.0.1-rc.10 → 0.0.1-rc.11

package/src/app/pages/NodePage/NodeGraphTab.jsx

@@ -36,13 +36,18 @@ const NodeLineage = djNode => {
       edges,
       direction = 'LR',
       nodeWidth = 800,
-      nodeHeight = 150,
     ) => {
       const isHorizontal = direction === 'TB';
       dagreGraph.setGraph({ rankdir: direction });
+      const nodeHeightTracker = {};
 
       nodes.forEach(node => {
-        dagreGraph.setNode(node.id, { width: nodeWidth, height: nodeHeight });
+        nodeHeightTracker[node.id] =
+          Math.min(node.data.column_names.length, 10) * 37 + 250;
+        dagreGraph.setNode(node.id, {
+          width: nodeWidth,
+          height: nodeHeightTracker[node.id],
+        });
       });
 
       edges.forEach(edge => {
@@ -57,21 +62,24 @@ const NodeLineage = djNode => {
         node.sourcePosition = isHorizontal ? 'right' : 'bottom';
         node.position = {
           x: nodeWithPosition.x - nodeWidth / 2,
-          y: nodeWithPosition.y - nodeHeight / 2,
+          y: nodeWithPosition.y - nodeHeightTracker[node.id] / 2,
         };
+        node.width = nodeWidth;
+        node.height = nodeHeightTracker[node.id];
         return node;
       });
-
       return { nodes, edges };
     };
 
     const dagFetch = async () => {
-      let upstreams = await djClient.upstreams(djNode.djNode.name);
-      let downstreams = await djClient.downstreams(djNode.djNode.name);
+      let related_nodes = await djClient.node_dag(djNode.djNode.name);
+      // djNode.djNode.is_current = true;
       var djNodes = [djNode.djNode];
-      for (const iterable of [upstreams, downstreams]) {
+      for (const iterable of [related_nodes]) {
         for (const item of iterable) {
-          djNodes.push(item);
+          if (item.type !== 'cube') {
+            djNodes.push(item);
+          }
         }
       }
       let edges = [];
@@ -80,24 +88,43 @@ const NodeLineage = djNode => {
           if (parent.name) {
             edges.push({
               id: obj.name + '-' + parent.name,
-              target: obj.name,
               source: parent.name,
+              sourceHandle: parent.name,
+              target: obj.name,
+              targetHandle: obj.name,
               animated: true,
               markerEnd: {
                 type: MarkerType.Arrow,
               },
+              style: {
+                strokeWidth: 3,
+                stroke: '#b0b9c2',
+              },
             });
           }
         });
 
         obj.columns.forEach(col => {
           if (col.dimension) {
-            edges.push({
-              id: obj.name + '-' + col.dimension.name,
-              target: obj.name,
+            const edge = {
+              id: col.dimension.name + '->' + obj.name + '.' + col.name,
               source: col.dimension.name,
+              sourceHandle: col.dimension.name,
+              target: obj.name,
+              targetHandle: obj.name + '.' + col.name,
               draggable: true,
-            });
+              markerStart: {
+                type: MarkerType.Arrow,
+                width: 20,
+                height: 20,
+                color: '#b0b9c2',
+              },
+              style: {
+                strokeWidth: 3,
+                stroke: '#b0b9c2',
+              },
+            };
+            edges.push(edge);
           }
         });
       });
@@ -126,14 +153,15 @@ const NodeLineage = djNode => {
             type: node.type,
             primary_key: primary_key,
             column_names: column_names,
-            // dimensions: dimensions,
+            is_current: node.name === djNode.djNode.name,
           },
-          // parentNode: [node.name.split(".").slice(-2, -1)],
-          // extent: 'parent',
         };
       });
       setNodes(nodes);
       setEdges(edges);
+
+      // use dagre to determine the position of the parents (the DJ nodes)
+      // the positions of the columns are relative to each DJ node
       setElementsLayout(nodes, edges);
     };
 
@@ -146,7 +174,7 @@ const NodeLineage = djNode => {
   );
 
   return (
-    <div style={{ height: '600px' }}>
+    <div style={{ height: '800px' }}>
       <ReactFlow
         nodes={nodes}
         edges={edges}

package/src/app/pages/NodePage/NodeHistory.jsx

@@ -0,0 +1,71 @@
+import { useEffect, useState } from 'react';
+
+export default function NodeHistory({ node, djClient }) {
+  const [history, setHistory] = useState([]);
+  const [revisions, setRevisions] = useState([]);
+
+  useEffect(() => {
+    const fetchData = async () => {
+      const data = await djClient.history('node', node.name);
+      setHistory(data);
+      const revisions = await djClient.revisions(node.name);
+      setRevisions(revisions);
+    };
+    fetchData().catch(console.error);
+  }, [djClient, node]);
+  const tableData = history => {
+    return history.map(event => (
+      <tr>
+        <td className="text-start">
+          <span
+            className={`history_type__${event.activity_type} badge node_type`}
+          >
+            {event.activity_type}
+          </span>
+        </td>
+        <td>{event.entity_type}</td>
+        <td>{event.entity_name}</td>
+        <td>{event.user ? event.user : 'unknown'}</td>
+        <td>{event.created_at}</td>
+      </tr>
+    ));
+  };
+
+  const revisionsTable = revisions => {
+    return revisions.map(revision => (
+      <tr>
+        <td className="text-start">
+          <span className={`badge node_type__source`}>{revision.version}</span>
+        </td>
+        <td>{revision.display_name}</td>
+        <td>{revision.description}</td>
+        <td>{revision.query}</td>
+        <td>{revision.tags}</td>
+      </tr>
+    ));
+  };
+  return (
+    <div className="table-responsive">
+      <table className="card-inner-table table">
+        <thead className="fs-7 fw-bold text-gray-400 border-bottom-0">
+          <th className="text-start">Version</th>
+          <th>Display Name</th>
+          <th>Description</th>
+          <th>Query</th>
+          <th>Tags</th>
+        </thead>
+        {revisionsTable(revisions)}
+      </table>
+      <table className="card-inner-table table">
+        <thead className="fs-7 fw-bold text-gray-400 border-bottom-0">
+          <th className="text-start">Activity</th>
+          <th>Type</th>
+          <th>Name</th>
+          <th>User</th>
+          <th>Timestamp</th>
+        </thead>
+        {tableData(history)}
+      </table>
+    </div>
+  );
+}

package/src/app/pages/NodePage/NodeInfoTab.jsx

@@ -1,18 +1,36 @@
-import { Component } from 'react';
+import { useState, useContext, useEffect } from 'react';
 import { Light as SyntaxHighlighter } from 'react-syntax-highlighter';
 import { foundation } from 'react-syntax-highlighter/src/styles/hljs';
 import sql from 'react-syntax-highlighter/dist/esm/languages/hljs/sql';
 import { format } from 'sql-formatter';
-
 import NodeStatus from './NodeStatus';
 import ListGroupItem from '../../components/ListGroupItem';
+import ToggleSwitch from '../../components/ToggleSwitch';
+import DJClientContext from '../../providers/djclient';
 
 SyntaxHighlighter.registerLanguage('sql', sql);
 foundation.hljs['padding'] = '2rem';
 
-export default class NodeInfoTab extends Component {
-  nodeTags = this.props.node?.tags.map(tag => <div>{tag}</div>);
-  queryDiv = this.props.node?.query ? (
+export default function NodeInfoTab({ node }) {
+  const [compiledSQL, setCompiledSQL] = useState('');
+  const [checked, setChecked] = useState(false);
+  const nodeTags = node?.tags.map(tag => <div>{tag}</div>);
+  const djClient = useContext(DJClientContext).DataJunctionAPI;
+  useEffect(() => {
+    const fetchData = async () => {
+      const data = await djClient.compiledSql(node.name);
+      if (data.sql) {
+        setCompiledSQL(data.sql);
+      } else {
+        setCompiledSQL('/* Ran into an issue while generating compiled SQL */');
+      }
+    };
+    fetchData().catch(console.error);
+  }, [node, djClient]);
+  function toggle(value) {
+    return !value;
+  }
+  const queryDiv = node?.query ? (
     <div className="list-group-item d-flex">
       <div className="d-flex gap-2 w-100 justify-content-between py-3">
         <div
@@ -21,15 +39,34 @@ export default class NodeInfoTab extends Component {
           }}
         >
           <h6 className="mb-0 w-100">Query</h6>
+          {['metric', 'dimension', 'transform'].indexOf(node?.type) > -1 ? (
+            <ToggleSwitch
+              id="toggleSwitch"
+              checked={checked}
+              onChange={() => setChecked(toggle)}
+              toggleName="Show Compiled SQL"
+            />
+          ) : (
+            <></>
+          )}
           <SyntaxHighlighter language="sql" style={foundation}>
-            {format(this.props.node?.query, {
-              language: 'spark',
-              tabWidth: 2,
-              keywordCase: 'upper',
-              denseOperators: true,
-              logicalOperatorNewline: 'before',
-              expressionWidth: 10,
-            })}
+            {checked
+              ? format(compiledSQL, {
+                  language: 'spark',
+                  tabWidth: 2,
+                  keywordCase: 'upper',
+                  denseOperators: true,
+                  logicalOperatorNewline: 'before',
+                  expressionWidth: 10,
+                })
+              : format(node?.query, {
+                  language: 'spark',
+                  tabWidth: 2,
+                  keywordCase: 'upper',
+                  denseOperators: true,
+                  logicalOperatorNewline: 'before',
+                  expressionWidth: 10,
+                })}
           </SyntaxHighlighter>
         </div>
       </div>
@@ -38,50 +75,96 @@ export default class NodeInfoTab extends Component {
     <></>
   );
 
-  render() {
-    return (
-      <div className="list-group align-items-center justify-content-between flex-md-row gap-2">
-        <ListGroupItem
-          label="Description"
-          value={this.props.node?.description}
-        />
-        <div className="list-group-item d-flex">
-          <div className="d-flex gap-2 w-100 justify-content-between py-3">
-            <div>
-              <h6 className="mb-0 w-100">Version</h6>
-
-              <p className="mb-0 opacity-75">
+  const cubeElementsDiv = node?.cube_elements ? (
+    <div className="list-group-item d-flex">
+      <div className="d-flex gap-2 w-100 justify-content-between py-3">
+        <div
+          style={{
+            width: window.innerWidth * 0.8,
+          }}
+        >
+          <h6 className="mb-0 w-100">Cube Elements</h6>
+          <div className={`list-group-item`}>
+            {node.cube_elements.map(cubeElem => (
+              <div className="button-3 cube-element">
+                <a href={`/nodes/${cubeElem.node_name}`}>
+                  {cubeElem.type === 'metric'
+                    ? cubeElem.node_name
+                    : cubeElem.name}
+                </a>
                 <span
-                  className="rounded-pill badge bg-secondary-soft"
-                  style={{ marginLeft: '0.5rem', fontSize: '100%' }}
+                  className={`badge node_type__${
+                    cubeElem.type === 'metric' ? cubeElem.type : 'dimension'
+                  }`}
                 >
-                  {this.props.node?.version}
+                  {cubeElem.type === 'metric' ? cubeElem.type : 'dimension'}
                 </span>
-              </p>
-            </div>
-            <div>
-              <h6 className="mb-0 w-100">Status</h6>
-              <p className="mb-0 opacity-75">
-                <NodeStatus node={this.props.node} />
-              </p>
-            </div>
+              </div>
+            ))}
+          </div>
+        </div>
+      </div>
+    </div>
+  ) : (
+    <></>
+  );
+  return (
+    <div className="list-group align-items-center justify-content-between flex-md-row gap-2">
+      <ListGroupItem label="Description" value={node?.description} />
+      <div className="list-group-item d-flex">
+        <div className="d-flex gap-2 w-100 justify-content-between py-3">
+          <div>
+            <h6 className="mb-0 w-100">Version</h6>
+
+            <p className="mb-0 opacity-75">
+              <span
+                className="rounded-pill badge bg-secondary-soft"
+                style={{ marginLeft: '0.5rem', fontSize: '100%' }}
+              >
+                {node?.version}
+              </span>
+            </p>
+          </div>
+          {node.type === 'source' ? (
             <div>
-              <h6 className="mb-0 w-100">Mode</h6>
+              <h6 className="mb-0 w-100">Table</h6>
               <p className="mb-0 opacity-75">
-                <span className="status">{this.props.node?.mode}</span>
+                {node?.catalog.name}.{node?.schema_}.{node?.table}
               </p>
             </div>
-            <div>
-              <h6 className="mb-0 w-100">Tags</h6>
-              <p className="mb-0 opacity-75">{this.nodeTags}</p>
-            </div>
+          ) : (
+            <></>
+          )}
+          <div>
+            <h6 className="mb-0 w-100">Status</h6>
+            <p className="mb-0 opacity-75">
+              <NodeStatus node={node} />
+            </p>
+          </div>
+          <div>
+            <h6 className="mb-0 w-100">Mode</h6>
+            <p className="mb-0 opacity-75">
+              <span className="status">{node?.mode}</span>
+            </p>
+          </div>
+          <div>
+            <h6 className="mb-0 w-100">Tags</h6>
+            <p className="mb-0 opacity-75">{nodeTags}</p>
+          </div>
+          <div>
+            <h6 className="mb-0 w-100">Primary Key</h6>
+            <p className="mb-0 opacity-75">{node?.primary_key}</p>
+          </div>
+          <div>
+            <h6 className="mb-0 w-100">Last Updated</h6>
+            <p className="mb-0 opacity-75">
+              {new Date(node?.updated_at).toDateString()}
+            </p>
           </div>
-        </div>
-        {this.queryDiv}
-        <div className="list-group-item d-flex">
-          {this.props.node?.primary_key}
         </div>
       </div>
-    );
-  }
+      {node?.type !== 'cube' ? queryDiv : ''}
+      {cubeElementsDiv}
+    </div>
+  );
 }

package/src/app/pages/NodePage/NodeMaterializationTab.jsx

@@ -0,0 +1,151 @@
+import { useEffect, useState } from 'react';
+import ClientCodePopover from './ClientCodePopover';
+import TableIcon from '../../icons/TableIcon';
+
+const cronstrue = require('cronstrue');
+
+export default function NodeMaterializationTab({ node, djClient }) {
+  const [materializations, setMaterializations] = useState([]);
+  useEffect(() => {
+    const fetchData = async () => {
+      const data = await djClient.materializations(node.name);
+      setMaterializations(data);
+    };
+    fetchData().catch(console.error);
+  }, [djClient, node]);
+
+  const rangePartition = partition => {
+    return (
+      <div>
+        <span className="badge partition_value">
+          <span className="badge partition_value">{partition.range[0]}</span>to
+          <span className="badge partition_value">{partition.range[1]}</span>
+        </span>
+      </div>
+    );
+  };
+
+  const cron = materialization => {
+    var parsedCron = '';
+    try {
+      parsedCron = cronstrue.toString(materialization.schedule);
+    } catch (e) {}
+    return parsedCron;
+  };
+
+  const materializationRows = materializations => {
+    return materializations.map(materialization => (
+      <tr>
+        <td className="text-start node_name">
+          <a href={materialization.urls[0]}>{materialization.name}</a>
+          <ClientCodePopover code={materialization.clientCode} />
+        </td>
+        <td>
+          <span className={`badge cron`}>{materialization.schedule}</span>
+          <div className={`cron-description`}>{cron(materialization)} </div>
+        </td>
+        <td>
+          {materialization.engine.name}
+          <br />
+          {materialization.engine.version}
+        </td>
+        <td>
+          {materialization.config.partitions ? (
+            materialization.config.partitions.map(partition =>
+              partition.type_ === 'categorical' ? (
+                <div className="partition__full">
+                  <div className="partition__header">{partition.name}</div>
+                  <div className="partition__body">
+                    {partition.values !== null && partition.values.length > 0
+                      ? partition.values.map(val => (
+                          <span className="badge partition_value">{val}</span>
+                        ))
+                      : null}
+                    {partition.range !== null && partition.range.length > 0
+                      ? rangePartition(partition)
+                      : null}
+                    {(partition.range === null && partition.values === null) ||
+                    (partition.range.length === 0 &&
+                      partition.values.length === 0) ? (
+                      <span className={`badge partition_value_highlight`}>
+                        ALL
+                      </span>
+                    ) : null}
+                  </div>
+                </div>
+              ) : null,
+            )
+          ) : (
+            <br />
+          )}
+        </td>
+        <td>
+          {materialization.output_tables.map(table => (
+            <div className={`table__full`}>
+              <div className="table__header">
+                <TableIcon />{' '}
+                <span className={`entity-info`}>
+                  {table.split('.')[0] + '.' + table.split('.')[1]}
+                </span>
+              </div>
+              <div className={`table__body upstream_tables`}>
+                {table.split('.')[2]}
+              </div>
+            </div>
+          ))}
+        </td>
+        {/*<td>{Object.keys(materialization.config.spark).map(key => <li className={`list-group-item`}>{key}: {materialization.config.spark[key]}</li>)}</td>*/}
+
+        <td>
+          {materialization.config.partitions ? (
+            materialization.config.partitions.map(partition =>
+              partition.type_ === 'temporal' ? (
+                <div className="partition__full">
+                  <div className="partition__header">{partition.name}</div>
+                  <div className="partition__body">
+                    {partition.values !== null && partition.values.length > 0
+                      ? partition.values.map(val => (
+                          <span className="badge partition_value">{val}</span>
+                        ))
+                      : null}
+                    {partition.range !== null && partition.range.length > 0
+                      ? rangePartition(partition)
+                      : null}
+                  </div>
+                </div>
+              ) : null,
+            )
+          ) : (
+            <br />
+          )}
+        </td>
+        <td>
+          {materialization.urls.map((url, idx) => (
+            <a href={url}>[{idx + 1}]</a>
+          ))}
+        </td>
+      </tr>
+    ));
+  };
+  return (
+    <div className="table-responsive">
+      <table className="card-inner-table table">
+        <thead className="fs-7 fw-bold text-gray-400 border-bottom-0">
+          <th className="text-start">Name</th>
+          <th>Schedule</th>
+          <th>Engine</th>
+          <th>Partitions</th>
+          <th>Output Tables</th>
+          <th>Backfills</th>
+          <th>URLs</th>
+        </thead>
+        {materializationRows(
+          materializations.filter(
+            materialization =>
+              !(materialization.name === 'default' && node.type === 'cube'),
+          ),
+        )}
+      </table>
+    </div>
+  );
+}

package/src/app/pages/NodePage/NodeSQLTab.jsx

@@ -0,0 +1,100 @@
+import { useContext, useEffect, useState } from 'react';
+import Select from 'react-select';
+import DJClientContext from '../../providers/djclient';
+import { Light as SyntaxHighlighter } from 'react-syntax-highlighter';
+import { foundation } from 'react-syntax-highlighter/src/styles/hljs';
+import { format } from 'sql-formatter';
+
+const NodeSQLTab = djNode => {
+  const djClient = useContext(DJClientContext).DataJunctionAPI;
+  const [query, setQuery] = useState('');
+
+  const [selection, setSelection] = useState({
+    dimensions: [],
+    filters: [],
+  });
+
+  useEffect(() => {
+    const fetchData = async () => {
+      const query = await djClient.sql(djNode.djNode.name, selection);
+      setQuery(query.sql);
+    };
+    fetchData().catch(console.error);
+  }, [djClient, djNode.djNode.name, selection]);
+  const dimensionsList = djNode.djNode.dimensions
+    ? djNode.djNode.dimensions.map(dim => ({
+        value: dim.name,
+        label: dim.name + ` (${dim.type})`,
+      }))
+    : [''];
+
+  // const options = [
+  //   { value: '>=', label: '>=' },
+  //   { value: '<=', label: '<=' },
+  //   { value: '>', label: '>' },
+  //   { value: '<', label: '<' },
+  //   { value: '=', label: '=' },
+  //   { value: '!=', label: '!=' },
+  //   { value: 'IN', label: 'IN' },
+  // ];
+
+  const handleSubmit = event => {
+    event.preventDefault();
+  };
+
+  const handleChange = event => {
+    setSelection({ filters: [], dimensions: event.map(dim => dim.value) });
+  };
+
+  return (
+    <form
+      id="retrieve-sql"
+      name="retrieve-sql"
+      onSubmit={handleSubmit.bind(this)}
+    >
+      <div>
+        <h4>Group By</h4>
+        <Select
+          name="dimensions"
+          options={dimensionsList}
+          isMulti
+          isClearable
+          onChange={handleChange}
+        />
+        {/*<h4>Filters</h4>*/}
+        {/*<Select*/}
+        {/*  name="filter_name"*/}
+        {/*  options={dimensionsList}*/}
+        {/*  className="filters_attribute"*/}
+        {/*/>*/}
+        {/*<Select*/}
+        {/*  name="filter_operator"*/}
+        {/*  options={options}*/}
+        {/*  className="filters_attribute"*/}
+        {/*/>*/}
+        {/*<textarea name="filter_value" className="filters_attribute" />*/}
+
+        <div
+          style={{
+            width: window.innerWidth * 0.8,
+            marginTop: '2rem',
+          }}
+        >
+          <h6 className="mb-0 w-100">Query</h6>
+          <SyntaxHighlighter language="sql" style={foundation}>
+            {format(query, {
+              language: 'spark',
+              tabWidth: 2,
+              keywordCase: 'upper',
+              denseOperators: true,
+              logicalOperatorNewline: 'before',
+              expressionWidth: 10,
+            })}
+          </SyntaxHighlighter>
+        </div>
+      </div>
+    </form>
+  );
+};
+
+export default NodeSQLTab;