datajunction-ui 0.0.1-rc.1 → 0.0.1-rc.11

package/src/app/pages/SQLBuilderPage/index.jsx

@@ -0,0 +1,344 @@
+import { useContext, useEffect, useState } from 'react';
+import NamespaceHeader from '../../components/NamespaceHeader';
+import { DataJunctionAPI } from '../../services/DJService';
+import DJClientContext from '../../providers/djclient';
+import { Light as SyntaxHighlighter } from 'react-syntax-highlighter';
+import { foundation } from 'react-syntax-highlighter/src/styles/hljs';
+import { format } from 'sql-formatter';
+import Select from 'react-select';
+import QueryInfo from '../../components/QueryInfo';
+
+export function SQLBuilderPage() {
+  const djClient = useContext(DJClientContext).DataJunctionAPI;
+  const [stagedMetrics, setStagedMetrics] = useState([]);
+  const [metrics, setMetrics] = useState([]);
+  const [commonDimensionsList, setCommonDimensionsList] = useState([]);
+  const [selectedDimensions, setSelectedDimensions] = useState([]);
+  const [stagedDimensions, setStagedDimensions] = useState([]);
+  const [selectedMetrics, setSelectedMetrics] = useState([]);
+  const [query, setQuery] = useState('');
+  const [submittedQueryInfo, setSubmittedQueryInfo] = useState(null);
+  const [data, setData] = useState(null);
+  const [loadingData, setLoadingData] = useState(false);
+  const [viewData, setViewData] = useState(false);
+  const [showHelp, setShowHelp] = useState(true);
+  const [showNumRows, setShowNumRows] = useState(100);
+  const [displayedRows, setDisplayedRows] = useState(<></>);
+  const numRowsOptions = [
+    {
+      value: 10,
+      label: '10 Rows',
+      isFixed: true,
+    },
+    {
+      value: 100,
+      label: '100 Rows',
+      isFixed: true,
+    },
+    {
+      value: 1000,
+      label: '1,000 Rows',
+      isFixed: true,
+    },
+  ];
+  const toggleViewData = () => setViewData(current => !current);
+
+  // Get data for the current selection of metrics and dimensions
+  const getData = () => {
+    setLoadingData(true);
+    const fetchData = async () => {
+      setData(null);
+      const queryInfo = await djClient.data(
+        selectedMetrics,
+        selectedDimensions,
+      );
+      setLoadingData(false);
+      setSubmittedQueryInfo(queryInfo);
+      queryInfo.numRows = 0;
+      if (queryInfo.results && queryInfo.results?.length) {
+        setData(queryInfo.results);
+        queryInfo.numRows = queryInfo.results[0].rows.length;
+        setViewData(true);
+        setShowNumRows(100);
+      }
+    };
+    fetchData().catch(console.error);
+  };
+
+  const resetView = () => {
+    setQuery('');
+    setData(null);
+    setViewData(false);
+  };
+  const handleMetricSelect = event => {
+    const metrics = event.map(m => m.value);
+    resetView();
+    setStagedMetrics(metrics);
+  };
+
+  const handleMetricSelectorClose = () => {
+    setSelectedMetrics(stagedMetrics);
+  };
+
+  const handleDimensionSelect = event => {
+    const dimensions = event.map(d => d.value);
+    resetView();
+    setStagedDimensions(dimensions);
+  };
+
+  const handleDimensionSelectorClose = () => {
+    setSelectedDimensions(stagedDimensions);
+  };
+
+  // Get metrics
+  useEffect(() => {
+    const fetchData = async () => {
+      const metrics = await djClient.metrics();
+      setMetrics(metrics.map(m => ({ value: m, label: m })));
+    };
+    fetchData().catch(console.error);
+  }, [djClient, djClient.metrics]);
+
+  // Get common dimensions
+  useEffect(() => {
+    const fetchData = async () => {
+      if (selectedMetrics.length) {
+        const commonDimensions = await djClient.commonDimensions(
+          selectedMetrics,
+        );
+        setCommonDimensionsList(
+          commonDimensions.map(d => ({
+            value: d.name,
+            label: d.name,
+            path: d.path.join(' ▶ '),
+          })),
+        );
+      } else {
+        setCommonDimensionsList([]);
+      }
+    };
+    fetchData().catch(console.error);
+  }, [selectedMetrics, djClient]);
+
+  // Get SQL
+  useEffect(() => {
+    const fetchData = async () => {
+      if (selectedMetrics.length && selectedDimensions.length) {
+        const query = await djClient.sqls(selectedMetrics, selectedDimensions);
+        setShowHelp(false);
+        setQuery(query);
+      } else {
+        resetView();
+        setShowHelp(true);
+      }
+    };
+    fetchData().catch(console.error);
+  }, [selectedMetrics, selectedDimensions, djClient]);
+
+  // Set number of rows to display
+  useEffect(() => {
+    if (data) {
+      setDisplayedRows(
+        data[0].rows.slice(0, showNumRows).map((rowData, index) => (
+          <tr key={`data-row:${index}`}>
+            {rowData.map(rowValue => (
+              <td key={rowValue}>{rowValue}</td>
+            ))}
+          </tr>
+        )),
+      );
+    }
+  }, [showNumRows, data]);
+  const formatOptionLabel = ({ value, label, path }) => (
+    <div className={`badge dimension_option`}>
+      <div>{label}</div>
+      <span className={`badge dimension_option_subheading`}>{path}</span>
+    </div>
+  );
+
+  // @ts-ignore
+  return (
+    <>
+      <div className="mid">
+        <NamespaceHeader namespace="" />
+        <div className="card">
+          <div className="card-header">
+            <h4>Metrics</h4>
+            <Select
+              name="metrics"
+              options={metrics}
+              noOptionsMessage={() => 'No metrics found.'}
+              placeholder={`${metrics.length} Available Metrics`}
+              isMulti
+              isClearable
+              closeMenuOnSelect={false}
+              onChange={handleMetricSelect}
+              onMenuClose={handleMetricSelectorClose}
+            />
+            <h4>Group By</h4>
+            <Select
+              name="dimensions"
+              formatOptionLabel={formatOptionLabel}
+              options={commonDimensionsList}
+              noOptionsMessage={() =>
+                'No shared dimensions found. Try selecting different metrics.'
+              }
+              placeholder={`${commonDimensionsList.length} Shared Dimensions`}
+              isMulti
+              isClearable
+              closeMenuOnSelect={false}
+              onChange={handleDimensionSelect}
+              onMenuClose={handleDimensionSelectorClose}
+            />
+            <h4>Filter By</h4>
+            <Select
+              name="filters"
+              className="filters"
+              formatOptionLabel={formatOptionLabel}
+              options={commonDimensionsList}
+              noOptionsMessage={() =>
+                'No shared dimensions found. Try selecting different metrics.'
+              }
+              placeholder={`${commonDimensionsList.length} Shared Dimensions`}
+              isClearable
+              closeMenuOnSelect={true}
+              // onChange={handleDimensionSelect}
+              // onMenuClose={handleDimensionSelectorClose}
+            />
+            <Select
+              name="filters"
+              className="filters-operator"
+              // formatOptionLabel={formatOptionLabel}
+              options={[{value: '=', label: '='}, {value: '>', label: '>'}, {value: '>', label: '>'}]}
+              placeholder={`Operator`}
+              closeMenuOnSelect={true}
+              // onChange={handleDimensionSelect}
+              // onMenuClose={handleDimensionSelectorClose}
+            />
+            <input value={`Value`}></input>
+          </div>
+          <div className="card-header">
+            {showHelp ? (
+              <div className="card-light-shadow">
+                <h6>Using the SQL Builder</h6>
+                <p>
+                  The sql builder allows you to group multiple metrics along
+                  with their shared dimensions. Using your selections,
+                  DataJunction will generate the corresponding SQL.
+                </p>
+                <ol>
+                  <li>
+                    <b>Select Metrics:</b> Start by selecting one or more
+                    metrics from the metrics dropdown.
+                  </li>
+                  <li>
+                    <b>Select Dimensions:</b> Next, select the dimension
+                    attributes you would like to include. As you select
+                    additional metrics, the list of available dimensions will be
+                    filtered to those shared by the selected metrics. If the
+                    dimensions list is empty, no shared dimensions were
+                    discovered.
+                  </li>
+                  <li>
+                    <b>View the generated SQL Query:</b> As you make your
+                    selections, the SQL required to retrieve the set of metrics
+                    and dimensions will be generated below.
+                  </li>
+                  <li>
+                    <b>Run the Query:</b> If query running is enabled by your
+                    server, you can also run the generated SQL query to view a
+                    sample of 100 records.
+                  </li>
+                </ol>
+              </div>
+            ) : (
+              <></>
+            )}
+            {query ? (
+              <>
+                {loadingData ? (
+                  <span className="button-3 executing-button">
+                    {'Running Query'}
+                  </span>
+                ) : (
+                  <span className="button-3 execute-button" onClick={getData}>
+                    {'Run Query'}
+                  </span>
+                )}
+                {data ? (
+                  viewData ? (
+                    <>
+                      <span
+                        className="button-3 neutral-button"
+                        onClick={toggleViewData}
+                      >
+                        {'View Query'}
+                      </span>
+                      <span style={{ display: 'inline-block' }}>
+                        <Select
+                          name="num-rows"
+                          defaultValue={numRowsOptions[1]}
+                          options={numRowsOptions}
+                          onChange={e => setShowNumRows(e.value)}
+                        />
+                      </span>
+                    </>
+                  ) : (
+                    <span
+                      className="button-3 neutral-button"
+                      onClick={toggleViewData}
+                    >
+                      {'View Data'}
+                    </span>
+                  )
+                ) : (
+                  <></>
+                )}
+              </>
+            ) : (
+              <></>
+            )}
+            {submittedQueryInfo ? <QueryInfo {...submittedQueryInfo} /> : <></>}
+            <div>
+              {/*{query.dialect}*/}
+              {query && !viewData ? (
+                <SyntaxHighlighter language="sql" style={foundation}>
+                  {format(query.sql, {
+                    language: 'spark',
+                    tabWidth: 2,
+                    keywordCase: 'upper',
+                    denseOperators: true,
+                    logicalOperatorNewline: 'before',
+                    expressionWidth: 10,
+                  })}
+                </SyntaxHighlighter>
+              ) : (
+                ''
+              )}
+            </div>
+            {data && viewData ? (
+              <div className="table-responsive">
+                <table className="card-inner-table table">
+                  <thead className="fs-7 fw-bold text-gray-400 border-bottom-0">
+                    <tr>
+                      {data[0].columns.map(columnName => (
+                        <th key={columnName.name}>{columnName.name}</th>
+                      ))}
+                    </tr>
+                  </thead>
+                  <tbody>{displayedRows}</tbody>
+                </table>
+              </div>
+            ) : (
+              <></>
+            )}
+          </div>
+        </div>
+      </div>
+    </>
+  );
+}
+
+SQLBuilderPage.defaultProps = {
+  djClient: DataJunctionAPI,
+};

package/src/app/providers/djclient.jsx

@@ -0,0 +1,5 @@
+import { DataJunctionAPI } from '../services/DJService';
+import * as React from 'react';
+
+const DJClientContext = React.createContext({ DataJunctionAPI });
+export default DJClientContext;

package/src/app/services/DJService.js

@@ -1,10 +1,17 @@
 import { MarkerType } from 'reactflow';
 
-const DJ_URL = 'http://localhost:8000'; //process.env.REACT_APP_DJ_URL;
+const DJ_URL = process.env.REACT_APP_DJ_URL
+  ? process.env.REACT_APP_DJ_URL
+  : 'http://localhost:8000';
 
 export const DataJunctionAPI = {
   node: async function (name) {
     const data = await (await fetch(DJ_URL + '/nodes/' + name + '/')).json();
+    data.primary_key = data.columns
+      .filter(col =>
+        col.attributes.some(attr => attr.attribute_type.name === 'primary_key'),
+      )
+      .map(col => col.name);
     return data;
   },
 
@@ -22,11 +29,64 @@ export const DataJunctionAPI = {
     return data;
   },
 
+  node_dag: async function (name) {
+    const data = await (
+      await fetch(DJ_URL + '/nodes/' + name + '/dag/')
+    ).json();
+    return data;
+  },
+
   metric: async function (name) {
     const data = await (await fetch(DJ_URL + '/metrics/' + name + '/')).json();
     return data;
   },
 
+  clientCode: async function (name) {
+    const data = await (
+      await fetch(DJ_URL + '/client/python/new_node/' + name)
+    ).json();
+    return data;
+  },
+
+  cube: async function (name) {
+    const data = await (await fetch(DJ_URL + '/cubes/' + name + '/')).json();
+    return data;
+  },
+
+  metrics: async function (name) {
+    const data = await (await fetch(DJ_URL + '/metrics/')).json();
+    return data;
+  },
+
+  commonDimensions: async function (metrics) {
+    const metricsQuery = '?' + metrics.map(m => `metric=${m}`).join('&');
+    const data = await (
+      await fetch(DJ_URL + '/metrics/common/dimensions/' + metricsQuery)
+    ).json();
+    return data;
+  },
+
+  history: async function (type, name, offset, limit) {
+    const data = await (
+      await fetch(
+        DJ_URL +
+          '/history/' +
+          type +
+          '/' +
+          name +
+          `/?offset=${offset ? offset : 0}&limit=${limit ? limit : 100}`,
+      )
+    ).json();
+    return data;
+  },
+
+  revisions: async function (name) {
+    const data = await (
+      await fetch(DJ_URL + '/nodes/' + name + '/revisions/')
+    ).json();
+    return data;
+  },
+
   namespace: async function (nmspce) {
     const data = await (
       await fetch(DJ_URL + '/namespaces/' + nmspce + '/')
@@ -39,8 +99,72 @@ export const DataJunctionAPI = {
     return data;
   },
 
+  sql: async function (metric_name, selection) {
+    const data = await (
+      await fetch(
+        DJ_URL + '/sql/' + metric_name + '?' + new URLSearchParams(selection),
+      )
+    ).json();
+    return data;
+  },
+
+  materializations: async function (node) {
+    const data = await (
+      await fetch(DJ_URL + `/nodes/${node}/materializations/`)
+    ).json();
+
+    return await Promise.all(
+      data.map(async materialization => {
+        materialization.clientCode = await (
+          await fetch(
+            DJ_URL +
+              `/client/python/add_materialization/${node}/${materialization.name}`,
+          )
+        ).json();
+        return materialization;
+      }),
+    );
+  },
+
+  columns: async function (node) {
+    return await Promise.all(
+      node.columns.map(async col => {
+        col.clientCode = await (
+          await fetch(
+            DJ_URL +
+              `/client/python/link_dimension/${node.name}/${col.name}/${col.dimension?.name}`,
+          )
+        ).json();
+        return col;
+      }),
+    );
+  },
+
+  sqls: async function (metricSelection, dimensionSelection) {
+    const params = new URLSearchParams();
+    metricSelection.map(metric => params.append('metrics', metric));
+    dimensionSelection.map(dimension => params.append('dimensions', dimension));
+    const data = await (await fetch(DJ_URL + '/sql/?' + params)).json();
+    return data;
+  },
+
+  data: async function (metricSelection, dimensionSelection) {
+    const params = new URLSearchParams();
+    metricSelection.map(metric => params.append('metrics', metric));
+    dimensionSelection.map(dimension => params.append('dimensions', dimension));
+    const data = await (
+      await fetch(DJ_URL + '/data/?' + params + '&limit=100&async_=true')
+    ).json();
+    return data;
+  },
+
   lineage: async function (node) {},
 
+  compiledSql: async function (node) {
+    const data = await (await fetch(DJ_URL + `/sql/${node}/`)).json();
+    return data;
+  },
+
   dag: async function (namespace = 'default') {
     const edges = [];
     const data = await (await fetch(DJ_URL + '/nodes/')).json();

package/src/styles/dag.css

@@ -46,7 +46,9 @@
   display: none;
 }
 
-.dj-node__header {
+.dj-node__header,
+.partition__header,
+.table__header {
   font-weight: 400;
   text-transform: uppercase;
   font-family: 'jetbrains-mono', monospace;
@@ -61,28 +63,84 @@
   border-color: #c4cbd1;
 }
 
-.dj-node__body {
+.dj-node__body,
+.partition__body,
+.table__body {
   background-color: rgb(254, 254, 254);
   padding: 0.5em 0.7em 0.9em 0.7em;
   font-family: 'nt-dapper', -apple-system, BlinkMacSystemFont, Segoe UI, Roboto,
     Oxygen, Ubuntu, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif;
   font-size: 1.7em;
+  border-radius: 0 0 8px 8px;
 }
 
-.dj-node__full {
+.dj-node_highlight {
+  border: 10px solid #cccccc !important;
+}
+
+.react-flow__node {
+  z-index: -1 !important;
+}
+
+.react-flow__node-DJNode,
+.dj-node__full,
+.partition__full,
+.table__full {
   display: flex;
   flex-direction: column;
-  height: 100%;
   border-width: 1px;
   border-style: solid;
   border-image: initial;
   border-color: #b0b9c2;
   border-radius: 8px;
-  overflow: hidden;
   box-shadow: rgba(0, 0, 0, 0.1) 0px 4px 6px -1px,
     rgba(0, 0, 0, 0.06) 0px 2px 4px -1px;
 }
 
+.partition__header,
+.partition__body,
+.partition__full,
+.table__body,
+.table__header {
+  font-size: 1em;
+  font-family: 'nt-dapper', -apple-system, BlinkMacSystemFont, Segoe UI, Roboto,
+    Oxygen, Ubuntu, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif;
+}
+
+.partition__full,
+.table__full {
+  margin-bottom: 1rem;
+  border-color: #e0e0e0;
+}
+
+.partition__header,
+.table__header {
+  background-color: #ccefff50;
+  color: #6daaa7;
+  text-transform: none;
+  border-bottom-style: none;
+}
+
+.partition_value {
+  background-color: #ccefff50;
+  color: #6daaa7;
+  border: 1px solid #6daaa750;
+}
+
+.table__header {
+  background-color: rgba(227, 255, 204, 0.31);
+  color: #0aa400;
+}
+
+.table__body {
+  padding: 16px;
+}
+
+.table__full {
+  /*border-radius: 0.2em;*/
+  /*border: none;*/
+}
+
 .dj-node__metadata {
   padding: 1em;
   font-family: 'nt-dapper', -apple-system, BlinkMacSystemFont, Segoe UI, Roboto,
@@ -116,3 +174,51 @@
   font-family: 'jetbrains-mono', monospace;
   text-transform: lowercase;
 }
+
+.react-flow__handle-left {
+  transform: translate(-180%, -50%);
+}
+
+.custom-node-subheader {
+  --bs-badge-padding-x: 0.4em;
+  --bs-badge-padding-y: 0.3em;
+  --bs-badge-font-size: 18px;
+  --bs-badge-font-weight: 400;
+  --bs-badge-color: #fff;
+  --bs-badge-border-radius: 0.375rem;
+  display: flex;
+  flex-direction: column;
+  flex-grow: 1;
+  padding: var(--bs-badge-padding-y) var(--bs-badge-padding-x);
+  margin: 0.55rem;
+  font-size: var(--bs-badge-font-size);
+  font-weight: var(--bs-badge-font-weight);
+  line-height: 1;
+  color: var(--bs-badge-color);
+  text-align: center;
+  white-space: nowrap;
+  vertical-align: baseline;
+  border-radius: var(--bs-badge-border-radius);
+  position: relative;
+  justify-content: flex-start;
+}
+.custom-node-port {
+  color: #636776;
+  font-size: 20px;
+  text-align: center;
+}
+
+.white_badge {
+  background-color: #ffffff !important;
+  display: inline-block;
+  margin: 0.3rem 0.3rem 0.3rem 0.3rem;
+  margin-top: 0.7rem;
+  color: #636776;
+}
+
+.dimension_attributes {
+  max-width: 500px;
+  display: inline-flex;
+  flex-wrap: wrap;
+  justify-content: center;
+}