datajunction-ui 0.0.1-rc.25 → 0.0.1-rc.26

package/src/app/pages/NodePage/__tests__/__snapshots__/NodePage.test.jsx.snap

@@ -104,9 +104,6 @@ exports[`<NodePage /> renders the NodeMaterialization tab with materializations
       <th
         class="text-start"
       >
-        Name
-      </th>
-      <th>
         Schedule
       </th>
       <th>
@@ -131,11 +128,22 @@ exports[`<NodePage /> renders the NodeMaterialization tab with materializations
       <td
         class="text-start node_name"
       >
-        <a
-          href="http://fake.url/job"
+        <span
+          class="badge cron"
         >
-          country_birth_date_contractor_id_379232101
-        </a>
+          0 * * * *
+        </span>
+        <div
+          class="cron-description"
+        >
+          Every hour
+           
+        </div>
+      </td>
+      <td>
+        spark
+        <br />
+        2.4.4
         <button
           aria-label="code-button"
           class="code-button"
@@ -222,79 +230,7 @@ exports[`<NodePage /> renders the NodeMaterialization tab with materializations
           </pre>
         </div>
       </td>
-      <td>
-        <span
-          class="badge cron"
-        >
-          0 * * * *
-        </span>
-        <div
-          class="cron-description"
-        >
-          Every hour
-           
-        </div>
-      </td>
-      <td>
-        spark
-        <br />
-        2.4.4
-      </td>
-      <td>
-        <div
-          class="partition__full"
-        >
-          <div
-            class="partition__header"
-          >
-            country
-          </div>
-          <div
-            class="partition__body"
-          >
-            <span
-              class="badge partition_value"
-            >
-              DE
-            </span>
-            <span
-              class="badge partition_value"
-            >
-              MY
-            </span>
-          </div>
-        </div>
-        <div
-          class="partition__full"
-        >
-          <div
-            class="partition__header"
-          >
-            contractor_id
-          </div>
-          <div
-            class="partition__body"
-          >
-            <div>
-              <span
-                class="badge partition_value"
-              >
-                <span
-                  class="badge partition_value"
-                >
-                  1
-                </span>
-                to
-                <span
-                  class="badge partition_value"
-                >
-                  10
-                </span>
-              </span>
-            </div>
-          </div>
-        </div>
-      </td>
+      <td />
       <td>
         <div
           class="table__full"
@@ -356,35 +292,100 @@ exports[`<NodePage /> renders the NodeMaterialization tab with materializations
         </div>
       </td>
       <td>
-        <div
-          class="partition__full"
+        <a
+          class="partitionLink"
         >
           <div
-            class="partition__header"
+            class="partition__full"
           >
-            birth_date
-          </div>
-          <div
-            class="partition__body"
-          >
-            <div>
+            <div
+              class="partition__header"
+            />
+            <div
+              class="partition__body"
+            >
               <span
                 class="badge partition_value"
               >
-                <span
-                  class="badge partition_value"
-                >
-                  20010101
-                </span>
-                to
-                <span
-                  class="badge partition_value"
-                >
-                  20020101
-                </span>
+                20230101
+              </span>
+              to
+              <span
+                class="badge partition_value"
+              >
+                20230102
               </span>
             </div>
           </div>
+        </a>
+        <button
+          aria-label="AddBackfill"
+          class="edit_button"
+          tabindex="0"
+        >
+          <span
+            class="add_node"
+          >
+            + Add Backfill
+          </span>
+        </button>
+        <div
+          class="fade modal-backdrop in"
+          style="display: none;"
+        />
+        <div
+          aria-label="client-code"
+          class="centerPopover"
+          role="dialog"
+          style="display: none; width: 50%;"
+        >
+          <form
+            action="#"
+          >
+            <h2>
+              Run Backfill
+            </h2>
+            <span
+              data-testid="edit-partition"
+            >
+              <label
+                for="engine"
+                style="padding-bottom: 1rem;"
+              >
+                Engine
+              </label>
+              <select
+                disabled=""
+                id="engine"
+                name="engine"
+              >
+                <option
+                  value="spark"
+                >
+                  spark
+                   
+                  2.4.4
+                </option>
+              </select>
+            </span>
+            <br />
+            <br />
+            <label
+              for="partition"
+              style="padding-bottom: 1rem;"
+            >
+              Partition Range
+            </label>
+            <br />
+            <button
+              aria-hidden="false"
+              aria-label="SaveEditColumn"
+              class="add_node"
+              type="submit"
+            >
+              Save
+            </button>
+          </form>
         </div>
       </td>
       <td>

package/src/app/services/DJService.js

@@ -26,6 +26,14 @@ export const DataJunctionAPI = {
     ).json();
   },
 
+  engines: async function () {
+    return await (
+      await fetch(`${DJ_URL}/engines`, {
+        credentials: 'include',
+      })
+    ).json();
+  },
+
   node: async function (name) {
     const data = await (
       await fetch(`${DJ_URL}/nodes/${name}/`, {
@@ -575,4 +583,83 @@ export const DataJunctionAPI = {
     });
     return { status: response.status, json: await response.json() };
   },
+  setPartition: async function (
+    nodeName,
+    columnName,
+    partitionType,
+    format,
+    granularity,
+  ) {
+    const body = {
+      type_: partitionType,
+    };
+    if (format) {
+      body.format = format;
+    }
+    if (granularity) {
+      body.granularity = granularity;
+    }
+    const response = await fetch(
+      `${DJ_URL}/nodes/${nodeName}/columns/${columnName}/partition`,
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(body),
+        credentials: 'include',
+      },
+    );
+    return { status: response.status, json: await response.json() };
+  },
+  materialize: async function (
+    nodeName,
+    engineName,
+    engineVersion,
+    schedule,
+    config,
+  ) {
+    const response = await fetch(
+      `${DJ_URL}/nodes/${nodeName}/materialization`,
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          engine: {
+            name: engineName,
+            version: engineVersion,
+          },
+          schedule: schedule,
+          config: JSON.parse(config),
+        }),
+        credentials: 'include',
+      },
+    );
+    return { status: response.status, json: await response.json() };
+  },
+  runBackfill: async function (
+    nodeName,
+    materializationName,
+    partitionColumn,
+    from,
+    to,
+  ) {
+    const response = await fetch(
+      `${DJ_URL}/nodes/${nodeName}/materializations/${materializationName}/backfill`,
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          column_name: partitionColumn,
+          range: [from, to],
+        }),
+        credentials: 'include',
+      },
+    );
+    return { status: response.status, json: await response.json() };
+  },
 };

package/src/app/services/__tests__/DJService.test.jsx

@@ -607,6 +607,18 @@ describe('DataJunctionAPI', () => {
     );
   });
 
+  it('calls addNamespace correctly', async () => {
+    fetch.mockResponseOnce(JSON.stringify({}));
+    await DataJunctionAPI.addNamespace('test');
+    expect(fetch).toHaveBeenCalledWith(`${DJ_URL}/namespaces/test`, {
+      credentials: 'include',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      method: 'POST',
+    });
+  });
+
   it('calls listTags correctly', async () => {
     fetch.mockResponseOnce(JSON.stringify(mocks.tags));
     const res = await DataJunctionAPI.listTags();
@@ -689,4 +701,104 @@ describe('DataJunctionAPI', () => {
       method: 'POST',
     });
   });
+
+  it('calls editTag correctly', async () => {
+    fetch.mockResponseOnce(JSON.stringify({}));
+    await DataJunctionAPI.editTag(
+      'report.financials',
+      'Financial reports',
+      'Financial Reports',
+    );
+    expect(fetch).toHaveBeenCalledWith(`${DJ_URL}/tags/report.financials`, {
+      credentials: 'include',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        description: 'Financial reports',
+        display_name: 'Financial Reports',
+      }),
+      method: 'PATCH',
+    });
+  });
+
+  it('calls setPartition correctly', async () => {
+    fetch.mockResponseOnce(JSON.stringify({}));
+    await DataJunctionAPI.setPartition(
+      'default.hard_hat',
+      'hire_date',
+      'temporal',
+      'yyyyMMdd',
+      'day',
+    );
+    expect(fetch).toHaveBeenCalledWith(
+      `${DJ_URL}/nodes/default.hard_hat/columns/hire_date/partition`,
+      {
+        credentials: 'include',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          type_: 'temporal',
+          format: 'yyyyMMdd',
+          granularity: 'day',
+        }),
+        method: 'POST',
+      },
+    );
+  });
+
+  it('calls materialize correctly', async () => {
+    fetch.mockResponseOnce(JSON.stringify({}));
+    await DataJunctionAPI.materialize(
+      'default.hard_hat',
+      'spark',
+      '3.3',
+      '@daily',
+      '{}',
+    );
+    expect(fetch).toHaveBeenCalledWith(
+      `${DJ_URL}/nodes/default.hard_hat/materialization`,
+      {
+        credentials: 'include',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          engine: {
+            name: 'spark',
+            version: '3.3',
+          },
+          schedule: '@daily',
+          config: {},
+        }),
+        method: 'POST',
+      },
+    );
+  });
+
+  it('calls runBackfill correctly', async () => {
+    fetch.mockResponseOnce(JSON.stringify({}));
+    await DataJunctionAPI.runBackfill(
+      'default.hard_hat',
+      'spark',
+      'hire_date',
+      '20230101',
+      '20230202',
+    );
+    expect(fetch).toHaveBeenCalledWith(
+      `${DJ_URL}/nodes/default.hard_hat/materializations/spark/backfill`,
+      {
+        credentials: 'include',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          column_name: 'hire_date',
+          range: ['20230101', '20230202'],
+        }),
+        method: 'POST',
+      },
+    );
+  });
 });

package/src/mocks/mockNodes.jsx

@@ -30,6 +30,7 @@ export const mocks = {
         display_name: 'Default DOT num repair orders',
         attributes: [],
         dimension: null,
+        partition: null,
       },
     ],
     primary_key: ['repair_order_id', 'country'],
@@ -247,6 +248,7 @@ export const mocks = {
       display_name: 'Default DOT avg repair price',
       attributes: [],
       dimension: null,
+      partition: null,
     },
   ],
   metricNodeHistory: [
@@ -338,32 +340,19 @@ export const mocks = {
   ],
   nodeMaterializations: [
     {
+      backfills: [
+        {
+          spec: {
+            column_name: 'default_DOT_hard_hat_DOT_hire_date',
+            values: null,
+            range: ['20230101', '20230102'],
+          },
+          urls: [],
+        },
+      ],
       name: 'country_birth_date_contractor_id_379232101',
       engine: { name: 'spark', version: '2.4.4', uri: null, dialect: 'spark' },
       config: {
-        partitions: [
-          {
-            name: 'country',
-            values: ['DE', 'MY'],
-            range: null,
-            expression: null,
-            type_: 'categorical',
-          },
-          {
-            name: 'birth_date',
-            values: null,
-            range: [20010101, 20020101],
-            expression: null,
-            type_: 'temporal',
-          },
-          {
-            name: 'contractor_id',
-            values: null,
-            range: [1, 10],
-            expression: null,
-            type_: 'categorical',
-          },
-        ],
         spark: {},
         query:
           "SELECT  default_DOT_hard_hats.address,\n\tdefault_DOT_hard_hats.birth_date,\n\tdefault_DOT_hard_hats.city,\n\tdefault_DOT_hard_hats.contractor_id,\n\tdefault_DOT_hard_hats.country,\n\tdefault_DOT_hard_hats.first_name,\n\tdefault_DOT_hard_hats.hard_hat_id,\n\tdefault_DOT_hard_hats.hire_date,\n\tdefault_DOT_hard_hats.last_name,\n\tdefault_DOT_hard_hats.manager,\n\tdefault_DOT_hard_hats.postal_code,\n\tdefault_DOT_hard_hats.state,\n\tdefault_DOT_hard_hats.title \n FROM roads.hard_hats AS default_DOT_hard_hats \n WHERE  default_DOT_hard_hats.country IN ('DE', 'MY') AND default_DOT_hard_hats.contractor_id BETWEEN 1 AND 10\n\n",
@@ -1235,66 +1224,66 @@ export const mocks = {
           },
         },
         {
-          "name": "hire_date",
-          "type": "date",
-          "attributes": [],
-          "dimension": {
-            "name": "default.date_dim"
+          name: 'hire_date',
+          type: 'date',
+          attributes: [],
+          dimension: {
+            name: 'default.date_dim',
           },
-          "display_name": "Hire Date"
+          display_name: 'Hire Date',
         },
         {
-          "name": "address",
-          "type": "string",
-          "attributes": [],
-          "dimension": null,
-          "display_name": "Address"
+          name: 'address',
+          type: 'string',
+          attributes: [],
+          dimension: null,
+          display_name: 'Address',
         },
         {
-          "name": "city",
-          "type": "string",
-          "attributes": [],
-          "dimension": null,
-          "display_name": "City"
+          name: 'city',
+          type: 'string',
+          attributes: [],
+          dimension: null,
+          display_name: 'City',
         },
         {
-          "name": "state",
-          "type": "string",
-          "attributes": [],
-          "dimension": {
-            "name": "default.us_state"
+          name: 'state',
+          type: 'string',
+          attributes: [],
+          dimension: {
+            name: 'default.us_state',
           },
-          "display_name": "State"
+          display_name: 'State',
         },
         {
-          "name": "postal_code",
-          "type": "string",
-          "attributes": [],
-          "dimension": null,
-          "display_name": "Postal Code"
+          name: 'postal_code',
+          type: 'string',
+          attributes: [],
+          dimension: null,
+          display_name: 'Postal Code',
         },
         {
-          "name": "country",
-          "type": "string",
-          "attributes": [],
-          "dimension": null,
-          "display_name": "Country"
+          name: 'country',
+          type: 'string',
+          attributes: [],
+          dimension: null,
+          display_name: 'Country',
         },
         {
-          "name": "manager",
-          "type": "int",
-          "attributes": [],
-          "dimension": null,
-          "display_name": "Manager"
+          name: 'manager',
+          type: 'int',
+          attributes: [],
+          dimension: null,
+          display_name: 'Manager',
         },
         {
-          "name": "contractor_id",
-          "type": "int",
-          "attributes": [],
-          "dimension": null,
-          "display_name": "Contractor Id"
-        }
-        ],
+          name: 'contractor_id',
+          type: 'int',
+          attributes: [],
+          dimension: null,
+          display_name: 'Contractor Id',
+        },
+      ],
       created_at: '2023-08-21T16:48:55.594537+00:00',
       tags: [],
     },

package/src/styles/dag.css

@@ -97,6 +97,10 @@
     rgba(0, 0, 0, 0.06) 0px 2px 4px -1px;
 }
 
+.partition__full:hover {
+  border: solid #195039 0.025em;
+}
+
 .partition__header,
 .partition__body,
 .partition__full,