@things-factory/dataset 5.0.0-alpha.19 → 5.0.0-alpha.21

package/README.md

@@ -0,0 +1,13 @@
+# dataset
+## partition keys
+At the early stage, partition keys are desinged for dynamic partitioning for Athena. But It will be required so many AWS Glue crawlers also by each 'data-sets'.
+Now partition keys are fixed with some cases. Default is daily dataset, which has S3 Key like 'domain={domain}/datasetid={datasetid}/year=2022/month=3/day=25'. It will be set default with empty value in 'data-sets'.
+
+In another case, it is necessary to divide by hours or minutes and store them. This datetime item must be included in the S3 key. It will be required a new Glue Crawler and a Glue Catalog Table. For example if you need to separate directory hourly, you should add "%H" for 'hour' to partition_keys for this. and also should add a new crawler. And it would be needed a column to know it is daily or hourly or minutely for reporting client side. It's not developed yet.
+Look at the below picture for understanding.
+
+![Architecture for partitioning](./assets/data-samples.jpg)
+
+These tasks are related for S3 request limitations. 
+> __3,500 PUT/COPY/POST/DELETE or 5,500 GET/HEAD requests per second per prefix in a bucket__
+- https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html

package/client/pages/data-item-list.js

@@ -290,7 +290,9 @@ class DataItemList extends localize(i18next)(LitElement) {
         }
       })
 
-      if (!response.errors) this.grist.fetch()
+      if (!response.errors) {
+        this.grist.fetch()
+      }
     }
   }
 

package/client/pages/data-ooc-view.js

@@ -1,4 +1,4 @@
-import '@operato/dataset/ox-data-sample-view.js'
+import '@operato/dataset/ox-data-ooc-view.js'
 
 import { LitElement, css, html } from 'lit'
 import { i18next, localize } from '@operato/i18n'
@@ -8,13 +8,6 @@ import { client } from '@operato/graphql'
 import gql from 'graphql-tag'
 
 class DataOocView extends localize(i18next)(LitElement) {
-  static get properties() {
-    return {
-      dataSet: Object,
-      dataSample: Object
-    }
-  }
-
   static get styles() {
     return [
       ScrollbarStyles,
@@ -30,11 +23,34 @@ class DataOocView extends localize(i18next)(LitElement) {
           flex: 1;
 
           display: flex;
-          overflow: scroll;
+          overflow: auto;
         }
 
-        ox-data-sample-view {
+        ox-data-ooc-view {
           flex: 1;
+          padding: var(--padding-wide);
+        }
+
+        label[comment] {
+          display: flex;
+          flex-direction: column;
+
+          padding: var(--padding-wide);
+        }
+
+        label[comment] div {
+          display: flex;
+        }
+
+        mwc-icon {
+          color: var(--status-danger-color);
+        }
+
+        textarea {
+          border: var(--input-field-border);
+          border-radius: var(--input-border-radius);
+          padding: var(--input-field-padding);
+          font: var(--input-field-font);
         }
 
         .button-container {
@@ -46,30 +62,56 @@ class DataOocView extends localize(i18next)(LitElement) {
     ]
   }
 
+  static get properties() {
+    return {
+      dataSet: Object,
+      dataOoc: Object
+    }
+  }
+
   get sampleView() {
-    return this.renderRoot.querySelector('ox-data-sample-view')
+    return this.renderRoot.querySelector('ox-data-ooc-view')
   }
 
   render() {
+    const state = this.dataOoc.state
+
     return html`
       <div content>
-        <ox-data-sample-view .dataSet=${this.dataSet} .dataSample=${this.dataSample}></ox-data-sample-view>
+        <ox-data-ooc-view .dataSet=${this.dataSet} .dataOoc=${this.dataOoc}></ox-data-ooc-view>
       </div>
 
+      ${state === 'CREATED' || state === 'REVIEWED'
+        ? html`
+            <label comment>
+              <div><mwc-icon>build_circle</mwc-icon> <span>correction activity</span></div>
+              <textarea placeholder="조치 내용을 입력해주세요."></textarea>
+            </label>
+          `
+        : html``}
+
       <div class="button-container">
-        <mwc-button raised @click=${this._updateDataSample.bind(this)}>${i18next.t('button.save')}</mwc-button>
+        ${state === 'CREATED'
+          ? html`<mwc-button raised @click=${() => this._processOoc('REVIEWED')}
+              >${i18next.t('button.reviewed')}</mwc-button
+            >`
+          : state === 'REVIEWED'
+          ? html`<mwc-button raised @click=${() => this._processOoc('CORRECTED')}
+              >${i18next.t('button.corrected')}</mwc-button
+            >`
+          : html``}
       </div>
     `
   }
 
   updated(changes) {
-    if (changes.has('dataSample')) {
+    if (changes.has('dataOoc')) {
       this.fetchDataSet()
     }
   }
 
   async fetchDataSet() {
-    const id = this.dataSample?.dataSet?.id
+    const id = this.dataOoc?.dataSet?.id
 
     if (id) {
       const response = await client.query({
@@ -94,7 +136,7 @@ class DataOocView extends localize(i18next)(LitElement) {
           }
         `,
         variables: {
-          id: this.dataSample.dataSet.id
+          id: this.dataOoc.dataSet.id
         }
       })
 
@@ -102,32 +144,38 @@ class DataOocView extends localize(i18next)(LitElement) {
     }
   }
 
-  async _updateDataSample() {
-    // const data = this.sampleView.buildValue()
-    // const dataSample = {
-    //   dataSet: {
-    //     id: this.dataSet.id
-    //   },
-    //   data
-    // }
-    // const response = await client.mutate({
-    //   mutation: gql`
-    //     mutation ($dataSample: NewDataSample!) {
-    //       createDataSample(dataSample: $dataSample) {
-    //         id
-    //         collectedAt
-    //       }
-    //     }
-    //   `,
-    //   variables: {
-    //     dataSample
-    //   }
-    // })
-    // if (!response.errors) {
-    //   document.dispatchEvent(
-    //     new CustomEvent('notify', { detail: { message: i18next.t('text.data sample created successfully') } })
-    //   )
-    // }
+  async _processOoc(state) {
+    const commentTextArea = this.renderRoot.querySelector('textarea')
+    const comment = commentTextArea && commentTextArea.value
+    if (!comment || !comment.trim()) {
+      commentTextArea.focus()
+      return
+    }
+
+    const patch = {
+      state,
+      correctiveAction: comment
+    }
+
+    const response = await client.mutate({
+      mutation: gql`
+        mutation ($id: String!, $patch: DataOocPatch!) {
+          updateDataOoc(id: $id, patch: $patch) {
+            id
+          }
+        }
+      `,
+      variables: {
+        id: this.dataOoc.id,
+        patch
+      }
+    })
+
+    if (!response.errors) {
+      document.dispatchEvent(
+        new CustomEvent('notify', { detail: { message: i18next.t('text.data ooc updated successfully') } })
+      )
+    }
   }
 }
 

package/client/pages/data-ooc.js

@@ -56,11 +56,11 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
       title: i18next.t('title.data-ooc list'),
       help: 'integration/ui/data-ooc',
       actions: [
-        {
-          title: i18next.t('button.save'),
-          action: this._updateDataOoc.bind(this),
-          ...CommonButtonStyles.save
-        }
+        // {
+        //   title: i18next.t('button.save'),
+        //   action: this._updateDataOoc.bind(this),
+        //   ...CommonButtonStyles.save
+        // }
       ],
       exportable: {
         name: i18next.t('title.data-ooc list'),
@@ -124,14 +124,17 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
           icon: 'assignment_turned_in',
           handlers: {
             click: (columns, data, column, record, rowIndex) => {
-              openPopup(
-                html` <data-ooc-view .dataSample=${record} style="background-color: white;"></data-ooc-view> `,
+              const popup = openPopup(
+                html` <data-ooc-view .dataOoc=${record} style="background-color: white;"></data-ooc-view> `,
                 {
                   backdrop: true,
                   size: 'large',
                   title: i18next.t('title.data-ooc view')
                 }
               )
+              popup.onclosed = () => {
+                this.grist.fetch()
+              }
             }
           }
         },
@@ -160,6 +163,12 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
           width: 150,
           imex: true
         },
+        {
+          type: 'string',
+          name: 'history',
+          hidden: true,
+          imex: true
+        },
         {
           type: 'checkbox',
           name: 'ooc',
@@ -179,14 +188,16 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
           width: 30
         },
         {
-          type: 'string',
+          type: 'select',
           name: 'state',
           label: true,
           header: i18next.t('field.state'),
           record: {
-            editable: false
+            editable: false,
+            options: ['', 'CREATED', 'REVIEWED', 'CORRECTED']
           },
-          width: 150,
+          width: 100,
+          filter: true,
           imex: true
         },
         {
@@ -211,17 +222,6 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
           width: 120,
           imex: true
         },
-        {
-          type: 'resource-object',
-          name: 'dataSample',
-          header: i18next.t('field.data-sample'),
-          record: {
-            editable: false
-          },
-          sortable: true,
-          width: 120,
-          imex: true
-        },
         {
           type: 'json5',
           name: 'partitionKeys',
@@ -338,6 +338,7 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
               spec
               ooc
               oos
+              history
               correctiveAction
               correctedAt
               corrector {
@@ -368,62 +369,62 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
     }
   }
 
-  async _updateDataOoc() {
-    let patches = this.grist.dirtyRecords
-    if (patches && patches.length) {
-      patches = patches.map(patch => {
-        let patchField = patch.id ? { id: patch.id } : {}
-        const dirtyFields = patch.__dirtyfields__
-        for (let key in dirtyFields) {
-          patchField[key] = dirtyFields[key].after
-        }
-        this._setDefaultFieldsValue(patchField)
-        patchField.cuFlag = patch.__dirty__
-
-        return patchField
-      })
-
-      const response = await client.mutate({
-        mutation: gql`
-          mutation ($patches: [DataOocPatch!]!) {
-            updateMultipleDataOoc(patches: $patches) {
-              name
-            }
-          }
-        `,
-        variables: {
-          patches
-        }
-      })
-
-      if (!response.errors) this.grist.fetch()
-    }
-  }
-
-  async _deleteDataOoc() {
-    if (confirm(i18next.t('text.sure_to_x', { x: i18next.t('text.delete') }))) {
-      const ids = this.grist.selected.map(record => record.id)
-      if (ids && ids.length > 0) {
-        const response = await client.mutate({
-          mutation: gql`
-            mutation ($ids: [String!]!) {
-              deleteDataOocs(ids: $ids)
-            }
-          `,
-          variables: {
-            ids
-          }
-        })
-
-        if (!response.errors) {
-          this.grist.fetch()
-          notify({
-            message: i18next.t('text.info_x_successfully', { x: i18next.t('text.delete') })
-          })
-        }
-      }
-    }
-  }
+  // async _updateDataOoc() {
+  //   let patches = this.grist.dirtyRecords
+  //   if (patches && patches.length) {
+  //     patches = patches.map(patch => {
+  //       let patchField = patch.id ? { id: patch.id } : {}
+  //       const dirtyFields = patch.__dirtyfields__
+  //       for (let key in dirtyFields) {
+  //         patchField[key] = dirtyFields[key].after
+  //       }
+  //       this._setDefaultFieldsValue(patchField)
+  //       patchField.cuFlag = patch.__dirty__
+
+  //       return patchField
+  //     })
+
+  //     const response = await client.mutate({
+  //       mutation: gql`
+  //         mutation ($patches: [DataOocPatch!]!) {
+  //           updateMultipleDataOoc(patches: $patches) {
+  //             name
+  //           }
+  //         }
+  //       `,
+  //       variables: {
+  //         patches
+  //       }
+  //     })
+
+  //     if (!response.errors) this.grist.fetch()
+  //   }
+  // }
+
+  // async _deleteDataOoc() {
+  //   if (confirm(i18next.t('text.sure_to_x', { x: i18next.t('text.delete') }))) {
+  //     const ids = this.grist.selected.map(record => record.id)
+  //     if (ids && ids.length > 0) {
+  //       const response = await client.mutate({
+  //         mutation: gql`
+  //           mutation ($ids: [String!]!) {
+  //             deleteDataOocs(ids: $ids)
+  //           }
+  //         `,
+  //         variables: {
+  //           ids
+  //         }
+  //       })
+
+  //       if (!response.errors) {
+  //         this.grist.fetch()
+  //         notify({
+  //           message: i18next.t('text.info_x_successfully', { x: i18next.t('text.delete') })
+  //         })
+  //       }
+  //     }
+  //   }
+  // }
 
   _exportableData() {
     let records = []

package/client/pages/data-sample-view.js

@@ -30,11 +30,12 @@ class DataSampleView extends localize(i18next)(LitElement) {
           flex: 1;
 
           display: flex;
-          overflow: scroll;
+          overflow: auto;
         }
 
         ox-data-sample-view {
           flex: 1;
+          padding: var(--padding-wide);
         }
       `
     ]

package/client/pages/data-sensor.js

@@ -153,6 +153,7 @@ export class DataSensor extends connect(store)(localize(i18next)(PageView)) {
             editable: true
           },
           sortable: true,
+          filter: true,
           width: 60
         },
         {
@@ -176,6 +177,7 @@ export class DataSensor extends connect(store)(localize(i18next)(PageView)) {
             editable: true
           },
           sortable: true,
+          filter: 'search',
           width: 150
         },
         {