@things-factory/dataset 5.0.0-alpha.2 → 5.0.0-alpha.20

package/README.md

@@ -0,0 +1,12 @@
+# dataset
+## partition keys
+At the early stage, partition keys are desinged for dynamic partitioning for Athena. But It will be required so many AWS Glue crawlers also by each 'data-sets'.
+Now partition keys are fixed with some cases. Default is daily dataset, which has S3 Key like 'domain={domain}/datasetid={datasetid}/year=2022/month=3/day=25'. It will be set default with empty value in 'data-sets'.
+
+In another case, it is necessary to divide by hours or minutes and store them. This datetime item must be included in the S3 key. It will be required a new Glue Crawler and a Glue Catalog Table. For example if you need to separate directory hourly, you should add "%H" for 'hour' to partition_keys for this. and also should add a new crawler. Look at the below picture for understanding.
+
+![Architecture for partitioning](./assets/data-samples.jpg)
+
+These tasks are related for S3 request limitations. 
+> __3,500 PUT/COPY/POST/DELETE or 5,500 GET/HEAD requests per second per prefix in a bucket__
+- https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html

package/client/bootstrap.js

@@ -1 +1,16 @@
-export default function bootstrap() {}
+import {
+  OxGristRendererJson5,
+  registerEditor as registerGristEditor,
+  registerRenderer as registerGristRenderer
+} from '@operato/data-grist'
+
+import { OxGristEditorDataItemSpec } from '@operato/dataset/grist-editor'
+import { OxGristEditorPartitionKeys } from '@operato/app/grist-editor/ox-grist-editor-partition-keys.js'
+
+export default function bootstrap() {
+  registerGristEditor('data-item-spec', OxGristEditorDataItemSpec)
+  registerGristRenderer('data-item-spec', OxGristRendererJson5)
+
+  registerGristEditor('partition-keys', OxGristEditorPartitionKeys)
+  registerGristRenderer('partition-keys', OxGristRendererJson5)
+}

package/client/pages/data-entry-form.js

@@ -0,0 +1,84 @@
+import '@operato/dataset/ox-data-entry-form.js'
+
+import { LitElement, css, html } from 'lit'
+import { i18next, localize } from '@operato/i18n'
+
+import { client } from '@operato/graphql'
+import gql from 'graphql-tag'
+
+class DataEntryForm extends localize(i18next)(LitElement) {
+  static get properties() {
+    return {
+      dataSet: Object
+    }
+  }
+
+  static get styles() {
+    return [
+      css`
+        :host {
+          display: flex;
+          flex-direction: column;
+
+          background-color: #fff;
+        }
+
+        ox-data-entry-form {
+          flex: 1;
+          margin: 10px;
+        }
+
+        .button-container {
+          display: flex;
+          margin-left: auto;
+          padding: var(--padding-default);
+        }
+      `
+    ]
+  }
+
+  get entryForm() {
+    return this.renderRoot.querySelector('ox-data-entry-form')
+  }
+
+  render() {
+    return html`
+      <ox-data-entry-form .dataSet=${this.dataSet}></ox-data-entry-form>
+      <div class="button-container">
+        <mwc-button raised @click=${this._updateDataItems.bind(this)}>${i18next.t('button.save')}</mwc-button>
+      </div>
+    `
+  }
+
+  async _updateDataItems() {
+    const data = this.entryForm.buildValue()
+    const dataSample = {
+      dataSet: {
+        id: this.dataSet.id
+      },
+      data
+    }
+
+    const response = await client.mutate({
+      mutation: gql`
+        mutation ($dataSample: NewDataSample!) {
+          createDataSample(dataSample: $dataSample) {
+            id
+            collectedAt
+          }
+        }
+      `,
+      variables: {
+        dataSample
+      }
+    })
+
+    if (!response.errors) {
+      document.dispatchEvent(
+        new CustomEvent('notify', { detail: { message: i18next.t('text.data sample created successfully') } })
+      )
+    }
+  }
+}
+
+window.customElements.define('data-entry-form', DataEntryForm)

package/client/pages/data-item-list.js

@@ -1,8 +1,8 @@
-import gql from 'graphql-tag'
-import { css, html, LitElement } from 'lit'
+import { LitElement, css, html } from 'lit'
+import { i18next, localize } from '@operato/i18n'
 
 import { client } from '@operato/graphql'
-import { i18next, localize } from '@operato/i18n'
+import gql from 'graphql-tag'
 import { isMobileDevice } from '@operato/utils'
 
 class DataItemList extends localize(i18next)(LitElement) {
@@ -33,9 +33,6 @@ class DataItemList extends localize(i18next)(LitElement) {
           padding: var(--padding-default);
         }
 
-        form {
-          position: relative;
-        }
         [danger] {
           --mdc-theme-primary: var(--mdc-danger-button-primary-color);
         }
@@ -46,7 +43,7 @@ class DataItemList extends localize(i18next)(LitElement) {
     ]
   }
 
-  get dataGrist() {
+  get grist() {
     return this.renderRoot.querySelector('ox-grist')
   }
 
@@ -64,8 +61,6 @@ class DataItemList extends localize(i18next)(LitElement) {
     `
   }
 
-  async updated(changedProps) {}
-
   async firstUpdated() {
     this.gristConfig = {
       list: { fields: ['name', 'description', 'active'] },
@@ -149,7 +144,44 @@ class DataItemList extends localize(i18next)(LitElement) {
           name: 'type',
           header: i18next.t('field.type'),
           record: {
-            options: ['number', 'text', 'select', 'boolean'],
+            options: ['', 'number', 'text', 'select', 'boolean', 'file'],
+            editable: true
+          },
+          width: 120
+        },
+        {
+          type: 'parameters',
+          name: 'options',
+          header: i18next.t('field.options'),
+          record: {
+            editable: true,
+            renderer: 'json5',
+            options: async (value, column, record, row, field) => {
+              return {
+                name: record.type,
+                help: '',
+                spec:
+                  record.type === 'select'
+                    ? [
+                        {
+                          type: 'options' /* property-editor type */,
+                          name: 'options',
+                          label: 'options'
+                        }
+                      ]
+                    : [],
+                context: this.grist,
+                objectified: true
+              }
+            }
+          },
+          width: 120
+        },
+        {
+          type: 'string',
+          name: 'unit',
+          header: i18next.t('field.unit'),
+          record: {
             editable: true
           },
           width: 120
@@ -164,11 +196,16 @@ class DataItemList extends localize(i18next)(LitElement) {
           width: 60
         },
         {
-          type: 'json',
+          type: 'data-item-spec',
           name: 'spec',
           header: i18next.t('field.spec'),
           record: {
-            editable: true
+            editable: true,
+            options: {
+              name: '',
+              help: '',
+              objectified: true /* prevent from stringifying */
+            }
           },
           width: 200
         }
@@ -207,7 +244,10 @@ class DataItemList extends localize(i18next)(LitElement) {
               description
               sequence
               active
+              tag
               type
+              options
+              unit
               quota
               spec
             }
@@ -224,7 +264,7 @@ class DataItemList extends localize(i18next)(LitElement) {
   }
 
   async _updateDataItems() {
-    let patches = this.dataGrist._data.records
+    let patches = this.grist._data.records
     if (patches && patches.length) {
       patches = patches.map(patch => {
         var patchField = {}
@@ -250,14 +290,16 @@ class DataItemList extends localize(i18next)(LitElement) {
         }
       })
 
-      if (!response.errors) this.dataGrist.fetch()
+      if (!response.errors) {
+        this.grist.fetch()
+      }
     }
   }
 
   async _deleteDataItems() {
     if (!confirm(i18next.t('text.sure_to_x', { x: i18next.t('text.delete') }))) return
 
-    const ids = this.dataGrist.selected.map(record => record.id)
+    const ids = this.grist.selected.map(record => record.id)
     if (!(ids && ids.length > 0)) return
 
     const response = await client.mutate({
@@ -273,7 +315,7 @@ class DataItemList extends localize(i18next)(LitElement) {
 
     if (response.errors) return
 
-    this.dataGrist.fetch()
+    this.grist.fetch()
     await document.dispatchEvent(
       new CustomEvent('notify', {
         detail: {

package/client/pages/data-ooc-view.js

@@ -0,0 +1,182 @@
+import '@operato/dataset/ox-data-ooc-view.js'
+
+import { LitElement, css, html } from 'lit'
+import { i18next, localize } from '@operato/i18n'
+
+import { ScrollbarStyles } from '@operato/styles'
+import { client } from '@operato/graphql'
+import gql from 'graphql-tag'
+
+class DataOocView extends localize(i18next)(LitElement) {
+  static get styles() {
+    return [
+      ScrollbarStyles,
+      css`
+        :host {
+          display: flex;
+          flex-direction: column;
+
+          background-color: #fff;
+        }
+
+        div[content] {
+          flex: 1;
+
+          display: flex;
+          overflow: auto;
+        }
+
+        ox-data-ooc-view {
+          flex: 1;
+          padding: var(--padding-wide);
+        }
+
+        label[comment] {
+          display: flex;
+          flex-direction: column;
+
+          padding: var(--padding-wide);
+        }
+
+        label[comment] div {
+          display: flex;
+        }
+
+        mwc-icon {
+          color: var(--status-danger-color);
+        }
+
+        textarea {
+          border: var(--input-field-border);
+          border-radius: var(--input-border-radius);
+          padding: var(--input-field-padding);
+          font: var(--input-field-font);
+        }
+
+        .button-container {
+          display: flex;
+          margin-left: auto;
+          padding: var(--padding-default);
+        }
+      `
+    ]
+  }
+
+  static get properties() {
+    return {
+      dataSet: Object,
+      dataOoc: Object
+    }
+  }
+
+  get sampleView() {
+    return this.renderRoot.querySelector('ox-data-ooc-view')
+  }
+
+  render() {
+    const state = this.dataOoc.state
+
+    return html`
+      <div content>
+        <ox-data-ooc-view .dataSet=${this.dataSet} .dataOoc=${this.dataOoc}></ox-data-ooc-view>
+      </div>
+
+      ${state === 'CREATED' || state === 'REVIEWED'
+        ? html`
+            <label comment>
+              <div><mwc-icon>build_circle</mwc-icon> <span>correction activity</span></div>
+              <textarea placeholder="조치 내용을 입력해주세요."></textarea>
+            </label>
+          `
+        : html``}
+
+      <div class="button-container">
+        ${state === 'CREATED'
+          ? html`<mwc-button raised @click=${() => this._processOoc('REVIEWED')}
+              >${i18next.t('button.reviewed')}</mwc-button
+            >`
+          : state === 'REVIEWED'
+          ? html`<mwc-button raised @click=${() => this._processOoc('CORRECTED')}
+              >${i18next.t('button.corrected')}</mwc-button
+            >`
+          : html``}
+      </div>
+    `
+  }
+
+  updated(changes) {
+    if (changes.has('dataOoc')) {
+      this.fetchDataSet()
+    }
+  }
+
+  async fetchDataSet() {
+    const id = this.dataOoc?.dataSet?.id
+
+    if (id) {
+      const response = await client.query({
+        query: gql`
+          query ($id: String!) {
+            dataSet(id: $id) {
+              id
+              name
+              description
+              useCase
+              dataItems {
+                id
+                name
+                description
+                active
+                unit
+                tag
+                type
+                spec
+              }
+            }
+          }
+        `,
+        variables: {
+          id: this.dataOoc.dataSet.id
+        }
+      })
+
+      this.dataSet = response.data.dataSet
+    }
+  }
+
+  async _processOoc(state) {
+    const commentTextArea = this.renderRoot.querySelector('textarea')
+    const comment = commentTextArea && commentTextArea.value
+    if (!comment || !comment.trim()) {
+      commentTextArea.focus()
+      return
+    }
+
+    const patch = {
+      state,
+      correctiveAction: comment
+    }
+
+    const response = await client.mutate({
+      mutation: gql`
+        mutation ($id: String!, $patch: DataOocPatch!) {
+          updateDataOoc(id: $id, patch: $patch) {
+            id
+          }
+        }
+      `,
+      variables: {
+        id: this.dataOoc.id,
+        patch
+      }
+    })
+
+    if (!response.errors) {
+      document.dispatchEvent(
+        new CustomEvent('notify', { detail: { message: i18next.t('text.data ooc updated successfully') } })
+      )
+    }
+  }
+}
+
+window.customElements.define('data-ooc-view', DataOocView)