@things-factory/dataset 5.0.0-alpha.18 → 5.0.0-alpha.20

package/README.md

@@ -0,0 +1,12 @@
+# dataset
+## partition keys
+At the early stage, partition keys are desinged for dynamic partitioning for Athena. But It will be required so many AWS Glue crawlers also by each 'data-sets'.
+Now partition keys are fixed with some cases. Default is daily dataset, which has S3 Key like 'domain={domain}/datasetid={datasetid}/year=2022/month=3/day=25'. It will be set default with empty value in 'data-sets'.
+
+In another case, it is necessary to divide by hours or minutes and store them. This datetime item must be included in the S3 key. It will be required a new Glue Crawler and a Glue Catalog Table. For example if you need to separate directory hourly, you should add "%H" for 'hour' to partition_keys for this. and also should add a new crawler. Look at the below picture for understanding.
+
+![Architecture for partitioning](./assets/data-samples.jpg)
+
+These tasks are related for S3 request limitations. 
+> __3,500 PUT/COPY/POST/DELETE or 5,500 GET/HEAD requests per second per prefix in a bucket__
+- https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html

package/client/pages/data-item-list.js

@@ -290,7 +290,9 @@ class DataItemList extends localize(i18next)(LitElement) {
         }
       })
 
-      if (!response.errors) this.grist.fetch()
+      if (!response.errors) {
+        this.grist.fetch()
+      }
     }
   }
 

package/client/pages/data-ooc-view.js

@@ -0,0 +1,182 @@
+import '@operato/dataset/ox-data-ooc-view.js'
+
+import { LitElement, css, html } from 'lit'
+import { i18next, localize } from '@operato/i18n'
+
+import { ScrollbarStyles } from '@operato/styles'
+import { client } from '@operato/graphql'
+import gql from 'graphql-tag'
+
+class DataOocView extends localize(i18next)(LitElement) {
+  static get styles() {
+    return [
+      ScrollbarStyles,
+      css`
+        :host {
+          display: flex;
+          flex-direction: column;
+
+          background-color: #fff;
+        }
+
+        div[content] {
+          flex: 1;
+
+          display: flex;
+          overflow: auto;
+        }
+
+        ox-data-ooc-view {
+          flex: 1;
+          padding: var(--padding-wide);
+        }
+
+        label[comment] {
+          display: flex;
+          flex-direction: column;
+
+          padding: var(--padding-wide);
+        }
+
+        label[comment] div {
+          display: flex;
+        }
+
+        mwc-icon {
+          color: var(--status-danger-color);
+        }
+
+        textarea {
+          border: var(--input-field-border);
+          border-radius: var(--input-border-radius);
+          padding: var(--input-field-padding);
+          font: var(--input-field-font);
+        }
+
+        .button-container {
+          display: flex;
+          margin-left: auto;
+          padding: var(--padding-default);
+        }
+      `
+    ]
+  }
+
+  static get properties() {
+    return {
+      dataSet: Object,
+      dataOoc: Object
+    }
+  }
+
+  get sampleView() {
+    return this.renderRoot.querySelector('ox-data-ooc-view')
+  }
+
+  render() {
+    const state = this.dataOoc.state
+
+    return html`
+      <div content>
+        <ox-data-ooc-view .dataSet=${this.dataSet} .dataOoc=${this.dataOoc}></ox-data-ooc-view>
+      </div>
+
+      ${state === 'CREATED' || state === 'REVIEWED'
+        ? html`
+            <label comment>
+              <div><mwc-icon>build_circle</mwc-icon> <span>correction activity</span></div>
+              <textarea placeholder="조치 내용을 입력해주세요."></textarea>
+            </label>
+          `
+        : html``}
+
+      <div class="button-container">
+        ${state === 'CREATED'
+          ? html`<mwc-button raised @click=${() => this._processOoc('REVIEWED')}
+              >${i18next.t('button.reviewed')}</mwc-button
+            >`
+          : state === 'REVIEWED'
+          ? html`<mwc-button raised @click=${() => this._processOoc('CORRECTED')}
+              >${i18next.t('button.corrected')}</mwc-button
+            >`
+          : html``}
+      </div>
+    `
+  }
+
+  updated(changes) {
+    if (changes.has('dataOoc')) {
+      this.fetchDataSet()
+    }
+  }
+
+  async fetchDataSet() {
+    const id = this.dataOoc?.dataSet?.id
+
+    if (id) {
+      const response = await client.query({
+        query: gql`
+          query ($id: String!) {
+            dataSet(id: $id) {
+              id
+              name
+              description
+              useCase
+              dataItems {
+                id
+                name
+                description
+                active
+                unit
+                tag
+                type
+                spec
+              }
+            }
+          }
+        `,
+        variables: {
+          id: this.dataOoc.dataSet.id
+        }
+      })
+
+      this.dataSet = response.data.dataSet
+    }
+  }
+
+  async _processOoc(state) {
+    const commentTextArea = this.renderRoot.querySelector('textarea')
+    const comment = commentTextArea && commentTextArea.value
+    if (!comment || !comment.trim()) {
+      commentTextArea.focus()
+      return
+    }
+
+    const patch = {
+      state,
+      correctiveAction: comment
+    }
+
+    const response = await client.mutate({
+      mutation: gql`
+        mutation ($id: String!, $patch: DataOocPatch!) {
+          updateDataOoc(id: $id, patch: $patch) {
+            id
+          }
+        }
+      `,
+      variables: {
+        id: this.dataOoc.id,
+        patch
+      }
+    })
+
+    if (!response.errors) {
+      document.dispatchEvent(
+        new CustomEvent('notify', { detail: { message: i18next.t('text.data ooc updated successfully') } })
+      )
+    }
+  }
+}
+
+window.customElements.define('data-ooc-view', DataOocView)

package/client/pages/data-ooc.js

@@ -1,15 +1,16 @@
 import '@operato/data-grist'
+import './data-ooc-view.js'
 
 import { CommonButtonStyles, ScrollbarStyles } from '@operato/styles'
 import { PageView, store } from '@operato/shell'
 import { css, html } from 'lit'
 import { i18next, localize } from '@operato/i18n'
+import { notify, openPopup } from '@operato/layout'
 
 import { client } from '@operato/graphql'
 import { connect } from 'pwa-helpers/connect-mixin'
 import gql from 'graphql-tag'
 import { isMobileDevice } from '@operato/utils'
-import { notify } from '@operato/layout'
 
 export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
   static get properties() {
@@ -55,11 +56,11 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
       title: i18next.t('title.data-ooc list'),
       help: 'integration/ui/data-ooc',
       actions: [
-        {
-          title: i18next.t('button.save'),
-          action: this._updateDataOoc.bind(this),
-          ...CommonButtonStyles.save
-        }
+        // {
+        //   title: i18next.t('button.save'),
+        //   action: this._updateDataOoc.bind(this),
+        //   ...CommonButtonStyles.save
+        // }
       ],
       exportable: {
         name: i18next.t('title.data-ooc list'),
@@ -117,6 +118,26 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
       columns: [
         { type: 'gutter', gutterName: 'sequence' },
         { type: 'gutter', gutterName: 'row-selector', multiple: true },
+        {
+          type: 'gutter',
+          gutterName: 'button',
+          icon: 'assignment_turned_in',
+          handlers: {
+            click: (columns, data, column, record, rowIndex) => {
+              const popup = openPopup(
+                html` <data-ooc-view .dataOoc=${record} style="background-color: white;"></data-ooc-view> `,
+                {
+                  backdrop: true,
+                  size: 'large',
+                  title: i18next.t('title.data-ooc view')
+                }
+              )
+              popup.onclosed = () => {
+                this.grist.fetch()
+              }
+            }
+          }
+        },
         {
           type: 'string',
           name: 'name',
@@ -142,6 +163,12 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
           width: 150,
           imex: true
         },
+        {
+          type: 'string',
+          name: 'history',
+          hidden: true,
+          imex: true
+        },
         {
           type: 'checkbox',
           name: 'ooc',
@@ -161,14 +188,16 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
           width: 30
         },
         {
-          type: 'string',
+          type: 'select',
           name: 'state',
           label: true,
           header: i18next.t('field.state'),
           record: {
-            editable: false
+            editable: false,
+            options: ['', 'CREATED', 'REVIEWED', 'CORRECTED']
           },
-          width: 150,
+          width: 100,
+          filter: true,
           imex: true
         },
         {
@@ -193,17 +222,6 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
           width: 120,
           imex: true
         },
-        {
-          type: 'resource-object',
-          name: 'dataSample',
-          header: i18next.t('field.data-sample'),
-          record: {
-            editable: false
-          },
-          sortable: true,
-          width: 120,
-          imex: true
-        },
         {
           type: 'json5',
           name: 'partitionKeys',
@@ -320,6 +338,7 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
               spec
               ooc
               oos
+              history
               correctiveAction
               correctedAt
               corrector {
@@ -350,62 +369,62 @@ export class DataOoc extends connect(store)(localize(i18next)(PageView)) {
     }
   }
 
-  async _updateDataOoc() {
-    let patches = this.grist.dirtyRecords
-    if (patches && patches.length) {
-      patches = patches.map(patch => {
-        let patchField = patch.id ? { id: patch.id } : {}
-        const dirtyFields = patch.__dirtyfields__
-        for (let key in dirtyFields) {
-          patchField[key] = dirtyFields[key].after
-        }
-        this._setDefaultFieldsValue(patchField)
-        patchField.cuFlag = patch.__dirty__
-
-        return patchField
-      })
-
-      const response = await client.mutate({
-        mutation: gql`
-          mutation ($patches: [DataOocPatch!]!) {
-            updateMultipleDataOoc(patches: $patches) {
-              name
-            }
-          }
-        `,
-        variables: {
-          patches
-        }
-      })
-
-      if (!response.errors) this.grist.fetch()
-    }
-  }
-
-  async _deleteDataOoc() {
-    if (confirm(i18next.t('text.sure_to_x', { x: i18next.t('text.delete') }))) {
-      const ids = this.grist.selected.map(record => record.id)
-      if (ids && ids.length > 0) {
-        const response = await client.mutate({
-          mutation: gql`
-            mutation ($ids: [String!]!) {
-              deleteDataOocs(ids: $ids)
-            }
-          `,
-          variables: {
-            ids
-          }
-        })
-
-        if (!response.errors) {
-          this.grist.fetch()
-          notify({
-            message: i18next.t('text.info_x_successfully', { x: i18next.t('text.delete') })
-          })
-        }
-      }
-    }
-  }
+  // async _updateDataOoc() {
+  //   let patches = this.grist.dirtyRecords
+  //   if (patches && patches.length) {
+  //     patches = patches.map(patch => {
+  //       let patchField = patch.id ? { id: patch.id } : {}
+  //       const dirtyFields = patch.__dirtyfields__
+  //       for (let key in dirtyFields) {
+  //         patchField[key] = dirtyFields[key].after
+  //       }
+  //       this._setDefaultFieldsValue(patchField)
+  //       patchField.cuFlag = patch.__dirty__
+
+  //       return patchField
+  //     })
+
+  //     const response = await client.mutate({
+  //       mutation: gql`
+  //         mutation ($patches: [DataOocPatch!]!) {
+  //           updateMultipleDataOoc(patches: $patches) {
+  //             name
+  //           }
+  //         }
+  //       `,
+  //       variables: {
+  //         patches
+  //       }
+  //     })
+
+  //     if (!response.errors) this.grist.fetch()
+  //   }
+  // }
+
+  // async _deleteDataOoc() {
+  //   if (confirm(i18next.t('text.sure_to_x', { x: i18next.t('text.delete') }))) {
+  //     const ids = this.grist.selected.map(record => record.id)
+  //     if (ids && ids.length > 0) {
+  //       const response = await client.mutate({
+  //         mutation: gql`
+  //           mutation ($ids: [String!]!) {
+  //             deleteDataOocs(ids: $ids)
+  //           }
+  //         `,
+  //         variables: {
+  //           ids
+  //         }
+  //       })
+
+  //       if (!response.errors) {
+  //         this.grist.fetch()
+  //         notify({
+  //           message: i18next.t('text.info_x_successfully', { x: i18next.t('text.delete') })
+  //         })
+  //       }
+  //     }
+  //   }
+  // }
 
   _exportableData() {
     let records = []

package/client/pages/data-sample-view.js

@@ -0,0 +1,97 @@
+import '@operato/dataset/ox-data-sample-view.js'
+
+import { LitElement, css, html } from 'lit'
+import { i18next, localize } from '@operato/i18n'
+
+import { ScrollbarStyles } from '@operato/styles'
+import { client } from '@operato/graphql'
+import gql from 'graphql-tag'
+
+class DataSampleView extends localize(i18next)(LitElement) {
+  static get properties() {
+    return {
+      dataSet: Object,
+      dataSample: Object
+    }
+  }
+
+  static get styles() {
+    return [
+      ScrollbarStyles,
+      css`
+        :host {
+          display: flex;
+          flex-direction: column;
+
+          background-color: #fff;
+        }
+
+        div[content] {
+          flex: 1;
+
+          display: flex;
+          overflow: auto;
+        }
+
+        ox-data-sample-view {
+          flex: 1;
+          padding: var(--padding-wide);
+        }
+      `
+    ]
+  }
+
+  get sampleView() {
+    return this.renderRoot.querySelector('ox-data-sample-view')
+  }
+
+  render() {
+    return html`
+      <div content>
+        <ox-data-sample-view .dataSet=${this.dataSet} .dataSample=${this.dataSample}></ox-data-sample-view>
+      </div>
+    `
+  }
+
+  updated(changes) {
+    if (changes.has('dataSample')) {
+      this.fetchDataSet()
+    }
+  }
+
+  async fetchDataSet() {
+    const id = this.dataSample?.dataSet?.id
+
+    if (id) {
+      const response = await client.query({
+        query: gql`
+          query ($id: String!) {
+            dataSet(id: $id) {
+              id
+              name
+              description
+              useCase
+              dataItems {
+                id
+                name
+                description
+                active
+                unit
+                tag
+                type
+                spec
+              }
+            }
+          }
+        `,
+        variables: {
+          id: this.dataSample.dataSet.id
+        }
+      })
+
+      this.dataSet = response.data.dataSet
+    }
+  }
+}
+
+window.customElements.define('data-sample-view', DataSampleView)

package/client/pages/data-sample.js

@@ -1,15 +1,16 @@
 import '@operato/data-grist'
+import './data-sample-view.js'
 
 import { PageView, store } from '@operato/shell'
 import { css, html } from 'lit'
 import { i18next, localize } from '@operato/i18n'
+import { notify, openPopup } from '@operato/layout'
 
 import { ScrollbarStyles } from '@operato/styles'
 import { client } from '@operato/graphql'
 import { connect } from 'pwa-helpers/connect-mixin'
 import gql from 'graphql-tag'
 import { isMobileDevice } from '@operato/utils'
-import { notify } from '@operato/layout'
 
 export class DataSample extends connect(store)(localize(i18next)(PageView)) {
   static get properties() {
@@ -109,6 +110,23 @@ export class DataSample extends connect(store)(localize(i18next)(PageView)) {
       columns: [
         { type: 'gutter', gutterName: 'sequence' },
         { type: 'gutter', gutterName: 'row-selector', multiple: true },
+        {
+          type: 'gutter',
+          gutterName: 'button',
+          icon: 'assignment',
+          handlers: {
+            click: (columns, data, column, record, rowIndex) => {
+              openPopup(
+                html` <data-sample-view .dataSample=${record} style="background-color: white;"></data-sample-view> `,
+                {
+                  backdrop: true,
+                  size: 'large',
+                  title: i18next.t('title.data-sample view')
+                }
+              )
+            }
+          }
+        },
         {
           type: 'string',
           name: 'name',

package/client/pages/data-sensor.js

@@ -153,6 +153,7 @@ export class DataSensor extends connect(store)(localize(i18next)(PageView)) {
             editable: true
           },
           sortable: true,
+          filter: true,
           width: 60
         },
         {
@@ -176,6 +177,7 @@ export class DataSensor extends connect(store)(localize(i18next)(PageView)) {
             editable: true
           },
           sortable: true,
+          filter: 'search',
           width: 150
         },
         {

package/client/pages/data-set.js

@@ -200,6 +200,7 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
           record: {
             editable: true
           },
+          filter: true,
           sortable: true,
           width: 60
         },
@@ -223,6 +224,7 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
             ]
           },
           sortable: true,
+          filter: true,
           width: 60
         },
         {
@@ -242,6 +244,7 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
             }
           },
           sortable: true,
+          filter: true,
           width: 80
         },
         {

package/dist-server/controllers/create-data-sample.js

@@ -4,13 +4,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createDataSample = void 0;
-const moment_1 = __importDefault(require("moment"));
+const data_ooc_1 = require("../service/data-ooc/data-ooc");
 const shell_1 = require("@things-factory/shell");
 const data_item_1 = require("../service/data-item/data-item");
-const data_ooc_1 = require("../service/data-ooc/data-ooc");
 const data_sample_1 = require("../service/data-sample/data-sample");
 const data_set_1 = require("../service/data-set/data-set");
 const data_use_case_1 = require("./data-use-case");
+const moment_1 = __importDefault(require("moment"));
 const debug = require('debug')('things-factory:dataset:controller/save-data-sample');
 // parse variable javascript string pattern
 const replaceVariables = (keys, dic) => {
@@ -53,12 +53,20 @@ async function createDataSample(dataSample, context) {
         spec[dataItem.tag] = Object.assign(Object.assign({}, dataItem.spec), { name: dataItem.name /* do we need ? */ });
         return spec;
     }, {});
-    var partitionKeys = Object.assign({}, dataSet.partitionKeys);
     const collectedAt = dataSample.collectedAt || new Date();
-    partitionKeys = formatDate(partitionKeys, (0, moment_1.default)(collectedAt).utc());
-    partitionKeys = replaceVariables(partitionKeys, Object.assign({ domain: domain.subdomain, dataSetId: dataSample.dataSet.id }, dataSample.data));
+    const momentUtc = (0, moment_1.default)(collectedAt).utc();
+    const defaultPartitionKeys = {
+        domain: domain.subdomain,
+        dataSetId: dataSample.dataSet.id,
+        year: momentUtc.format('Y'),
+        month: momentUtc.format('M'),
+        day: momentUtc.format('D')
+    };
+    var partitionKeys = Object.assign(Object.assign({}, defaultPartitionKeys), dataSet.partitionKeys);
+    partitionKeys = formatDate(partitionKeys, momentUtc);
+    partitionKeys = replaceVariables(partitionKeys, Object.assign({}, dataSample.data));
     const { ooc, oos } = data_use_case_1.DataUseCase.evaluate(dataSet, dataItems, dataSample.data) || {};
-    const result = await tx.getRepository(data_sample_1.DataSample).save(Object.assign(Object.assign({ name: dataSet.name, description: dataSet.description }, dataSample), { domain,
+    const result = await tx.getRepository(data_sample_1.DataSample).save(Object.assign(Object.assign({ name: dataSet.name, description: dataSet.description, useCase: dataSet.useCase }, dataSample), { domain,
         partitionKeys,
         spec,
         ooc,
@@ -68,6 +76,7 @@ async function createDataSample(dataSample, context) {
         const dataOoc = await tx.getRepository(data_ooc_1.DataOoc).save({
             name: dataSet.name,
             description: dataSet.description,
+            useCase: dataSet.useCase,
             dataSet,
             dataSample: result,
             data: dataSample.data,
@@ -76,6 +85,16 @@ async function createDataSample(dataSample, context) {
             spec,
             ooc,
             oos,
+            history: [
+                {
+                    user: {
+                        id: user.id,
+                        name: user.name
+                    },
+                    state: data_ooc_1.DataOocStatus.CREATED,
+                    timestamp: Date.now()
+                }
+            ],
             state: data_ooc_1.DataOocStatus.CREATED,
             collectedAt,
             creator: user,