@things-factory/dataset 5.0.0-alpha.3 → 5.0.0-alpha.32

package/client/pages/{data-set.js → data-set/data-set-list-page.js}

@@ -1,53 +1,81 @@
 import '@operato/data-grist'
-import './data-item-list'
-import './data-set-importer'
+import './data-item-list.js'
+import './data-set-importer.js'
+import '../data-entry/data-entry-form.js'
 
 import gql from 'graphql-tag'
+import JSON5 from 'json5'
 import { css, html } from 'lit'
 import moment from 'moment-timezone'
 import { connect } from 'pwa-helpers/connect-mixin'
 
+import { getEditor, getRenderer } from '@operato/data-grist'
+import { OxDataUseCase } from '@operato/dataset'
 import { client } from '@operato/graphql'
 import { i18next, localize } from '@operato/i18n'
 import { notify, openPopup } from '@operato/layout'
 import { PageView, store } from '@operato/shell'
-import { CommonButtonStyles, ScrollbarStyles } from '@operato/styles'
+import { CommonButtonStyles, CommonGristStyles, ScrollbarStyles } from '@operato/styles'
 import { isMobileDevice } from '@operato/utils'
 
-export class DataSet extends connect(store)(localize(i18next)(PageView)) {
+const DEFAULT_TZ = Intl.DateTimeFormat().resolvedOptions().timeZone
+const TIMEZONE_OPTIONS = ['', DEFAULT_TZ, ...moment.tz.names().filter(tz => tz !== DEFAULT_TZ)]
+
+const VIEW_TYPES = [
+  {
+    display: '',
+    value: ''
+  },
+  {
+    display: 'Generated',
+    value: 'generated'
+  },
+  {
+    display: 'Board',
+    value: 'board'
+  },
+  {
+    display: 'Page',
+    value: 'page'
+  },
+  {
+    display: 'External URL',
+    value: 'external'
+  }
+]
+
+const USECASE_OPTIONS = () => {
+  return ['', ...OxDataUseCase.getUseCaseNames()].map(name => {
+    return {
+      display: name,
+      value: name
+    }
+  })
+}
+
+export class DataSetListPage extends connect(store)(localize(i18next)(PageView)) {
   static get properties() {
     return {
       active: String,
-      gristConfig: Object
+      gristConfig: Object,
+      filters: Object,
+      sorters: Object,
+      mode: String
     }
   }
 
   static get styles() {
     return [
       ScrollbarStyles,
+      CommonGristStyles,
       css`
         :host {
           display: flex;
-          flex-direction: column;
-
-          overflow: hidden;
-        }
-
-        ox-grist {
-          overflow-y: auto;
-          flex: 1;
-        }
-
-        #filters {
-          display: flex;
-          flex-direction: row;
-          justify-content: space-between;
 
-          background-color: white;
-        }
+          width: 100%;
 
-        #filters > * {
-          padding: var(--padding-default) var(--padding-wide);
+          --grid-record-emphasized-background-color: red;
+          --grid-record-emphasized-color: yellow;
         }
       `
     ]
@@ -56,7 +84,7 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
   get context() {
     return {
       title: i18next.t('title.data-set list'),
-      help: 'integration/ui/data-set',
+      help: 'dataset/data-set',
       actions: [
         {
           title: i18next.t('button.copy'),
@@ -85,14 +113,43 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
   }
 
   render() {
+    const mode = this.mode || (isMobileDevice() ? 'LIST' : 'GRID')
+
     return html`
       <ox-grist
-        .mode=${isMobileDevice() ? 'LIST' : 'GRID'}
+        .mode=${mode}
         .config=${this.gristConfig}
+        .filters=${this.filters}
+        .orders=${this.orders}
         .fetchHandler=${this.fetchHandler.bind(this)}
       >
         <div slot="headroom" id="filters">
-          <ox-filters-form></ox-filters-form>
+          <div id="filters">
+            <ox-filters-form></ox-filters-form>
+          </div>
+
+          <div id="sorters">
+            Sort
+            <mwc-icon
+              @click=${e => {
+                const target = e.currentTarget
+                this.renderRoot.querySelector('#sorter-control').open({
+                  right: 0,
+                  top: target.offsetTop + target.offsetHeight
+                })
+              }}
+              >expand_more</mwc-icon
+            >
+            <ox-popup id="sorter-control">
+              <ox-sorters-control> </ox-sorters-control>
+            </ox-popup>
+          </div>
+
+          <div id="modes">
+            <mwc-icon @click=${() => (this.mode = 'GRID')} ?active=${mode == 'GRID'}>grid_on</mwc-icon>
+            <mwc-icon @click=${() => (this.mode = 'LIST')} ?active=${mode == 'LIST'}>format_list_bulleted</mwc-icon>
+            <mwc-icon @click=${() => (this.mode = 'CARD')} ?active=${mode == 'CARD'}>apps</mwc-icon>
+          </div>
         </div>
       </ox-grist>
     `
@@ -102,32 +159,12 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
     return this.renderRoot.querySelector('ox-grist')
   }
 
-  // update with url params value
-  _updateSearchConfig(lifecycle) {
-    // this.searchConfig = this.searchConfig.map(conf => {
-    // if (conf.name in lifecycle.params) {
-    //   conf.value = lifecycle.params[conf.name]
-    // } else {
-    //   delete conf.value
-    // }
-    // return conf
-    // })
-  }
-
-  // set default field value to record with searchConfig
-  _setDefaultFieldsValue(fields) {
-    // this.searchConfig.forEach(conf => {
-    //   if (!fields[conf.name] && conf.value) {
-    //     fields[conf.name] = conf.value
-    //   }
-    // })
-  }
-
   async pageInitialized(lifecycle) {
-    this._updateSearchConfig(lifecycle)
-
     this.gristConfig = {
-      list: { fields: ['name', 'description', 'schedule', 'active'] },
+      list: {
+        fields: ['name', 'description'],
+        details: ['schedule', 'active']
+      },
       columns: [
         { type: 'gutter', gutterName: 'sequence' },
         { type: 'gutter', gutterName: 'row-selector', multiple: true },
@@ -138,19 +175,38 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
           handlers: {
             click: (columns, data, column, record, rowIndex) => {
               if (!record.id) return
-              openPopup(html` <data-item-list .dataSet=${record}></data-item-list> `, {
+              const popup = openPopup(html` <data-item-list .dataSet=${record}></data-item-list> `, {
                 backdrop: true,
                 help: 'data-set/ui/data-item-list',
                 size: 'large',
                 title: i18next.t('title.data-item list')
               })
+              popup.onclosed = () => {
+                this.grist.fetch()
+              }
+            }
+          }
+        },
+        {
+          type: 'gutter',
+          gutterName: 'button',
+          icon: 'fact_check',
+          handlers: {
+            click: (columns, data, column, record, rowIndex) => {
+              openPopup(
+                html` <data-entry-form .dataSet=${record} style="background-color: white;"></data-entry-form> `,
+                {
+                  backdrop: true,
+                  size: 'large',
+                  title: i18next.t('title.data-entry-form')
+                }
+              )
             }
           }
         },
         {
           type: 'string',
           name: 'name',
-          label: true,
           header: i18next.t('field.name'),
           record: {
             editable: true
@@ -162,7 +218,6 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
         {
           type: 'string',
           name: 'description',
-          label: true,
           header: i18next.t('field.description'),
           record: {
             editable: true
@@ -178,15 +233,58 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
           record: {
             editable: true
           },
+          filter: true,
+          sortable: true,
+          width: 60
+        },
+        {
+          type: 'select',
+          name: 'type',
+          label: true,
+          header: i18next.t('field.type'),
+          record: {
+            editable: true,
+            options: [
+              {},
+              {
+                display: 'Manually Collected',
+                value: 'manual'
+              },
+              {
+                display: 'Automatically Collected',
+                value: 'automatic'
+              }
+            ]
+          },
           sortable: true,
+          filter: true,
           width: 60
         },
         {
-          type: 'json',
+          type: 'select',
+          name: 'useCase',
+          label: true,
+          header: i18next.t('field.use-case'),
+          record: {
+            editable: true,
+            options: USECASE_OPTIONS
+          },
+          sortable: true,
+          filter: {
+            operator: 'eq',
+            options: USECASE_OPTIONS /* in case select options type is a function, filter should have its own options */
+          },
+          width: 80
+        },
+        {
+          type: 'partition-keys',
           name: 'partitionKeys',
           header: i18next.t('field.partition-keys'),
           record: {
-            editable: true
+            editable: true,
+            options: {
+              objectified: true /* transfered as a object type */
+            }
           },
           width: 200
         },
@@ -196,7 +294,10 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
           label: true,
           header: i18next.t('field.schedule'),
           record: {
-            editable: true
+            editable: true,
+            options: {
+              objectified: true
+            }
           },
           width: 80,
           label: true
@@ -207,12 +308,81 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
           header: i18next.t('field.timezone'),
           record: {
             editable: true,
-            options: ['', ...moment.tz.names()]
+            options: TIMEZONE_OPTIONS
+          },
+          width: 120
+        },
+        {
+          type: 'resource-object',
+          name: 'supervisoryRole',
+          header: i18next.t('field.supervisory-role'),
+          record: {
+            editable: true,
+            options: {
+              queryName: 'roles'
+            }
           },
+          sortable: true,
           width: 120
         },
         {
-          type: 'object',
+          type: 'select',
+          name: 'entryType',
+          label: true,
+          header: i18next.t('field.entry-type'),
+          record: {
+            editable: true,
+            options: VIEW_TYPES
+          },
+          width: 80
+        },
+        {
+          type: 'string',
+          name: 'entryView',
+          header: i18next.t('field.entry-view'),
+          record: {
+            editable: true,
+            editor: function (value, column, record, rowIndex, field) {
+              var type = record.entryType !== 'board' ? 'string' : 'board'
+              return getEditor(type)(value, column, record, rowIndex, field)
+            },
+            renderer: function (value, column, record, rowIndex, field) {
+              var type = record.entryType !== 'board' ? 'string' : 'board'
+              return getRenderer(type)(value, column, record, rowIndex, field)
+            }
+          },
+          width: 140
+        },
+        {
+          type: 'select',
+          name: 'monitorType',
+          label: true,
+          header: i18next.t('field.monitor-type'),
+          record: {
+            editable: true,
+            options: VIEW_TYPES
+          },
+          width: 80
+        },
+        {
+          type: 'string',
+          name: 'monitorView',
+          header: i18next.t('field.monitor-view'),
+          record: {
+            editable: true,
+            editor: function (value, column, record, rowIndex, field) {
+              var type = record.monitorType !== 'board' ? 'string' : 'board'
+              return getEditor(type)(value, column, record, rowIndex, field)
+            },
+            renderer: function (value, column, record, rowIndex, field) {
+              var type = record.monitorType !== 'board' ? 'string' : 'board'
+              return getRenderer(type)(value, column, record, rowIndex, field)
+            }
+          },
+          width: 140
+        },
+        {
+          type: 'resource-object',
           name: 'updater',
           header: i18next.t('field.updater'),
           record: {
@@ -251,10 +421,28 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
 
   async pageUpdated(changes, lifecycle) {
     if (this.active) {
-      // update with url params value
-      this._updateSearchConfig(lifecycle)
       await this.updateComplete
 
+      var filters = lifecycle.params?.['filters']
+      if (filters) {
+        try {
+          filters = JSON5.parse(filters)
+          this.filters = filters
+        } catch (e) {
+          console.error(`filters parameter parsing error: ${e}`)
+        }
+      }
+
+      var sorters = lifecycle.params?.['sorters']
+      if (sorters) {
+        try {
+          sorters = JSON5.parse(sorters)
+          this.sorters = sorters
+        } catch (e) {
+          console.error(`sorters parameter parsing error: ${e}`)
+        }
+      }
+
       this.grist.fetch()
     }
   }
@@ -269,10 +457,19 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
               name
               description
               partitionKeys
-              slugger
               active
+              type
+              useCase
               schedule
               timezone
+              supervisoryRole {
+                id
+                name
+              }
+              entryType
+              entryView
+              monitorType
+              monitorView
               updater {
                 id
                 name
@@ -283,7 +480,10 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
                 description
                 sequence
                 active
+                tag
                 type
+                unit
+                options
                 quota
                 spec
               }
@@ -330,13 +530,6 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
     }
   }
 
-  async stateChanged(state) {
-    if (this.active && this._currentPopupName && !state.layout.viewparts[this._currentPopupName]) {
-      this.grist.fetch()
-      this._currentPopupName = null
-    }
-  }
-
   async _copyDataSet() {
     var selected = this.grist.selected
     if (selected.length == 0) return
@@ -355,7 +548,9 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
       }
     })
 
-    if (!response.errors) this.grist.fetch()
+    if (!response.errors) {
+      this.grist.fetch()
+    }
   }
 
   async _updateDataSet() {
@@ -367,7 +562,6 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
         for (let key in dirtyFields) {
           patchField[key] = dirtyFields[key].after
         }
-        this._setDefaultFieldsValue(patchField)
         patchField.cuFlag = patch.__dirty__
 
         return patchField
@@ -386,13 +580,25 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
         }
       })
 
-      if (!response.errors) this.grist.fetch()
+      if (!response.errors) {
+        this.grist.fetch()
+      }
     }
   }
 
   async exportHandler() {
     const exportTargets = this.grist.selected.length ? this.grist.selected : this.grist.dirtyData.records
-    const targetFieldSet = new Set(['id', 'name', 'type', 'description', 'schedule', 'timezone', 'active', 'dataItems'])
+    const targetFieldSet = new Set([
+      'id',
+      'name',
+      'type',
+      'description',
+      'tag',
+      'schedule',
+      'timezone',
+      'active',
+      'dataItems'
+    ])
 
     return exportTargets.map(dataSet => {
       let tempObj = {}
@@ -408,11 +614,11 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
     openPopup(
       html`
         <data-set-importer
-          .dataSets="${records}"
-          @imported="${() => {
+          .dataSets=${records}
+          @imported=${() => {
             history.back()
             this.grist.fetch()
-          }}"
+          }}
         ></data-set-importer>
       `,
       {
@@ -424,4 +630,4 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
   }
 }
 
-window.customElements.define('data-set-page', DataSet)
+window.customElements.define('data-set-list-page', DataSetListPage)

package/client/route.js

@@ -1,15 +1,23 @@
 export default function route(page) {
   switch (page) {
-    case 'data-set':
-      import('./pages/data-set')
+    case 'data-set-list':
+      import('./pages/data-set/data-set-list-page.js')
       return page
 
-    case 'data-sensor':
-      import('./pages/data-sensor')
+    case 'data-sensor-list':
+      import('./pages/data-sensor/data-sensor-list-page.js')
       return page
 
-    case 'data-sample':
-      import('./pages/data-sample')
+    case 'data-sample-list':
+      import('./pages/data-sample/data-sample-list-page.js')
+      return page
+
+    case 'data-ooc-list':
+      import('./pages/data-ooc/data-ooc-list-page.js')
+      return page
+
+    case 'data-entry-list':
+      import('./pages/data-entry/data-entry-list-page.js')
       return page
   }
 }

package/dist-server/controllers/create-data-sample.js

@@ -0,0 +1,133 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDataSample = void 0;
+const moment_timezone_1 = __importDefault(require("moment-timezone"));
+const shell_1 = require("@things-factory/shell");
+const data_item_1 = require("../service/data-item/data-item");
+const data_ooc_1 = require("../service/data-ooc/data-ooc");
+const data_sample_1 = require("../service/data-sample/data-sample");
+const data_set_1 = require("../service/data-set/data-set");
+const data_use_case_1 = require("./data-use-case");
+const work_shift_1 = require("@things-factory/work-shift");
+const debug = require('debug')('things-factory:dataset:controller/save-data-sample');
+// parse variable javascript string pattern
+const replaceVariables = (keys, dic) => {
+    for (const k in keys) {
+        const matches = keys[k].match(/\$\{\w*\}/g);
+        matches &&
+            matches.forEach(m => {
+                keys[k] = keys[k].replace(m, dic[m.slice(2, -1)]);
+            });
+    }
+    return keys;
+};
+// It is required UTC date for Partitioning File System like AWS S3 from Athena.
+// ex) %YYYY, %MM, %DD
+const formatDate = (keys, _moment) => {
+    for (const k in keys) {
+        const matches = keys[k].match(/%\w*/g);
+        matches &&
+            matches.forEach(m => {
+                keys[k] = keys[k].replace(m, _moment.format(m.substr(1)));
+            });
+    }
+    return keys;
+};
+async function createDataSample(dataSample, context) {
+    const { domain, user, tx } = context.state;
+    const dataSet = await tx.getRepository(data_set_1.DataSet).findOne({
+        where: { id: dataSample.dataSet.id }
+    });
+    const dataItems = await tx.getRepository(data_item_1.DataItem).find({
+        where: {
+            domain,
+            dataSet
+        },
+        order: {
+            sequence: 'DESC'
+        }
+    });
+    const spec = dataItems.reduce((spec, dataItem) => {
+        spec[dataItem.tag] = Object.assign(Object.assign({}, dataItem.spec), { name: dataItem.name, hidden: dataItem.hidden });
+        return spec;
+    }, {});
+    const collectedAt = dataSample.collectedAt || new Date();
+    // workDate ex) 2022-04-04
+    const { workDate, workShift } = await (0, work_shift_1.getWorkDateAndShift)(domain, collectedAt);
+    const dateFormat = 'YYYY-MM-DD';
+    // local time dataSet timezone or domain timezone or default 'UTC'
+    const timezone = dataSet['timezone'] || domain['timezone'] || 'UTC';
+    // const collectedAt = dataSample.collectedAt || new Date()
+    const localDateTz = (0, moment_timezone_1.default)(collectedAt).tz(timezone);
+    const defaultPartitionKeys = {
+        domain: domain.subdomain,
+        datasetid: dataSample.dataSet.id,
+        date: localDateTz.format(dateFormat),
+        workdate: workDate,
+        workshift: workShift
+    };
+    var partitionKeys = Object.assign(Object.assign({}, defaultPartitionKeys), dataSet.partitionKeys);
+    partitionKeys = formatDate(partitionKeys, localDateTz);
+    partitionKeys = replaceVariables(partitionKeys, Object.assign({}, dataSample.data));
+    const { ooc, oos } = data_use_case_1.DataUseCase.evaluate(dataSet, dataItems, dataSample.data) || {};
+    const result = await tx.getRepository(data_sample_1.DataSample).save(Object.assign(Object.assign({ name: dataSet.name, description: dataSet.description, useCase: dataSet.useCase }, dataSample), { domain,
+        partitionKeys,
+        spec,
+        ooc,
+        oos,
+        collectedAt,
+        workDate,
+        workShift, creator: user, updater: user }));
+    if (ooc || oos) {
+        const dataOoc = await tx.getRepository(data_ooc_1.DataOoc).save({
+            name: dataSet.name,
+            description: dataSet.description,
+            useCase: dataSet.useCase,
+            dataSet,
+            dataSample: result,
+            data: dataSample.data,
+            rawData: dataSample.rawData,
+            domain,
+            partitionKeys,
+            spec,
+            ooc,
+            oos,
+            history: [
+                {
+                    user: {
+                        id: user.id,
+                        name: user.name
+                    },
+                    state: data_ooc_1.DataOocStatus.CREATED,
+                    timestamp: Date.now()
+                }
+            ],
+            state: data_ooc_1.DataOocStatus.CREATED,
+            workDate,
+            workShift,
+            collectedAt,
+            creator: user,
+            updater: user
+        });
+        shell_1.pubsub.publish('data-ooc', {
+            dataOoc,
+            supervisoryRoleId: dataSet.supervisoryRoleId
+        });
+        shell_1.pubsub.publish('notification', {
+            notification: {
+                domain,
+                type: 'error',
+                title: `Data OOC occurred on '${dataSet.name}'`,
+                body: `Data OOC occurred on '${dataSet.name}'`,
+                url: (0, shell_1.getRedirectSubdomainPath)(context, domain.subdomain, `/data-ooc/${dataOoc.id}`),
+                timestamp: collectedAt
+            }
+        });
+    }
+    return result;
+}
+exports.createDataSample = createDataSample;
+//# sourceMappingURL=create-data-sample.js.map