@things-factory/dataset 5.0.0-alpha.3 → 5.0.0-alpha.30

package/README.md

@@ -0,0 +1,13 @@
+# dataset
+## partition keys
+At the early stage, partition keys are desinged for dynamic partitioning for Athena. But It will be required so many AWS Glue crawlers also by each 'data-sets'.
+Now partition keys are fixed with some cases. Default is daily dataset, which has S3 Key like 'domain={domain}/datasetid={datasetid}/year=2022/month=3/day=25'. It will be set default with empty value in 'data-sets'.
+
+In another case, it is necessary to divide by hours or minutes and store them. This datetime item must be included in the S3 key. It will be required a new Glue Crawler and a Glue Catalog Table. For example if you need to separate directory hourly, you should add "%H" for 'hour' to partition_keys for this. and also should add a new crawler. And it would be needed a column to know it is daily or hourly or minutely for reporting client side. It's not developed yet.
+Look at the below picture for understanding.
+
+![Architecture for partitioning](./assets/data-samples.jpg)
+
+These tasks are related for S3 request limitations. 
+> __3,500 PUT/COPY/POST/DELETE or 5,500 GET/HEAD requests per second per prefix in a bucket__
+- https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html

package/client/bootstrap.js

@@ -1 +1,16 @@
-export default function bootstrap() {}
+import {
+  OxGristRendererJson5,
+  registerEditor as registerGristEditor,
+  registerRenderer as registerGristRenderer
+} from '@operato/data-grist'
+
+import { OxGristEditorDataItemSpec } from '@operato/dataset/grist-editor'
+import { OxGristEditorPartitionKeys } from '@operato/app/grist-editor/ox-grist-editor-partition-keys.js'
+
+export default function bootstrap() {
+  registerGristEditor('data-item-spec', OxGristEditorDataItemSpec)
+  registerGristRenderer('data-item-spec', OxGristRendererJson5)
+
+  registerGristEditor('partition-keys', OxGristEditorPartitionKeys)
+  registerGristRenderer('partition-keys', OxGristRendererJson5)
+}

package/client/pages/data-entry/data-entry-form.js

@@ -0,0 +1,85 @@
+import '@operato/dataset/ox-data-entry-form.js'
+
+import { LitElement, css, html } from 'lit'
+import { i18next, localize } from '@operato/i18n'
+
+import { client } from '@operato/graphql'
+import gql from 'graphql-tag'
+
+class DataEntryForm extends localize(i18next)(LitElement) {
+  static get properties() {
+    return {
+      dataSet: Object
+    }
+  }
+
+  static get styles() {
+    return [
+      css`
+        :host {
+          display: flex;
+          flex-direction: column;
+
+          background-color: #fff;
+          overflow: auto;
+        }
+
+        ox-data-entry-form {
+          flex: 1;
+          margin: 10px;
+        }
+
+        .button-container {
+          display: flex;
+          margin-left: auto;
+          padding: var(--padding-default);
+        }
+      `
+    ]
+  }
+
+  get entryForm() {
+    return this.renderRoot.querySelector('ox-data-entry-form')
+  }
+
+  render() {
+    return html`
+      <ox-data-entry-form .dataSet=${this.dataSet}></ox-data-entry-form>
+      <div class="button-container">
+        <mwc-button raised @click=${this._updateDataItems.bind(this)}>${i18next.t('button.save')}</mwc-button>
+      </div>
+    `
+  }
+
+  async _updateDataItems() {
+    const data = this.entryForm.buildValue()
+    const dataSample = {
+      dataSet: {
+        id: this.dataSet.id
+      },
+      data
+    }
+
+    const response = await client.mutate({
+      mutation: gql`
+        mutation ($dataSample: NewDataSample!) {
+          createDataSample(dataSample: $dataSample) {
+            id
+            collectedAt
+          }
+        }
+      `,
+      variables: {
+        dataSample
+      }
+    })
+
+    if (!response.errors) {
+      document.dispatchEvent(
+        new CustomEvent('notify', { detail: { message: i18next.t('text.data sample created successfully') } })
+      )
+    }
+  }
+}
+
+window.customElements.define('data-entry-form', DataEntryForm)

package/client/pages/data-entry/data-entry-list-page.js

@@ -0,0 +1,464 @@
+import '@operato/data-grist'
+import './data-entry-form'
+import '@operato/board/ox-board-viewer.js'
+
+import gql from 'graphql-tag'
+import JSON5 from 'json5'
+import { css, html } from 'lit'
+import { connect } from 'pwa-helpers/connect-mixin'
+
+import { getRenderer } from '@operato/data-grist'
+import { OxDataUseCase } from '@operato/dataset'
+import { client } from '@operato/graphql'
+import { i18next, localize } from '@operato/i18n'
+import { openPopup } from '@operato/layout'
+import { navigate, PageView, store } from '@operato/shell'
+import { CommonGristStyles, ScrollbarStyles } from '@operato/styles'
+import { provider } from '@things-factory/board-ui'
+
+const USECASE_OPTIONS = () => {
+  return ['', ...OxDataUseCase.getUseCaseNames()].map(name => {
+    return {
+      display: name,
+      value: name
+    }
+  })
+}
+
+const showEntryView = async (columns, data, column, record, rowIndex) => {
+  const { name, entryType, entryView } = record
+  const title = `${name} - ${i18next.t('title.data-entry-form')}`
+
+  switch (entryType) {
+    case 'generated':
+      openPopup(html` <data-entry-form .dataSet=${record} style="background-color: white;"></data-entry-form> `, {
+        closable: true,
+        backdrop: true,
+        size: 'large',
+        title
+      })
+      break
+
+    case 'board':
+      const board = {
+        id: entryView
+      }
+      openPopup(
+        html`
+          <ox-board-viewer
+            style="background-color: white;"
+            .board=${board}
+            .provider=${provider}
+            hide-fullscreen
+            hide-navigation
+          ></ox-board-viewer>
+        `,
+        {
+          closable: true,
+          backdrop: true,
+          size: 'large',
+          title
+        }
+      )
+
+      // navigate(`board-viewer/${entryView}?interactive=true&title=${title}`)
+      break
+
+    case 'page':
+      navigate(entryView)
+      break
+
+    case 'external':
+      window.open(entryView, '_blank')
+      break
+  }
+}
+
+const showMonitorView = (columns, data, column, record, rowIndex) => {
+  const { name, monitorType, monitorView } = record
+  const title = `${name} - ${i18next.t('title.data-entry-form')}`
+
+  switch (monitorType) {
+    case 'generated':
+      openPopup(html` <div style="background-color: white;">Under construction</div> `, {
+        backdrop: true,
+        size: 'large',
+        title
+      })
+      break
+
+    case 'board':
+      const board = {
+        id: monitorView
+      }
+      openPopup(
+        html`
+          <ox-board-viewer
+            style="background-color: white;"
+            .board=${board}
+            .provider=${provider}
+            hide-fullscreen
+            hide-navigation
+          ></ox-board-viewer>
+        `,
+        {
+          closable: true,
+          backdrop: true,
+          size: 'large',
+          title
+        }
+      )
+
+      // navigate(`board-viewer/${monitorView}?interactive=true&title=${title}`)
+      break
+
+    case 'page':
+      navigate(monitorView)
+      break
+
+    case 'external':
+      window.open(monitorView, '_blank')
+      break
+  }
+}
+
+export class DataEntryListPage extends connect(store)(localize(i18next)(PageView)) {
+  static get properties() {
+    return {
+      active: String,
+      gristConfig: Object,
+      filters: Object,
+      sorters: Object,
+      mode: String
+    }
+  }
+
+  static get styles() {
+    return [
+      ScrollbarStyles,
+      CommonGristStyles,
+      css`
+        :host {
+          display: flex;
+
+          width: 100%;
+
+          --grid-record-emphasized-background-color: red;
+          --grid-record-emphasized-color: yellow;
+        }
+      `
+    ]
+  }
+
+  get context() {
+    return {
+      title: i18next.t('title.data-entry list'),
+      help: 'dataset/data-entry-list'
+    }
+  }
+
+  render() {
+    const mode = 'CARD'
+
+    return html`
+      <ox-grist
+        .mode=${mode}
+        .config=${this.gristConfig}
+        .filters=${this.filters}
+        .orders=${this.orders}
+        .fetchHandler=${this.fetchHandler.bind(this)}
+      >
+        <div slot="headroom" id="filters">
+          <div id="filters">
+            <ox-filters-form></ox-filters-form>
+          </div>
+
+          <div id="sorters">
+            Sort
+            <mwc-icon
+              @click=${e => {
+                const target = e.currentTarget
+                this.renderRoot.querySelector('#sorter-control').open({
+                  right: 0,
+                  top: target.offsetTop + target.offsetHeight
+                })
+              }}
+              >expand_more</mwc-icon
+            >
+            <ox-popup id="sorter-control">
+              <ox-sorters-control> </ox-sorters-control>
+            </ox-popup>
+          </div>
+        </div>
+      </ox-grist>
+    `
+  }
+
+  get grist() {
+    return this.renderRoot.querySelector('ox-grist')
+  }
+
+  async pageInitialized(lifecycle) {
+    this.gristConfig = {
+      list: {
+        thumbnail: 'entryView',
+        fields: ['name', 'description'],
+        details: ['schedule', 'type', 'useCase', 'latestCollectedAt', 'prevSchedule', 'nextSchedule']
+      },
+      columns: [
+        {
+          type: 'gutter',
+          gutterName: 'button',
+          icon: 'fact_check',
+          handlers: {
+            click: showEntryView
+          }
+        },
+        {
+          type: 'gutter',
+          gutterName: 'button',
+          icon: 'analytics',
+          handlers: {
+            click: showMonitorView
+          }
+        },
+        {
+          type: 'string',
+          name: 'name',
+          header: i18next.t('field.name'),
+          record: {
+            editable: false
+          },
+          filter: 'search',
+          sortable: true,
+          width: 150
+        },
+        {
+          type: 'string',
+          name: 'description',
+          header: i18next.t('field.description'),
+          record: {
+            editable: false
+          },
+          filter: 'search',
+          width: 200
+        },
+        {
+          type: 'select',
+          name: 'type',
+          label: true,
+          header: i18next.t('field.type'),
+          record: {
+            editable: false,
+            options: [
+              {},
+              {
+                display: 'Manually Collected',
+                value: 'manual'
+              },
+              {
+                display: 'Automatically Collected',
+                value: 'automatic'
+              }
+            ]
+          },
+          sortable: true,
+          filter: true,
+          width: 60
+        },
+        {
+          type: 'select',
+          name: 'useCase',
+          label: true,
+          header: i18next.t('field.use-case'),
+          record: {
+            editable: false,
+            options: USECASE_OPTIONS
+          },
+          sortable: true,
+          filter: {
+            operator: 'eq',
+            options: USECASE_OPTIONS /* in case select options type is a function, filter should have its own options */
+          },
+          width: 80
+        },
+        {
+          type: 'crontab',
+          name: 'schedule',
+          label: true,
+          header: i18next.t('field.schedule'),
+          record: {
+            editable: false
+          },
+          width: 80,
+          label: true
+        },
+        {
+          type: 'resource-object',
+          name: 'supervisoryRole',
+          header: i18next.t('field.supervisory-role'),
+          record: {
+            editable: false
+          },
+          width: 120
+        },
+        {
+          type: 'datetime',
+          name: 'latestCollectedAt',
+          label: true,
+          header: i18next.t('field.latest-collected-at'),
+          record: {
+            editable: false
+          },
+          width: 180
+        },
+        {
+          type: 'datetime',
+          name: 'prevSchedule',
+          label: true,
+          header: i18next.t('field.prev-schedule'),
+          record: {
+            editable: false
+          },
+          width: 180
+        },
+        {
+          type: 'datetime',
+          name: 'nextSchedule',
+          label: true,
+          header: i18next.t('field.next-schedule'),
+          record: {
+            editable: false
+          },
+          width: 180
+        },
+        {
+          type: 'string',
+          name: 'entryView',
+          hidden: true,
+          record: {
+            editable: false,
+            renderer: function (value, column, record, rowIndex, field) {
+              const type = record.entryType !== 'board' ? 'string' : 'image'
+              value = record.entryType !== 'board' ? value : record.entryBoard?.thumbnail
+
+              return getRenderer(type)(value, column, record, rowIndex, field)
+            }
+          }
+        }
+      ],
+      rows: {
+        selectable: {
+          multiple: false
+        },
+        handlers: {
+          click: showEntryView
+        },
+        classifier: function (record, rowIndex) {}
+      },
+      sorters: [
+        {
+          name: 'name'
+        }
+      ]
+    }
+
+    await this.updateComplete
+
+    this.grist.fetch()
+  }
+
+  async pageUpdated(changes, lifecycle) {
+    if (this.active) {
+      await this.updateComplete
+
+      var filters = lifecycle.params?.['filters']
+      if (filters) {
+        try {
+          filters = JSON5.parse(filters)
+          this.filters = filters
+        } catch (e) {
+          console.error(`filters parameter parsing error: ${e}`)
+        }
+      }
+
+      var sorters = lifecycle.params?.['sorters']
+      if (sorters) {
+        try {
+          sorters = JSON5.parse(sorters)
+          this.sorters = sorters
+        } catch (e) {
+          console.error(`sorters parameter parsing error: ${e}`)
+        }
+      }
+
+      this.grist.fetch()
+    }
+  }
+
+  async fetchHandler({ page, limit, sortings = [], filters = [] }) {
+    const response = await client.query({
+      query: gql`
+        query ($filters: [Filter!], $pagination: Pagination, $sortings: [Sorting!]) {
+          responses: dataSetsForEntry(filters: $filters, pagination: $pagination, sortings: $sortings) {
+            items {
+              id
+              name
+              description
+              partitionKeys
+              active
+              type
+              useCase
+              schedule
+              timezone
+              supervisoryRole {
+                id
+                name
+              }
+              entryType
+              entryView
+              entryBoard {
+                thumbnail
+              }
+              monitorType
+              monitorView
+              updater {
+                id
+                name
+              }
+              updatedAt
+              dataItems {
+                name
+                description
+                sequence
+                active
+                tag
+                type
+                unit
+                options
+                quota
+                spec
+              }
+              latestCollectedAt
+              nextSchedule
+              prevSchedule
+            }
+            total
+          }
+        }
+      `,
+      variables: {
+        filters,
+        pagination: { page, limit },
+        sortings
+      }
+    })
+
+    return {
+      total: response.data.responses.total || 0,
+      records: response.data.responses.items || []
+    }
+  }
+}
+
+window.customElements.define('data-entry-list-page', DataEntryListPage)