@things-factory/dataset 5.0.14 → 5.0.15

package/README.md

@@ -1,6 +1,12 @@
 # dataset
 ## Architecture for data-samples
 ![Architecture for data-samples](./assets/data-samples.jpg)
+## data-report process
+![data-report process](./assets/data-report.jpg)
+
+'shiny' uses the next lambda. But 'jasper' is not using this lambda now. Consider using the same data service.
+- https://github.com/operatochef/serverless/tree/main/func-transform-data-samples
+
 ## Partition Keys
 At the early stage, partition keys are desinged for dynamic partitioning for Athena. But It will be required so many AWS Glue crawlers also by each 'data-sets'.
 
@@ -17,7 +23,21 @@ Now partition keys are fixed
 - workdate: working date
 - workshift
 
+### Glue Catalog Table Indices
+Partitions can be indices. You can create new indices with combination of partitions.
+It affects query performance and finding partitions.
+![partitions and indices](./assets/glue-table-indices.png)
+## Data Samples
+sample data could have an own timestamp if it is from a sensor data.
+'collected_at' uses this timestamp.
+Manual type of data is used 'Date.now() and new Date()'
+Graphql might affect timezone of server os for Date type field.
+'TZ=UTC node' can help to solve this when server starts.
+Or use ``` process.env.TZ = 'UTC' ```.
+Now applied in 'create-data-samples.ts'
+
 ## PostgreSQL query for dataset
+This is not using now.
 ```
 select 
 	data::json#>>'{product,0}' as product,

package/client/pages/data-archive/data-archive-list-page.js

@@ -0,0 +1,290 @@
+import '@operato/data-grist'
+
+import gql from 'graphql-tag'
+import { css, html } from 'lit'
+import { connect } from 'pwa-helpers/connect-mixin'
+
+import { client } from '@operato/graphql'
+import { i18next, localize } from '@operato/i18n'
+import { openPopup } from '@operato/layout'
+import { PageView, store } from '@operato/shell'
+import { CommonGristStyles, ScrollbarStyles } from '@operato/styles'
+import { isMobileDevice } from '@operato/utils'
+
+import './data-archive-request-popup'
+
+export class DataArchiveListPage extends connect(store)(localize(i18next)(PageView)) {
+  static get properties() {
+    return {
+      active: String,
+      gristConfig: Object,
+      mode: String
+    }
+  }
+
+  static get styles() {
+    return [
+      ScrollbarStyles,
+      CommonGristStyles,
+      css`
+        :host {
+          display: flex;
+          flex-direction: column;
+
+          overflow: hidden;
+        }
+
+        ox-grist {
+          overflow-y: auto;
+          flex: 1;
+        }
+      `
+    ]
+  }
+
+  get context() {
+    return {
+      title: i18next.t('title.data-archive list'),
+      help: 'dataset/data-archive',
+      actions: [],
+      exportable: {
+        name: i18next.t('title.data-archive list'),
+        data: this._exportableData.bind(this)
+      }
+    }
+  }
+
+  render() {
+    const mode = this.mode || (isMobileDevice() ? 'LIST' : 'GRID')
+
+    return html`
+      <ox-grist
+        .mode=${mode}
+        .config=${this.gristConfig}
+        .fetchHandler=${this.fetchHandler.bind(this)}
+        ?url-params-sensitive=${false /* this.active */}
+      >
+        <div slot="headroom">
+          <div id="filters">
+            <ox-filters-form autofocus></ox-filters-form>
+          </div>
+
+          <div id="sorters">
+            Sort
+            <mwc-icon
+              @click=${e => {
+                const target = e.currentTarget
+                this.renderRoot.querySelector('#sorter-control').open({
+                  right: 0,
+                  top: target.offsetTop + target.offsetHeight
+                })
+              }}
+              >expand_more</mwc-icon
+            >
+            <ox-popup id="sorter-control">
+              <ox-sorters-control> </ox-sorters-control>
+            </ox-popup>
+          </div>
+
+          <div id="modes">
+            <mwc-icon @click=${() => (this.mode = 'GRID')} ?active=${mode == 'GRID'}>grid_on</mwc-icon>
+            <mwc-icon @click=${() => (this.mode = 'LIST')} ?active=${mode == 'LIST'}>format_list_bulleted</mwc-icon>
+            <mwc-icon @click=${() => (this.mode = 'CARD')} ?active=${mode == 'CARD'}>apps</mwc-icon>
+          </div>
+
+          <mwc-button
+            dense
+            raised
+            label=${i18next.t('button.request')}
+            @click=${this.openArchivePopup.bind(this)}
+            icon="archive"
+          ></mwc-button>
+        </div>
+      </ox-grist>
+    `
+  }
+
+  get grist() {
+    return this.renderRoot.querySelector('ox-grist')
+  }
+
+  async pageInitialized(lifecycle) {
+    const today = new Date().toISOString().split('T')[0]
+
+    this.gristConfig = {
+      list: { fields: ['updater', 'updatedAt'] },
+      columns: [
+        { type: 'gutter', gutterName: 'sequence' },
+        // { type: 'gutter', gutterName: 'row-selector', multiple: true },
+        {
+          type: 'resource-object',
+          name: 'creator',
+          header: i18next.t('field.creator'),
+          sortable: true,
+          width: 120,
+          imex: true
+        },
+        {
+          type: 'datetime',
+          name: 'createdAt',
+          header: i18next.t('field.created_at'),
+          sortable: true,
+          width: 180,
+          imex: true
+        },
+        {
+          type: 'datetime',
+          name: 'updatedAt',
+          header: i18next.t('field.updated_at'),
+          sortable: true,
+          width: 180,
+          imex: true
+        },
+        {
+          type: 'json5',
+          name: 'requestParams',
+          header: i18next.t('field.request-params'),
+          record: {
+            editable: false
+          },
+          width: 200,
+          imex: true
+        },
+        {
+          type: 'string',
+          name: 'downloadUrl',
+          label: true,
+          header: i18next.t('field.download-url'),
+          record: {
+            editable: false
+          },
+          width: 240,
+          imex: true
+        },
+        {
+          type: 'string',
+          name: 'status',
+          label: true,
+          header: i18next.t('field.status'),
+          record: {
+            editable: false
+          },
+          sortable: true,
+          width: 120,
+          imex: true
+        }
+      ],
+      rows: {
+        appendable: false,
+        selectable: {
+          multiple: true
+        }
+      },
+      sorters: [
+        {
+          name: 'createdAt',
+          desc: true
+        }
+      ]
+    }
+  }
+
+  closePopupAndRefesh() {
+    this.popup && this.popup.close()
+    this.grist.fetch()
+  }
+
+  async openArchivePopup() {
+    this.popup = openPopup(
+      html`
+      <data-archive-request-popup
+        @requested="${this.closePopupAndRefesh.bind(this)}"
+        @created="${this.closePopupAndRefesh.bind(this)}"
+      ></data-archive-request-popup>`, 
+      {
+        backdrop: true,
+        size: 'small',
+        title: i18next.t('title.data-archive request popup')
+      })
+  }
+
+  async fetchHandler({ page, limit, sortings = [], filters = [] }) {
+    const response = await client.query({
+      query: gql`
+        query ($filters: [Filter!], $pagination: Pagination, $sortings: [Sorting!]) {
+          responses: dataArchives(filters: $filters, pagination: $pagination, sortings: $sortings) {
+            items {
+              id
+              type
+              requestParams
+              downloadUrl
+              status
+              creator {
+                id
+                name
+              }
+              updater {
+                id
+                name
+              }
+              updatedAt
+              createdAt
+            }
+            total
+          }
+        }
+      `,
+      variables: {
+        filters,
+        pagination: { page, limit },
+        sortings
+      }
+    })
+
+    return {
+      total: response.data.responses.total || 0,
+      records: response.data.responses.items || []
+    }
+  }
+
+  _exportableData() {
+    let records = []
+    if (this.grist.selected && this.grist.selected.length > 0) {
+      records = this.grist.selected
+    } else {
+      records = this.grist.data.records
+    }
+
+    var headerSetting = this.grist.compiledConfig.columns
+      .filter(column => column.type !== 'gutter' && column.record !== undefined && column.imex !== undefined)
+      .map(column => {
+        return column.imex === true
+          ? {
+              header: column.header.renderer(),
+              key: column.name,
+              width: column.width,
+              type: column.type
+            }
+          : column.imex
+      })
+
+    var data = records.map(item => {
+      return {
+        id: item.id,
+        ...this.gristConfig.columns
+          .filter(column => column.type !== 'gutter' && column.record !== undefined && column.imex !== undefined)
+          .reduce((record, column) => {
+            const key = column.imex === true ? column.name : column.imex.key
+            record[key] = key
+              .split('.')
+              .reduce((obj, key) => (obj && obj[key] !== 'undefined' ? obj[key] : undefined), item)
+            return record
+          }, {})
+      }
+    })
+
+    return { header: headerSetting, data: data }
+  }
+}
+
+window.customElements.define('data-archive-list-page', DataArchiveListPage)

package/client/pages/data-archive/data-archive-request-popup.js

@@ -0,0 +1,198 @@
+import { SingleColumnFormStyles } from '@things-factory/form-ui'
+import '@things-factory/grist-ui'
+import { i18next, localize } from '@things-factory/i18n-base'
+import { client, CustomAlert } from '@things-factory/shell'
+import gql from 'graphql-tag'
+import { css, html, LitElement } from 'lit-element'
+import { notify } from '@things-factory/notification'
+import moment from 'moment-timezone'
+
+class DataArchiveRequestPopup extends localize(i18next)(LitElement) {
+  static get properties() {
+    return {
+      dataSetTypes: Array
+    }
+  }
+
+  static get styles() {
+    return [
+      SingleColumnFormStyles,
+      css`
+        :host {
+          padding: 10px;
+          display: flex;
+          flex-direction: column;
+          overflow-x: overlay;
+          background-color: var(--main-section-background-color);
+        }
+        .button-container {
+          padding: var(--button-container-padding);
+          margin: var(--button-container-margin);
+          text-align: var(--button-container-align);
+          background-color: var(--button-container-background);
+          height: var(--button-container-height);
+        }
+      `
+    ]
+  }
+
+  render() {
+    return html`
+      <form id="input-form" name="generation" class="single-column-form" @submit=${e => this.onSubmit(e)}>
+        <fieldset>
+          <label>${i18next.t('label.start-date')}</label>
+          <input type="month" name="startDate" required/>
+          <label>${i18next.t('label.end-date')}</label>
+          <input type="month" name="endDate" .value=${moment().format('YYYY-MM')}/>
+          <!-- not implemented on server side, yet -->
+          <!-- 
+          <label>${i18next.t('label.data-set-type')}</label>
+          <select name="type" required>
+            ${(this.dataSetTypes || []).map(
+              t => html`<option value="${t && t.value}">${t && t.display}</option>`
+            )}
+          </select> 
+          -->
+        </fieldset>
+      </form>
+
+      <div class="button-container">
+        <mwc-button raised @click="${this.requestArchive}" label="${i18next.t('button.submit')}"></mwc-button>
+      </div>
+    `
+  }
+
+  async firstUpdated() {
+    
+  }
+
+  async onSubmit(e) {
+    e.preventDefault()
+    this.requestArchive()
+  }
+
+
+  serializeFormData() {
+    const obj = {}
+    
+    Array.from(this.shadowRoot.querySelectorAll('form#input-form input')).forEach(field => {
+      if (!field.hasAttribute('hidden') && field.value) {
+        obj[field.name] = field.type === 'checkbox' ? field.checked : field.value
+      }
+    })
+
+    if (obj['startDate']) {
+      obj['startDate'] += '-01'
+    }
+
+    if (obj['endDate']) {
+      const endDate = moment(obj['endDate']).endOf('month')
+      const today = moment()
+      const format = 'YYYY-MM-DD'
+
+      obj['endDate'] = endDate > today ? today.format(format) : endDate.format(format)
+    }
+
+    return obj
+  }
+
+  /** request download url. */
+  async requestArchive() {
+    // #1 request download url
+    const dataArchive = {
+      requestParams: this.serializeFormData(),
+      status: 'requested'
+    }
+
+    const response = await client.mutate({
+      mutation: gql`
+        mutation createDataArchive($dataArchive: NewDataArchive!) {
+          createDataArchive(dataArchive: $dataArchive) {
+            id
+            status
+          }
+        }
+      `,
+      variables: { dataArchive }
+    })
+
+    if (!response.errors) {
+      this.dispatchEvent(new CustomEvent('requested', {}))
+
+      const { createDataArchive: { id } } = response.data
+      dataArchive['id'] = id
+
+      this._generateArchiveAndDownloadUrl(dataArchive)
+
+      await CustomAlert({
+        type: 'info',
+        title: i18next.t('title.ready'),
+        text: i18next.t('text.data-archive waits'),
+        confirmButton: { text: i18next.t('button.confirm') }
+      })
+      
+    } else {
+      console.error(response.errors)
+      this.showToast(i18next.t('text.failed'))
+    }
+  }
+
+  async _generateArchiveAndDownloadUrl(dataArchive) {
+    const response = await client.mutate({
+      mutation: gql`
+        mutation generatePresignedUrl($patch: DataArchivePatch!) {
+          generatePresignedUrl(patch: $patch) {
+            id
+            downloadUrl
+            status
+          }
+        }
+      `,
+      variables: { patch: dataArchive }
+    })
+
+    if (!response.errors) {
+
+      const  { generatePresignedUrl: { downloadUrl } } = response.data
+
+      this.showToast(i18next.t('title.data-archive downloads ready'))
+
+      notify({
+        receivers: '',
+        title: i18next.t('title.data-archive downloads ready'),
+        body: '',
+        image: '',
+        mode: 'inapp',
+        url: downloadUrl
+      })
+
+      this.dispatchEvent(new CustomEvent('created', {}))
+      
+    } else {
+      console.error(response.errors)
+      this.showToast(i18next.t('text.failed'))
+    }
+  }
+
+  showToast(message) {
+    document.dispatchEvent(new CustomEvent('notify', { detail: { message } }))
+  }
+
+  constructor() {
+    super()
+
+    this.dataSetTypes = [
+      {},
+      {
+        display: i18next.t('text.manually collected'),
+        value: 'manual'
+      },
+      {
+        display: i18next.t('text.automatically collected'),
+        value: 'automatic'
+      }
+    ]
+  }
+}
+
+window.customElements.define('data-archive-request-popup', DataArchiveRequestPopup)

package/client/pages/data-key-set/data-key-set-list-page.js

@@ -12,7 +12,40 @@ import { notify, openPopup } from '@operato/layout'
 import { navigate, PageView, store } from '@operato/shell'
 import { CommonButtonStyles, CommonGristStyles, ScrollbarStyles } from '@operato/styles'
 import { isMobileDevice } from '@operato/utils'
+import { encodeUrlParams } from '@things-factory/utils'
 
+import { getEditor, getRenderer } from '@operato/data-grist'
+
+const REPORT_TYPES = [
+  {
+    display: '',
+    value: ''
+  },
+  {
+    display: 'Generated',
+    value: 'generated'
+  },
+  {
+    display: 'Embed',
+    value: 'embed'
+  },
+  {
+    display: 'Page',
+    value: 'page'
+  },
+  {
+    display: 'External URL',
+    value: 'external'
+  },
+  {
+    display: 'Jasper',
+    value: 'jasper'
+  },
+  {
+    display: 'Shiny',
+    value: 'shiny'
+  }
+]
 export class DataKeySetListPage extends connect(store)(localize(i18next)(PageView)) {
   static get properties() {
     return {
@@ -136,13 +169,45 @@ export class DataKeySetListPage extends connect(store)(localize(i18next)(PageVie
         {
           type: 'gutter',
           gutterName: 'button',
-          icon: 'newspaper',
+          icon: 'checklist',
           handlers: {
             click: (columns, data, column, record, rowIndex) => {
               navigate(`data-sample-search/${record.id}`)
             }
           }
         },
+        {
+          type: 'gutter',
+          gutterName: 'button',
+          icon: 'newspaper',
+          handlers: {
+            click: (columns, data, column, record, rowIndex) => {
+              try {
+                const { id:dataKeySetId, reportType, reportView } = record
+                switch (reportType) {
+                  case 'generated':
+                    break
+                  case 'page':
+                    break
+                  case 'embed':
+                    break
+                  case 'external':
+                    reportView && window.open(reportView, '_blank')
+                    break
+                  case 'jasper':
+                  case 'shiny':
+                    const urlParams = encodeUrlParams({ dataKeySetId, reportType, reportView })
+                    navigate(`data-report-samples/?${urlParams}`)
+                    break
+                  default:
+                    break
+                }
+              } catch(ex) {
+                console.log(ex)
+              }
+            }
+          }
+        },
         {
           type: 'string',
           name: 'name',
@@ -176,6 +241,34 @@ export class DataKeySetListPage extends connect(store)(localize(i18next)(PageVie
           sortable: true,
           width: 60
         },
+        {
+          type: 'select',
+          name: 'reportType',
+          label: true,
+          header: i18next.t('field.report-type'),
+          record: {
+            editable: true,
+            options: REPORT_TYPES
+          },
+          width: 80
+        },
+        {
+          type: 'string',
+          name: 'reportView',
+          header: i18next.t('field.report-view'),
+          record: {
+            editable: true,
+            editor: function (value, column, record, rowIndex, field) {
+              var type = record.reportType !== 'custom' ? 'string' : 'script'
+              return getEditor(type)(value, column, record, rowIndex, field)
+            },
+            renderer: function (value, column, record, rowIndex, field) {
+              var type = record.reportType !== 'custom' ? 'string' : 'string'
+              return getRenderer(type)(value, column, record, rowIndex, field)
+            }
+          },
+          width: 140
+        },
         {
           type: 'resource-object',
           name: 'updater',
@@ -226,6 +319,8 @@ export class DataKeySetListPage extends connect(store)(localize(i18next)(PageVie
               name
               description
               active
+              reportType
+              reportView
               dataKeyItems {
                 name
                 description
@@ -315,7 +410,7 @@ export class DataKeySetListPage extends connect(store)(localize(i18next)(PageVie
 
   async exportHandler() {
     const exportTargets = this.grist.selected.length ? this.grist.selected : this.grist.dirtyData.records
-    const targetFieldSet = new Set(['id', 'name', 'description', 'active'])
+    const targetFieldSet = new Set(['id', 'name', 'description', 'active', 'reportType', 'reportView'])
 
     return exportTargets.map(dataKeySet => {
       let tempObj = {}