@things-factory/dataset 5.0.14 → 6.0.0-alpha.0

package/README.md

@@ -1,6 +1,12 @@
 # dataset
 ## Architecture for data-samples
 ![Architecture for data-samples](./assets/data-samples.jpg)
+## data-report process
+![data-report process](./assets/data-report.jpg)
+
+'shiny' uses the next lambda. But 'jasper' is not using this lambda now. Consider using the same data service.
+- https://github.com/operatochef/serverless/tree/main/func-transform-data-samples
+
 ## Partition Keys
 At the early stage, partition keys are desinged for dynamic partitioning for Athena. But It will be required so many AWS Glue crawlers also by each 'data-sets'.
 
@@ -17,7 +23,21 @@ Now partition keys are fixed
 - workdate: working date
 - workshift
 
+### Glue Catalog Table Indices
+Partitions can be indices. You can create new indices with combination of partitions.
+It affects query performance and finding partitions.
+![partitions and indices](./assets/glue-table-indices.png)
+## Data Samples
+sample data could have an own timestamp if it is from a sensor data.
+'collected_at' uses this timestamp.
+Manual type of data is used 'Date.now() and new Date()'
+Graphql might affect timezone of server os for Date type field.
+'TZ=UTC node' can help to solve this when server starts.
+Or use ``` process.env.TZ = 'UTC' ```.
+Now applied in 'create-data-samples.ts'
+
 ## PostgreSQL query for dataset
+This is not using now.
 ```
 select 
 	data::json#>>'{product,0}' as product,

package/client/pages/data-key-set/data-key-set-importer.js

@@ -75,11 +75,9 @@ export class DataKeySetImporter extends LitElement {
       <ox-grist
         .mode=${isMobileDevice() ? 'LIST' : 'GRID'}
         .config=${this.columns}
-        .data=${
-          { 
-            records: this.dataKeySets 
-          }
-        }
+        .data=${{
+          records: this.dataKeySets
+        }}
       ></ox-grist>
 
       <div class="button-container">
@@ -98,7 +96,7 @@ export class DataKeySetImporter extends LitElement {
       variables: { dataKeySets: this.dataKeySets }
     })
 
-    if (response.error?.length) return
+    if (response.errors?.length) return
 
     this.dispatchEvent(new CustomEvent('imported'))
   }

package/client/pages/data-key-set/data-key-set-list-page.js

@@ -12,7 +12,40 @@ import { notify, openPopup } from '@operato/layout'
 import { navigate, PageView, store } from '@operato/shell'
 import { CommonButtonStyles, CommonGristStyles, ScrollbarStyles } from '@operato/styles'
 import { isMobileDevice } from '@operato/utils'
+import { encodeUrlParams } from '@things-factory/utils'
 
+import { getEditor, getRenderer } from '@operato/data-grist'
+
+const REPORT_TYPES = [
+  {
+    display: '',
+    value: ''
+  },
+  {
+    display: 'Generated',
+    value: 'generated'
+  },
+  {
+    display: 'Embed',
+    value: 'embed'
+  },
+  {
+    display: 'Page',
+    value: 'page'
+  },
+  {
+    display: 'External URL',
+    value: 'external'
+  },
+  {
+    display: 'Jasper',
+    value: 'jasper'
+  },
+  {
+    display: 'Shiny',
+    value: 'shiny'
+  }
+]
 export class DataKeySetListPage extends connect(store)(localize(i18next)(PageView)) {
   static get properties() {
     return {
@@ -136,13 +169,45 @@ export class DataKeySetListPage extends connect(store)(localize(i18next)(PageVie
         {
           type: 'gutter',
           gutterName: 'button',
-          icon: 'newspaper',
+          icon: 'checklist',
           handlers: {
             click: (columns, data, column, record, rowIndex) => {
               navigate(`data-sample-search/${record.id}`)
             }
           }
         },
+        {
+          type: 'gutter',
+          gutterName: 'button',
+          icon: 'newspaper',
+          handlers: {
+            click: (columns, data, column, record, rowIndex) => {
+              try {
+                const { id:dataKeySetId, reportType, reportView } = record
+                switch (reportType) {
+                  case 'generated':
+                    break
+                  case 'page':
+                    break
+                  case 'embed':
+                    break
+                  case 'external':
+                    reportView && window.open(reportView, '_blank')
+                    break
+                  case 'jasper':
+                  case 'shiny':
+                    const urlParams = encodeUrlParams({ dataKeySetId, reportType, reportView })
+                    navigate(`data-report-samples/?${urlParams}`)
+                    break
+                  default:
+                    break
+                }
+              } catch(ex) {
+                console.log(ex)
+              }
+            }
+          }
+        },
         {
           type: 'string',
           name: 'name',
@@ -176,6 +241,34 @@ export class DataKeySetListPage extends connect(store)(localize(i18next)(PageVie
           sortable: true,
           width: 60
         },
+        {
+          type: 'select',
+          name: 'reportType',
+          label: true,
+          header: i18next.t('field.report-type'),
+          record: {
+            editable: true,
+            options: REPORT_TYPES
+          },
+          width: 80
+        },
+        {
+          type: 'string',
+          name: 'reportView',
+          header: i18next.t('field.report-view'),
+          record: {
+            editable: true,
+            editor: function (value, column, record, rowIndex, field) {
+              var type = record.reportType !== 'custom' ? 'string' : 'script'
+              return getEditor(type)(value, column, record, rowIndex, field)
+            },
+            renderer: function (value, column, record, rowIndex, field) {
+              var type = record.reportType !== 'custom' ? 'string' : 'string'
+              return getRenderer(type)(value, column, record, rowIndex, field)
+            }
+          },
+          width: 140
+        },
         {
           type: 'resource-object',
           name: 'updater',
@@ -226,6 +319,8 @@ export class DataKeySetListPage extends connect(store)(localize(i18next)(PageVie
               name
               description
               active
+              reportType
+              reportView
               dataKeyItems {
                 name
                 description
@@ -315,7 +410,7 @@ export class DataKeySetListPage extends connect(store)(localize(i18next)(PageVie
 
   async exportHandler() {
     const exportTargets = this.grist.selected.length ? this.grist.selected : this.grist.dirtyData.records
-    const targetFieldSet = new Set(['id', 'name', 'description', 'active'])
+    const targetFieldSet = new Set(['id', 'name', 'description', 'active', 'reportType', 'reportView'])
 
     return exportTargets.map(dataKeySet => {
       let tempObj = {}

package/client/pages/data-report/data-report-embed-page.js

@@ -1,3 +1,7 @@
+/**
+ * report page from 'data-set-list-page'
+ * datasetId is required
+ */
 import '@operato/data-grist'
 import '@things-factory/form-ui'
 
@@ -6,7 +10,7 @@ import { css, html } from 'lit'
 import { i18next, localize } from '@operato/i18n'
 import { CommonButtonStyles, ScrollbarStyles } from '@operato/styles'
 import { PageView } from '@things-factory/shell'
-import { encodeFormParams, getCookie } from '@things-factory/utils'
+import { encodeUrlParams, getCookie } from '@things-factory/utils'
 
 class DataReportEmbedPage extends localize(i18next)(PageView) {
   static get properties() {
@@ -57,7 +61,7 @@ class DataReportEmbedPage extends localize(i18next)(PageView) {
       }
     ]
     return {
-      title: 'jasper-report',
+      title: 'data-report-embed',
       actions,
       filters
     }
@@ -68,7 +72,7 @@ class DataReportEmbedPage extends localize(i18next)(PageView) {
         .filters=${this.context.filters}
         @filters-change=${e => {
           console.log('filters changed', e.detail)
-          this._reportTemplate(e.detail.filters)
+          this.reportTemplate(e.detail.filters)
         }}
         ?url-params-sensitive=${this.active}
       ></ox-filters-form-base>
@@ -82,12 +86,12 @@ class DataReportEmbedPage extends localize(i18next)(PageView) {
     }
   }
 
-  async _reportTemplate(filters) {
+  async reportTemplate(filters) {
     // convert filters array to object
     const params = filters.reduce((acc, curr) => ((acc[curr.name] = curr.value), acc), {})
     Object.assign(params, this.lifecycle.params)
 
-    const { workDateRange, workShift, reportView } = params
+    const { id, workDateRange, workShift, reportType, reportView } = params
 
     /** ignoring date conditions  */
     if (!workDateRange[0] || !workDateRange[1] || workDateRange[0] > workDateRange[1]) {
@@ -97,16 +101,16 @@ class DataReportEmbedPage extends localize(i18next)(PageView) {
     /** urlencoded params including test values */
     const urlParams = {
       table: 'samples',
-      dataSetId: this.lifecycle.resourceId,
-      fromWorkDate: workDateRange[0],
-      toWorkDate: workDateRange[1],
-      token: getCookie('access_token'),
-      workShift
+      dataSetId: id,
+      startDate: workDateRange[0],
+      endDate: workDateRange[1],
+      workShift,
+      reportView
     }
 
-    const encodedUrlParams = encodeFormParams(urlParams)
+    const encodedUrlParams = encodeUrlParams(urlParams)
 
-    this.shadowRoot.querySelector('#container').src = `${reportView}?params=${encodedUrlParams}`
+    this.shadowRoot.querySelector('#container').src = `/data-report/${reportType}/?${encodedUrlParams}`
   }
 }
 

package/client/pages/data-report/data-report-list-page.js

@@ -13,6 +13,7 @@ import { openPopup } from '@operato/layout'
 import { navigate, PageView, store } from '@operato/shell'
 import { CommonGristStyles, ScrollbarStyles } from '@operato/styles'
 import { provider } from '@things-factory/board-ui'
+import { encodeUrlParams } from '@things-factory/utils'
 
 const USECASE_OPTIONS = () => {
   return ['', ...OxDataUseCase.getUseCaseNames()].map(name => {
@@ -72,7 +73,8 @@ const showMonitorView = (columns, data, column, record, rowIndex) => {
 }
 
 const showReportView = (columns, data, column, record, rowIndex) => {
-  const { id, name, reportType, reportView, reportTemplate } = record
+  const { id:datasetId, dataKeySet, name, reportType, reportView, reportTemplate } = record
+  const { id:dataKeySetId } = dataKeySet || { id: '' }
   const title = `${name} - ${i18next.t('title.data-report-view')}`
 
   switch (reportType) {
@@ -85,26 +87,23 @@ const showReportView = (columns, data, column, record, rowIndex) => {
       break
 
     case 'embed':
-      const board = {
-        id: reportView
-      }
-      navigate(`/data-report-embed/${id}?reportView=${encodeURIComponent(reportView)}`)
-      // openPopup(html` <div style="background-color: white;">Under construction</div> `, {
-      //   closable: true,
-      //   backdrop: true,
-      //   size: 'large',
-      //   title
-      // })
-
       break
 
     case 'page':
-      navigate(reportView + `/${id}?template=${encodeURIComponent(reportTemplate)}`)
       break
 
     case 'external':
       window.open(reportView, '_blank')
       break
+    
+    case 'jasper':
+    case 'shiny':
+      const encodedUrlParams = encodeUrlParams({ datasetId, dataKeySetId, reportType, reportView, reportTemplate })
+      navigate(`/data-report-samples/?${encodedUrlParams}`)
+      break
+    
+    default:
+      break
   }
 }
 
@@ -366,6 +365,10 @@ export class DataReportListPage extends connect(store)(localize(i18next)(PageVie
           responses: dataSetsForReport(filters: $filters, pagination: $pagination, sortings: $sortings) {
             items {
               id
+              dataKeySet {
+                id
+                name
+              }
               name
               description
               partitionKeys

package/client/pages/data-report/data-report-samples-page.js

@@ -0,0 +1,186 @@
+/**
+ * data sample report page from 'data-key-set-list-page'
+ * dateKeySetId is required
+ */
+import '@operato/data-grist'
+
+import gql from 'graphql-tag'
+import { css, html } from 'lit'
+import { connect } from 'pwa-helpers/connect-mixin'
+
+import { client } from '@operato/graphql'
+import { i18next, localize } from '@operato/i18n'
+import { PageView, store } from '@operato/shell'
+import { ScrollbarStyles } from '@operato/styles'
+
+import { encodeUrlParams } from '@things-factory/utils'
+
+export class DataReportSamplesPage extends connect(store)(localize(i18next)(PageView)) {
+  static get properties() {
+    return {
+      dataKeySetId: String,
+      dataKeySet: Object,
+      active: String
+    }
+  }
+
+  static get styles() {
+    return [
+      ScrollbarStyles,
+      css`
+        :host {
+          display: flex;
+          flex-direction: column;
+          padding: 0;
+        }
+
+        #container {
+          flex: 1;
+          padding: 0;
+          margin: 0;
+          border: 0;
+        }
+      `
+    ]
+  }
+
+  get context() {
+
+    const today = new Date().toISOString().split('T')[0]
+
+    const filters = [
+      {
+        name: 'workDateRange',
+        type: 'date',
+        label: i18next.t('field.work-date'),
+        operator: 'between',
+        value: [today, today] // not working...
+      },
+      ...this.getDataKeyFilters()
+    ]
+
+    return {
+      title: i18next.t('title.data-report samples'),
+      help: 'dataset/data-report-samples',
+      filters
+    }
+  }
+
+  getDataKeyFilters() {
+    return (
+      this.dataKeySet?.dataKeyItems.map((item, index) => {
+        return {
+          type: 'string',
+          name: `key_0${index + 1}`,
+          label: i18next.t(item.tKey),
+          operator: 'i_like',
+          width: 120
+        }
+      }) || []
+    )
+  }
+
+  render() {
+    // clear if pre-loaded
+    const container = this.shadowRoot.querySelector('#container')
+    if (container) {
+      container.src = ""
+      // @TODO: Clear filters
+      /** */
+    }
+
+    return html`
+      <ox-filters-form-base
+        .filters=${this.context.filters}
+        @filters-change=${e => {
+          console.log('filters changed', e.detail)
+          this.renderTemplate(e.detail.filters)
+        }}
+        ?url-params-sensitive=${this.active}
+      ></ox-filters-form-base>
+      <iframe id="container"></iframe>
+    `
+  }
+
+  renderTemplate(filters) {
+    const params = filters.reduce((acc, curr) => ((acc[curr.name] = curr.value), acc), {})
+    Object.assign(params, this.lifecycle.params)
+
+    const filterNames = this.getDataKeyFilters().map(x => x.name)
+    const dataKeyFilters = filters.filter(x => filterNames.includes(x.name))
+    const { workDateRange } = params
+
+    /** ignoring date conditions  */
+    if (!workDateRange[0] || !workDateRange[1] || workDateRange[0] > workDateRange[1]) {
+      return
+    }
+
+    /** urlencoded params including test values */
+    const urlParams = {
+      reportType: params.reportType,
+      reportView: params.reportView,
+      reportTemplate: params.reportTemplate,
+      table: 'samples',
+      datasetId: params.datasetId,
+      dataKeySetId: params.dataKeySetId,
+      startDate: workDateRange[0],
+      endDate: workDateRange[1],
+      dataKeySetItems: JSON.stringify(this.dataKeySet?.dataKeyItems || []),
+      dataKeyFilters: JSON.stringify(dataKeyFilters)
+    }
+
+    const encodedUrlParams = encodeUrlParams(urlParams)
+
+    const reportUrl = `/data-report/${urlParams.reportType}/`
+    this.shadowRoot.querySelector('#container').src = `${reportUrl}?${encodedUrlParams}`
+  }
+
+  pageUpdated(changes, lifecycle) {
+    if (this.active) {
+      const { dataKeySetId } = lifecycle.params
+      this.dataKeySetId = dataKeySetId
+      return
+    }
+  }
+
+  async fetchHandler({ page, limit, sortings = [], filters = [] }) {
+
+  }
+
+  async updated(changes) {
+    if (changes.has('dataKeySetId')) {
+      if (!this.dataKeySetId) {
+        this.dataKeySet  = null
+        return
+      }
+
+      const response = await client.query({
+        query: gql`
+          query ($id: String!) {
+            dataKeySet(id: $id) {
+              id
+              name
+              description
+              reportType
+              reportView
+              dataKeyItems {
+                name
+                description
+                dataKey
+                tKey
+              }
+            }
+          }
+        `,
+        variables: {
+          id: this.dataKeySetId
+        }
+      })
+
+      this.dataKeySet = response.data.dataKeySet
+    }
+  }
+
+}
+
+window.customElements.define('data-report-samples-page', DataReportSamplesPage)

package/client/pages/data-set/data-set-importer.js

@@ -94,7 +94,7 @@ export class DataSetImporter extends LitElement {
       variables: { dataSets: this.dataSets }
     })
 
-    if (response.error?.length) return
+    if (response.errors?.length) return
 
     this.dispatchEvent(new CustomEvent('imported'))
   }

package/client/pages/data-set/data-set-list-page.js

@@ -86,6 +86,14 @@ const REPORT_TYPES = [
   {
     display: 'External URL',
     value: 'external'
+  },
+  {
+    display: 'Jasper',
+    value: 'jasper'
+  },
+  {
+    display: 'Shiny',
+    value: 'shiny'
   }
 ]
 

package/client/route.js

@@ -16,6 +16,10 @@ export default function route(page) {
       import('./pages/data-sample/data-sample-search-page.js')
       return page
 
+    case 'data-report-samples':
+      import('./pages/data-report/data-report-samples-page.js')
+      return page
+
     case 'data-ooc-list':
       import('./pages/data-ooc/data-ooc-list-page.js')
       return page
@@ -28,21 +32,21 @@ export default function route(page) {
       import('./pages/data-report/data-report-list-page.js')
       return page
 
-    case 'jasper-report-samples':
-      import('./pages/data-report/jasper-report-samples-page.js')
-      return page
+    // case 'jasper-report-samples':
+    //   import('./pages/data-report/jasper-report-samples-page.js')
+    //   return page
 
-    case 'jasper-report-samples-crosstab':
-      import('./pages/data-report/jasper-report-samples-crosstab-page.js')
-      return page
+    // case 'jasper-report-samples-crosstab':
+    //   import('./pages/data-report/jasper-report-samples-crosstab-page.js')
+    //   return page
 
-    case 'jasper-report-oocs':
-      import('./pages/data-report/jasper-report-oocs-page.js')
-      return page
+    // case 'jasper-report-oocs':
+    //   import('./pages/data-report/jasper-report-oocs-page.js')
+    //   return page
 
-    case 'data-report-embed':
-      import('./pages/data-report/data-report-embed-page')
-      return page
+    // case 'data-report-embed':
+    //   import('./pages/data-report/data-report-embed-page')
+    //   return page
 
     case 'data-key-set-list':
       import('./pages/data-key-set/data-key-set-list-page.js')

package/config/config.development.js

@@ -5,6 +5,16 @@ module.exports = {
                 host: 'localhost',
                 port: 8090
             },
+            datasource: {
+                database: ''
+            },
+            method: 'POST'
+        },
+        shiny: {
+            endpoint: {
+                host: 'localhost',
+                port: 3838
+            },
             datasource: {
                 database: ''
             }

package/config/config.production.js

@@ -1,13 +1,23 @@
 module.exports = {
-  dataReport: {
-      jasper: {
-          endpoint: {
-              host: 'localhost',
-              port: 8090
-          },
-          datasource: {
-              database: ''
-          }
-      }
-  }
+    dataReport: {
+        jasper: {
+            endpoint: {
+                host: 'localhost',
+                port: 8090
+            },
+            datasource: {
+                database: ''
+            },
+            method: 'POST'
+        },
+        shiny: {
+            endpoint: {
+                host: 'localhost',
+                port: 3838
+            },
+            datasource: {
+                database: ''
+            }
+        }
+    }
 }

package/dist-server/controllers/create-data-sample.js

@@ -9,6 +9,8 @@ const data_ooc_1 = require("../service/data-ooc/data-ooc");
 const data_sample_1 = require("../service/data-sample/data-sample");
 const data_set_1 = require("../service/data-set/data-set");
 const data_use_case_1 = require("./data-use-case");
+// See README.md at ## Data Samples
+process.env.TZ = 'UTC';
 const fillDataKeys = (dataKeySet, data) => {
     const keys = (dataKeySet === null || dataKeySet === void 0 ? void 0 : dataKeySet.dataKeyItems) || [];
     return keys.reduce((sum, key, index) => {
@@ -68,7 +70,7 @@ async function createDataSample(dataSample, context) {
     partitionKeys = replaceVariables(partitionKeys, dataSample.data);
     const dataKeys = fillDataKeys(dataSet === null || dataSet === void 0 ? void 0 : dataSet.dataKeySet, dataSample.data);
     const { ooc, oos, judgment } = data_use_case_1.DataUseCase.evaluate(dataSet, dataItems, dataSample.data) || {};
-    const result = await tx.getRepository(data_sample_1.DataSample).save(Object.assign(Object.assign(Object.assign({ name: dataSet.name, description: dataSet.description, useCase: dataSet.useCase }, dataSample), dataKeys), { dataSetVersion: dataSet.version, domain,
+    const result = await tx.getRepository(data_sample_1.DataSample).save(Object.assign(Object.assign(Object.assign({ name: dataSet.name, description: dataSet.description, useCase: dataSet.useCase, type: dataSet.type }, dataSample), dataKeys), { dataSetVersion: dataSet.version, domain,
         partitionKeys,
         ooc,
         oos,