@things-factory/dataset 5.0.0-zeta.7 → 5.0.0

package/server/controllers/create-data-sample.ts

@@ -3,15 +3,13 @@ import { EntityManager } from 'typeorm'
 
 import { User } from '@things-factory/auth-base'
 import { Domain, getRedirectSubdomainPath, pubsub } from '@things-factory/shell'
+import { getWorkDateAndShift } from '@things-factory/work-shift'
 
-import { DataItem } from '../service/data-item/data-item'
 import { DataOoc, DataOocStatus } from '../service/data-ooc/data-ooc'
 import { DataSample } from '../service/data-sample/data-sample'
 import { NewDataSample } from '../service/data-sample/data-sample-type'
 import { DataSet } from '../service/data-set/data-set'
 import { DataUseCase } from './data-use-case'
-import { getWorkDateAndShift } from '@things-factory/work-shift'
-
 
 const debug = require('debug')('things-factory:dataset:controller/save-data-sample')
 
@@ -56,26 +54,7 @@ export async function createDataSample(
     where: { id: dataSample.dataSet.id }
   })
 
-  const dataItems = await tx.getRepository(DataItem).find({
-    where: {
-      domain,
-      dataSet
-    },
-    order: {
-      sequence: 'DESC'
-    }
-  })
-
-  const spec = dataItems.reduce((spec, dataItem) => {
-    spec[dataItem.tag] = {
-      ...dataItem.spec,
-      name: dataItem.name, /* do we need ? */
-      hidden: dataItem.hidden
-    }
-
-    return spec
-  }, {})
-
+  const dataItems = dataSet.dataItems
   const collectedAt = dataSample.collectedAt || new Date()
 
   const timezone = dataSet.timezone || domain.timezone || 'UTC'
@@ -83,16 +62,15 @@ export async function createDataSample(
 
   // workDate ex) 2022-04-04
   const { workDate, workShift } = await getWorkDateAndShift(domain, collectedAt, { timezone, format })
-  
+
   // local time dataSet timezone or domain timezone or default 'UTC'
-  
-  // const collectedAt = dataSample.collectedAt || new Date()
+
   const localDateTz = moment(collectedAt).tz(timezone)
   const defaultPartitionKeys = {
     domain: domain.subdomain,
-    datasetid: dataSample.dataSet.id, /* It should not be 'data_set_id' as column name duplicated for Glue */
-    date: localDateTz.format(format), /* local time date */
-    workdate: workDate, /* working date */
+    datasetid: dataSample.dataSet.id /* It should not be 'data_set_id' as column name duplicated for Glue */,
+    date: localDateTz.format(format) /* local time date */,
+    workdate: workDate /* working date */,
     workshift: workShift
   }
 
@@ -106,17 +84,18 @@ export async function createDataSample(
     ...dataSample.data
   })
 
-  const { ooc, oos } = DataUseCase.evaluate(dataSet, dataItems, dataSample.data) || {}
+  const { ooc, oos, judgment } = DataUseCase.evaluate(dataSet, dataItems, dataSample.data) || {}
   const result = await tx.getRepository(DataSample).save({
     name: dataSet.name,
     description: dataSet.description,
     useCase: dataSet.useCase,
     ...dataSample,
+    dataSetVersion: dataSet.version,
     domain,
     partitionKeys,
-    spec,
     ooc,
     oos,
+    judgment,
     collectedAt,
     workDate,
     workShift,
@@ -126,18 +105,7 @@ export async function createDataSample(
 
   if (ooc || oos) {
     const dataOoc = await tx.getRepository(DataOoc).save({
-      name: dataSet.name,
-      description: dataSet.description,
-      useCase: dataSet.useCase,
-      dataSet,
-      dataSample: result,
-      data: dataSample.data,
-      rawData: dataSample.rawData,
-      domain,
-      partitionKeys,
-      spec,
-      ooc,
-      oos,
+      ...result,
       history: [
         {
           user: {
@@ -148,12 +116,7 @@ export async function createDataSample(
           timestamp: Date.now()
         }
       ],
-      state: DataOocStatus.CREATED,
-      workDate,
-      workShift,
-      collectedAt,
-      creator: user,
-      updater: user
+      state: DataOocStatus.CREATED
     })
 
     pubsub.publish('data-ooc', {
@@ -174,4 +137,4 @@ export async function createDataSample(
   }
 
   return result
-}
+}

package/server/controllers/data-use-case.ts

@@ -1,4 +1,5 @@
-import { DataItem, DataSet } from 'server/service'
+import { DataItem } from '../service/data-set/data-item-type'
+import { DataSet } from '../service/data-set/data-set'
 
 export type DataItemSpec = {
   type: string
@@ -13,7 +14,11 @@ export type DataItemSpecSet = {
   specs: DataItemSpec[]
 }
 
-export type EvaluationResult = { oos: boolean; ooc: boolean }
+export type EvaluationResult = {
+  oos: boolean
+  ooc: boolean
+  judgment?: { [tag: string]: { ooc: boolean; oos: boolean } }
+}
 
 export abstract class DataUseCase {
   static registry: { [name: string]: DataUseCase } = {}
@@ -33,6 +38,7 @@ export abstract class DataUseCase {
   public static evaluate(dataSet: DataSet, dataItems: DataItem[], data: any): EvaluationResult {
     var ooc = false
     var oos = false
+    var judgment: { [tag: string]: { ooc: boolean; oos: boolean } } = {}
 
     if (!dataSet.useCase) {
       return { ooc, oos }
@@ -49,7 +55,7 @@ export abstract class DataUseCase {
       }
 
       let values: any | any[] = data[tag]
-      if (typeof values === 'undefined') {
+      if (values == null) {
         continue // TODO what if in case no value ?
       }
 
@@ -57,6 +63,9 @@ export abstract class DataUseCase {
         values = [values]
       }
 
+      let oocForTag = false
+      let oosForTag = false
+
       for (let j = 0; j < useCases.length; j++) {
         const useCase = useCases[j]
 
@@ -68,17 +77,20 @@ export abstract class DataUseCase {
         const result = useCase.evaluate(specs, values)
 
         if (result) {
+          oocForTag ||= result.ooc
+          oosForTag ||= result.oos
           ooc ||= result.ooc
           oos ||= result.oos
         }
+      }
 
-        // if (ooc && oos) {
-        //   return { ooc, oos }
-        // }
+      judgment[tag] = {
+        ooc: oocForTag,
+        oos: oosForTag
       }
     }
 
-    return { ooc, oos }
+    return { ooc, oos, judgment }
   }
 
   public abstract evaluate(specs: any, values: any[]): EvaluationResult

package/server/controllers/index.ts

@@ -1,2 +1,2 @@
-export * from './data-use-case'
 export * from './create-data-sample'
+export * from './data-use-case'

package/server/controllers/jasper-report.ts

@@ -6,47 +6,53 @@ import { AthenaController } from '@things-factory/aws-base'
 import { config } from '@things-factory/env'
 
 const dataReportConfig = config.get('dataReport')
-const { jasper: { 
-  endpoint: { protocol: PROTOCOL, host:HOST, port:PORT }, 
-  datasource: { database: DATABASE } } 
-} = dataReportConfig
+const {
+  jasper: {
+    endpoint: { protocol: PROTOCOL, host: HOST, port: PORT },
+    datasource: { database: DATABASE }
+  }
+} = dataReportConfig || {
+  jasper: {
+    endpoint: {},
+    datasource: {}
+  }
+}
 
-/** author: ywnam123 */
 function transformValuesToRows(queryResult) {
   var parseData = []
-    let index = 1
-    for (let i = 0; i < queryResult.Items.length; i++) {
-      var j = 0
-      const data = JSON.parse(queryResult.Items[i].data)
-      const spec = JSON.parse(queryResult.Items[i].spec)
-
-      for (let key in data) {
-        if (Array.isArray(data[key])) {
-          for (j = 0; j < data[key].length; j++) {
-            for (let specKey in spec) {
-              if (key === specKey) {
-                parseData.push({
-                  item: spec[specKey].name,
-                  index: index + j,
-                  value: String(data[key][j])
-                })
-              }
+  let index = 1
+  for (let i = 0; i < queryResult.Items.length; i++) {
+    var j = 0
+    const data = JSON.parse(queryResult.Items[i].data)
+    const spec = JSON.parse(queryResult.Items[i].spec)
+
+    for (let key in data) {
+      if (Array.isArray(data[key])) {
+        for (j = 0; j < data[key].length; j++) {
+          for (let specKey in spec) {
+            if (key === specKey) {
+              parseData.push({
+                item: spec[specKey].name,
+                index: index + j,
+                value: String(data[key][j])
+              })
             }
           }
-        } else {
-          parseData.push({
-            item: key,
-            index,
-            value: String(data[key])
-          })
         }
-      }
-      if (j !== 0) {
-        index = index + j
       } else {
-        index = index + 1
+        parseData.push({
+          item: key,
+          index,
+          value: String(data[key])
+        })
       }
     }
+    if (j !== 0) {
+      index = index + j
+    } else {
+      index = index + 1
+    }
+  }
 }
 
 /** @todo considering trasformation in lambda, as massive dataset */
@@ -61,11 +67,12 @@ function pivotData(rows) {
     for (let key in data) {
       /** @todo rule to display or not, about unspecified spec */
       const value = data[key]
-      !(spec[key]?.hidden) && parsedData.push({
-        item: spec[key]?.name || key,
-        index,
-        value: Array.isArray(value) ? value.join(', ') : value
-      })
+      !spec[key]?.hidden &&
+        parsedData.push({
+          item: spec[key]?.name || key,
+          index,
+          value: Array.isArray(value) ? value.join(', ') : value
+        })
     }
     if (j !== 0) {
       index = index + j
@@ -95,6 +102,7 @@ function parseJsonDataField(rows) {
 }
 
 const athenaClient = new AthenaController()
+
 async function queryAthena(params) {
   const { table, domain, dataSetId, fromWorkDate, toWorkDate, workShift, timezone } = params
   const queryData = {
@@ -108,7 +116,7 @@ async function queryAthena(params) {
     AND datasetid = '${dataSetId}'
     AND workdate >= '${fromWorkDate}'
     AND workdate <= '${toWorkDate}'
-    ${workShift ? 'AND workshift = \'' + workShift + '\'' : ''}
+    ${workShift ? "AND workshift = '" + workShift + "'" : ''}
     ORDER BY collected_at`,
     db: DATABASE
   }
@@ -116,19 +124,23 @@ async function queryAthena(params) {
 
   return await athenaClient.query(queryData)
 }
-export async function renderJasperReport(context: any) { 
-  const { state: { domain }, query } = context
+
+export async function renderJasperReport(context: any) {
+  const {
+    state: { domain },
+    query
+  } = context
 
   const template = await STORAGE.readFile(query['template'] || 'dynamic_header_sample.jrxml', 'utf-8')
   let templateType = query['templateType'] || 'crosstab'
   let parsedData = []
 
   // @todo: get dataset timezone
-  /**  
+  /**
    * const variables = await gql(dataSet(id:${dataSetId}) {
    *  name, description, partition_keys, timezone
    * })
-  */
+   */
 
   query['domain'] = domain?.subdomain
   query['timezone'] = domain?.timezone
@@ -137,8 +149,7 @@ export async function renderJasperReport(context: any) {
 
   if (!rows.length) {
     return '<h3>Not found result.</h3>'
-  }
-  else {
+  } else {
     const firstRow = rows[0]
     // uses the first row values as data-set has no history data.
     const parameters = {
@@ -146,18 +157,18 @@ export async function renderJasperReport(context: any) {
       description: firstRow.description,
       ...query
     }
-  
+
     if (templateType === 'crosstab') {
       parsedData = pivotData(rows)
     } else {
       parsedData = parseJsonDataField(rows)
     }
-  
+
     const formData = new FormData()
     formData.append('template', template)
     formData.append('jsonString', JSON.stringify(parsedData))
     formData.append('parameters', JSON.stringify(parameters))
-  
+
     const reportUrl = `${PROTOCOL || 'http'}://${HOST}:${PORT}/rest/report/show_html`
     const response = await fetch(reportUrl, {
       method: 'POST',
@@ -166,5 +177,4 @@ export async function renderJasperReport(context: any) {
 
     return await response.text()
   }
-
 }

package/server/service/data-item/index.ts

@@ -1,6 +1,3 @@
 import { DataItem } from './data-item'
-import { DataItemQuery } from './data-item-query'
-import { DataItemMutation } from './data-item-mutation'
 
 export const entities = [DataItem]
-export const resolvers = [DataItemQuery, DataItemMutation]

package/server/service/data-ooc/data-ooc-query.ts

@@ -4,7 +4,8 @@ import { getRepository } from 'typeorm'
 import { User } from '@things-factory/auth-base'
 import { Domain, getQueryBuilderFromListParams, ListParam } from '@things-factory/shell'
 
-import { DataSample } from '../data-sample/data-sample'
+import { DataSetHistory } from '../data-set-history/data-set-history'
+import { DataItem } from '../data-set/data-item-type'
 import { DataSet } from '../data-set/data-set'
 import { DataOoc } from './data-ooc'
 import { DataOocList } from './data-ooc-type'
@@ -38,6 +39,17 @@ export class DataOocQuery {
     return { items, total }
   }
 
+  @FieldResolver(type => [DataItem])
+  async dataItems(@Root() dataOoc: DataOoc): Promise<DataItem[]> {
+    const dataSetHistory: DataSetHistory = await getRepository(DataSetHistory).findOne({
+      where: {
+        originalId: dataOoc.dataSetId,
+        version: dataOoc.dataSetVersion
+      }
+    })
+    return dataSetHistory?.dataItems || []
+  }
+
   @FieldResolver(type => DataSet)
   async dataSet(@Root() dataOoc: DataOoc): Promise<DataSet> {
     return await getRepository(DataSet).findOne({
@@ -45,13 +57,6 @@ export class DataOocQuery {
     })
   }
 
-  @FieldResolver(type => DataSample)
-  async dataSample(@Root() dataOoc: DataOoc): Promise<DataSample> {
-    return await getRepository(DataSample).findOne({
-      id: dataOoc.dataSampleId
-    })
-  }
-
   @FieldResolver(type => Domain)
   async domain(@Root() dataOoc: DataOoc): Promise<Domain> {
     return await getRepository(Domain).findOne(dataOoc.domainId)

package/server/service/data-ooc/data-ooc-type.ts

@@ -15,6 +15,9 @@ export class NewDataOoc {
   @Field(type => ObjectRef, { nullable: true })
   dataSet?: ObjectRef
 
+  @Field({ nullable: true })
+  dataSetVersion?: number
+
   @Field(type => ObjectRef, { nullable: true })
   dataSample?: ObjectRef
 

package/server/service/data-ooc/data-ooc.ts

@@ -4,9 +4,7 @@ import {
   CreateDateColumn,
   Entity,
   Index,
-  JoinColumn,
   ManyToOne,
-  OneToOne,
   PrimaryGeneratedColumn,
   RelationId,
   UpdateDateColumn
@@ -16,7 +14,7 @@ import { User } from '@things-factory/auth-base'
 import { config } from '@things-factory/env'
 import { Domain, ScalarObject } from '@things-factory/shell'
 
-import { DataSample } from '../data-sample/data-sample'
+import { DataItem } from '../data-set/data-item-type'
 import { DataSet } from '../data-set/data-set'
 
 const ORMCONFIG = config.get('ormconfig', {})
@@ -34,9 +32,7 @@ registerEnumType(DataOocStatus, {
 })
 
 @Entity()
-@Index('ix_data_ooc_0', (dataOoc: DataOoc) => [dataOoc.domain, dataOoc.dataSet], { unique: false })
-@Index('ix_data_ooc_1', (dataOoc: DataOoc) => [dataOoc.domain, dataOoc.dataSample], { unique: true })
-@Index('ix_data_ooc_2', (dataOoc: DataOoc) => [dataOoc.domain, dataOoc.collectedAt], { unique: false })
+@Index('ix_data_ooc_0', (dataOoc: DataOoc) => [dataOoc.domain, dataOoc.dataSet, dataOoc.collectedAt], { unique: false })
 @ObjectType({ description: 'Entity for Out of control data' })
 export class DataOoc {
   @PrimaryGeneratedColumn('uuid')
@@ -67,13 +63,9 @@ export class DataOoc {
   @RelationId((dataOoc: DataOoc) => dataOoc.dataSet)
   dataSetId?: string
 
-  @OneToOne(type => DataSample)
-  @JoinColumn()
-  @Field(type => DataSample, { nullable: true })
-  dataSample?: DataSample
-
-  @RelationId((dataOoc: DataOoc) => dataOoc.dataSample)
-  dataSampleId?: string
+  @Column({ nullable: true })
+  @Field({ nullable: true })
+  dataSetVersion?: number
 
   @Column({
     nullable: true
@@ -127,7 +119,10 @@ export class DataOoc {
 
   @Column('simple-json', { nullable: true })
   @Field(type => ScalarObject, { nullable: true })
-  spec?: ScalarObject
+  judgment?: ScalarObject
+
+  @Field(type => [DataItem], { nullable: true })
+  dataItems?: DataItem[]
 
   @Column('simple-json', { nullable: true })
   @Field(type => ScalarObject, { nullable: true })

package/server/service/data-sample/data-sample-query.ts

@@ -4,6 +4,8 @@ import { getRepository } from 'typeorm'
 import { User } from '@things-factory/auth-base'
 import { Domain, getQueryBuilderFromListParams, ListParam } from '@things-factory/shell'
 
+import { DataSetHistory } from '../data-set-history/data-set-history'
+import { DataItem } from '../data-set/data-item-type'
 import { DataSet } from '../data-set/data-set'
 import { DataSample } from './data-sample'
 import { DataSampleList } from './data-sample-type'
@@ -37,8 +39,19 @@ export class DataSampleQuery {
     return { items, total }
   }
 
+  @FieldResolver(type => [DataItem])
+  async dataItems(@Root() dataSample: DataSample): Promise<DataItem[]> {
+    const dataSetHistory: DataSetHistory = await getRepository(DataSetHistory).findOne({
+      where: {
+        originalId: dataSample.dataSetId,
+        version: dataSample.dataSetVersion
+      }
+    })
+    return dataSetHistory?.dataItems || []
+  }
+
   @FieldResolver(type => DataSet)
-  async dataSet(@Root() dataSample: DataSample): Promise<Domain> {
+  async dataSet(@Root() dataSample: DataSample): Promise<DataSet> {
     return await getRepository(DataSet).findOne(dataSample.dataSetId)
   }
 

package/server/service/data-sample/data-sample-type.ts

@@ -15,6 +15,9 @@ export class NewDataSample {
   @Field(type => ObjectRef, { nullable: true })
   dataSet?: ObjectRef
 
+  @Field({ nullable: true })
+  dataSetVersion?: number
+
   @Field(type => ScalarObject, { nullable: true })
   data?: ScalarObject
 

package/server/service/data-sample/data-sample.ts

@@ -14,14 +14,14 @@ import { User } from '@things-factory/auth-base'
 import { config } from '@things-factory/env'
 import { Domain, ScalarObject } from '@things-factory/shell'
 
+import { DataItem } from '../data-set/data-item-type'
 import { DataSet } from '../data-set/data-set'
 
 const ORMCONFIG = config.get('ormconfig', {})
 const DATABASE_TYPE = ORMCONFIG.type
 @Entity()
-@Index('ix_data_sample_0', (dataSample: DataSample) => [dataSample.domain, dataSample.dataSet], { unique: false })
 @Index(
-  'ix_data_sample_2',
+  'ix_data_sample_0',
   (dataSample: DataSample) => [dataSample.domain, dataSample.dataSet, dataSample.collectedAt],
   { unique: false }
 )
@@ -71,13 +71,20 @@ export class DataSample {
   @RelationId((dataSample: DataSample) => dataSample.dataSet)
   dataSetId?: string
 
+  @Column({ nullable: true })
+  @Field({ nullable: true })
+  dataSetVersion?: number
+
   @Column('simple-json', { nullable: true })
   @Field(type => ScalarObject, { nullable: true })
   data?: ScalarObject
 
   @Column('simple-json', { nullable: true })
   @Field(type => ScalarObject, { nullable: true })
-  spec?: ScalarObject
+  judgment?: ScalarObject
+
+  @Field(type => [DataItem], { nullable: true })
+  dataItems?: DataItem[]
 
   @Column({
     nullable: true,

package/server/service/data-set/data-item-type.ts

@@ -0,0 +1,82 @@
+import { Field, InputType, Int, ObjectType, registerEnumType } from 'type-graphql'
+
+import { ScalarObject } from '@things-factory/shell'
+
+export enum DataItemType {
+  number = 'number',
+  text = 'text',
+  boolean = 'boolean',
+  select = 'select',
+  file = 'file'
+}
+
+registerEnumType(DataItemType, {
+  name: 'DataItemType',
+  description: 'state enumeration of a data-item'
+})
+
+@ObjectType({ description: 'Entity for DataItem' })
+export class DataItem {
+  @Field()
+  name: string
+
+  @Field({ nullable: true })
+  description?: string
+
+  @Field({ nullable: true })
+  tag?: string
+
+  @Field({ nullable: true })
+  active?: boolean
+
+  @Field({ nullable: true })
+  hidden?: boolean
+
+  @Field({ nullable: true })
+  type?: DataItemType
+
+  @Field(type => ScalarObject, { nullable: true })
+  options?: ScalarObject
+
+  @Field({ nullable: true })
+  unit?: string
+
+  @Field(type => Int, { nullable: true })
+  quota: number
+
+  @Field(type => ScalarObject, { nullable: true })
+  spec?: ScalarObject
+}
+
+@InputType()
+export class DataItemPatch {
+  @Field({ nullable: true })
+  name?: string
+
+  @Field({ nullable: true })
+  description?: string
+
+  @Field({ nullable: true })
+  tag?: string
+
+  @Field(type => DataItemType, { nullable: true })
+  type?: DataItemType
+
+  @Field(type => ScalarObject, { nullable: true })
+  options?: ScalarObject
+
+  @Field({ nullable: true })
+  unit?: string
+
+  @Field(type => Int, { nullable: true })
+  quota?: number
+
+  @Field({ nullable: true })
+  active?: boolean
+
+  @Field({ nullable: true })
+  hidden?: boolean
+
+  @Field(type => ScalarObject, { nullable: true })
+  spec?: ScalarObject
+}