@things-factory/dataset 6.0.73 → 6.0.75

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@things-factory/dataset",
-  "version": "6.0.73",
+  "version": "6.0.75",
   "main": "dist-server/index.js",
   "browser": "dist-client/index.js",
   "things-factory": true,
@@ -35,17 +35,19 @@
     "@operato/shell": "^1.0.1",
     "@operato/styles": "^1.0.0",
     "@operato/utils": "^1.0.1",
-    "@things-factory/auth-base": "^6.0.73",
-    "@things-factory/aws-base": "^6.0.73",
-    "@things-factory/board-service": "^6.0.73",
+    "@things-factory/auth-base": "^6.0.75",
+    "@things-factory/aws-base": "^6.0.75",
+    "@things-factory/board-service": "^6.0.75",
     "@things-factory/env": "^6.0.71",
-    "@things-factory/organization": "^6.0.73",
-    "@things-factory/scheduler-client": "^6.0.73",
-    "@things-factory/shell": "^6.0.73",
-    "@things-factory/work-shift": "^6.0.73",
-    "@things-factory/worklist": "^6.0.73",
+    "@things-factory/organization": "^6.0.75",
+    "@things-factory/scheduler-client": "^6.0.75",
+    "@things-factory/shell": "^6.0.75",
+    "@things-factory/work-shift": "^6.0.75",
+    "@things-factory/worklist": "^6.0.75",
     "cron-parser": "^4.3.0",
-    "moment-timezone": "^0.5.40"
+    "moment-timezone": "^0.5.40",
+    "simple-statistics": "^7.8.3",
+    "statistics": "^3.3.0"
   },
-  "gitHead": "cf00adc3f06bebbe492c63f0f743eb874155f9b5"
+  "gitHead": "f7ca179a7fabfc129a9981a7bd734380a3ac64f0"
 }

package/server/controllers/generate-data-summary.ts

@@ -1,3 +1,6 @@
+const statistics = require('simple-statistics')
+const deepClone = require('lodash/cloneDeep')
+
 import { Sorting, getQueryBuilderFromListParams } from '@things-factory/shell'
 import { logger } from '@things-factory/env'
 
@@ -7,6 +10,20 @@ import { DataSet } from '../service/data-set/data-set'
 import { DataSummary } from '../service/data-summary/data-summary'
 import { DataKeyItem } from '../service/data-key-set/data-key-item-type'
 
+import { DataItem } from 'service'
+
+const STAT_FUNCTION_MAP = {
+  sum: 'sum',
+  mean: 'mean',
+  stddev: 'standardDeviation',
+  variance: 'variance',
+  min: 'min',
+  max: 'max',
+  range: 'range',
+  median: 'median',
+  mode: 'mode'
+}
+
 const compareKeys = (dataKeyItems: DataKeyItem[], summary: Partial<DataSummary>, sample: DataSample): boolean => {
   return dataKeyItems.every((item, index) => {
     const prop = `key0${index + 1}`
@@ -31,6 +48,42 @@ const buildKeySortingList = (dataKeyItems: DataKeyItem[]): Sorting[] => {
   }, [])
 }
 
+const calculateSummary = (dataItems: DataItem[], base: { [tag: string]: any[] }) => {
+  return dataItems.reduce((summary, item) => {
+    const tag = item.tag
+
+    const data = base[tag]
+      .flat(Infinity)
+      .map(Number)
+      .filter(item => !isNaN(item))
+
+    switch (item.stat) {
+      case 'range':
+        summary[tag] = statistics.max(data) - statistics.min(data)
+        break
+
+      default:
+        const functionName = STAT_FUNCTION_MAP[item.stat]
+        summary[tag] = (functionName && statistics[functionName](data)) || ''
+    }
+
+    return summary
+  }, {})
+}
+
+const fillSummaryResult = (
+  dataSummary: Partial<DataSummary>,
+  dataItems: DataItem[],
+  base: { [tag: string]: any[] }
+): void => {
+  const summary = calculateSummary(dataItems, base)
+
+  dataSummary.summary = summary
+  dataItems.slice(0, 4).forEach((dataItem, idx) => {
+    dataSummary[`data0${idx + 1}`] = summary[dataItem.tag]
+  })
+}
+
 export async function generateDataSummary(
   dataSetId: string,
   workDate: string,
@@ -46,6 +99,11 @@ export async function generateDataSummary(
     })
 
     const dataKeyItems = dataSet.dataKeySet?.dataKeyItems || []
+    const dataItems = dataSet.dataItems.filter(item => item.stat)
+    const initialSummary = dataItems.reduce((sum, item) => {
+      sum[item.tag] = []
+      return sum
+    }, {})
 
     const limit = 100
     var page = 1
@@ -71,6 +129,7 @@ export async function generateDataSummary(
       for (const sample of samples) {
         if (!summary || !compareKeys(dataKeyItems, summary, sample)) {
           if (summary) {
+            fillSummaryResult(summary, dataItems, summary.summary)
             summaries.push(summary)
           }
 
@@ -85,6 +144,7 @@ export async function generateDataSummary(
             count: 0,
             countOoc: 0,
             countOos: 0,
+            summary: deepClone(initialSummary),
             updater: user,
             creator: user
           }
@@ -93,10 +153,17 @@ export async function generateDataSummary(
         summary.count++
         sample.ooc && summary.countOoc++
         sample.oos && summary.countOos++
+
+        dataItems.forEach(item => {
+          summary.summary[item.tag].push(sample.data[item.tag])
+        })
       }
 
       if (samples.length < limit) {
-        summary && summaries.push(summary)
+        if (summary) {
+          fillSummaryResult(summary, dataItems, summary.summary)
+          summaries.push(summary)
+        }
         break
       }
 

package/server/service/data-sample/data-sample-query.ts

@@ -39,6 +39,42 @@ export class DataSampleQuery {
     return { items, total }
   }
 
+  @Directive('@privilege(category: "data-sample", privilege: "query", domainOwnerGranted: true)')
+  @Query(returns => DataSampleList, { description: 'To fetch multiple data samples by data set' })
+  async dataSamplesByDataSet(
+    @Arg('dataSetId') dataSetId: string,
+    @Args() params: ListParam,
+    @Ctx() context: ResolverContext
+  ): Promise<DataSampleList> {
+    const { domain } = context.state
+
+    const dataSet = await getRepository(DataSet).findOne({
+      where: { domain: { id: domain.id }, id: dataSetId },
+      relations: ['dataKeySet']
+    })
+
+    if (!dataSet) {
+      throw new Error(`dataSet not found by the given dataSetId(${dataSetId})`)
+    }
+
+    const dataKeyItems = dataSet.dataKeySet?.dataKeyItems || []
+    const searchables = dataKeyItems.map((item, index) => `key0${index + 1}`)
+
+    const queryBuilder = getQueryBuilderFromListParams({
+      repository: getRepository(DataSample),
+      params,
+      domain,
+      alias: 'sample',
+      searchables: ['name', 'description'].concat(searchables)
+    }).innerJoin('sample.dataSet', 'ds', 'ds.id = :dataSetId', {
+      dataSetId
+    })
+
+    const [items, total] = await queryBuilder.getManyAndCount()
+
+    return { items, total }
+  }
+
   @Directive('@privilege(category: "data-sample", privilege: "query", domainOwnerGranted: true)')
   @Query(returns => DataSampleList, { description: 'To fetch multiple data samples by data key set' })
   async dataSamplesByDataKeySet(
@@ -56,9 +92,9 @@ export class DataSampleQuery {
       repository: getRepository(DataSample),
       params,
       domain,
-      alias: 'dks',
+      alias: 'sample',
       searchables: ['name', 'description'].concat(searchables)
-    }).innerJoin('dks.dataSet', 'ds', 'ds.dataKeySet = :dataKeySetId', {
+    }).innerJoin('sample.dataSet', 'ds', 'ds.dataKeySet = :dataKeySetId', {
       dataKeySetId
     })
 

package/server/service/data-set/data-item-type.ts

@@ -17,23 +17,21 @@ registerEnumType(DataItemType, {
   description: 'type enumeration of a data-item'
 })
 
-export enum DataItemReducerType {
+export enum DataItemStatType {
   sum = 'sum',
   mean = 'mean',
   stddev = 'stddev',
-  var = 'var',
+  variance = 'variance',
   min = 'min',
   max = 'max',
   range = 'range',
   median = 'median',
-  mode = 'mode',
-  percentile = 'percentile',
-  histogram = 'histogram'
+  mode = 'mode'
 }
 
-registerEnumType(DataItemReducerType, {
-  name: 'DataItemReducerType',
-  description: 'reducer type enumeration of a data-item'
+registerEnumType(DataItemStatType, {
+  name: 'DataItemStatType',
+  description: 'stat type enumeration of a data-item'
 })
 
 @ObjectType({ description: 'Entity for DataItem' })
@@ -59,8 +57,11 @@ export class DataItem {
   @Field(type => ScalarObject, { nullable: true })
   options?: { [option: string]: any }
 
-  @Field({ nullable: true })
-  reducer?: DataItemReducerType
+  @Field({
+    nullable: true,
+    description: 'The grouping logic for data summarization in the given field during periodic task deadlines.'
+  })
+  stat?: DataItemStatType
 
   @Field({ nullable: true })
   unit?: string
@@ -89,6 +90,12 @@ export class DataItemPatch {
   @Field(type => ScalarObject, { nullable: true })
   options?: { [option: string]: any }
 
+  @Field({
+    nullable: true,
+    description: 'The grouping logic for data summarization in the given field during periodic task deadlines.'
+  })
+  stat?: DataItemStatType
+
   @Field({ nullable: true })
   unit?: string
 

package/server/service/data-summary/data-summary-query.ts

@@ -35,6 +35,42 @@ export class DataSummaryQuery {
     return { items, total }
   }
 
+  @Directive('@privilege(category: "data-summary", privilege: "query", domainOwnerGranted: true)')
+  @Query(returns => DataSummaryList, { description: 'To fetch multiple data summaries by data set' })
+  async dataSummariesByDataSet(
+    @Arg('dataSetId') dataSetId: string,
+    @Args() params: ListParam,
+    @Ctx() context: ResolverContext
+  ): Promise<DataSummaryList> {
+    const { domain } = context.state
+
+    const dataSet = await getRepository(DataSet).findOne({
+      where: { domain: { id: domain.id }, id: dataSetId },
+      relations: ['dataKeySet']
+    })
+
+    if (!dataSet) {
+      throw new Error(`dataSet not found by the given dataSetId(${dataSetId})`)
+    }
+
+    const dataKeyItems = dataSet.dataKeySet?.dataKeyItems || []
+    const searchables = dataKeyItems.map((item, index) => `key0${index + 1}`)
+
+    const queryBuilder = getQueryBuilderFromListParams({
+      repository: getRepository(DataSummary),
+      params,
+      domain,
+      alias: 'summary',
+      searchables: ['name', 'description'].concat(searchables)
+    }).innerJoin('summary.dataSet', 'ds', 'ds.id = :dataSetId', {
+      dataSetId
+    })
+
+    const [items, total] = await queryBuilder.getManyAndCount()
+
+    return { items, total }
+  }
+
   @Directive('@privilege(category: "data-summary", privilege: "query", domainOwnerGranted: true)')
   @Query(returns => DataSummaryList, { description: 'To fetch multiple data summaries by data key set' })
   async dataSummariesByDataKeySet(
@@ -52,9 +88,9 @@ export class DataSummaryQuery {
       repository: getRepository(DataSummary),
       params,
       domain,
-      alias: 'dks',
+      alias: 'summary',
       searchables: ['name', 'description'].concat(searchables)
-    }).innerJoin('dks.dataSet', 'ds', 'ds.dataKeySet = :dataKeySetId', {
+    }).innerJoin('summary.dataSet', 'ds', 'ds.dataKeySet = :dataKeySetId', {
       dataKeySetId
     })
 
@@ -65,13 +101,13 @@ export class DataSummaryQuery {
 
   @FieldResolver(type => [DataItem])
   async dataItems(@Root() dataSummary: DataSummary): Promise<DataItem[]> {
-    const dataSetHistory: DataSetHistory = await getRepository(DataSetHistory).findOne({
+    const dataSet = await getRepository(DataSet).findOne({
       where: {
-        originalId: dataSummary.dataSetId,
-        version: dataSummary.dataSetVersion
+        id: dataSummary.dataSetId
       }
     })
-    return dataSetHistory?.dataItems || []
+
+    return dataSet?.dataItems || []
   }
 
   @FieldResolver(type => DataSet)

package/server/service/data-summary/data-summary.ts

@@ -141,6 +141,26 @@ export class DataSummary {
   @Field({ nullable: true })
   key05?: string = ''
 
+  @Column({ type: 'real', nullable: true })
+  @Field({ nullable: true })
+  data01?: number
+
+  @Column({ type: 'real', nullable: true })
+  @Field({ nullable: true })
+  data02?: number
+
+  @Column({ type: 'real', nullable: true })
+  @Field({ nullable: true })
+  data03?: number
+
+  @Column({ type: 'real', nullable: true })
+  @Field({ nullable: true })
+  data04?: number
+
+  @Column({ type: 'real', nullable: true })
+  @Field({ nullable: true })
+  data05?: number
+
   @CreateDateColumn()
   @Field({ nullable: true })
   createdAt?: Date

package/translations/en.json

@@ -26,6 +26,11 @@
   "field.key-03": "key 3",
   "field.key-04": "key 4",
   "field.key-05": "key 5",
+  "field.data-01": "data 1",
+  "field.data-02": "data 2",
+  "field.data-03": "data 3",
+  "field.data-04": "data 4",
+  "field.data-05": "data 5",
   "field.latest-collected-at": "latest collected at",
   "field.monitor-type": "monitor type",
   "field.monitor-view": "monitor view",
@@ -34,6 +39,7 @@
   "field.oos": "out of critical limit",
   "field.ooc": "out of control limit",
   "field.summary": "summary",
+  "field.stat-function": "stat func.",
   "field.count": "count",
   "field.count-oos": "oos count",
   "field.count-ooc": "oos count",

package/translations/ko.json

@@ -21,11 +21,16 @@
   "field.entry-view": "입력용 화면",
   "field.hidden": "숨기기",
   "field.item": "항목",
-  "field.key-01": "데이타키 1",
-  "field.key-02": "데이타키 2",
-  "field.key-03": "데이타키 3",
-  "field.key-04": "데이타키 4",
-  "field.key-05": "데이타키 5",
+  "field.key-01": "키 1",
+  "field.key-02": "키 2",
+  "field.key-03": "키 3",
+  "field.key-04": "키 4",
+  "field.key-05": "키 5",
+  "field.data-01": "데이타 1",
+  "field.data-02": "데이타 2",
+  "field.data-03": "데이타 3",
+  "field.data-04": "데이타 4",
+  "field.data-05": "데이타 5",
   "field.latest-collected-at": "최근 수집 시간",
   "field.monitor-type": "모니터용 화면종류",
   "field.monitor-view": "모니터용 화면",
@@ -34,9 +39,10 @@
   "field.ooc": "관리한계 이탈여부",
   "field.oos": "허용한계 이탈여부",
   "field.summary": "마감 자료",
+  "field.stat-function": "통계함수",
   "field.count": "건수",
-  "field.count-oos": "oos 건수",
-  "field.count-ooc": "oos 건수",
+  "field.count-oos": "허용한계이탈 건수",
+  "field.count-ooc": "관리한계이탈 건수",
   "field.options": "선택옵션",
   "field.partition-keys": "파티션 키",
   "field.prev-schedule": "이전 수집계획",

package/translations/ms.json

@@ -25,6 +25,11 @@
   "field.key-03": "key 3",
   "field.key-04": "key 4",
   "field.key-05": "key 5",
+  "field.data-01": "data 1",
+  "field.data-02": "data 2",
+  "field.data-03": "data 3",
+  "field.data-04": "data 4",
+  "field.data-05": "data 5",
   "field.latest-collected-at": "latest collected at",
   "field.monitor-type": "monitor type",
   "field.monitor-view": "monitor view",
@@ -33,6 +38,7 @@
   "field.oos": "out of critical limit",
   "field.ooc": "out of control limit",
   "field.summary": "summary",
+  "field.stat-function": "stat func.",
   "field.count": "count",
   "field.count-oos": "oos count",
   "field.count-ooc": "oos count",

package/translations/zh.json

@@ -25,6 +25,11 @@
   "field.key-03": "key 3",
   "field.key-04": "key 4",
   "field.key-05": "key 5",
+  "field.data-01": "data 1",
+  "field.data-02": "data 2",
+  "field.data-03": "data 3",
+  "field.data-04": "data 4",
+  "field.data-05": "data 5",
   "field.latest-collected-at": "latest collected at",
   "field.monitor-type": "monitor type",
   "field.monitor-view": "monitor view",
@@ -33,6 +38,7 @@
   "field.oos": "out of critical limit",
   "field.ooc": "out of control limit",
   "field.summary": "summary",
+  "field.stat-function": "stat func.",
   "field.count": "count",
   "field.count-oos": "oos count",
   "field.count-ooc": "oos count",