@things-factory/reference-app 5.0.0-zeta.7 → 5.0.0

package/server/controllers/create-data-sample-mockup.ts

@@ -0,0 +1,268 @@
+import moment from 'moment-timezone'
+import { EntityManager } from 'typeorm'
+
+import { User } from '@things-factory/auth-base'
+import { DataSet, DataSample, DataUseCase } from '@things-factory/dataset'
+import { Product } from '@things-factory/product-base'
+import { Operation } from '@things-factory/routing-base'
+import { Domain } from '@things-factory/shell'
+import { getWorkDateAndShift } from '@things-factory/work-shift'
+
+import { DataSampleMockupInfo } from '../service/data-sample-mockup/data-sample-mockup-type'
+
+const randomWords =  require('random-words')
+const debug = require('debug')('things-factory:reference-app:controller/create-data-sample-mockup')
+
+// parse variable javascript string pattern
+const replaceVariables = (keys, dic) => {
+  for (const k in keys) {
+    const matches = keys[k].match(/\$\{\w*\}/g)
+    matches &&
+      matches.forEach(m => {
+        keys[k] = keys[k].replace(m, dic[m.slice(2, -1)])
+      })
+  }
+  return keys
+}
+
+// It is required UTC date for Partitioning File System like AWS S3 from Athena.
+// ex) %YYYY, %MM, %DD
+const formatDate = (keys, _moment) => {
+  for (const k in keys) {
+    const matches = keys[k].match(/%\w*/g)
+    matches &&
+      matches.forEach(m => {
+        keys[k] = keys[k].replace(m, _moment.format(m.substr(1)))
+      })
+  }
+  return keys
+}
+
+/** 
+ * Normal Distribution
+ * https://stackoverflow.com/questions/25582882/javascript-math-random-normal-distribution-gaussian-bell-curve
+ */
+function normal_dist(min, max, skew) {
+  let u = 0, v = 0;
+  while(u === 0) u = Math.random() //Converting [0,1) to (0,1)
+  while(v === 0) v = Math.random()
+  let num = Math.sqrt( -2.0 * Math.log( u ) ) * Math.cos( 2.0 * Math.PI * v )
+  
+  num = num / 10.0 + 0.5 // Translate to 0 -> 1
+  if (num > 1 || num < 0) 
+    num = normal_dist(min, max, skew) // resample between 0 and 1 if out of range
+  
+  else{
+    num = Math.pow(num, skew) // Skew
+    num *= max - min // Stretch to fill range
+    num += min // offset to min
+  }
+  return num
+}
+
+function randomIndex(max = 2) {
+  return Math.floor(Math.random() * max)
+}
+
+function fillColumns(prev, next, dict) {
+  let values = []
+  try {
+    const gap = 20
+    const timeFormat = 'HH:mm'
+    const dateFormat = 'YYYY-MM-DD'
+    const atFormat = 'YYYY-MM-DD HH:mm:ss'
+    const now = moment()
+    const { type, spec, quota } = next
+    const isValidSpec = spec && Object.keys(spec).length
+    const arr = Array.from(Array(quota).keys())
+
+    if(type == 'number') {
+      if (isValidSpec) { /* generate values refer to spec info. */
+        const { CCP: { criticalLimits: { minimum, maximum } } } = spec
+        while (values.length < quota) {
+          let value = 0
+          if (typeof minimum === 'number' && typeof maximum === 'number') {
+            value = Math.floor(normal_dist(minimum, maximum, 1))
+          } else if (typeof minimum === 'number' && typeof maximum !== 'number') {
+            const max = minimum + gap
+            value = Math.floor(normal_dist(minimum, max, 1))
+          } else if (typeof minimum !== 'number' && typeof maximum === 'number') {
+            const min = maximum - gap
+            value = Math.floor(normal_dist(min, maximum, 1))
+          }
+          values.push(value)
+        }
+      } else { /* random number */
+        const p = Math.random()
+        const q = () => randomIndex(10) + 1
+        // fixed
+        if (p > 0.33) {
+          values = Array(quota).fill(q())
+        } else if(p > 0.66) {  /* or random */
+          values = arr.map(i => q())
+          
+        } else { /* or incremental */
+          values = arr.reduce((prev, i) => {
+            prev.push(!!prev.length ? prev[0] + i: q())
+            return prev
+          }, [])
+        }
+      }
+    } else if (type == 'select') {
+      const options = next['options']['options']
+      values = arr.map(i => options[randomIndex(options.length)]['value'])
+    } else if (type == 'boolean') {
+      values = arr.map(i => !!randomIndex())
+    } else if (next['tag'].toLowerCase().indexOf('time') >= 0) {
+      /* ex) startTime => 09:21 */
+      arr.reduce(curr => {
+        values.push(curr.format(timeFormat))
+        curr.add(randomIndex(60), 'minutes')
+        return curr
+      }, now)
+    } else if (next['tag'].indexOf('edAt') > 0) { 
+      /* ex) createdAt => '2022-06-15 09:21:30 */
+      arr.reduce(curr => {
+        values.push(curr.format(atFormat))
+        curr.add(randomIndex(12), 'hours')
+        return curr
+      }, now)
+    } else if (next['tag'].toLowerCase().indexOf('date') > 0) {
+      /* ex) startDate => '2022-06-15 */
+      arr.reduce(curr => {
+        values.push(curr.format(dateFormat))
+        curr.add(randomIndex(7), 'days')
+        return curr
+      }, now)
+    } else if (dict.hasOwnProperty(next['tag'])) {
+      values = Array(quota).fill(dict[next['tag']])
+    } else {
+      if (type == 'text') {
+        values = randomWords(quota)
+      }
+    }
+  } finally {
+    prev[next['tag']] = values
+    return prev
+  }
+}
+
+async function _generateMockupData(dataSources:any, tx:EntityManager) {
+  const { user, domain, dataSet, spec, products, operations } = dataSources
+
+  const collectedAt = new Date()
+
+  const timezone = dataSet.timezone || domain.timezone || 'UTC'
+  const format = 'YYYY-MM-DD'
+
+  // workDate ex) 2022-04-04
+  const { workDate, workShift } = await getWorkDateAndShift(domain, collectedAt, { timezone, format })
+  
+  // local time dataSet timezone or domain timezone or default 'UTC'
+  
+  // const collectedAt = dataSample.collectedAt || new Date()
+  const localDateTz = moment(collectedAt).tz(timezone)
+  const defaultPartitionKeys = {
+    domain: domain.subdomain,
+    datasetid: dataSet.id, /* It should not be 'data_set_id' as column name duplicated for Glue */
+    date: localDateTz.format(format), /* local time date */
+    workdate: workDate, /* working date */
+    workshift: workShift
+  }
+
+  var partitionKeys = {
+    ...defaultPartitionKeys,
+    ...dataSet.partitionKeys
+  }
+
+  // get product, operation as a map
+  const oper = operations[Math.floor(Math.random() * operations.length)]
+  const prod = products[Math.floor(Math.random() * products.length)]
+  const dict = {
+    operation: oper['name'],
+    operationId: oper['id'],
+    product: prod['name'],
+    productId: prod['id']
+  }
+
+  const sample = dataSet.dataItems.reduce((prev, next) => fillColumns(prev, next, dict), {})
+  debug('sample', sample)
+
+  partitionKeys = formatDate(partitionKeys, localDateTz)
+  partitionKeys = replaceVariables(partitionKeys, {
+    ...sample
+  })
+
+  const { ooc, oos, judgment } = DataUseCase.evaluate(dataSet, dataSet.dataItems, sample) || {}
+  const result = await tx.getRepository(DataSample).save({
+    name: dataSet.name,
+    description: dataSet.description,
+    useCase: dataSet.useCase,
+    dataSet: dataSet,
+    dataSetVersion: dataSet.version,
+    data: sample,
+    domain,
+    partitionKeys,
+    spec,
+    ooc,
+    oos,
+    judgment,
+    collectedAt,
+    workDate,
+    workShift,
+    creator: user,
+    updater: user
+  })
+
+  return true
+}
+
+export async function generateMockupData(
+  params: DataSampleMockupInfo,
+  context: {
+    state: {
+      domain: Domain
+      user: User
+      tx: EntityManager
+    }
+  }
+): Promise<Boolean> {
+  const { domain, user, tx } = context.state
+
+  const dataSet: DataSet = await tx.getRepository(DataSet).findOne({
+    where: { id: params.dataSetId }
+  })
+
+  const products = await tx.getRepository(Product).find({
+    select: [ 'id', 'name', 'description' ],
+    where: {
+      domain
+    }
+  })
+
+  const operations = await tx.getRepository(Operation).find({
+    select: [ 'id', 'name', 'description' ],
+    where: {
+      domain
+    }
+  })
+
+  const dataItems = dataSet.dataItems
+
+  const spec = dataItems.reduce((spec, dataItem) => {
+    spec[dataItem.tag] = {
+      ...dataItem.spec,
+      name: dataItem.name, /* do we need ? */
+      hidden: dataItem.hidden
+    }
+
+    return spec
+  }, {})
+
+  const promises = Array(params.numSamples).fill(0).map(x => {
+    return _generateMockupData({ domain, user, dataSet, products, operations, spec }, tx)
+  })
+
+  await Promise.all(promises)
+  return true
+}

package/server/controllers/index.ts

@@ -0,0 +1 @@
+export * from './create-data-sample-mockup'

package/server/service/data-sample-mockup/data-sample-mockup-mutation.ts

@@ -0,0 +1,18 @@
+import { Arg, Ctx, Directive, Mutation, Resolver } from 'type-graphql'
+
+import { DataSample } from '@things-factory/dataset'
+
+import { generateMockupData } from '../../controllers/create-data-sample-mockup'
+import { DataSampleMockupInfo } from './data-sample-mockup-type'
+
+const debug = require('debug')('things-factory:dataset:data-sample:data-sample-mutation')
+
+@Resolver(DataSample)
+export class DataSampleMockupMutation {
+  @Directive('@privilege(category: "data-sample", privilege: "mutation", domainOwnerGranted: true)')
+  @Directive('@transaction')
+  @Mutation(returns => Boolean, { description: 'To generate data sample one' })
+  async generateMockupData(@Arg('params') params: DataSampleMockupInfo, @Ctx() context: any): Promise<Boolean> {
+    return await generateMockupData(params, context)
+  }
+}

package/server/service/data-sample-mockup/data-sample-mockup-type.ts

@@ -0,0 +1,10 @@
+import { Field, InputType } from 'type-graphql'
+
+@InputType()
+export class DataSampleMockupInfo {
+  @Field()
+  dataSetId?: string
+
+  @Field({ nullable: true })
+  numSamples?: number
+}

package/server/service/data-sample-mockup/index.ts

@@ -0,0 +1,4 @@
+import { DataSampleMockupMutation } from './data-sample-mockup-mutation'
+
+export const entities = []
+export const resolvers = [DataSampleMockupMutation]

package/server/service/index.ts

@@ -2,15 +2,18 @@
 
 /* IMPORT ENTITIES AND RESOLVERS */
 import { entities as ReferenceEntities, resolvers as ReferenceResolvers } from './reference'
+import { entities as DataSampleMockupEntities, resolvers as DataSampleMockupMutation } from './data-sample-mockup'
 
 export const entities = [
   /* ENTITIES */
-  ...ReferenceEntities
+  ...ReferenceEntities,
+  ...DataSampleMockupEntities
 ]
 
 export const schema = {
   resolverClasses: [
     /* RESOLVER CLASSES */
-    ...ReferenceResolvers
+    ...ReferenceResolvers,
+    ...DataSampleMockupMutation
   ]
 }

package/server/service/reference/reference-mutation.ts

@@ -1,8 +1,10 @@
-import { Resolver, Mutation, Root, Arg, Ctx, Directive } from 'type-graphql'
-import { GraphQLUpload, FileUpload } from 'graphql-upload'
+import type { FileUpload } from 'graphql-upload/GraphQLUpload.js'
+import GraphQLUpload from 'graphql-upload/GraphQLUpload.js'
+import { Arg, Ctx, Mutation, Resolver, Root } from 'type-graphql'
+
+import { Attachment, createAttachments } from '@things-factory/attachment-base'
 import { publishProgress } from '@things-factory/shell'
 import { sleep } from '@things-factory/utils'
-import { Attachment, createAttachments } from '@things-factory/attachment-base'
 
 @Resolver()
 export class ReferenceMutation {

package/server/service/reference/reference-query.ts

@@ -1,4 +1,5 @@
-import { FileUpload, GraphQLUpload } from 'graphql-upload'
+import type { FileUpload } from 'graphql-upload/GraphQLUpload.js'
+import GraphQLUpload from 'graphql-upload/GraphQLUpload.js'
 import { Arg, Ctx, Query, Resolver, Root } from 'type-graphql'
 
 import { Attachment, createAttachments } from '@things-factory/attachment-base'
@@ -41,7 +42,7 @@ export class ReferenceQuery {
           return { data: image, name: filename, format: formatArr[formatArr.length - 1] }
         })
       )
-      const res = await extractInfos(files)
+      const res: any = await extractInfos(files)
       console.log('interpretImages response', JSON.stringify(res, null, ' '))
       return res.data?.extractInfos
     }
@@ -49,12 +50,12 @@ export class ReferenceQuery {
     return []
   }
 
-  streamToString (stream) {
-    const chunks = [];
+  streamToString(stream) {
+    const chunks = []
     return new Promise((resolve, reject) => {
-      stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
-      stream.on('error', (err) => reject(err));
-      stream.on('end', () => resolve(Buffer.concat(chunks).toString('base64')));
+      stream.on('data', chunk => chunks.push(Buffer.from(chunk)))
+      stream.on('error', err => reject(err))
+      stream.on('end', () => resolve(Buffer.concat(chunks).toString('base64')))
     })
   }
 

package/things-factory.config.js

@@ -12,6 +12,14 @@ export default {
       tagname: 'reference-app-main',
       page: 'reference-app-main'
     },
+    {
+      tagname: 'operation-master',
+      page: 'operation_master'
+    },
+    {
+      tagname: 'product-master',
+      page: 'product_master'
+    },
     {
       tagname: 'upload-page',
       page: 'upload-page'

package/translations/en.json

@@ -1,4 +1,5 @@
 {
+  "button.generate_mockup": "Generate Mockup Data",
   "field.company": "company",
   "field.barcode": "barcode",
   "field.active": "active",
@@ -7,5 +8,9 @@
   "field.role": "role",
   "field.dynamic_type": "dynamic type",
   "field.dynamic_value": "dynamic value",
-  "text.ox-data-report-grand-total": "grand total"
+  "label.num_samples": "Number of Samples",
+  "text.mockup data created successfully": "mockup data created successfully",
+  "text.ox-data-report-grand-total": "grand total",
+  "title.edit code": "edit code",
+  "title.generate_mockup": "Generate Mockup Data"
 }

package/translations/ko.json

@@ -1,4 +1,5 @@
 {
+  "button.generate_mockup": "테스트 데이타 생성",
   "field.company": "회사",
   "field.barcode": "바코드",
   "field.active": "활성화",
@@ -7,5 +8,10 @@
   "field.role": "역할",
   "field.dynamic_type": "변화 타입",
   "field.dynamic_value": "변화타입 값",
-  "text.ox-data-report-grand-total": "총합계"
+  "label.maximum value": "최대값",
+  "label.num_samples": "샘플 수",
+  "text.ox-data-report-grand-total": "총합계",
+  "text.mockup data created successfully": "테스트 데이타가 성공적으로 생성되었습니다",
+  "title.edit code": "코드 편집",
+  "title.generate_mockup": "테스트 데이타 생성"
 }

package/logs/application-2022-06-20-13.log

@@ -1,4 +0,0 @@
-2022-06-20T13:41:05+09:00 info: File Storage is Ready.
-2022-06-20T13:41:07+09:00 info: Database connection established
-2022-06-20T13:41:08+09:00 info: 🚀 Server ready at http://0.0.0.0:3000/graphql
-2022-06-20T13:41:08+09:00 info: 🚀 Subscriptions ready at ws://0.0.0.0:3000/subscriptions