@things-factory/dataset 5.0.0-alpha.4 → 5.0.0-alpha.40

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@things-factory/dataset",
-  "version": "5.0.0-alpha.4",
+  "version": "5.0.0-alpha.40",
   "main": "dist-server/index.js",
   "browser": "client/index.js",
   "things-factory": true,
@@ -24,17 +24,22 @@
     "migration:create": "node ../../node_modules/typeorm/cli.js migration:create -d ./server/migrations"
   },
   "dependencies": {
-    "@operato/data-grist": "1.0.0-alpha.4",
-    "@operato/dataset": "1.0.0-alpha.4",
-    "@operato/graphql": "1.0.0-alpha.4",
-    "@operato/i18n": "1.0.0-alpha.4",
-    "@operato/layout": "1.0.0-alpha.4",
-    "@operato/shell": "1.0.0-alpha.4",
-    "@operato/styles": "1.0.0-alpha.4",
-    "@operato/utils": "1.0.0-alpha.4",
-    "@things-factory/auth-base": "^5.0.0-alpha.4",
-    "@things-factory/env": "^5.0.0-alpha.4",
-    "@things-factory/shell": "^5.0.0-alpha.4"
+    "@operato/app": "1.0.0-beta.7",
+    "@operato/data-grist": "1.0.0-beta.7",
+    "@operato/dataset": "1.0.0-beta.7",
+    "@operato/graphql": "1.0.0-beta.7",
+    "@operato/i18n": "1.0.0-beta.7",
+    "@operato/layout": "1.0.0-beta.7",
+    "@operato/shell": "1.0.0-beta.7",
+    "@operato/styles": "1.0.0-beta.7",
+    "@operato/utils": "1.0.0-beta.7",
+    "@things-factory/auth-base": "^5.0.0-alpha.40",
+    "@things-factory/board-service": "^5.0.0-alpha.40",
+    "@things-factory/env": "^5.0.0-alpha.40",
+    "@things-factory/shell": "^5.0.0-alpha.40",
+    "@things-factory/work-shift": "^5.0.0-alpha.40",
+    "cron-parser": "^4.3.0",
+    "moment-timezone": "^0.5.34"
   },
-  "gitHead": "77253f86954dc6f5c14356ea85887c323fd2511a"
+  "gitHead": "c64462e85c9b744250f5649d28c0ec4bd56a1b38"
 }

package/server/controllers/create-data-sample.ts

@@ -0,0 +1,177 @@
+import moment from 'moment-timezone'
+import { EntityManager } from 'typeorm'
+
+import { User } from '@things-factory/auth-base'
+import { Domain, getRedirectSubdomainPath, pubsub } from '@things-factory/shell'
+
+import { DataItem } from '../service/data-item/data-item'
+import { DataOoc, DataOocStatus } from '../service/data-ooc/data-ooc'
+import { DataSample } from '../service/data-sample/data-sample'
+import { NewDataSample } from '../service/data-sample/data-sample-type'
+import { DataSet } from '../service/data-set/data-set'
+import { DataUseCase } from './data-use-case'
+import { getWorkDateAndShift } from '@things-factory/work-shift'
+
+
+const debug = require('debug')('things-factory:dataset:controller/save-data-sample')
+
+// parse variable javascript string pattern
+const replaceVariables = (keys, dic) => {
+  for (const k in keys) {
+    const matches = keys[k].match(/\$\{\w*\}/g)
+    matches &&
+      matches.forEach(m => {
+        keys[k] = keys[k].replace(m, dic[m.slice(2, -1)])
+      })
+  }
+  return keys
+}
+
+// It is required UTC date for Partitioning File System like AWS S3 from Athena.
+// ex) %YYYY, %MM, %DD
+const formatDate = (keys, _moment) => {
+  for (const k in keys) {
+    const matches = keys[k].match(/%\w*/g)
+    matches &&
+      matches.forEach(m => {
+        keys[k] = keys[k].replace(m, _moment.format(m.substr(1)))
+      })
+  }
+  return keys
+}
+
+export async function createDataSample(
+  dataSample: NewDataSample,
+  context: {
+    state: {
+      domain: Domain
+      user: User
+      tx: EntityManager
+    }
+  }
+): Promise<DataSample> {
+  const { domain, user, tx } = context.state
+
+  const dataSet = await tx.getRepository(DataSet).findOne({
+    where: { id: dataSample.dataSet.id }
+  })
+
+  const dataItems = await tx.getRepository(DataItem).find({
+    where: {
+      domain,
+      dataSet
+    },
+    order: {
+      sequence: 'DESC'
+    }
+  })
+
+  const spec = dataItems.reduce((spec, dataItem) => {
+    spec[dataItem.tag] = {
+      ...dataItem.spec,
+      name: dataItem.name, /* do we need ? */
+      hidden: dataItem.hidden
+    }
+
+    return spec
+  }, {})
+
+  const collectedAt = dataSample.collectedAt || new Date()
+
+  const timezone = dataSet.timezone || domain.timezone || 'UTC'
+  const format = 'YYYY-MM-DD'
+
+  // workDate ex) 2022-04-04
+  const { workDate, workShift } = await getWorkDateAndShift(domain, collectedAt, { timezone, format })
+  
+  // local time dataSet timezone or domain timezone or default 'UTC'
+  
+  // const collectedAt = dataSample.collectedAt || new Date()
+  const localDateTz = moment(collectedAt).tz(timezone)
+  const defaultPartitionKeys = {
+    domain: domain.subdomain,
+    datasetid: dataSample.dataSet.id, /* It should not be 'data_set_id' as column name duplicated for Glue */
+    date: localDateTz.format(format), /* local time date */
+    workdate: workDate, /* working date */
+    workshift: workShift
+  }
+
+  var partitionKeys = {
+    ...defaultPartitionKeys,
+    ...dataSet.partitionKeys
+  }
+
+  partitionKeys = formatDate(partitionKeys, localDateTz)
+  partitionKeys = replaceVariables(partitionKeys, {
+    ...dataSample.data
+  })
+
+  const { ooc, oos } = DataUseCase.evaluate(dataSet, dataItems, dataSample.data) || {}
+  const result = await tx.getRepository(DataSample).save({
+    name: dataSet.name,
+    description: dataSet.description,
+    useCase: dataSet.useCase,
+    ...dataSample,
+    domain,
+    partitionKeys,
+    spec,
+    ooc,
+    oos,
+    collectedAt,
+    workDate,
+    workShift,
+    creator: user,
+    updater: user
+  })
+
+  if (ooc || oos) {
+    const dataOoc = await tx.getRepository(DataOoc).save({
+      name: dataSet.name,
+      description: dataSet.description,
+      useCase: dataSet.useCase,
+      dataSet,
+      dataSample: result,
+      data: dataSample.data,
+      rawData: dataSample.rawData,
+      domain,
+      partitionKeys,
+      spec,
+      ooc,
+      oos,
+      history: [
+        {
+          user: {
+            id: user.id,
+            name: user.name
+          },
+          state: DataOocStatus.CREATED,
+          timestamp: Date.now()
+        }
+      ],
+      state: DataOocStatus.CREATED,
+      workDate,
+      workShift,
+      collectedAt,
+      creator: user,
+      updater: user
+    })
+
+    pubsub.publish('data-ooc', {
+      dataOoc,
+      supervisoryRoleId: dataSet.supervisoryRoleId
+    })
+
+    pubsub.publish('notification', {
+      notification: {
+        domain,
+        type: 'error',
+        title: `Data OOC occurred on '${dataSet.name}'`,
+        body: `Data OOC occurred on '${dataSet.name}'`,
+        url: getRedirectSubdomainPath(context, domain.subdomain, `/data-ooc/${dataOoc.id}`),
+        timestamp: collectedAt
+      }
+    })
+  }
+
+  return result
+}

package/server/controllers/data-use-case.ts

@@ -0,0 +1,85 @@
+import { DataItem, DataSet } from 'server/service'
+
+export type DataItemSpec = {
+  type: string
+  label: string
+  name: string
+  property: { [option: string]: any }
+}
+
+export type DataItemSpecSet = {
+  name: string
+  description: string
+  specs: DataItemSpec[]
+}
+
+export type EvaluationResult = { oos: boolean; ooc: boolean }
+
+export abstract class DataUseCase {
+  static registry: { [name: string]: DataUseCase } = {}
+
+  public static registerUseCase(name: string, provider: DataUseCase) {
+    DataUseCase.registry[name] = provider
+  }
+
+  public static getUseCaseNames() {
+    return Object.keys(DataUseCase.registry)
+  }
+
+  public static getUseCase(name: string): DataUseCase | undefined {
+    return DataUseCase.registry[name]
+  }
+
+  public static evaluate(dataSet: DataSet, dataItems: DataItem[], data: any): EvaluationResult {
+    var ooc = false
+    var oos = false
+
+    if (!dataSet.useCase) {
+      return { ooc, oos }
+    }
+
+    const useCaseNames = dataSet.useCase.split(',').map(useCaseName => useCaseName.trim())
+    const useCases = useCaseNames.map(useCaseName => DataUseCase.getUseCase(useCaseName)).filter(useCase => !!useCase)
+
+    for (let i = 0; i < dataItems.length; i++) {
+      const dataItem = dataItems[i]
+      const { active, tag } = dataItem
+      if (!active || !tag) {
+        continue
+      }
+
+      let values: any | any[] = data[tag]
+      if (typeof values === 'undefined') {
+        continue // TODO what if in case no value ?
+      }
+
+      if (!(values instanceof Array)) {
+        values = [values]
+      }
+
+      for (let j = 0; j < useCases.length; j++) {
+        const useCase = useCases[j]
+
+        const specs = dataItem.spec?.[dataSet.useCase]
+        if (!specs) {
+          continue
+        }
+
+        const result = useCase.evaluate(specs, values)
+
+        if (result) {
+          ooc ||= result.ooc
+          oos ||= result.oos
+        }
+
+        // if (ooc && oos) {
+        //   return { ooc, oos }
+        // }
+      }
+    }
+
+    return { ooc, oos }
+  }
+
+  public abstract evaluate(specs: any, values: any[]): EvaluationResult
+}

package/server/controllers/index.ts

@@ -0,0 +1 @@
+export * from './data-use-case'

package/server/index.ts

@@ -1,5 +1,6 @@
 import './routes'
 
+export * from './controllers'
 export * from './migrations'
 export * from './middlewares'
 export * from './service'

package/server/routes.ts

@@ -1,7 +1,8 @@
 import { getConnection } from 'typeorm'
 
-import { DataSample } from './service'
-import { DataItem } from './service/data-item/data-item'
+import { User } from '@things-factory/auth-base'
+
+import { createDataSample } from './controllers/create-data-sample'
 import { DataSensor } from './service/data-sensor/data-sensor'
 
 const debug = require('debug')('things-factory:dataset:routes')
@@ -47,41 +48,26 @@ process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRout
 
       const domain = sensor.domain
       const dataSet = sensor.dataSet
-      const user = sensor.appliance
-
-      const dataItems = await tx.getRepository(DataItem).find({
+      const user: User = await tx.getRepository(User).findOne({
         where: {
-          domain,
-          dataSet
-        },
-        order: {
-          sequence: 'DESC'
+          reference: sensor.appliance.id,
+          userType: 'appliance'
         }
       })
 
-      var spec = {} as any
-
-      dataItems.forEach(dataItem => {
-        spec[dataItem.name] = dataItem.spec
-      })
-
-      await tx.getRepository(DataSample).save({
-        domain,
-        name: dataSet.name,
-        description: dataSet.description,
-        partitionKeys: dataSet.partitionKeys,
-        dataSet,
-        data,
-        rawData,
-        spec,
-        source: deviceId,
-        collectedAt: new Date(timestamp),
-        creator: user,
-        updater: user
-      })
+      return await createDataSample(
+        {
+          dataSet,
+          data,
+          rawData,
+          source: deviceId,
+          collectedAt: new Date(timestamp)
+        },
+        { state: { domain, user, tx } }
+      )
     })
 
-    return 'OK'
+    context.status = 200
   })
 })
 

package/server/service/data-item/data-item-mutation.ts

@@ -24,9 +24,14 @@ export class DataItemMutation {
     await dataItemRepo.delete({ domain, dataSet: dataSetId })
 
     for (let i = 0; i < patches.length; i++) {
+      const patch = patches[i]
+      if (!patch.options) {
+        patch.options = {}
+      }
+
       const result = await dataItemRepo.save({
         quota: 1,
-        ...patches[i],
+        ...patch,
         sequence: i,
         dataSet,
         domain,

package/server/service/data-item/data-item-query.ts

@@ -2,7 +2,7 @@ import { Arg, Args, Ctx, Directive, FieldResolver, Query, Resolver, Root } from
 import { getRepository } from 'typeorm'
 
 import { User } from '@things-factory/auth-base'
-import { convertListParams, Domain, ListParam } from '@things-factory/shell'
+import { Domain, getQueryBuilderFromListParams, ListParam } from '@things-factory/shell'
 
 import { DataSet } from '../data-set/data-set'
 import { DataItem } from './data-item'
@@ -25,8 +25,14 @@ export class DataItemQuery {
   async dataItems(@Args() params: ListParam, @Ctx() context: any): Promise<DataItemList> {
     const { domain } = context.state
 
-    const convertedParams = convertListParams(params, domain.id)
-    const [items, total] = await getRepository(DataItem).findAndCount(convertedParams)
+    const queryBuilder = getQueryBuilderFromListParams({
+      repository: getRepository(DataItem),
+      params,
+      domain,
+      alias: 'dataitem'
+    })
+
+    const [items, total] = await queryBuilder.getManyAndCount()
 
     return { items, total }
   }

package/server/service/data-item/data-item-type.ts

@@ -1,16 +1,14 @@
-import { Field, ID, InputType, Int, ObjectType } from 'type-graphql'
-
-import { ScalarObject } from '@things-factory/shell'
-
 import { DataItem, DataItemType } from './data-item'
+import { Field, ID, InputType, Int, ObjectType } from 'type-graphql'
+import { ObjectRef, ScalarObject } from '@things-factory/shell'
 
 @InputType()
 export class DataItemPatch {
   @Field(type => ID, { nullable: true })
   id?: string
 
-  @Field({ nullable: true })
-  dataSetId?: string
+  @Field(type => ObjectRef, { nullable: true })
+  dataSet?: ObjectRef
 
   @Field({ nullable: true })
   name?: string
@@ -30,12 +28,18 @@ export class DataItemPatch {
   @Field(type => ScalarObject, { nullable: true })
   options?: ScalarObject
 
+  @Field({ nullable: true })
+  unit?: string
+
   @Field(type => Int, { nullable: true })
   quota?: number
 
   @Field({ nullable: true })
   active?: boolean
 
+  @Field({ nullable: true })
+  hidden?: boolean
+
   @Field(type => ScalarObject, { nullable: true })
   spec?: ScalarObject
 

package/server/service/data-item/data-item.ts

@@ -1,4 +1,3 @@
-import { Field, ID, Int, ObjectType, registerEnumType } from 'type-graphql'
 import {
   Column,
   CreateDateColumn,
@@ -9,11 +8,11 @@ import {
   RelationId,
   UpdateDateColumn
 } from 'typeorm'
-
-import { User } from '@things-factory/auth-base'
 import { Domain, ScalarObject } from '@things-factory/shell'
+import { Field, ID, Int, ObjectType, registerEnumType } from 'type-graphql'
 
 import { DataSet } from '../data-set/data-set'
+import { User } from '@things-factory/auth-base'
 
 export enum DataItemType {
   number = 'number',
@@ -54,7 +53,7 @@ export class DataItem {
   description?: string
 
   @ManyToOne(type => DataSet, dataSet => dataSet.dataItems, { onDelete: 'CASCADE' })
-  @Field(type => DataSet, { nullable: true })
+  @Field(type => DataSet, { nullable: false })
   dataSet?: DataSet
 
   @RelationId((dataItem: DataItem) => dataItem.dataSet)
@@ -76,6 +75,12 @@ export class DataItem {
   @Field({ nullable: true })
   active?: boolean
 
+  @Column({
+    nullable: true
+  })
+  @Field({ nullable: true })
+  hidden?: boolean
+
   @Column({
     nullable: true
   })
@@ -86,6 +91,12 @@ export class DataItem {
   @Field(type => ScalarObject, { nullable: true })
   options?: ScalarObject
 
+  @Column({
+    nullable: true
+  })
+  @Field({ nullable: true })
+  unit?: string
+
   @Column({
     nullable: true
   })

package/server/service/data-ooc/data-ooc-mutation.ts

@@ -0,0 +1,150 @@
+import { Arg, Ctx, Directive, Mutation, Resolver } from 'type-graphql'
+
+import { DataOoc, DataOocStatus } from './data-ooc'
+import { DataOocPatch, NewDataOoc } from './data-ooc-type'
+
+@Resolver(DataOoc)
+export class DataOocMutation {
+  @Directive('@privilege(category: "data-ooc", privilege: "mutation", domainOwnerGranted: true)')
+  @Directive('@transaction')
+  @Mutation(returns => DataOoc, { description: 'To create new DataOoc' })
+  async createDataOoc(@Arg('dataOoc') dataOoc: NewDataOoc, @Ctx() context: any): Promise<DataOoc> {
+    const { domain, user, tx } = context.state
+
+    const state = dataOoc.state || DataOocStatus.CREATED
+    const history = [
+      {
+        user: {
+          id: user.id,
+          name: user.name
+        },
+        state,
+        timestamp: Date.now()
+      }
+    ]
+
+    return await tx.getRepository(DataOoc).save({
+      ...dataOoc,
+      state,
+      domain,
+      history,
+      creator: user,
+      updater: user
+    })
+  }
+
+  @Directive('@privilege(category: "data-ooc", privilege: "mutation", domainOwnerGranted: true)')
+  @Directive('@transaction')
+  @Mutation(returns => DataOoc, { description: 'To modify DataOoc information' })
+  async updateDataOoc(@Arg('id') id: string, @Arg('patch') patch: DataOocPatch, @Ctx() context: any): Promise<DataOoc> {
+    const { domain, user, tx } = context.state
+
+    const repository = tx.getRepository(DataOoc)
+    const dataOoc = await repository.findOne({
+      where: { domain, id }
+    })
+
+    const state = patch.state || dataOoc.state
+
+    const history = dataOoc.history || []
+    history.push({
+      user: {
+        id: user.id,
+        name: user.name
+      },
+      state,
+      comment: patch.correctiveAction || '',
+      timestamp: Date.now()
+    })
+
+    const more = {} as any
+
+    if (dataOoc.state !== patch.state) {
+      if (patch.state === DataOocStatus.CORRECTED) {
+        more.corrector = user
+      } else {
+        more.corrector = null
+      }
+    }
+
+    return await repository.save({
+      ...dataOoc,
+      ...patch,
+      ...more,
+      state,
+      history,
+      updater: user
+    })
+  }
+
+  // @Directive('@privilege(category: "data-ooc", privilege: "mutation", domainOwnerGranted: true)')
+  // @Directive('@transaction')
+  // @Mutation(returns => [DataOoc], { description: "To modify multiple DataOoc' information" })
+  // async updateMultipleDataOoc(
+  //   @Arg('patches', type => [DataOocPatch]) patches: DataOocPatch[],
+  //   @Ctx() context: any
+  // ): Promise<DataOoc[]> {
+  //   const { domain, user, tx } = context.state
+
+  //   let results = []
+  //   const _createRecords = patches.filter((patch: any) => patch.cuFlag.toUpperCase() === '+')
+  //   const _updateRecords = patches.filter((patch: any) => patch.cuFlag.toUpperCase() === 'M')
+  //   const dataOocRepo = tx.getRepository(DataOoc)
+
+  //   if (_createRecords.length > 0) {
+  //     for (let i = 0; i < _createRecords.length; i++) {
+  //       const newRecord = _createRecords[i]
+
+  //       const result = await dataOocRepo.save({
+  //         ...newRecord,
+  //         domain,
+  //         creator: user,
+  //         updater: user
+  //       })
+
+  //       results.push({ ...result, cuFlag: '+' })
+  //     }
+  //   }
+
+  //   if (_updateRecords.length > 0) {
+  //     for (let i = 0; i < _updateRecords.length; i++) {
+  //       const newRecord = _updateRecords[i]
+  //       const dataOoc = await dataOocRepo.findOne(newRecord.id)
+
+  //       const result = await dataOocRepo.save({
+  //         ...dataOoc,
+  //         ...newRecord,
+  //         updater: user
+  //       })
+
+  //       results.push({ ...result, cuFlag: 'M' })
+  //     }
+  //   }
+
+  //   return results
+  // }
+
+  // @Directive('@privilege(category: "data-ooc", privilege: "mutation", domainOwnerGranted: true)')
+  // @Directive('@transaction')
+  // @Mutation(returns => Boolean, { description: 'To delete DataOoc' })
+  // async deleteDataOoc(@Arg('id') id: string, @Ctx() context: any): Promise<boolean> {
+  //   const { domain, tx } = context.state
+
+  //   await tx.getRepository(DataOoc).delete({ domain, id })
+  //   return true
+  // }
+
+  // @Directive('@privilege(category: "data-ooc", privilege: "mutation", domainOwnerGranted: true)')
+  // @Directive('@transaction')
+  // @Mutation(returns => Boolean, { description: 'To delete multiple dataOocs' })
+  // async deleteDataOocs(@Arg('ids', type => [String]) ids: string[], @Ctx() context: any): Promise<boolean> {
+  //   const { domain, tx } = context.state
+
+  //   await tx.getRepository(DataOoc).delete({
+  //     domain,
+  //     id: In(ids)
+  //   })
+
+  //   return true
+  // }
+}