@things-factory/dataset 8.0.0-beta.9 → 8.0.0

package/server/controllers/create-data-sample.ts

@@ -0,0 +1,426 @@
+import moment from 'moment-timezone'
+import { In } from 'typeorm'
+
+import { Attachment, createAttachment } from '@things-factory/attachment-base'
+import { Role } from '@things-factory/auth-base'
+import { logger } from '@things-factory/env'
+import { getRedirectSubdomainPath, pubsub } from '@things-factory/shell'
+import { getWorkDateAndShift } from '@things-factory/work-shift'
+import { Scenario, publishData } from '@things-factory/integration-base'
+import { Activity } from '@things-factory/worklist'
+import { issue } from '@things-factory/worklist/dist-server/controllers/activity-instance/issue'
+
+import { DataOoc, DataOocStatus } from '../service/data-ooc/data-ooc'
+import { DataSample } from '../service/data-sample/data-sample'
+import { NewDataSample } from '../service/data-sample/data-sample-type'
+import { DataSet } from '../service/data-set/data-set'
+import { DataUseCase } from './data-use-case'
+
+// See README.md at ## Data Samples
+process.env.TZ = 'UTC'
+
+const fillDataKeys = (dataKeySet, data = {}) => {
+  const keys = dataKeySet?.dataKeyItems || []
+  return keys.reduce((sum, key, index) => {
+    const value = data[key.dataKey]
+    if (value != null) {
+      sum[`key0${index + 1}`] = value instanceof Array ? value[0] : value
+    }
+    return sum
+  }, {})
+}
+
+// parse variable javascript string pattern
+const replaceVariables = (keys, dic) => {
+  for (const k in keys) {
+    const matches = keys[k].match(/\$\{\w*\}/g)
+    matches &&
+      matches.forEach(m => {
+        keys[k] = keys[k].replace(m, dic[m.slice(2, -1)])
+      })
+  }
+  return keys
+}
+
+// It is required UTC date for Partitioning File System like AWS S3 from Athena.
+// ex) %YYYY, %MM, %DD
+const formatDate = (keys, _moment) => {
+  for (const k in keys) {
+    const matches = keys[k].match(/%\w*/g)
+    matches &&
+      matches.forEach(m => {
+        keys[k] = keys[k].replace(m, _moment.format(m.substr(1)))
+      })
+  }
+  return keys
+}
+
+export async function createDataSample(newDataSample: NewDataSample, context: ResolverContext): Promise<DataSample> {
+  const { domain, user, tx } = context.state
+
+  const dataSet = await tx.getRepository(DataSet).findOne({
+    where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: newDataSample.dataSet.id },
+    relations: ['dataKeySet']
+  })
+
+  const { dataItems = [], tag: publishTag, normalScenarioId, outlierScenarioId } = dataSet
+  const collectedAt = newDataSample.collectedAt || new Date()
+
+  const timezone = dataSet.timezone || domain.timezone || 'UTC'
+  const format = 'YYYY-MM-DD'
+
+  // workDate ex) 2022-04-04
+  const { workDate, workShift } = await getWorkDateAndShift(domain, collectedAt, { timezone, format })
+
+  // local time dataSet timezone or domain timezone or default 'UTC'
+
+  const localDateTz = moment(collectedAt).tz(timezone)
+  const defaultPartitionKeys = {
+    domain: domain.subdomain,
+    datasetid: newDataSample.dataSet.id /* It should not be 'data_set_id' as column name duplicated for Glue */,
+    date: localDateTz.format(format) /* local time date */,
+    workdate: workDate /* working date */,
+    workshift: workShift
+  }
+
+  var partitionKeys = {
+    ...defaultPartitionKeys,
+    ...dataSet.partitionKeys
+  }
+
+  partitionKeys = formatDate(partitionKeys, localDateTz)
+  partitionKeys = replaceVariables(partitionKeys, newDataSample.data)
+
+  const dataKeys = fillDataKeys(dataSet?.dataKeySet, newDataSample.data)
+
+  const { ooc, oos, judgment } = DataUseCase.evaluate(dataSet, dataItems, newDataSample.data) || {}
+
+  const old = await tx.getRepository(DataSample).findOne({
+    where: {
+      domain: { id: domain.id },
+      dataSet: { id: dataSet.id },
+      collectedAt,
+      ...dataKeys
+    }
+  })
+
+  /* 
+    pre-processing for file attachment.
+    currently only support type of [FileUpload].
+    If [FileUpload[]] type needed, add 'files' type for dataset
+  */
+
+  const data = newDataSample.data
+  const attachments = []
+
+  for (let dataItem of dataItems) {
+    if (dataItem.type == 'file') {
+      const tag = dataItem.tag
+      const filesArray = data[tag]
+
+      if (tag && filesArray && filesArray.length > 0) {
+        let pathsArray = []
+
+        for (let files of filesArray) {
+          let paths = []
+
+          if (files instanceof Array) {
+            for (let file of files) {
+              if (file) {
+                const attachment = await createAttachment(
+                  null,
+                  {
+                    attachment: {
+                      file: file.file,
+                      refType: DataSample.name
+                    }
+                  },
+                  context
+                )
+
+                const fetched = await tx.getRepository(Attachment).findOneBy({ id: attachment.id })
+                if (fetched) {
+                  attachments.push(fetched)
+                  paths.push({
+                    id: fetched.id,
+                    mimetype: fetched.mimetype,
+                    name: fetched.name,
+                    fullpath: fetched.fullpath
+                  })
+                } else {
+                  throw `Failed to save file(${attachment.name})`
+                }
+              } else {
+                paths.push(null)
+              }
+            }
+          }
+
+          pathsArray.push(paths)
+        }
+
+        data[tag] = pathsArray
+      }
+    }
+  }
+
+  const dataSample = await tx.getRepository(DataSample).save({
+    ...old,
+    name: dataSet.name,
+    description: dataSet.description,
+    useCase: dataSet.useCase,
+    type: dataSet.type,
+    ...newDataSample,
+    ...dataKeys,
+    dataSetVersion: dataSet.version,
+    domain,
+    partitionKeys,
+    ooc,
+    oos,
+    judgment,
+    collectedAt,
+    workDate,
+    workShift,
+    creator: user,
+    updater: user
+  })
+
+  if (publishTag) {
+    publishData(publishTag, data, { domain, user })
+  }
+
+  /* post-process for for file attachment. */
+  if (attachments.length > 0) {
+    attachments.forEach(attachment => (attachment.refId = dataSample.id))
+    tx.getRepository(Attachment).save(attachments)
+  }
+
+  if (ooc || oos) {
+    const dataOoc = await tx.getRepository(DataOoc).save({
+      ...dataSample,
+      history: [
+        {
+          user: {
+            id: user?.id,
+            name: user?.name
+          },
+          state: DataOocStatus.ISSUED,
+          timestamp: collectedAt
+        }
+      ],
+      state: DataOocStatus.ISSUED,
+      dataSample: dataSample
+    })
+
+    if (outlierScenarioId) {
+      const scenario = await tx.getRepository(Scenario).findOne({
+        where: {
+          domain: domain.parentId ? { id: In([domain.id, domain.parentId]) } : { id: domain.id },
+          id: outlierScenarioId
+        },
+        relations: ['domain', 'steps', 'updater']
+      })
+
+      if (scenario) {
+        scenario.start({
+          instanceName: scenario.name + ':' + dataSet.name + ':' + dataSample.id,
+          domain,
+          user,
+          variables: {
+            dataOocId: dataOoc.id,
+            dataSampleId: dataSample.id,
+            dataSet: dataSet.id,
+            data,
+            ooc,
+            oos,
+            judgment,
+            collectedAt,
+            workDate,
+            workShift,
+            domain: {
+              id: domain.id,
+              subdomain: domain.subdomain,
+              name: domain.name
+            },
+            updator: {
+              id: user.id,
+              email: user.email,
+              name: user.name
+            }
+          }
+        })
+      } else {
+        console.error(`Cannot find the set outlier-scenario for the dataset(${dataSet.name}).`)
+      }
+    }
+
+    const activity = (await tx.getRepository(Activity).findOneBy({
+      domain: domain.parentId ? { id: In([domain.id, domain.parentId]) } : { id: domain.id },
+      name: 'OOC Review'
+    })) as Activity
+
+    if (activity) {
+      const assigneeRole =
+        dataSet.supervisoryRoleId &&
+        (await tx.getRepository(Role).findOneBy({
+          domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
+          id: dataSet.supervisoryRoleId
+        }))
+
+      // const assignees = dataSet.supervisoryRoleId
+      //   ? [{ type: 'Role', value: dataSet.supervisoryRoleId, assigneeRole }]
+      //   : []
+
+      /* 해당 dataset의 supervisor로 하여금, OOC를 리뷰하고 instruction을 작성해서, OOC 해결을 위한 태스크를 dataset assignees에게 지시하도록 한다. */
+      if (assigneeRole) {
+        const activityInstance = {
+          name: `[OOC 검토] ${dataSet.name}`,
+          description: dataSet.description,
+          activityId: activity.id,
+          dueAt: new Date(collectedAt.getTime() + (activity.standardTime || 24 * 60 * 60) * 1000),
+          input: {
+            dataOocId: dataOoc.id
+          },
+          assigneeRole,
+          threadsMin: 1,
+          threadsMax: 1,
+          approvalLine: []
+        }
+
+        dataOoc.reviewActivityInstance = await issue(activityInstance as any, context)
+        await tx.getRepository(DataOoc).save(dataOoc)
+      } else {
+        console.error(
+          `Assignees are not set. So Data OOC Review task for ${dataOoc.name}(${dataOoc.id}) could not be issued.`
+        )
+      }
+    } else {
+      console.warn('OOC Review Activity not installed.')
+    }
+
+    try {
+      // pubsub.publish('data-ooc', {
+      //   dataOoc,
+      //   supervisoryRoleId: dataSet.supervisoryRoleId
+      // })
+
+      pubsub.publish('notification', {
+        notification: {
+          domain,
+          type: 'error',
+          title: `[OOC] ${dataSet.name}`,
+          body: `Data OOC occurred on '${dataSet.name}'`,
+          url: getRedirectSubdomainPath(context, domain.subdomain, `/data-ooc/${dataOoc.id}`),
+          timestamp: collectedAt
+        }
+      })
+    } catch (err) {
+      logger.error('Notification', err)
+    }
+  } else {
+    if (normalScenarioId) {
+      const scenario = await tx.getRepository(Scenario).findOne({
+        where: {
+          domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
+          id: normalScenarioId
+        },
+        relations: ['domain', 'steps', 'updater']
+      })
+
+      if (scenario) {
+        scenario.start({
+          instanceName: scenario.name + ':' + dataSet.name + ':' + dataSample.id,
+          domain,
+          user,
+          variables: {
+            dataSampleId: dataSample.id,
+            dataSet: dataSet.id,
+            data,
+            ooc,
+            oos,
+            judgment,
+            collectedAt,
+            workDate,
+            workShift,
+            domain: {
+              id: domain.id,
+              subdomain: domain.subdomain,
+              name: domain.name
+            },
+            updator: {
+              id: user.id,
+              email: user.email,
+              name: user.name
+            }
+          }
+        })
+      } else {
+        console.error(`Cannot find the set normal-scenario for the dataset(${dataSet.name}).`)
+      }
+    }
+
+    if (dataSet.requiresReview) {
+      const activity = (await tx.getRepository(Activity).findOneBy({
+        domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
+        name: 'Data Review'
+      })) as Activity
+
+      if (activity) {
+        const assigneeRole =
+          dataSet.supervisoryRoleId &&
+          (await tx.getRepository(Role).findOneBy({
+            domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
+            id: dataSet.supervisoryRoleId
+          }))
+
+        // const assignees = dataSet.supervisoryRoleId
+        //   ? [{ type: 'Role', value: dataSet.supervisoryRoleId, assigneeRole }]
+        //   : []
+
+        /* 해당 dataset의 supervisor로 하여금, data를 리뷰하고 instruction을 작성해서, approvalLine을 이용해서 승인을 한다. */
+        if (assigneeRole) {
+          dataSample.dataItems = dataItems
+          const activityInstance = {
+            name: `[Data 검토] ${dataSet.name}`,
+            description: dataSet.description,
+            activityId: activity.id,
+            dueAt: new Date(collectedAt.getTime() + (activity.standardTime || 24 * 60 * 60) * 1000),
+            input: {
+              dataSampleId: dataSample.id
+            },
+            assigneeRole,
+            threadsMin: 1,
+            threadsMax: 1,
+            approvalLine: dataSet.reviewApprovalLine
+          }
+
+          dataSample.reviewActivityInstance = await issue(activityInstance as any, context)
+          await tx.getRepository(DataSample).save(dataSample)
+
+          try {
+            pubsub.publish('notification', {
+              notification: {
+                domain,
+                type: 'info',
+                title: `[Data Review] ${dataSet.name}`,
+                body: `Data Review occurred on '${dataSet.name}'`,
+                url: getRedirectSubdomainPath(context, domain.subdomain, `/data-sample/${dataSample.id}`),
+                timestamp: collectedAt
+              }
+            })
+          } catch (err) {
+            logger.error('Notification', err)
+          }
+        } else {
+          console.error(
+            `Assignees are not set. So Data Review task for ${dataSet.name}(${dataSet.id}) could not be issued.`
+          )
+        }
+      } else {
+        console.error('Data Review Activity not installed.')
+      }
+    }
+  }
+
+  return dataSample
+}

package/server/controllers/data-use-case.ts

@@ -0,0 +1,98 @@
+import { DataItem } from '../service/data-set/data-item-type'
+import { DataSet } from '../service/data-set/data-set'
+
+export type DataItemSpec = {
+  type: string
+  label: string
+  name: string
+  property?: { [option: string]: any }
+}
+
+export type DataItemSpecSet = {
+  name: string
+  description: string
+  help?: string
+  specs: DataItemSpec[]
+}
+
+export type EvaluationResult = {
+  oos: boolean
+  ooc: boolean
+  judgment?: { [tag: string]: { ooc: boolean; oos: boolean } }
+}
+
+export abstract class DataUseCase {
+  static registry: { [name: string]: DataUseCase } = {}
+
+  public static registerUseCase(name: string, provider: DataUseCase) {
+    DataUseCase.registry[name] = provider
+  }
+
+  public static getUseCaseNames() {
+    return Object.keys(DataUseCase.registry)
+  }
+
+  public static getUseCase(name: string): DataUseCase | undefined {
+    return DataUseCase.registry[name]
+  }
+
+  public static evaluate(dataSet: DataSet, dataItems: DataItem[], data: any): EvaluationResult {
+    var ooc = false
+    var oos = false
+    var judgment: { [tag: string]: { ooc: boolean; oos: boolean } } = {}
+
+    if (!dataSet.useCase) {
+      return { ooc, oos }
+    }
+
+    const useCaseNames = dataSet.useCase.split(',').map(useCaseName => useCaseName.trim())
+    const useCases = useCaseNames.map(useCaseName => DataUseCase.getUseCase(useCaseName)).filter(useCase => !!useCase)
+
+    for (let i = 0; i < dataItems.length; i++) {
+      const dataItem = dataItems[i]
+      const { active, tag } = dataItem
+      if (!active || !tag) {
+        continue
+      }
+
+      let values: any | any[] = data && data[tag]
+      if (values == null) {
+        continue // TODO what if in case no value ?
+      }
+
+      if (!(values instanceof Array)) {
+        values = [values]
+      }
+
+      let oocForTag = false
+      let oosForTag = false
+
+      for (let j = 0; j < useCases.length; j++) {
+        const useCase = useCases[j]
+
+        const specs = dataItem.spec?.[dataSet.useCase]
+        if (!specs) {
+          continue
+        }
+
+        const result = useCase.evaluate(specs, values)
+
+        if (result) {
+          oocForTag ||= result.ooc
+          oosForTag ||= result.oos
+          ooc ||= result.ooc
+          oos ||= result.oos
+        }
+      }
+
+      judgment[tag] = {
+        ooc: oocForTag,
+        oos: oosForTag
+      }
+    }
+
+    return { ooc, oos, judgment }
+  }
+
+  public abstract evaluate(specs: any, values: any[]): EvaluationResult
+}