@things-factory/dataset 8.0.0 → 9.0.0-beta.3

package/server/controllers/create-data-sample.ts

@@ -1,426 +0,0 @@
-import moment from 'moment-timezone'
-import { In } from 'typeorm'
-
-import { Attachment, createAttachment } from '@things-factory/attachment-base'
-import { Role } from '@things-factory/auth-base'
-import { logger } from '@things-factory/env'
-import { getRedirectSubdomainPath, pubsub } from '@things-factory/shell'
-import { getWorkDateAndShift } from '@things-factory/work-shift'
-import { Scenario, publishData } from '@things-factory/integration-base'
-import { Activity } from '@things-factory/worklist'
-import { issue } from '@things-factory/worklist/dist-server/controllers/activity-instance/issue'
-
-import { DataOoc, DataOocStatus } from '../service/data-ooc/data-ooc'
-import { DataSample } from '../service/data-sample/data-sample'
-import { NewDataSample } from '../service/data-sample/data-sample-type'
-import { DataSet } from '../service/data-set/data-set'
-import { DataUseCase } from './data-use-case'
-
-// See README.md at ## Data Samples
-process.env.TZ = 'UTC'
-
-const fillDataKeys = (dataKeySet, data = {}) => {
-  const keys = dataKeySet?.dataKeyItems || []
-  return keys.reduce((sum, key, index) => {
-    const value = data[key.dataKey]
-    if (value != null) {
-      sum[`key0${index + 1}`] = value instanceof Array ? value[0] : value
-    }
-    return sum
-  }, {})
-}
-
-// parse variable javascript string pattern
-const replaceVariables = (keys, dic) => {
-  for (const k in keys) {
-    const matches = keys[k].match(/\$\{\w*\}/g)
-    matches &&
-      matches.forEach(m => {
-        keys[k] = keys[k].replace(m, dic[m.slice(2, -1)])
-      })
-  }
-  return keys
-}
-
-// It is required UTC date for Partitioning File System like AWS S3 from Athena.
-// ex) %YYYY, %MM, %DD
-const formatDate = (keys, _moment) => {
-  for (const k in keys) {
-    const matches = keys[k].match(/%\w*/g)
-    matches &&
-      matches.forEach(m => {
-        keys[k] = keys[k].replace(m, _moment.format(m.substr(1)))
-      })
-  }
-  return keys
-}
-
-export async function createDataSample(newDataSample: NewDataSample, context: ResolverContext): Promise<DataSample> {
-  const { domain, user, tx } = context.state
-
-  const dataSet = await tx.getRepository(DataSet).findOne({
-    where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: newDataSample.dataSet.id },
-    relations: ['dataKeySet']
-  })
-
-  const { dataItems = [], tag: publishTag, normalScenarioId, outlierScenarioId } = dataSet
-  const collectedAt = newDataSample.collectedAt || new Date()
-
-  const timezone = dataSet.timezone || domain.timezone || 'UTC'
-  const format = 'YYYY-MM-DD'
-
-  // workDate ex) 2022-04-04
-  const { workDate, workShift } = await getWorkDateAndShift(domain, collectedAt, { timezone, format })
-
-  // local time dataSet timezone or domain timezone or default 'UTC'
-
-  const localDateTz = moment(collectedAt).tz(timezone)
-  const defaultPartitionKeys = {
-    domain: domain.subdomain,
-    datasetid: newDataSample.dataSet.id /* It should not be 'data_set_id' as column name duplicated for Glue */,
-    date: localDateTz.format(format) /* local time date */,
-    workdate: workDate /* working date */,
-    workshift: workShift
-  }
-
-  var partitionKeys = {
-    ...defaultPartitionKeys,
-    ...dataSet.partitionKeys
-  }
-
-  partitionKeys = formatDate(partitionKeys, localDateTz)
-  partitionKeys = replaceVariables(partitionKeys, newDataSample.data)
-
-  const dataKeys = fillDataKeys(dataSet?.dataKeySet, newDataSample.data)
-
-  const { ooc, oos, judgment } = DataUseCase.evaluate(dataSet, dataItems, newDataSample.data) || {}
-
-  const old = await tx.getRepository(DataSample).findOne({
-    where: {
-      domain: { id: domain.id },
-      dataSet: { id: dataSet.id },
-      collectedAt,
-      ...dataKeys
-    }
-  })
-
-  /* 
-    pre-processing for file attachment.
-    currently only support type of [FileUpload].
-    If [FileUpload[]] type needed, add 'files' type for dataset
-  */
-
-  const data = newDataSample.data
-  const attachments = []
-
-  for (let dataItem of dataItems) {
-    if (dataItem.type == 'file') {
-      const tag = dataItem.tag
-      const filesArray = data[tag]
-
-      if (tag && filesArray && filesArray.length > 0) {
-        let pathsArray = []
-
-        for (let files of filesArray) {
-          let paths = []
-
-          if (files instanceof Array) {
-            for (let file of files) {
-              if (file) {
-                const attachment = await createAttachment(
-                  null,
-                  {
-                    attachment: {
-                      file: file.file,
-                      refType: DataSample.name
-                    }
-                  },
-                  context
-                )
-
-                const fetched = await tx.getRepository(Attachment).findOneBy({ id: attachment.id })
-                if (fetched) {
-                  attachments.push(fetched)
-                  paths.push({
-                    id: fetched.id,
-                    mimetype: fetched.mimetype,
-                    name: fetched.name,
-                    fullpath: fetched.fullpath
-                  })
-                } else {
-                  throw `Failed to save file(${attachment.name})`
-                }
-              } else {
-                paths.push(null)
-              }
-            }
-          }
-
-          pathsArray.push(paths)
-        }
-
-        data[tag] = pathsArray
-      }
-    }
-  }
-
-  const dataSample = await tx.getRepository(DataSample).save({
-    ...old,
-    name: dataSet.name,
-    description: dataSet.description,
-    useCase: dataSet.useCase,
-    type: dataSet.type,
-    ...newDataSample,
-    ...dataKeys,
-    dataSetVersion: dataSet.version,
-    domain,
-    partitionKeys,
-    ooc,
-    oos,
-    judgment,
-    collectedAt,
-    workDate,
-    workShift,
-    creator: user,
-    updater: user
-  })
-
-  if (publishTag) {
-    publishData(publishTag, data, { domain, user })
-  }
-
-  /* post-process for for file attachment. */
-  if (attachments.length > 0) {
-    attachments.forEach(attachment => (attachment.refId = dataSample.id))
-    tx.getRepository(Attachment).save(attachments)
-  }
-
-  if (ooc || oos) {
-    const dataOoc = await tx.getRepository(DataOoc).save({
-      ...dataSample,
-      history: [
-        {
-          user: {
-            id: user?.id,
-            name: user?.name
-          },
-          state: DataOocStatus.ISSUED,
-          timestamp: collectedAt
-        }
-      ],
-      state: DataOocStatus.ISSUED,
-      dataSample: dataSample
-    })
-
-    if (outlierScenarioId) {
-      const scenario = await tx.getRepository(Scenario).findOne({
-        where: {
-          domain: domain.parentId ? { id: In([domain.id, domain.parentId]) } : { id: domain.id },
-          id: outlierScenarioId
-        },
-        relations: ['domain', 'steps', 'updater']
-      })
-
-      if (scenario) {
-        scenario.start({
-          instanceName: scenario.name + ':' + dataSet.name + ':' + dataSample.id,
-          domain,
-          user,
-          variables: {
-            dataOocId: dataOoc.id,
-            dataSampleId: dataSample.id,
-            dataSet: dataSet.id,
-            data,
-            ooc,
-            oos,
-            judgment,
-            collectedAt,
-            workDate,
-            workShift,
-            domain: {
-              id: domain.id,
-              subdomain: domain.subdomain,
-              name: domain.name
-            },
-            updator: {
-              id: user.id,
-              email: user.email,
-              name: user.name
-            }
-          }
-        })
-      } else {
-        console.error(`Cannot find the set outlier-scenario for the dataset(${dataSet.name}).`)
-      }
-    }
-
-    const activity = (await tx.getRepository(Activity).findOneBy({
-      domain: domain.parentId ? { id: In([domain.id, domain.parentId]) } : { id: domain.id },
-      name: 'OOC Review'
-    })) as Activity
-
-    if (activity) {
-      const assigneeRole =
-        dataSet.supervisoryRoleId &&
-        (await tx.getRepository(Role).findOneBy({
-          domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
-          id: dataSet.supervisoryRoleId
-        }))
-
-      // const assignees = dataSet.supervisoryRoleId
-      //   ? [{ type: 'Role', value: dataSet.supervisoryRoleId, assigneeRole }]
-      //   : []
-
-      /* 해당 dataset의 supervisor로 하여금, OOC를 리뷰하고 instruction을 작성해서, OOC 해결을 위한 태스크를 dataset assignees에게 지시하도록 한다. */
-      if (assigneeRole) {
-        const activityInstance = {
-          name: `[OOC 검토] ${dataSet.name}`,
-          description: dataSet.description,
-          activityId: activity.id,
-          dueAt: new Date(collectedAt.getTime() + (activity.standardTime || 24 * 60 * 60) * 1000),
-          input: {
-            dataOocId: dataOoc.id
-          },
-          assigneeRole,
-          threadsMin: 1,
-          threadsMax: 1,
-          approvalLine: []
-        }
-
-        dataOoc.reviewActivityInstance = await issue(activityInstance as any, context)
-        await tx.getRepository(DataOoc).save(dataOoc)
-      } else {
-        console.error(
-          `Assignees are not set. So Data OOC Review task for ${dataOoc.name}(${dataOoc.id}) could not be issued.`
-        )
-      }
-    } else {
-      console.warn('OOC Review Activity not installed.')
-    }
-
-    try {
-      // pubsub.publish('data-ooc', {
-      //   dataOoc,
-      //   supervisoryRoleId: dataSet.supervisoryRoleId
-      // })
-
-      pubsub.publish('notification', {
-        notification: {
-          domain,
-          type: 'error',
-          title: `[OOC] ${dataSet.name}`,
-          body: `Data OOC occurred on '${dataSet.name}'`,
-          url: getRedirectSubdomainPath(context, domain.subdomain, `/data-ooc/${dataOoc.id}`),
-          timestamp: collectedAt
-        }
-      })
-    } catch (err) {
-      logger.error('Notification', err)
-    }
-  } else {
-    if (normalScenarioId) {
-      const scenario = await tx.getRepository(Scenario).findOne({
-        where: {
-          domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
-          id: normalScenarioId
-        },
-        relations: ['domain', 'steps', 'updater']
-      })
-
-      if (scenario) {
-        scenario.start({
-          instanceName: scenario.name + ':' + dataSet.name + ':' + dataSample.id,
-          domain,
-          user,
-          variables: {
-            dataSampleId: dataSample.id,
-            dataSet: dataSet.id,
-            data,
-            ooc,
-            oos,
-            judgment,
-            collectedAt,
-            workDate,
-            workShift,
-            domain: {
-              id: domain.id,
-              subdomain: domain.subdomain,
-              name: domain.name
-            },
-            updator: {
-              id: user.id,
-              email: user.email,
-              name: user.name
-            }
-          }
-        })
-      } else {
-        console.error(`Cannot find the set normal-scenario for the dataset(${dataSet.name}).`)
-      }
-    }
-
-    if (dataSet.requiresReview) {
-      const activity = (await tx.getRepository(Activity).findOneBy({
-        domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
-        name: 'Data Review'
-      })) as Activity
-
-      if (activity) {
-        const assigneeRole =
-          dataSet.supervisoryRoleId &&
-          (await tx.getRepository(Role).findOneBy({
-            domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
-            id: dataSet.supervisoryRoleId
-          }))
-
-        // const assignees = dataSet.supervisoryRoleId
-        //   ? [{ type: 'Role', value: dataSet.supervisoryRoleId, assigneeRole }]
-        //   : []
-
-        /* 해당 dataset의 supervisor로 하여금, data를 리뷰하고 instruction을 작성해서, approvalLine을 이용해서 승인을 한다. */
-        if (assigneeRole) {
-          dataSample.dataItems = dataItems
-          const activityInstance = {
-            name: `[Data 검토] ${dataSet.name}`,
-            description: dataSet.description,
-            activityId: activity.id,
-            dueAt: new Date(collectedAt.getTime() + (activity.standardTime || 24 * 60 * 60) * 1000),
-            input: {
-              dataSampleId: dataSample.id
-            },
-            assigneeRole,
-            threadsMin: 1,
-            threadsMax: 1,
-            approvalLine: dataSet.reviewApprovalLine
-          }
-
-          dataSample.reviewActivityInstance = await issue(activityInstance as any, context)
-          await tx.getRepository(DataSample).save(dataSample)
-
-          try {
-            pubsub.publish('notification', {
-              notification: {
-                domain,
-                type: 'info',
-                title: `[Data Review] ${dataSet.name}`,
-                body: `Data Review occurred on '${dataSet.name}'`,
-                url: getRedirectSubdomainPath(context, domain.subdomain, `/data-sample/${dataSample.id}`),
-                timestamp: collectedAt
-              }
-            })
-          } catch (err) {
-            logger.error('Notification', err)
-          }
-        } else {
-          console.error(
-            `Assignees are not set. So Data Review task for ${dataSet.name}(${dataSet.id}) could not be issued.`
-          )
-        }
-      } else {
-        console.error('Data Review Activity not installed.')
-      }
-    }
-  }
-
-  return dataSample
-}

package/server/controllers/data-use-case.ts

@@ -1,98 +0,0 @@
-import { DataItem } from '../service/data-set/data-item-type'
-import { DataSet } from '../service/data-set/data-set'
-
-export type DataItemSpec = {
-  type: string
-  label: string
-  name: string
-  property?: { [option: string]: any }
-}
-
-export type DataItemSpecSet = {
-  name: string
-  description: string
-  help?: string
-  specs: DataItemSpec[]
-}
-
-export type EvaluationResult = {
-  oos: boolean
-  ooc: boolean
-  judgment?: { [tag: string]: { ooc: boolean; oos: boolean } }
-}
-
-export abstract class DataUseCase {
-  static registry: { [name: string]: DataUseCase } = {}
-
-  public static registerUseCase(name: string, provider: DataUseCase) {
-    DataUseCase.registry[name] = provider
-  }
-
-  public static getUseCaseNames() {
-    return Object.keys(DataUseCase.registry)
-  }
-
-  public static getUseCase(name: string): DataUseCase | undefined {
-    return DataUseCase.registry[name]
-  }
-
-  public static evaluate(dataSet: DataSet, dataItems: DataItem[], data: any): EvaluationResult {
-    var ooc = false
-    var oos = false
-    var judgment: { [tag: string]: { ooc: boolean; oos: boolean } } = {}
-
-    if (!dataSet.useCase) {
-      return { ooc, oos }
-    }
-
-    const useCaseNames = dataSet.useCase.split(',').map(useCaseName => useCaseName.trim())
-    const useCases = useCaseNames.map(useCaseName => DataUseCase.getUseCase(useCaseName)).filter(useCase => !!useCase)
-
-    for (let i = 0; i < dataItems.length; i++) {
-      const dataItem = dataItems[i]
-      const { active, tag } = dataItem
-      if (!active || !tag) {
-        continue
-      }
-
-      let values: any | any[] = data && data[tag]
-      if (values == null) {
-        continue // TODO what if in case no value ?
-      }
-
-      if (!(values instanceof Array)) {
-        values = [values]
-      }
-
-      let oocForTag = false
-      let oosForTag = false
-
-      for (let j = 0; j < useCases.length; j++) {
-        const useCase = useCases[j]
-
-        const specs = dataItem.spec?.[dataSet.useCase]
-        if (!specs) {
-          continue
-        }
-
-        const result = useCase.evaluate(specs, values)
-
-        if (result) {
-          oocForTag ||= result.ooc
-          oosForTag ||= result.oos
-          ooc ||= result.ooc
-          oos ||= result.oos
-        }
-      }
-
-      judgment[tag] = {
-        ooc: oocForTag,
-        oos: oosForTag
-      }
-    }
-
-    return { ooc, oos, judgment }
-  }
-
-  public abstract evaluate(specs: any, values: any[]): EvaluationResult
-}