@things-factory/dataset 8.0.0-beta.9 → 8.0.2

package/server/controllers/finalize-data-collection.ts

@@ -0,0 +1,388 @@
+const statistics = require('simple-statistics')
+const deepClone = require('lodash/cloneDeep')
+
+import moment from 'moment-timezone'
+import { In } from 'typeorm'
+
+import { Sorting, getQueryBuilderFromListParams } from '@things-factory/shell'
+import { logger } from '@things-factory/env'
+import {
+  getDateRangeForWorkDate,
+  getDateRangeForWorkShift,
+  getLatestWorkDateAndShift,
+  getSummaryScheduleForWorkDate,
+  getSummaryScheduleForWorkShift
+} from '@things-factory/work-shift'
+
+import { DataSample } from '../service/data-sample/data-sample'
+import { DataSet, DataSetSummaryPeriodType } from '../service/data-set/data-set'
+
+import { DataSummary } from '../service/data-summary/data-summary'
+import { DataKeyItem } from '../service/data-key-set/data-key-item-type'
+
+import { DataItem } from 'service'
+
+const STAT_FUNCTION_MAP = {
+  sum: 'sum',
+  mean: 'mean',
+  stddev: 'standardDeviation',
+  variance: 'variance',
+  min: 'min',
+  max: 'max',
+  range: 'range',
+  median: 'median',
+  mode: 'mode'
+}
+
+const compareKeys = (dataKeyItems: DataKeyItem[], summary: Partial<DataSummary>, sample: DataSample): boolean => {
+  return dataKeyItems.every((item, index) => {
+    const prop = `key0${index + 1}`
+    return sample[prop] === summary[prop]
+  })
+}
+
+const buildKeysFromSample = (dataKeyItems: DataKeyItem[], sample: DataSample): Partial<DataSummary> => {
+  return dataKeyItems.reduce((sum, item, index) => {
+    const prop = `key0${index + 1}`
+    sum[prop] = sample[prop]
+
+    return sum
+  }, {} as Partial<DataSummary>)
+}
+
+const buildKeySortingList = (dataKeyItems: DataKeyItem[]): Sorting[] => {
+  return dataKeyItems.reduce((sum, item, index) => {
+    const name = `key0${index + 1}`
+    sum.push({ name, desc: true })
+    return sum
+  }, [])
+}
+
+const calculateSummary = (dataItems: DataItem[], base: { [tag: string]: any[] }) => {
+  return dataItems.reduce((summary, item) => {
+    const tag = item.tag
+
+    const data = base[tag]
+      .flat(Infinity)
+      .map(Number)
+      .filter(item => !isNaN(item))
+
+    if (data.length > 0) {
+      try {
+        switch (item.stat) {
+          case 'range':
+            summary[tag] = statistics.max(data) - statistics.min(data)
+            break
+
+          default:
+            const functionName = STAT_FUNCTION_MAP[item.stat]
+            summary[tag] = (functionName && statistics[functionName](data)) || ''
+        }
+      } catch (err) {
+        summary[tag] = null
+        console.error(err)
+      }
+    } else {
+      summary[tag] = null
+    }
+
+    return summary
+  }, {})
+}
+
+const fillSummaryResult = (dataSummary: Partial<DataSummary>, dataItems: DataItem[], base: { [tag: string]: any[] }): void => {
+  const summary = calculateSummary(dataItems, base)
+
+  dataSummary.summary = summary
+  dataItems.slice(0, 4).forEach((dataItem, idx) => {
+    const value = Number(summary[dataItem.tag])
+    dataSummary[`data0${idx + 1}`] = isNaN(value) ? null : value
+  })
+}
+
+async function getLatestTimesForPeriod(periodType: DataSetSummaryPeriodType, context: ResolverContext): Promise<{ date?: string; period?: string; range: Date[] }> {
+  const { domain } = context.state
+  const now = moment()
+
+  if (periodType == DataSetSummaryPeriodType.Hour) {
+    const begin = now.clone().subtract(1, 'hour').startOf('hour')
+    const end = now.clone().startOf('hour')
+    const date = begin.clone().tz(domain.timezone)
+
+    return {
+      date: date.format('YYYY-MM-DD'),
+      period: date.format('HH'),
+      range: [begin.toDate(), end.toDate()]
+    }
+  } else if (periodType == DataSetSummaryPeriodType.WorkShift) {
+    const { workDate, workShift, shiftRange } = await getLatestWorkDateAndShift(domain, new Date())
+
+    return { date: workDate, period: workShift, range: shiftRange }
+  } else if (periodType == DataSetSummaryPeriodType.WorkDate) {
+    const { workDate, dateRange } = await getLatestWorkDateAndShift(domain, new Date())
+
+    return { date: workDate, range: dateRange }
+  } else if (periodType == DataSetSummaryPeriodType.Day) {
+    const begin = now.clone().subtract(1, 'day').startOf('day')
+    const end = now.clone().startOf('day')
+    const date = begin.clone().tz(domain.timezone)
+
+    return {
+      date: date.format('YYYY-MM-DD'),
+      range: [begin.toDate(), end.toDate()]
+    }
+  }
+}
+
+async function getTimesForPeriod(
+  periodType: DataSetSummaryPeriodType,
+  date: string,
+  period: string,
+  context: ResolverContext
+): Promise<{ date?: string; period?: string; range: Date[] }> {
+  const { domain } = context.state
+
+  if (periodType == DataSetSummaryPeriodType.Hour) {
+    const theDate = moment.tz(`${date} ${period}:00:00`, 'YYYY-MM-DD HH:mm:ss', domain.timezone)
+
+    const begin = theDate.clone().startOf('hour').toDate()
+    const end = theDate.clone().add(+1, 'hour').startOf('hour').toDate()
+
+    return {
+      date,
+      period,
+      range: [begin, end]
+    }
+  } else if (periodType == DataSetSummaryPeriodType.WorkShift) {
+    const range = await getDateRangeForWorkShift(domain, date, period)
+
+    return { date, period, range }
+  } else if (periodType == DataSetSummaryPeriodType.WorkDate) {
+    const range = await getDateRangeForWorkDate(domain, date)
+
+    return { date, range }
+  } else if (periodType == DataSetSummaryPeriodType.Day) {
+    const theDate = moment.tz(`${date} 00:00:00`, 'YYYY-MM-DD HH:mm:ss', domain.timezone)
+
+    const begin = theDate.clone().startOf('day').toDate()
+    const end = theDate.clone().add(1, 'day').startOf('day').toDate()
+
+    return {
+      date: moment(begin).tz(domain.timezone).format('YYYY-MM-DD'),
+      range: [begin, end]
+    }
+  }
+}
+
+export async function getDataFinalizeCrontabSchedule(dataSet: DataSet, context: ResolverContext): Promise<string> {
+  const { domain, user, tx } = context.state
+
+  try {
+    const { summaryPeriod } = dataSet
+
+    if (summaryPeriod == DataSetSummaryPeriodType.Hour) {
+      return '0 5 * * * *'
+    } else if (summaryPeriod == DataSetSummaryPeriodType.WorkShift) {
+      return await getSummaryScheduleForWorkShift(domain)
+    } else if (summaryPeriod == DataSetSummaryPeriodType.WorkDate) {
+      return await getSummaryScheduleForWorkDate(domain)
+    } else if (summaryPeriod == DataSetSummaryPeriodType.Day) {
+      return '0 10 0 * * *'
+    }
+  } catch (err) {
+    console.error(err)
+  }
+}
+
+export async function finalizeLatestDataCollection(dataSetId: string, context: ResolverContext): Promise<boolean> {
+  const { domain, user, tx } = context.state
+
+  try {
+    const dataSet = await tx.getRepository(DataSet).findOne({
+      where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: dataSetId },
+      relations: ['dataKeySet']
+    })
+
+    const dataKeyItems = dataSet.dataKeySet?.dataKeyItems || []
+    const dataItems = dataSet.dataItems.filter(item => item.stat)
+    const initialSummary = dataItems.reduce((sum, item) => {
+      sum[item.tag] = []
+      return sum
+    }, {})
+
+    const { date, period, range } = await getLatestTimesForPeriod(dataSet.summaryPeriod, context)
+    const limit = 100
+    var page = 1
+
+    var summaries: Partial<DataSummary>[] = []
+    var summary: Partial<DataSummary>
+
+    do {
+      const samples = await getQueryBuilderFromListParams({
+        repository: tx.getRepository(DataSample),
+        domain,
+        params: {
+          filters: [{ name: 'dataSetId', operator: 'eq', value: dataSetId }],
+          pagination: { page, limit },
+          sortings: [...buildKeySortingList(dataKeyItems), { name: 'collectedAt', desc: true }]
+        },
+        alias: 'datasample'
+      })
+        // The 'Between' operator includes the 'to' time in the filtering, making it unsuitable for the desired use case.
+        // .andWhere({ collectedAt: Between.apply(null, range) })
+        .andWhere('datasample.collectedAt >= :from', { from: range[0] })
+        .andWhere('datasample.collectedAt < :to', { to: range[1] })
+        .getMany()
+
+      for (const sample of samples) {
+        if (!summary || !compareKeys(dataKeyItems, summary, sample)) {
+          if (summary) {
+            fillSummaryResult(summary, dataItems, summary.summary)
+            summaries.push(summary)
+          }
+
+          summary = {
+            domain,
+            name: dataSet.name,
+            description: dataSet.description,
+            date,
+            period,
+            dataSet,
+            ...buildKeysFromSample(dataKeyItems, sample),
+            count: 0,
+            countOoc: 0,
+            countOos: 0,
+            summary: deepClone(initialSummary),
+            updater: user,
+            creator: user
+          }
+        }
+
+        summary.count++
+        sample.ooc && summary.countOoc++
+        sample.oos && summary.countOos++
+
+        dataItems.forEach(item => {
+          summary.summary[item.tag].push(sample.data[item.tag])
+        })
+      }
+
+      if (samples.length < limit) {
+        if (summary) {
+          fillSummaryResult(summary, dataItems, summary.summary)
+          summaries.push(summary)
+        }
+        break
+      }
+
+      page++
+    } while (true)
+
+    tx.getRepository(DataSummary).upsert(summaries, ['domain', 'dataSet', 'key01', 'key02', 'key03', 'key04', 'key05', 'date', 'period'])
+
+    return true
+  } catch (e) {
+    logger.error(e)
+  }
+
+  return false
+}
+
+export async function finalizeDataCollection(dataSetId: string, date: string, period: string, context: ResolverContext): Promise<boolean> {
+  const { domain, user, tx } = context.state
+
+  try {
+    const dataSet =
+      dataSetId &&
+      (await tx.getRepository(DataSet).findOne({
+        where: { domain: In([domain.id, domain.parentId].filter(Boolean)), id: dataSetId },
+        relations: ['dataKeySet']
+      }))
+
+    const dataKeyItems = dataSet.dataKeySet?.dataKeyItems || []
+    const dataItems = dataSet.dataItems.filter(item => item.stat)
+    const initialSummary = dataItems.reduce((sum, item) => {
+      sum[item.tag] = []
+      return sum
+    }, {})
+
+    const times = await getTimesForPeriod(dataSet.summaryPeriod, date, period, context)
+    const range = times.range
+    period = times.period
+
+    const limit = 100
+    var page = 1
+
+    var summaries: Partial<DataSummary>[] = []
+    var summary: Partial<DataSummary>
+
+    do {
+      const samples = await getQueryBuilderFromListParams({
+        repository: tx.getRepository(DataSample),
+        params: {
+          filters: [{ name: 'dataSetId', operator: 'eq', value: dataSetId }],
+          pagination: { page, limit },
+          sortings: [...buildKeySortingList(dataKeyItems), { name: 'collectedAt', desc: true }]
+        },
+        domain,
+        alias: 'datasample'
+      })
+        // The 'Between' operator includes the 'to' time in the filtering, making it unsuitable for the desired use case.
+        // .andWhere({ collectedAt: Between.apply(null, range) })
+        .andWhere('datasample.collectedAt >= :from', { from: range[0] })
+        .andWhere('datasample.collectedAt < :to', { to: range[1] })
+        .getMany()
+
+      for (const sample of samples) {
+        if (!summary || !compareKeys(dataKeyItems, summary, sample)) {
+          if (summary) {
+            fillSummaryResult(summary, dataItems, summary.summary)
+            summaries.push(summary)
+          }
+
+          summary = {
+            domain,
+            name: dataSet.name,
+            description: dataSet.description,
+            date,
+            period,
+            dataSet,
+            ...buildKeysFromSample(dataKeyItems, sample),
+            count: 0,
+            countOoc: 0,
+            countOos: 0,
+            summary: deepClone(initialSummary),
+            updater: user,
+            creator: user
+          }
+        }
+
+        summary.count++
+        sample.ooc && summary.countOoc++
+        sample.oos && summary.countOos++
+
+        dataItems.forEach(item => {
+          summary.summary[item.tag].push(sample.data[item.tag])
+        })
+      }
+
+      if (samples.length < limit) {
+        if (summary) {
+          fillSummaryResult(summary, dataItems, summary.summary)
+          summaries.push(summary)
+        }
+        break
+      }
+
+      page++
+    } while (true)
+
+    tx.getRepository(DataSummary).upsert(summaries, ['domain', 'dataSet', 'key01', 'key02', 'key03', 'key04', 'key05', 'date', 'period'])
+
+    return true
+  } catch (e) {
+    logger.error(e)
+  }
+
+  return false
+}

package/server/controllers/index.ts

@@ -0,0 +1,6 @@
+export * from './create-data-sample'
+export * from './create-data-ooc'
+export * from './issue-ooc-review'
+export * from './issue-ooc-resolve'
+export * from './data-use-case'
+export * from './query-data-summary-by-period'

package/server/controllers/issue-data-collection-task.ts

@@ -0,0 +1,70 @@
+import { In } from 'typeorm'
+
+import { Domain, getDataSource } from '@things-factory/shell'
+import { Activity, ActivityInstance, issue } from '@things-factory/worklist'
+
+import { DataSet } from '../service/data-set/data-set'
+
+export async function issueDataCollectionTask(
+  domainId: string,
+  dataSetId: string,
+  context: ResolverContext
+): Promise<ActivityInstance | void> {
+  await getDataSource().transaction(async tx => {
+    const domain = await tx.getRepository(Domain).findOneBy({ id: domainId })
+
+    if (!domain) {
+      throw new Error(`domain(${domainId}) not found`)
+    }
+
+    const dataSet = await tx.getRepository(DataSet).findOne({
+      where: {
+        domain: {
+          id: In([domain.id, domain.parentId].filter(Boolean))
+        },
+        id: dataSetId
+      },
+      relations: ['entryRole']
+    })
+
+    const activity = (await tx.getRepository(Activity).findOneBy({
+      domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
+      name: 'Collect Data'
+    })) as Activity
+
+    if (activity) {
+      const { entryRole, timeLimit } = dataSet
+
+      /* 해당 dataset에 대한 데이타 수집 태스크를 dataset entryRole에게 할당한다. */
+      if (entryRole) {
+        const activityInstance = {
+          name: `[Data 수집] ${dataSet.name}`,
+          description: dataSet.description,
+          activityId: activity.id,
+          dueAt: new Date(Date.now() + (timeLimit || activity.standardTime || 24 * 60 * 60) * 1000),
+          input: {
+            dataSetId: dataSet.id,
+            dataSetName: dataSet.name
+          },
+          assigneeRole: entryRole,
+          threadsMin: 1,
+          threadsMax: 1
+        }
+
+        context.state = {
+          ...context.state,
+          domain,
+          tx
+        }
+
+        return await issue(activityInstance, context)
+      } else {
+        throw new Error(
+          `Data Entry Role not set. So Data Collect Activity for ${dataSet.name}($dataSet.id) could not be issued.`
+        )
+      }
+    } else {
+      throw new Error(`Data Collect Activity is not installed.`)
+    }
+  })
+}

package/server/controllers/issue-ooc-resolve.ts

@@ -0,0 +1,58 @@
+import { In } from 'typeorm'
+import { Activity } from '@things-factory/worklist'
+import { issue } from '@things-factory/worklist/dist-server/controllers/activity-instance/issue'
+import { Role } from '@things-factory/auth-base'
+
+import { DataOoc } from '../service/data-ooc/data-ooc'
+import { DataSet } from 'service'
+
+export async function issueOocResolve(
+  dataOoc: DataOoc,
+  dataSet: DataSet,
+  correctiveInstruction: string,
+  context: ResolverContext
+): Promise<void> {
+  const { domain, user, tx } = context.state
+  const { resolverRoleId, outlierApprovalLine } = dataSet
+
+  const activity = (await tx.getRepository(Activity).findOneBy({
+    domain: domain.parentId ? In([domain.id, domain.parentId]) : { id: domain.id },
+    name: 'OOC Resolve'
+  })) as Activity
+
+  if (activity) {
+    const assigneeRole =
+      resolverRoleId &&
+      (await tx.getRepository(Role).findOneBy({
+        domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
+        id: resolverRoleId
+      }))
+
+    /* 해당 dataset의 작업 담당자(resolverRole)에게 OOC 해결을 위한 태스크를 지시한다. */
+    if (assigneeRole) {
+      const activityInstance = {
+        name: `[OOC 조치] ${dataSet.name}`,
+        description: dataSet.description,
+        activityId: activity.id,
+        dueAt: new Date(Date.now() + (activity.standardTime || 24 * 60 * 60) * 1000),
+        input: {
+          dataOocId: dataOoc.id,
+          instruction: correctiveInstruction
+        },
+        assigneeRole,
+        threadsMin: 1,
+        threadsMax: 1,
+        approvalLine: outlierApprovalLine
+      }
+
+      dataOoc.resolveActivityInstance = await issue(activityInstance, context)
+      await tx.getRepository(DataOoc).save(dataOoc)
+    } else {
+      console.error(
+        `Assignees are not set. So, Data OOC Resolve task for ${dataOoc.name}(${dataOoc.id}) could not be issued.`
+      )
+    }
+  } else {
+    console.error('OOC Resolve Activity not installed.')
+  }
+}

package/server/controllers/issue-ooc-review.ts

@@ -0,0 +1,52 @@
+import { In } from 'typeorm'
+
+import { Role } from '@things-factory/auth-base'
+import { Activity } from '@things-factory/worklist'
+import { issue } from '@things-factory/worklist/dist-server/controllers/activity-instance/issue'
+
+import { DataSet } from '../service/data-set/data-set'
+import { DataOoc } from '../service/data-ooc/data-ooc'
+
+export async function issueOocReview(dataOoc: DataOoc, dataSet: DataSet, context: ResolverContext): Promise<void> {
+  const { domain, tx } = context.state
+  const { collectedAt } = dataOoc
+
+  const activity = (await tx.getRepository(Activity).findOneBy({
+    domain: domain.parentId ? { id: In([domain.id, domain.parentId]) } : { id: domain.id },
+    name: 'OOC Review'
+  })) as Activity
+
+  if (activity) {
+    const assigneeRole =
+      dataSet.supervisoryRoleId &&
+      (await tx.getRepository(Role).findOneBy({
+        domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
+        id: dataSet.supervisoryRoleId
+      }))
+
+    if (assigneeRole) {
+      const activityInstance = {
+        name: `[OOC 검토] ${dataSet.name}`,
+        description: dataSet.description,
+        activityId: activity.id,
+        dueAt: new Date(collectedAt.getTime() + (activity.standardTime || 24 * 60 * 60) * 1000),
+        input: {
+          dataOocId: dataOoc.id
+        },
+        assigneeRole,
+        threadsMin: 1,
+        threadsMax: 1,
+        approvalLine: []
+      }
+
+      dataOoc.reviewActivityInstance = await issue(activityInstance as any, context)
+      await tx.getRepository(DataOoc).save(dataOoc)
+    } else {
+      console.error(
+        `Assignees are not set. So Data OOC Review task for ${dataOoc.name}(${dataOoc.id}) could not be issued.`
+      )
+    }
+  } else {
+    console.warn('OOC Review Activity not installed.')
+  }
+}