@things-factory/dataset 8.0.0 → 9.0.0-beta.3

package/server/controllers/finalize-data-collection.ts

@@ -1,388 +0,0 @@
-const statistics = require('simple-statistics')
-const deepClone = require('lodash/cloneDeep')
-
-import moment from 'moment-timezone'
-import { In } from 'typeorm'
-
-import { Sorting, getQueryBuilderFromListParams } from '@things-factory/shell'
-import { logger } from '@things-factory/env'
-import {
-  getDateRangeForWorkDate,
-  getDateRangeForWorkShift,
-  getLatestWorkDateAndShift,
-  getSummaryScheduleForWorkDate,
-  getSummaryScheduleForWorkShift
-} from '@things-factory/work-shift'
-
-import { DataSample } from '../service/data-sample/data-sample'
-import { DataSet, DataSetSummaryPeriodType } from '../service/data-set/data-set'
-
-import { DataSummary } from '../service/data-summary/data-summary'
-import { DataKeyItem } from '../service/data-key-set/data-key-item-type'
-
-import { DataItem } from 'service'
-
-const STAT_FUNCTION_MAP = {
-  sum: 'sum',
-  mean: 'mean',
-  stddev: 'standardDeviation',
-  variance: 'variance',
-  min: 'min',
-  max: 'max',
-  range: 'range',
-  median: 'median',
-  mode: 'mode'
-}
-
-const compareKeys = (dataKeyItems: DataKeyItem[], summary: Partial<DataSummary>, sample: DataSample): boolean => {
-  return dataKeyItems.every((item, index) => {
-    const prop = `key0${index + 1}`
-    return sample[prop] === summary[prop]
-  })
-}
-
-const buildKeysFromSample = (dataKeyItems: DataKeyItem[], sample: DataSample): Partial<DataSummary> => {
-  return dataKeyItems.reduce((sum, item, index) => {
-    const prop = `key0${index + 1}`
-    sum[prop] = sample[prop]
-
-    return sum
-  }, {} as Partial<DataSummary>)
-}
-
-const buildKeySortingList = (dataKeyItems: DataKeyItem[]): Sorting[] => {
-  return dataKeyItems.reduce((sum, item, index) => {
-    const name = `key0${index + 1}`
-    sum.push({ name, desc: true })
-    return sum
-  }, [])
-}
-
-const calculateSummary = (dataItems: DataItem[], base: { [tag: string]: any[] }) => {
-  return dataItems.reduce((summary, item) => {
-    const tag = item.tag
-
-    const data = base[tag]
-      .flat(Infinity)
-      .map(Number)
-      .filter(item => !isNaN(item))
-
-    if (data.length > 0) {
-      try {
-        switch (item.stat) {
-          case 'range':
-            summary[tag] = statistics.max(data) - statistics.min(data)
-            break
-
-          default:
-            const functionName = STAT_FUNCTION_MAP[item.stat]
-            summary[tag] = (functionName && statistics[functionName](data)) || ''
-        }
-      } catch (err) {
-        summary[tag] = null
-        console.error(err)
-      }
-    } else {
-      summary[tag] = null
-    }
-
-    return summary
-  }, {})
-}
-
-const fillSummaryResult = (dataSummary: Partial<DataSummary>, dataItems: DataItem[], base: { [tag: string]: any[] }): void => {
-  const summary = calculateSummary(dataItems, base)
-
-  dataSummary.summary = summary
-  dataItems.slice(0, 4).forEach((dataItem, idx) => {
-    const value = Number(summary[dataItem.tag])
-    dataSummary[`data0${idx + 1}`] = isNaN(value) ? null : value
-  })
-}
-
-async function getLatestTimesForPeriod(periodType: DataSetSummaryPeriodType, context: ResolverContext): Promise<{ date?: string; period?: string; range: Date[] }> {
-  const { domain } = context.state
-  const now = moment()
-
-  if (periodType == DataSetSummaryPeriodType.Hour) {
-    const begin = now.clone().subtract(1, 'hour').startOf('hour')
-    const end = now.clone().startOf('hour')
-    const date = begin.clone().tz(domain.timezone)
-
-    return {
-      date: date.format('YYYY-MM-DD'),
-      period: date.format('HH'),
-      range: [begin.toDate(), end.toDate()]
-    }
-  } else if (periodType == DataSetSummaryPeriodType.WorkShift) {
-    const { workDate, workShift, shiftRange } = await getLatestWorkDateAndShift(domain, new Date())
-
-    return { date: workDate, period: workShift, range: shiftRange }
-  } else if (periodType == DataSetSummaryPeriodType.WorkDate) {
-    const { workDate, dateRange } = await getLatestWorkDateAndShift(domain, new Date())
-
-    return { date: workDate, range: dateRange }
-  } else if (periodType == DataSetSummaryPeriodType.Day) {
-    const begin = now.clone().subtract(1, 'day').startOf('day')
-    const end = now.clone().startOf('day')
-    const date = begin.clone().tz(domain.timezone)
-
-    return {
-      date: date.format('YYYY-MM-DD'),
-      range: [begin.toDate(), end.toDate()]
-    }
-  }
-}
-
-async function getTimesForPeriod(
-  periodType: DataSetSummaryPeriodType,
-  date: string,
-  period: string,
-  context: ResolverContext
-): Promise<{ date?: string; period?: string; range: Date[] }> {
-  const { domain } = context.state
-
-  if (periodType == DataSetSummaryPeriodType.Hour) {
-    const theDate = moment.tz(`${date} ${period}:00:00`, 'YYYY-MM-DD HH:mm:ss', domain.timezone)
-
-    const begin = theDate.clone().startOf('hour').toDate()
-    const end = theDate.clone().add(+1, 'hour').startOf('hour').toDate()
-
-    return {
-      date,
-      period,
-      range: [begin, end]
-    }
-  } else if (periodType == DataSetSummaryPeriodType.WorkShift) {
-    const range = await getDateRangeForWorkShift(domain, date, period)
-
-    return { date, period, range }
-  } else if (periodType == DataSetSummaryPeriodType.WorkDate) {
-    const range = await getDateRangeForWorkDate(domain, date)
-
-    return { date, range }
-  } else if (periodType == DataSetSummaryPeriodType.Day) {
-    const theDate = moment.tz(`${date} 00:00:00`, 'YYYY-MM-DD HH:mm:ss', domain.timezone)
-
-    const begin = theDate.clone().startOf('day').toDate()
-    const end = theDate.clone().add(1, 'day').startOf('day').toDate()
-
-    return {
-      date: moment(begin).tz(domain.timezone).format('YYYY-MM-DD'),
-      range: [begin, end]
-    }
-  }
-}
-
-export async function getDataFinalizeCrontabSchedule(dataSet: DataSet, context: ResolverContext): Promise<string> {
-  const { domain, user, tx } = context.state
-
-  try {
-    const { summaryPeriod } = dataSet
-
-    if (summaryPeriod == DataSetSummaryPeriodType.Hour) {
-      return '0 5 * * * *'
-    } else if (summaryPeriod == DataSetSummaryPeriodType.WorkShift) {
-      return await getSummaryScheduleForWorkShift(domain)
-    } else if (summaryPeriod == DataSetSummaryPeriodType.WorkDate) {
-      return await getSummaryScheduleForWorkDate(domain)
-    } else if (summaryPeriod == DataSetSummaryPeriodType.Day) {
-      return '0 10 0 * * *'
-    }
-  } catch (err) {
-    console.error(err)
-  }
-}
-
-export async function finalizeLatestDataCollection(dataSetId: string, context: ResolverContext): Promise<boolean> {
-  const { domain, user, tx } = context.state
-
-  try {
-    const dataSet = await tx.getRepository(DataSet).findOne({
-      where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: dataSetId },
-      relations: ['dataKeySet']
-    })
-
-    const dataKeyItems = dataSet.dataKeySet?.dataKeyItems || []
-    const dataItems = dataSet.dataItems.filter(item => item.stat)
-    const initialSummary = dataItems.reduce((sum, item) => {
-      sum[item.tag] = []
-      return sum
-    }, {})
-
-    const { date, period, range } = await getLatestTimesForPeriod(dataSet.summaryPeriod, context)
-    const limit = 100
-    var page = 1
-
-    var summaries: Partial<DataSummary>[] = []
-    var summary: Partial<DataSummary>
-
-    do {
-      const samples = await getQueryBuilderFromListParams({
-        repository: tx.getRepository(DataSample),
-        domain,
-        params: {
-          filters: [{ name: 'dataSetId', operator: 'eq', value: dataSetId }],
-          pagination: { page, limit },
-          sortings: [...buildKeySortingList(dataKeyItems), { name: 'collectedAt', desc: true }]
-        },
-        alias: 'datasample'
-      })
-        // The 'Between' operator includes the 'to' time in the filtering, making it unsuitable for the desired use case.
-        // .andWhere({ collectedAt: Between.apply(null, range) })
-        .andWhere('datasample.collectedAt >= :from', { from: range[0] })
-        .andWhere('datasample.collectedAt < :to', { to: range[1] })
-        .getMany()
-
-      for (const sample of samples) {
-        if (!summary || !compareKeys(dataKeyItems, summary, sample)) {
-          if (summary) {
-            fillSummaryResult(summary, dataItems, summary.summary)
-            summaries.push(summary)
-          }
-
-          summary = {
-            domain,
-            name: dataSet.name,
-            description: dataSet.description,
-            date,
-            period,
-            dataSet,
-            ...buildKeysFromSample(dataKeyItems, sample),
-            count: 0,
-            countOoc: 0,
-            countOos: 0,
-            summary: deepClone(initialSummary),
-            updater: user,
-            creator: user
-          }
-        }
-
-        summary.count++
-        sample.ooc && summary.countOoc++
-        sample.oos && summary.countOos++
-
-        dataItems.forEach(item => {
-          summary.summary[item.tag].push(sample.data[item.tag])
-        })
-      }
-
-      if (samples.length < limit) {
-        if (summary) {
-          fillSummaryResult(summary, dataItems, summary.summary)
-          summaries.push(summary)
-        }
-        break
-      }
-
-      page++
-    } while (true)
-
-    tx.getRepository(DataSummary).upsert(summaries, ['domain', 'dataSet', 'key01', 'key02', 'key03', 'key04', 'key05', 'date', 'period'])
-
-    return true
-  } catch (e) {
-    logger.error(e)
-  }
-
-  return false
-}
-
-export async function finalizeDataCollection(dataSetId: string, date: string, period: string, context: ResolverContext): Promise<boolean> {
-  const { domain, user, tx } = context.state
-
-  try {
-    const dataSet =
-      dataSetId &&
-      (await tx.getRepository(DataSet).findOne({
-        where: { domain: In([domain.id, domain.parentId].filter(Boolean)), id: dataSetId },
-        relations: ['dataKeySet']
-      }))
-
-    const dataKeyItems = dataSet.dataKeySet?.dataKeyItems || []
-    const dataItems = dataSet.dataItems.filter(item => item.stat)
-    const initialSummary = dataItems.reduce((sum, item) => {
-      sum[item.tag] = []
-      return sum
-    }, {})
-
-    const times = await getTimesForPeriod(dataSet.summaryPeriod, date, period, context)
-    const range = times.range
-    period = times.period
-
-    const limit = 100
-    var page = 1
-
-    var summaries: Partial<DataSummary>[] = []
-    var summary: Partial<DataSummary>
-
-    do {
-      const samples = await getQueryBuilderFromListParams({
-        repository: tx.getRepository(DataSample),
-        params: {
-          filters: [{ name: 'dataSetId', operator: 'eq', value: dataSetId }],
-          pagination: { page, limit },
-          sortings: [...buildKeySortingList(dataKeyItems), { name: 'collectedAt', desc: true }]
-        },
-        domain,
-        alias: 'datasample'
-      })
-        // The 'Between' operator includes the 'to' time in the filtering, making it unsuitable for the desired use case.
-        // .andWhere({ collectedAt: Between.apply(null, range) })
-        .andWhere('datasample.collectedAt >= :from', { from: range[0] })
-        .andWhere('datasample.collectedAt < :to', { to: range[1] })
-        .getMany()
-
-      for (const sample of samples) {
-        if (!summary || !compareKeys(dataKeyItems, summary, sample)) {
-          if (summary) {
-            fillSummaryResult(summary, dataItems, summary.summary)
-            summaries.push(summary)
-          }
-
-          summary = {
-            domain,
-            name: dataSet.name,
-            description: dataSet.description,
-            date,
-            period,
-            dataSet,
-            ...buildKeysFromSample(dataKeyItems, sample),
-            count: 0,
-            countOoc: 0,
-            countOos: 0,
-            summary: deepClone(initialSummary),
-            updater: user,
-            creator: user
-          }
-        }
-
-        summary.count++
-        sample.ooc && summary.countOoc++
-        sample.oos && summary.countOos++
-
-        dataItems.forEach(item => {
-          summary.summary[item.tag].push(sample.data[item.tag])
-        })
-      }
-
-      if (samples.length < limit) {
-        if (summary) {
-          fillSummaryResult(summary, dataItems, summary.summary)
-          summaries.push(summary)
-        }
-        break
-      }
-
-      page++
-    } while (true)
-
-    tx.getRepository(DataSummary).upsert(summaries, ['domain', 'dataSet', 'key01', 'key02', 'key03', 'key04', 'key05', 'date', 'period'])
-
-    return true
-  } catch (e) {
-    logger.error(e)
-  }
-
-  return false
-}

package/server/controllers/index.ts

@@ -1,3 +0,0 @@
-export * from './create-data-sample'
-export * from './data-use-case'
-export * from './query-data-summary-by-period'

package/server/controllers/issue-data-collection-task.ts

@@ -1,70 +0,0 @@
-import { In } from 'typeorm'
-
-import { Domain, getDataSource } from '@things-factory/shell'
-import { Activity, ActivityInstance, issue } from '@things-factory/worklist'
-
-import { DataSet } from '../service/data-set/data-set'
-
-export async function issueDataCollectionTask(
-  domainId: string,
-  dataSetId: string,
-  context: ResolverContext
-): Promise<ActivityInstance | void> {
-  await getDataSource().transaction(async tx => {
-    const domain = await tx.getRepository(Domain).findOneBy({ id: domainId })
-
-    if (!domain) {
-      throw new Error(`domain(${domainId}) not found`)
-    }
-
-    const dataSet = await tx.getRepository(DataSet).findOne({
-      where: {
-        domain: {
-          id: In([domain.id, domain.parentId].filter(Boolean))
-        },
-        id: dataSetId
-      },
-      relations: ['entryRole']
-    })
-
-    const activity = (await tx.getRepository(Activity).findOneBy({
-      domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
-      name: 'Collect Data'
-    })) as Activity
-
-    if (activity) {
-      const { entryRole, timeLimit } = dataSet
-
-      /* 해당 dataset에 대한 데이타 수집 태스크를 dataset entryRole에게 할당한다. */
-      if (entryRole) {
-        const activityInstance = {
-          name: `[Data 수집] ${dataSet.name}`,
-          description: dataSet.description,
-          activityId: activity.id,
-          dueAt: new Date(Date.now() + (timeLimit || activity.standardTime || 24 * 60 * 60) * 1000),
-          input: {
-            dataSetId: dataSet.id,
-            dataSetName: dataSet.name
-          },
-          assigneeRole: entryRole,
-          threadsMin: 1,
-          threadsMax: 1
-        }
-
-        context.state = {
-          ...context.state,
-          domain,
-          tx
-        }
-
-        return await issue(activityInstance, context)
-      } else {
-        throw new Error(
-          `Data Entry Role not set. So Data Collect Activity for ${dataSet.name}($dataSet.id) could not be issued.`
-        )
-      }
-    } else {
-      throw new Error(`Data Collect Activity is not installed.`)
-    }
-  })
-}

package/server/controllers/jasper-report.ts

@@ -1,186 +0,0 @@
-import FormData from 'form-data'
-import fetch from 'node-fetch'
-
-import { getEndpointUrl } from '../utils/config-resolver'
-
-import { STORAGE } from '@things-factory/attachment-base'
-import { AthenaController } from '@things-factory/aws-base'
-import { config } from '@things-factory/env'
-
-const dataReportConfig = config.get('dataReport')
-const {
-  jasper: {
-    endpoint: ENDPOINT,
-    datasource: { database: DATABASE }
-  }
-} = dataReportConfig || {
-  jasper: {
-    endpoint: {},
-    datasource: {}
-  }
-}
-
-function transformValuesToRows(queryResult) {
-  var parseData = []
-  let index = 1
-  for (let i = 0; i < queryResult.Items.length; i++) {
-    var j = 0
-    const data = JSON.parse(queryResult.Items[i].data)
-    const spec = JSON.parse(queryResult.Items[i].spec)
-
-    for (let key in data) {
-      if (Array.isArray(data[key])) {
-        for (j = 0; j < data[key].length; j++) {
-          for (let specKey in spec) {
-            if (key === specKey) {
-              parseData.push({
-                item: spec[specKey].name,
-                index: index + j,
-                value: String(data[key][j])
-              })
-            }
-          }
-        }
-      } else {
-        parseData.push({
-          item: key,
-          index,
-          value: String(data[key])
-        })
-      }
-    }
-    if (j !== 0) {
-      index = index + j
-    } else {
-      index = index + 1
-    }
-  }
-}
-
-/** @todo considering trasformation in lambda, as massive dataset */
-function pivotData(rows) {
-  let parsedData = []
-  let index = 1
-  for (let i = 0; i < rows.length; i++) {
-    let j = 0
-    const data = JSON.parse(rows[i].data)
-    const spec = JSON.parse(rows[i].spec)
-
-    for (let key in data) {
-      /** @todo rule to display or not, about unspecified spec */
-      const value = data[key]
-      !spec[key]?.hidden &&
-        parsedData.push({
-          item: spec[key]?.name || key,
-          index,
-          value: Array.isArray(value) ? value.join(', ') : value
-        })
-    }
-    if (j !== 0) {
-      index = index + j
-    } else {
-      index = index + 1
-    }
-  }
-
-  return parsedData
-}
-
-function parseJsonDataField(rows) {
-  let parsedData = []
-  for (let i = 0; i < rows.length; i++) {
-    const row = rows[i]
-    const data = JSON.parse(row.data)
-    for (let key in data) {
-      if (Array.isArray(data[key])) {
-        data[key] = data[key].toString()
-      }
-    }
-    delete row.data
-    parsedData.push({ ...row, ...data })
-  }
-
-  return parsedData
-}
-
-const athenaClient = new AthenaController()
-
-async function queryAthena(params) {
-  const { table, domain, datasetId, startDate, endDate, workShift, timezone } = params
-  const queryData = {
-    sql: `SELECT ds.name, ds.description, ds.data, dsh.data_items as spec, ds.workdate, ds.workshift,
-    DATE_FORMAT(
-      FROM_UNIXTIME(collected_at / 1000 / 1000) AT TIME ZONE '${timezone || 'UTC'}', 
-      '%Y-%m-%d %H:%i:%s'
-    ) AS dscollected_at
-    FROM ${table} ds
-    JOIN data_set_histories dsh 
-      ON (ds.datasetid = dsh.original_id
-        and ds.data_set_version = dsh.version)
-    WHERE ds.domain='${domain}' 
-    AND ds.datasetid = '${datasetId}'
-    AND ds.workdate >= '${startDate}'
-    AND ds.workdate <= '${endDate}'
-    ${workShift ? "AND ds.workshift = '" + workShift + "'" : ''}
-    ORDER BY ds.collected_at`,
-    db: DATABASE
-  }
-  // and json_extract_scalar(data, '$.dauid') = 'A8032AD81730'
-
-  return await athenaClient.query(queryData)
-}
-
-export async function renderJasperReport(context: any) {
-  const {
-    state: { domain },
-    query
-  } = context
-
-  const template = await STORAGE.readFile(query['reportTemplate'] || 'dynamic_header_sample.jrxml', 'utf-8')
-  let templateType = query['templateType'] || 'crosstab'
-  let parsedData = []
-
-  // @todo: get dataset timezone
-  /**
-   * const variables = await gql(dataSet(id:${dataSetId}) {
-   *  name, description, partition_keys, timezone
-   * })
-   */
-
-  query['domain'] = domain?.subdomain
-  query['timezone'] = domain?.timezone
-  const queryResult = await queryAthena(query)
-  const rows = queryResult.Items
-
-  if (!rows.length) {
-    return '<h3>Not found result.</h3>'
-  } else {
-    const firstRow = rows[0]
-    // uses the first row values as data-set has no history data.
-    const parameters = {
-      name: firstRow.name,
-      description: firstRow.description,
-      ...query
-    }
-
-    if (templateType === 'crosstab') {
-      parsedData = pivotData(rows)
-    } else {
-      parsedData = parseJsonDataField(rows)
-    }
-
-    const formData = new FormData()
-    formData.append('template', template)
-    formData.append('jsonString', JSON.stringify(parsedData))
-    formData.append('parameters', JSON.stringify(parameters))
-
-    const { reportView } = query
-    const url = getEndpointUrl(ENDPOINT, reportView)
-    const response = await fetch(url, {
-      method: 'POST',
-      body: formData
-    })
-
-    return await response.text()
-  }
-}