@things-factory/dataset 8.0.0-beta.9 → 8.0.0

package/server/controllers/finalize-data-collection.ts

@@ -0,0 +1,388 @@
+const statistics = require('simple-statistics')
+const deepClone = require('lodash/cloneDeep')
+
+import moment from 'moment-timezone'
+import { In } from 'typeorm'
+
+import { Sorting, getQueryBuilderFromListParams } from '@things-factory/shell'
+import { logger } from '@things-factory/env'
+import {
+  getDateRangeForWorkDate,
+  getDateRangeForWorkShift,
+  getLatestWorkDateAndShift,
+  getSummaryScheduleForWorkDate,
+  getSummaryScheduleForWorkShift
+} from '@things-factory/work-shift'
+
+import { DataSample } from '../service/data-sample/data-sample'
+import { DataSet, DataSetSummaryPeriodType } from '../service/data-set/data-set'
+
+import { DataSummary } from '../service/data-summary/data-summary'
+import { DataKeyItem } from '../service/data-key-set/data-key-item-type'
+
+import { DataItem } from 'service'
+
+const STAT_FUNCTION_MAP = {
+  sum: 'sum',
+  mean: 'mean',
+  stddev: 'standardDeviation',
+  variance: 'variance',
+  min: 'min',
+  max: 'max',
+  range: 'range',
+  median: 'median',
+  mode: 'mode'
+}
+
+const compareKeys = (dataKeyItems: DataKeyItem[], summary: Partial<DataSummary>, sample: DataSample): boolean => {
+  return dataKeyItems.every((item, index) => {
+    const prop = `key0${index + 1}`
+    return sample[prop] === summary[prop]
+  })
+}
+
+const buildKeysFromSample = (dataKeyItems: DataKeyItem[], sample: DataSample): Partial<DataSummary> => {
+  return dataKeyItems.reduce((sum, item, index) => {
+    const prop = `key0${index + 1}`
+    sum[prop] = sample[prop]
+
+    return sum
+  }, {} as Partial<DataSummary>)
+}
+
+const buildKeySortingList = (dataKeyItems: DataKeyItem[]): Sorting[] => {
+  return dataKeyItems.reduce((sum, item, index) => {
+    const name = `key0${index + 1}`
+    sum.push({ name, desc: true })
+    return sum
+  }, [])
+}
+
+const calculateSummary = (dataItems: DataItem[], base: { [tag: string]: any[] }) => {
+  return dataItems.reduce((summary, item) => {
+    const tag = item.tag
+
+    const data = base[tag]
+      .flat(Infinity)
+      .map(Number)
+      .filter(item => !isNaN(item))
+
+    if (data.length > 0) {
+      try {
+        switch (item.stat) {
+          case 'range':
+            summary[tag] = statistics.max(data) - statistics.min(data)
+            break
+
+          default:
+            const functionName = STAT_FUNCTION_MAP[item.stat]
+            summary[tag] = (functionName && statistics[functionName](data)) || ''
+        }
+      } catch (err) {
+        summary[tag] = null
+        console.error(err)
+      }
+    } else {
+      summary[tag] = null
+    }
+
+    return summary
+  }, {})
+}
+
+const fillSummaryResult = (dataSummary: Partial<DataSummary>, dataItems: DataItem[], base: { [tag: string]: any[] }): void => {
+  const summary = calculateSummary(dataItems, base)
+
+  dataSummary.summary = summary
+  dataItems.slice(0, 4).forEach((dataItem, idx) => {
+    const value = Number(summary[dataItem.tag])
+    dataSummary[`data0${idx + 1}`] = isNaN(value) ? null : value
+  })
+}
+
+async function getLatestTimesForPeriod(periodType: DataSetSummaryPeriodType, context: ResolverContext): Promise<{ date?: string; period?: string; range: Date[] }> {
+  const { domain } = context.state
+  const now = moment()
+
+  if (periodType == DataSetSummaryPeriodType.Hour) {
+    const begin = now.clone().subtract(1, 'hour').startOf('hour')
+    const end = now.clone().startOf('hour')
+    const date = begin.clone().tz(domain.timezone)
+
+    return {
+      date: date.format('YYYY-MM-DD'),
+      period: date.format('HH'),
+      range: [begin.toDate(), end.toDate()]
+    }
+  } else if (periodType == DataSetSummaryPeriodType.WorkShift) {
+    const { workDate, workShift, shiftRange } = await getLatestWorkDateAndShift(domain, new Date())
+
+    return { date: workDate, period: workShift, range: shiftRange }
+  } else if (periodType == DataSetSummaryPeriodType.WorkDate) {
+    const { workDate, dateRange } = await getLatestWorkDateAndShift(domain, new Date())
+
+    return { date: workDate, range: dateRange }
+  } else if (periodType == DataSetSummaryPeriodType.Day) {
+    const begin = now.clone().subtract(1, 'day').startOf('day')
+    const end = now.clone().startOf('day')
+    const date = begin.clone().tz(domain.timezone)
+
+    return {
+      date: date.format('YYYY-MM-DD'),
+      range: [begin.toDate(), end.toDate()]
+    }
+  }
+}
+
+async function getTimesForPeriod(
+  periodType: DataSetSummaryPeriodType,
+  date: string,
+  period: string,
+  context: ResolverContext
+): Promise<{ date?: string; period?: string; range: Date[] }> {
+  const { domain } = context.state
+
+  if (periodType == DataSetSummaryPeriodType.Hour) {
+    const theDate = moment.tz(`${date} ${period}:00:00`, 'YYYY-MM-DD HH:mm:ss', domain.timezone)
+
+    const begin = theDate.clone().startOf('hour').toDate()
+    const end = theDate.clone().add(+1, 'hour').startOf('hour').toDate()
+
+    return {
+      date,
+      period,
+      range: [begin, end]
+    }
+  } else if (periodType == DataSetSummaryPeriodType.WorkShift) {
+    const range = await getDateRangeForWorkShift(domain, date, period)
+
+    return { date, period, range }
+  } else if (periodType == DataSetSummaryPeriodType.WorkDate) {
+    const range = await getDateRangeForWorkDate(domain, date)
+
+    return { date, range }
+  } else if (periodType == DataSetSummaryPeriodType.Day) {
+    const theDate = moment.tz(`${date} 00:00:00`, 'YYYY-MM-DD HH:mm:ss', domain.timezone)
+
+    const begin = theDate.clone().startOf('day').toDate()
+    const end = theDate.clone().add(1, 'day').startOf('day').toDate()
+
+    return {
+      date: moment(begin).tz(domain.timezone).format('YYYY-MM-DD'),
+      range: [begin, end]
+    }
+  }
+}
+
+export async function getDataFinalizeCrontabSchedule(dataSet: DataSet, context: ResolverContext): Promise<string> {
+  const { domain, user, tx } = context.state
+
+  try {
+    const { summaryPeriod } = dataSet
+
+    if (summaryPeriod == DataSetSummaryPeriodType.Hour) {
+      return '0 5 * * * *'
+    } else if (summaryPeriod == DataSetSummaryPeriodType.WorkShift) {
+      return await getSummaryScheduleForWorkShift(domain)
+    } else if (summaryPeriod == DataSetSummaryPeriodType.WorkDate) {
+      return await getSummaryScheduleForWorkDate(domain)
+    } else if (summaryPeriod == DataSetSummaryPeriodType.Day) {
+      return '0 10 0 * * *'
+    }
+  } catch (err) {
+    console.error(err)
+  }
+}
+
+export async function finalizeLatestDataCollection(dataSetId: string, context: ResolverContext): Promise<boolean> {
+  const { domain, user, tx } = context.state
+
+  try {
+    const dataSet = await tx.getRepository(DataSet).findOne({
+      where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: dataSetId },
+      relations: ['dataKeySet']
+    })
+
+    const dataKeyItems = dataSet.dataKeySet?.dataKeyItems || []
+    const dataItems = dataSet.dataItems.filter(item => item.stat)
+    const initialSummary = dataItems.reduce((sum, item) => {
+      sum[item.tag] = []
+      return sum
+    }, {})
+
+    const { date, period, range } = await getLatestTimesForPeriod(dataSet.summaryPeriod, context)
+    const limit = 100
+    var page = 1
+
+    var summaries: Partial<DataSummary>[] = []
+    var summary: Partial<DataSummary>
+
+    do {
+      const samples = await getQueryBuilderFromListParams({
+        repository: tx.getRepository(DataSample),
+        domain,
+        params: {
+          filters: [{ name: 'dataSetId', operator: 'eq', value: dataSetId }],
+          pagination: { page, limit },
+          sortings: [...buildKeySortingList(dataKeyItems), { name: 'collectedAt', desc: true }]
+        },
+        alias: 'datasample'
+      })
+        // The 'Between' operator includes the 'to' time in the filtering, making it unsuitable for the desired use case.
+        // .andWhere({ collectedAt: Between.apply(null, range) })
+        .andWhere('datasample.collectedAt >= :from', { from: range[0] })
+        .andWhere('datasample.collectedAt < :to', { to: range[1] })
+        .getMany()
+
+      for (const sample of samples) {
+        if (!summary || !compareKeys(dataKeyItems, summary, sample)) {
+          if (summary) {
+            fillSummaryResult(summary, dataItems, summary.summary)
+            summaries.push(summary)
+          }
+
+          summary = {
+            domain,
+            name: dataSet.name,
+            description: dataSet.description,
+            date,
+            period,
+            dataSet,
+            ...buildKeysFromSample(dataKeyItems, sample),
+            count: 0,
+            countOoc: 0,
+            countOos: 0,
+            summary: deepClone(initialSummary),
+            updater: user,
+            creator: user
+          }
+        }
+
+        summary.count++
+        sample.ooc && summary.countOoc++
+        sample.oos && summary.countOos++
+
+        dataItems.forEach(item => {
+          summary.summary[item.tag].push(sample.data[item.tag])
+        })
+      }
+
+      if (samples.length < limit) {
+        if (summary) {
+          fillSummaryResult(summary, dataItems, summary.summary)
+          summaries.push(summary)
+        }
+        break
+      }
+
+      page++
+    } while (true)
+
+    tx.getRepository(DataSummary).upsert(summaries, ['domain', 'dataSet', 'key01', 'key02', 'key03', 'key04', 'key05', 'date', 'period'])
+
+    return true
+  } catch (e) {
+    logger.error(e)
+  }
+
+  return false
+}
+
+export async function finalizeDataCollection(dataSetId: string, date: string, period: string, context: ResolverContext): Promise<boolean> {
+  const { domain, user, tx } = context.state
+
+  try {
+    const dataSet =
+      dataSetId &&
+      (await tx.getRepository(DataSet).findOne({
+        where: { domain: In([domain.id, domain.parentId].filter(Boolean)), id: dataSetId },
+        relations: ['dataKeySet']
+      }))
+
+    const dataKeyItems = dataSet.dataKeySet?.dataKeyItems || []
+    const dataItems = dataSet.dataItems.filter(item => item.stat)
+    const initialSummary = dataItems.reduce((sum, item) => {
+      sum[item.tag] = []
+      return sum
+    }, {})
+
+    const times = await getTimesForPeriod(dataSet.summaryPeriod, date, period, context)
+    const range = times.range
+    period = times.period
+
+    const limit = 100
+    var page = 1
+
+    var summaries: Partial<DataSummary>[] = []
+    var summary: Partial<DataSummary>
+
+    do {
+      const samples = await getQueryBuilderFromListParams({
+        repository: tx.getRepository(DataSample),
+        params: {
+          filters: [{ name: 'dataSetId', operator: 'eq', value: dataSetId }],
+          pagination: { page, limit },
+          sortings: [...buildKeySortingList(dataKeyItems), { name: 'collectedAt', desc: true }]
+        },
+        domain,
+        alias: 'datasample'
+      })
+        // The 'Between' operator includes the 'to' time in the filtering, making it unsuitable for the desired use case.
+        // .andWhere({ collectedAt: Between.apply(null, range) })
+        .andWhere('datasample.collectedAt >= :from', { from: range[0] })
+        .andWhere('datasample.collectedAt < :to', { to: range[1] })
+        .getMany()
+
+      for (const sample of samples) {
+        if (!summary || !compareKeys(dataKeyItems, summary, sample)) {
+          if (summary) {
+            fillSummaryResult(summary, dataItems, summary.summary)
+            summaries.push(summary)
+          }
+
+          summary = {
+            domain,
+            name: dataSet.name,
+            description: dataSet.description,
+            date,
+            period,
+            dataSet,
+            ...buildKeysFromSample(dataKeyItems, sample),
+            count: 0,
+            countOoc: 0,
+            countOos: 0,
+            summary: deepClone(initialSummary),
+            updater: user,
+            creator: user
+          }
+        }
+
+        summary.count++
+        sample.ooc && summary.countOoc++
+        sample.oos && summary.countOos++
+
+        dataItems.forEach(item => {
+          summary.summary[item.tag].push(sample.data[item.tag])
+        })
+      }
+
+      if (samples.length < limit) {
+        if (summary) {
+          fillSummaryResult(summary, dataItems, summary.summary)
+          summaries.push(summary)
+        }
+        break
+      }
+
+      page++
+    } while (true)
+
+    tx.getRepository(DataSummary).upsert(summaries, ['domain', 'dataSet', 'key01', 'key02', 'key03', 'key04', 'key05', 'date', 'period'])
+
+    return true
+  } catch (e) {
+    logger.error(e)
+  }
+
+  return false
+}

package/server/controllers/index.ts

@@ -0,0 +1,3 @@
+export * from './create-data-sample'
+export * from './data-use-case'
+export * from './query-data-summary-by-period'

package/server/controllers/issue-data-collection-task.ts

@@ -0,0 +1,70 @@
+import { In } from 'typeorm'
+
+import { Domain, getDataSource } from '@things-factory/shell'
+import { Activity, ActivityInstance, issue } from '@things-factory/worklist'
+
+import { DataSet } from '../service/data-set/data-set'
+
+export async function issueDataCollectionTask(
+  domainId: string,
+  dataSetId: string,
+  context: ResolverContext
+): Promise<ActivityInstance | void> {
+  await getDataSource().transaction(async tx => {
+    const domain = await tx.getRepository(Domain).findOneBy({ id: domainId })
+
+    if (!domain) {
+      throw new Error(`domain(${domainId}) not found`)
+    }
+
+    const dataSet = await tx.getRepository(DataSet).findOne({
+      where: {
+        domain: {
+          id: In([domain.id, domain.parentId].filter(Boolean))
+        },
+        id: dataSetId
+      },
+      relations: ['entryRole']
+    })
+
+    const activity = (await tx.getRepository(Activity).findOneBy({
+      domain: { id: In([domain.id, domain.parentId].filter(Boolean)) },
+      name: 'Collect Data'
+    })) as Activity
+
+    if (activity) {
+      const { entryRole, timeLimit } = dataSet
+
+      /* 해당 dataset에 대한 데이타 수집 태스크를 dataset entryRole에게 할당한다. */
+      if (entryRole) {
+        const activityInstance = {
+          name: `[Data 수집] ${dataSet.name}`,
+          description: dataSet.description,
+          activityId: activity.id,
+          dueAt: new Date(Date.now() + (timeLimit || activity.standardTime || 24 * 60 * 60) * 1000),
+          input: {
+            dataSetId: dataSet.id,
+            dataSetName: dataSet.name
+          },
+          assigneeRole: entryRole,
+          threadsMin: 1,
+          threadsMax: 1
+        }
+
+        context.state = {
+          ...context.state,
+          domain,
+          tx
+        }
+
+        return await issue(activityInstance, context)
+      } else {
+        throw new Error(
+          `Data Entry Role not set. So Data Collect Activity for ${dataSet.name}($dataSet.id) could not be issued.`
+        )
+      }
+    } else {
+      throw new Error(`Data Collect Activity is not installed.`)
+    }
+  })
+}

package/server/controllers/jasper-report.ts

@@ -0,0 +1,186 @@
+import FormData from 'form-data'
+import fetch from 'node-fetch'
+
+import { getEndpointUrl } from '../utils/config-resolver'
+
+import { STORAGE } from '@things-factory/attachment-base'
+import { AthenaController } from '@things-factory/aws-base'
+import { config } from '@things-factory/env'
+
+const dataReportConfig = config.get('dataReport')
+const {
+  jasper: {
+    endpoint: ENDPOINT,
+    datasource: { database: DATABASE }
+  }
+} = dataReportConfig || {
+  jasper: {
+    endpoint: {},
+    datasource: {}
+  }
+}
+
+function transformValuesToRows(queryResult) {
+  var parseData = []
+  let index = 1
+  for (let i = 0; i < queryResult.Items.length; i++) {
+    var j = 0
+    const data = JSON.parse(queryResult.Items[i].data)
+    const spec = JSON.parse(queryResult.Items[i].spec)
+
+    for (let key in data) {
+      if (Array.isArray(data[key])) {
+        for (j = 0; j < data[key].length; j++) {
+          for (let specKey in spec) {
+            if (key === specKey) {
+              parseData.push({
+                item: spec[specKey].name,
+                index: index + j,
+                value: String(data[key][j])
+              })
+            }
+          }
+        }
+      } else {
+        parseData.push({
+          item: key,
+          index,
+          value: String(data[key])
+        })
+      }
+    }
+    if (j !== 0) {
+      index = index + j
+    } else {
+      index = index + 1
+    }
+  }
+}
+
+/** @todo considering trasformation in lambda, as massive dataset */
+function pivotData(rows) {
+  let parsedData = []
+  let index = 1
+  for (let i = 0; i < rows.length; i++) {
+    let j = 0
+    const data = JSON.parse(rows[i].data)
+    const spec = JSON.parse(rows[i].spec)
+
+    for (let key in data) {
+      /** @todo rule to display or not, about unspecified spec */
+      const value = data[key]
+      !spec[key]?.hidden &&
+        parsedData.push({
+          item: spec[key]?.name || key,
+          index,
+          value: Array.isArray(value) ? value.join(', ') : value
+        })
+    }
+    if (j !== 0) {
+      index = index + j
+    } else {
+      index = index + 1
+    }
+  }
+
+  return parsedData
+}
+
+function parseJsonDataField(rows) {
+  let parsedData = []
+  for (let i = 0; i < rows.length; i++) {
+    const row = rows[i]
+    const data = JSON.parse(row.data)
+    for (let key in data) {
+      if (Array.isArray(data[key])) {
+        data[key] = data[key].toString()
+      }
+    }
+    delete row.data
+    parsedData.push({ ...row, ...data })
+  }
+
+  return parsedData
+}
+
+const athenaClient = new AthenaController()
+
+async function queryAthena(params) {
+  const { table, domain, datasetId, startDate, endDate, workShift, timezone } = params
+  const queryData = {
+    sql: `SELECT ds.name, ds.description, ds.data, dsh.data_items as spec, ds.workdate, ds.workshift,
+    DATE_FORMAT(
+      FROM_UNIXTIME(collected_at / 1000 / 1000) AT TIME ZONE '${timezone || 'UTC'}', 
+      '%Y-%m-%d %H:%i:%s'
+    ) AS dscollected_at
+    FROM ${table} ds
+    JOIN data_set_histories dsh 
+      ON (ds.datasetid = dsh.original_id
+        and ds.data_set_version = dsh.version)
+    WHERE ds.domain='${domain}' 
+    AND ds.datasetid = '${datasetId}'
+    AND ds.workdate >= '${startDate}'
+    AND ds.workdate <= '${endDate}'
+    ${workShift ? "AND ds.workshift = '" + workShift + "'" : ''}
+    ORDER BY ds.collected_at`,
+    db: DATABASE
+  }
+  // and json_extract_scalar(data, '$.dauid') = 'A8032AD81730'
+
+  return await athenaClient.query(queryData)
+}
+
+export async function renderJasperReport(context: any) {
+  const {
+    state: { domain },
+    query
+  } = context
+
+  const template = await STORAGE.readFile(query['reportTemplate'] || 'dynamic_header_sample.jrxml', 'utf-8')
+  let templateType = query['templateType'] || 'crosstab'
+  let parsedData = []
+
+  // @todo: get dataset timezone
+  /**
+   * const variables = await gql(dataSet(id:${dataSetId}) {
+   *  name, description, partition_keys, timezone
+   * })
+   */
+
+  query['domain'] = domain?.subdomain
+  query['timezone'] = domain?.timezone
+  const queryResult = await queryAthena(query)
+  const rows = queryResult.Items
+
+  if (!rows.length) {
+    return '<h3>Not found result.</h3>'
+  } else {
+    const firstRow = rows[0]
+    // uses the first row values as data-set has no history data.
+    const parameters = {
+      name: firstRow.name,
+      description: firstRow.description,
+      ...query
+    }
+
+    if (templateType === 'crosstab') {
+      parsedData = pivotData(rows)
+    } else {
+      parsedData = parseJsonDataField(rows)
+    }
+
+    const formData = new FormData()
+    formData.append('template', template)
+    formData.append('jsonString', JSON.stringify(parsedData))
+    formData.append('parameters', JSON.stringify(parameters))
+
+    const { reportView } = query
+    const url = getEndpointUrl(ENDPOINT, reportView)
+    const response = await fetch(url, {
+      method: 'POST',
+      body: formData
+    })
+
+    return await response.text()
+  }
+}