@things-factory/dataset 8.0.0-beta.9 → 8.0.2

package/server/controllers/jasper-report.ts

@@ -0,0 +1,186 @@
+import FormData from 'form-data'
+import fetch from 'node-fetch'
+
+import { getEndpointUrl } from '../utils/config-resolver'
+
+import { STORAGE } from '@things-factory/attachment-base'
+import { AthenaController } from '@things-factory/aws-base'
+import { config } from '@things-factory/env'
+
+const dataReportConfig = config.get('dataReport')
+const {
+  jasper: {
+    endpoint: ENDPOINT,
+    datasource: { database: DATABASE }
+  }
+} = dataReportConfig || {
+  jasper: {
+    endpoint: {},
+    datasource: {}
+  }
+}
+
+function transformValuesToRows(queryResult) {
+  var parseData = []
+  let index = 1
+  for (let i = 0; i < queryResult.Items.length; i++) {
+    var j = 0
+    const data = JSON.parse(queryResult.Items[i].data)
+    const spec = JSON.parse(queryResult.Items[i].spec)
+
+    for (let key in data) {
+      if (Array.isArray(data[key])) {
+        for (j = 0; j < data[key].length; j++) {
+          for (let specKey in spec) {
+            if (key === specKey) {
+              parseData.push({
+                item: spec[specKey].name,
+                index: index + j,
+                value: String(data[key][j])
+              })
+            }
+          }
+        }
+      } else {
+        parseData.push({
+          item: key,
+          index,
+          value: String(data[key])
+        })
+      }
+    }
+    if (j !== 0) {
+      index = index + j
+    } else {
+      index = index + 1
+    }
+  }
+}
+
+/** @todo considering trasformation in lambda, as massive dataset */
+function pivotData(rows) {
+  let parsedData = []
+  let index = 1
+  for (let i = 0; i < rows.length; i++) {
+    let j = 0
+    const data = JSON.parse(rows[i].data)
+    const spec = JSON.parse(rows[i].spec)
+
+    for (let key in data) {
+      /** @todo rule to display or not, about unspecified spec */
+      const value = data[key]
+      !spec[key]?.hidden &&
+        parsedData.push({
+          item: spec[key]?.name || key,
+          index,
+          value: Array.isArray(value) ? value.join(', ') : value
+        })
+    }
+    if (j !== 0) {
+      index = index + j
+    } else {
+      index = index + 1
+    }
+  }
+
+  return parsedData
+}
+
+function parseJsonDataField(rows) {
+  let parsedData = []
+  for (let i = 0; i < rows.length; i++) {
+    const row = rows[i]
+    const data = JSON.parse(row.data)
+    for (let key in data) {
+      if (Array.isArray(data[key])) {
+        data[key] = data[key].toString()
+      }
+    }
+    delete row.data
+    parsedData.push({ ...row, ...data })
+  }
+
+  return parsedData
+}
+
+const athenaClient = new AthenaController()
+
+async function queryAthena(params) {
+  const { table, domain, datasetId, startDate, endDate, workShift, timezone } = params
+  const queryData = {
+    sql: `SELECT ds.name, ds.description, ds.data, dsh.data_items as spec, ds.workdate, ds.workshift,
+    DATE_FORMAT(
+      FROM_UNIXTIME(collected_at / 1000 / 1000) AT TIME ZONE '${timezone || 'UTC'}', 
+      '%Y-%m-%d %H:%i:%s'
+    ) AS dscollected_at
+    FROM ${table} ds
+    JOIN data_set_histories dsh 
+      ON (ds.datasetid = dsh.original_id
+        and ds.data_set_version = dsh.version)
+    WHERE ds.domain='${domain}' 
+    AND ds.datasetid = '${datasetId}'
+    AND ds.workdate >= '${startDate}'
+    AND ds.workdate <= '${endDate}'
+    ${workShift ? "AND ds.workshift = '" + workShift + "'" : ''}
+    ORDER BY ds.collected_at`,
+    db: DATABASE
+  }
+  // and json_extract_scalar(data, '$.dauid') = 'A8032AD81730'
+
+  return await athenaClient.query(queryData)
+}
+
+export async function renderJasperReport(context: any) {
+  const {
+    state: { domain },
+    query
+  } = context
+
+  const template = await STORAGE.readFile(query['reportTemplate'] || 'dynamic_header_sample.jrxml', 'utf-8')
+  let templateType = query['templateType'] || 'crosstab'
+  let parsedData = []
+
+  // @todo: get dataset timezone
+  /**
+   * const variables = await gql(dataSet(id:${dataSetId}) {
+   *  name, description, partition_keys, timezone
+   * })
+   */
+
+  query['domain'] = domain?.subdomain
+  query['timezone'] = domain?.timezone
+  const queryResult = await queryAthena(query)
+  const rows = queryResult.Items
+
+  if (!rows.length) {
+    return '<h3>Not found result.</h3>'
+  } else {
+    const firstRow = rows[0]
+    // uses the first row values as data-set has no history data.
+    const parameters = {
+      name: firstRow.name,
+      description: firstRow.description,
+      ...query
+    }
+
+    if (templateType === 'crosstab') {
+      parsedData = pivotData(rows)
+    } else {
+      parsedData = parseJsonDataField(rows)
+    }
+
+    const formData = new FormData()
+    formData.append('template', template)
+    formData.append('jsonString', JSON.stringify(parsedData))
+    formData.append('parameters', JSON.stringify(parameters))
+
+    const { reportView } = query
+    const url = getEndpointUrl(ENDPOINT, reportView)
+    const response = await fetch(url, {
+      method: 'POST',
+      body: formData
+    })
+
+    return await response.text()
+  }
+}

package/server/controllers/query-data-summary-by-period.ts

@@ -0,0 +1,178 @@
+import { In } from 'typeorm'
+
+import { getQueryBuilderFromListParams, getRepository, getTimesForPeriod, ListParam, Sorting } from '@things-factory/shell'
+
+import { DataSet, DataSetSummaryPeriodType } from '../service/data-set/data-set'
+import { DataSummary } from '../service/data-summary/data-summary'
+
+const STAT_FUNCTION_MAP = {
+  sum: 'sum',
+  mean: 'mean',
+  stddev: 'standardDeviation',
+  variance: 'variance',
+  min: 'min',
+  max: 'max',
+  range: 'range',
+  median: 'median',
+  mode: 'mode'
+}
+
+export async function queryDataSummaryByPeriod(
+  period: 'today' | 'this month' | '30 days' | 'this year' | '12 months',
+  dataSetName: string,
+  dataKeys: string[] | null,
+  params: ListParam,
+  context: ResolverContext
+): Promise<DataSummary[]> {
+  const { domain, user, tx } = context.state
+  const { t } = context
+
+  const dataSet = await getRepository(DataSet).findOne({
+    where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, name: dataSetName },
+    relations: ['dataKeySet']
+  })
+
+  if (!dataSet) {
+    throw new Error(t('error.dataset not found', { dataSetName }))
+  }
+
+  // limitations
+  const summaryPeriodType = dataSet.summaryPeriod
+  if ((summaryPeriodType == DataSetSummaryPeriodType.Day || summaryPeriodType == DataSetSummaryPeriodType.WorkDate) && period == 'today') {
+    throw new Error(t('error.summary not supported', { dataSetName, period: t(`label.period-${period}`) }))
+  }
+
+  // dataKeys 가 설정되지 않았다면, dataSet의 dataKeySet을 그대로 적용한다는 의미임.
+  // dataKeys == [] 라면, dataKeySet을 적용하지 않는다는 의미임.
+  // dataKeys에는 dataKeySet의 dataKey 값을 따라야 한다.
+  const dataKeyItems = !dataSet.dataKeySet?.dataKeyItems
+    ? []
+    : dataKeys
+    ? dataKeys
+        .map(dataKey => {
+          return dataSet.dataKeySet.dataKeyItems.find(item => item.dataKey == dataKey)
+        })
+        .filter(Boolean)
+    : dataSet.dataKeySet.dataKeyItems
+
+  const searchables = dataKeyItems.map((item, index) => `key0${index + 1}`)
+  const dataItems = dataSet.dataItems.filter(item => item.agg)
+
+  const { from, to } = await getTimesForPeriod(period, context)
+
+  const selectPeriod = queryBuilder => {
+    if (period == 'today') {
+      queryBuilder.addSelect('summary.date', 'date').addSelect('summary.period', 'period')
+    } else if (['this year', '12 months'].includes(period)) {
+      queryBuilder.addSelect('SUBSTRING(summary.date, 1, 7) AS month')
+    } else {
+      queryBuilder.addSelect('summary.date', 'date')
+    }
+
+    return queryBuilder
+  }
+  const selectKeys = dataKeyItems.map((item, index) => {
+    const aliasName = `key0${index + 1}`
+    const columnName = `summary.${aliasName}`
+
+    return `${columnName} as ${aliasName}`
+  }) as string[]
+  const selectData = dataItems.map((item, index) => {
+    const aliasName = `data0${index + 1}`
+    const columnName = `summary.${aliasName}`
+
+    switch (item.agg) {
+      case STAT_FUNCTION_MAP.sum:
+        return `SUM(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.mean:
+        // 일차 mean 된 값들을 다시 계산하므로, 무의미할 수 있다.
+        return `AVG(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.stddev:
+        // 일차 stddev 된 값들을 다시 계산하므로, 무의미할 수 있다.
+        return `STDDEV(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.variance:
+        // 일차 variance 된 값들을 다시 계산하므로, 무의미할 수 있다.
+        return `VARIANCE(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.min:
+        return `MIN(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.max:
+        return `MAX(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.range:
+        // 일차 range 값들을 다시 계산하므로, 무의미할 수 있다.
+        return `MAX(${columnName}) - MIN(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.mode:
+        // not guaranteed
+        return `MODE() WITHIN GROUP (ORDER BY ${columnName})`
+      default:
+        return `AVG(${columnName}) as ${aliasName}`
+    }
+  })
+
+  const groupByPeriod =
+    period == 'today' ? 'summary.date, summary.period' : ['this year', '12 months'].includes(period) ? 'SUBSTRING(summary.date, 1, 7)' : 'summary.date'
+  const groupByKeys = dataKeyItems
+    .map((item, index) => {
+      return `summary.key0${index + 1}`
+    })
+    .join(', ')
+
+  const desc = false
+  var orderByPeriod: Sorting[] =
+    period == 'today'
+      ? [
+          {
+            name: 'date',
+            desc
+          },
+          {
+            name: 'period',
+            desc
+          }
+        ]
+      : ['this year', '12 months'].includes(period)
+      ? [{ name: 'month', desc }]
+      : [{ name: 'date', desc }]
+
+  if (params?.sortings && params.sortings.length > 0) {
+    orderByPeriod = params.sortings
+  }
+
+  var queryBuilder = getQueryBuilderFromListParams({
+    repository: getRepository(DataSummary),
+    params: {
+      ...params,
+      sortings: []
+    },
+    domain,
+    alias: 'summary',
+    searchables: searchables
+  })
+    .select('summary.dataSet')
+    .addSelect(selectData)
+    .addSelect('SUM(summary.count)', 'count')
+    .addSelect('SUM(summary.countOoc)', 'countOoc')
+    .addSelect('SUM(summary.countOos)', 'countOos')
+
+  queryBuilder = selectPeriod(queryBuilder)
+    .innerJoin('summary.dataSet', 'ds', 'ds.id = :dataSetName', {
+      dataSetName: dataSet.id
+    })
+    .andWhere('summary.domain = :domain', { domain: domain.id })
+    .andWhere('summary.date >= :from', { from })
+    .andWhere('summary.date < :to', { to })
+    .addGroupBy('summary.dataSet')
+    .addGroupBy(groupByPeriod)
+
+  if (dataKeyItems.length > 0) {
+    queryBuilder.addSelect(selectKeys).addGroupBy(groupByKeys)
+  }
+
+  orderByPeriod.map(orderBy => {
+    const { name: sort, desc } = orderBy
+    queryBuilder.addOrderBy(sort, desc ? 'DESC' : 'ASC')
+  })
+
+  const items = await queryBuilder.getRawMany()
+
+  return items
+}

package/server/controllers/shiny-report.ts

@@ -0,0 +1,54 @@
+import { config } from '@things-factory/env'
+import { getEndpointUrl } from '../utils/config-resolver'
+
+import debugLib from 'debug'
+const debug = debugLib('things-factory:dataset:shiny-report')
+const dataReportConfig = config.get('dataReport')
+const {
+  shiny: {
+    endpoint: ENDPOINT,
+    datasource: { database: DATABASE }
+  }
+} = Object.assign(
+  {
+    shiny: {
+      endpoint: {},
+      datasource: {}
+    }
+  },
+  dataReportConfig
+)
+
+export async function renderShinyReport(context: any) {
+  debug('render:context', context)
+
+  try {
+    const cookies = new URLSearchParams(context.header.cookie.replace(/; /g, '&'))
+    const token = cookies.get('access_token')
+    const {
+      state: { domain },
+      query,
+      response
+    } = context
+
+    const urlParams = new URLSearchParams({
+      token,
+      ...query,
+      database: DATABASE,
+      domain: domain?.subdomain
+    }).toString()
+
+    const subpath = query?.reportView || ''
+
+    if (!subpath) {
+      return (context.body = '')
+    }
+
+    const url = getEndpointUrl(ENDPOINT, subpath)
+    const reportUrl = `${url}/?${urlParams}`
+
+    response.redirect(reportUrl)
+  } catch (ex) {
+    context.body = `Error: ${ex.message} || ''`
+  }
+}

package/server/engine/index.ts

@@ -0,0 +1 @@
+import './task'

package/server/engine/task/create-data-sample.ts

@@ -0,0 +1,100 @@
+import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
+import { getDataSource } from '@things-factory/shell'
+import { access } from '@things-factory/utils'
+import i18next from 'i18next'
+
+import { DataSet } from '../../service/data-set/data-set'
+
+import { createDataSample } from '../../controllers/create-data-sample'
+import { TaskHandler } from '@things-factory/integration-base/dist-server/engine/types'
+
+async function CreateDataSample(step: InputStep, context: Context) {
+  const { data, domain, user, lng } = context
+  var {
+    params: {
+      dataset: dataSetId,
+      source: sourceAccessor,
+      rawData: rawDataAccessor,
+      data: dataAccessor,
+      timestamp: timestampAccessor
+    }
+  } = step
+
+  if (!dataSetId) {
+    throw new Error(`no dataset found`)
+  }
+
+  // make new data-sample
+  return await getDataSource().transaction(async tx => {
+    const dataSet = await tx.getRepository(DataSet).findOne({
+      where: {
+        id: dataSetId
+      }
+    })
+
+    var source = await access(sourceAccessor, data)
+    var extractedData = await access(dataAccessor, data)
+    var extractedRawData = rawDataAccessor && (await access(rawDataAccessor, data))
+    var timestamp = timestampAccessor && (await access(timestampAccessor, data))
+
+    const dataSample = await createDataSample(
+      {
+        dataSet,
+        data: extractedData,
+        rawData: extractedRawData,
+        source,
+        collectedAt: new Date(timestamp || new Date())
+      },
+      {
+        t: i18next.t,
+        state: {
+          domain,
+          user,
+          lng,
+          tx
+        }
+      }
+    )
+
+    return {
+      data: dataSample
+    }
+  })
+}
+
+CreateDataSample.parameterSpec = [
+  {
+    type: 'entity-selector',
+    name: 'dataset',
+    label: 'data-set',
+    property: {
+      queryName: 'dataSets',
+      valueKey: 'id'
+    }
+  },
+  {
+    type: 'string',
+    name: 'source',
+    label: 'source'
+  },
+  {
+    type: 'string',
+    name: 'rawData',
+    label: 'raw-data'
+  },
+  {
+    type: 'string',
+    name: 'data',
+    label: 'data'
+  },
+  {
+    type: 'string',
+    name: 'timestamp',
+    label: 'timestamp'
+  }
+]
+
+CreateDataSample.connectorFree = true
+CreateDataSample.help = 'dataset/task/create-data-sample'
+
+TaskRegistry.registerTaskHandler('create-data-sample', CreateDataSample as TaskHandler)

package/server/engine/task/index.ts

@@ -0,0 +1,2 @@
+import './create-data-sample'
+import './issue-collect-data'

package/server/engine/task/issue-collect-data.ts

@@ -0,0 +1,45 @@
+import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
+import { issueDataCollectionTask } from '../../controllers/issue-data-collection-task'
+import i18next from 'i18next'
+
+async function IssueCollectData(step: InputStep, context: Context) {
+  const { logger, data, domain, user, lng } = context
+  var {
+    params: { dataset: dataSetId }
+  } = step
+
+  if (!dataSetId) {
+    throw new Error(`no dataset found`)
+  }
+
+  const activityInstance = await issueDataCollectionTask(domain.id, dataSetId, {
+    t: i18next.t,
+    state: {
+      domain,
+      user,
+      lng,
+      tx: undefined
+    }
+  })
+
+  return {
+    data: activityInstance
+  }
+}
+
+IssueCollectData.parameterSpec = [
+  {
+    type: 'entity-selector',
+    name: 'dataset',
+    label: 'data-set',
+    property: {
+      queryName: 'dataSets',
+      valueKey: 'id'
+    }
+  }
+]
+
+IssueCollectData.connectorFree = true
+IssueCollectData.help = 'dataset/task/issue-collect-data'
+
+TaskRegistry.registerTaskHandler('issue-collect-data', IssueCollectData)

package/server/index.ts

@@ -0,0 +1,8 @@
+import './routes'
+import './activities'
+import './engine'
+
+export * from './controllers'
+export * from './service'
+
+export * from './service/data-spec/data-spec-manager'