@things-factory/dataset 8.0.39 → 9.0.0-9.0.0-beta.59.0

package/server/controllers/jasper-report.ts

@@ -1,186 +0,0 @@
-import FormData from 'form-data'
-import fetch from 'node-fetch'
-
-import { getEndpointUrl } from '../utils/config-resolver'
-
-import { STORAGE } from '@things-factory/attachment-base'
-import { AthenaController } from '@things-factory/aws-base'
-import { config } from '@things-factory/env'
-
-const dataReportConfig = config.get('dataReport')
-const {
-  jasper: {
-    endpoint: ENDPOINT,
-    datasource: { database: DATABASE }
-  }
-} = dataReportConfig || {
-  jasper: {
-    endpoint: {},
-    datasource: {}
-  }
-}
-
-function transformValuesToRows(queryResult) {
-  var parseData = []
-  let index = 1
-  for (let i = 0; i < queryResult.Items.length; i++) {
-    var j = 0
-    const data = JSON.parse(queryResult.Items[i].data)
-    const spec = JSON.parse(queryResult.Items[i].spec)
-
-    for (let key in data) {
-      if (Array.isArray(data[key])) {
-        for (j = 0; j < data[key].length; j++) {
-          for (let specKey in spec) {
-            if (key === specKey) {
-              parseData.push({
-                item: spec[specKey].name,
-                index: index + j,
-                value: String(data[key][j])
-              })
-            }
-          }
-        }
-      } else {
-        parseData.push({
-          item: key,
-          index,
-          value: String(data[key])
-        })
-      }
-    }
-    if (j !== 0) {
-      index = index + j
-    } else {
-      index = index + 1
-    }
-  }
-}
-
-/** @todo considering trasformation in lambda, as massive dataset */
-function pivotData(rows) {
-  let parsedData = []
-  let index = 1
-  for (let i = 0; i < rows.length; i++) {
-    let j = 0
-    const data = JSON.parse(rows[i].data)
-    const spec = JSON.parse(rows[i].spec)
-
-    for (let key in data) {
-      /** @todo rule to display or not, about unspecified spec */
-      const value = data[key]
-      !spec[key]?.hidden &&
-        parsedData.push({
-          item: spec[key]?.name || key,
-          index,
-          value: Array.isArray(value) ? value.join(', ') : value
-        })
-    }
-    if (j !== 0) {
-      index = index + j
-    } else {
-      index = index + 1
-    }
-  }
-
-  return parsedData
-}
-
-function parseJsonDataField(rows) {
-  let parsedData = []
-  for (let i = 0; i < rows.length; i++) {
-    const row = rows[i]
-    const data = JSON.parse(row.data)
-    for (let key in data) {
-      if (Array.isArray(data[key])) {
-        data[key] = data[key].toString()
-      }
-    }
-    delete row.data
-    parsedData.push({ ...row, ...data })
-  }
-
-  return parsedData
-}
-
-const athenaClient = new AthenaController()
-
-async function queryAthena(params) {
-  const { table, domain, datasetId, startDate, endDate, workShift, timezone } = params
-  const queryData = {
-    sql: `SELECT ds.name, ds.description, ds.data, dsh.data_items as spec, ds.workdate, ds.workshift,
-    DATE_FORMAT(
-      FROM_UNIXTIME(collected_at / 1000 / 1000) AT TIME ZONE '${timezone || 'UTC'}', 
-      '%Y-%m-%d %H:%i:%s'
-    ) AS dscollected_at
-    FROM ${table} ds
-    JOIN data_set_histories dsh 
-      ON (ds.datasetid = dsh.original_id
-        and ds.data_set_version = dsh.version)
-    WHERE ds.domain='${domain}' 
-    AND ds.datasetid = '${datasetId}'
-    AND ds.workdate >= '${startDate}'
-    AND ds.workdate <= '${endDate}'
-    ${workShift ? "AND ds.workshift = '" + workShift + "'" : ''}
-    ORDER BY ds.collected_at`,
-    db: DATABASE
-  }
-  // and json_extract_scalar(data, '$.dauid') = 'A8032AD81730'
-
-  return await athenaClient.query(queryData)
-}
-
-export async function renderJasperReport(context: any) {
-  const {
-    state: { domain },
-    query
-  } = context
-
-  const template = await STORAGE.readFile(query['reportTemplate'] || 'dynamic_header_sample.jrxml', 'utf-8')
-  let templateType = query['templateType'] || 'crosstab'
-  let parsedData = []
-
-  // @todo: get dataset timezone
-  /**
-   * const variables = await gql(dataSet(id:${dataSetId}) {
-   *  name, description, partition_keys, timezone
-   * })
-   */
-
-  query['domain'] = domain?.subdomain
-  query['timezone'] = domain?.timezone
-  const queryResult = await queryAthena(query)
-  const rows = queryResult.Items
-
-  if (!rows.length) {
-    return '<h3>Not found result.</h3>'
-  } else {
-    const firstRow = rows[0]
-    // uses the first row values as data-set has no history data.
-    const parameters = {
-      name: firstRow.name,
-      description: firstRow.description,
-      ...query
-    }
-
-    if (templateType === 'crosstab') {
-      parsedData = pivotData(rows)
-    } else {
-      parsedData = parseJsonDataField(rows)
-    }
-
-    const formData = new FormData()
-    formData.append('template', template)
-    formData.append('jsonString', JSON.stringify(parsedData))
-    formData.append('parameters', JSON.stringify(parameters))
-
-    const { reportView } = query
-    const url = getEndpointUrl(ENDPOINT, reportView)
-    const response = await fetch(url, {
-      method: 'POST',
-      body: formData
-    })
-
-    return await response.text()
-  }
-}

package/server/controllers/query-data-summary-by-period.ts

@@ -1,178 +0,0 @@
-import { In } from 'typeorm'
-
-import { getQueryBuilderFromListParams, getRepository, getTimesForPeriod, ListParam, Sorting } from '@things-factory/shell'
-
-import { DataSet, DataSetSummaryPeriodType } from '../service/data-set/data-set'
-import { DataSummary } from '../service/data-summary/data-summary'
-
-const STAT_FUNCTION_MAP = {
-  sum: 'sum',
-  mean: 'mean',
-  stddev: 'standardDeviation',
-  variance: 'variance',
-  min: 'min',
-  max: 'max',
-  range: 'range',
-  median: 'median',
-  mode: 'mode'
-}
-
-export async function queryDataSummaryByPeriod(
-  period: 'today' | 'this month' | '30 days' | 'this year' | '12 months',
-  dataSetName: string,
-  dataKeys: string[] | null,
-  params: ListParam,
-  context: ResolverContext
-): Promise<DataSummary[]> {
-  const { domain, user, tx } = context.state
-  const { t } = context
-
-  const dataSet = await getRepository(DataSet).findOne({
-    where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, name: dataSetName },
-    relations: ['dataKeySet']
-  })
-
-  if (!dataSet) {
-    throw new Error(t('error.dataset not found', { dataSetName }))
-  }
-
-  // limitations
-  const summaryPeriodType = dataSet.summaryPeriod
-  if ((summaryPeriodType == DataSetSummaryPeriodType.Day || summaryPeriodType == DataSetSummaryPeriodType.WorkDate) && period == 'today') {
-    throw new Error(t('error.summary not supported', { dataSetName, period: t(`label.period-${period}`) }))
-  }
-
-  // dataKeys 가 설정되지 않았다면, dataSet의 dataKeySet을 그대로 적용한다는 의미임.
-  // dataKeys == [] 라면, dataKeySet을 적용하지 않는다는 의미임.
-  // dataKeys에는 dataKeySet의 dataKey 값을 따라야 한다.
-  const dataKeyItems = !dataSet.dataKeySet?.dataKeyItems
-    ? []
-    : dataKeys
-    ? dataKeys
-        .map(dataKey => {
-          return dataSet.dataKeySet.dataKeyItems.find(item => item.dataKey == dataKey)
-        })
-        .filter(Boolean)
-    : dataSet.dataKeySet.dataKeyItems
-
-  const searchables = dataKeyItems.map((item, index) => `key0${index + 1}`)
-  const dataItems = dataSet.dataItems.filter(item => item.agg)
-
-  const { from, to } = await getTimesForPeriod(period, context)
-
-  const selectPeriod = queryBuilder => {
-    if (period == 'today') {
-      queryBuilder.addSelect('summary.date', 'date').addSelect('summary.period', 'period')
-    } else if (['this year', '12 months'].includes(period)) {
-      queryBuilder.addSelect('SUBSTRING(summary.date, 1, 7) AS month')
-    } else {
-      queryBuilder.addSelect('summary.date', 'date')
-    }
-
-    return queryBuilder
-  }
-  const selectKeys = dataKeyItems.map((item, index) => {
-    const aliasName = `key0${index + 1}`
-    const columnName = `summary.${aliasName}`
-
-    return `${columnName} as ${aliasName}`
-  }) as string[]
-  const selectData = dataItems.map((item, index) => {
-    const aliasName = `data0${index + 1}`
-    const columnName = `summary.${aliasName}`
-
-    switch (item.agg) {
-      case STAT_FUNCTION_MAP.sum:
-        return `SUM(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.mean:
-        // 일차 mean 된 값들을 다시 계산하므로, 무의미할 수 있다.
-        return `AVG(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.stddev:
-        // 일차 stddev 된 값들을 다시 계산하므로, 무의미할 수 있다.
-        return `STDDEV(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.variance:
-        // 일차 variance 된 값들을 다시 계산하므로, 무의미할 수 있다.
-        return `VARIANCE(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.min:
-        return `MIN(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.max:
-        return `MAX(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.range:
-        // 일차 range 값들을 다시 계산하므로, 무의미할 수 있다.
-        return `MAX(${columnName}) - MIN(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.mode:
-        // not guaranteed
-        return `MODE() WITHIN GROUP (ORDER BY ${columnName})`
-      default:
-        return `AVG(${columnName}) as ${aliasName}`
-    }
-  })
-
-  const groupByPeriod =
-    period == 'today' ? 'summary.date, summary.period' : ['this year', '12 months'].includes(period) ? 'SUBSTRING(summary.date, 1, 7)' : 'summary.date'
-  const groupByKeys = dataKeyItems
-    .map((item, index) => {
-      return `summary.key0${index + 1}`
-    })
-    .join(', ')
-
-  const desc = false
-  var orderByPeriod: Sorting[] =
-    period == 'today'
-      ? [
-          {
-            name: 'date',
-            desc
-          },
-          {
-            name: 'period',
-            desc
-          }
-        ]
-      : ['this year', '12 months'].includes(period)
-      ? [{ name: 'month', desc }]
-      : [{ name: 'date', desc }]
-
-  if (params?.sortings && params.sortings.length > 0) {
-    orderByPeriod = params.sortings
-  }
-
-  var queryBuilder = getQueryBuilderFromListParams({
-    repository: getRepository(DataSummary),
-    params: {
-      ...params,
-      sortings: []
-    },
-    domain,
-    alias: 'summary',
-    searchables: searchables
-  })
-    .select('summary.dataSet')
-    .addSelect(selectData)
-    .addSelect('SUM(summary.count)', 'count')
-    .addSelect('SUM(summary.countOoc)', 'countOoc')
-    .addSelect('SUM(summary.countOos)', 'countOos')
-
-  queryBuilder = selectPeriod(queryBuilder)
-    .innerJoin('summary.dataSet', 'ds', 'ds.id = :dataSetName', {
-      dataSetName: dataSet.id
-    })
-    .andWhere('summary.domain = :domain', { domain: domain.id })
-    .andWhere('summary.date >= :from', { from })
-    .andWhere('summary.date < :to', { to })
-    .addGroupBy('summary.dataSet')
-    .addGroupBy(groupByPeriod)
-
-  if (dataKeyItems.length > 0) {
-    queryBuilder.addSelect(selectKeys).addGroupBy(groupByKeys)
-  }
-
-  orderByPeriod.map(orderBy => {
-    const { name: sort, desc } = orderBy
-    queryBuilder.addOrderBy(sort, desc ? 'DESC' : 'ASC')
-  })
-
-  const items = await queryBuilder.getRawMany()
-
-  return items
-}

package/server/controllers/shiny-report.ts

@@ -1,54 +0,0 @@
-import { config } from '@things-factory/env'
-import { getEndpointUrl } from '../utils/config-resolver'
-
-import debugLib from 'debug'
-const debug = debugLib('things-factory:dataset:shiny-report')
-const dataReportConfig = config.get('dataReport')
-const {
-  shiny: {
-    endpoint: ENDPOINT,
-    datasource: { database: DATABASE }
-  }
-} = Object.assign(
-  {
-    shiny: {
-      endpoint: {},
-      datasource: {}
-    }
-  },
-  dataReportConfig
-)
-
-export async function renderShinyReport(context: any) {
-  debug('render:context', context)
-
-  try {
-    const cookies = new URLSearchParams(context.header.cookie.replace(/; /g, '&'))
-    const token = cookies.get('access_token')
-    const {
-      state: { domain },
-      query,
-      response
-    } = context
-
-    const urlParams = new URLSearchParams({
-      token,
-      ...query,
-      database: DATABASE,
-      domain: domain?.subdomain
-    }).toString()
-
-    const subpath = query?.reportView || ''
-
-    if (!subpath) {
-      return (context.body = '')
-    }
-
-    const url = getEndpointUrl(ENDPOINT, subpath)
-    const reportUrl = `${url}/?${urlParams}`
-
-    response.redirect(reportUrl)
-  } catch (ex) {
-    context.body = `Error: ${ex.message} || ''`
-  }
-}

package/server/engine/index.ts

@@ -1 +0,0 @@
-import './task'

package/server/engine/task/create-data-sample.ts

@@ -1,100 +0,0 @@
-import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
-import { getDataSource } from '@things-factory/shell'
-import { access } from '@things-factory/utils'
-import i18next from 'i18next'
-
-import { DataSet } from '../../service/data-set/data-set'
-
-import { createDataSample } from '../../controllers/create-data-sample'
-import { TaskHandler } from '@things-factory/integration-base/dist-server/engine/types'
-
-async function CreateDataSample(step: InputStep, context: Context) {
-  const { data, domain, user, lng } = context
-  var {
-    params: {
-      dataset: dataSetId,
-      source: sourceAccessor,
-      rawData: rawDataAccessor,
-      data: dataAccessor,
-      timestamp: timestampAccessor
-    }
-  } = step
-
-  if (!dataSetId) {
-    throw new Error(`no dataset found`)
-  }
-
-  // make new data-sample
-  return await getDataSource().transaction(async tx => {
-    const dataSet = await tx.getRepository(DataSet).findOne({
-      where: {
-        id: dataSetId
-      }
-    })
-
-    var source = await access(sourceAccessor, data)
-    var extractedData = await access(dataAccessor, data)
-    var extractedRawData = rawDataAccessor && (await access(rawDataAccessor, data))
-    var timestamp = timestampAccessor && (await access(timestampAccessor, data))
-
-    const dataSample = await createDataSample(
-      {
-        dataSet,
-        data: extractedData,
-        rawData: extractedRawData,
-        source,
-        collectedAt: new Date(timestamp || new Date())
-      },
-      {
-        t: i18next.t,
-        state: {
-          domain,
-          user,
-          lng,
-          tx
-        }
-      }
-    )
-
-    return {
-      data: dataSample
-    }
-  })
-}
-
-CreateDataSample.parameterSpec = [
-  {
-    type: 'entity-selector',
-    name: 'dataset',
-    label: 'data-set',
-    property: {
-      queryName: 'dataSets',
-      valueKey: 'id'
-    }
-  },
-  {
-    type: 'string',
-    name: 'source',
-    label: 'source'
-  },
-  {
-    type: 'string',
-    name: 'rawData',
-    label: 'raw-data'
-  },
-  {
-    type: 'string',
-    name: 'data',
-    label: 'data'
-  },
-  {
-    type: 'string',
-    name: 'timestamp',
-    label: 'timestamp'
-  }
-]
-
-CreateDataSample.connectorFree = true
-CreateDataSample.help = 'dataset/task/create-data-sample'
-
-TaskRegistry.registerTaskHandler('create-data-sample', CreateDataSample as TaskHandler)

package/server/engine/task/index.ts

@@ -1,2 +0,0 @@
-import './create-data-sample'
-import './issue-collect-data'

package/server/engine/task/issue-collect-data.ts

@@ -1,45 +0,0 @@
-import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
-import { issueDataCollectionTask } from '../../controllers/issue-data-collection-task'
-import i18next from 'i18next'
-
-async function IssueCollectData(step: InputStep, context: Context) {
-  const { logger, data, domain, user, lng } = context
-  var {
-    params: { dataset: dataSetId }
-  } = step
-
-  if (!dataSetId) {
-    throw new Error(`no dataset found`)
-  }
-
-  const activityInstance = await issueDataCollectionTask(domain.id, dataSetId, {
-    t: i18next.t,
-    state: {
-      domain,
-      user,
-      lng,
-      tx: undefined
-    }
-  })
-
-  return {
-    data: activityInstance
-  }
-}
-
-IssueCollectData.parameterSpec = [
-  {
-    type: 'entity-selector',
-    name: 'dataset',
-    label: 'data-set',
-    property: {
-      queryName: 'dataSets',
-      valueKey: 'id'
-    }
-  }
-]
-
-IssueCollectData.connectorFree = true
-IssueCollectData.help = 'dataset/task/issue-collect-data'
-
-TaskRegistry.registerTaskHandler('issue-collect-data', IssueCollectData)

package/server/index.ts

@@ -1,8 +0,0 @@
-import './routes'
-import './activities'
-import './engine'
-
-export * from './controllers'
-export * from './service'
-
-export * from './service/data-spec/data-spec-manager'