@things-factory/dataset 8.0.0 → 9.0.0-beta.3

package/server/controllers/query-data-summary-by-period.ts

@@ -1,178 +0,0 @@
-import { In } from 'typeorm'
-
-import { getQueryBuilderFromListParams, getRepository, getTimesForPeriod, ListParam, Sorting } from '@things-factory/shell'
-
-import { DataSet, DataSetSummaryPeriodType } from '../service/data-set/data-set'
-import { DataSummary } from '../service/data-summary/data-summary'
-
-const STAT_FUNCTION_MAP = {
-  sum: 'sum',
-  mean: 'mean',
-  stddev: 'standardDeviation',
-  variance: 'variance',
-  min: 'min',
-  max: 'max',
-  range: 'range',
-  median: 'median',
-  mode: 'mode'
-}
-
-export async function queryDataSummaryByPeriod(
-  period: 'today' | 'this month' | '30 days' | 'this year' | '12 months',
-  dataSetName: string,
-  dataKeys: string[] | null,
-  params: ListParam,
-  context: ResolverContext
-): Promise<DataSummary[]> {
-  const { domain, user, tx } = context.state
-  const { t } = context
-
-  const dataSet = await getRepository(DataSet).findOne({
-    where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, name: dataSetName },
-    relations: ['dataKeySet']
-  })
-
-  if (!dataSet) {
-    throw new Error(t('error.dataset not found', { dataSetName }))
-  }
-
-  // limitations
-  const summaryPeriodType = dataSet.summaryPeriod
-  if ((summaryPeriodType == DataSetSummaryPeriodType.Day || summaryPeriodType == DataSetSummaryPeriodType.WorkDate) && period == 'today') {
-    throw new Error(t('error.summary not supported', { dataSetName, period: t(`label.period-${period}`) }))
-  }
-
-  // dataKeys 가 설정되지 않았다면, dataSet의 dataKeySet을 그대로 적용한다는 의미임.
-  // dataKeys == [] 라면, dataKeySet을 적용하지 않는다는 의미임.
-  // dataKeys에는 dataKeySet의 dataKey 값을 따라야 한다.
-  const dataKeyItems = !dataSet.dataKeySet?.dataKeyItems
-    ? []
-    : dataKeys
-    ? dataKeys
-        .map(dataKey => {
-          return dataSet.dataKeySet.dataKeyItems.find(item => item.dataKey == dataKey)
-        })
-        .filter(Boolean)
-    : dataSet.dataKeySet.dataKeyItems
-
-  const searchables = dataKeyItems.map((item, index) => `key0${index + 1}`)
-  const dataItems = dataSet.dataItems.filter(item => item.agg)
-
-  const { from, to } = await getTimesForPeriod(period, context)
-
-  const selectPeriod = queryBuilder => {
-    if (period == 'today') {
-      queryBuilder.addSelect('summary.date', 'date').addSelect('summary.period', 'period')
-    } else if (['this year', '12 months'].includes(period)) {
-      queryBuilder.addSelect('SUBSTRING(summary.date, 1, 7) AS month')
-    } else {
-      queryBuilder.addSelect('summary.date', 'date')
-    }
-
-    return queryBuilder
-  }
-  const selectKeys = dataKeyItems.map((item, index) => {
-    const aliasName = `key0${index + 1}`
-    const columnName = `summary.${aliasName}`
-
-    return `${columnName} as ${aliasName}`
-  }) as string[]
-  const selectData = dataItems.map((item, index) => {
-    const aliasName = `data0${index + 1}`
-    const columnName = `summary.${aliasName}`
-
-    switch (item.agg) {
-      case STAT_FUNCTION_MAP.sum:
-        return `SUM(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.mean:
-        // 일차 mean 된 값들을 다시 계산하므로, 무의미할 수 있다.
-        return `AVG(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.stddev:
-        // 일차 stddev 된 값들을 다시 계산하므로, 무의미할 수 있다.
-        return `STDDEV(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.variance:
-        // 일차 variance 된 값들을 다시 계산하므로, 무의미할 수 있다.
-        return `VARIANCE(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.min:
-        return `MIN(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.max:
-        return `MAX(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.range:
-        // 일차 range 값들을 다시 계산하므로, 무의미할 수 있다.
-        return `MAX(${columnName}) - MIN(${columnName}) as ${aliasName}`
-      case STAT_FUNCTION_MAP.mode:
-        // not guaranteed
-        return `MODE() WITHIN GROUP (ORDER BY ${columnName})`
-      default:
-        return `AVG(${columnName}) as ${aliasName}`
-    }
-  })
-
-  const groupByPeriod =
-    period == 'today' ? 'summary.date, summary.period' : ['this year', '12 months'].includes(period) ? 'SUBSTRING(summary.date, 1, 7)' : 'summary.date'
-  const groupByKeys = dataKeyItems
-    .map((item, index) => {
-      return `summary.key0${index + 1}`
-    })
-    .join(', ')
-
-  const desc = false
-  var orderByPeriod: Sorting[] =
-    period == 'today'
-      ? [
-          {
-            name: 'date',
-            desc
-          },
-          {
-            name: 'period',
-            desc
-          }
-        ]
-      : ['this year', '12 months'].includes(period)
-      ? [{ name: 'month', desc }]
-      : [{ name: 'date', desc }]
-
-  if (params?.sortings && params.sortings.length > 0) {
-    orderByPeriod = params.sortings
-  }
-
-  var queryBuilder = getQueryBuilderFromListParams({
-    repository: getRepository(DataSummary),
-    params: {
-      ...params,
-      sortings: []
-    },
-    domain,
-    alias: 'summary',
-    searchables: searchables
-  })
-    .select('summary.dataSet')
-    .addSelect(selectData)
-    .addSelect('SUM(summary.count)', 'count')
-    .addSelect('SUM(summary.countOoc)', 'countOoc')
-    .addSelect('SUM(summary.countOos)', 'countOos')
-
-  queryBuilder = selectPeriod(queryBuilder)
-    .innerJoin('summary.dataSet', 'ds', 'ds.id = :dataSetName', {
-      dataSetName: dataSet.id
-    })
-    .andWhere('summary.domain = :domain', { domain: domain.id })
-    .andWhere('summary.date >= :from', { from })
-    .andWhere('summary.date < :to', { to })
-    .addGroupBy('summary.dataSet')
-    .addGroupBy(groupByPeriod)
-
-  if (dataKeyItems.length > 0) {
-    queryBuilder.addSelect(selectKeys).addGroupBy(groupByKeys)
-  }
-
-  orderByPeriod.map(orderBy => {
-    const { name: sort, desc } = orderBy
-    queryBuilder.addOrderBy(sort, desc ? 'DESC' : 'ASC')
-  })
-
-  const items = await queryBuilder.getRawMany()
-
-  return items
-}

package/server/controllers/shiny-report.ts

@@ -1,54 +0,0 @@
-import { config } from '@things-factory/env'
-import { getEndpointUrl } from '../utils/config-resolver'
-
-import debugLib from 'debug'
-const debug = debugLib('things-factory:dataset:shiny-report')
-const dataReportConfig = config.get('dataReport')
-const {
-  shiny: {
-    endpoint: ENDPOINT,
-    datasource: { database: DATABASE }
-  }
-} = Object.assign(
-  {
-    shiny: {
-      endpoint: {},
-      datasource: {}
-    }
-  },
-  dataReportConfig
-)
-
-export async function renderShinyReport(context: any) {
-  debug('render:context', context)
-
-  try {
-    const cookies = new URLSearchParams(context.header.cookie.replace(/; /g, '&'))
-    const token = cookies.get('access_token')
-    const {
-      state: { domain },
-      query,
-      response
-    } = context
-
-    const urlParams = new URLSearchParams({
-      token,
-      ...query,
-      database: DATABASE,
-      domain: domain?.subdomain
-    }).toString()
-
-    const subpath = query?.reportView || ''
-
-    if (!subpath) {
-      return (context.body = '')
-    }
-
-    const url = getEndpointUrl(ENDPOINT, subpath)
-    const reportUrl = `${url}/?${urlParams}`
-
-    response.redirect(reportUrl)
-  } catch (ex) {
-    context.body = `Error: ${ex.message} || ''`
-  }
-}

package/server/engine/index.ts

@@ -1 +0,0 @@
-import './task'

package/server/engine/task/create-data-sample.ts

@@ -1,100 +0,0 @@
-import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
-import { getDataSource } from '@things-factory/shell'
-import { access } from '@things-factory/utils'
-import i18next from 'i18next'
-
-import { DataSet } from '../../service/data-set/data-set'
-
-import { createDataSample } from '../../controllers/create-data-sample'
-import { TaskHandler } from '@things-factory/integration-base/dist-server/engine/types'
-
-async function CreateDataSample(step: InputStep, context: Context) {
-  const { data, domain, user, lng } = context
-  var {
-    params: {
-      dataset: dataSetId,
-      source: sourceAccessor,
-      rawData: rawDataAccessor,
-      data: dataAccessor,
-      timestamp: timestampAccessor
-    }
-  } = step
-
-  if (!dataSetId) {
-    throw new Error(`no dataset found`)
-  }
-
-  // make new data-sample
-  return await getDataSource().transaction(async tx => {
-    const dataSet = await tx.getRepository(DataSet).findOne({
-      where: {
-        id: dataSetId
-      }
-    })
-
-    var source = await access(sourceAccessor, data)
-    var extractedData = await access(dataAccessor, data)
-    var extractedRawData = rawDataAccessor && (await access(rawDataAccessor, data))
-    var timestamp = timestampAccessor && (await access(timestampAccessor, data))
-
-    const dataSample = await createDataSample(
-      {
-        dataSet,
-        data: extractedData,
-        rawData: extractedRawData,
-        source,
-        collectedAt: new Date(timestamp || new Date())
-      },
-      {
-        t: i18next.t,
-        state: {
-          domain,
-          user,
-          lng,
-          tx
-        }
-      }
-    )
-
-    return {
-      data: dataSample
-    }
-  })
-}
-
-CreateDataSample.parameterSpec = [
-  {
-    type: 'entity-selector',
-    name: 'dataset',
-    label: 'data-set',
-    property: {
-      queryName: 'dataSets',
-      valueKey: 'id'
-    }
-  },
-  {
-    type: 'string',
-    name: 'source',
-    label: 'source'
-  },
-  {
-    type: 'string',
-    name: 'rawData',
-    label: 'raw-data'
-  },
-  {
-    type: 'string',
-    name: 'data',
-    label: 'data'
-  },
-  {
-    type: 'string',
-    name: 'timestamp',
-    label: 'timestamp'
-  }
-]
-
-CreateDataSample.connectorFree = true
-CreateDataSample.help = 'dataset/task/create-data-sample'
-
-TaskRegistry.registerTaskHandler('create-data-sample', CreateDataSample as TaskHandler)

package/server/engine/task/index.ts

@@ -1,2 +0,0 @@
-import './create-data-sample'
-import './issue-collect-data'

package/server/engine/task/issue-collect-data.ts

@@ -1,45 +0,0 @@
-import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
-import { issueDataCollectionTask } from '../../controllers/issue-data-collection-task'
-import i18next from 'i18next'
-
-async function IssueCollectData(step: InputStep, context: Context) {
-  const { logger, data, domain, user, lng } = context
-  var {
-    params: { dataset: dataSetId }
-  } = step
-
-  if (!dataSetId) {
-    throw new Error(`no dataset found`)
-  }
-
-  const activityInstance = await issueDataCollectionTask(domain.id, dataSetId, {
-    t: i18next.t,
-    state: {
-      domain,
-      user,
-      lng,
-      tx: undefined
-    }
-  })
-
-  return {
-    data: activityInstance
-  }
-}
-
-IssueCollectData.parameterSpec = [
-  {
-    type: 'entity-selector',
-    name: 'dataset',
-    label: 'data-set',
-    property: {
-      queryName: 'dataSets',
-      valueKey: 'id'
-    }
-  }
-]
-
-IssueCollectData.connectorFree = true
-IssueCollectData.help = 'dataset/task/issue-collect-data'
-
-TaskRegistry.registerTaskHandler('issue-collect-data', IssueCollectData)

package/server/index.ts

@@ -1,8 +0,0 @@
-import './routes'
-import './activities'
-import './engine'
-
-export * from './controllers'
-export * from './service'
-
-export * from './service/data-spec/data-spec-manager'

package/server/routes.ts

@@ -1,188 +0,0 @@
-import { In } from 'typeorm'
-
-import { Domain, getDataSource } from '@things-factory/shell'
-import { User } from '@things-factory/auth-base'
-import { runScenario } from '@things-factory/integration-base'
-import { ScheduleRegisterRequest } from '@things-factory/scheduler-client'
-
-import { createDataSample } from './controllers/create-data-sample'
-import { renderJasperReport } from './controllers/jasper-report'
-import { renderShinyReport } from './controllers/shiny-report'
-import { issueDataCollectionTask } from './controllers/issue-data-collection-task'
-import { DataSensor } from './service/data-sensor/data-sensor'
-import { DataSet } from './service/data-set/data-set'
-
-process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRouter) => {
-  /*
-   * can add global public routes to application (auth not required, tenancy not required)
-   *
-   * ex) routes.get('/path', async(context, next) => {})
-   * ex) routes.post('/path', async(context, next) => {})
-   */
-
-  globalPublicRouter.post('/sensor-data', async (context, next) => {
-    // 데이타 검증
-    const { deviceId, data, rawData, timestamp = Date.now() } = context.request.body
-    if (!deviceId) {
-      throw new Error(`The deviceId(${deviceId}) property are mandatory`)
-    }
-
-    // make new data-sample
-    await getDataSource().transaction(async tx => {
-      // find sensor through deviceId
-      const sensor = await tx.getRepository(DataSensor).findOne({
-        where: { deviceId },
-        relations: ['domain', 'appliance', 'dataSet', 'decoder']
-      })
-
-      if (!sensor) {
-        throw new Error(`Sensor having given deviceId(${deviceId}) not found`)
-      }
-
-      if (!sensor.active) {
-        throw new Error(`State of the sensor given deviceId(${deviceId}) is not active`)
-      }
-
-      if (!sensor.appliance) {
-        throw new Error(`Appliance of the sensor given deviceId(${deviceId}) is not set up`)
-      }
-
-      const domain = sensor.domain
-      const dataSet = sensor.dataSet
-
-      /* 
-        원칙적으로 user 인증정보를 가지고 요청해야 하지만, 인증정보를 보낼 수 없는 상황이라고 전제하여 처리한다.
-        앞으로, 보완되어야 한다.
-      */
-      const user: User =
-        context.user ||
-        (await tx.getRepository(User).findOne({
-          where: {
-            reference: sensor.appliance.id,
-            userType: 'appliance'
-          }
-        }))
-
-      context.state = {
-        user,
-        ...context.state,
-        domain,
-        tx
-      }
-
-      var decoded
-      if (sensor.decoder) {
-        const { name: scenarioName } = sensor.decoder
-        const variables = {
-          dataSensor: sensor,
-          source: deviceId,
-          data,
-          rawData,
-          timestamp
-        }
-
-        decoded = (await runScenario(null, scenarioName, variables, context))?.result
-      }
-
-      if (dataSet) {
-        await createDataSample(
-          {
-            dataSet,
-            data,
-            rawData,
-            source: deviceId,
-            collectedAt: new Date(timestamp),
-            ...decoded
-          },
-          context
-        )
-      }
-
-      context.status = 200
-      context.body = {
-        data,
-        ...decoded
-      }
-    })
-  })
-
-  /* When a callback occurs from the scheduler when a scheduled dataset is on schedule, data collection task for the dataset should be issued. */
-  globalPublicRouter.post('/callback-schedule-for-dataset', async (context, next) => {
-    const { client } = context.request.body as ScheduleRegisterRequest
-
-    if (!client || typeof client !== 'object') {
-      throw new Error('client property should be a part of callback body.')
-    }
-
-    const { group: domainId, key: dataSetId } = client
-
-    if (!domainId || !dataSetId) {
-      throw new Error(`group(${domainId}) and key(${dataSetId}) properties should not be empty`)
-    }
-
-    await issueDataCollectionTask(domainId, dataSetId, context)
-
-    context.status = 200
-  })
-
-  /* When a callback occurs from the scheduler when a scheduled summary is on schedule, data summary procedure for the dataset should be executed. */
-  globalPublicRouter.post('/callback-schedule-for-data-summary', async (context, next) => {
-    const { client } = context.request.body as ScheduleRegisterRequest
-
-    if (!client || typeof client !== 'object') {
-      throw new Error('client property should be a part of callback body.')
-    }
-
-    const { group: domainId, key: dataSetId } = client
-
-    if (!domainId || !dataSetId) {
-      throw new Error(`group(${domainId}) and key(${dataSetId}) properties should not be empty`)
-    }
-
-    await getDataSource().transaction(async tx => {
-      const domain = await tx.getRepository(Domain).findOneBy({ id: domainId })
-
-      if (!domain) {
-        throw new Error(`domain(${domainId}) not found`)
-      }
-
-      const dataSet = await tx
-        .getRepository(DataSet)
-        .findOne({ where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: dataSetId } })
-
-      // do what you gotta do
-    })
-
-    context.status = 200
-  })
-})
-
-process.on('bootstrap-module-global-private-route' as any, (app, globalPrivateRouter) => {
-  /*
-   * can add global private routes to application (auth required, tenancy not required)
-   */
-})
-
-process.on('bootstrap-module-domain-public-route' as any, (app, domainPublicRouter) => {
-  /*
-   * can add domain public routes to application (auth not required, tenancy required)
-   */
-})
-
-process.on('bootstrap-module-domain-private-route' as any, (app, domainPrivateRouter) => {
-  /*
-   * can add domain private routes to application (auth required, tenancy required)
-   */
-  domainPrivateRouter.get('/data-report/jasper', async (context, next) => {
-    context.body = await renderJasperReport(context)
-  })
-
-  domainPrivateRouter.get('/data-report/shiny', async (context, next) => {
-    await renderShinyReport(context)
-  })
-
-  domainPrivateRouter.get('/data-report/:reportType', async (context, next) => {
-    let { reportType } = context.params
-    context.body = `${reportType} Not Implemented.`
-  })
-})