@things-factory/dataset 8.0.0-beta.9 → 8.0.0

package/server/controllers/query-data-summary-by-period.ts

@@ -0,0 +1,178 @@
+import { In } from 'typeorm'
+
+import { getQueryBuilderFromListParams, getRepository, getTimesForPeriod, ListParam, Sorting } from '@things-factory/shell'
+
+import { DataSet, DataSetSummaryPeriodType } from '../service/data-set/data-set'
+import { DataSummary } from '../service/data-summary/data-summary'
+
+const STAT_FUNCTION_MAP = {
+  sum: 'sum',
+  mean: 'mean',
+  stddev: 'standardDeviation',
+  variance: 'variance',
+  min: 'min',
+  max: 'max',
+  range: 'range',
+  median: 'median',
+  mode: 'mode'
+}
+
+export async function queryDataSummaryByPeriod(
+  period: 'today' | 'this month' | '30 days' | 'this year' | '12 months',
+  dataSetName: string,
+  dataKeys: string[] | null,
+  params: ListParam,
+  context: ResolverContext
+): Promise<DataSummary[]> {
+  const { domain, user, tx } = context.state
+  const { t } = context
+
+  const dataSet = await getRepository(DataSet).findOne({
+    where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, name: dataSetName },
+    relations: ['dataKeySet']
+  })
+
+  if (!dataSet) {
+    throw new Error(t('error.dataset not found', { dataSetName }))
+  }
+
+  // limitations
+  const summaryPeriodType = dataSet.summaryPeriod
+  if ((summaryPeriodType == DataSetSummaryPeriodType.Day || summaryPeriodType == DataSetSummaryPeriodType.WorkDate) && period == 'today') {
+    throw new Error(t('error.summary not supported', { dataSetName, period: t(`label.period-${period}`) }))
+  }
+
+  // dataKeys 가 설정되지 않았다면, dataSet의 dataKeySet을 그대로 적용한다는 의미임.
+  // dataKeys == [] 라면, dataKeySet을 적용하지 않는다는 의미임.
+  // dataKeys에는 dataKeySet의 dataKey 값을 따라야 한다.
+  const dataKeyItems = !dataSet.dataKeySet?.dataKeyItems
+    ? []
+    : dataKeys
+    ? dataKeys
+        .map(dataKey => {
+          return dataSet.dataKeySet.dataKeyItems.find(item => item.dataKey == dataKey)
+        })
+        .filter(Boolean)
+    : dataSet.dataKeySet.dataKeyItems
+
+  const searchables = dataKeyItems.map((item, index) => `key0${index + 1}`)
+  const dataItems = dataSet.dataItems.filter(item => item.agg)
+
+  const { from, to } = await getTimesForPeriod(period, context)
+
+  const selectPeriod = queryBuilder => {
+    if (period == 'today') {
+      queryBuilder.addSelect('summary.date', 'date').addSelect('summary.period', 'period')
+    } else if (['this year', '12 months'].includes(period)) {
+      queryBuilder.addSelect('SUBSTRING(summary.date, 1, 7) AS month')
+    } else {
+      queryBuilder.addSelect('summary.date', 'date')
+    }
+
+    return queryBuilder
+  }
+  const selectKeys = dataKeyItems.map((item, index) => {
+    const aliasName = `key0${index + 1}`
+    const columnName = `summary.${aliasName}`
+
+    return `${columnName} as ${aliasName}`
+  }) as string[]
+  const selectData = dataItems.map((item, index) => {
+    const aliasName = `data0${index + 1}`
+    const columnName = `summary.${aliasName}`
+
+    switch (item.agg) {
+      case STAT_FUNCTION_MAP.sum:
+        return `SUM(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.mean:
+        // 일차 mean 된 값들을 다시 계산하므로, 무의미할 수 있다.
+        return `AVG(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.stddev:
+        // 일차 stddev 된 값들을 다시 계산하므로, 무의미할 수 있다.
+        return `STDDEV(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.variance:
+        // 일차 variance 된 값들을 다시 계산하므로, 무의미할 수 있다.
+        return `VARIANCE(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.min:
+        return `MIN(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.max:
+        return `MAX(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.range:
+        // 일차 range 값들을 다시 계산하므로, 무의미할 수 있다.
+        return `MAX(${columnName}) - MIN(${columnName}) as ${aliasName}`
+      case STAT_FUNCTION_MAP.mode:
+        // not guaranteed
+        return `MODE() WITHIN GROUP (ORDER BY ${columnName})`
+      default:
+        return `AVG(${columnName}) as ${aliasName}`
+    }
+  })
+
+  const groupByPeriod =
+    period == 'today' ? 'summary.date, summary.period' : ['this year', '12 months'].includes(period) ? 'SUBSTRING(summary.date, 1, 7)' : 'summary.date'
+  const groupByKeys = dataKeyItems
+    .map((item, index) => {
+      return `summary.key0${index + 1}`
+    })
+    .join(', ')
+
+  const desc = false
+  var orderByPeriod: Sorting[] =
+    period == 'today'
+      ? [
+          {
+            name: 'date',
+            desc
+          },
+          {
+            name: 'period',
+            desc
+          }
+        ]
+      : ['this year', '12 months'].includes(period)
+      ? [{ name: 'month', desc }]
+      : [{ name: 'date', desc }]
+
+  if (params?.sortings && params.sortings.length > 0) {
+    orderByPeriod = params.sortings
+  }
+
+  var queryBuilder = getQueryBuilderFromListParams({
+    repository: getRepository(DataSummary),
+    params: {
+      ...params,
+      sortings: []
+    },
+    domain,
+    alias: 'summary',
+    searchables: searchables
+  })
+    .select('summary.dataSet')
+    .addSelect(selectData)
+    .addSelect('SUM(summary.count)', 'count')
+    .addSelect('SUM(summary.countOoc)', 'countOoc')
+    .addSelect('SUM(summary.countOos)', 'countOos')
+
+  queryBuilder = selectPeriod(queryBuilder)
+    .innerJoin('summary.dataSet', 'ds', 'ds.id = :dataSetName', {
+      dataSetName: dataSet.id
+    })
+    .andWhere('summary.domain = :domain', { domain: domain.id })
+    .andWhere('summary.date >= :from', { from })
+    .andWhere('summary.date < :to', { to })
+    .addGroupBy('summary.dataSet')
+    .addGroupBy(groupByPeriod)
+
+  if (dataKeyItems.length > 0) {
+    queryBuilder.addSelect(selectKeys).addGroupBy(groupByKeys)
+  }
+
+  orderByPeriod.map(orderBy => {
+    const { name: sort, desc } = orderBy
+    queryBuilder.addOrderBy(sort, desc ? 'DESC' : 'ASC')
+  })
+
+  const items = await queryBuilder.getRawMany()
+
+  return items
+}

package/server/controllers/shiny-report.ts

@@ -0,0 +1,54 @@
+import { config } from '@things-factory/env'
+import { getEndpointUrl } from '../utils/config-resolver'
+
+import debugLib from 'debug'
+const debug = debugLib('things-factory:dataset:shiny-report')
+const dataReportConfig = config.get('dataReport')
+const {
+  shiny: {
+    endpoint: ENDPOINT,
+    datasource: { database: DATABASE }
+  }
+} = Object.assign(
+  {
+    shiny: {
+      endpoint: {},
+      datasource: {}
+    }
+  },
+  dataReportConfig
+)
+
+export async function renderShinyReport(context: any) {
+  debug('render:context', context)
+
+  try {
+    const cookies = new URLSearchParams(context.header.cookie.replace(/; /g, '&'))
+    const token = cookies.get('access_token')
+    const {
+      state: { domain },
+      query,
+      response
+    } = context
+
+    const urlParams = new URLSearchParams({
+      token,
+      ...query,
+      database: DATABASE,
+      domain: domain?.subdomain
+    }).toString()
+
+    const subpath = query?.reportView || ''
+
+    if (!subpath) {
+      return (context.body = '')
+    }
+
+    const url = getEndpointUrl(ENDPOINT, subpath)
+    const reportUrl = `${url}/?${urlParams}`
+
+    response.redirect(reportUrl)
+  } catch (ex) {
+    context.body = `Error: ${ex.message} || ''`
+  }
+}

package/server/engine/index.ts

@@ -0,0 +1 @@
+import './task'

package/server/engine/task/create-data-sample.ts

@@ -0,0 +1,100 @@
+import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
+import { getDataSource } from '@things-factory/shell'
+import { access } from '@things-factory/utils'
+import i18next from 'i18next'
+
+import { DataSet } from '../../service/data-set/data-set'
+
+import { createDataSample } from '../../controllers/create-data-sample'
+import { TaskHandler } from '@things-factory/integration-base/dist-server/engine/types'
+
+async function CreateDataSample(step: InputStep, context: Context) {
+  const { data, domain, user, lng } = context
+  var {
+    params: {
+      dataset: dataSetId,
+      source: sourceAccessor,
+      rawData: rawDataAccessor,
+      data: dataAccessor,
+      timestamp: timestampAccessor
+    }
+  } = step
+
+  if (!dataSetId) {
+    throw new Error(`no dataset found`)
+  }
+
+  // make new data-sample
+  return await getDataSource().transaction(async tx => {
+    const dataSet = await tx.getRepository(DataSet).findOne({
+      where: {
+        id: dataSetId
+      }
+    })
+
+    var source = await access(sourceAccessor, data)
+    var extractedData = await access(dataAccessor, data)
+    var extractedRawData = rawDataAccessor && (await access(rawDataAccessor, data))
+    var timestamp = timestampAccessor && (await access(timestampAccessor, data))
+
+    const dataSample = await createDataSample(
+      {
+        dataSet,
+        data: extractedData,
+        rawData: extractedRawData,
+        source,
+        collectedAt: new Date(timestamp || new Date())
+      },
+      {
+        t: i18next.t,
+        state: {
+          domain,
+          user,
+          lng,
+          tx
+        }
+      }
+    )
+
+    return {
+      data: dataSample
+    }
+  })
+}
+
+CreateDataSample.parameterSpec = [
+  {
+    type: 'entity-selector',
+    name: 'dataset',
+    label: 'data-set',
+    property: {
+      queryName: 'dataSets',
+      valueKey: 'id'
+    }
+  },
+  {
+    type: 'string',
+    name: 'source',
+    label: 'source'
+  },
+  {
+    type: 'string',
+    name: 'rawData',
+    label: 'raw-data'
+  },
+  {
+    type: 'string',
+    name: 'data',
+    label: 'data'
+  },
+  {
+    type: 'string',
+    name: 'timestamp',
+    label: 'timestamp'
+  }
+]
+
+CreateDataSample.connectorFree = true
+CreateDataSample.help = 'dataset/task/create-data-sample'
+
+TaskRegistry.registerTaskHandler('create-data-sample', CreateDataSample as TaskHandler)

package/server/engine/task/index.ts

@@ -0,0 +1,2 @@
+import './create-data-sample'
+import './issue-collect-data'

package/server/engine/task/issue-collect-data.ts

@@ -0,0 +1,45 @@
+import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
+import { issueDataCollectionTask } from '../../controllers/issue-data-collection-task'
+import i18next from 'i18next'
+
+async function IssueCollectData(step: InputStep, context: Context) {
+  const { logger, data, domain, user, lng } = context
+  var {
+    params: { dataset: dataSetId }
+  } = step
+
+  if (!dataSetId) {
+    throw new Error(`no dataset found`)
+  }
+
+  const activityInstance = await issueDataCollectionTask(domain.id, dataSetId, {
+    t: i18next.t,
+    state: {
+      domain,
+      user,
+      lng,
+      tx: undefined
+    }
+  })
+
+  return {
+    data: activityInstance
+  }
+}
+
+IssueCollectData.parameterSpec = [
+  {
+    type: 'entity-selector',
+    name: 'dataset',
+    label: 'data-set',
+    property: {
+      queryName: 'dataSets',
+      valueKey: 'id'
+    }
+  }
+]
+
+IssueCollectData.connectorFree = true
+IssueCollectData.help = 'dataset/task/issue-collect-data'
+
+TaskRegistry.registerTaskHandler('issue-collect-data', IssueCollectData)

package/server/index.ts

@@ -0,0 +1,8 @@
+import './routes'
+import './activities'
+import './engine'
+
+export * from './controllers'
+export * from './service'
+
+export * from './service/data-spec/data-spec-manager'

package/server/routes.ts

@@ -0,0 +1,188 @@
+import { In } from 'typeorm'
+
+import { Domain, getDataSource } from '@things-factory/shell'
+import { User } from '@things-factory/auth-base'
+import { runScenario } from '@things-factory/integration-base'
+import { ScheduleRegisterRequest } from '@things-factory/scheduler-client'
+
+import { createDataSample } from './controllers/create-data-sample'
+import { renderJasperReport } from './controllers/jasper-report'
+import { renderShinyReport } from './controllers/shiny-report'
+import { issueDataCollectionTask } from './controllers/issue-data-collection-task'
+import { DataSensor } from './service/data-sensor/data-sensor'
+import { DataSet } from './service/data-set/data-set'
+
+process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRouter) => {
+  /*
+   * can add global public routes to application (auth not required, tenancy not required)
+   *
+   * ex) routes.get('/path', async(context, next) => {})
+   * ex) routes.post('/path', async(context, next) => {})
+   */
+
+  globalPublicRouter.post('/sensor-data', async (context, next) => {
+    // 데이타 검증
+    const { deviceId, data, rawData, timestamp = Date.now() } = context.request.body
+    if (!deviceId) {
+      throw new Error(`The deviceId(${deviceId}) property are mandatory`)
+    }
+
+    // make new data-sample
+    await getDataSource().transaction(async tx => {
+      // find sensor through deviceId
+      const sensor = await tx.getRepository(DataSensor).findOne({
+        where: { deviceId },
+        relations: ['domain', 'appliance', 'dataSet', 'decoder']
+      })
+
+      if (!sensor) {
+        throw new Error(`Sensor having given deviceId(${deviceId}) not found`)
+      }
+
+      if (!sensor.active) {
+        throw new Error(`State of the sensor given deviceId(${deviceId}) is not active`)
+      }
+
+      if (!sensor.appliance) {
+        throw new Error(`Appliance of the sensor given deviceId(${deviceId}) is not set up`)
+      }
+
+      const domain = sensor.domain
+      const dataSet = sensor.dataSet
+
+      /* 
+        원칙적으로 user 인증정보를 가지고 요청해야 하지만, 인증정보를 보낼 수 없는 상황이라고 전제하여 처리한다.
+        앞으로, 보완되어야 한다.
+      */
+      const user: User =
+        context.user ||
+        (await tx.getRepository(User).findOne({
+          where: {
+            reference: sensor.appliance.id,
+            userType: 'appliance'
+          }
+        }))
+
+      context.state = {
+        user,
+        ...context.state,
+        domain,
+        tx
+      }
+
+      var decoded
+      if (sensor.decoder) {
+        const { name: scenarioName } = sensor.decoder
+        const variables = {
+          dataSensor: sensor,
+          source: deviceId,
+          data,
+          rawData,
+          timestamp
+        }
+
+        decoded = (await runScenario(null, scenarioName, variables, context))?.result
+      }
+
+      if (dataSet) {
+        await createDataSample(
+          {
+            dataSet,
+            data,
+            rawData,
+            source: deviceId,
+            collectedAt: new Date(timestamp),
+            ...decoded
+          },
+          context
+        )
+      }
+
+      context.status = 200
+      context.body = {
+        data,
+        ...decoded
+      }
+    })
+  })
+
+  /* When a callback occurs from the scheduler when a scheduled dataset is on schedule, data collection task for the dataset should be issued. */
+  globalPublicRouter.post('/callback-schedule-for-dataset', async (context, next) => {
+    const { client } = context.request.body as ScheduleRegisterRequest
+
+    if (!client || typeof client !== 'object') {
+      throw new Error('client property should be a part of callback body.')
+    }
+
+    const { group: domainId, key: dataSetId } = client
+
+    if (!domainId || !dataSetId) {
+      throw new Error(`group(${domainId}) and key(${dataSetId}) properties should not be empty`)
+    }
+
+    await issueDataCollectionTask(domainId, dataSetId, context)
+
+    context.status = 200
+  })
+
+  /* When a callback occurs from the scheduler when a scheduled summary is on schedule, data summary procedure for the dataset should be executed. */
+  globalPublicRouter.post('/callback-schedule-for-data-summary', async (context, next) => {
+    const { client } = context.request.body as ScheduleRegisterRequest
+
+    if (!client || typeof client !== 'object') {
+      throw new Error('client property should be a part of callback body.')
+    }
+
+    const { group: domainId, key: dataSetId } = client
+
+    if (!domainId || !dataSetId) {
+      throw new Error(`group(${domainId}) and key(${dataSetId}) properties should not be empty`)
+    }
+
+    await getDataSource().transaction(async tx => {
+      const domain = await tx.getRepository(Domain).findOneBy({ id: domainId })
+
+      if (!domain) {
+        throw new Error(`domain(${domainId}) not found`)
+      }
+
+      const dataSet = await tx
+        .getRepository(DataSet)
+        .findOne({ where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: dataSetId } })
+
+      // do what you gotta do
+    })
+
+    context.status = 200
+  })
+})
+
+process.on('bootstrap-module-global-private-route' as any, (app, globalPrivateRouter) => {
+  /*
+   * can add global private routes to application (auth required, tenancy not required)
+   */
+})
+
+process.on('bootstrap-module-domain-public-route' as any, (app, domainPublicRouter) => {
+  /*
+   * can add domain public routes to application (auth not required, tenancy required)
+   */
+})
+
+process.on('bootstrap-module-domain-private-route' as any, (app, domainPrivateRouter) => {
+  /*
+   * can add domain private routes to application (auth required, tenancy required)
+   */
+  domainPrivateRouter.get('/data-report/jasper', async (context, next) => {
+    context.body = await renderJasperReport(context)
+  })
+
+  domainPrivateRouter.get('/data-report/shiny', async (context, next) => {
+    await renderShinyReport(context)
+  })
+
+  domainPrivateRouter.get('/data-report/:reportType', async (context, next) => {
+    let { reportType } = context.params
+    context.body = `${reportType} Not Implemented.`
+  })
+})