@things-factory/dataset 5.0.0-alpha.8 → 5.0.0-zeta.0

package/server/controllers/jasper-report.ts

@@ -0,0 +1,170 @@
+import FormData from 'form-data'
+import fetch from 'node-fetch'
+
+import { STORAGE } from '@things-factory/attachment-base'
+import { AthenaController } from '@things-factory/aws-base'
+import { config } from '@things-factory/env'
+
+const dataReportConfig = config.get('dataReport')
+const { jasper: { 
+  endpoint: { protocol: PROTOCOL, host:HOST, port:PORT }, 
+  datasource: { database: DATABASE } } 
+} = dataReportConfig
+
+/** author: ywnam123 */
+function transformValuesToRows(queryResult) {
+  var parseData = []
+    let index = 1
+    for (let i = 0; i < queryResult.Items.length; i++) {
+      var j = 0
+      const data = JSON.parse(queryResult.Items[i].data)
+      const spec = JSON.parse(queryResult.Items[i].spec)
+
+      for (let key in data) {
+        if (Array.isArray(data[key])) {
+          for (j = 0; j < data[key].length; j++) {
+            for (let specKey in spec) {
+              if (key === specKey) {
+                parseData.push({
+                  item: spec[specKey].name,
+                  index: index + j,
+                  value: String(data[key][j])
+                })
+              }
+            }
+          }
+        } else {
+          parseData.push({
+            item: key,
+            index,
+            value: String(data[key])
+          })
+        }
+      }
+      if (j !== 0) {
+        index = index + j
+      } else {
+        index = index + 1
+      }
+    }
+}
+
+/** @todo considering trasformation in lambda, as massive dataset */
+function pivotData(rows) {
+  let parsedData = []
+  let index = 1
+  for (let i = 0; i < rows.length; i++) {
+    let j = 0
+    const data = JSON.parse(rows[i].data)
+    const spec = JSON.parse(rows[i].spec)
+
+    for (let key in data) {
+      /** @todo rule to display or not, about unspecified spec */
+      const value = data[key]
+      !(spec[key]?.hidden) && parsedData.push({
+        item: spec[key]?.name || key,
+        index,
+        value: Array.isArray(value) ? value.join(', ') : value
+      })
+    }
+    if (j !== 0) {
+      index = index + j
+    } else {
+      index = index + 1
+    }
+  }
+
+  return parsedData
+}
+
+function parseJsonDataField(rows) {
+  let parsedData = []
+  for (let i = 0; i < rows.length; i++) {
+    const row = rows[i]
+    const data = JSON.parse(row.data)
+    for (let key in data) {
+      if (Array.isArray(data[key])) {
+        data[key] = data[key].toString()
+      }
+    }
+    delete row.data
+    parsedData.push({ ...row, ...data })
+  }
+
+  return parsedData
+}
+
+const athenaClient = new AthenaController()
+async function queryAthena(params) {
+  const { table, domain, dataSetId, fromWorkDate, toWorkDate, workShift, timezone } = params
+  const queryData = {
+    sql: `SELECT name, description, data, spec, workdate, workshift, 
+    DATE_FORMAT(
+      FROM_UNIXTIME(collected_at / 1000 / 1000) AT TIME ZONE '${timezone || 'UTC'}', 
+      '%Y-%m-%d %H:%i:%s'
+    ) AS collected_at
+    FROM ${table} 
+    WHERE domain='${domain}' 
+    AND datasetid = '${dataSetId}'
+    AND workdate >= '${fromWorkDate}'
+    AND workdate <= '${toWorkDate}'
+    ${workShift ? 'AND workshift = \'' + workShift + '\'' : ''}
+    ORDER BY collected_at`,
+    db: DATABASE
+  }
+  // and json_extract_scalar(data, '$.dauid') = 'A8032AD81730'
+
+  return await athenaClient.query(queryData)
+}
+export async function renderJasperReport(context: any) { 
+  const { state: { domain }, query } = context
+
+  const template = await STORAGE.readFile(query['template'] || 'dynamic_header_sample.jrxml', 'utf-8')
+  let templateType = query['templateType'] || 'crosstab'
+  let parsedData = []
+
+  // @todo: get dataset timezone
+  /**  
+   * const variables = await gql(dataSet(id:${dataSetId}) {
+   *  name, description, partition_keys, timezone
+   * })
+  */
+
+  query['domain'] = domain?.subdomain
+  query['timezone'] = domain?.timezone
+  const queryResult = await queryAthena(query)
+  const rows = queryResult.Items
+
+  if (!rows.length) {
+    return '<h3>Not found result.</h3>'
+  }
+  else {
+    const firstRow = rows[0]
+    // uses the first row values as data-set has no history data.
+    const parameters = {
+      name: firstRow.name,
+      description: firstRow.description,
+      ...query
+    }
+  
+    if (templateType === 'crosstab') {
+      parsedData = pivotData(rows)
+    } else {
+      parsedData = parseJsonDataField(rows)
+    }
+  
+    const formData = new FormData()
+    formData.append('template', template)
+    formData.append('jsonString', JSON.stringify(parsedData))
+    formData.append('parameters', JSON.stringify(parameters))
+  
+    const reportUrl = `${PROTOCOL || 'http'}://${HOST}:${PORT}/rest/report/show_html`
+    const response = await fetch(reportUrl, {
+      method: 'POST',
+      body: formData
+    })
+
+    return await response.text()
+  }
+
+}

package/server/index.ts

@@ -1,5 +1,6 @@
 import './routes'
 
+export * from './controllers'
 export * from './migrations'
 export * from './middlewares'
 export * from './service'

package/server/routes.ts

@@ -1,7 +1,9 @@
 import { getConnection } from 'typeorm'
 
-import { DataSample } from './service'
-import { DataItem } from './service/data-item/data-item'
+import { User } from '@things-factory/auth-base'
+
+import { createDataSample } from './controllers/create-data-sample'
+import { renderJasperReport } from './controllers/jasper-report'
 import { DataSensor } from './service/data-sensor/data-sensor'
 
 const debug = require('debug')('things-factory:dataset:routes')
@@ -47,41 +49,26 @@ process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRout
 
       const domain = sensor.domain
       const dataSet = sensor.dataSet
-      const user = sensor.appliance
-
-      const dataItems = await tx.getRepository(DataItem).find({
+      const user: User = await tx.getRepository(User).findOne({
         where: {
-          domain,
-          dataSet
-        },
-        order: {
-          sequence: 'DESC'
+          reference: sensor.appliance.id,
+          userType: 'appliance'
         }
       })
 
-      var spec = {} as any
-
-      dataItems.forEach(dataItem => {
-        spec[dataItem.name] = dataItem.spec
-      })
-
-      await tx.getRepository(DataSample).save({
-        domain,
-        name: dataSet.name,
-        description: dataSet.description,
-        partitionKeys: dataSet.partitionKeys,
-        dataSet,
-        data,
-        rawData,
-        spec,
-        source: deviceId,
-        collectedAt: new Date(timestamp),
-        creator: user,
-        updater: user
-      })
+      return await createDataSample(
+        {
+          dataSet,
+          data,
+          rawData,
+          source: deviceId,
+          collectedAt: new Date(timestamp)
+        },
+        { state: { domain, user, tx } }
+      )
     })
 
-    return 'OK'
+    context.status = 200
   })
 })
 
@@ -101,4 +88,7 @@ process.on('bootstrap-module-domain-private-route' as any, (app, domainPrivateRo
   /*
    * can add domain private routes to application (auth required, tenancy required)
    */
+  domainPrivateRouter.get('/data-report/jasper', async (context, next) => {
+    context.body = await renderJasperReport(context)
+  })
 })

package/server/service/data-item/data-item-mutation.ts

@@ -24,9 +24,14 @@ export class DataItemMutation {
     await dataItemRepo.delete({ domain, dataSet: dataSetId })
 
     for (let i = 0; i < patches.length; i++) {
+      const patch = patches[i]
+      if (!patch.options) {
+        patch.options = {}
+      }
+
       const result = await dataItemRepo.save({
         quota: 1,
-        ...patches[i],
+        ...patch,
         sequence: i,
         dataSet,
         domain,

package/server/service/data-item/data-item-query.ts

@@ -2,7 +2,7 @@ import { Arg, Args, Ctx, Directive, FieldResolver, Query, Resolver, Root } from
 import { getRepository } from 'typeorm'
 
 import { User } from '@things-factory/auth-base'
-import { convertListParams, Domain, ListParam } from '@things-factory/shell'
+import { Domain, getQueryBuilderFromListParams, ListParam } from '@things-factory/shell'
 
 import { DataSet } from '../data-set/data-set'
 import { DataItem } from './data-item'
@@ -25,8 +25,13 @@ export class DataItemQuery {
   async dataItems(@Args() params: ListParam, @Ctx() context: any): Promise<DataItemList> {
     const { domain } = context.state
 
-    const convertedParams = convertListParams(params, domain.id)
-    const [items, total] = await getRepository(DataItem).findAndCount(convertedParams)
+    const queryBuilder = getQueryBuilderFromListParams({
+      repository: getRepository(DataItem),
+      params,
+      domain
+    })
+
+    const [items, total] = await queryBuilder.getManyAndCount()
 
     return { items, total }
   }

package/server/service/data-item/data-item-type.ts

@@ -1,16 +1,14 @@
-import { Field, ID, InputType, Int, ObjectType } from 'type-graphql'
-
-import { ScalarObject } from '@things-factory/shell'
-
 import { DataItem, DataItemType } from './data-item'
+import { Field, ID, InputType, Int, ObjectType } from 'type-graphql'
+import { ObjectRef, ScalarObject } from '@things-factory/shell'
 
 @InputType()
 export class DataItemPatch {
   @Field(type => ID, { nullable: true })
   id?: string
 
-  @Field({ nullable: true })
-  dataSetId?: string
+  @Field(type => ObjectRef, { nullable: true })
+  dataSet?: ObjectRef
 
   @Field({ nullable: true })
   name?: string
@@ -30,12 +28,18 @@ export class DataItemPatch {
   @Field(type => ScalarObject, { nullable: true })
   options?: ScalarObject
 
+  @Field({ nullable: true })
+  unit?: string
+
   @Field(type => Int, { nullable: true })
   quota?: number
 
   @Field({ nullable: true })
   active?: boolean
 
+  @Field({ nullable: true })
+  hidden?: boolean
+
   @Field(type => ScalarObject, { nullable: true })
   spec?: ScalarObject
 

package/server/service/data-item/data-item.ts

@@ -1,4 +1,3 @@
-import { Field, ID, Int, ObjectType, registerEnumType } from 'type-graphql'
 import {
   Column,
   CreateDateColumn,
@@ -9,11 +8,11 @@ import {
   RelationId,
   UpdateDateColumn
 } from 'typeorm'
-
-import { User } from '@things-factory/auth-base'
 import { Domain, ScalarObject } from '@things-factory/shell'
+import { Field, ID, Int, ObjectType, registerEnumType } from 'type-graphql'
 
 import { DataSet } from '../data-set/data-set'
+import { User } from '@things-factory/auth-base'
 
 export enum DataItemType {
   number = 'number',
@@ -54,7 +53,7 @@ export class DataItem {
   description?: string
 
   @ManyToOne(type => DataSet, dataSet => dataSet.dataItems, { onDelete: 'CASCADE' })
-  @Field(type => DataSet, { nullable: true })
+  @Field(type => DataSet, { nullable: false })
   dataSet?: DataSet
 
   @RelationId((dataItem: DataItem) => dataItem.dataSet)
@@ -76,6 +75,12 @@ export class DataItem {
   @Field({ nullable: true })
   active?: boolean
 
+  @Column({
+    nullable: true
+  })
+  @Field({ nullable: true })
+  hidden?: boolean
+
   @Column({
     nullable: true
   })
@@ -86,6 +91,12 @@ export class DataItem {
   @Field(type => ScalarObject, { nullable: true })
   options?: ScalarObject
 
+  @Column({
+    nullable: true
+  })
+  @Field({ nullable: true })
+  unit?: string
+
   @Column({
     nullable: true
   })

package/server/service/data-ooc/data-ooc-mutation.ts

@@ -0,0 +1,150 @@
+import { Arg, Ctx, Directive, Mutation, Resolver } from 'type-graphql'
+
+import { DataOoc, DataOocStatus } from './data-ooc'
+import { DataOocPatch, NewDataOoc } from './data-ooc-type'
+
+@Resolver(DataOoc)
+export class DataOocMutation {
+  @Directive('@privilege(category: "data-ooc", privilege: "mutation", domainOwnerGranted: true)')
+  @Directive('@transaction')
+  @Mutation(returns => DataOoc, { description: 'To create new DataOoc' })
+  async createDataOoc(@Arg('dataOoc') dataOoc: NewDataOoc, @Ctx() context: any): Promise<DataOoc> {
+    const { domain, user, tx } = context.state
+
+    const state = dataOoc.state || DataOocStatus.CREATED
+    const history = [
+      {
+        user: {
+          id: user.id,
+          name: user.name
+        },
+        state,
+        timestamp: Date.now()
+      }
+    ]
+
+    return await tx.getRepository(DataOoc).save({
+      ...dataOoc,
+      state,
+      domain,
+      history,
+      creator: user,
+      updater: user
+    })
+  }
+
+  @Directive('@privilege(category: "data-ooc", privilege: "mutation", domainOwnerGranted: true)')
+  @Directive('@transaction')
+  @Mutation(returns => DataOoc, { description: 'To modify DataOoc information' })
+  async updateDataOoc(@Arg('id') id: string, @Arg('patch') patch: DataOocPatch, @Ctx() context: any): Promise<DataOoc> {
+    const { domain, user, tx } = context.state
+
+    const repository = tx.getRepository(DataOoc)
+    const dataOoc = await repository.findOne({
+      where: { domain, id }
+    })
+
+    const state = patch.state || dataOoc.state
+
+    const history = dataOoc.history || []
+    history.push({
+      user: {
+        id: user.id,
+        name: user.name
+      },
+      state,
+      comment: patch.correctiveAction || '',
+      timestamp: Date.now()
+    })
+
+    const more = {} as any
+
+    if (dataOoc.state !== patch.state) {
+      if (patch.state === DataOocStatus.CORRECTED) {
+        more.corrector = user
+      } else {
+        more.corrector = null
+      }
+    }
+
+    return await repository.save({
+      ...dataOoc,
+      ...patch,
+      ...more,
+      state,
+      history,
+      updater: user
+    })
+  }
+
+  // @Directive('@privilege(category: "data-ooc", privilege: "mutation", domainOwnerGranted: true)')
+  // @Directive('@transaction')
+  // @Mutation(returns => [DataOoc], { description: "To modify multiple DataOoc' information" })
+  // async updateMultipleDataOoc(
+  //   @Arg('patches', type => [DataOocPatch]) patches: DataOocPatch[],
+  //   @Ctx() context: any
+  // ): Promise<DataOoc[]> {
+  //   const { domain, user, tx } = context.state
+
+  //   let results = []
+  //   const _createRecords = patches.filter((patch: any) => patch.cuFlag.toUpperCase() === '+')
+  //   const _updateRecords = patches.filter((patch: any) => patch.cuFlag.toUpperCase() === 'M')
+  //   const dataOocRepo = tx.getRepository(DataOoc)
+
+  //   if (_createRecords.length > 0) {
+  //     for (let i = 0; i < _createRecords.length; i++) {
+  //       const newRecord = _createRecords[i]
+
+  //       const result = await dataOocRepo.save({
+  //         ...newRecord,
+  //         domain,
+  //         creator: user,
+  //         updater: user
+  //       })
+
+  //       results.push({ ...result, cuFlag: '+' })
+  //     }
+  //   }
+
+  //   if (_updateRecords.length > 0) {
+  //     for (let i = 0; i < _updateRecords.length; i++) {
+  //       const newRecord = _updateRecords[i]
+  //       const dataOoc = await dataOocRepo.findOne(newRecord.id)
+
+  //       const result = await dataOocRepo.save({
+  //         ...dataOoc,
+  //         ...newRecord,
+  //         updater: user
+  //       })
+
+  //       results.push({ ...result, cuFlag: 'M' })
+  //     }
+  //   }
+
+  //   return results
+  // }
+
+  // @Directive('@privilege(category: "data-ooc", privilege: "mutation", domainOwnerGranted: true)')
+  // @Directive('@transaction')
+  // @Mutation(returns => Boolean, { description: 'To delete DataOoc' })
+  // async deleteDataOoc(@Arg('id') id: string, @Ctx() context: any): Promise<boolean> {
+  //   const { domain, tx } = context.state
+
+  //   await tx.getRepository(DataOoc).delete({ domain, id })
+  //   return true
+  // }
+
+  // @Directive('@privilege(category: "data-ooc", privilege: "mutation", domainOwnerGranted: true)')
+  // @Directive('@transaction')
+  // @Mutation(returns => Boolean, { description: 'To delete multiple dataOocs' })
+  // async deleteDataOocs(@Arg('ids', type => [String]) ids: string[], @Ctx() context: any): Promise<boolean> {
+  //   const { domain, tx } = context.state
+
+  //   await tx.getRepository(DataOoc).delete({
+  //     domain,
+  //     id: In(ids)
+  //   })
+
+  //   return true
+  // }
+}

package/server/service/data-ooc/data-ooc-query.ts

@@ -0,0 +1,69 @@
+import { Arg, Args, Ctx, Directive, FieldResolver, Query, Resolver, Root } from 'type-graphql'
+import { getRepository } from 'typeorm'
+
+import { User } from '@things-factory/auth-base'
+import { Domain, getQueryBuilderFromListParams, ListParam } from '@things-factory/shell'
+
+import { DataSample } from '../data-sample/data-sample'
+import { DataSet } from '../data-set/data-set'
+import { DataOoc } from './data-ooc'
+import { DataOocList } from './data-ooc-type'
+
+@Resolver(DataOoc)
+export class DataOocQuery {
+  @Directive('@privilege(category: "data-ooc", privilege: "query", domainOwnerGranted: true)')
+  @Query(returns => DataOoc, { description: 'To fetch a DataOoc' })
+  async dataOoc(@Arg('id') id: string, @Ctx() context: any): Promise<DataOoc> {
+    const { domain } = context.state
+
+    return await getRepository(DataOoc).findOne({
+      where: { domain, id }
+    })
+  }
+
+  @Directive('@privilege(category: "data-ooc", privilege: "query", domainOwnerGranted: true)')
+  @Query(returns => DataOocList, { description: 'To fetch multiple DataOoc' })
+  async dataOocs(@Args() params: ListParam, @Ctx() context: any): Promise<DataOocList> {
+    const { domain } = context.state
+
+    const queryBuilder = getQueryBuilderFromListParams({
+      repository: getRepository(DataOoc),
+      params,
+      domain,
+      searchables: ['name', 'description']
+    })
+
+    const [items, total] = await queryBuilder.getManyAndCount()
+
+    return { items, total }
+  }
+
+  @FieldResolver(type => DataSet)
+  async dataSet(@Root() dataOoc: DataOoc): Promise<DataSet> {
+    return await getRepository(DataSet).findOne({
+      id: dataOoc.dataSetId
+    })
+  }
+
+  @FieldResolver(type => DataSample)
+  async dataSample(@Root() dataOoc: DataOoc): Promise<DataSample> {
+    return await getRepository(DataSample).findOne({
+      id: dataOoc.dataSampleId
+    })
+  }
+
+  @FieldResolver(type => Domain)
+  async domain(@Root() dataOoc: DataOoc): Promise<Domain> {
+    return await getRepository(Domain).findOne(dataOoc.domainId)
+  }
+
+  @FieldResolver(type => User)
+  async updater(@Root() dataOoc: DataOoc): Promise<User> {
+    return await getRepository(User).findOne(dataOoc.updaterId)
+  }
+
+  @FieldResolver(type => User)
+  async creator(@Root() dataOoc: DataOoc): Promise<User> {
+    return await getRepository(User).findOne(dataOoc.creatorId)
+  }
+}

package/server/service/data-ooc/data-ooc-subscription.ts

@@ -0,0 +1,51 @@
+import { Resolver, Subscription, Root } from 'type-graphql'
+import { withFilter } from 'graphql-subscriptions'
+import { pubsub } from '@things-factory/shell'
+import { User } from '@things-factory/auth-base'
+import { DataOoc } from './data-ooc'
+import { getRepository } from 'typeorm'
+
+const debug = require('debug')('things-factory:dataset:data-ooc-subscription')
+
+@Resolver(DataOoc)
+export class DataOocSubscription {
+  @Subscription({
+    subscribe: (_, args, context, info) => {
+      const { domain, user } = context.state
+      const subdomain = domain?.subdomain
+
+      debug('subscribe', subdomain)
+
+      if (!domain) {
+        throw new Error('domain required')
+      }
+
+      if (!user.domains?.find(d => d.subdomain === subdomain)) {
+        throw new Error(`domain(${subdomain}) is not working for user(${user.email}).`)
+      }
+
+      return withFilter(
+        () => pubsub.asyncIterator('data-ooc'),
+        async (payload, variables, context, info) => {
+          const { dataOoc, supervisoryRoleId } = payload
+          const { domain } = dataOoc
+
+          if (subdomain !== domain?.subdomain) {
+            return false
+          }
+
+          // check if the user has that role
+          const userWithRoles: User = await getRepository(User).findOne({
+            where: { id: user.id },
+            relations: ['roles']
+          })
+
+          return userWithRoles.roles.find(role => role.id === supervisoryRoleId)
+        }
+      )(_, args, context, info)
+    }
+  })
+  dataOoc(@Root() payload: { dataOoc: DataOoc; supervisoryRoleId: string }): DataOoc {
+    return payload.dataOoc
+  }
+}