@things-factory/dataset 7.0.0-alpha.6 → 7.0.0-alpha.8

package/server/controllers/create-data-sample.ts

@@ -55,16 +55,16 @@ const formatDate = (keys, _moment) => {
   return keys
 }
 
-export async function createDataSample(dataSample: NewDataSample, context: ResolverContext): Promise<DataSample> {
+export async function createDataSample(newDataSample: NewDataSample, context: ResolverContext): Promise<DataSample> {
   const { domain, user, tx } = context.state
 
   const dataSet = await tx.getRepository(DataSet).findOne({
-    where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: dataSample.dataSet.id },
+    where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: newDataSample.dataSet.id },
     relations: ['dataKeySet']
   })
 
   const { dataItems, tag: publishTag, normalScenarioId, outlierScenarioId } = dataSet
-  const collectedAt = dataSample.collectedAt || new Date()
+  const collectedAt = newDataSample.collectedAt || new Date()
 
   const timezone = dataSet.timezone || domain.timezone || 'UTC'
   const format = 'YYYY-MM-DD'
@@ -77,7 +77,7 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
   const localDateTz = moment(collectedAt).tz(timezone)
   const defaultPartitionKeys = {
     domain: domain.subdomain,
-    datasetid: dataSample.dataSet.id /* It should not be 'data_set_id' as column name duplicated for Glue */,
+    datasetid: newDataSample.dataSet.id /* It should not be 'data_set_id' as column name duplicated for Glue */,
     date: localDateTz.format(format) /* local time date */,
     workdate: workDate /* working date */,
     workshift: workShift
@@ -89,11 +89,11 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
   }
 
   partitionKeys = formatDate(partitionKeys, localDateTz)
-  partitionKeys = replaceVariables(partitionKeys, dataSample.data)
+  partitionKeys = replaceVariables(partitionKeys, newDataSample.data)
 
-  const dataKeys = fillDataKeys(dataSet?.dataKeySet, dataSample.data)
+  const dataKeys = fillDataKeys(dataSet?.dataKeySet, newDataSample.data)
 
-  const { ooc, oos, judgment } = DataUseCase.evaluate(dataSet, dataItems, dataSample.data) || {}
+  const { ooc, oos, judgment } = DataUseCase.evaluate(dataSet, dataItems, newDataSample.data) || {}
 
   const old = await tx.getRepository(DataSample).findOne({
     where: {
@@ -110,7 +110,7 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
     If [FileUpload[]] type needed, add 'files' type for dataset
   */
 
-  const data = dataSample.data
+  const data = newDataSample.data
   const attachments = []
 
   for (let dataItem of dataItems) {
@@ -164,13 +164,13 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
     }
   }
 
-  const result = await tx.getRepository(DataSample).save({
+  const dataSample = await tx.getRepository(DataSample).save({
     ...old,
     name: dataSet.name,
     description: dataSet.description,
     useCase: dataSet.useCase,
     type: dataSet.type,
-    ...dataSample,
+    ...newDataSample,
     ...dataKeys,
     dataSetVersion: dataSet.version,
     domain,
@@ -197,13 +197,13 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
 
   /* post-process for for file attachment. */
   if (attachments.length > 0) {
-    attachments.forEach(attachment => (attachment.refId = result.id))
+    attachments.forEach(attachment => (attachment.refId = dataSample.id))
     tx.getRepository(Attachment).save(attachments)
   }
 
   if (ooc || oos) {
     const dataOoc = await tx.getRepository(DataOoc).save({
-      ...result,
+      ...dataSample,
       history: [
         {
           user: {
@@ -214,7 +214,8 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
           timestamp: collectedAt
         }
       ],
-      state: DataOocStatus.ISSUED
+      state: DataOocStatus.ISSUED,
+      dataSample: dataSample
     })
 
     if (outlierScenarioId) {
@@ -227,9 +228,9 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
       })
 
       if (scenario) {
-        scenario.start(scenario.name + ':' + dataSet.name + ':' + result.id, {
+        scenario.start(scenario.name + ':' + dataSet.name + ':' + dataSample.id, {
           dataOocId: dataOoc.id,
-          dataSampleId: result.id,
+          dataSampleId: dataSample.id,
           dataSet: dataSet.id,
           data,
           ooc,
@@ -260,9 +261,7 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
     })) as Activity
 
     if (activity) {
-      const assignee =
-        dataSet.supervisoryRoleId &&
-        (await tx.getRepository(Role).findOneBy({ domain: { id: domain.id }, id: dataSet.supervisoryRoleId }))
+      const assignee = dataSet.supervisoryRoleId && (await tx.getRepository(Role).findOneBy({ domain: { id: domain.id }, id: dataSet.supervisoryRoleId }))
 
       const assignees = dataSet.supervisoryRoleId ? [{ type: 'Role', value: dataSet.supervisoryRoleId, assignee }] : []
 
@@ -280,11 +279,10 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
           approvalLine: []
         }
 
-        await issue(activityInstance, context)
+        dataOoc.reviewActivityInstance = await issue(activityInstance, context)
+        await tx.getRepository(DataOoc).save(dataOoc)
       } else {
-        console.error(
-          `Assignees are not set. So Data OOC Review task for ${dataOoc.name}(${dataOoc.id}) could not be issued.`
-        )
+        console.error(`Assignees are not set. So Data OOC Review task for ${dataOoc.name}(${dataOoc.id}) could not be issued.`)
       }
     } else {
       console.error('OOC Review Activity not installed.')
@@ -320,8 +318,8 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
       })
 
       if (scenario) {
-        scenario.start(scenario.name + ':' + dataSet.name + ':' + result.id, {
-          dataSampleId: result.id,
+        scenario.start(scenario.name + ':' + dataSet.name + ':' + dataSample.id, {
+          dataSampleId: dataSample.id,
           dataSet: dataSet.id,
           data,
           ooc,
@@ -353,30 +351,27 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
       })) as Activity
 
       if (activity) {
-        const assignee =
-          dataSet.supervisoryRoleId &&
-          (await tx.getRepository(Role).findOneBy({ domain: { id: domain.id }, id: dataSet.supervisoryRoleId }))
+        const assignee = dataSet.supervisoryRoleId && (await tx.getRepository(Role).findOneBy({ domain: { id: domain.id }, id: dataSet.supervisoryRoleId }))
 
-        const assignees = dataSet.supervisoryRoleId
-          ? [{ type: 'Role', value: dataSet.supervisoryRoleId, assignee }]
-          : []
+        const assignees = dataSet.supervisoryRoleId ? [{ type: 'Role', value: dataSet.supervisoryRoleId, assignee }] : []
 
         /* 해당 dataset의 supervisor로 하여금, data를 리뷰하고 instruction을 작성해서, approvalLine을 이용해서 승인을 한다. */
         if (assignees && assignees instanceof Array && assignees.length > 0) {
-          result.dataItems = dataItems
+          dataSample.dataItems = dataItems
           const activityInstance = {
             name: `[Data 검토] ${dataSet.name}`,
             description: dataSet.description,
             activityId: activity.id,
             dueAt: new Date(collectedAt.getTime() + 24 * 60 * 60 * 1000),
             input: {
-              dataSample: result
+              dataSampleId: dataSample.id
             },
             assignees,
             approvalLine: dataSet.reviewApprovalLine
           }
 
-          await issue(activityInstance, context)
+          dataSample.reviewActivityInstance = await issue(activityInstance, context)
+          await tx.getRepository(DataSample).save(dataSample)
 
           try {
             pubsub.publish('notification', {
@@ -385,7 +380,7 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
                 type: 'info',
                 title: `[Data Review] ${dataSet.name}`,
                 body: `Data Review occurred on '${dataSet.name}'`,
-                url: getRedirectSubdomainPath(context, domain.subdomain, `/data-sample/${result.id}`),
+                url: getRedirectSubdomainPath(context, domain.subdomain, `/data-sample/${dataSample.id}`),
                 timestamp: collectedAt
               }
             })
@@ -393,9 +388,7 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
             logger.error('Notification', err)
           }
         } else {
-          console.error(
-            `Assignees are not set. So Data Review task for ${dataSet.name}(${dataSet.id}) could not be issued.`
-          )
+          console.error(`Assignees are not set. So Data Review task for ${dataSet.name}(${dataSet.id}) could not be issued.`)
         }
       } else {
         console.error('Data Review Activity not installed.')
@@ -403,5 +396,5 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
     }
   }
 
-  return result
+  return dataSample
 }

package/server/controllers/issue-collect-data.ts

@@ -0,0 +1,61 @@
+import { In } from 'typeorm'
+
+import { Domain, getDataSource } from '@things-factory/shell'
+import { Activity, ActivityInstance, issue } from '@things-factory/worklist'
+
+import { DataSet } from '../service/data-set/data-set'
+
+export async function issueCollectData(domainId: string, dataSetId: string, context: ResolverContext): Promise<ActivityInstance> {
+  await getDataSource().transaction(async tx => {
+    const domain = await tx.getRepository(Domain).findOneBy({ id: domainId })
+
+    if (!domain) {
+      throw new Error(`domain(${domainId}) not found`)
+    }
+
+    const dataSet = await tx.getRepository(DataSet).findOne({
+      where: {
+        domain: {
+          id: In([domain.id, domain.parentId].filter(Boolean))
+        },
+        id: dataSetId
+      }
+    })
+
+    const activity = (await tx.getRepository(Activity).findOneBy({
+      domain: { id: domain.id },
+      name: 'Collect Data'
+    })) as Activity
+
+    if (activity) {
+      const { assignees } = dataSet
+
+      /* 해당 dataset에 대한 데이타 수집 태스크를 dataset assignees에게 할당한다. */
+      if (assignees && assignees instanceof Array && assignees.length > 0) {
+        const activityInstance = {
+          name: `[Data 수집] ${dataSet.name}`,
+          description: dataSet.description,
+          activityId: activity.id,
+          dueAt: new Date(Date.now() + 24 * 60 * 60 * 1000),
+          input: {
+            dataSetId: dataSet.id,
+            dataSetName: dataSet.name
+          },
+          assignees
+        }
+
+        context.state = {
+          ...context.state,
+          domain,
+          tx
+        }
+
+        return await issue(activityInstance, context)
+      } else {
+        throw new Error(`Assignees not set. So Data Collect Activity for ${dataSet.name}($dataSet.id) could not be issued.`)
+      }
+    } else {
+      throw new Error(`Data Collect Activity is not installed.`)
+    }
+  })
+}

package/server/engine/index.ts

@@ -0,0 +1 @@
+import './task'

package/server/engine/task/create-data-sample.ts

@@ -0,0 +1,92 @@
+import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
+import { getDataSource } from '@things-factory/shell'
+import { access } from '@things-factory/utils'
+import i18next from 'i18next'
+
+import { DataSet } from '../../service/data-set/data-set'
+
+import { createDataSample } from '../../controllers/create-data-sample'
+
+async function CreateDataSample(step: InputStep, context: Context) {
+  const { logger, data, domain, user, lng } = context
+  var {
+    params: { dataset: dataSetId, source: sourceAccessor, rawData: rawDataAccessor, data: dataAccessor, timestamp }
+  } = step
+
+  if (!dataSetId) {
+    throw new Error(`no dataset found`)
+  }
+
+  // make new data-sample
+  await getDataSource().transaction(async tx => {
+    const dataSet = await tx.getRepository(DataSet).findOne({
+      where: {
+        id: dataSetId
+      }
+    })
+
+    var source = await access(sourceAccessor, data)
+    var extractedData = await access(dataAccessor, data)
+    var extractedRawData = rawDataAccessor && (await access(rawDataAccessor, data))
+
+    const dataSample = await createDataSample(
+      {
+        dataSet,
+        data: extractedData,
+        rawData: extractedRawData,
+        source,
+        collectedAt: new Date(timestamp || new Date())
+      },
+      {
+        t: i18next.t,
+        state: {
+          domain,
+          user,
+          lng,
+          tx
+        }
+      }
+    )
+
+    return {
+      data: dataSample
+    }
+  })
+}
+
+CreateDataSample.parameterSpec = [
+  {
+    type: 'entity-selector',
+    name: 'dataset',
+    label: 'data-set',
+    property: {
+      queryName: 'dataSets',
+      valueKey: 'id'
+    }
+  },
+  {
+    type: 'string',
+    name: 'source',
+    label: 'source'
+  },
+  {
+    type: 'string',
+    name: 'rawData',
+    label: 'raw-data'
+  },
+  {
+    type: 'string',
+    name: 'data',
+    label: 'data'
+  },
+  {
+    type: 'string',
+    name: 'timestamp',
+    label: 'timestamp'
+  }
+]
+
+CreateDataSample.connectorFree = true
+CreateDataSample.help = 'dataset/task/create-data-sample'
+
+TaskRegistry.registerTaskHandler('create-data-sample', CreateDataSample)

package/server/engine/task/index.ts

@@ -0,0 +1,2 @@
+import './create-data-sample'
+import './issue-collect-data'

package/server/engine/task/issue-collect-data.ts

@@ -0,0 +1,45 @@
+import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
+import { issueCollectData } from '../../controllers/issue-collect-data'
+import i18next from 'i18next'
+
+async function IssueCollectData(step: InputStep, context: Context) {
+  const { logger, data, domain, user, lng } = context
+  var {
+    params: { dataset: dataSetId }
+  } = step
+
+  if (!dataSetId) {
+    throw new Error(`no dataset found`)
+  }
+
+  const activityInstance = await issueCollectData(domain.id, dataSetId, {
+    t: i18next.t,
+    state: {
+      domain,
+      user,
+      lng,
+      tx: undefined
+    }
+  })
+
+  return {
+    data: activityInstance
+  }
+}
+
+IssueCollectData.parameterSpec = [
+  {
+    type: 'entity-selector',
+    name: 'dataset',
+    label: 'data-set',
+    property: {
+      queryName: 'dataSets',
+      valueKey: 'id'
+    }
+  }
+]
+
+IssueCollectData.connectorFree = true
+IssueCollectData.help = 'dataset/task/issue-collect-data'
+
+TaskRegistry.registerTaskHandler('issue-collect-data', IssueCollectData)

package/server/index.ts

@@ -1,5 +1,6 @@
 import './routes'
 import './activities'
+import './engine'
 
 export * from './controllers'
 export * from './service'

package/server/routes.ts

@@ -2,13 +2,13 @@ import { In } from 'typeorm'
 
 import { Domain, getDataSource } from '@things-factory/shell'
 import { User } from '@things-factory/auth-base'
-import { Activity } from '@things-factory/worklist'
-import { issue } from '@things-factory/worklist/dist-server/controllers/activity-instance/issue'
+import { runScenario } from '@things-factory/integration-base'
 import { ScheduleRegisterRequest } from '@things-factory/scheduler-client'
 
 import { createDataSample } from './controllers/create-data-sample'
 import { renderJasperReport } from './controllers/jasper-report'
 import { renderShinyReport } from './controllers/shiny-report'
+import { issueCollectData } from './controllers/issue-collect-data'
 import { DataSensor } from './service/data-sensor/data-sensor'
 import { DataSet } from './service/data-set/data-set'
 
@@ -23,8 +23,8 @@ process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRout
   globalPublicRouter.post('/sensor-data', async (context, next) => {
     // 데이타 검증
     const { deviceId, data, rawData, timestamp = Date.now() } = context.request.body
-    if (!deviceId || !data) {
-      throw new Error(`deviceId(${deviceId}) and data(${JSON.stringify(data)}) properties are mandatory`)
+    if (!deviceId) {
+      throw new Error(`The deviceId(${deviceId}) property are mandatory`)
     }
 
     // make new data-sample
@@ -32,7 +32,7 @@ process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRout
       // find sensor through deviceId
       const sensor = await tx.getRepository(DataSensor).findOne({
         where: { deviceId },
-        relations: ['domain', 'appliance', 'dataSet']
+        relations: ['domain', 'appliance', 'dataSet', 'decoder']
       })
 
       if (!sensor) {
@@ -47,18 +47,18 @@ process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRout
         throw new Error(`Appliance of the sensor given deviceId(${deviceId}) is not set up`)
       }
 
-      if (!sensor.dataSet) {
-        throw new Error(`DataSet of the sensor given deviceId(${deviceId}) is not set up`)
-      }
+      // if (!sensor.dataSet) {
+      //   throw new Error(`DataSet of the sensor given deviceId(${deviceId}) is not set up`)
+      // }
 
       const domain = sensor.domain
       const dataSet = sensor.dataSet
-      const user: User = await tx.getRepository(User).findOne({
-        where: {
-          reference: sensor.appliance.id,
-          userType: 'appliance'
-        }
-      })
+      // const user: User = await tx.getRepository(User).findOne({
+      //   where: {
+      //     reference: sensor.appliance.id,
+      //     userType: 'appliance'
+      //   }
+      // })
 
       context.state = {
         ...context.state,
@@ -66,19 +66,40 @@ process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRout
         tx
       }
 
-      return await createDataSample(
-        {
-          dataSet,
+      var decoded
+      if (sensor.decoder) {
+        const { name: scenarioName } = sensor.decoder
+        const variables = {
+          dataSensor: sensor,
+          source: deviceId,
           data,
           rawData,
-          source: deviceId,
-          collectedAt: new Date(timestamp)
-        },
-        context
-      )
-    })
+          timestamp
+        }
 
-    context.status = 200
+        decoded = (await runScenario(null, scenarioName, variables, context))?.result
+      }
+
+      if (dataSet) {
+        await createDataSample(
+          {
+            dataSet,
+            data,
+            rawData,
+            source: deviceId,
+            collectedAt: new Date(timestamp),
+            ...decoded
+          },
+          context
+        )
+      }
+
+      context.status = 200
+      context.body = {
+        data,
+        ...decoded
+      }
+    })
   })
 
   /* When a callback occurs from the scheduler when a scheduled dataset is on schedule, data collection task for the dataset should be issued. */
@@ -95,55 +116,7 @@ process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRout
       throw new Error(`group(${domainId}) and key(${dataSetId}) properties should not be empty`)
     }
 
-    await getDataSource().transaction(async tx => {
-      const domain = await tx.getRepository(Domain).findOneBy({ id: domainId })
-
-      if (!domain) {
-        throw new Error(`domain(${domainId}) not found`)
-      }
-
-      const dataSet = await tx
-        .getRepository(DataSet)
-        .findOne({ where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: dataSetId } })
-
-      const activity = (await tx.getRepository(Activity).findOneBy({
-        domain: { id: domainId },
-        name: 'Collect Data'
-      })) as Activity
-
-      if (activity) {
-        const { assignees } = dataSet
-
-        /* 해당 dataset에 대한 데이타 수집 태스크를 dataset assignees에게 할당한다. */
-        if (assignees && assignees instanceof Array && assignees.length > 0) {
-          const activityInstance = {
-            name: `[Data 수집] ${dataSet.name}`,
-            description: dataSet.description,
-            activityId: activity.id,
-            dueAt: new Date(Date.now() + 24 * 60 * 60 * 1000),
-            input: {
-              dataSetId: dataSet.id,
-              dataSetName: dataSet.name
-            },
-            assignees
-          }
-
-          context.state = {
-            ...context.state,
-            domain,
-            tx
-          }
-
-          await issue(activityInstance, context)
-        } else {
-          throw new Error(
-            `Assignees not set. So Data Collect Activity for ${dataSet.name}($dataSet.id) could not be issued.`
-          )
-        }
-      } else {
-        throw new Error(`Data Collect Activity is not installed.`)
-      }
-    })
+    await issueCollectData(domainId, dataSetId, context)
 
     context.status = 200
   })
@@ -169,9 +142,7 @@ process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRout
         throw new Error(`domain(${domainId}) not found`)
       }
 
-      const dataSet = await tx
-        .getRepository(DataSet)
-        .findOne({ where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: dataSetId } })
+      const dataSet = await tx.getRepository(DataSet).findOne({ where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: dataSetId } })
 
       // do what you gotta do
     })

package/server/service/data-ooc/data-ooc-query.ts

@@ -2,10 +2,12 @@ import { Arg, Args, Ctx, Directive, FieldResolver, Query, Resolver, Root } from
 
 import { User } from '@things-factory/auth-base'
 import { Domain, getQueryBuilderFromListParams, getRepository, ListParam } from '@things-factory/shell'
+import { ActivityInstance } from '@things-factory/worklist'
 
 import { DataSetHistory } from '../data-set-history/data-set-history'
 import { DataItem } from '../data-set/data-item-type'
 import { DataSet } from '../data-set/data-set'
+import { DataSample } from '../data-sample/data-sample'
 import { DataOoc } from './data-ooc'
 import { DataOocList } from './data-ooc-type'
 
@@ -51,11 +53,26 @@ export class DataOocQuery {
     return dataSetHistory?.dataItems || []
   }
 
+  @FieldResolver(type => DataSample)
+  async dataSample(@Root() dataOoc: DataOoc): Promise<DataSample> {
+    return dataOoc.dataSampleId && (await getRepository(DataSample).findOneBy({ id: dataOoc.dataSampleId }))
+  }
+
   @FieldResolver(type => DataSet)
   async dataSet(@Root() dataOoc: DataOoc): Promise<DataSet> {
     return dataOoc.dataSetId && (await getRepository(DataSet).findOneBy({ id: dataOoc.dataSetId }))
   }
 
+  @FieldResolver(type => ActivityInstance)
+  async reviewActivityInstance(@Root() dataOoc: DataOoc): Promise<ActivityInstance> {
+    return dataOoc.reviewActivityInstanceId && (await getRepository(ActivityInstance).findOneBy({ id: dataOoc.reviewActivityInstanceId }))
+  }
+
+  @FieldResolver(type => ActivityInstance)
+  async resolveActivityInstance(@Root() dataOoc: DataOoc): Promise<ActivityInstance> {
+    return dataOoc.resolveActivityInstanceId && (await getRepository(ActivityInstance).findOneBy({ id: dataOoc.resolveActivityInstanceId }))
+  }
+
   @FieldResolver(type => User)
   async corrector(@Root() dataOoc: DataOoc): Promise<User> {
     return dataOoc.correctorId && (await getRepository(User).findOneBy({ id: dataOoc.correctorId }))