@things-factory/dataset 7.0.0-alpha.0 → 7.0.0-alpha.18

package/server/controllers/create-data-sample.ts

@@ -55,16 +55,16 @@ const formatDate = (keys, _moment) => {
   return keys
 }
 
-export async function createDataSample(dataSample: NewDataSample, context: ResolverContext): Promise<DataSample> {
+export async function createDataSample(newDataSample: NewDataSample, context: ResolverContext): Promise<DataSample> {
   const { domain, user, tx } = context.state
 
   const dataSet = await tx.getRepository(DataSet).findOne({
-    where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: dataSample.dataSet.id },
+    where: { domain: { id: In([domain.id, domain.parentId].filter(Boolean)) }, id: newDataSample.dataSet.id },
     relations: ['dataKeySet']
   })
 
   const { dataItems, tag: publishTag, normalScenarioId, outlierScenarioId } = dataSet
-  const collectedAt = dataSample.collectedAt || new Date()
+  const collectedAt = newDataSample.collectedAt || new Date()
 
   const timezone = dataSet.timezone || domain.timezone || 'UTC'
   const format = 'YYYY-MM-DD'
@@ -77,7 +77,7 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
   const localDateTz = moment(collectedAt).tz(timezone)
   const defaultPartitionKeys = {
     domain: domain.subdomain,
-    datasetid: dataSample.dataSet.id /* It should not be 'data_set_id' as column name duplicated for Glue */,
+    datasetid: newDataSample.dataSet.id /* It should not be 'data_set_id' as column name duplicated for Glue */,
     date: localDateTz.format(format) /* local time date */,
     workdate: workDate /* working date */,
     workshift: workShift
@@ -89,11 +89,11 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
   }
 
   partitionKeys = formatDate(partitionKeys, localDateTz)
-  partitionKeys = replaceVariables(partitionKeys, dataSample.data)
+  partitionKeys = replaceVariables(partitionKeys, newDataSample.data)
 
-  const dataKeys = fillDataKeys(dataSet?.dataKeySet, dataSample.data)
+  const dataKeys = fillDataKeys(dataSet?.dataKeySet, newDataSample.data)
 
-  const { ooc, oos, judgment } = DataUseCase.evaluate(dataSet, dataItems, dataSample.data) || {}
+  const { ooc, oos, judgment } = DataUseCase.evaluate(dataSet, dataItems, newDataSample.data) || {}
 
   const old = await tx.getRepository(DataSample).findOne({
     where: {
@@ -110,7 +110,7 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
     If [FileUpload[]] type needed, add 'files' type for dataset
   */
 
-  const data = dataSample.data
+  const data = newDataSample.data
   const attachments = []
 
   for (let dataItem of dataItems) {
@@ -164,13 +164,13 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
     }
   }
 
-  const result = await tx.getRepository(DataSample).save({
+  const dataSample = await tx.getRepository(DataSample).save({
     ...old,
     name: dataSet.name,
     description: dataSet.description,
     useCase: dataSet.useCase,
     type: dataSet.type,
-    ...dataSample,
+    ...newDataSample,
     ...dataKeys,
     dataSetVersion: dataSet.version,
     domain,
@@ -197,13 +197,13 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
 
   /* post-process for for file attachment. */
   if (attachments.length > 0) {
-    attachments.forEach(attachment => (attachment.refId = result.id))
+    attachments.forEach(attachment => (attachment.refId = dataSample.id))
     tx.getRepository(Attachment).save(attachments)
   }
 
   if (ooc || oos) {
     const dataOoc = await tx.getRepository(DataOoc).save({
-      ...result,
+      ...dataSample,
       history: [
         {
           user: {
@@ -214,7 +214,8 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
           timestamp: collectedAt
         }
       ],
-      state: DataOocStatus.ISSUED
+      state: DataOocStatus.ISSUED,
+      dataSample: dataSample
     })
 
     if (outlierScenarioId) {
@@ -227,9 +228,9 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
       })
 
       if (scenario) {
-        scenario.start(scenario.name + ':' + dataSet.name + ':' + result.id, {
+        scenario.start(scenario.name + ':' + dataSet.name + ':' + dataSample.id, {
           dataOocId: dataOoc.id,
-          dataSampleId: result.id,
+          dataSampleId: dataSample.id,
           dataSet: dataSet.id,
           data,
           ooc,
@@ -260,9 +261,7 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
     })) as Activity
 
     if (activity) {
-      const assignee =
-        dataSet.supervisoryRoleId &&
-        (await tx.getRepository(Role).findOneBy({ domain: { id: domain.id }, id: dataSet.supervisoryRoleId }))
+      const assignee = dataSet.supervisoryRoleId && (await tx.getRepository(Role).findOneBy({ domain: { id: domain.id }, id: dataSet.supervisoryRoleId }))
 
       const assignees = dataSet.supervisoryRoleId ? [{ type: 'Role', value: dataSet.supervisoryRoleId, assignee }] : []
 
@@ -280,11 +279,10 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
           approvalLine: []
         }
 
-        await issue(activityInstance, context)
+        dataOoc.reviewActivityInstance = await issue(activityInstance, context)
+        await tx.getRepository(DataOoc).save(dataOoc)
       } else {
-        console.error(
-          `Assignees are not set. So Data OOC Review task for ${dataOoc.name}(${dataOoc.id}) could not be issued.`
-        )
+        console.error(`Assignees are not set. So Data OOC Review task for ${dataOoc.name}(${dataOoc.id}) could not be issued.`)
       }
     } else {
       console.error('OOC Review Activity not installed.')
@@ -320,8 +318,8 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
       })
 
       if (scenario) {
-        scenario.start(scenario.name + ':' + dataSet.name + ':' + result.id, {
-          dataSampleId: result.id,
+        scenario.start(scenario.name + ':' + dataSet.name + ':' + dataSample.id, {
+          dataSampleId: dataSample.id,
           dataSet: dataSet.id,
           data,
           ooc,
@@ -353,30 +351,27 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
       })) as Activity
 
       if (activity) {
-        const assignee =
-          dataSet.supervisoryRoleId &&
-          (await tx.getRepository(Role).findOneBy({ domain: { id: domain.id }, id: dataSet.supervisoryRoleId }))
+        const assignee = dataSet.supervisoryRoleId && (await tx.getRepository(Role).findOneBy({ domain: { id: domain.id }, id: dataSet.supervisoryRoleId }))
 
-        const assignees = dataSet.supervisoryRoleId
-          ? [{ type: 'Role', value: dataSet.supervisoryRoleId, assignee }]
-          : []
+        const assignees = dataSet.supervisoryRoleId ? [{ type: 'Role', value: dataSet.supervisoryRoleId, assignee }] : []
 
         /* 해당 dataset의 supervisor로 하여금, data를 리뷰하고 instruction을 작성해서, approvalLine을 이용해서 승인을 한다. */
         if (assignees && assignees instanceof Array && assignees.length > 0) {
-          result.dataItems = dataItems
+          dataSample.dataItems = dataItems
           const activityInstance = {
             name: `[Data 검토] ${dataSet.name}`,
             description: dataSet.description,
             activityId: activity.id,
             dueAt: new Date(collectedAt.getTime() + 24 * 60 * 60 * 1000),
             input: {
-              dataSample: result
+              dataSampleId: dataSample.id
             },
             assignees,
             approvalLine: dataSet.reviewApprovalLine
           }
 
-          await issue(activityInstance, context)
+          dataSample.reviewActivityInstance = await issue(activityInstance, context)
+          await tx.getRepository(DataSample).save(dataSample)
 
           try {
             pubsub.publish('notification', {
@@ -385,7 +380,7 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
                 type: 'info',
                 title: `[Data Review] ${dataSet.name}`,
                 body: `Data Review occurred on '${dataSet.name}'`,
-                url: getRedirectSubdomainPath(context, domain.subdomain, `/data-sample/${result.id}`),
+                url: getRedirectSubdomainPath(context, domain.subdomain, `/data-sample/${dataSample.id}`),
                 timestamp: collectedAt
               }
             })
@@ -393,9 +388,7 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
             logger.error('Notification', err)
           }
         } else {
-          console.error(
-            `Assignees are not set. So Data Review task for ${dataSet.name}(${dataSet.id}) could not be issued.`
-          )
+          console.error(`Assignees are not set. So Data Review task for ${dataSet.name}(${dataSet.id}) could not be issued.`)
         }
       } else {
         console.error('Data Review Activity not installed.')
@@ -403,5 +396,5 @@ export async function createDataSample(dataSample: NewDataSample, context: Resol
     }
   }
 
-  return result
+  return dataSample
 }

package/server/controllers/{generate-data-summary.ts → finalize-data-collection.ts}

@@ -90,11 +90,7 @@ const calculateSummary = (dataItems: DataItem[], base: { [tag: string]: any[] })
   }, {})
 }
 
-const fillSummaryResult = (
-  dataSummary: Partial<DataSummary>,
-  dataItems: DataItem[],
-  base: { [tag: string]: any[] }
-): void => {
+const fillSummaryResult = (dataSummary: Partial<DataSummary>, dataItems: DataItem[], base: { [tag: string]: any[] }): void => {
   const summary = calculateSummary(dataItems, base)
 
   dataSummary.summary = summary
@@ -104,10 +100,7 @@ const fillSummaryResult = (
   })
 }
 
-async function getLatestTimesForPeriod(
-  periodType: DataSetSummaryPeriodType,
-  context: ResolverContext
-): Promise<{ date?: string; period?: string; range: Date[] }> {
+async function getLatestTimesForPeriod(periodType: DataSetSummaryPeriodType, context: ResolverContext): Promise<{ date?: string; period?: string; range: Date[] }> {
   const { domain } = context.state
   const now = moment()
 
@@ -181,7 +174,7 @@ async function getTimesForPeriod(
   }
 }
 
-export async function getDataSummaryCrontabSchedule(dataSet: DataSet, context: ResolverContext): Promise<string> {
+export async function getDataFinalizeCrontabSchedule(dataSet: DataSet, context: ResolverContext): Promise<string> {
   const { domain, user, tx } = context.state
 
   try {
@@ -201,7 +194,7 @@ export async function getDataSummaryCrontabSchedule(dataSet: DataSet, context: R
   }
 }
 
-export async function generateLatestDataSummaries(dataSetId: string, context: ResolverContext): Promise<boolean> {
+export async function finalizeLatestDataCollection(dataSetId: string, context: ResolverContext): Promise<boolean> {
   const { domain, user, tx } = context.state
 
   try {
@@ -285,17 +278,7 @@ export async function generateLatestDataSummaries(dataSetId: string, context: Re
       page++
     } while (true)
 
-    tx.getRepository(DataSummary).upsert(summaries, [
-      'domain',
-      'dataSet',
-      'key01',
-      'key02',
-      'key03',
-      'key04',
-      'key05',
-      'date',
-      'period'
-    ])
+    tx.getRepository(DataSummary).upsert(summaries, ['domain', 'dataSet', 'key01', 'key02', 'key03', 'key04', 'key05', 'date', 'period'])
 
     return true
   } catch (e) {
@@ -305,12 +288,7 @@ export async function generateLatestDataSummaries(dataSetId: string, context: Re
   return false
 }
 
-export async function generateDataSummary(
-  dataSetId: string,
-  date: string,
-  period: string,
-  context: ResolverContext
-): Promise<boolean> {
+export async function finalizeDataCollection(dataSetId: string, date: string, period: string, context: ResolverContext): Promise<boolean> {
   const { domain, user, tx } = context.state
 
   try {
@@ -399,17 +377,7 @@ export async function generateDataSummary(
       page++
     } while (true)
 
-    tx.getRepository(DataSummary).upsert(summaries, [
-      'domain',
-      'dataSet',
-      'key01',
-      'key02',
-      'key03',
-      'key04',
-      'key05',
-      'date',
-      'period'
-    ])
+    tx.getRepository(DataSummary).upsert(summaries, ['domain', 'dataSet', 'key01', 'key02', 'key03', 'key04', 'key05', 'date', 'period'])
 
     return true
   } catch (e) {

package/server/controllers/issue-collect-data.ts

@@ -0,0 +1,61 @@
+import { In } from 'typeorm'
+
+import { Domain, getDataSource } from '@things-factory/shell'
+import { Activity, ActivityInstance, issue } from '@things-factory/worklist'
+
+import { DataSet } from '../service/data-set/data-set'
+
+export async function issueCollectData(domainId: string, dataSetId: string, context: ResolverContext): Promise<ActivityInstance> {
+  await getDataSource().transaction(async tx => {
+    const domain = await tx.getRepository(Domain).findOneBy({ id: domainId })
+
+    if (!domain) {
+      throw new Error(`domain(${domainId}) not found`)
+    }
+
+    const dataSet = await tx.getRepository(DataSet).findOne({
+      where: {
+        domain: {
+          id: In([domain.id, domain.parentId].filter(Boolean))
+        },
+        id: dataSetId
+      }
+    })
+
+    const activity = (await tx.getRepository(Activity).findOneBy({
+      domain: { id: domain.id },
+      name: 'Collect Data'
+    })) as Activity
+
+    if (activity) {
+      const { assignees } = dataSet
+
+      /* 해당 dataset에 대한 데이타 수집 태스크를 dataset assignees에게 할당한다. */
+      if (assignees && assignees instanceof Array && assignees.length > 0) {
+        const activityInstance = {
+          name: `[Data 수집] ${dataSet.name}`,
+          description: dataSet.description,
+          activityId: activity.id,
+          dueAt: new Date(Date.now() + 24 * 60 * 60 * 1000),
+          input: {
+            dataSetId: dataSet.id,
+            dataSetName: dataSet.name
+          },
+          assignees
+        }
+
+        context.state = {
+          ...context.state,
+          domain,
+          tx
+        }
+
+        return await issue(activityInstance, context)
+      } else {
+        throw new Error(`Assignees not set. So Data Collect Activity for ${dataSet.name}($dataSet.id) could not be issued.`)
+      }
+    } else {
+      throw new Error(`Data Collect Activity is not installed.`)
+    }
+  })
+}

package/server/controllers/query-data-summary-by-period.ts

@@ -1,12 +1,6 @@
 import { In } from 'typeorm'
 
-import {
-  getQueryBuilderFromListParams,
-  getRepository,
-  getTimesForPeriod,
-  ListParam,
-  Sorting
-} from '@things-factory/shell'
+import { getQueryBuilderFromListParams, getRepository, getTimesForPeriod, ListParam, Sorting } from '@things-factory/shell'
 
 import { DataSet, DataSetSummaryPeriodType } from '../service/data-set/data-set'
 import { DataSummary } from '../service/data-summary/data-summary'
@@ -44,10 +38,7 @@ export async function queryDataSummaryByPeriod(
 
   // limitations
   const summaryPeriodType = dataSet.summaryPeriod
-  if (
-    (summaryPeriodType == DataSetSummaryPeriodType.Day || summaryPeriodType == DataSetSummaryPeriodType.WorkDate) &&
-    period == 'today'
-  ) {
+  if ((summaryPeriodType == DataSetSummaryPeriodType.Day || summaryPeriodType == DataSetSummaryPeriodType.WorkDate) && period == 'today') {
     throw new Error(t('error.summary not supported', { dataSetName, period: t(`label.period-${period}`) }))
   }
 
@@ -65,7 +56,7 @@ export async function queryDataSummaryByPeriod(
     : dataSet.dataKeySet.dataKeyItems
 
   const searchables = dataKeyItems.map((item, index) => `key0${index + 1}`)
-  const dataItems = dataSet.dataItems.filter(item => item.stat)
+  const dataItems = dataSet.dataItems.filter(item => item.agg)
 
   const { from, to } = await getTimesForPeriod(period, context)
 
@@ -90,7 +81,7 @@ export async function queryDataSummaryByPeriod(
     const aliasName = `data0${index + 1}`
     const columnName = `summary.${aliasName}`
 
-    switch (item.stat) {
+    switch (item.agg) {
       case STAT_FUNCTION_MAP.sum:
         return `SUM(${columnName}) as ${aliasName}`
       case STAT_FUNCTION_MAP.mean:
@@ -118,11 +109,7 @@ export async function queryDataSummaryByPeriod(
   })
 
   const groupByPeriod =
-    period == 'today'
-      ? 'summary.date, summary.period'
-      : ['this year', '12 months'].includes(period)
-      ? 'SUBSTRING(summary.date, 1, 7)'
-      : 'summary.date'
+    period == 'today' ? 'summary.date, summary.period' : ['this year', '12 months'].includes(period) ? 'SUBSTRING(summary.date, 1, 7)' : 'summary.date'
   const groupByKeys = dataKeyItems
     .map((item, index) => {
       return `summary.key0${index + 1}`

package/server/engine/index.ts

@@ -0,0 +1 @@
+import './task'

package/server/engine/task/create-data-sample.ts

@@ -0,0 +1,92 @@
+import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
+import { getDataSource } from '@things-factory/shell'
+import { access } from '@things-factory/utils'
+import i18next from 'i18next'
+
+import { DataSet } from '../../service/data-set/data-set'
+
+import { createDataSample } from '../../controllers/create-data-sample'
+
+async function CreateDataSample(step: InputStep, context: Context) {
+  const { logger, data, domain, user, lng } = context
+  var {
+    params: { dataset: dataSetId, source: sourceAccessor, rawData: rawDataAccessor, data: dataAccessor, timestamp }
+  } = step
+
+  if (!dataSetId) {
+    throw new Error(`no dataset found`)
+  }
+
+  // make new data-sample
+  await getDataSource().transaction(async tx => {
+    const dataSet = await tx.getRepository(DataSet).findOne({
+      where: {
+        id: dataSetId
+      }
+    })
+
+    var source = await access(sourceAccessor, data)
+    var extractedData = await access(dataAccessor, data)
+    var extractedRawData = rawDataAccessor && (await access(rawDataAccessor, data))
+
+    const dataSample = await createDataSample(
+      {
+        dataSet,
+        data: extractedData,
+        rawData: extractedRawData,
+        source,
+        collectedAt: new Date(timestamp || new Date())
+      },
+      {
+        t: i18next.t,
+        state: {
+          domain,
+          user,
+          lng,
+          tx
+        }
+      }
+    )
+
+    return {
+      data: dataSample
+    }
+  })
+}
+
+CreateDataSample.parameterSpec = [
+  {
+    type: 'entity-selector',
+    name: 'dataset',
+    label: 'data-set',
+    property: {
+      queryName: 'dataSets',
+      valueKey: 'id'
+    }
+  },
+  {
+    type: 'string',
+    name: 'source',
+    label: 'source'
+  },
+  {
+    type: 'string',
+    name: 'rawData',
+    label: 'raw-data'
+  },
+  {
+    type: 'string',
+    name: 'data',
+    label: 'data'
+  },
+  {
+    type: 'string',
+    name: 'timestamp',
+    label: 'timestamp'
+  }
+]
+
+CreateDataSample.connectorFree = true
+CreateDataSample.help = 'dataset/task/create-data-sample'
+
+TaskRegistry.registerTaskHandler('create-data-sample', CreateDataSample)

package/server/engine/task/index.ts

@@ -0,0 +1,2 @@
+import './create-data-sample'
+import './issue-collect-data'

package/server/engine/task/issue-collect-data.ts

@@ -0,0 +1,45 @@
+import { TaskRegistry, InputStep, Context } from '@things-factory/integration-base'
+import { issueCollectData } from '../../controllers/issue-collect-data'
+import i18next from 'i18next'
+
+async function IssueCollectData(step: InputStep, context: Context) {
+  const { logger, data, domain, user, lng } = context
+  var {
+    params: { dataset: dataSetId }
+  } = step
+
+  if (!dataSetId) {
+    throw new Error(`no dataset found`)
+  }
+
+  const activityInstance = await issueCollectData(domain.id, dataSetId, {
+    t: i18next.t,
+    state: {
+      domain,
+      user,
+      lng,
+      tx: undefined
+    }
+  })
+
+  return {
+    data: activityInstance
+  }
+}
+
+IssueCollectData.parameterSpec = [
+  {
+    type: 'entity-selector',
+    name: 'dataset',
+    label: 'data-set',
+    property: {
+      queryName: 'dataSets',
+      valueKey: 'id'
+    }
+  }
+]
+
+IssueCollectData.connectorFree = true
+IssueCollectData.help = 'dataset/task/issue-collect-data'
+
+TaskRegistry.registerTaskHandler('issue-collect-data', IssueCollectData)

package/server/index.ts

@@ -1,5 +1,6 @@
 import './routes'
 import './activities'
+import './engine'
 
 export * from './controllers'
 export * from './service'