@things-factory/dataset 5.0.0-alpha.1 → 5.0.0-alpha.4

package/dist-server/service/index.js

@@ -16,25 +16,33 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.schema = exports.entities = void 0;
 /* IMPORT ENTITIES AND RESOLVERS */
+const data_sensor_1 = require("./data-sensor");
 const data_sample_1 = require("./data-sample");
 const data_item_1 = require("./data-item");
 const data_set_1 = require("./data-set");
+const data_spec_1 = require("./data-spec");
 /* EXPORT ENTITY TYPES */
+__exportStar(require("./data-sensor/data-sensor"), exports);
 __exportStar(require("./data-sample/data-sample"), exports);
 __exportStar(require("./data-item/data-item"), exports);
 __exportStar(require("./data-set/data-set"), exports);
+__exportStar(require("./data-spec/data-spec"), exports);
 exports.entities = [
     /* ENTITIES */
+    ...data_sensor_1.entities,
     ...data_sample_1.entities,
     ...data_item_1.entities,
-    ...data_set_1.entities
+    ...data_set_1.entities,
+    ...data_spec_1.entities
 ];
 exports.schema = {
     resolverClasses: [
         /* RESOLVER CLASSES */
+        ...data_sensor_1.resolvers,
         ...data_sample_1.resolvers,
         ...data_item_1.resolvers,
-        ...data_set_1.resolvers
+        ...data_set_1.resolvers,
+        ...data_spec_1.resolvers
     ]
 };
 //# sourceMappingURL=index.js.map

package/dist-server/service/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../server/service/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,mCAAmC;AACnC,+CAAgG;AAChG,2CAA0F;AAC1F,yCAAuF;AAEvF,yBAAyB;AACzB,4DAAyC;AACzC,wDAAqC;AACrC,sDAAmC;AAEtB,QAAA,QAAQ,GAAG;IACtB,cAAc;IACf,GAAG,sBAAkB;IACpB,GAAG,oBAAgB;IACnB,GAAG,mBAAe;CACnB,CAAA;AAEY,QAAA,MAAM,GAAG;IACpB,eAAe,EAAE;QACf,sBAAsB;QACxB,GAAG,uBAAmB;QACpB,GAAG,qBAAiB;QACpB,GAAG,oBAAgB;KACpB;CACF,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../server/service/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,mCAAmC;AACnC,+CAAgG;AAChG,+CAAgG;AAChG,2CAA0F;AAC1F,yCAAuF;AACvF,2CAA0F;AAE1F,yBAAyB;AACzB,4DAAyC;AACzC,4DAAyC;AACzC,wDAAqC;AACrC,sDAAmC;AACnC,wDAAqC;AAExB,QAAA,QAAQ,GAAG;IACtB,cAAc;IACd,GAAG,sBAAkB;IACrB,GAAG,sBAAkB;IACrB,GAAG,oBAAgB;IACnB,GAAG,mBAAe;IAClB,GAAG,oBAAgB;CACpB,CAAA;AAEY,QAAA,MAAM,GAAG;IACpB,eAAe,EAAE;QACf,sBAAsB;QACtB,GAAG,uBAAmB;QACtB,GAAG,uBAAmB;QACtB,GAAG,qBAAiB;QACpB,GAAG,oBAAgB;QACnB,GAAG,qBAAiB;KACrB;CACF,CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@things-factory/dataset",
-  "version": "5.0.0-alpha.1",
+  "version": "5.0.0-alpha.4",
   "main": "dist-server/index.js",
   "browser": "client/index.js",
   "things-factory": true,
@@ -24,15 +24,17 @@
     "migration:create": "node ../../node_modules/typeorm/cli.js migration:create -d ./server/migrations"
   },
   "dependencies": {
-    "@operato/data-grist": "^0.4.1",
-    "@operato/graphql": "^0.4.1",
-    "@operato/i18n": "^0.4.1",
-    "@operato/layout": "^0.4.1",
-    "@operato/shell": "^0.4.1",
-    "@operato/styles": "^0.4.1",
-    "@operato/utils": "^0.4.1",
-    "@things-factory/auth-base": "^5.0.0-alpha.1",
-    "@things-factory/shell": "^5.0.0-alpha.1"
+    "@operato/data-grist": "1.0.0-alpha.4",
+    "@operato/dataset": "1.0.0-alpha.4",
+    "@operato/graphql": "1.0.0-alpha.4",
+    "@operato/i18n": "1.0.0-alpha.4",
+    "@operato/layout": "1.0.0-alpha.4",
+    "@operato/shell": "1.0.0-alpha.4",
+    "@operato/styles": "1.0.0-alpha.4",
+    "@operato/utils": "1.0.0-alpha.4",
+    "@things-factory/auth-base": "^5.0.0-alpha.4",
+    "@things-factory/env": "^5.0.0-alpha.4",
+    "@things-factory/shell": "^5.0.0-alpha.4"
   },
-  "gitHead": "2b3e3818a5d7ab7fdfff9214cc65f56885b27ad0"
+  "gitHead": "77253f86954dc6f5c14356ea85887c323fd2511a"
 }

package/server/index.ts

@@ -3,3 +3,5 @@ import './routes'
 export * from './migrations'
 export * from './middlewares'
 export * from './service'
+
+export * from './service/data-spec/data-spec-manager'

package/server/routes.ts

@@ -1,3 +1,9 @@
+import { getConnection } from 'typeorm'
+
+import { DataSample } from './service'
+import { DataItem } from './service/data-item/data-item'
+import { DataSensor } from './service/data-sensor/data-sensor'
+
 const debug = require('debug')('things-factory:dataset:routes')
 
 process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRouter) => {
@@ -7,6 +13,76 @@ process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRout
    * ex) routes.get('/path', async(context, next) => {})
    * ex) routes.post('/path', async(context, next) => {})
    */
+
+  globalPublicRouter.post('/sensor-data', async (context, next) => {
+    // 데이타 검증
+    const { deviceId, data, rawData, timestamp = Date.now() } = context.request.body
+    if (!deviceId || !data) {
+      throw new Error(`deviceId(${deviceId}) and data(${JSON.stringify(data)}) properties are mandatory`)
+    }
+
+    // make new data-sample
+    await getConnection().transaction(async tx => {
+      // find sensor through deviceId
+      const sensor = await tx.getRepository(DataSensor).findOne({
+        where: { deviceId },
+        relations: ['domain', 'appliance', 'dataSet']
+      })
+
+      if (!sensor) {
+        throw new Error(`Sensor having given deviceId(${deviceId}) not found`)
+      }
+
+      if (!sensor.active) {
+        throw new Error(`State of the sensor given deviceId(${deviceId}) is not active`)
+      }
+
+      if (!sensor.appliance) {
+        throw new Error(`Appliance of the sensor given deviceId(${deviceId}) is not set up`)
+      }
+
+      if (!sensor.dataSet) {
+        throw new Error(`DataSet of the sensor given deviceId(${deviceId}) is not set up`)
+      }
+
+      const domain = sensor.domain
+      const dataSet = sensor.dataSet
+      const user = sensor.appliance
+
+      const dataItems = await tx.getRepository(DataItem).find({
+        where: {
+          domain,
+          dataSet
+        },
+        order: {
+          sequence: 'DESC'
+        }
+      })
+
+      var spec = {} as any
+
+      dataItems.forEach(dataItem => {
+        spec[dataItem.name] = dataItem.spec
+      })
+
+      await tx.getRepository(DataSample).save({
+        domain,
+        name: dataSet.name,
+        description: dataSet.description,
+        partitionKeys: dataSet.partitionKeys,
+        dataSet,
+        data,
+        rawData,
+        spec,
+        source: deviceId,
+        collectedAt: new Date(timestamp),
+        creator: user,
+        updater: user
+      })
+    })
+
+    return 'OK'
+  })
 })
 
 process.on('bootstrap-module-global-private-route' as any, (app, globalPrivateRouter) => {

package/server/service/data-item/data-item-type.ts

@@ -21,20 +21,23 @@ export class DataItemPatch {
   @Field(type => Int, { nullable: true })
   sequence?: number
 
+  @Field({ nullable: true })
+  tag?: string
+
   @Field(type => DataItemType, { nullable: true })
   type?: DataItemType
 
+  @Field(type => ScalarObject, { nullable: true })
+  options?: ScalarObject
+
   @Field(type => Int, { nullable: true })
   quota?: number
 
-  @Field({ nullable: true })
-  options?: string
-
   @Field({ nullable: true })
   active?: boolean
 
   @Field(type => ScalarObject, { nullable: true })
-  spec?: object
+  spec?: ScalarObject
 
   @Field({ nullable: true })
   cuFlag?: string

package/server/service/data-item/data-item.ts

@@ -19,7 +19,8 @@ export enum DataItemType {
   number = 'number',
   text = 'text',
   boolean = 'boolean',
-  select = 'select'
+  select = 'select',
+  file = 'file'
 }
 
 registerEnumType(DataItemType, {
@@ -63,6 +64,12 @@ export class DataItem {
   @Field(type => Int, { nullable: true })
   sequence: number
 
+  @Column({
+    nullable: true
+  })
+  @Field({ nullable: true })
+  tag?: string
+
   @Column({
     nullable: true
   })
@@ -75,6 +82,10 @@ export class DataItem {
   @Field({ nullable: true })
   type?: DataItemType
 
+  @Column('simple-json', { nullable: true })
+  @Field(type => ScalarObject, { nullable: true })
+  options?: ScalarObject
+
   @Column({
     nullable: true
   })
@@ -83,7 +94,7 @@ export class DataItem {
 
   @Column('simple-json', { nullable: true })
   @Field(type => ScalarObject, { nullable: true })
-  spec?: object
+  spec?: ScalarObject
 
   @CreateDateColumn()
   @Field({ nullable: true })

package/server/service/data-sample/data-sample-mutation.ts

@@ -6,6 +6,35 @@ import { DataSet } from '../data-set/data-set'
 import { DataSample } from './data-sample'
 import { DataSamplePatch, NewDataSample } from './data-sample-type'
 
+import moment from 'moment'
+
+const debug = require('debug')('things-factory:dataset:data-sample:data-sample-mutation')
+
+// parse variable javascript string pattern
+const replaceVariables = (keys, dic) => {
+  for (const k in keys) {
+    const matches = keys[k].match(/\$\{\w*\}/g)
+    matches &&
+      matches.forEach(m => {
+        keys[k] = keys[k].replace(m, dic[m.slice(2, -1)])
+      })
+  }
+  return keys
+}
+
+// It is required UTC date for Partitioning File System like AWS S3.
+// ex) %YYYY, %MM, %DD
+const formatDate = (keys, _moment) => {
+  for (const k in keys) {
+    const matches = keys[k].match(/%\w*/g)
+    matches &&
+      matches.forEach(m => {
+        keys[k] = keys[k].replace(m, _moment.format(m.substr(1)))
+      })
+  }
+  return keys
+}
+
 @Resolver(DataSample)
 export class DataSampleMutation {
   @Directive('@privilege(category: "data-sample", privilege: "mutation", domainOwnerGranted: true)')
@@ -16,19 +45,38 @@ export class DataSampleMutation {
 
     const dataSet = await tx.getRepository(DataSet).findOne({ id: dataSample.dataSetId })
     const dataItems = await tx.getRepository(DataItem).find({
-      select: {
+      where: {
         domain,
-        dataSetId: dataSample.dataSetId
+        dataSet
       },
       order: {
         sequence: 'DESC'
       }
     })
 
-    var spec = {} as any
+    const spec = dataItems.reduce((spec, dataItem) => {
+      spec[dataItem.tag] = {
+        ...dataItem.spec,
+        name: dataItem.name /* do we need ? */
+      }
+
+      return spec
+    }, {})
+
+    var partitionKeys = {
+      ...dataSet.partitionKeys,
+      year: '%YYYY',
+      month: '%MM',
+      date: '%DD',
+      variableX: '${x}',
+      variableDataSetId: '${dataSetId}'
+    }
 
-    dataItems.sortforEach(dataItem => {
-      spec[dataItem.name] = dataItem.spec
+    const collectedAt = dataSample.collectedAt || new Date()
+    partitionKeys = formatDate(partitionKeys, moment(collectedAt).utc())
+    partitionKeys = replaceVariables(partitionKeys, {
+      dataSetId: dataSample.dataSetId,
+      ...dataSample.data
     })
 
     return await tx.getRepository(DataSample).save({
@@ -36,7 +84,9 @@ export class DataSampleMutation {
       description: dataSet.description,
       ...dataSample,
       domain,
+      partitionKeys,
       spec,
+      collectedAt,
       creator: user,
       updater: user
     })

package/server/service/data-sample/data-sample-type.ts

@@ -16,7 +16,16 @@ export class NewDataSample {
   dataSetId?: string
 
   @Field(type => ScalarObject, { nullable: true })
-  data?: object
+  data?: ScalarObject
+
+  @Field({ nullable: true })
+  rawData?: string
+
+  @Field({ nullable: true })
+  source?: string
+
+  @Field({ nullable: true })
+  collectedAt?: Date
 }
 
 @InputType()
@@ -34,7 +43,13 @@ export class DataSamplePatch {
   dataSetId?: string
 
   @Field(type => ScalarObject, { nullable: true })
-  data?: object
+  data?: ScalarObject
+
+  @Field({ nullable: true })
+  rawData?: string
+
+  @Field({ nullable: true })
+  source?: string
 
   @Field()
   cuFlag: string

package/server/service/data-sample/data-sample.ts

@@ -11,10 +11,13 @@ import {
 } from 'typeorm'
 
 import { User } from '@things-factory/auth-base'
+import { config } from '@things-factory/env'
 import { Domain, ScalarObject } from '@things-factory/shell'
 
 import { DataSet } from '../data-set/data-set'
 
+const ORMCONFIG = config.get('ormconfig', {})
+const DATABASE_TYPE = ORMCONFIG.type
 @Entity()
 @Index('ix_data_sample_0', (dataSample: DataSample) => [dataSample.domain, dataSample.dataSet], { unique: false })
 @ObjectType({ description: 'Entity for DataSample' })
@@ -40,6 +43,16 @@ export class DataSample {
   @Field({ nullable: true })
   description?: string
 
+  @Column({
+    nullable: true
+  })
+  @Field({ nullable: true })
+  type?: string
+
+  @Column('simple-json', { nullable: true })
+  @Field(type => ScalarObject, { nullable: true })
+  partitionKeys?: ScalarObject
+
   @ManyToOne(type => DataSet, dataSet => dataSet.dataSamples)
   @Field(type => DataSet, { nullable: true })
   dataSet?: DataSet
@@ -50,11 +63,33 @@ export class DataSample {
 
   @Column('simple-json', { nullable: true })
   @Field(type => ScalarObject, { nullable: true })
-  data?: object
+  data?: ScalarObject
 
   @Column('simple-json', { nullable: true })
   @Field(type => ScalarObject, { nullable: true })
-  spec?: object
+  spec?: ScalarObject
+
+  @Column({
+    nullable: true,
+    type:
+      DATABASE_TYPE == 'mysql' || DATABASE_TYPE == 'mariadb'
+        ? 'longtext'
+        : DATABASE_TYPE == 'oracle'
+        ? 'clob'
+        : 'varchar'
+  })
+  @Field({ nullable: true })
+  rawData?: string
+
+  @Column({
+    nullable: true
+  })
+  @Field({ nullable: true })
+  source?: string
+
+  @Column({ nullable: true })
+  @Field({ nullable: true })
+  collectedAt?: Date
 
   @CreateDateColumn()
   @Field({ nullable: true })

package/server/service/data-sensor/data-sensor-mutation.ts

@@ -0,0 +1,110 @@
+import { Arg, Ctx, Directive, Mutation, Resolver } from 'type-graphql'
+import { In } from 'typeorm'
+
+import { DataSensor } from './data-sensor'
+import { DataSensorPatch, NewDataSensor } from './data-sensor-type'
+
+@Resolver(DataSensor)
+export class DataSensorMutation {
+  @Directive('@transaction')
+  @Mutation(returns => DataSensor, { description: 'To create new DataSensor' })
+  async createDataSensor(@Arg('dataSensor') dataSensor: NewDataSensor, @Ctx() context: any): Promise<DataSensor> {
+    const { domain, user, tx } = context.state
+
+    return await tx.getRepository(DataSensor).save({
+      ...dataSensor,
+      domain,
+      creator: user,
+      updater: user
+    })
+  }
+
+  @Directive('@transaction')
+  @Mutation(returns => DataSensor, { description: 'To modify DataSensor information' })
+  async updateDataSensor(
+    @Arg('id') id: string,
+    @Arg('patch') patch: DataSensorPatch,
+    @Ctx() context: any
+  ): Promise<DataSensor> {
+    const { domain, user, tx } = context.state
+
+    const repository = tx.getRepository(DataSensor)
+    const dataSensor = await repository.findOne({
+      where: { domain, id }
+    })
+
+    return await repository.save({
+      ...dataSensor,
+      ...patch,
+      updater: user
+    })
+  }
+
+  @Directive('@transaction')
+  @Mutation(returns => [DataSensor], { description: "To modify multiple DataSensors' information" })
+  async updateMultipleDataSensor(
+    @Arg('patches', type => [DataSensorPatch]) patches: DataSensorPatch[],
+    @Ctx() context: any
+  ): Promise<DataSensor[]> {
+    const { domain, user, tx } = context.state
+
+    let results = []
+    const _createRecords = patches.filter((patch: any) => patch.cuFlag.toUpperCase() === '+')
+    const _updateRecords = patches.filter((patch: any) => patch.cuFlag.toUpperCase() === 'M')
+    const dataSensorRepo = tx.getRepository(DataSensor)
+
+    if (_createRecords.length > 0) {
+      for (let i = 0; i < _createRecords.length; i++) {
+        const newRecord = _createRecords[i]
+
+        const result = await dataSensorRepo.save({
+          ...newRecord,
+          domain,
+          creator: user,
+          updater: user
+        })
+
+        results.push({ ...result, cuFlag: '+' })
+      }
+    }
+
+    if (_updateRecords.length > 0) {
+      for (let i = 0; i < _updateRecords.length; i++) {
+        const newRecord = _updateRecords[i]
+        const dataSensor = await dataSensorRepo.findOne(newRecord.id)
+
+        const result = await dataSensorRepo.save({
+          ...dataSensor,
+          ...newRecord,
+          updater: user
+        })
+
+        results.push({ ...result, cuFlag: 'M' })
+      }
+    }
+
+    return results
+  }
+
+  @Directive('@transaction')
+  @Mutation(returns => Boolean, { description: 'To delete DataSensor' })
+  async deleteDataSensor(@Arg('id') id: string, @Ctx() context: any): Promise<boolean> {
+    const { domain, tx } = context.state
+
+    await tx.getRepository(DataSensor).delete({ domain, id })
+    return true
+  }
+
+  @Directive('@transaction')
+  @Mutation(returns => Boolean, { description: 'To delete multiple dataSensors' })
+  async deleteDataSensors(@Arg('ids', type => [String]) ids: string[], @Ctx() context: any): Promise<boolean> {
+    const { domain, tx } = context.state
+
+    await tx.getRepository(DataSensor).delete({
+      domain,
+      id: In(ids)
+    })
+
+    return true
+  }
+}

package/server/service/data-sensor/data-sensor-query.ts

@@ -0,0 +1,56 @@
+import { Arg, Args, Ctx, FieldResolver, Query, Resolver, Root } from 'type-graphql'
+import { getRepository } from 'typeorm'
+
+import { Appliance, User } from '@things-factory/auth-base'
+import { convertListParams, Domain, ListParam } from '@things-factory/shell'
+
+import { DataSet } from '../data-set/data-set'
+import { DataSensor } from './data-sensor'
+import { DataSensorList } from './data-sensor-type'
+
+@Resolver(DataSensor)
+export class DataSensorQuery {
+  @Query(returns => DataSensor, { description: 'To fetch a DataSensor' })
+  async dataSensor(@Arg('id') id: string, @Ctx() context: any): Promise<DataSensor> {
+    const { domain } = context.state
+
+    return await getRepository(DataSensor).findOne({
+      where: { domain, id }
+    })
+  }
+
+  @Query(returns => DataSensorList, { description: 'To fetch multiple DataSensors' })
+  async dataSensors(@Args() params: ListParam, @Ctx() context: any): Promise<DataSensorList> {
+    const { domain } = context.state
+
+    const convertedParams = convertListParams(params, domain.id)
+    const [items, total] = await getRepository(DataSensor).findAndCount(convertedParams)
+
+    return { items, total }
+  }
+
+  @FieldResolver(type => Appliance)
+  async appliance(@Root() dataSensor: DataSensor): Promise<Appliance> {
+    return await getRepository(Appliance).findOne(dataSensor.applianceId || '')
+  }
+
+  @FieldResolver(type => Appliance)
+  async dataSet(@Root() dataSensor: DataSensor): Promise<DataSet> {
+    return await getRepository(DataSet).findOne(dataSensor.dataSetId || '')
+  }
+
+  @FieldResolver(type => Domain)
+  async domain(@Root() dataSensor: DataSensor): Promise<Domain> {
+    return await getRepository(Domain).findOne(dataSensor.domainId)
+  }
+
+  @FieldResolver(type => User)
+  async updater(@Root() dataSensor: DataSensor): Promise<User> {
+    return await getRepository(User).findOne(dataSensor.updaterId)
+  }
+
+  @FieldResolver(type => User)
+  async creator(@Root() dataSensor: DataSensor): Promise<User> {
+    return await getRepository(User).findOne(dataSensor.creatorId)
+  }
+}