@things-factory/integration-base 8.0.0-beta.9 → 8.0.2

package/server/engine/connector/socket-server.ts

@@ -0,0 +1,86 @@
+import net from 'net'
+import { EventEmitter } from 'events'
+
+import { Connector } from '../types'
+import { ConnectionManager } from '../connection-manager'
+import { InputConnection } from '../../service/connection/connection-type'
+
+export class SocketServer implements Connector {
+  // socket: any
+
+  async ready(connectionConfigs: InputConnection[]) {
+    await Promise.all(connectionConfigs.map(this.connect))
+    ConnectionManager.logger.info('socket servers are ready')
+  }
+
+  async connect(config: InputConnection): Promise<void> {
+    var em = new EventEmitter()
+    var [host = '0.0.0.0', port = 8124] = config.endpoint.split(':')
+    return new Promise((resolve, reject) => {
+      var server = net.createServer(tcp => {
+        ConnectionManager.logger.info('Client connection: ')
+        ConnectionManager.logger.info('   local = %s:%s', tcp.localAddress, tcp.localPort)
+        ConnectionManager.logger.info('   remote = %s:%s', tcp.remoteAddress, tcp.remotePort)
+        tcp.on('data', (data: any) => {
+          em['__tcp__'] = tcp
+          var message = data.toString()
+          message = message.replace('SB#', '')
+          message = message.replace(/(\r\n|\n|\r)/gm, '')
+          em.emit('socket-message-arrive', message)
+          ConnectionManager.logger.info(`socket ${message}`)
+          tcp.write('ok')
+        })
+
+        tcp.on('end', () => {
+          ConnectionManager.logger.info('Client disconnected')
+        })
+
+        tcp.on('error', ex => {
+          ConnectionManager.logger.info('Server socket error: ')
+          var logger = ConnectionManager.logger
+          let message = ex.stack ? ex.stack : ex
+          let msg = typeof message == 'object' ? JSON.stringify(message, null, 2) : message
+          logger.error(msg, ex)
+          reject(ex)
+        })
+      })
+
+      server.listen(port, async () => {
+        ConnectionManager.logger.info(`tcp echo server listening on ${host}:${port}`)
+
+        /* default client connection */
+        try {
+          em['__server__'] = server
+          ConnectionManager.addConnectionInstance(config, em)
+          ConnectionManager.logger.info(`socket server connection(${config.name}:${config.endpoint}) is connected`)
+
+          resolve()
+        } catch (err) {
+          ConnectionManager.logger.error(err)
+          reject(err)
+        }
+      })
+    })
+  }
+
+  async disconnect(connection: InputConnection) {
+    let em = ConnectionManager.removeConnectionInstance(connection)
+    var tcp = em['__tcp__']
+    var server = em['__server__']
+
+    tcp && (await tcp.destroy())
+    server && (await server.close())
+
+    ConnectionManager.logger.info(`socket server connection(${connection.name}) is disconnected`)
+  }
+
+  get parameterSpec() {
+    return []
+  }
+
+  get taskPrefixes() {
+    return ['socket']
+  }
+}
+
+ConnectionManager.registerConnector('socket-server', new SocketServer())

package/server/engine/connector/sqlite-connector.ts

@@ -0,0 +1,69 @@
+import { promisify } from 'util'
+
+import { logger } from '@things-factory/env'
+
+import { ConnectionManager } from '../connection-manager'
+import { Connector } from '../types'
+import { InputConnection } from '../../service/connection/connection-type'
+
+try {
+  var sqlite3 = require('sqlite3').verbose()
+} catch (err) {
+  logger.error('sqlite3 module loading failed', err)
+}
+
+export class SqliteConnector implements Connector {
+  async ready(connectionConfigs: InputConnection[]) {
+    await Promise.all(connectionConfigs.map(this.connect.bind(this)))
+
+    ConnectionManager.logger.info('sqlite-connector connections are ready')
+  }
+
+  async connect(connection: InputConnection) {
+    var { endpoint } = connection
+
+    if (!sqlite3) {
+      throw new Error('sqlite3 module loading failed')
+    }
+
+    var database = new sqlite3.Database(endpoint, sqlite3.OPEN_READWRITE, err => {
+      if (err) {
+        ConnectionManager.logger.error(`SQLite Database(${connection.name}) at ${endpoint} not connected.`, err)
+        return
+      }
+
+      ConnectionManager.addConnectionInstance(connection, {
+        query: promisify(database.all.bind(database)),
+        close: promisify(database.close.bind(database))
+      })
+
+      ConnectionManager.logger.info(`SQLite Database(${connection.name}) at ${endpoint} connected.`)
+    })
+  }
+
+  async disconnect(connection: InputConnection) {
+    var database = ConnectionManager.getConnectionInstance(connection)
+    try {
+      await database.close()
+      ConnectionManager.logger.info(`SQLite Database(${connection.name}) closed.`)
+    } catch (e) {
+      ConnectionManager.logger.error(e)
+    }
+
+    ConnectionManager.removeConnectionInstance(connection)
+  }
+
+  get parameterSpec() {
+    return []
+  }
+
+  get taskPrefixes() {
+    return ['database']
+  }
+
+  get help() {
+    return 'integration/connector/sqlite-connector'
+  }
+}
+
+ConnectionManager.registerConnector('sqlite-connector', new SqliteConnector())

package/server/engine/edge-client.ts

@@ -0,0 +1,45 @@
+import { Connection } from '../service/connection/connection-type'
+import { Step } from '../service/step/step-type'
+import { Context, TaskHandler } from './types'
+
+export type EdgeClient = {
+  handler: TaskHandler
+  syncConnections(connections: Connection[], context?: ResolverContext): Promise<any>
+  connectConnections(connections: Connection[], context?: ResolverContext): Promise<any>
+  disconnectConnections(connections: Connection[], context?: ResolverContext): Promise<any>
+}
+
+var edgeClient: EdgeClient = {
+  async handler(step: Step, scenarioContext: Context) {
+    throw 'edgeClient not supported'
+  },
+  async syncConnections(connections: Connection[], context: ResolverContext): Promise<any> {
+    throw 'edgeClient not supported'
+  },
+  async connectConnections(connections: Connection[], context: ResolverContext): Promise<any> {
+    throw 'edgeClient not supported'
+  },
+  async disconnectConnections(connections: Connection[], context: ResolverContext): Promise<any> {
+    throw 'edgeClient not supported'
+  }
+}
+
+export async function handler(step: Step, scenarioContext: Context) {
+  return await edgeClient.handler(step, scenarioContext)
+}
+
+export async function syncConnections(connections: Connection[], context?: ResolverContext): Promise<any> {
+  return await edgeClient.syncConnections(connections, context)
+}
+
+export async function connectConnections(connections: Connection[], context?: ResolverContext): Promise<any> {
+  return await edgeClient.connectConnections(connections, context)
+}
+
+export async function disconnectConnections(connections: Connection[], context?: ResolverContext): Promise<any> {
+  return await edgeClient.disconnectConnections(connections, context)
+}
+
+export function setEdgeClient(edge: EdgeClient) {
+  edgeClient = edge
+}

package/server/engine/index.ts

@@ -0,0 +1,10 @@
+import './connector'
+import './task'
+
+export * from './connection-manager'
+export * from './scenario-engine'
+export * from './task-registry'
+export * from './analyzer/analyze-integration'
+export * from './edge-client'
+
+export { Connector, Context, TaskHandler } from './types'

package/server/engine/pending-queue.ts

@@ -0,0 +1,97 @@
+const debug = require('debug')('things-factory:integration-base:pending-queue')
+
+export type PendingObject = {
+  stuff
+  due
+  priority
+  tag
+}
+
+export class PendingQueue {
+  private _queue: PendingObject[] = []
+  private _changedCallback
+
+  constructor(changedCallback) {
+    this._changedCallback = changedCallback
+  }
+
+  get queue() {
+    return this._queue
+  }
+
+  get status() {
+    return {
+      total: this._queue.length
+    }
+  }
+
+  reset() {
+    this._queue = []
+    this._changedCallback(this._queue)
+  }
+
+  put({ delay, stuff, priority = 0, tag = '' }) {
+    const due = Date.now() + (Number(delay) || 0) * 1000
+    const insertBefore = this._queue.findIndex(pended => pended.due > due)
+
+    if (insertBefore == -1) {
+      this._queue.push({
+        due,
+        priority: Number(priority) || 0,
+        stuff,
+        tag: tag || ''
+      })
+    } else {
+      this._queue.splice(insertBefore, 0, {
+        due,
+        priority,
+        stuff,
+        tag
+      })
+    }
+
+    this._changedCallback(this._queue)
+
+    debug('put', this.status)
+  }
+
+  pick(tag = '') {
+    const due = Date.now()
+
+    var toppick
+    var index = -1
+
+    for (let i = 0; i < this._queue.length; i++) {
+      var pended = this._queue[i]
+
+      if (pended.due > due) {
+        break
+      }
+
+      if (pended.tag !== tag) {
+        continue
+      }
+
+      if (!toppick || toppick.priority < pended.priority) {
+        toppick = pended
+        index = i
+      }
+    }
+
+    if (index !== -1) {
+      this._queue.splice(index, 1)
+      this._changedCallback(this._queue)
+    }
+
+    toppick && debug('pick', toppick, this.status, due)
+
+    return toppick
+  }
+
+  cancel(stuff) {
+    const index = this._queue.findIndex(pended => pended.stuff === stuff)
+    index !== -1 && this._queue.splice(index, 1)
+
+    this._changedCallback(this._queue)
+  }
+}

package/server/engine/scenario-engine.ts

@@ -0,0 +1,106 @@
+import { getRepository, GraphqlLocalClient, pubsub } from '@things-factory/shell'
+
+import { Scenario } from '../service/scenario/scenario'
+import { ScenarioInstance } from '../service/scenario-instance/scenario-instance-type'
+
+import { PendingQueue } from './pending-queue'
+
+const debug = require('debug')('things-factory:integration-base:scenario-engine')
+
+export class ScenarioEngine {
+  private static scenarioInstances = {} // TODO impl. by WeakMap
+  private static pendingQueues = {}
+
+  public static getPendingQueue(domain) {
+    if (!ScenarioEngine.pendingQueues[domain.id]) {
+      ScenarioEngine.pendingQueues[domain.id] = new PendingQueue(queue => {
+        pubsub.publish('scenario-queue-state', {
+          scenarioQueueState: {
+            domain,
+            queue
+          }
+        })
+      })
+    }
+
+    return ScenarioEngine.pendingQueues[domain.id]
+  }
+
+  public static getScenarioInstance(domain, instanceName): ScenarioInstance {
+    return ScenarioEngine.scenarioInstances[domain.id]?.[instanceName]
+  }
+
+  public static getScenarioInstances(domain, scenarioName?): ScenarioInstance[] {
+    const instances: ScenarioInstance[] = Object.values(ScenarioEngine.scenarioInstances[domain.id] || {}) || []
+    return instances.filter(instance => !scenarioName || instance.scenarioName == scenarioName)
+  }
+
+  public static async load(instanceName, scenarioConfig, context?) {
+    const { domain: scenarioDomain } = scenarioConfig
+    const domain = context?.domain || scenarioDomain
+
+    debug('load', domain.subdomain, instanceName)
+
+    var scenarioInstances = ScenarioEngine.scenarioInstances[domain.id]
+    if (!scenarioInstances) {
+      scenarioInstances = ScenarioEngine.scenarioInstances[domain.id] = {}
+    }
+
+    var scenarioInstance = scenarioInstances[instanceName]
+    if (scenarioInstance) {
+      debug('load - duplicated', instanceName)
+      return scenarioInstance
+    }
+
+    var instance = new ScenarioInstance(instanceName, scenarioConfig, {
+      ...context,
+      client: GraphqlLocalClient.client,
+      disposer: () => {
+        this.unload(domain, instanceName)
+      }
+    })
+
+    instance.start().catch(err => {
+      /* IMPORTANT. if you didn't catch exception here, system will be exit */
+      console.error(
+        `An exception occurred while executing the scenario instance named "${instanceName}" on the domain "${domain.name}"`,
+        err
+      )
+    })
+
+    scenarioInstances[instanceName] = instance
+
+    return instance
+  }
+
+  public static async unload(domain, instanceName) {
+    var scenarioInstances = ScenarioEngine.scenarioInstances[domain.id]
+    if (!scenarioInstances) {
+      return
+    }
+
+    var instance = scenarioInstances[instanceName]
+    if (!instance) {
+      return
+    }
+
+    delete scenarioInstances[instanceName]
+
+    instance.unload()
+  }
+
+  // will be deprecated soon
+  public static async loadAll() {
+    const SCENARIOS = await getRepository(Scenario).find({
+      where: { active: true },
+      relations: ['domain', 'steps']
+    })
+
+    debug(
+      'load-all',
+      SCENARIOS.map(scenario => `${scenario.domain.subdomain}: ${scenario.name}`)
+    )
+
+    SCENARIOS.forEach(scenario => ScenarioEngine.load(scenario.name, scenario))
+  }
+}

package/server/engine/task/book-up-scenario.ts

@@ -0,0 +1,73 @@
+import { access, deepClone } from '@things-factory/utils'
+import { TaskRegistry } from '../task-registry'
+import { substitute } from './utils/substitute'
+import { ScenarioEngine } from '../scenario-engine'
+import { InputStep } from '../../service/step/step-type'
+import { Context } from '../types'
+
+const debug = require('debug')('things-factory:integration-base:book-up-scenarios')
+
+async function BookUpScenario(step: InputStep, { logger, data, domain }: Context) {
+  var {
+    params: { scenario, delay, priority, variables, tag }
+  } = step
+
+  const pendingQueue = ScenarioEngine.getPendingQueue(domain)
+  delay = Number(substitute(delay, data))
+
+  pendingQueue.put({
+    stuff: {
+      scenario,
+      variables: deepClone(access(variables, data))
+    },
+    delay,
+    priority: Number(priority) || 0,
+    tag: tag || ''
+  })
+
+  debug('pending', scenario, delay, tag, priority)
+
+  return {
+    data: pendingQueue.status
+  }
+}
+
+BookUpScenario.parameterSpec = [
+  {
+    type: 'entity-selector',
+    name: 'scenario',
+    label: 'scenario',
+    property: {
+      queryName: 'scenarios'
+    }
+  },
+  {
+    type: 'string',
+    name: 'delay',
+    label: 'delay',
+    placeHolder: 'seconds'
+  },
+  {
+    type: 'select',
+    name: 'priority',
+    label: 'priority',
+    property: {
+      options: ['', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
+    }
+  },
+  {
+    type: 'scenario-step-input',
+    name: 'variables',
+    label: 'variables'
+  },
+  {
+    type: 'string',
+    name: 'tag',
+    label: 'tag'
+  }
+]
+
+BookUpScenario.connectorFree = true
+BookUpScenario.help = 'integration/task/book-up-scenario'
+
+TaskRegistry.registerTaskHandler('book-up-scenario', BookUpScenario)

package/server/engine/task/csv-readline.ts

@@ -0,0 +1,127 @@
+const fs = require('fs')
+const { URL } = require('url')
+const path = require('path')
+const readline = require('readline')
+const { Readable } = require('stream')
+
+import { TaskRegistry } from '../task-registry'
+
+import { InputStep } from '../../service/step/step-type'
+import { Context } from '../types'
+
+async function createReadStream(source) {
+  let stream
+
+  try {
+    const url = new URL(source)
+    const protocol = url.protocol.slice(0, -1) // Remove ':' from protocol
+    const filePath = url.pathname
+
+    switch (protocol) {
+      case 'file':
+        stream = fs.createReadStream(filePath)
+        break
+      case 'http':
+      case 'https':
+        const response = await fetch(source)
+        stream = Readable.from(await response.text())
+        // 아래의 경우는 문제 발생함. /attachment/:attachment로 테스트함 - 동일 서버에서 요청과 응답을 하는 상홍에서의 문제인가 ?
+        // TODO 스트림을 제대로 사용해야 서버의 자원 부담이 덜할텐데.
+        // TODO attachment 서비스가 authorization을 지원하는 상황에서는 인증에서 통과하지 못할 수도 있음.
+        // stream = response.body.getReader()
+        break
+      default:
+        stream = fs.createReadStream(path.resolve(source))
+    }
+  } catch (e) {
+    stream = fs.createReadStream(path.resolve(source))
+  }
+
+  return stream
+}
+
+async function CsvReadline(step: InputStep, context: Context) {
+  var {
+    params: { path }
+  } = step
+
+  var { logger, __csv_resources, closures } = context
+
+  if (!path) {
+    throw new Error('csv filepath should be given')
+  }
+
+  if (!__csv_resources) {
+    __csv_resources = context.__csv_resources = [] as ReadableStream[]
+  }
+
+  if (!__csv_resources[path]) {
+    let stream
+
+    try {
+      stream = await createReadStream(path)
+    } catch (e) {
+      throw new Error(e)
+    }
+
+    if (!stream) {
+      throw new Error('csv file loading failed')
+    }
+
+    const rl = readline.createInterface({
+      input: stream,
+      crlfDelay: Infinity
+    })
+
+    const iterator = rl[Symbol.asyncIterator]()
+    const { value: header } = await iterator.next()
+
+    __csv_resources[path] = {
+      stream,
+      iterator,
+      header: header ? header.split(',').map(h => h.replace(/"/g, '')) : []
+    }
+
+    closures.push(async () => {
+      try {
+        stream && (await stream.close())
+        logger.info(`csv file(${path}) is released before ending the scenario.`)
+        delete __csv_resources[path]
+      } catch (e) {
+        logger.error(e)
+      }
+    })
+  }
+
+  const { iterator, header } = __csv_resources[path]
+
+  const data = {} as { [key: string]: string }
+  const { value: line } = await iterator.next()
+
+  if (!line) {
+    return
+  }
+
+  const lineData = line.split(',')
+
+  for (let i = 0; i < header.length; i++) {
+    data[header[i]] = lineData[i]
+  }
+
+  return {
+    data
+  }
+}
+
+CsvReadline.parameterSpec = [
+  {
+    type: 'string',
+    name: 'path',
+    label: 'path'
+  }
+]
+
+CsvReadline.connectorFree = true
+CsvReadline.help = 'integration/task/csv-readline'
+
+TaskRegistry.registerTaskHandler('csv-readline', CsvReadline)

package/server/engine/task/data-accessor.ts

@@ -0,0 +1,36 @@
+import { access } from '@things-factory/utils'
+import { TaskRegistry } from '../task-registry'
+
+import { InputStep } from '../../service/step/step-type'
+import { Context } from '../types'
+
+async function DataAccessor(step: InputStep, { logger, data }: Context) {
+  var {
+    params: { target, accessor }
+  } = step
+
+  const input = access(target, data)
+  const output = access(accessor, input)
+
+  return {
+    data: output
+  }
+}
+
+DataAccessor.parameterSpec = [
+  {
+    type: 'scenario-step-input',
+    name: 'target',
+    label: 'target'
+  },
+  {
+    type: 'data-accessor',
+    name: 'accessor',
+    label: 'accessor'
+  }
+]
+
+DataAccessor.connectorFree = true
+DataAccessor.help = 'integration/task/data-accessor'
+
+TaskRegistry.registerTaskHandler('data-accessor', DataAccessor)

package/server/engine/task/data-mapper.ts

@@ -0,0 +1,47 @@
+import { access } from '@things-factory/utils'
+import { TaskRegistry } from '../task-registry'
+
+import { InputStep } from '../../service/step/step-type'
+import { Context } from '../types'
+
+function mapping(rule, input) {
+  if (Array.isArray(input)) {
+    return input.map(i => mapping(rule, i))
+  }
+
+  return Object.keys(rule).reduce((sum, key) => {
+    sum[key] = input[rule[key]]
+    return sum
+  }, {} as any)
+}
+
+async function DataMapper(step: InputStep, { logger, data }: Context) {
+  var {
+    params: { accessor, mappingRule }
+  } = step
+
+  const input = access(accessor, data)
+  const output = mapping(mappingRule || {}, input)
+
+  return {
+    data: output
+  }
+}
+
+DataMapper.parameterSpec = [
+  {
+    type: 'scenario-step-input',
+    name: 'accessor',
+    label: 'accessor'
+  },
+  {
+    type: 'key-values',
+    name: 'mappingRule',
+    label: 'mapping-rule'
+  }
+]
+
+DataMapper.connectorFree = true
+DataMapper.help = 'integration/task/data-mapper'
+
+TaskRegistry.registerTaskHandler('data-mapper', DataMapper)