@things-factory/integration-base 8.0.0-beta.9 → 8.0.2

package/server/engine/connector/operato-connector.ts

@@ -0,0 +1,264 @@
+import 'cross-fetch/polyfill'
+
+import { ApolloClient, InMemoryCache, createHttpLink, split } from '@apollo/client/core'
+import { setContext } from '@apollo/client/link/context'
+
+import WebSocket from 'ws'
+import { createClient } from 'graphql-ws'
+import { GraphQLWsLink } from '@apollo/client/link/subscriptions'
+import { getMainDefinition } from '@apollo/client/utilities'
+import gql from 'graphql-tag'
+
+import { ConnectionManager } from '../connection-manager'
+import { Connector } from '../types'
+import { InputConnection } from '../../service/connection/connection-type'
+
+import { Scenario } from '../../service/scenario/scenario'
+import { ScenarioInstance } from '../../service/scenario-instance/scenario-instance-type'
+
+import { getRepository, GraphqlLocalClient, Domain } from '@things-factory/shell'
+import { User, checkUserHasRole } from '@things-factory/auth-base'
+
+const debug = require('debug')('things-factory:integration-base:operato-connector')
+
+const defaultOptions: any = {
+  watchQuery: {
+    fetchPolicy: 'no-cache',
+    errorPolicy: 'ignore'
+  },
+  query: {
+    fetchPolicy: 'no-cache', //'network-only'
+    errorPolicy: 'all'
+  },
+  mutate: {
+    errorPolicy: 'all'
+  }
+}
+
+export const GRAPHQL_URI = '/graphql'
+export const SUBSCRIPTION_URI = GRAPHQL_URI
+
+interface SubscriberData {
+  tag: string
+  scenario: any
+  subscriptionObserver: any
+}
+
+export class OperatoConnector implements Connector {
+  private context: any
+
+  async ready(connectionConfigs: InputConnection[]) {
+    await Promise.all(connectionConfigs.map(this.connect.bind(this)))
+
+    ConnectionManager.logger.info('operato-connector connections are ready')
+  }
+
+  async connect(connection: InputConnection) {
+    const {
+      endpoint: uri,
+      params: { authKey, domain, subscriptionHandlers = {} }
+    } = connection
+
+    if (!authKey || !domain) {
+      throw new Error('some connection paramter missing.')
+    }
+
+    const domainOwner = await getRepository(User).findOne({
+      where: {
+        id: connection.domain.owner
+      }
+    })
+
+    this.context = {
+      domain: connection.domain,
+      user: domainOwner
+      /* TODO: domainOwner 대신 특정 유저를 지정할 수 있도록 개선해야함. 모든 커넥션에 유저를 지정하는 기능으로 일반화할 필요가 있는 지 고민해야함 */
+    }
+
+    const httpLink = createHttpLink({
+      uri: uri
+    })
+
+    /* 
+    CHECKPOINT: 
+      1. GraphqQLWsLink를 사용하면 setContext를 통한 추가 헤더 설정이 무시됩니다.
+      따라서, GraphQLWsLink를 사용하려면, connectionParams를 통해 헤더를 설정해야 합니다.
+    
+      2. 서버에서 실행시, webSocketImpl을 명시적으로 지정해야 합니다.
+    */
+    const wsLink = new GraphQLWsLink(
+      createClient({
+        url: uri.replace(/^http/, 'ws'),
+        keepAlive: 10_000,
+        retryAttempts: 1_000_000,
+        shouldRetry: e => true,
+        webSocketImpl: WebSocket,
+        connectionParams: {
+          headers: {
+            'x-things-factory-domain': domain,
+            authorization: authKey ? `Bearer ${authKey}` : ''
+          }
+        }
+      })
+    )
+
+    const splitLink = split(
+      ({ query }) => {
+        const def = getMainDefinition(query)
+        return def.kind === 'OperationDefinition' && def.operation === 'subscription'
+      },
+      wsLink,
+      setContext((_, { headers }) => {
+        return {
+          headers: {
+            ...headers,
+            'x-things-factory-domain': domain,
+            authorization: authKey ? `Bearer ${authKey}` : ''
+          }
+        }
+      }).concat(httpLink)
+    )
+
+    const cache = new InMemoryCache({
+      addTypename: false
+    })
+
+    const client = new ApolloClient({
+      defaultOptions,
+      cache,
+      link: splitLink
+    })
+
+    const subscriptions: SubscriberData[] = []
+    Object.keys(subscriptionHandlers).forEach(async tag => {
+      if (!tag || !subscriptionHandlers[tag]) return
+
+      const scenarioName = subscriptionHandlers[tag]
+
+      // fetch a scenario
+      const selectedScenario = await getRepository(Scenario).findOne({
+        where: {
+          name: scenarioName
+        },
+        relations: ['steps', 'domain']
+      })
+
+      const subscription = client.subscribe({
+        query: gql`
+          subscription {
+            data(tag: "${tag}") {
+              tag
+              data
+            }
+          }
+        `
+      })
+
+      const subscriptionObserver = subscription.subscribe({
+        next: async data => {
+          debug('received pubsub msg.:', data?.data)
+          await this.runScenario(subscriptions, data?.data?.data)
+        },
+        error: error => {
+          ConnectionManager.logger.error(`(${connection.name}:${connection.endpoint}) subscription error`, error)
+        },
+        complete: () => {
+          ConnectionManager.logger.info(`(${connection.name}:${connection.endpoint}) subscription complete`)
+        }
+      })
+
+      ConnectionManager.logger.info(
+        `(${connection.name}:${connection.endpoint}) subscription closed flag: ${subscriptionObserver.closed}`
+      )
+
+      subscriptions.push({
+        tag,
+        scenario: selectedScenario,
+        subscriptionObserver
+      })
+      ConnectionManager.logger.info(`(${tag}:${scenarioName}) subscription closed flag: ${subscriptionObserver.closed}`)
+    })
+
+    client['subscriptions'] = subscriptions
+    ConnectionManager.addConnectionInstance(connection, client)
+
+    ConnectionManager.logger.info(
+      `operato-connector connection(${connection.name}:${connection.endpoint}) is connected`
+    )
+  }
+
+  async disconnect(connection: InputConnection) {
+    const client = ConnectionManager.getConnectionInstance(connection)
+    const subscriptions: SubscriberData[] = client['subscriptions']
+    subscriptions.forEach(subscription => subscription.subscriptionObserver.unsubscribe())
+    client.stop()
+    ConnectionManager.removeConnectionInstance(connection)
+
+    ConnectionManager.logger.info(`operato-connector connection(${connection.name}) is disconnected`)
+  }
+
+  async runScenario(subscriptions: SubscriberData[], variables: any): Promise<ScenarioInstance> {
+    const { domain, user } = this.context
+    const { tag } = variables
+
+    if (!tag) {
+      throw new Error(`tag is invalid - ${tag}`)
+    }
+
+    const scenario = subscriptions.find(subscription => subscription.tag === tag)?.scenario
+    if (!scenario) {
+      throw new Error(`scenario is not found - ${tag}`)
+    }
+
+    if (!(await checkUserHasRole(scenario.roleId, domain, user))) {
+      throw new Error(`Unauthorized! ${scenario.name} doesn't have required role.`)
+    }
+
+    /* create a scenario instance */
+    const instanceName = scenario.name + '-' + String(Date.now())
+    const instance = new ScenarioInstance(instanceName, scenario, {
+      user,
+      domain,
+      variables,
+      client: GraphqlLocalClient.client
+    })
+
+    // run scenario
+    await instance.run()
+    return instance
+  }
+
+  get parameterSpec() {
+    return [
+      {
+        type: 'string',
+        name: 'authKey',
+        label: 'auth-key'
+      },
+      {
+        type: 'string',
+        name: 'domain',
+        label: 'domain'
+      },
+      {
+        type: 'tag-scenarios',
+        name: 'subscriptionHandlers',
+        label: 'subscription-handlers'
+      }
+    ]
+  }
+
+  get taskPrefixes() {
+    return ['graphql']
+  }
+
+  get help() {
+    return 'integration/connector/operato-connector'
+  }
+
+  get description() {
+    return 'Operato Graphql Connector'
+  }
+}
+
+ConnectionManager.registerConnector('operato-connector', new OperatoConnector())

package/server/engine/connector/oracle-connector.ts

@@ -0,0 +1,218 @@
+import { logger } from '@things-factory/env'
+
+import { ConnectionManager } from '../connection-manager'
+import { Connector } from '../types'
+import { InputConnection } from '../../service/connection/connection-type'
+
+try {
+  var oracledb = require('oracledb')
+} catch (err) {
+  logger.error('oracledb module loading failed', err)
+}
+
+export class OracleConnector implements Connector {
+  async ready(connectionConfigs: InputConnection[]) {
+    await Promise.all(connectionConfigs.map(this.connect))
+
+    ConnectionManager.logger.info('oracle-connector connections are ready')
+  }
+
+  async recreatePool(connection: InputConnection) {
+    const {
+      name,
+      endpoint,
+      params: { user, password, database, poolMin, poolMax, poolIncrement },
+      domain
+    } = connection
+
+    if (!oracledb) {
+      throw new Error('oracledb module loading failed')
+    }
+
+    const poolAlias = `${domain.name}-${name}`
+    await oracledb.getPool(poolAlias).close(10)
+    await oracledb.createPool({
+      user,
+      password,
+      // when oracle not using default port must add connection string with port like localhost:port
+      connectString: `${endpoint.trim()}/${database}`,
+      poolMin,
+      poolMax,
+      poolIncrement,
+      poolAlias
+    })
+
+    ConnectionManager.logger.info(`Oracle Database(${connection.name}:${database}) at ${endpoint} recreated.`)
+  }
+
+  async connect(connection: InputConnection) {
+    const {
+      name,
+      endpoint,
+      params: { user, password, database, poolMin, poolMax, poolIncrement },
+      domain
+    } = connection
+
+    if (!oracledb) {
+      throw new Error('oracledb module loading failed')
+    }
+
+    const poolAlias = `${domain.name}-${name}`
+
+    var enableStatistics = true
+    const pool = await oracledb.createPool({
+      user,
+      password,
+      // when oracle not using default port must add connection string with port like localhost:port
+      connectString: `${endpoint.trim()}/${database}`,
+      poolMin,
+      poolMax,
+      poolIncrement,
+      poolAlias,
+      enableStatistics
+    })
+
+    ConnectionManager.addConnectionInstance(connection, {
+      query: async (query, params) => {
+        let dbConnection: any = {}
+        let taskResult: any = {}
+        try {
+          dbConnection = await oracledb.getConnection(poolAlias)
+
+          taskResult = (
+            await dbConnection.execute(query, params, {
+              outFormat: oracledb.OBJECT
+            })
+          ).rows
+        } catch (e) {
+          if (e.name === 'Error' && e.message.includes('NJS-040')) {
+            await this.recreatePool(connection)
+          }
+          throw e
+        } finally {
+          await dbConnection.close()
+        }
+        return taskResult
+      },
+      execute: async (procedure, params) => {
+        let dbConnection: any = {}
+        let taskResult: any = {}
+        try {
+          // TODO: need to check if this is available when procedure string is a common query.
+          procedure = `BEGIN
+            ${procedure}
+          END;`
+          dbConnection = await oracledb.getConnection(poolAlias)
+
+          // console.log('\n************* stat after get ****************')
+          // await oracledb.getPool(poolAlias).logStatistics()
+
+          const result = await dbConnection.execute(procedure, params, {
+            outFormat: oracledb.OBJECT
+          })
+
+          let paramKeys = Object.keys(params)
+
+          for (const paramKey of paramKeys) {
+            if (params[paramKey].dir === oracledb?.BIND_OUT) {
+              if (params[paramKey].type === oracledb?.CURSOR) {
+                const resultSetTemp = result.outBinds[paramKey]
+                taskResult[paramKey] = await resultSetTemp.getRows()
+                await resultSetTemp.close()
+              } else {
+                taskResult[paramKey] = result.outBinds[paramKey]
+              }
+            }
+          }
+        } catch (e) {
+          if (e.name === 'Error' && e.message.includes('NJS-040')) {
+            await this.recreatePool(connection)
+          }
+          throw e
+        } finally {
+          await dbConnection.close()
+
+          // console.log('\n************* stat after close ****************')
+          // await oracledb.getPool(poolAlias).logStatistics()
+        }
+        return taskResult
+      },
+      close: async () => {
+        await oracledb.getPool(poolAlias).close(10)
+      }
+    })
+
+    ConnectionManager.logger.info(`Oracle Database(${connection.name}:${database}) at ${endpoint} connected.`)
+  }
+
+  async disconnect(connection: InputConnection) {
+    var client = ConnectionManager.getConnectionInstance(connection)
+
+    try {
+      await client.close()
+      ConnectionManager.logger.info(`Oracle Database(${connection.name}) closed.`)
+    } catch (e) {
+      ConnectionManager.logger.error(e)
+    }
+
+    ConnectionManager.removeConnectionInstance(connection)
+  }
+
+  get parameterSpec() {
+    return [
+      {
+        type: 'string',
+        name: 'user',
+        label: 'user'
+      },
+      {
+        type: 'password',
+        name: 'password',
+        label: 'password'
+      },
+      {
+        type: 'string',
+        name: 'database',
+        placeholder: 'SID',
+        label: 'database'
+      },
+      {
+        type: 'number',
+        name: 'poolMin',
+        placeholder: 'minimum connection-pool size',
+        label: 'pool-min',
+        value: 0
+      },
+      {
+        type: 'number',
+        name: 'poolMax',
+        placeholder: 'maximum connection-pool size',
+        label: 'pool-max',
+        value: 4
+      },
+      {
+        type: 'number',
+        name: 'poolIncrement',
+        placeholder: 'connection incremental size',
+        label: 'pool-increment',
+        value: 1
+      }
+    ]
+  }
+
+  get taskPrefixes() {
+    return ['database', 'oracle']
+  }
+
+  get help() {
+    return 'integration/connector/oracle-connector'
+  }
+}
+
+ConnectionManager.registerConnector('oracle-connector', new OracleConnector())
+
+// need reference:
+// https://download.oracle.com/otn_software/mac/instantclient/193000/instantclient-basiclite-macos.x64-19.3.0.0.0dbru.zip
+// https://node-oracledb.readthedocs.io/en/latest/index.html
+
+// docker pull store/oracle/database-instantclient:12.2.0.1

package/server/engine/connector/postgresql-connector.ts

@@ -0,0 +1,152 @@
+import { logger } from '@things-factory/env'
+
+import { ConnectionManager } from '../connection-manager'
+import { Connector } from '../types'
+import { InputConnection } from '../../service/connection/connection-type'
+
+try {
+  var { Client, Pool } = require('pg')
+} catch (err) {
+  logger.error('postgresql module loading failed', err)
+}
+
+export class PostgresqlConnector implements Connector {
+  async ready(connectionConfigs: InputConnection[]) {
+    await Promise.all(connectionConfigs.map(this.connect.bind(this)))
+
+    ConnectionManager.logger.info('postgresql-connector connections are ready')
+  }
+
+  async connect(connection: InputConnection) {
+    const {
+      endpoint,
+      params: { user, password, database, maxPoolConnection, idleTimeoutMillis, connectionTimeoutMillis }
+    } = connection
+    const [host, port = 5432] = endpoint.split(':')
+
+    if (!Pool) {
+      throw new Error('postgresql module loading failed')
+    }
+
+    // create a new pool
+    const pool = new Pool({
+      user,
+      host,
+      database,
+      password,
+      port: Number(port),
+      max: maxPoolConnection || 10,
+      idleTimeoutMillis: idleTimeoutMillis || 30000,
+      connectionTimeoutMillis: connectionTimeoutMillis || 2000
+    })
+
+    pool.on('error', (err, client) => {
+      logger.error('unexpected error on postgres pool.', err)
+    })
+
+    try {
+      // try to connect to database to check the connection at initial time
+      const entryConnection = await pool.connect().catch(async e => {
+        ConnectionManager.logger.error(e)
+        throw e
+      })
+      entryConnection.release()
+
+      ConnectionManager.addConnectionInstance(connection, {
+        query: async (query, params) => {
+          try {
+            // get a connection from the pool
+            var client = await pool.connect().catch(async e => {
+              ConnectionManager.logger.error(e)
+              throw e
+            })
+
+            client?.on('error', async err => {
+              logger.error('unexpected error on postgres client.', err)
+              ConnectionManager.logger.error(err)
+            })
+
+            // try to query
+            var result = (await client.query(query, params)).rows
+          } catch (e) {
+            ConnectionManager.logger.error(e)
+            throw e
+          } finally {
+            // release the connection back to the pool
+            client?.release()
+          }
+          return result
+        },
+        close: async () => {
+          // close the pool
+          await pool?.end()
+        }
+      })
+
+      ConnectionManager.logger.info(`PostgresSQL Database(${connection.name}:${database}) at ${endpoint} connected.`)
+    } catch (e) {
+      ConnectionManager.logger.error(
+        `PostgresSQL Database(${connection.name}:${database}) at ${endpoint} not connected.`,
+        e
+      )
+    }
+  }
+
+  async disconnect(connection: InputConnection) {
+    var connectionInstance = ConnectionManager.getConnectionInstance(connection)
+    try {
+      await connectionInstance.close()
+      ConnectionManager.logger.info(`PostgresSQL Database(${connection.name}) closed.`)
+    } catch (e) {
+      ConnectionManager.logger.error(e)
+    }
+    ConnectionManager.removeConnectionInstance(connection)
+  }
+
+  get parameterSpec() {
+    return [
+      {
+        type: 'string',
+        name: 'user',
+        label: 'user'
+      },
+      {
+        type: 'password',
+        name: 'password',
+        label: 'password'
+      },
+      {
+        type: 'string',
+        name: 'database',
+        label: 'database'
+      },
+      {
+        type: 'number',
+        name: 'maxPoolConnection',
+        label: 'maxPoolConnection'
+      },
+      {
+        type: 'number',
+        name: 'idleTimeoutMillis',
+        label: 'idleTimeoutMillis',
+        placeHolder: 'milli-seconds'
+      },
+      {
+        type: 'number',
+        name: 'connectionTimeoutMillis',
+        label: 'connectionTimeoutMillis',
+        placeHolder: 'milli-seconds'
+      }
+    ]
+  }
+
+  get taskPrefixes() {
+    return ['database']
+  }
+
+  get help() {
+    return 'integration/connector/postgresql-connector'
+  }
+}
+
+ConnectionManager.registerConnector('postgresql-connector', new PostgresqlConnector())

package/server/engine/connector/proxy-connector.ts

@@ -0,0 +1,53 @@
+import { ConnectionManager } from '../connection-manager'
+import { Connector } from '../types'
+import { InputConnection } from '../../service/connection/connection-type'
+import { connectConnections, disconnectConnections } from '../edge-client'
+
+/**
+ * This connector is a proxy connector installed on the edge server to manage connections from the host.
+ * It interacts with connections established on the edge server and provides synchronization functionality.
+ */
+export class ProxyConnector implements Connector {
+  public static instance = new ProxyConnector()
+
+  async ready(connectionConfigs: InputConnection[]) {
+    await Promise.all(connectionConfigs.map(this.connect.bind(this)))
+
+    ConnectionManager.logger.info('proxy-connector connections are ready')
+  }
+
+  async connect(connection: InputConnection) {
+    // TODO 원래 커넥션과 에지설정을 참고하여, 에지 서버로 CONNECT/DISCONNECT 명령을 보낸다.
+    const proxy = {
+      disconnect: async () => {
+        ConnectionManager.logger.info('[proxy-connector] trying disconnect')
+        await disconnectConnections([connection])
+      }
+    }
+
+    await connectConnections([connection])
+
+    ConnectionManager.addConnectionInstance(connection, proxy)
+
+    ConnectionManager.logger.info(`proxy-connector connection(${connection.name}:${connection.endpoint}) is connected`)
+  }
+
+  async disconnect(connection: InputConnection) {
+    const proxy = ConnectionManager.removeConnectionInstance(connection)
+    await proxy.disconnect()
+
+    ConnectionManager.logger.info(`proxy-connector connection(${connection.name}) is disconnected`)
+  }
+
+  get parameterSpec() {
+    return []
+  }
+
+  get taskPrefixes() {
+    return []
+  }
+
+  get description() {
+    return 'Operato Proxy Connector'
+  }
+}