@things-factory/integration-base 8.0.0 → 9.0.0-beta.3

package/server/engine/connection-manager.ts

@@ -1,239 +0,0 @@
-import moment from 'moment-timezone'
-import { createLogger, format, transports } from 'winston'
-
-import { Domain, getRepository, pubsub, PubSubLogTransport } from '@things-factory/shell'
-
-import { Connection, ConnectionStatus } from '../service'
-import { Connector } from './types'
-import { ProxyConnector } from './connector/proxy-connector'
-
-const { combine, splat, printf, errors } = format
-const debug = require('debug')('things-factory:integration-base:connections')
-
-function getSystemTimeZone() {
-  try {
-    const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone
-    if (!timeZone) {
-      throw new Error('Unable to resolve timeZone')
-    }
-    return timeZone
-  } catch (e) {
-    console.warn('Failed to get system timeZone, falling back to UTC.', e)
-    return 'UTC'
-  }
-}
-
-const SYSTEM_TZ = getSystemTimeZone()
-const systemTimestamp = format((info, opts: { tz?: string }) => {
-  if (opts.tz) info.timestamp = moment().tz(opts.tz).format()
-  return info
-})
-
-export class ConnectionManager {
-  private static connectors: { [propName: string]: Connector } = {}
-  private static connections: { [domainId: string]: { [name: string]: any } } = {}
-  private static entities = {}
-  private static logFormat = printf(({ level, message, timestamp, stack }) => {
-    return `${timestamp} ${level}: ${stack || message}`
-  })
-
-  public static logger = createLogger({
-    format: combine(errors({ stack: true }), systemTimestamp({ tz: SYSTEM_TZ }), splat(), ConnectionManager.logFormat),
-    transports: [
-      new (transports as any).DailyRotateFile({
-        filename: `logs/connections-%DATE%.log`,
-        datePattern: 'YYYY-MM-DD-HH',
-        zippedArchive: false,
-        maxSize: '20m',
-        maxFiles: '14d',
-        level: 'info'
-      }),
-      new PubSubLogTransport({
-        topic: 'connection-log'
-      })
-    ]
-  })
-
-  static async ready() {
-    const CONNECTIONS = (
-      await getRepository(Connection).find({
-        where: { active: true },
-        relations: ['domain', 'edge', 'creator', 'updater']
-      })
-    ).map(connection => {
-      var params = {}
-      try {
-        params = JSON.parse(connection.params || '{}')
-      } catch (ex) {
-        ConnectionManager.logger.error(`connection '${connection.name}' params should be JSON format`, ex)
-      }
-
-      return {
-        ...connection,
-        params
-      }
-    })
-
-    ConnectionManager.logger.info('Initializing ConnectionManager...')
-
-    return await Promise.all(
-      [...Object.keys(ConnectionManager.connectors), 'proxy-connector'].map(type => {
-        const connector = type == 'proxy-connector' ? ProxyConnector.instance : ConnectionManager.getConnector(type)
-
-        ConnectionManager.logger.info(`Connector '${type}' started to ready`)
-
-        return connector
-          .ready(
-            CONNECTIONS.filter(connection => {
-              if (type == 'proxy-connector') {
-                return !!connection.edge
-              } else {
-                return !connection.edge && connection.type == type
-              }
-            }) as any
-          )
-          .catch(error => {
-            ConnectionManager.logger.error(error)
-          })
-          .then(() => {
-            ConnectionManager.logger.info(`All connector for '${type}' ready`)
-          })
-      })
-    ).then(() => {
-      ConnectionManager.logger.info('ConnectionManager initialization done:')
-      Object.keys(ConnectionManager.connections).forEach(key => {
-        var connections = ConnectionManager.connections[key]
-        ConnectionManager.logger.info('For domain(%s) : %s', key, JSON.stringify(Object.keys(connections)))
-      })
-    })
-  }
-
-  static registerConnector(type: string, connector: Connector) {
-    ConnectionManager.connectors[type] = connector
-  }
-
-  static getConnector(type: string): Connector {
-    return ConnectionManager.connectors[type]
-  }
-
-  static getConnectors(): { [connectorName: string]: Connector } {
-    return {
-      ...ConnectionManager.connectors
-    }
-  }
-
-  static unregisterConnector(type: string) {
-    delete ConnectionManager.connectors[type]
-  }
-
-  static getConnections() {
-    return ConnectionManager.connections
-  }
-
-  static getEntities() {
-    return ConnectionManager.entities
-  }
-
-  static getConnectionInstance(connection: Connection): any {
-    const { domain, name } = connection
-    return ConnectionManager.connections[domain.id]?.[name]
-  }
-
-  static getConnectionInstanceByName(domain: Domain, name: string) {
-    const connections = ConnectionManager.connections[domain.id]
-    const connection = connections?.[name]
-
-    if (!connection) {
-      throw `The connection with the given name(${name}) cannot be found`
-    }
-
-    return connection
-  }
-
-  static getConnectionInstanceEntityByName(domain: Domain, name: string): any {
-    const connection = ConnectionManager.entities[domain.id]?.[name]
-    if (connection) {
-      return connection
-    }
-
-    if (domain.parentId) {
-      return ConnectionManager.entities[domain.id]?.[name]
-    }
-  }
-
-  static getConnectionInstances(domain: Domain): { [connectionName: string]: any } {
-    const connections = ConnectionManager.connections[domain.id]
-    const parentConnections = domain.parentId && ConnectionManager.connections[domain.parentId]
-
-    return {
-      ...parentConnections,
-      ...connections
-    }
-  }
-
-  static getConnectionInstanceEntities(domain: Domain): { [connectionName: string]: any } {
-    const connections = ConnectionManager.entities[domain.id]
-
-    return {
-      ...connections
-    }
-  }
-
-  static addConnectionInstance(connection: Connection, instance: any) {
-    const { domain, name } = connection
-
-    var connections = ConnectionManager.connections[domain.id]
-    if (!connections) {
-      connections = ConnectionManager.connections[domain.id] = {}
-    }
-
-    var entities = ConnectionManager.entities[domain.id]
-    if (!entities) {
-      entities = ConnectionManager.entities[domain.id] = {}
-    }
-
-    connections[name] = instance
-    entities[name] = connection
-
-    ConnectionManager.publishState(connection, ConnectionStatus.CONNECTED)
-    debug('add-connection', domain.subdomain, name)
-  }
-
-  static removeConnectionInstance(connection: Connection): any {
-    const { domain, name } = connection
-    var connections = ConnectionManager.connections[domain.id]
-    var entities = ConnectionManager.entities[domain.id]
-
-    var instance = connections?.[name]
-
-    if (!connections || !instance) {
-      debug('remove-connection', `'${name}' connection not found in domain '${domain.subdomain}'`)
-      return
-    }
-
-    delete connections[name]
-    delete entities[name]
-
-    ConnectionManager.publishState(connection, ConnectionStatus.DISCONNECTED)
-    debug('remove-connection', `'${name}' connection is removed from domain '${domain.subdomain}'`)
-
-    return instance
-  }
-
-  private static async publishState(connection: Connection, state) {
-    const { domain, id, name, description, type, edge } = connection
-
-    pubsub.publish('connection-state', {
-      connectionState: {
-        domain,
-        id,
-        name,
-        description,
-        type,
-        edge,
-        state,
-        timestamp: new Date()
-      }
-    })
-  }
-}

package/server/engine/connector/echo-back-connector.ts

@@ -1,51 +0,0 @@
-import net from 'net'
-import PromiseSocket from 'promise-socket'
-
-import { Connector } from '../types'
-import { ConnectionManager } from '../connection-manager'
-import { InputConnection } from '../../service/connection/connection-type'
-
-export class EchoBack implements Connector {
-  async ready(connectionConfigs) {
-    await Promise.all(connectionConfigs.map(this.connect.bind(this)))
-
-    ConnectionManager.logger.info('echo-back connections are ready')
-  }
-
-  async connect(connection: InputConnection) {
-    let socket = new PromiseSocket(new net.Socket())
-    let [host, port = 8124] = connection.endpoint.split(':')
-
-    try {
-      await socket.connect(Number(port), host)
-      ConnectionManager.addConnectionInstance(connection, socket)
-
-      ConnectionManager.logger.info(
-        `echo-back-connector connection(${connection.name}:${connection.endpoint}) is connected`
-      )
-    } catch (e) {
-      ConnectionManager.logger.error(
-        `echo-back-connector connection(${connection.name}:${connection.endpoint}) is not connected.`,
-        e
-      )
-    }
-  }
-
-  async disconnect(connection: InputConnection) {
-    let socket = ConnectionManager.removeConnectionInstance(connection)
-
-    await socket.destroy()
-
-    ConnectionManager.logger.info(`echo-back-connector connection(${connection.name}) is disconnected`)
-  }
-
-  get parameterSpec() {
-    return []
-  }
-
-  get taskPrefixes() {
-    return ['echo']
-  }
-}
-
-ConnectionManager.registerConnector('echo-back', new EchoBack())

package/server/engine/connector/echo-back-server.ts

@@ -1,72 +0,0 @@
-import net from 'net'
-import PromiseSocket from 'promise-socket'
-
-import { Connector } from '../types'
-import { ConnectionManager } from '../connection-manager'
-import { InputConnection } from '../../service/connection/connection-type'
-
-export class EchoBack implements Connector {
-  async ready(connectionConfigs: InputConnection[]) {
-    await Promise.all(connectionConfigs.map(this.connect.bind(this)))
-
-    ConnectionManager.logger.info('echo-back-servers are ready')
-  }
-
-  async connect(config: InputConnection): Promise<void> {
-    var [host = '0.0.0.0', port = 8124] = config.endpoint.split(':')
-
-    return new Promise((resolve, reject) => {
-      var server = net.createServer(socket => {
-        socket.on('data', function (data) {
-          socket.write(data.toString())
-        })
-
-        socket.on('error', function (err) {
-          ConnectionManager.logger.error(err)
-          reject(err)
-        })
-      })
-
-      server.listen(port, async () => {
-        ConnectionManager.logger.info(`Echo-back server listening on ${host}:${port}`)
-
-        /* default client connection */
-        let socket = new net.Socket()
-        socket.on('error', console.error)
-
-        try {
-          let promiseSocket = new PromiseSocket(socket)
-          await promiseSocket.connect(Number(port), 'localhost')
-          promiseSocket['__server__'] = server
-
-          ConnectionManager.addConnectionInstance(config, promiseSocket)
-
-          ConnectionManager.logger.info(`echo-back-server connection(${config.name}:${config.endpoint}) is connected`)
-
-          resolve()
-        } catch (err) {
-          ConnectionManager.logger.error(
-            `echo-back-server connection(${config.name}:${config.endpoint}) is connected.\ncause: ${err}`
-          )
-          reject(err)
-        }
-      })
-    })
-  }
-
-  async disconnect(connection: InputConnection) {
-    let socket = ConnectionManager.removeConnectionInstance(connection)
-    var server = socket['__server__']
-
-    await socket.destroy()
-    server && (await server.close())
-
-    ConnectionManager.logger.info(`echo-back-server connection(${connection.name}) is disconnected`)
-  }
-
-  get parameterSpec() {
-    return []
-  }
-}
-
-ConnectionManager.registerConnector('echo-back-server', new EchoBack())

package/server/engine/connector/graphql-connector.ts

@@ -1,126 +0,0 @@
-import 'cross-fetch/polyfill'
-
-import { ApolloClient, ApolloLink, from, HttpLink, InMemoryCache } from '@apollo/client/core'
-import { onError } from '@apollo/client/link/error'
-import { Oauth2Client } from '@things-factory/oauth2-client'
-import { getRepository } from '@things-factory/shell'
-
-import { ConnectionManager } from '../connection-manager'
-import { Connector } from '../types'
-import { InputConnection } from '../../service/connection/connection-type'
-
-const defaultOptions: any = {
-  watchQuery: {
-    fetchPolicy: 'no-cache',
-    errorPolicy: 'ignore'
-  },
-  query: {
-    fetchPolicy: 'no-cache', //'network-only'
-    errorPolicy: 'all'
-  },
-  mutate: {
-    errorPolicy: 'all'
-  }
-}
-
-export class GraphqlConnector implements Connector {
-  async ready(connectionConfigs: InputConnection[]) {
-    await Promise.all(connectionConfigs.map(this.connect.bind(this)))
-
-    ConnectionManager.logger.info('graphql-connector connections are ready')
-  }
-
-  async connect(connection: InputConnection) {
-    const {
-      endpoint: uri,
-      params: { authClient }
-    } = connection
-
-    const ERROR_HANDLER: any = ({ graphQLErrors, networkError }) => {
-      if (graphQLErrors)
-        graphQLErrors.map(({ message, locations, path }) => {
-          ConnectionManager.logger.error(
-            `[GraphQL error] Message: ${message}, Location: ${locations}, Path: ${path}`,
-            graphQLErrors
-          )
-        })
-
-      if (networkError) {
-        ConnectionManager.logger.error(`[Network error - ${networkError.statusCode}]`, networkError)
-      }
-    }
-
-    const oauth2Client: Oauth2Client = await getRepository(Oauth2Client).findOneBy({ id: authClient })
-
-    const authMiddleware = new ApolloLink((operation, forward) => {
-      // add the authorization to the headers
-      operation.setContext(({ headers = {} }) => ({
-        headers: {
-          ...headers,
-          ...oauth2Client.getAuthHeaders()
-        }
-      }))
-
-      return forward(operation)
-    })
-
-    const cache = new InMemoryCache({
-      addTypename: false
-    })
-
-    ConnectionManager.addConnectionInstance(
-      connection,
-      new ApolloClient({
-        defaultOptions,
-        cache,
-        link: from([
-          authMiddleware,
-          onError(ERROR_HANDLER),
-          new HttpLink({
-            uri,
-            credentials: 'include'
-          })
-        ])
-      })
-    )
-
-    ConnectionManager.logger.info(
-      `graphql-connector connection(${connection.name}:${connection.endpoint}) is connected`
-    )
-  }
-
-  async disconnect(connection: InputConnection) {
-    const client = ConnectionManager.getConnectionInstance(connection)
-    client.stop()
-    ConnectionManager.removeConnectionInstance(connection)
-
-    ConnectionManager.logger.info(`graphql-connector connection(${connection.name}) is disconnected`)
-  }
-
-  get parameterSpec() {
-    return [
-      {
-        type: 'entity-selector',
-        name: 'authClient',
-        label: 'auth-client',
-        property: {
-          queryName: 'oauth2Clients'
-        }
-      }
-    ]
-  }
-
-  get taskPrefixes() {
-    return ['graphql']
-  }
-
-  get description() {
-    return 'Graphql Http Connector'
-  }
-
-  get help() {
-    return 'integration/connector/graphql-connector'
-  }
-}
-
-ConnectionManager.registerConnector('graphql-connector', new GraphqlConnector())

package/server/engine/connector/http-connector.ts

@@ -1,65 +0,0 @@
-import { Oauth2Client } from '@things-factory/oauth2-client'
-import { getRepository } from '@things-factory/shell'
-
-import { ConnectionManager } from '../connection-manager'
-import { Connector } from '../types'
-import { InputConnection } from '../../service/connection/connection-type'
-
-export class HttpConnector implements Connector {
-  async ready(connectionConfigs: InputConnection[]) {
-    await Promise.all(connectionConfigs.map(this.connect.bind(this)))
-
-    ConnectionManager.logger.info('http-connector connections are ready')
-  }
-
-  async connect(connection: InputConnection) {
-    var { params } = connection
-    params.rejectUnauthorized = (params.rejectUnauthorized || 'Y') === 'Y'
-
-    if (params.authClient) {
-      const oauth2Client: Oauth2Client = await getRepository(Oauth2Client).findOneBy({ id: params.authClient })
-      var authHeaders = oauth2Client.getAuthHeaders()
-    }
-
-    ConnectionManager.addConnectionInstance(connection, {
-      ...connection,
-      authHeaders: authHeaders || {},
-      params
-    })
-
-    ConnectionManager.logger.info(`http-connector connection(${connection.name}:${connection.endpoint}) is connected`)
-  }
-
-  async disconnect(connection: InputConnection) {
-    ConnectionManager.removeConnectionInstance(connection)
-
-    ConnectionManager.logger.info(`http-connector connection(${connection.name}) is disconnected`)
-  }
-
-  get parameterSpec() {
-    return [
-      {
-        type: 'entity-selector',
-        name: 'authClient',
-        label: 'auth-client',
-        property: {
-          queryName: 'oauth2Clients'
-        }
-      },
-      {
-        type: 'select',
-        name: 'rejectUnauthorized',
-        label: 'reject-unauthorized',
-        property: {
-          options: ['Y', 'N']
-        }
-      }
-    ]
-  }
-
-  get taskPrefixes() {
-    return ['http', 'headless-post', 'headless-scrap']
-  }
-}
-
-ConnectionManager.registerConnector('http-connector', new HttpConnector())

package/server/engine/connector/index.ts

@@ -1,12 +0,0 @@
-import './echo-back-server'
-import './echo-back-connector'
-import './http-connector'
-import './graphql-connector'
-import './sqlite-connector'
-import './postgresql-connector'
-import './mqtt-connector'
-import './mssql-connector'
-import './oracle-connector'
-import './mysql-connector'
-import './socket-server'
-import './operato-connector'

package/server/engine/connector/mqtt-connector.ts

@@ -1,78 +0,0 @@
-import { Connector } from '../types'
-import { ConnectionManager } from '../connection-manager'
-import { InputConnection } from '../../service/connection/connection-type'
-
-import mqtt from 'async-mqtt'
-
-export class MqttConnector implements Connector {
-  async ready(connectionConfigs: InputConnection[]) {
-    await Promise.all(connectionConfigs.map(this.connect.bind(this)))
-
-    ConnectionManager.logger.info('mqtt-connector connections are ready')
-  }
-
-  async connect(connection: InputConnection) {
-    const {
-      endpoint: uri,
-      params: { user, password }
-    } = connection
-
-    try {
-      var client = null
-      if (user && password) {
-        client = await mqtt.connectAsync(uri, { username: user, password: password })
-      } else {
-        client = await mqtt.connectAsync(uri)
-      }
-
-      if (!client) {
-        throw new Error(
-          `the initialization of mqtt connector is failed. uri: ${uri}, user: ${user} password: ${password}`
-        )
-      }
-
-      ConnectionManager.addConnectionInstance(connection, {
-        client,
-        connection
-      })
-
-      ConnectionManager.logger.info(`mqtt-connector connection(${connection.name}:${connection.endpoint}) is connected`)
-    } catch (err) {
-      ConnectionManager.logger.error(
-        `mqtt-connector connection(${connection.name}:${connection.endpoint}) is failed.`,
-        err
-      )
-    }
-  }
-
-  async disconnect(connection: InputConnection) {
-    const { client } = ConnectionManager.removeConnectionInstance(connection)
-
-    client && (await client.end())
-  }
-
-  get parameterSpec() {
-    return [
-      {
-        type: 'string',
-        name: 'user',
-        label: 'user'
-      },
-      {
-        type: 'password',
-        name: 'password',
-        label: 'password'
-      }
-    ]
-  }
-
-  get taskPrefixes() {
-    return ['mqtt']
-  }
-
-  get help() {
-    return 'integration/connector/mqtt-connector'
-  }
-}
-
-ConnectionManager.registerConnector('mqtt-connector', new MqttConnector())