@things-factory/integration-base 8.0.0-beta.1 → 8.0.0-beta.2

package/server/engine/pending-queue.ts

@@ -1,97 +0,0 @@
-const debug = require('debug')('things-factory:integration-base:pending-queue')
-
-export type PendingObject = {
-  stuff
-  due
-  priority
-  tag
-}
-
-export class PendingQueue {
-  private _queue: PendingObject[] = []
-  private _changedCallback
-
-  constructor(changedCallback) {
-    this._changedCallback = changedCallback
-  }
-
-  get queue() {
-    return this._queue
-  }
-
-  get status() {
-    return {
-      total: this._queue.length
-    }
-  }
-
-  reset() {
-    this._queue = []
-    this._changedCallback(this._queue)
-  }
-
-  put({ delay, stuff, priority = 0, tag = '' }) {
-    const due = Date.now() + (Number(delay) || 0) * 1000
-    const insertBefore = this._queue.findIndex(pended => pended.due > due)
-
-    if (insertBefore == -1) {
-      this._queue.push({
-        due,
-        priority: Number(priority) || 0,
-        stuff,
-        tag: tag || ''
-      })
-    } else {
-      this._queue.splice(insertBefore, 0, {
-        due,
-        priority,
-        stuff,
-        tag
-      })
-    }
-
-    this._changedCallback(this._queue)
-
-    debug('put', this.status)
-  }
-
-  pick(tag = '') {
-    const due = Date.now()
-
-    var toppick
-    var index = -1
-
-    for (let i = 0; i < this._queue.length; i++) {
-      var pended = this._queue[i]
-
-      if (pended.due > due) {
-        break
-      }
-
-      if (pended.tag !== tag) {
-        continue
-      }
-
-      if (!toppick || toppick.priority < pended.priority) {
-        toppick = pended
-        index = i
-      }
-    }
-
-    if (index !== -1) {
-      this._queue.splice(index, 1)
-      this._changedCallback(this._queue)
-    }
-
-    toppick && debug('pick', toppick, this.status, due)
-
-    return toppick
-  }
-
-  cancel(stuff) {
-    const index = this._queue.findIndex(pended => pended.stuff === stuff)
-    index !== -1 && this._queue.splice(index, 1)
-
-    this._changedCallback(this._queue)
-  }
-}

package/server/engine/resource-pool/headless-pool.ts

@@ -1,136 +0,0 @@
-import * as genericPool from 'generic-pool'
-import { config, logger } from '@things-factory/env'
-
-try {
-  var puppeteer = require('puppeteer')
-} catch (err) {
-  logger.error(err)
-}
-
-let headlessPool
-
-export function getHeadlessPool() {
-  if (!headlessPool) {
-    headlessPool = createHeadlessPool({ min: 2, max: 20, acquireTimeoutMillis: 15000, testOnBorrow: true })
-  }
-
-  return headlessPool
-}
-
-function createHeadlessPool(options: genericPool.Options) {
-  return genericPool.createPool(
-    {
-      create() {
-        console.log('headless instance in headless-pool-integration about to create')
-        return initializeChromium()
-      },
-      validate(browser) {
-        return Promise.race([
-          new Promise(res => setTimeout(() => res(false), 1500)),
-          browser
-            //@ts-ignore
-            .version()
-            .then(() => true)
-            .catch(() => false)
-        ])
-      },
-      destroy(browser) {
-        //@ts-ignore
-        return browser.close()
-      }
-    } as genericPool.Factory<any>,
-    options
-  )
-}
-
-async function destroyHeadlessPool() {
-  if (headlessPool) {
-    console.log('headless-pool-integration about to destroy')
-
-    try {
-      await Promise.race([
-        headlessPool.drain().then(() => headlessPool.clear()), // 정리 작업
-        new Promise(
-          (_, reject) => setTimeout(() => reject(new Error('Destroy timeout')), 5000) // 5초 타임아웃 설정
-        )
-      ])
-      console.log('Headless pool destroyed')
-    } catch (err) {
-      logger.error('Failed to destroy headless pool:', err)
-    }
-  }
-}
-
-const CHROMIUM_PATH = config.get('CHROMIUM_PATH')
-
-async function initializeChromium() {
-  try {
-    if (!puppeteer) {
-      return
-    }
-
-    const launchSetting = {
-      args: ['--hide-scrollbars', '--mute-audio', '--no-sandbox', '--use-gl=egl'],
-      headless: 'shell'
-    }
-
-    if (CHROMIUM_PATH) {
-      launchSetting['executablePath'] = CHROMIUM_PATH
-    }
-
-    const browser = await puppeteer.launch(launchSetting)
-
-    return browser
-  } catch (err) {
-    logger.error(err)
-  }
-}
-
-// Graceful shutdown logic
-function setupProcessExitHandlers() {
-  let isCleaningUp = false // 중복 정리를 방지
-
-  const cleanup = async () => {
-    if (isCleaningUp) return // 이미 정리 중이면 무시
-    isCleaningUp = true
-
-    console.log('Application is shutting down. Cleaning up resources...')
-    try {
-      // Pool 정리 작업 실행
-      await destroyHeadlessPool()
-    } catch (err) {
-      logger.error('Error during cleanup:', err)
-    } finally {
-      console.log('Cleanup completed.')
-    }
-  }
-
-  const onExit = async (signal?: string) => {
-    console.log(`Received signal: ${signal || 'unknown'}`)
-    await cleanup()
-
-    // 다른 핸들러가 실행될 수 있도록 exit 호출을 지연
-    process.nextTick(() => process.exit(0)) // Tick 뒤에 종료
-  }
-
-  // Handle termination signals
-  process.once('SIGINT', () => onExit('SIGINT')) // Ctrl+C
-  process.once('SIGTERM', () => onExit('SIGTERM')) // Termination signal
-
-  // Handle uncaught exceptions
-  process.once('uncaughtException', async err => {
-    logger.error('Uncaught Exception:', err)
-    await cleanup()
-    process.nextTick(() => process.exit(1)) // Tick 뒤에 종료
-  })
-
-  // Handle unhandled promise rejections
-  process.once('unhandledRejection', async reason => {
-    logger.error('Unhandled Rejection:', reason)
-    await cleanup()
-    process.nextTick(() => process.exit(1)) // Tick 뒤에 종료
-  })
-}
-
-// Initialize process exit handlers
-setupProcessExitHandlers()

package/server/engine/resource-pool/index.ts

@@ -1 +0,0 @@
-export * from './headless-pool'

package/server/engine/scenario-engine.ts

@@ -1,106 +0,0 @@
-import { getRepository, GraphqlLocalClient, pubsub } from '@things-factory/shell'
-
-import { Scenario } from '../service/scenario/scenario'
-import { ScenarioInstance } from '../service/scenario-instance/scenario-instance-type'
-
-import { PendingQueue } from './pending-queue'
-
-const debug = require('debug')('things-factory:integration-base:scenario-engine')
-
-export class ScenarioEngine {
-  private static scenarioInstances = {} // TODO impl. by WeakMap
-  private static pendingQueues = {}
-
-  public static getPendingQueue(domain) {
-    if (!ScenarioEngine.pendingQueues[domain.id]) {
-      ScenarioEngine.pendingQueues[domain.id] = new PendingQueue(queue => {
-        pubsub.publish('scenario-queue-state', {
-          scenarioQueueState: {
-            domain,
-            queue
-          }
-        })
-      })
-    }
-
-    return ScenarioEngine.pendingQueues[domain.id]
-  }
-
-  public static getScenarioInstance(domain, instanceName): ScenarioInstance {
-    return ScenarioEngine.scenarioInstances[domain.id]?.[instanceName]
-  }
-
-  public static getScenarioInstances(domain, scenarioName?): ScenarioInstance[] {
-    const instances: ScenarioInstance[] = Object.values(ScenarioEngine.scenarioInstances[domain.id] || {}) || []
-    return instances.filter(instance => !scenarioName || instance.scenarioName == scenarioName)
-  }
-
-  public static async load(instanceName, scenarioConfig, context?) {
-    const { domain: scenarioDomain } = scenarioConfig
-    const domain = context?.domain || scenarioDomain
-
-    debug('load', domain.subdomain, instanceName)
-
-    var scenarioInstances = ScenarioEngine.scenarioInstances[domain.id]
-    if (!scenarioInstances) {
-      scenarioInstances = ScenarioEngine.scenarioInstances[domain.id] = {}
-    }
-
-    var scenarioInstance = scenarioInstances[instanceName]
-    if (scenarioInstance) {
-      debug('load - duplicated', instanceName)
-      return scenarioInstance
-    }
-
-    var instance = new ScenarioInstance(instanceName, scenarioConfig, {
-      ...context,
-      client: GraphqlLocalClient.client,
-      disposer: () => {
-        this.unload(domain, instanceName)
-      }
-    })
-
-    instance.start().catch(err => {
-      /* IMPORTANT. if you didn't catch exception here, system will be exit */
-      console.error(
-        `An exception occurred while executing the scenario instance named "${instanceName}" on the domain "${domain.name}"`,
-        err
-      )
-    })
-
-    scenarioInstances[instanceName] = instance
-
-    return instance
-  }
-
-  public static async unload(domain, instanceName) {
-    var scenarioInstances = ScenarioEngine.scenarioInstances[domain.id]
-    if (!scenarioInstances) {
-      return
-    }
-
-    var instance = scenarioInstances[instanceName]
-    if (!instance) {
-      return
-    }
-
-    delete scenarioInstances[instanceName]
-
-    instance.unload()
-  }
-
-  // will be deprecated soon
-  public static async loadAll() {
-    const SCENARIOS = await getRepository(Scenario).find({
-      where: { active: true },
-      relations: ['domain', 'steps']
-    })
-
-    debug(
-      'load-all',
-      SCENARIOS.map(scenario => `${scenario.domain.subdomain}: ${scenario.name}`)
-    )
-
-    SCENARIOS.forEach(scenario => ScenarioEngine.load(scenario.name, scenario))
-  }
-}

package/server/engine/task/book-up-scenario.ts

@@ -1,73 +0,0 @@
-import { access, deepClone } from '@things-factory/utils'
-import { TaskRegistry } from '../task-registry'
-import { substitute } from './utils/substitute'
-import { ScenarioEngine } from '../scenario-engine'
-import { InputStep } from '../../service/step/step-type'
-import { Context } from '../types'
-
-const debug = require('debug')('things-factory:integration-base:book-up-scenarios')
-
-async function BookUpScenario(step: InputStep, { logger, data, domain }: Context) {
-  var {
-    params: { scenario, delay, priority, variables, tag }
-  } = step
-
-  const pendingQueue = ScenarioEngine.getPendingQueue(domain)
-  delay = Number(substitute(delay, data))
-
-  pendingQueue.put({
-    stuff: {
-      scenario,
-      variables: deepClone(access(variables, data))
-    },
-    delay,
-    priority: Number(priority) || 0,
-    tag: tag || ''
-  })
-
-  debug('pending', scenario, delay, tag, priority)
-
-  return {
-    data: pendingQueue.status
-  }
-}
-
-BookUpScenario.parameterSpec = [
-  {
-    type: 'entity-selector',
-    name: 'scenario',
-    label: 'scenario',
-    property: {
-      queryName: 'scenarios'
-    }
-  },
-  {
-    type: 'string',
-    name: 'delay',
-    label: 'delay',
-    placeHolder: 'seconds'
-  },
-  {
-    type: 'select',
-    name: 'priority',
-    label: 'priority',
-    property: {
-      options: ['', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
-    }
-  },
-  {
-    type: 'scenario-step-input',
-    name: 'variables',
-    label: 'variables'
-  },
-  {
-    type: 'string',
-    name: 'tag',
-    label: 'tag'
-  }
-]
-
-BookUpScenario.connectorFree = true
-BookUpScenario.help = 'integration/task/book-up-scenario'
-
-TaskRegistry.registerTaskHandler('book-up-scenario', BookUpScenario)

package/server/engine/task/csv-readline.ts

@@ -1,127 +0,0 @@
-const fs = require('fs')
-const { URL } = require('url')
-const path = require('path')
-const readline = require('readline')
-const { Readable } = require('stream')
-
-import { TaskRegistry } from '../task-registry'
-
-import { InputStep } from '../../service/step/step-type'
-import { Context } from '../types'
-
-async function createReadStream(source) {
-  let stream
-
-  try {
-    const url = new URL(source)
-    const protocol = url.protocol.slice(0, -1) // Remove ':' from protocol
-    const filePath = url.pathname
-
-    switch (protocol) {
-      case 'file':
-        stream = fs.createReadStream(filePath)
-        break
-      case 'http':
-      case 'https':
-        const response = await fetch(source)
-        stream = Readable.from(await response.text())
-        // 아래의 경우는 문제 발생함. /attachment/:attachment로 테스트함 - 동일 서버에서 요청과 응답을 하는 상홍에서의 문제인가 ?
-        // TODO 스트림을 제대로 사용해야 서버의 자원 부담이 덜할텐데.
-        // TODO attachment 서비스가 authorization을 지원하는 상황에서는 인증에서 통과하지 못할 수도 있음.
-        // stream = response.body.getReader()
-        break
-      default:
-        stream = fs.createReadStream(path.resolve(source))
-    }
-  } catch (e) {
-    stream = fs.createReadStream(path.resolve(source))
-  }
-
-  return stream
-}
-
-async function CsvReadline(step: InputStep, context: Context) {
-  var {
-    params: { path }
-  } = step
-
-  var { logger, __csv_resources, closures } = context
-
-  if (!path) {
-    throw new Error('csv filepath should be given')
-  }
-
-  if (!__csv_resources) {
-    __csv_resources = context.__csv_resources = [] as ReadableStream[]
-  }
-
-  if (!__csv_resources[path]) {
-    let stream
-
-    try {
-      stream = await createReadStream(path)
-    } catch (e) {
-      throw new Error(e)
-    }
-
-    if (!stream) {
-      throw new Error('csv file loading failed')
-    }
-
-    const rl = readline.createInterface({
-      input: stream,
-      crlfDelay: Infinity
-    })
-
-    const iterator = rl[Symbol.asyncIterator]()
-    const { value: header } = await iterator.next()
-
-    __csv_resources[path] = {
-      stream,
-      iterator,
-      header: header ? header.split(',').map(h => h.replace(/"/g, '')) : []
-    }
-
-    closures.push(async () => {
-      try {
-        stream && (await stream.close())
-        logger.info(`csv file(${path}) is released before ending the scenario.`)
-        delete __csv_resources[path]
-      } catch (e) {
-        logger.error(e)
-      }
-    })
-  }
-
-  const { iterator, header } = __csv_resources[path]
-
-  const data = {} as { [key: string]: string }
-  const { value: line } = await iterator.next()
-
-  if (!line) {
-    return
-  }
-
-  const lineData = line.split(',')
-
-  for (let i = 0; i < header.length; i++) {
-    data[header[i]] = lineData[i]
-  }
-
-  return {
-    data
-  }
-}
-
-CsvReadline.parameterSpec = [
-  {
-    type: 'string',
-    name: 'path',
-    label: 'path'
-  }
-]
-
-CsvReadline.connectorFree = true
-CsvReadline.help = 'integration/task/csv-readline'
-
-TaskRegistry.registerTaskHandler('csv-readline', CsvReadline)

package/server/engine/task/data-accessor.ts

@@ -1,36 +0,0 @@
-import { access } from '@things-factory/utils'
-import { TaskRegistry } from '../task-registry'
-
-import { InputStep } from '../../service/step/step-type'
-import { Context } from '../types'
-
-async function DataAccessor(step: InputStep, { logger, data }: Context) {
-  var {
-    params: { target, accessor }
-  } = step
-
-  const input = access(target, data)
-  const output = access(accessor, input)
-
-  return {
-    data: output
-  }
-}
-
-DataAccessor.parameterSpec = [
-  {
-    type: 'scenario-step-input',
-    name: 'target',
-    label: 'target'
-  },
-  {
-    type: 'data-accessor',
-    name: 'accessor',
-    label: 'accessor'
-  }
-]
-
-DataAccessor.connectorFree = true
-DataAccessor.help = 'integration/task/data-accessor'
-
-TaskRegistry.registerTaskHandler('data-accessor', DataAccessor)

package/server/engine/task/data-mapper.ts

@@ -1,47 +0,0 @@
-import { access } from '@things-factory/utils'
-import { TaskRegistry } from '../task-registry'
-
-import { InputStep } from '../../service/step/step-type'
-import { Context } from '../types'
-
-function mapping(rule, input) {
-  if (Array.isArray(input)) {
-    return input.map(i => mapping(rule, i))
-  }
-
-  return Object.keys(rule).reduce((sum, key) => {
-    sum[key] = input[rule[key]]
-    return sum
-  }, {} as any)
-}
-
-async function DataMapper(step: InputStep, { logger, data }: Context) {
-  var {
-    params: { accessor, mappingRule }
-  } = step
-
-  const input = access(accessor, data)
-  const output = mapping(mappingRule || {}, input)
-
-  return {
-    data: output
-  }
-}
-
-DataMapper.parameterSpec = [
-  {
-    type: 'scenario-step-input',
-    name: 'accessor',
-    label: 'accessor'
-  },
-  {
-    type: 'key-values',
-    name: 'mappingRule',
-    label: 'mapping-rule'
-  }
-]
-
-DataMapper.connectorFree = true
-DataMapper.help = 'integration/task/data-mapper'
-
-TaskRegistry.registerTaskHandler('data-mapper', DataMapper)

package/server/engine/task/database-query.ts

@@ -1,56 +0,0 @@
-import { ConnectionManager } from '../connection-manager'
-import { TaskRegistry } from '../task-registry'
-import { InputStep } from '../../service/step/step-type'
-import { Context } from '../types'
-import 'ses'
-
-async function DatabaseQuery(step: InputStep, context: Context) {
-  var { domain, user, data, variables, lng } = context
-  var {
-    connection: connectionName,
-    params: { query }
-  } = step
-
-  var dbconnection = ConnectionManager.getConnectionInstanceByName(domain, connectionName)
-
-  const compartment = new Compartment({
-    domain,
-    user,
-    lng,
-    data,
-    variables,
-    console
-  })
-
-  let evalQuery
-  try {
-    evalQuery = compartment.evaluate('`' + query + '`')
-  } catch (err) {
-    throw new Error(`Failed to evaluate query: ${err.message}`)
-  }
-
-  var queryResult = await dbconnection.query(evalQuery, [])
-
-  return {
-    data: queryResult
-  }
-}
-
-DatabaseQuery.parameterSpec = [
-  {
-    type: 'textarea',
-    name: 'query',
-    label: 'query',
-    property: {
-      language: 'sql',
-      showLineNumbers: true
-    },
-    styles: {
-      flex: '1'
-    }
-  }
-]
-
-DatabaseQuery.help = 'integration/task/database-query'
-
-TaskRegistry.registerTaskHandler('database-query', DatabaseQuery)

package/server/engine/task/echo-receive.ts

@@ -1,21 +0,0 @@
-import { TaskRegistry } from '../task-registry'
-import { ConnectionManager } from '../connection-manager'
-
-import { InputStep } from '../../service/step/step-type'
-import { Context } from '../types'
-
-async function EchoReceive(step: InputStep, { logger, domain }: Context) {
-  var { connection: connectionName } = step
-
-  var connection = ConnectionManager.getConnectionInstanceByName(domain, connectionName)
-
-  var message = await connection.read()
-
-  return {
-    data: message.toString()
-  }
-}
-
-EchoReceive.parameterSpec = []
-
-TaskRegistry.registerTaskHandler('echo-receive', EchoReceive)