@things-factory/integration-base 8.0.0-alpha.30 → 8.0.0-alpha.34

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@things-factory/integration-base",
-  "version": "8.0.0-alpha.30",
+  "version": "8.0.0-alpha.34",
   "main": "dist-server/index.js",
   "browser": "client/index.js",
   "things-factory": true,
@@ -32,7 +32,7 @@
     "@things-factory/cache-service": "^8.0.0-alpha.29",
     "@things-factory/env": "^8.0.0-alpha.8",
     "@things-factory/oauth2-client": "^8.0.0-alpha.29",
-    "@things-factory/scheduler-client": "^8.0.0-alpha.29",
+    "@things-factory/scheduler-client": "^8.0.0-alpha.31",
     "@things-factory/shell": "^8.0.0-alpha.29",
     "@things-factory/utils": "^8.0.0-alpha.29",
     "async-mqtt": "^2.5.0",
@@ -41,8 +41,9 @@
     "ieee754": "^1.2.1",
     "node-fetch": "^2.6.0",
     "promise-socket": "^7.0.0",
+    "puppeteer": "^23.0.0",
     "readline": "^1.3.0",
     "ses": "^1.5.0"
   },
-  "gitHead": "f01fac46630cf06e1697edc185dfb4cddcd61365"
+  "gitHead": "0d43b419fc888429ed0bfa56c82b2456c2e74056"
 }

package/server/engine/connector/headless-connector.ts

@@ -0,0 +1,341 @@
+import { ConnectionManager } from '../connection-manager'
+import { Connector } from '../types'
+import { getHeadlessPool } from '../resource-pool/headless-pool'
+import { Browser, Page } from 'puppeteer'
+
+/*
+  Functionality of the headless-connector:
+  - Provides a mechanism to acquire an active session page.
+  - Performs login when necessary to obtain valid cookies.
+    - Applies these cookies to the page for session management.
+  - Pages are acquired from the `headlessPool`, which manages browser instances.
+  - During the login process, pages from the pool are used for actions like form filling and navigation.
+  - Valid cookies are saved after login and reused for subsequent page acquisitions.
+  - Users must explicitly release the page after use through the `releasePage` method.
+  - Released pages are returned to the `headlessPool` for reuse.
+*/
+
+export class HeadlessConnector implements Connector {
+  async ready(connectionConfigs) {
+    await Promise.all(connectionConfigs.map(this.connect.bind(this)))
+    ConnectionManager.logger.info('headless-connector connections are ready')
+  }
+
+  async connect(connection) {
+    const {
+      endpoint: uri = '1',
+      params: {
+        username = '',
+        password = '',
+        loginPath = '/login',
+        usernameSelector = '#username',
+        passwordSelector = '#password',
+        submitSelector = '#submit',
+        successSelector = null,
+        shadowDomSelectors = '', // Comma separated shadow DOM selectors
+        timeout = 15000, // Default timeout for operations
+        retries = 3 // Default number of retries for login or page actions
+      } = {}
+    } = connection
+
+    const loginInfo = {
+      loginRequired: Boolean(username), // Determine if login is required
+      username,
+      password,
+      loginPath,
+      timeout,
+      retries,
+      loginSelectors: {
+        usernameSelector,
+        passwordSelector,
+        submitSelector,
+        successSelector,
+        shadowDomSelectors: shadowDomSelectors
+          .split(',')
+          .map(selector => selector.trim())
+          .filter(Boolean)
+      }
+    }
+
+    async function acquireBrowser() {
+      try {
+        const pool = getHeadlessPool()
+        const browser = await pool.acquire()
+        return browser
+      } catch (error) {
+        ConnectionManager.logger.error('Failed to acquire browser:', error)
+        throw error
+      }
+    }
+
+    async function releaseBrowser(browser: Browser) {
+      try {
+        const pool = getHeadlessPool()
+        await pool.release(browser)
+      } catch (error) {
+        ConnectionManager.logger.error('Failed to release browser:', error)
+      }
+    }
+
+    ConnectionManager.addConnectionInstance(connection, {
+      endpoint: connection.endpoint,
+      params: connection.params,
+      acquireSessionPage: async () => {
+        const browser = await acquireBrowser()
+        let page
+        let cookies = connection.cookies
+
+        try {
+          page = await browser.newPage()
+
+          page.on('console', async msg => {
+            console.log(`[browser ${msg.type()}] ${msg.text()}`)
+          })
+
+          await page.setRequestInterception(true)
+
+          // page.on('request', request => {
+          //   const resourceType = request.resourceType()
+          //   if (resourceType === 'image' || resourceType === 'stylesheet' || resourceType === 'font') {
+          //     request.abort() // 이미지, 스타일시트, 폰트 요청 차단
+          //   } else {
+          //     request.continue() // 나머지 요청은 진행
+          //   }
+          // })
+
+          page.on('requestfailed', request => {
+            console.log('Request failed:')
+            console.log(`- URL: ${request.url()}`)
+            console.log(`- Method: ${request.method()}`)
+            console.log(`- Failure Text: ${request.failure()?.errorText}`)
+            console.log(`- Headers:`, request.headers())
+
+            // POST 데이터 (필요한 경우)
+            if (request.postData()) {
+              console.log(`- Post Data: ${request.postData()}`)
+            }
+          })
+
+          if (cookies && isCookieValid(cookies)) {
+            await this.applyCookiesAndVerifySession(page, cookies, loginInfo)
+            return page
+          }
+
+          if (loginInfo.loginRequired) {
+            await this.performLogin(page, uri, loginInfo)
+            cookies = await page.cookies()
+            connection.cookies = cookies
+          }
+
+          return page
+        } catch (error) {
+          ConnectionManager.logger.error('Failed to acquire session page:', error)
+          throw error
+        }
+      },
+      releasePage: async (page: Page) => {
+        try {
+          if (page) {
+            const browser = page.browser()
+            await page.close()
+            await releaseBrowser(browser)
+          }
+        } catch (error) {
+          ConnectionManager.logger.error('Failed to release page:', error)
+        }
+      },
+      acquireBrowser,
+      releaseBrowser
+    })
+
+    ConnectionManager.logger.info(
+      `headless-connector connection(${connection.name}:${connection.endpoint}) is connected`
+    )
+  }
+
+  async applyCookiesAndVerifySession(page, cookies, loginInfo) {
+    await page.setCookie(...cookies)
+    await page.reload({ waitUntil: 'networkidle2', timeout: loginInfo.timeout })
+
+    if (loginInfo.loginRequired && loginInfo.loginSelectors.successSelector) {
+      const success = await this.resolveShadowDom(
+        page,
+        loginInfo.loginSelectors.shadowDomSelectors,
+        loginInfo.loginSelectors.successSelector
+      )
+      if (!success) {
+        throw new Error('Session invalid, login required')
+      }
+    }
+  }
+
+  async performLogin(page, uri, loginInfo) {
+    for (let attempt = 1; attempt <= loginInfo.retries; attempt++) {
+      try {
+        await page.goto(`${uri}${loginInfo.loginPath}`, { waitUntil: 'networkidle2', timeout: loginInfo.timeout })
+
+        const usernameInput = await this.resolveShadowDom(
+          page,
+          loginInfo.loginSelectors.shadowDomSelectors,
+          loginInfo.loginSelectors.usernameSelector
+        )
+        const passwordInput = await this.resolveShadowDom(
+          page,
+          loginInfo.loginSelectors.shadowDomSelectors,
+          loginInfo.loginSelectors.passwordSelector
+        )
+        const submitButton = await this.resolveShadowDom(
+          page,
+          loginInfo.loginSelectors.shadowDomSelectors,
+          loginInfo.loginSelectors.submitSelector
+        )
+
+        if (!usernameInput || !passwordInput || !submitButton) {
+          throw new Error('Failed to locate input elements in shadow DOM')
+        }
+
+        await usernameInput.type(loginInfo.username)
+        await passwordInput.type(loginInfo.password)
+
+        // Capture the response of the form submission
+        const [response] = await Promise.all([
+          page.waitForNavigation({ waitUntil: 'networkidle2', timeout: loginInfo.timeout }),
+          submitButton.click()
+        ])
+
+        // Check response status code
+        if (response) {
+          const status = response.status()
+          if (status >= 200 && status < 300) {
+            ConnectionManager.logger.info(`Login successful with status code: ${status}`)
+            return // Login successful
+          } else if (status >= 400) {
+            throw new Error(`Login failed with status code: ${status}`)
+          }
+        } else {
+          throw new Error('No response received during login')
+        }
+
+        if (loginInfo.loginSelectors.successSelector) {
+          const success = await this.resolveShadowDom(
+            page,
+            loginInfo.loginSelectors.shadowDomSelectors,
+            loginInfo.loginSelectors.successSelector
+          )
+          if (!success) {
+            throw new Error('Login failed: Success selector not found')
+          }
+        }
+      } catch (error) {
+        ConnectionManager.logger.warn(`Login attempt ${attempt} failed:`, error)
+
+        if (attempt === loginInfo.retries) {
+          throw new Error(`Login failed after ${loginInfo.retries} attempts`)
+        }
+      }
+    }
+  }
+
+  async resolveShadowDom(page, shadowSelectors, targetSelector) {
+    let context
+
+    if (!shadowSelectors || shadowSelectors.length === 0) {
+      // No Shadow DOM path; use document root as the context
+      context = page.mainFrame() // Puppeteer uses frames to represent document
+      return context.$(targetSelector) // Search directly in the document root
+    }
+
+    context = page // Start with the page as the context
+    for (const selector of shadowSelectors) {
+      const shadowHost = await context.$(selector)
+      if (!shadowHost) {
+        throw new Error(`Shadow host not found: ${selector}`)
+      }
+      context = await page.evaluateHandle(host => host.shadowRoot, shadowHost)
+    }
+    return context.evaluateHandle((shadowRoot, selector) => shadowRoot.querySelector(selector), targetSelector)
+  }
+
+  async disconnect(connection) {
+    const connectionInstance = ConnectionManager.getConnectionInstance(connection)
+    ConnectionManager.removeConnectionInstance(connection)
+    ConnectionManager.logger.info(`headless-connector connection(${connection.name}) is disconnected`)
+  }
+
+  get parameterSpec() {
+    return [
+      {
+        type: 'string',
+        name: 'username',
+        label: 'username'
+      },
+      {
+        type: 'password',
+        name: 'password',
+        label: 'password'
+      },
+      {
+        type: 'string',
+        name: 'loginPath',
+        label: 'login-path'
+      },
+      {
+        type: 'string',
+        name: 'usernameSelector',
+        label: 'username-selector'
+      },
+      {
+        type: 'string',
+        name: 'passwordSelector',
+        label: 'password-selector'
+      },
+      {
+        type: 'string',
+        name: 'submitSelector',
+        label: 'submit-selector'
+      },
+      {
+        type: 'string',
+        name: 'successSelector',
+        label: 'success-selector'
+      },
+      {
+        type: 'string',
+        name: 'shadowDomSelectors',
+        label: 'shadow-dom-selectors'
+      },
+      {
+        type: 'number',
+        name: 'timeout',
+        label: 'timeout',
+        value: 15000
+      },
+      {
+        type: 'number',
+        name: 'retries',
+        label: 'maximum-retries',
+        value: 3
+      }
+    ]
+  }
+
+  get taskPrefixes() {
+    return ['headless']
+  }
+
+  get description() {
+    return 'Headless Pool Connector with login capabilities'
+  }
+
+  get help() {
+    return 'integration/connector/headless-connector'
+  }
+}
+
+ConnectionManager.registerConnector('headless-connector', new HeadlessConnector())
+
+function isCookieValid(cookies) {
+  if (!cookies || cookies.length === 0) return false
+  const now = Date.now() / 1000 // Current time in seconds
+  return cookies.some(cookie => cookie.expires && cookie.expires > now)
+}

package/server/engine/connector/http-connector.ts

@@ -58,7 +58,7 @@ export class HttpConnector implements Connector {
   }
 
   get taskPrefixes() {
-    return ['http', 'headless-post', 'headless-scrap']
+    return ['http']
   }
 }
 

package/server/engine/connector/index.ts

@@ -10,3 +10,4 @@ import './oracle-connector'
 import './mysql-connector'
 import './socket-server'
 import './operato-connector'
+import './headless-connector'

package/server/engine/index.ts

@@ -6,5 +6,6 @@ export * from './scenario-engine'
 export * from './task-registry'
 export * from './analyzer/analyze-integration'
 export * from './edge-client'
+export * from './resource-pool'
 
 export { Connector, Context, TaskHandler } from './types'

package/server/engine/resource-pool/headless-pool.ts

@@ -0,0 +1,136 @@
+import * as genericPool from 'generic-pool'
+import { config, logger } from '@things-factory/env'
+
+try {
+  var puppeteer = require('puppeteer')
+} catch (err) {
+  logger.error(err)
+}
+
+let headlessPool
+
+export function getHeadlessPool() {
+  if (!headlessPool) {
+    headlessPool = createHeadlessPool({ min: 2, max: 20, acquireTimeoutMillis: 15000, testOnBorrow: true })
+  }
+
+  return headlessPool
+}
+
+function createHeadlessPool(options: genericPool.Options) {
+  return genericPool.createPool(
+    {
+      create() {
+        console.log('headless instance in headless-pool-integration about to create')
+        return initializeChromium()
+      },
+      validate(browser) {
+        return Promise.race([
+          new Promise(res => setTimeout(() => res(false), 1500)),
+          browser
+            //@ts-ignore
+            .version()
+            .then(() => true)
+            .catch(() => false)
+        ])
+      },
+      destroy(browser) {
+        //@ts-ignore
+        return browser.close()
+      }
+    } as genericPool.Factory<any>,
+    options
+  )
+}
+
+async function destroyHeadlessPool() {
+  if (headlessPool) {
+    console.log('headless-pool-integration about to destroy')
+
+    try {
+      await Promise.race([
+        headlessPool.drain().then(() => headlessPool.clear()), // 정리 작업
+        new Promise(
+          (_, reject) => setTimeout(() => reject(new Error('Destroy timeout')), 5000) // 5초 타임아웃 설정
+        )
+      ])
+      console.log('Headless pool destroyed')
+    } catch (err) {
+      logger.error('Failed to destroy headless pool:', err)
+    }
+  }
+}
+
+const CHROMIUM_PATH = config.get('CHROMIUM_PATH')
+
+async function initializeChromium() {
+  try {
+    if (!puppeteer) {
+      return
+    }
+
+    const launchSetting = {
+      args: ['--hide-scrollbars', '--mute-audio', '--no-sandbox', '--use-gl=egl'],
+      headless: 'shell'
+    }
+
+    if (CHROMIUM_PATH) {
+      launchSetting['executablePath'] = CHROMIUM_PATH
+    }
+
+    const browser = await puppeteer.launch(launchSetting)
+
+    return browser
+  } catch (err) {
+    logger.error(err)
+  }
+}
+
+// Graceful shutdown logic
+function setupProcessExitHandlers() {
+  let isCleaningUp = false // 중복 정리를 방지
+
+  const cleanup = async () => {
+    if (isCleaningUp) return // 이미 정리 중이면 무시
+    isCleaningUp = true
+
+    console.log('Application is shutting down. Cleaning up resources...')
+    try {
+      // Pool 정리 작업 실행
+      await destroyHeadlessPool()
+    } catch (err) {
+      logger.error('Error during cleanup:', err)
+    } finally {
+      console.log('Cleanup completed.')
+    }
+  }
+
+  const onExit = async (signal?: string) => {
+    console.log(`Received signal: ${signal || 'unknown'}`)
+    await cleanup()
+
+    // 다른 핸들러가 실행될 수 있도록 exit 호출을 지연
+    process.nextTick(() => process.exit(0)) // Tick 뒤에 종료
+  }
+
+  // Handle termination signals
+  process.once('SIGINT', () => onExit('SIGINT')) // Ctrl+C
+  process.once('SIGTERM', () => onExit('SIGTERM')) // Termination signal
+
+  // Handle uncaught exceptions
+  process.once('uncaughtException', async err => {
+    logger.error('Uncaught Exception:', err)
+    await cleanup()
+    process.nextTick(() => process.exit(1)) // Tick 뒤에 종료
+  })
+
+  // Handle unhandled promise rejections
+  process.once('unhandledRejection', async reason => {
+    logger.error('Unhandled Rejection:', reason)
+    await cleanup()
+    process.nextTick(() => process.exit(1)) // Tick 뒤에 종료
+  })
+}
+
+// Initialize process exit handlers
+setupProcessExitHandlers()

package/server/engine/resource-pool/index.ts

@@ -0,0 +1 @@
+export * from './headless-pool'

package/server/engine/task/headless-post.ts

@@ -5,29 +5,23 @@ import { access } from '@things-factory/utils'
 import { TaskRegistry } from '../task-registry'
 import { ConnectionManager } from '../connection-manager'
 
-import { getHeadlessPool } from './utils/headless-pool-for-scenario'
+async function HeadlessPost(step, { logger, data, domain }) {
+  const { connection: connectionName, params: stepOptions } = step
+  const { headers: requestHeaders, contentType, path, accessor } = stepOptions || {}
 
-import { InputStep } from '../../service/step/step-type'
-import { Context } from '../types'
-
-async function HeadlessPost(step: InputStep, { logger, data, domain }: Context) {
-  var { connection: connectionName, params: stepOptions } = step
-  var { headers: requestHeaders, contentType, path, accessor } = stepOptions || {}
-
-  var connection = ConnectionManager.getConnectionInstanceByName(domain, connectionName)
+  const connection = ConnectionManager.getConnectionInstanceByName(domain, connectionName)
 
   if (!connection) {
-    throw new Error(`connection '${connectionName}' is not established.`)
+    throw new Error(`Connection '${connectionName}' is not established.`)
   }
 
-  var { endpoint, params: connectionParams, authHeaders = {} } = connection
+  const { endpoint, params: connectionParams, acquireSessionPage, releasePage } = connection
 
-  var headers = {
-    ...authHeaders,
+  const headers = {
     ...requestHeaders
   }
 
-  var body = access(accessor, data)
+  let body = access(accessor, data)
   if (contentType && body) {
     headers['content-type'] = contentType
     switch (contentType) {
@@ -42,18 +36,18 @@ async function HeadlessPost(step: InputStep, { logger, data, domain }: Context)
         for (const prop in body) {
           searchParams.set(prop, body[prop])
         }
-        body = searchParams
+        body = searchParams.toString()
         break
     }
   }
 
-  var options: any = {
+  const options = {
     method: 'POST',
     headers,
     body
-  }
+  } as any
 
-  var { rejectUnauthorized } = connectionParams
+  const { rejectUnauthorized } = connectionParams
 
   if (!rejectUnauthorized) {
     const httpsAgent = new https.Agent({
@@ -62,8 +56,7 @@ async function HeadlessPost(step: InputStep, { logger, data, domain }: Context)
     options.agent = httpsAgent
   }
 
-  const browser = (await getHeadlessPool().acquire()) as any
-  const page = await browser.newPage()
+  const page = await acquireSessionPage()
 
   try {
     page.on('console', async msg => {
@@ -93,11 +86,11 @@ async function HeadlessPost(step: InputStep, { logger, data, domain }: Context)
     return {
       data: response
     }
-  } catch (e) {
-    console.error(e)
+  } catch (error) {
+    logger.error('Error in HeadlessPost:', error)
+    throw error
   } finally {
-    page.close()
-    getHeadlessPool().release(browser)
+    await releasePage(page)
   }
 }
 
@@ -118,22 +111,10 @@ HeadlessPost.parameterSpec = [
     label: 'content-type',
     property: {
       options: [
-        {
-          display: '',
-          value: ''
-        },
-        {
-          display: 'application/json',
-          value: 'application/json'
-        },
-        {
-          display: 'text/plain',
-          value: 'text/plain'
-        },
-        {
-          display: 'application/x-www-form-urlencoded',
-          value: 'application/x-www-form-urlencoded'
-        }
+        { display: '', value: '' },
+        { display: 'application/json', value: 'application/json' },
+        { display: 'text/plain', value: 'text/plain' },
+        { display: 'application/x-www-form-urlencoded', value: 'application/x-www-form-urlencoded' }
       ]
     }
   },