@things-factory/integration-base 8.0.0-alpha.8 → 8.0.0-beta.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@things-factory/integration-base",
-  "version": "8.0.0-alpha.8",
+  "version": "8.0.0-beta.1",
   "main": "dist-server/index.js",
   "browser": "client/index.js",
   "things-factory": true,
@@ -26,23 +26,24 @@
   },
   "dependencies": {
     "@apollo/client": "^3.6.9",
-    "@operato/moment-timezone-es": "^8.0.0-alpha",
-    "@things-factory/api": "^8.0.0-alpha.8",
-    "@things-factory/auth-base": "^8.0.0-alpha.8",
-    "@things-factory/cache-service": "^8.0.0-alpha.8",
-    "@things-factory/env": "^8.0.0-alpha.8",
-    "@things-factory/oauth2-client": "^8.0.0-alpha.8",
-    "@things-factory/scheduler-client": "^8.0.0-alpha.8",
-    "@things-factory/shell": "^8.0.0-alpha.8",
-    "@things-factory/utils": "^8.0.0-alpha.0",
+    "@operato/moment-timezone-es": "^8.0.0-beta",
+    "@things-factory/api": "^8.0.0-beta.1",
+    "@things-factory/auth-base": "^8.0.0-beta.1",
+    "@things-factory/cache-service": "^8.0.0-beta.1",
+    "@things-factory/env": "^8.0.0-beta.1",
+    "@things-factory/oauth2-client": "^8.0.0-beta.1",
+    "@things-factory/scheduler-client": "^8.0.0-beta.1",
+    "@things-factory/shell": "^8.0.0-beta.1",
+    "@things-factory/utils": "^8.0.0-beta.1",
     "async-mqtt": "^2.5.0",
     "chance": "^1.1.11",
     "cross-fetch": "^3.0.4",
     "ieee754": "^1.2.1",
     "node-fetch": "^2.6.0",
     "promise-socket": "^7.0.0",
+    "puppeteer": "^23.0.0",
     "readline": "^1.3.0",
     "ses": "^1.5.0"
   },
-  "gitHead": "0a9fb3ab431934982294b58c743d01b6f782a15f"
+  "gitHead": "36c494e587640c1490318ef7b95adab02606e0c2"
 }

package/server/controllers/scenario-controller.ts

@@ -1,14 +1,10 @@
 import { getRepository, Domain, GraphqlLocalClient } from '@things-factory/shell'
-import { PrivilegeObject, User, checkPermission } from '@things-factory/auth-base'
+import { checkUserHasRole } from '@things-factory/auth-base'
 import { cacheService } from '@things-factory/cache-service'
 
 import { ScenarioEngine } from '../engine/scenario-engine'
 import { Scenario } from '../service/scenario/scenario'
-import {
-  ScenarioInstance,
-  ScenarioInstanceRunResult,
-  ScenarioInstanceStatus
-} from '../service/scenario-instance/scenario-instance-type'
+import { ScenarioInstance, ScenarioInstanceRunResult } from '../service/scenario-instance/scenario-instance-type'
 import { Step } from '../service/step/step-type'
 
 const debug = require('debug')('things-factory:integration-base:controller:run-scenario')
@@ -16,31 +12,48 @@ const debug = require('debug')('things-factory:integration-base:controller:run-s
 async function findScenario(
   scenarioName: string,
   domain: Domain
-): Promise<{ id: string; name: string; steps: Step[]; domain: Domain; privilege?: PrivilegeObject; ttl?: number }> {
+): Promise<{
+  id: string
+  ttl: number
+  name: string
+  steps: Step[]
+  domain: Domain
+}> {
   var repository = getRepository(Scenario)
 
   var scenario = await repository.findOne({
     where: { domain: { id: domain.id }, name: scenarioName },
-    relations: ['domain', 'steps', 'creator', 'updater']
+    relations: ['domain', 'steps', 'role', 'creator', 'updater']
   })
 
   if (!scenario && domain.parentId) {
     scenario = await repository.findOne({
       where: { domain: { id: domain.parentId }, name: scenarioName },
-      relations: ['domain', 'steps', 'creator', 'updater']
+      relations: ['domain', 'steps', 'role', 'creator', 'updater']
     })
   }
 
   return scenario as any
 }
 
+export async function checkHasRole(scenario: Partial<Scenario>, context: ResolverContext): Promise<void> {
+  const { domain, user } = context.state
+  if (!(await checkUserHasRole(scenario.roleId, domain, user))) {
+    throw new Error(
+      context.t('error.scenario run unauthorized', {
+        scenario: scenario.name
+      })
+    )
+  }
+}
+
 export async function runScenario(
   instanceName: string,
   scenarioName: string,
   variables: any,
   context: ResolverContext
 ): Promise<ScenarioInstanceRunResult> {
-  const { domain, user, lng, unsafeIP, prohibitedPrivileges } = context.state
+  const { domain, user, lng } = context.state
 
   debug('runScenario', scenarioName, instanceName, variables)
 
@@ -54,19 +67,7 @@ export async function runScenario(
     )
   }
 
-  if (!(await checkPermission(scenario.privilege, user, domain, unsafeIP, prohibitedPrivileges))) {
-    const { category, privilege } = scenario.privilege || {}
-
-    console.error(
-      `Unauthorized! ${category && privilege ? category + ':' + privilege + ' privilege' : 'ownership granted'} required`
-    )
-
-    throw new Error(
-      context.t('error.scenario run unauthorized', {
-        scenario: scenarioName
-      })
-    )
-  }
+  await checkHasRole(scenario, context)
 
   if (scenario.ttl > 0) {
     const cachedValue = await cacheService.getFromCache(scenario.id, { domain: domain.id, variables: variables || {} })
@@ -104,7 +105,7 @@ export async function startScenario(
   variables: any,
   context: ResolverContext
 ): Promise<ScenarioInstance> {
-  const { domain, user, lng, unsafeIP, prohibitedPrivileges } = context.state
+  const { domain, user, lng } = context.state
 
   debug('startScenario', instanceName, scenarioName, variables)
 
@@ -118,12 +119,7 @@ export async function startScenario(
     )
   }
 
-  if (!(await checkPermission(scenario.privilege, user, domain, unsafeIP, prohibitedPrivileges))) {
-    const { category, privilege } = scenario.privilege || {}
-    throw new Error(
-      `Unauthorized! ${category && privilege ? category + ':' + privilege + ' privilege' : 'ownership granted'} required`
-    )
-  }
+  await checkHasRole(scenario, context)
 
   instanceName = instanceName || scenarioName
   return await ScenarioEngine.load(instanceName, scenario, { domain, user, lng, variables })
@@ -152,12 +148,7 @@ export async function stopScenario(
 
   var scenario = await findScenario(scenarioInstance.scenarioName, domain)
 
-  if (!(await checkPermission(scenario.privilege, user, domain, unsafeIP, prohibitedPrivileges))) {
-    const { category, privilege } = scenario.privilege || {}
-    throw new Error(
-      `Unauthorized! ${category && privilege ? category + ':' + privilege + ' privilege' : 'ownership granted'} required`
-    )
-  }
+  await checkHasRole(scenario, context)
 
   await ScenarioEngine.unload(domain, instanceName)
 

package/server/engine/connection-manager.ts

@@ -24,7 +24,7 @@ function getSystemTimeZone() {
 }
 
 const SYSTEM_TZ = getSystemTimeZone()
-const systemTimestamp = format((info, opts) => {
+const systemTimestamp = format((info, opts: { tz?: string }) => {
   if (opts.tz) info.timestamp = moment().tz(opts.tz).format()
   return info
 })

package/server/engine/connector/headless-connector.ts

@@ -0,0 +1,341 @@
+import { ConnectionManager } from '../connection-manager'
+import { Connector } from '../types'
+import { getHeadlessPool } from '../resource-pool/headless-pool'
+import { Browser, Page } from 'puppeteer'
+
+/*
+  Functionality of the headless-connector:
+  - Provides a mechanism to acquire an active session page.
+  - Performs login when necessary to obtain valid cookies.
+    - Applies these cookies to the page for session management.
+  - Pages are acquired from the `headlessPool`, which manages browser instances.
+  - During the login process, pages from the pool are used for actions like form filling and navigation.
+  - Valid cookies are saved after login and reused for subsequent page acquisitions.
+  - Users must explicitly release the page after use through the `releasePage` method.
+  - Released pages are returned to the `headlessPool` for reuse.
+*/
+
+export class HeadlessConnector implements Connector {
+  async ready(connectionConfigs) {
+    await Promise.all(connectionConfigs.map(this.connect.bind(this)))
+    ConnectionManager.logger.info('headless-connector connections are ready')
+  }
+
+  async connect(connection) {
+    const {
+      endpoint: uri = '1',
+      params: {
+        username = '',
+        password = '',
+        loginPath = '/login',
+        usernameSelector = '#username',
+        passwordSelector = '#password',
+        submitSelector = '#submit',
+        successSelector = null,
+        shadowDomSelectors = '', // Comma separated shadow DOM selectors
+        timeout = 15000, // Default timeout for operations
+        retries = 3 // Default number of retries for login or page actions
+      } = {}
+    } = connection
+
+    const loginInfo = {
+      loginRequired: Boolean(username), // Determine if login is required
+      username,
+      password,
+      loginPath,
+      timeout,
+      retries,
+      loginSelectors: {
+        usernameSelector,
+        passwordSelector,
+        submitSelector,
+        successSelector,
+        shadowDomSelectors: shadowDomSelectors
+          .split(',')
+          .map(selector => selector.trim())
+          .filter(Boolean)
+      }
+    }
+
+    async function acquireBrowser() {
+      try {
+        const pool = getHeadlessPool()
+        const browser = await pool.acquire()
+        return browser
+      } catch (error) {
+        ConnectionManager.logger.error('Failed to acquire browser:', error)
+        throw error
+      }
+    }
+
+    async function releaseBrowser(browser: Browser) {
+      try {
+        const pool = getHeadlessPool()
+        await pool.release(browser)
+      } catch (error) {
+        ConnectionManager.logger.error('Failed to release browser:', error)
+      }
+    }
+
+    ConnectionManager.addConnectionInstance(connection, {
+      endpoint: connection.endpoint,
+      params: connection.params,
+      acquireSessionPage: async () => {
+        const browser = await acquireBrowser()
+        let page
+        let cookies = connection.cookies
+
+        try {
+          page = await browser.newPage()
+
+          page.on('console', async msg => {
+            console.log(`[browser ${msg.type()}] ${msg.text()}`)
+          })
+
+          await page.setRequestInterception(true)
+
+          // page.on('request', request => {
+          //   const resourceType = request.resourceType()
+          //   if (resourceType === 'image' || resourceType === 'stylesheet' || resourceType === 'font') {
+          //     request.abort() // 이미지, 스타일시트, 폰트 요청 차단
+          //   } else {
+          //     request.continue() // 나머지 요청은 진행
+          //   }
+          // })
+
+          page.on('requestfailed', request => {
+            console.log('Request failed:')
+            console.log(`- URL: ${request.url()}`)
+            console.log(`- Method: ${request.method()}`)
+            console.log(`- Failure Text: ${request.failure()?.errorText}`)
+            console.log(`- Headers:`, request.headers())
+
+            // POST 데이터 (필요한 경우)
+            if (request.postData()) {
+              console.log(`- Post Data: ${request.postData()}`)
+            }
+          })
+
+          if (cookies && isCookieValid(cookies)) {
+            await this.applyCookiesAndVerifySession(page, cookies, loginInfo)
+            return page
+          }
+
+          if (loginInfo.loginRequired) {
+            await this.performLogin(page, uri, loginInfo)
+            cookies = await page.cookies()
+            connection.cookies = cookies
+          }
+
+          return page
+        } catch (error) {
+          ConnectionManager.logger.error('Failed to acquire session page:', error)
+          throw error
+        }
+      },
+      releasePage: async (page: Page) => {
+        try {
+          if (page) {
+            const browser = page.browser()
+            await page.close()
+            await releaseBrowser(browser)
+          }
+        } catch (error) {
+          ConnectionManager.logger.error('Failed to release page:', error)
+        }
+      },
+      acquireBrowser,
+      releaseBrowser
+    })
+
+    ConnectionManager.logger.info(
+      `headless-connector connection(${connection.name}:${connection.endpoint}) is connected`
+    )
+  }
+
+  async applyCookiesAndVerifySession(page, cookies, loginInfo) {
+    await page.setCookie(...cookies)
+    await page.reload({ waitUntil: 'networkidle2', timeout: loginInfo.timeout })
+
+    if (loginInfo.loginRequired && loginInfo.loginSelectors.successSelector) {
+      const success = await this.resolveShadowDom(
+        page,
+        loginInfo.loginSelectors.shadowDomSelectors,
+        loginInfo.loginSelectors.successSelector
+      )
+      if (!success) {
+        throw new Error('Session invalid, login required')
+      }
+    }
+  }
+
+  async performLogin(page, uri, loginInfo) {
+    for (let attempt = 1; attempt <= loginInfo.retries; attempt++) {
+      try {
+        await page.goto(`${uri}${loginInfo.loginPath}`, { waitUntil: 'networkidle2', timeout: loginInfo.timeout })
+
+        const usernameInput = await this.resolveShadowDom(
+          page,
+          loginInfo.loginSelectors.shadowDomSelectors,
+          loginInfo.loginSelectors.usernameSelector
+        )
+        const passwordInput = await this.resolveShadowDom(
+          page,
+          loginInfo.loginSelectors.shadowDomSelectors,
+          loginInfo.loginSelectors.passwordSelector
+        )
+        const submitButton = await this.resolveShadowDom(
+          page,
+          loginInfo.loginSelectors.shadowDomSelectors,
+          loginInfo.loginSelectors.submitSelector
+        )
+
+        if (!usernameInput || !passwordInput || !submitButton) {
+          throw new Error('Failed to locate input elements in shadow DOM')
+        }
+
+        await usernameInput.type(loginInfo.username)
+        await passwordInput.type(loginInfo.password)
+
+        // Capture the response of the form submission
+        const [response] = await Promise.all([
+          page.waitForNavigation({ waitUntil: 'networkidle2', timeout: loginInfo.timeout }),
+          submitButton.click()
+        ])
+
+        // Check response status code
+        if (response) {
+          const status = response.status()
+          if (status >= 200 && status < 300) {
+            ConnectionManager.logger.info(`Login successful with status code: ${status}`)
+            return // Login successful
+          } else if (status >= 400) {
+            throw new Error(`Login failed with status code: ${status}`)
+          }
+        } else {
+          throw new Error('No response received during login')
+        }
+
+        if (loginInfo.loginSelectors.successSelector) {
+          const success = await this.resolveShadowDom(
+            page,
+            loginInfo.loginSelectors.shadowDomSelectors,
+            loginInfo.loginSelectors.successSelector
+          )
+          if (!success) {
+            throw new Error('Login failed: Success selector not found')
+          }
+        }
+      } catch (error) {
+        ConnectionManager.logger.warn(`Login attempt ${attempt} failed:`, error)
+
+        if (attempt === loginInfo.retries) {
+          throw new Error(`Login failed after ${loginInfo.retries} attempts`)
+        }
+      }
+    }
+  }
+
+  async resolveShadowDom(page, shadowSelectors, targetSelector) {
+    let context
+
+    if (!shadowSelectors || shadowSelectors.length === 0) {
+      // No Shadow DOM path; use document root as the context
+      context = page.mainFrame() // Puppeteer uses frames to represent document
+      return context.$(targetSelector) // Search directly in the document root
+    }
+
+    context = page // Start with the page as the context
+    for (const selector of shadowSelectors) {
+      const shadowHost = await context.$(selector)
+      if (!shadowHost) {
+        throw new Error(`Shadow host not found: ${selector}`)
+      }
+      context = await page.evaluateHandle(host => host.shadowRoot, shadowHost)
+    }
+    return context.evaluateHandle((shadowRoot, selector) => shadowRoot.querySelector(selector), targetSelector)
+  }
+
+  async disconnect(connection) {
+    const connectionInstance = ConnectionManager.getConnectionInstance(connection)
+    ConnectionManager.removeConnectionInstance(connection)
+    ConnectionManager.logger.info(`headless-connector connection(${connection.name}) is disconnected`)
+  }
+
+  get parameterSpec() {
+    return [
+      {
+        type: 'string',
+        name: 'username',
+        label: 'username'
+      },
+      {
+        type: 'password',
+        name: 'password',
+        label: 'password'
+      },
+      {
+        type: 'string',
+        name: 'loginPath',
+        label: 'login-path'
+      },
+      {
+        type: 'string',
+        name: 'usernameSelector',
+        label: 'username-selector'
+      },
+      {
+        type: 'string',
+        name: 'passwordSelector',
+        label: 'password-selector'
+      },
+      {
+        type: 'string',
+        name: 'submitSelector',
+        label: 'submit-selector'
+      },
+      {
+        type: 'string',
+        name: 'successSelector',
+        label: 'success-selector'
+      },
+      {
+        type: 'string',
+        name: 'shadowDomSelectors',
+        label: 'shadow-dom-selectors'
+      },
+      {
+        type: 'number',
+        name: 'timeout',
+        label: 'timeout',
+        value: 15000
+      },
+      {
+        type: 'number',
+        name: 'retries',
+        label: 'maximum-retries',
+        value: 3
+      }
+    ]
+  }
+
+  get taskPrefixes() {
+    return ['headless']
+  }
+
+  get description() {
+    return 'Headless Pool Connector with login capabilities'
+  }
+
+  get help() {
+    return 'integration/connector/headless-connector'
+  }
+}
+
+ConnectionManager.registerConnector('headless-connector', new HeadlessConnector())
+
+function isCookieValid(cookies) {
+  if (!cookies || cookies.length === 0) return false
+  const now = Date.now() / 1000 // Current time in seconds
+  return cookies.some(cookie => cookie.expires && cookie.expires > now)
+}

package/server/engine/connector/http-connector.ts

@@ -58,7 +58,7 @@ export class HttpConnector implements Connector {
   }
 
   get taskPrefixes() {
-    return ['http', 'headless-post', 'headless-scrap']
+    return ['http']
   }
 }
 

package/server/engine/connector/index.ts

@@ -10,3 +10,4 @@ import './oracle-connector'
 import './mysql-connector'
 import './socket-server'
 import './operato-connector'
+import './headless-connector'

package/server/engine/connector/operato-connector.ts

@@ -17,7 +17,7 @@ import { Scenario } from '../../service/scenario/scenario'
 import { ScenarioInstance } from '../../service/scenario-instance/scenario-instance-type'
 
 import { getRepository, GraphqlLocalClient, Domain } from '@things-factory/shell'
-import { User, checkPermission } from '@things-factory/auth-base'
+import { User, checkUserHasRole } from '@things-factory/auth-base'
 
 const debug = require('debug')('things-factory:integration-base:operato-connector')
 
@@ -198,7 +198,7 @@ export class OperatoConnector implements Connector {
   }
 
   async runScenario(subscriptions: SubscriberData[], variables: any): Promise<ScenarioInstance> {
-    const { domain, user, unsafeIP, prohibitedPrivileges } = this.context
+    const { domain, user } = this.context
     const { tag } = variables
 
     if (!tag) {
@@ -210,11 +210,8 @@ export class OperatoConnector implements Connector {
       throw new Error(`scenario is not found - ${tag}`)
     }
 
-    if (!(await checkPermission(scenario.privilege, user, domain, unsafeIP, prohibitedPrivileges))) {
-      const { category, privilege } = scenario.privilege || {}
-      throw new Error(
-        `Unauthorized! ${category && privilege ? category + ':' + privilege + ' privilege' : 'ownership granted'} required`
-      )
+    if (!(await checkUserHasRole(scenario.roleId, domain, user))) {
+      throw new Error(`Unauthorized! ${scenario.name} doesn't have required role.`)
     }
 
     /* create a scenario instance */

package/server/engine/index.ts

@@ -6,5 +6,6 @@ export * from './scenario-engine'
 export * from './task-registry'
 export * from './analyzer/analyze-integration'
 export * from './edge-client'
+export * from './resource-pool'
 
 export { Connector, Context, TaskHandler } from './types'