@things-factory/integration-base 6.0.67 → 6.0.69

package/helps/integration/task/data-mapper.md

@@ -0,0 +1,14 @@
+# data-mapper Task
+
+특정 accessor의 결과 값을 새로운 데이터로 변환하는 기능을 제공한다.
+기존 데이터가 단일 오브젝트이면 새로운 데이터도 단일 오브젝트 형태로 변환되며,
+기존 데이터가 오브젝트의 배열 형태이면 새로운 데이터도 오브젝트의 배열 형태로 반환된다.
+
+## parameters
+
+- [accessor](../concept/data-accessor.md)
+  - 변환하고자 하는 대상 데이터를 설정한다.
+- 매핑 규칙(mapping-rule)
+  - 기존 데이터를 기반으로 새로운 데이터를 만드는 규칙을 설정한다.
+  - key 부분에 설정된 값들은 새로 만들어질 데이터의 key들이 된다.
+  - value 부분에는 기존 데이터의 key를 입력한다. 기존 데이터의 key에 해당되는 값이 새로운 데이터의 값이 된다.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@things-factory/integration-base",
-  "version": "6.0.67",
+  "version": "6.0.69",
   "main": "dist-server/index.js",
   "browser": "client/index.js",
   "things-factory": true,
@@ -45,5 +45,5 @@
   "devDependencies": {
     "@types/cron": "^2.0.1"
   },
-  "gitHead": "69de24faac65e08ba200e67402b19ed564125a87"
+  "gitHead": "d0198bba193b275f29fffd8c3963f556db4dda92"
 }

package/server/engine/connector/http-connector.ts

@@ -57,7 +57,7 @@ export class HttpConnector implements Connector {
   }
 
   get taskPrefixes() {
-    return ['http']
+    return ['http', 'headless']
   }
 }
 

package/server/engine/task/data-mapper.ts

@@ -0,0 +1,44 @@
+import { access } from '@things-factory/utils'
+import { TaskRegistry } from '../task-registry'
+
+function mapping(rule, input) {
+  if (Array.isArray(input)) {
+    return input.map(i => mapping(rule, i))
+  }
+
+  return Object.keys(rule).reduce((sum, key) => {
+    sum[key] = input[rule[key]]
+    return sum
+  }, {} as any)
+}
+
+async function DataMapper(step, { logger, data }) {
+  var {
+    params: { accessor, mappingRule }
+  } = step
+
+  const input = access(accessor, data)
+  const output = mapping(mappingRule || {}, input)
+
+  return {
+    data: output
+  }
+}
+
+DataMapper.parameterSpec = [
+  {
+    type: 'scenario-step-input',
+    name: 'accessor',
+    label: 'accessor'
+  },
+  {
+    type: 'key-values',
+    name: 'mappingRule',
+    label: 'mapping-rule'
+  }
+]
+
+DataMapper.connectorFree = true
+DataMapper.help = 'integration/task/data-mapper'
+
+TaskRegistry.registerTaskHandler('data-mapper', DataMapper)

package/server/engine/task/headless-post.ts

@@ -0,0 +1,141 @@
+import https from 'https'
+import { URL } from 'url'
+
+import { access } from '@things-factory/utils'
+import { TaskRegistry } from '../task-registry'
+import { ConnectionManager } from '../connection-manager'
+
+import { getHeadlessPool } from './utils/headless-pool-for-scenario'
+
+async function HeadlessPost(step, { logger, data, domain }) {
+  var { connection: connectionName, params: stepOptions } = step
+  var { headers: requestHeaders, contentType, path, accessor } = stepOptions || {}
+
+  var connection = ConnectionManager.getConnectionInstanceByName(domain, connectionName)
+
+  if (!connection) {
+    throw new Error(`connection '${connectionName}' is not established.`)
+  }
+
+  var { endpoint, params: connectionParams, authHeaders = {} } = connection
+
+  var headers = {
+    ...authHeaders,
+    ...requestHeaders
+  }
+
+  var body = access(accessor, data)
+  if (contentType && body) {
+    headers['content-type'] = contentType
+    switch (contentType) {
+      case 'text/plain':
+        body = JSON.stringify(body)
+        break
+      case 'application/json':
+        body = JSON.stringify(body)
+        break
+      case 'application/x-www-form-urlencoded':
+        const searchParams = new URLSearchParams()
+        for (const prop in body) {
+          searchParams.set(prop, body[prop])
+        }
+        body = searchParams
+        break
+    }
+  }
+
+  var options: any = {
+    method: 'POST',
+    headers,
+    body
+  }
+
+  var { rejectUnauthorized } = connectionParams
+
+  if (!rejectUnauthorized) {
+    const httpsAgent = new https.Agent({
+      rejectUnauthorized
+    })
+    options.agent = httpsAgent
+  }
+
+  const browser = (await getHeadlessPool().acquire()) as any
+  const page = await browser.newPage()
+
+  try {
+    page.on('console', msg => {
+      console.log(`[browser ${msg.type()}] ${msg.text()}`)
+      for (let i = 0; i < msg.args().length; ++i) console.log(`${i}: ${msg.args()[i]}`)
+    })
+
+    await page.goto(endpoint, { waitUntil: 'networkidle2' })
+
+    const response = await page.evaluate(
+      async (urlString, options) => {
+        const response = await fetch(urlString, options)
+
+        if (response.ok && response.headers.get('content-type').includes('application/json')) {
+          return await response.json()
+        } else {
+          return await response.text()
+        }
+      },
+      new URL(path, endpoint),
+      options
+    )
+
+    return {
+      data: response
+    }
+  } catch (e) {
+    console.error(e)
+  } finally {
+    page.close()
+    getHeadlessPool().release(browser)
+  }
+}
+
+HeadlessPost.parameterSpec = [
+  {
+    type: 'string',
+    name: 'path',
+    label: 'path'
+  },
+  {
+    type: 'http-headers',
+    name: 'headers',
+    label: 'headers'
+  },
+  {
+    type: 'select',
+    name: 'contentType',
+    label: 'content-type',
+    property: {
+      options: [
+        {
+          display: '',
+          value: ''
+        },
+        {
+          display: 'application/json',
+          value: 'application/json'
+        },
+        {
+          display: 'text/plain',
+          value: 'text/plain'
+        },
+        {
+          display: 'application/x-www-form-urlencoded',
+          value: 'application/x-www-form-urlencoded'
+        }
+      ]
+    }
+  },
+  {
+    type: 'scenario-step-input',
+    name: 'accessor',
+    label: 'accessor'
+  }
+]
+
+TaskRegistry.registerTaskHandler('headless-post', HeadlessPost)

package/server/engine/task/http-post.ts

@@ -61,15 +61,11 @@ async function HttpPost(step, { logger, data, domain }) {
 
   var response = await fetch(url, fetchOptions)
 
-  var responseData = await response.text()
-
-  const responseContentType = response.headers.get('content-type')
-  if (responseContentType && responseContentType.indexOf('application/json') !== -1) {
-    responseData = JSON.stringify(responseData)
-  }
-
   return {
-    data: responseData
+    data:
+      response.ok && response.headers.get('content-type').includes('application/json')
+        ? await response.json()
+        : await response.text()
   }
 }
 

package/server/engine/task/index.ts

@@ -32,3 +32,5 @@ import './floating-point'
 import './socket-listener'
 import './random'
 import './csv-readline'
+import './data-mapper'
+import './headless-post'

package/server/engine/task/utils/headless-pool-for-scenario.ts

@@ -0,0 +1,71 @@
+import * as genericPool from 'generic-pool'
+
+import { config, logger } from '@things-factory/env'
+
+try {
+  var puppeteer = require('puppeteer')
+} catch (err) {
+  logger.error(err)
+}
+
+var headlessPool
+
+export function getHeadlessPool() {
+  if (!headlessPool) {
+    headlessPool = genericPool.createPool(
+      {
+        create() {
+          console.log('headless-pool-for-scensrio about to create')
+          return initializeChromium()
+        },
+        validate(browser) {
+          return Promise.race([
+            new Promise(res => setTimeout(() => res(false), 1500)),
+            browser
+              //@ts-ignore
+              .version()
+              .then(_ => true)
+              .catch(_ => false)
+          ])
+        },
+        destroy(browser) {
+          //@ts-ignore
+          return browser.close()
+        }
+      },
+      {
+        min: 2,
+        max: 10,
+        testOnBorrow: true,
+        acquireTimeoutMillis: 15000
+      }
+    )
+  }
+
+  return headlessPool
+}
+
+const CHROMIUM_PATH = config.get('CHROMIUM_PATH')
+
+async function initializeChromium() {
+  try {
+    if (!puppeteer) {
+      return
+    }
+
+    var launchSetting = {
+      args: ['--mute-audio', '--no-sandbox'],
+      headless: 'new'
+    }
+
+    if (CHROMIUM_PATH) {
+      launchSetting['executablePath'] = CHROMIUM_PATH
+    }
+
+    const browser = await puppeteer.launch(launchSetting)
+
+    return browser
+  } catch (err) {
+    logger.error(err)
+  }
+}

package/translations/en.json

@@ -1,4 +1,4 @@
 {
   "error.scenario not found": "scenario '{scenario}' not found.",
   "error.scenario instance not found": "scenario instance '{instance}' not found."
-}
+}

package/translations/ko.json

@@ -1,4 +1,4 @@
 {
   "error.scenario not found": "시나리오 '{scenario}'를 찾을 수 없습니다.",
   "error.scenario instance not found": "시나리오 인스턴스 '{instance}'를 찾을 수 없습니다. 이미 종료되었을 수 있습니다."
-}
+}

package/translations/ms.json

@@ -1,4 +1,4 @@
 {
   "error.scenario not found": "[ms] scenario '{scenario}' not found.",
   "error.scenario instance not found": "[ms] scenario instance '{instance}' not found."
-}
+}

package/translations/zh.json

@@ -1,4 +1,4 @@
 {
   "error.scenario not found": "未找到场景'{scenario}'",
   "error.scenario instance not found": "[zh] scenario instance '{instance}' not found."
-}
+}