@things-factory/integration-base 6.0.68 → 6.0.69

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@things-factory/integration-base",
-  "version": "6.0.68",
+  "version": "6.0.69",
   "main": "dist-server/index.js",
   "browser": "client/index.js",
   "things-factory": true,
@@ -45,5 +45,5 @@
   "devDependencies": {
     "@types/cron": "^2.0.1"
   },
-  "gitHead": "832b257b49d4fcca76d5a428df8a2fcf44b4db1a"
+  "gitHead": "d0198bba193b275f29fffd8c3963f556db4dda92"
 }

package/server/engine/connector/http-connector.ts

@@ -57,7 +57,7 @@ export class HttpConnector implements Connector {
   }
 
   get taskPrefixes() {
-    return ['http']
+    return ['http', 'headless']
   }
 }
 

package/server/engine/task/headless-post.ts

@@ -0,0 +1,141 @@
+import https from 'https'
+import { URL } from 'url'
+
+import { access } from '@things-factory/utils'
+import { TaskRegistry } from '../task-registry'
+import { ConnectionManager } from '../connection-manager'
+
+import { getHeadlessPool } from './utils/headless-pool-for-scenario'
+
+async function HeadlessPost(step, { logger, data, domain }) {
+  var { connection: connectionName, params: stepOptions } = step
+  var { headers: requestHeaders, contentType, path, accessor } = stepOptions || {}
+
+  var connection = ConnectionManager.getConnectionInstanceByName(domain, connectionName)
+
+  if (!connection) {
+    throw new Error(`connection '${connectionName}' is not established.`)
+  }
+
+  var { endpoint, params: connectionParams, authHeaders = {} } = connection
+
+  var headers = {
+    ...authHeaders,
+    ...requestHeaders
+  }
+
+  var body = access(accessor, data)
+  if (contentType && body) {
+    headers['content-type'] = contentType
+    switch (contentType) {
+      case 'text/plain':
+        body = JSON.stringify(body)
+        break
+      case 'application/json':
+        body = JSON.stringify(body)
+        break
+      case 'application/x-www-form-urlencoded':
+        const searchParams = new URLSearchParams()
+        for (const prop in body) {
+          searchParams.set(prop, body[prop])
+        }
+        body = searchParams
+        break
+    }
+  }
+
+  var options: any = {
+    method: 'POST',
+    headers,
+    body
+  }
+
+  var { rejectUnauthorized } = connectionParams
+
+  if (!rejectUnauthorized) {
+    const httpsAgent = new https.Agent({
+      rejectUnauthorized
+    })
+    options.agent = httpsAgent
+  }
+
+  const browser = (await getHeadlessPool().acquire()) as any
+  const page = await browser.newPage()
+
+  try {
+    page.on('console', msg => {
+      console.log(`[browser ${msg.type()}] ${msg.text()}`)
+      for (let i = 0; i < msg.args().length; ++i) console.log(`${i}: ${msg.args()[i]}`)
+    })
+
+    await page.goto(endpoint, { waitUntil: 'networkidle2' })
+
+    const response = await page.evaluate(
+      async (urlString, options) => {
+        const response = await fetch(urlString, options)
+
+        if (response.ok && response.headers.get('content-type').includes('application/json')) {
+          return await response.json()
+        } else {
+          return await response.text()
+        }
+      },
+      new URL(path, endpoint),
+      options
+    )
+
+    return {
+      data: response
+    }
+  } catch (e) {
+    console.error(e)
+  } finally {
+    page.close()
+    getHeadlessPool().release(browser)
+  }
+}
+
+HeadlessPost.parameterSpec = [
+  {
+    type: 'string',
+    name: 'path',
+    label: 'path'
+  },
+  {
+    type: 'http-headers',
+    name: 'headers',
+    label: 'headers'
+  },
+  {
+    type: 'select',
+    name: 'contentType',
+    label: 'content-type',
+    property: {
+      options: [
+        {
+          display: '',
+          value: ''
+        },
+        {
+          display: 'application/json',
+          value: 'application/json'
+        },
+        {
+          display: 'text/plain',
+          value: 'text/plain'
+        },
+        {
+          display: 'application/x-www-form-urlencoded',
+          value: 'application/x-www-form-urlencoded'
+        }
+      ]
+    }
+  },
+  {
+    type: 'scenario-step-input',
+    name: 'accessor',
+    label: 'accessor'
+  }
+]
+
+TaskRegistry.registerTaskHandler('headless-post', HeadlessPost)

package/server/engine/task/http-post.ts

@@ -61,15 +61,11 @@ async function HttpPost(step, { logger, data, domain }) {
 
   var response = await fetch(url, fetchOptions)
 
-  var responseData = await response.text()
-
-  const responseContentType = response.headers.get('content-type')
-  if (responseContentType && responseContentType.indexOf('application/json') !== -1) {
-    responseData = JSON.stringify(responseData)
-  }
-
   return {
-    data: responseData
+    data:
+      response.ok && response.headers.get('content-type').includes('application/json')
+        ? await response.json()
+        : await response.text()
   }
 }
 

package/server/engine/task/index.ts

@@ -33,3 +33,4 @@ import './socket-listener'
 import './random'
 import './csv-readline'
 import './data-mapper'
+import './headless-post'

package/server/engine/task/utils/headless-pool-for-scenario.ts

@@ -0,0 +1,71 @@
+import * as genericPool from 'generic-pool'
+
+import { config, logger } from '@things-factory/env'
+
+try {
+  var puppeteer = require('puppeteer')
+} catch (err) {
+  logger.error(err)
+}
+
+var headlessPool
+
+export function getHeadlessPool() {
+  if (!headlessPool) {
+    headlessPool = genericPool.createPool(
+      {
+        create() {
+          console.log('headless-pool-for-scensrio about to create')
+          return initializeChromium()
+        },
+        validate(browser) {
+          return Promise.race([
+            new Promise(res => setTimeout(() => res(false), 1500)),
+            browser
+              //@ts-ignore
+              .version()
+              .then(_ => true)
+              .catch(_ => false)
+          ])
+        },
+        destroy(browser) {
+          //@ts-ignore
+          return browser.close()
+        }
+      },
+      {
+        min: 2,
+        max: 10,
+        testOnBorrow: true,
+        acquireTimeoutMillis: 15000
+      }
+    )
+  }
+
+  return headlessPool
+}
+
+const CHROMIUM_PATH = config.get('CHROMIUM_PATH')
+
+async function initializeChromium() {
+  try {
+    if (!puppeteer) {
+      return
+    }
+
+    var launchSetting = {
+      args: ['--mute-audio', '--no-sandbox'],
+      headless: 'new'
+    }
+
+    if (CHROMIUM_PATH) {
+      launchSetting['executablePath'] = CHROMIUM_PATH
+    }
+
+    const browser = await puppeteer.launch(launchSetting)
+
+    return browser
+  } catch (err) {
+    logger.error(err)
+  }
+}