@things-factory/integration-base 6.1.175 → 6.1.177
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-server/engine/task/headless-post.js +4 -3
- package/dist-server/engine/task/headless-post.js.map +1 -1
- package/dist-server/engine/task/headless-scrap.js +4 -3
- package/dist-server/engine/task/headless-scrap.js.map +1 -1
- package/dist-server/tsconfig.tsbuildinfo +1 -1
- package/package.json +6 -6
- package/server/engine/task/headless-post.ts +5 -2
- package/server/engine/task/headless-scrap.ts +5 -2
|
@@ -50,10 +50,11 @@ async function HeadlessPost(step, { logger, data, domain }) {
|
|
|
50
50
|
const browser = (await (0, headless_pool_for_scenario_1.getHeadlessPool)().acquire());
|
|
51
51
|
const page = await browser.newPage();
|
|
52
52
|
try {
|
|
53
|
-
page.on('console', msg => {
|
|
53
|
+
page.on('console', async (msg) => {
|
|
54
54
|
console.log(`[browser ${msg.type()}] ${msg.text()}`);
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
});
|
|
56
|
+
page.on('requestfailed', request => {
|
|
57
|
+
console.log('Request failed:', request.url());
|
|
57
58
|
});
|
|
58
59
|
await page.goto(endpoint, { waitUntil: 'networkidle2' });
|
|
59
60
|
const response = await page.evaluate(async (urlString, options) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"headless-post.js","sourceRoot":"","sources":["../../../server/engine/task/headless-post.ts"],"names":[],"mappings":";;;AAAA,0DAAyB;AACzB,6BAAyB;AAEzB,iDAA8C;AAC9C,oDAA+C;AAC/C,8DAAyD;AAEzD,mFAAoE;AAEpE,KAAK,UAAU,YAAY,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IACxD,IAAI,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAA;IAC9D,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,WAAW,IAAI,EAAE,CAAA;IAEhF,IAAI,UAAU,GAAG,sCAAiB,CAAC,2BAA2B,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAEtF,IAAI,CAAC,UAAU,EAAE;QACf,MAAM,IAAI,KAAK,CAAC,eAAe,cAAc,uBAAuB,CAAC,CAAA;KACtE;IAED,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,gBAAgB,EAAE,WAAW,GAAG,EAAE,EAAE,GAAG,UAAU,CAAA;IAEzE,IAAI,OAAO,mCACN,WAAW,GACX,cAAc,CAClB,CAAA;IAED,IAAI,IAAI,GAAG,IAAA,cAAM,EAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;IACjC,IAAI,WAAW,IAAI,IAAI,EAAE;QACvB,OAAO,CAAC,cAAc,CAAC,GAAG,WAAW,CAAA;QACrC,QAAQ,WAAW,EAAE;YACnB,KAAK,YAAY;gBACf,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBAC3B,MAAK;YACP,KAAK,kBAAkB;gBACrB,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBAC3B,MAAK;YACP,KAAK,mCAAmC;gBACtC,MAAM,YAAY,GAAG,IAAI,eAAe,EAAE,CAAA;gBAC1C,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE;oBACvB,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;iBACnC;gBACD,IAAI,GAAG,YAAY,CAAA;gBACnB,MAAK;SACR;KACF;IAED,IAAI,OAAO,GAAQ;QACjB,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI;KACL,CAAA;IAED,IAAI,EAAE,kBAAkB,EAAE,GAAG,gBAAgB,CAAA;IAE7C,IAAI,CAAC,kBAAkB,EAAE;QACvB,MAAM,UAAU,GAAG,IAAI,eAAK,CAAC,KAAK,CAAC;YACjC,kBAAkB;SACnB,CAAC,CAAA;QACF,OAAO,CAAC,KAAK,GAAG,UAAU,CAAA;KAC3B;IAED,MAAM,OAAO,GAAG,CAAC,MAAM,IAAA,4CAAe,GAAE,CAAC,OAAO,EAAE,CAAQ,CAAA;IAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAA;IAEpC,IAAI;QACF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,
|
|
1
|
+
{"version":3,"file":"headless-post.js","sourceRoot":"","sources":["../../../server/engine/task/headless-post.ts"],"names":[],"mappings":";;;AAAA,0DAAyB;AACzB,6BAAyB;AAEzB,iDAA8C;AAC9C,oDAA+C;AAC/C,8DAAyD;AAEzD,mFAAoE;AAEpE,KAAK,UAAU,YAAY,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IACxD,IAAI,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAA;IAC9D,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,WAAW,IAAI,EAAE,CAAA;IAEhF,IAAI,UAAU,GAAG,sCAAiB,CAAC,2BAA2B,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAEtF,IAAI,CAAC,UAAU,EAAE;QACf,MAAM,IAAI,KAAK,CAAC,eAAe,cAAc,uBAAuB,CAAC,CAAA;KACtE;IAED,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,gBAAgB,EAAE,WAAW,GAAG,EAAE,EAAE,GAAG,UAAU,CAAA;IAEzE,IAAI,OAAO,mCACN,WAAW,GACX,cAAc,CAClB,CAAA;IAED,IAAI,IAAI,GAAG,IAAA,cAAM,EAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;IACjC,IAAI,WAAW,IAAI,IAAI,EAAE;QACvB,OAAO,CAAC,cAAc,CAAC,GAAG,WAAW,CAAA;QACrC,QAAQ,WAAW,EAAE;YACnB,KAAK,YAAY;gBACf,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBAC3B,MAAK;YACP,KAAK,kBAAkB;gBACrB,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBAC3B,MAAK;YACP,KAAK,mCAAmC;gBACtC,MAAM,YAAY,GAAG,IAAI,eAAe,EAAE,CAAA;gBAC1C,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE;oBACvB,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;iBACnC;gBACD,IAAI,GAAG,YAAY,CAAA;gBACnB,MAAK;SACR;KACF;IAED,IAAI,OAAO,GAAQ;QACjB,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI;KACL,CAAA;IAED,IAAI,EAAE,kBAAkB,EAAE,GAAG,gBAAgB,CAAA;IAE7C,IAAI,CAAC,kBAAkB,EAAE;QACvB,MAAM,UAAU,GAAG,IAAI,eAAK,CAAC,KAAK,CAAC;YACjC,kBAAkB;SACnB,CAAC,CAAA;QACF,OAAO,CAAC,KAAK,GAAG,UAAU,CAAA;KAC3B;IAED,MAAM,OAAO,GAAG,CAAC,MAAM,IAAA,4CAAe,GAAE,CAAC,OAAO,EAAE,CAAQ,CAAA;IAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAA;IAEpC,IAAI;QACF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAC,GAAG,EAAC,EAAE;YAC7B,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QACtD,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE;YACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAA;QAExD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAClC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE;YAC3B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;YAEhD,IAAI,QAAQ,CAAC,EAAE,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE;gBACpF,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;aAC7B;iBAAM;gBACL,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;aAC7B;QACH,CAAC,EACD,IAAI,SAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,EACvB,OAAO,CACR,CAAA;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;SACf,CAAA;KACF;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;KACjB;YAAS;QACR,IAAI,CAAC,KAAK,EAAE,CAAA;QACZ,IAAA,4CAAe,GAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;KACnC;AACH,CAAC;AAED,YAAY,CAAC,aAAa,GAAG;IAC3B;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,MAAM;KACd;IACD;QACE,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,SAAS;KACjB;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,cAAc;QACrB,QAAQ,EAAE;YACR,OAAO,EAAE;gBACP;oBACE,OAAO,EAAE,EAAE;oBACX,KAAK,EAAE,EAAE;iBACV;gBACD;oBACE,OAAO,EAAE,kBAAkB;oBAC3B,KAAK,EAAE,kBAAkB;iBAC1B;gBACD;oBACE,OAAO,EAAE,YAAY;oBACrB,KAAK,EAAE,YAAY;iBACpB;gBACD;oBACE,OAAO,EAAE,mCAAmC;oBAC5C,KAAK,EAAE,mCAAmC;iBAC3C;aACF;SACF;KACF;IACD;QACE,IAAI,EAAE,qBAAqB;QAC3B,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,UAAU;KAClB;CACF,CAAA;AAED,4BAAY,CAAC,mBAAmB,CAAC,eAAe,EAAE,YAAY,CAAC,CAAA","sourcesContent":["import https from 'https'\nimport { URL } from 'url'\n\nimport { access } from '@things-factory/utils'\nimport { TaskRegistry } from '../task-registry'\nimport { ConnectionManager } from '../connection-manager'\n\nimport { getHeadlessPool } from './utils/headless-pool-for-scenario'\n\nasync function HeadlessPost(step, { logger, data, domain }) {\n var { connection: connectionName, params: stepOptions } = step\n var { headers: requestHeaders, contentType, path, accessor } = stepOptions || {}\n\n var connection = ConnectionManager.getConnectionInstanceByName(domain, connectionName)\n\n if (!connection) {\n throw new Error(`connection '${connectionName}' is not established.`)\n }\n\n var { endpoint, params: connectionParams, authHeaders = {} } = connection\n\n var headers = {\n ...authHeaders,\n ...requestHeaders\n }\n\n var body = access(accessor, data)\n if (contentType && body) {\n headers['content-type'] = contentType\n switch (contentType) {\n case 'text/plain':\n body = JSON.stringify(body)\n break\n case 'application/json':\n body = JSON.stringify(body)\n break\n case 'application/x-www-form-urlencoded':\n const searchParams = new URLSearchParams()\n for (const prop in body) {\n searchParams.set(prop, body[prop])\n }\n body = searchParams\n break\n }\n }\n\n var options: any = {\n method: 'POST',\n headers,\n body\n }\n\n var { rejectUnauthorized } = connectionParams\n\n if (!rejectUnauthorized) {\n const httpsAgent = new https.Agent({\n rejectUnauthorized\n })\n options.agent = httpsAgent\n }\n\n const browser = (await getHeadlessPool().acquire()) as any\n const page = await browser.newPage()\n\n try {\n page.on('console', async msg => {\n console.log(`[browser ${msg.type()}] ${msg.text()}`)\n })\n\n page.on('requestfailed', request => {\n console.log('Request failed:', request.url())\n })\n\n await page.goto(endpoint, { waitUntil: 'networkidle2' })\n\n const response = await page.evaluate(\n async (urlString, options) => {\n const response = await fetch(urlString, options)\n\n if (response.ok && response.headers.get('content-type').includes('application/json')) {\n return await response.json()\n } else {\n return await response.text()\n }\n },\n new URL(path, endpoint),\n options\n )\n\n return {\n data: response\n }\n } catch (e) {\n console.error(e)\n } finally {\n page.close()\n getHeadlessPool().release(browser)\n }\n}\n\nHeadlessPost.parameterSpec = [\n {\n type: 'string',\n name: 'path',\n label: 'path'\n },\n {\n type: 'http-headers',\n name: 'headers',\n label: 'headers'\n },\n {\n type: 'select',\n name: 'contentType',\n label: 'content-type',\n property: {\n options: [\n {\n display: '',\n value: ''\n },\n {\n display: 'application/json',\n value: 'application/json'\n },\n {\n display: 'text/plain',\n value: 'text/plain'\n },\n {\n display: 'application/x-www-form-urlencoded',\n value: 'application/x-www-form-urlencoded'\n }\n ]\n }\n },\n {\n type: 'scenario-step-input',\n name: 'accessor',\n label: 'accessor'\n }\n]\n\nTaskRegistry.registerTaskHandler('headless-post', HeadlessPost)\n"]}
|
|
@@ -16,10 +16,11 @@ async function HeadlessScrap(step, { logger, data, domain }) {
|
|
|
16
16
|
const browser = (await (0, headless_pool_for_scenario_1.getHeadlessPool)().acquire());
|
|
17
17
|
const page = await browser.newPage();
|
|
18
18
|
try {
|
|
19
|
-
page.on('console', msg => {
|
|
19
|
+
page.on('console', async (msg) => {
|
|
20
20
|
console.log(`[browser ${msg.type()}] ${msg.text()}`);
|
|
21
|
-
|
|
22
|
-
|
|
21
|
+
});
|
|
22
|
+
page.on('requestfailed', request => {
|
|
23
|
+
console.log('Request failed:', request.url());
|
|
23
24
|
});
|
|
24
25
|
await page.setExtraHTTPHeaders(headers);
|
|
25
26
|
await page.goto(new url_1.URL(path, endpoint), { waitUntil: 'networkidle2' });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"headless-scrap.js","sourceRoot":"","sources":["../../../server/engine/task/headless-scrap.ts"],"names":[],"mappings":";;AAAA,6BAAyB;AAEzB,oDAA+C;AAC/C,8DAAyD;AAEzD,mFAAoE;AAEpE,KAAK,UAAU,aAAa,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IACzD,IAAI,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAA;IAC9D,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,WAAW,IAAI,EAAE,CAAA;IAEzE,IAAI,UAAU,GAAG,sCAAiB,CAAC,2BAA2B,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAEtF,IAAI,CAAC,UAAU,EAAE;QACf,MAAM,IAAI,KAAK,CAAC,eAAe,cAAc,uBAAuB,CAAC,CAAA;KACtE;IAED,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,gBAAgB,EAAE,WAAW,GAAG,EAAE,EAAE,GAAG,UAAU,CAAA;IAEzE,IAAI,OAAO,mCACN,WAAW,GACX,cAAc,CAClB,CAAA;IAED,MAAM,OAAO,GAAG,CAAC,MAAM,IAAA,4CAAe,GAAE,CAAC,OAAO,EAAE,CAAQ,CAAA;IAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAA;IAEpC,IAAI;QACF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,
|
|
1
|
+
{"version":3,"file":"headless-scrap.js","sourceRoot":"","sources":["../../../server/engine/task/headless-scrap.ts"],"names":[],"mappings":";;AAAA,6BAAyB;AAEzB,oDAA+C;AAC/C,8DAAyD;AAEzD,mFAAoE;AAEpE,KAAK,UAAU,aAAa,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IACzD,IAAI,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAA;IAC9D,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,WAAW,IAAI,EAAE,CAAA;IAEzE,IAAI,UAAU,GAAG,sCAAiB,CAAC,2BAA2B,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAEtF,IAAI,CAAC,UAAU,EAAE;QACf,MAAM,IAAI,KAAK,CAAC,eAAe,cAAc,uBAAuB,CAAC,CAAA;KACtE;IAED,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,gBAAgB,EAAE,WAAW,GAAG,EAAE,EAAE,GAAG,UAAU,CAAA;IAEzE,IAAI,OAAO,mCACN,WAAW,GACX,cAAc,CAClB,CAAA;IAED,MAAM,OAAO,GAAG,CAAC,MAAM,IAAA,4CAAe,GAAE,CAAC,OAAO,EAAE,CAAQ,CAAA;IAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAA;IAEpC,IAAI;QACF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAC,GAAG,EAAC,EAAE;YAC7B,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QACtD,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE;YACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAA;QACvC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,SAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAA;QAEvE,MAAM,MAAM,GAAG,EAAE,CAAA;QAEjB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;YAChC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAA;YAChC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE;gBACjD,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAA;YAC5D,CAAC,CAAC,CAAA;SACH;QAED,OAAO;YACL,IAAI,EAAE,MAAM;SACb,CAAA;KACF;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;KACjB;YAAS;QACR,IAAI,CAAC,KAAK,EAAE,CAAA;QACZ,IAAA,4CAAe,GAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;KACnC;AACH,CAAC;AAED,aAAa,CAAC,aAAa,GAAG;IAC5B;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,MAAM;KACd;IACD;QACE,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,SAAS;KACjB;IACD;QACE,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,WAAW;KACnB;CACF,CAAA;AAED,4BAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAA","sourcesContent":["import { URL } from 'url'\n\nimport { TaskRegistry } from '../task-registry'\nimport { ConnectionManager } from '../connection-manager'\n\nimport { getHeadlessPool } from './utils/headless-pool-for-scenario'\n\nasync function HeadlessScrap(step, { logger, data, domain }) {\n var { connection: connectionName, params: stepOptions } = step\n var { headers: requestHeaders, path, selectors = [] } = stepOptions || {}\n\n var connection = ConnectionManager.getConnectionInstanceByName(domain, connectionName)\n\n if (!connection) {\n throw new Error(`connection '${connectionName}' is not established.`)\n }\n\n var { endpoint, params: connectionParams, authHeaders = {} } = connection\n\n var headers = {\n ...authHeaders,\n ...requestHeaders\n }\n\n const browser = (await getHeadlessPool().acquire()) as any\n const page = await browser.newPage()\n\n try {\n page.on('console', async msg => {\n console.log(`[browser ${msg.type()}] ${msg.text()}`)\n })\n\n page.on('requestfailed', request => {\n console.log('Request failed:', request.url())\n })\n\n await page.setExtraHTTPHeaders(headers)\n await page.goto(new URL(path, endpoint), { waitUntil: 'networkidle2' })\n\n const result = {}\n\n for (const selector of selectors) {\n const { text, value } = selector\n result[text] = await page.$$eval(value, elements => {\n return elements.map(element => element.textContent.trim())\n })\n }\n\n return {\n data: result\n }\n } catch (e) {\n console.error(e)\n } finally {\n page.close()\n getHeadlessPool().release(browser)\n }\n}\n\nHeadlessScrap.parameterSpec = [\n {\n type: 'string',\n name: 'path',\n label: 'path'\n },\n {\n type: 'http-headers',\n name: 'headers',\n label: 'headers'\n },\n {\n type: 'options',\n name: 'selectors',\n label: 'selectors'\n }\n]\n\nTaskRegistry.registerTaskHandler('headless-scrap', HeadlessScrap)\n"]}
|