@things-factory/integration-base 9.0.0 → 9.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -56,7 +56,16 @@ async function HeadlessScrap(step, { logger, data, domain }) {
|
|
|
56
56
|
for (const selector of selectors) {
|
|
57
57
|
const { text, value } = selector;
|
|
58
58
|
result[text] = await page.$$eval(value, elements => {
|
|
59
|
-
return elements.map(element =>
|
|
59
|
+
return elements.map(element => {
|
|
60
|
+
if (element instanceof HTMLInputElement ||
|
|
61
|
+
element instanceof HTMLTextAreaElement ||
|
|
62
|
+
element instanceof HTMLSelectElement) {
|
|
63
|
+
return element.value;
|
|
64
|
+
}
|
|
65
|
+
else {
|
|
66
|
+
return element.textContent?.trim();
|
|
67
|
+
}
|
|
68
|
+
});
|
|
60
69
|
});
|
|
61
70
|
}
|
|
62
71
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"headless-scrap.js","sourceRoot":"","sources":["../../../server/engine/task/headless-scrap.ts"],"names":[],"mappings":";;AAAA,6BAAyB;AAEzB,oDAA+C;AAC/C,8DAAyD;AAEzD,KAAK,UAAU,aAAa,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IACzD,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAA;IAChE,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,GAAG,EAAE,EAAE,gBAAgB,EAAE,cAAc,EAAE,GAAG,WAAW,IAAI,EAAE,CAAA;IAE7G,MAAM,UAAU,GAAG,sCAAiB,CAAC,2BAA2B,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAExF,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,eAAe,cAAc,uBAAuB,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,WAAW,EAAE,GAAG,UAAU,CAAA;IAE1F,MAAM,OAAO,GAAG;QACd,GAAG,cAAc;KAClB,CAAA;IAED,MAAM,IAAI,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAEvC,IAAI,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAC,GAAG,EAAC,EAAE;YAC7B,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QACtD,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE;YACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;YAC9B,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;YACtC,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;YAC5C,OAAO,CAAC,GAAG,CAAC,mBAAmB,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;YAC9D,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;YAE5C,oBAAoB;YACpB,IAAI,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,gBAAgB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;YACnD,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAA;QACvC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,SAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAA;QAEvE,yCAAyC;QACzC,IAAI,gBAAgB,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,eAAe,CACxB,eAAe,CAAC,EAAE;oBAChB,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;oBAC/D,OAAO,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;wBAChC,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;wBAC3C,OAAO,EAAE,IAAI,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAA;oBACjE,CAAC,CAAC,CAAA;gBACJ,CAAC,EACD,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,EAC5D,gBAAgB,CAAC,kBAAkB;iBACpC,CAAA;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,oBAAoB,gBAAgB,kBAAkB,EAAE,CAAC,CAAC,CAAA;gBACvE,MAAM,CAAC,CAAA;YACT,CAAC;QACH,CAAC;aAAM,IAAI,cAAc,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAA;QACnD,CAAC;QAED,MAAM,MAAM,GAAG,EAAE,CAAA;QAEjB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAA;YAChC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE;gBACjD,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"headless-scrap.js","sourceRoot":"","sources":["../../../server/engine/task/headless-scrap.ts"],"names":[],"mappings":";;AAAA,6BAAyB;AAEzB,oDAA+C;AAC/C,8DAAyD;AAEzD,KAAK,UAAU,aAAa,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IACzD,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAA;IAChE,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,GAAG,EAAE,EAAE,gBAAgB,EAAE,cAAc,EAAE,GAAG,WAAW,IAAI,EAAE,CAAA;IAE7G,MAAM,UAAU,GAAG,sCAAiB,CAAC,2BAA2B,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAExF,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,eAAe,cAAc,uBAAuB,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,WAAW,EAAE,GAAG,UAAU,CAAA;IAE1F,MAAM,OAAO,GAAG;QACd,GAAG,cAAc;KAClB,CAAA;IAED,MAAM,IAAI,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAEvC,IAAI,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAC,GAAG,EAAC,EAAE;YAC7B,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QACtD,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE;YACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;YAC9B,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;YACtC,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;YAC5C,OAAO,CAAC,GAAG,CAAC,mBAAmB,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;YAC9D,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;YAE5C,oBAAoB;YACpB,IAAI,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,gBAAgB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;YACnD,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAA;QACvC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,SAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAA;QAEvE,yCAAyC;QACzC,IAAI,gBAAgB,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,eAAe,CACxB,eAAe,CAAC,EAAE;oBAChB,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;oBAC/D,OAAO,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;wBAChC,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;wBAC3C,OAAO,EAAE,IAAI,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAA;oBACjE,CAAC,CAAC,CAAA;gBACJ,CAAC,EACD,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,EAC5D,gBAAgB,CAAC,kBAAkB;iBACpC,CAAA;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,oBAAoB,gBAAgB,kBAAkB,EAAE,CAAC,CAAC,CAAA;gBACvE,MAAM,CAAC,CAAA;YACT,CAAC;QACH,CAAC;aAAM,IAAI,cAAc,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAA;QACnD,CAAC;QAED,MAAM,MAAM,GAAG,EAAE,CAAA;QAEjB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAA;YAChC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE;gBACjD,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;oBAC5B,IACE,OAAO,YAAY,gBAAgB;wBACnC,OAAO,YAAY,mBAAmB;wBACtC,OAAO,YAAY,iBAAiB,EACpC,CAAC;wBACD,OAAO,OAAO,CAAC,KAAK,CAAA;oBACtB,CAAC;yBAAM,CAAC;wBACN,OAAO,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,CAAA;oBACpC,CAAC;gBACH,CAAC,CAAC,CAAA;YACJ,CAAC,CAAC,CAAA;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;SACb,CAAA;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,CAAA;IACT,CAAC;YAAS,CAAC;QACT,MAAM,WAAW,CAAC,IAAI,CAAC,CAAA;IACzB,CAAC;AACH,CAAC;AAED,aAAa,CAAC,aAAa,GAAG;IAC5B;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,MAAM;KACd;IACD;QACE,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,SAAS;KACjB;IACD;QACE,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,WAAW;KACnB;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,kBAAkB;QACxB,KAAK,EAAE,oBAAoB;KAC5B;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,kBAAkB;KAC1B;CACF,CAAA;AAED,aAAa,CAAC,IAAI,GAAG,iCAAiC,CAAA;AAEtD,4BAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAA","sourcesContent":["import { URL } from 'url'\n\nimport { TaskRegistry } from '../task-registry'\nimport { ConnectionManager } from '../connection-manager'\n\nasync function HeadlessScrap(step, { logger, data, domain }) {\n const { connection: connectionName, params: stepOptions } = step\n const { headers: requestHeaders, path, selectors = [], waitForSelectors, waitForTimeout } = stepOptions || {}\n\n const connection = ConnectionManager.getConnectionInstanceByName(domain, connectionName)\n\n if (!connection) {\n throw new Error(`Connection '${connectionName}' is not established.`)\n }\n\n const { endpoint, params: connectionParams, acquireSessionPage, releasePage } = connection\n\n const headers = {\n ...requestHeaders\n }\n\n const page = await acquireSessionPage()\n\n try {\n page.on('console', async msg => {\n console.log(`[browser ${msg.type()}] ${msg.text()}`)\n })\n\n page.on('requestfailed', request => {\n console.log('Request failed:')\n console.log(`- URL: ${request.url()}`)\n console.log(`- Method: ${request.method()}`)\n console.log(`- Failure Text: ${request.failure()?.errorText}`)\n console.log(`- Headers:`, request.headers())\n\n // POST 데이터 (필요한 경우)\n if (request.postData()) {\n console.log(`- Post Data: ${request.postData()}`)\n }\n })\n\n await page.setExtraHTTPHeaders(headers)\n await page.goto(new URL(path, endpoint), { waitUntil: 'networkidle2' })\n\n // waitForSelectors, waitForTimeout 처리 추가\n if (waitForSelectors) {\n try {\n await page.waitForFunction(\n selectorsString => {\n const selectors = selectorsString.split(',').map(s => s.trim())\n return selectors.every(selector => {\n const el = document.querySelector(selector)\n return el && el.textContent && el.textContent.trim().length > 0\n })\n },\n { timeout: waitForTimeout ? Number(waitForTimeout) : 10000 },\n waitForSelectors // 콤마로 구분된 셀렉터 문자열\n )\n } catch (e) {\n logger.error(`waitForSelectors(${waitForSelectors}) 값이 모두 채워지지 않음:`, e)\n throw e\n }\n } else if (waitForTimeout) {\n await page.waitForTimeout(Number(waitForTimeout))\n }\n\n const result = {}\n\n for (const selector of selectors) {\n const { text, value } = selector\n result[text] = await page.$$eval(value, elements => {\n return elements.map(element => {\n if (\n element instanceof HTMLInputElement ||\n element instanceof HTMLTextAreaElement ||\n element instanceof HTMLSelectElement\n ) {\n return element.value\n } else {\n return element.textContent?.trim()\n }\n })\n })\n }\n\n return {\n data: result\n }\n } catch (e) {\n logger.error('Error in HeadlessScrap:', e)\n throw e\n } finally {\n await releasePage(page)\n }\n}\n\nHeadlessScrap.parameterSpec = [\n {\n type: 'string',\n name: 'path',\n label: 'path'\n },\n {\n type: 'http-headers',\n name: 'headers',\n label: 'headers'\n },\n {\n type: 'options',\n name: 'selectors',\n label: 'selectors'\n },\n {\n type: 'string',\n name: 'waitForSelectors',\n label: 'wait-for-selectors'\n },\n {\n type: 'string',\n name: 'waitForTimeout',\n label: 'wait-for-timeout'\n }\n]\n\nHeadlessScrap.help = 'integration/task/headless-scrap'\n\nTaskRegistry.registerTaskHandler('headless-scrap', HeadlessScrap)\n"]}
|