@hanivanrizky/nestjs-browser-action 0.20.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -31
- package/dist/browser-action.module-definition.d.ts +1 -0
- package/dist/browser-action.module.d.ts +3 -0
- package/dist/browser-action.module.js +146 -27
- package/dist/browser-action.module.js.map +1 -1
- package/dist/common/index.d.ts +1 -0
- package/dist/common/index.js +18 -0
- package/dist/common/index.js.map +1 -0
- package/dist/common/tokens.d.ts +7 -0
- package/dist/common/tokens.js +36 -0
- package/dist/common/tokens.js.map +1 -0
- package/dist/decorators/index.d.ts +1 -0
- package/dist/decorators/index.js +1 -0
- package/dist/decorators/index.js.map +1 -1
- package/dist/decorators/inject-browser.decorator.d.ts +2 -1
- package/dist/decorators/inject-browser.decorator.js +2 -1
- package/dist/decorators/inject-browser.decorator.js.map +1 -1
- package/dist/decorators/inject-page-controller.decorator.d.ts +2 -0
- package/dist/decorators/inject-page-controller.decorator.js +8 -0
- package/dist/decorators/inject-page-controller.decorator.js.map +1 -0
- package/dist/decorators/inject-page.decorator.d.ts +2 -1
- package/dist/decorators/inject-page.decorator.js +2 -1
- package/dist/decorators/inject-page.decorator.js.map +1 -1
- package/dist/examples/named-browser-example.d.ts +1 -0
- package/dist/examples/named-browser-example.js +183 -0
- package/dist/examples/named-browser-example.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/interfaces/browser-action-options.d.ts +1 -0
- package/dist/operators/capture.operator.d.ts +11 -0
- package/dist/operators/capture.operator.js +62 -0
- package/dist/operators/capture.operator.js.map +1 -0
- package/dist/operators/container.operator.d.ts +13 -0
- package/dist/operators/container.operator.js +149 -0
- package/dist/operators/container.operator.js.map +1 -0
- package/dist/operators/extraction.operator.d.ts +22 -0
- package/dist/operators/extraction.operator.js +204 -0
- package/dist/operators/extraction.operator.js.map +1 -0
- package/dist/operators/index.d.ts +5 -0
- package/dist/operators/index.js +14 -0
- package/dist/operators/index.js.map +1 -0
- package/dist/operators/pagination.operator.d.ts +24 -0
- package/dist/operators/pagination.operator.js +182 -0
- package/dist/operators/pagination.operator.js.map +1 -0
- package/dist/operators/workflow.operator.d.ts +35 -0
- package/dist/operators/workflow.operator.js +501 -0
- package/dist/operators/workflow.operator.js.map +1 -0
- package/dist/services/browser-action.service.d.ts +10 -30
- package/dist/services/browser-action.service.js +72 -995
- package/dist/services/browser-action.service.js.map +1 -1
- package/dist/services/browser-holder.d.ts +7 -0
- package/dist/services/browser-holder.js +26 -0
- package/dist/services/browser-holder.js.map +1 -0
- package/dist/services/browser-manager.service.js +3 -2
- package/dist/services/browser-manager.service.js.map +1 -1
- package/dist/services/browser-pool.service.d.ts +4 -4
- package/dist/services/browser-pool.service.js +45 -104
- package/dist/services/browser-pool.service.js.map +1 -1
- package/dist/services/named-browser-shutdown.d.ts +9 -0
- package/dist/services/named-browser-shutdown.js +47 -0
- package/dist/services/named-browser-shutdown.js.map +1 -0
- package/dist/services/page-controller.d.ts +39 -0
- package/dist/services/page-controller.js +325 -0
- package/dist/services/page-controller.js.map +1 -0
- package/dist/services/page-session.d.ts +26 -0
- package/dist/services/page-session.js +135 -0
- package/dist/services/page-session.js.map +1 -0
- package/dist/services/page.service.js +14 -3
- package/dist/services/page.service.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/dist/utils/browser-launcher.d.ts +6 -0
- package/dist/utils/browser-launcher.js +100 -0
- package/dist/utils/browser-launcher.js.map +1 -0
- package/package.json +2 -2
|
@@ -15,59 +15,63 @@ var BrowserActionService_1;
|
|
|
15
15
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
16
|
exports.BrowserActionService = void 0;
|
|
17
17
|
const common_1 = require("@nestjs/common");
|
|
18
|
-
const
|
|
19
|
-
const path_1 = require("path");
|
|
20
|
-
const class_validator_1 = require("class-validator");
|
|
18
|
+
const core_1 = require("@nestjs/core");
|
|
21
19
|
const path_util_1 = require("../utils/path.util");
|
|
22
20
|
const workflow_validator_1 = require("../validators/workflow.validator");
|
|
23
21
|
const page_service_1 = require("./page.service");
|
|
24
22
|
const cookie_service_1 = require("./cookie.service");
|
|
25
23
|
const cleansing_service_1 = require("./cleansing.service");
|
|
26
24
|
const pipe_engine_1 = require("../pipes/pipe-engine");
|
|
25
|
+
const operators_1 = require("../operators");
|
|
27
26
|
const user_agent_util_1 = require("../utils/user-agent.util");
|
|
28
27
|
const logger_util_1 = require("../utils/logger.util");
|
|
29
|
-
const delay_util_1 = require("../utils/delay.util");
|
|
30
28
|
const dom_util_1 = require("../utils/dom.util");
|
|
31
29
|
const truncate_log_util_1 = require("../utils/truncate-log.util");
|
|
32
30
|
const browser_action_constants_1 = require("../constants/browser-action.constants");
|
|
33
|
-
const isTargetClosed = (err) => err instanceof Error &&
|
|
34
|
-
(err.message.includes('Target closed') ||
|
|
35
|
-
err.message.includes('No target with given id') ||
|
|
36
|
-
err.message.includes('detached Frame') ||
|
|
37
|
-
err.message.includes('Attempted to use detached') ||
|
|
38
|
-
err.message.includes('Execution context was destroyed') ||
|
|
39
|
-
err.message.includes('context was destroyed') ||
|
|
40
|
-
err.message.includes('Session closed') ||
|
|
41
|
-
err.message.includes('Navigating frame was detached'));
|
|
42
31
|
let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
43
32
|
pageService;
|
|
44
33
|
cookieService;
|
|
45
34
|
cleansingService;
|
|
35
|
+
moduleRef;
|
|
46
36
|
moduleOptions;
|
|
47
37
|
logger;
|
|
48
38
|
pipeEngine = new pipe_engine_1.PipeEngine();
|
|
49
39
|
activeDebugLogMaxLength;
|
|
50
|
-
|
|
40
|
+
extraction;
|
|
41
|
+
capture;
|
|
42
|
+
container;
|
|
43
|
+
workflow;
|
|
44
|
+
pagination;
|
|
45
|
+
constructor(pageService, cookieService, cleansingService, moduleRef, moduleOptions) {
|
|
51
46
|
this.pageService = pageService;
|
|
52
47
|
this.cookieService = cookieService;
|
|
53
48
|
this.cleansingService = cleansingService;
|
|
49
|
+
this.moduleRef = moduleRef;
|
|
54
50
|
this.moduleOptions = moduleOptions;
|
|
55
51
|
this.logger = new logger_util_1.LoggerWithLevel(BrowserActionService_1.name, this.pageService.getLogLevel());
|
|
56
52
|
this.activeDebugLogMaxLength =
|
|
57
53
|
moduleOptions?.debugLogMaxLength ?? browser_action_constants_1.DEFAULT_DEBUG_LOG_MAX_LENGTH;
|
|
54
|
+
this.extraction = new operators_1.ExtractionOperator(this.pipeEngine, this.logger);
|
|
55
|
+
this.capture = new operators_1.CaptureOperator(this.logger);
|
|
56
|
+
this.container = new operators_1.ContainerOperator(this.extraction, this.pipeEngine, this.logger);
|
|
57
|
+
this.workflow = new operators_1.WorkflowOperator(this.extraction, this.container, this.pipeEngine, this.cleansingService, this.logger, () => this.cookieService);
|
|
58
|
+
this.pagination = new operators_1.PaginationOperator(this.logger);
|
|
59
|
+
}
|
|
60
|
+
async newPage() {
|
|
61
|
+
return this.moduleRef.resolve(page_service_1.PageService, undefined, { strict: false });
|
|
58
62
|
}
|
|
59
63
|
async takeScreenshot(url, path, options, scraperOptions) {
|
|
60
64
|
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Taking screenshot of ${url}`));
|
|
61
65
|
const cloak = scraperOptions?.useRandomUserAgent
|
|
62
66
|
? { ...scraperOptions?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
|
|
63
67
|
: scraperOptions?.cloak;
|
|
64
|
-
const
|
|
68
|
+
const pageService = await this.newPage();
|
|
69
|
+
const page = await pageService.navigateTo(url, this.buildNavOptions(scraperOptions), cloak, scraperOptions?.interceptResource);
|
|
65
70
|
try {
|
|
66
|
-
|
|
67
|
-
return result;
|
|
71
|
+
return await this.capture.screenshot(page, path, options);
|
|
68
72
|
}
|
|
69
73
|
finally {
|
|
70
|
-
await
|
|
74
|
+
await pageService.closePage();
|
|
71
75
|
}
|
|
72
76
|
}
|
|
73
77
|
async generatePDF(url, path, options, scraperOptions) {
|
|
@@ -75,99 +79,39 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
75
79
|
const cloak = scraperOptions?.useRandomUserAgent
|
|
76
80
|
? { ...scraperOptions?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
|
|
77
81
|
: scraperOptions?.cloak;
|
|
78
|
-
const
|
|
82
|
+
const pageService = await this.newPage();
|
|
83
|
+
const page = await pageService.navigateTo(url, this.buildNavOptions(scraperOptions), cloak, scraperOptions?.interceptResource);
|
|
79
84
|
try {
|
|
80
|
-
|
|
81
|
-
return Buffer.from(pdf);
|
|
85
|
+
return await this.capture.pdf(page, path, options);
|
|
82
86
|
}
|
|
83
87
|
finally {
|
|
84
|
-
await
|
|
88
|
+
await pageService.closePage();
|
|
85
89
|
}
|
|
86
90
|
}
|
|
87
91
|
async captureTlsFingerprint(path, url = browser_action_constants_1.TLS_CAPTURE_URL) {
|
|
88
92
|
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Capturing TLS fingerprint from ${url}`));
|
|
89
|
-
const
|
|
90
|
-
|
|
93
|
+
const pageService = await this.newPage();
|
|
94
|
+
const page = await pageService.navigateTo(url);
|
|
91
95
|
try {
|
|
92
|
-
|
|
96
|
+
return await this.capture.tlsFingerprint(page, path);
|
|
93
97
|
}
|
|
94
98
|
finally {
|
|
95
|
-
await
|
|
99
|
+
await pageService.closePage();
|
|
96
100
|
}
|
|
97
|
-
const data = JSON.parse(rawJson);
|
|
98
|
-
const fingerprint = this.buildTlsFingerprint(data);
|
|
99
|
-
await fs_1.promises.mkdir((0, path_1.dirname)(path), { recursive: true });
|
|
100
|
-
await fs_1.promises.writeFile(path, JSON.stringify(fingerprint, null, 2), 'utf-8');
|
|
101
|
-
this.logger.log(`Saved TLS fingerprint to ${path}`);
|
|
102
|
-
return fingerprint;
|
|
103
|
-
}
|
|
104
|
-
buildTlsFingerprint(data) {
|
|
105
|
-
const tls = (data.tls ?? {});
|
|
106
|
-
const http2 = (data.http2 ?? {});
|
|
107
|
-
const extensions = Array.isArray(tls.extensions)
|
|
108
|
-
? tls.extensions
|
|
109
|
-
: [];
|
|
110
|
-
const sentFrames = Array.isArray(http2.sent_frames)
|
|
111
|
-
? http2.sent_frames
|
|
112
|
-
: [];
|
|
113
|
-
const headersFrame = sentFrames.find((frame) => frame.frame_type === 'HEADERS');
|
|
114
|
-
return {
|
|
115
|
-
capturedAt: new Date().toISOString(),
|
|
116
|
-
ip: data.ip ?? '',
|
|
117
|
-
httpVersion: data.http_version ?? '',
|
|
118
|
-
method: data.method ?? '',
|
|
119
|
-
userAgent: data.user_agent ?? '',
|
|
120
|
-
ja3: tls.ja3 ?? '',
|
|
121
|
-
ja3Hash: tls.ja3_hash ?? '',
|
|
122
|
-
ja4: tls.ja4 ?? '',
|
|
123
|
-
ja4_r: tls.ja4_r,
|
|
124
|
-
peetprint: tls.peetprint ?? '',
|
|
125
|
-
peetprintHash: tls.peetprint_hash ?? '',
|
|
126
|
-
ciphers: Array.isArray(tls.ciphers) ? tls.ciphers : [],
|
|
127
|
-
tlsExtensions: extensions.map((ext) => String(ext.name)),
|
|
128
|
-
akamaiFingerprint: http2.akamai_fingerprint ?? '',
|
|
129
|
-
akamaiFingerprintHash: http2.akamai_fingerprint_hash ?? '',
|
|
130
|
-
headers: Array.isArray(headersFrame?.headers)
|
|
131
|
-
? headersFrame.headers
|
|
132
|
-
: [],
|
|
133
|
-
raw: data,
|
|
134
|
-
};
|
|
135
101
|
}
|
|
136
102
|
async scrape(url, selectors, options) {
|
|
137
103
|
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Scraping ${url}`));
|
|
138
104
|
const cloak = options?.useRandomUserAgent
|
|
139
105
|
? { ...options?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
|
|
140
106
|
: options?.cloak;
|
|
141
|
-
const
|
|
142
|
-
const
|
|
143
|
-
const evalOne = page['$eval'].bind(page);
|
|
107
|
+
const pageService = await this.newPage();
|
|
108
|
+
const page = await pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
|
|
144
109
|
try {
|
|
145
|
-
await
|
|
146
|
-
const { selector, attribute } = this.parseSelector(rawSelector);
|
|
147
|
-
try {
|
|
148
|
-
const value = attribute
|
|
149
|
-
? await evalOne(selector, (el, attr) => el.getAttribute(attr), attribute)
|
|
150
|
-
: await evalOne(selector, (el) => el.textContent);
|
|
151
|
-
result[key] = options?.pipes?.[key]
|
|
152
|
-
? this.pipeEngine.apply(typeof value === 'string' ? value : String(value ?? ''), options.pipes[key], url)
|
|
153
|
-
: value;
|
|
154
|
-
}
|
|
155
|
-
catch (err) {
|
|
156
|
-
this.logger.warn(`Failed to scrape '${key}' (${rawSelector}): ${err instanceof Error ? err.message : String(err)}`);
|
|
157
|
-
}
|
|
158
|
-
}));
|
|
110
|
+
return await this.extraction.extractSingle(page, url, selectors, options?.pipes);
|
|
159
111
|
}
|
|
160
112
|
finally {
|
|
161
|
-
await
|
|
162
|
-
}
|
|
163
|
-
return result;
|
|
164
|
-
}
|
|
165
|
-
parseSelector(raw) {
|
|
166
|
-
const match = raw.match(/^(.*)@([A-Za-z_][\w-]*)$/);
|
|
167
|
-
if (match && match[1].trim() !== '') {
|
|
168
|
-
return { selector: match[1], attribute: match[2] };
|
|
113
|
+
await pageService.closePage();
|
|
169
114
|
}
|
|
170
|
-
return { selector: raw };
|
|
171
115
|
}
|
|
172
116
|
buildNavOptions(options) {
|
|
173
117
|
if (!options?.waitUntil && !options?.timeout)
|
|
@@ -182,12 +126,13 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
182
126
|
const cloak = options?.useRandomUserAgent
|
|
183
127
|
? { ...options?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
|
|
184
128
|
: options?.cloak;
|
|
185
|
-
const
|
|
129
|
+
const pageService = await this.newPage();
|
|
130
|
+
const page = await pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
|
|
186
131
|
const result = {};
|
|
187
132
|
const evalAll = page['$$eval'].bind(page);
|
|
188
|
-
Object.entries(selectors).forEach(([key, selector]) => this.validateSelector(key, selector));
|
|
133
|
+
Object.entries(selectors).forEach(([key, selector]) => this.extraction.validateSelector(key, selector));
|
|
189
134
|
await Promise.all(Object.entries(selectors).map(async ([key, rawSelector]) => {
|
|
190
|
-
const { selector, attribute } = this.parseSelector(rawSelector);
|
|
135
|
+
const { selector, attribute } = this.extraction.parseSelector(rawSelector);
|
|
191
136
|
try {
|
|
192
137
|
const isXPath = (0, dom_util_1.isXPathSelector)(selector);
|
|
193
138
|
let values;
|
|
@@ -232,7 +177,7 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
232
177
|
this.logger.warn(`Failed to scrape '${key}' (${rawSelector}): ${err instanceof Error ? err.message : String(err)}`);
|
|
233
178
|
}
|
|
234
179
|
}));
|
|
235
|
-
await
|
|
180
|
+
await pageService.closePage();
|
|
236
181
|
return result;
|
|
237
182
|
}
|
|
238
183
|
async scrapeContainerFields(url, descriptor, options) {
|
|
@@ -240,12 +185,13 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
240
185
|
const cloak = options?.useRandomUserAgent
|
|
241
186
|
? { ...options?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
|
|
242
187
|
: options?.cloak;
|
|
188
|
+
const pageService = await this.newPage();
|
|
243
189
|
let page;
|
|
244
190
|
try {
|
|
245
|
-
page = await
|
|
191
|
+
page = await pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
|
|
246
192
|
}
|
|
247
193
|
catch (err) {
|
|
248
|
-
await
|
|
194
|
+
await pageService.closePage();
|
|
249
195
|
throw err;
|
|
250
196
|
}
|
|
251
197
|
try {
|
|
@@ -273,7 +219,7 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
273
219
|
return { items, pagination: raw.pagination };
|
|
274
220
|
}
|
|
275
221
|
finally {
|
|
276
|
-
await
|
|
222
|
+
await pageService.closePage();
|
|
277
223
|
}
|
|
278
224
|
}
|
|
279
225
|
async evaluateWebsite(options) {
|
|
@@ -325,7 +271,7 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
325
271
|
const template = pagination.urlTemplate ?? url;
|
|
326
272
|
const startPage = pagination.startPage ?? 2;
|
|
327
273
|
const { items: page1Items } = await this.scrapeContainerFields(url, descriptor, scraperOpts);
|
|
328
|
-
const { items: restItems, pages: restPages } = await this.paginateUrlIncrement(template, async (pageUrl) => {
|
|
274
|
+
const { items: restItems, pages: restPages } = await this.pagination.paginateUrlIncrement(template, async (pageUrl) => {
|
|
329
275
|
const { items } = await this.scrapeContainerFields(pageUrl, descriptor, scraperOpts);
|
|
330
276
|
return items;
|
|
331
277
|
}, pagination, startPage);
|
|
@@ -337,12 +283,13 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
337
283
|
const cloak = scraperOpts.useRandomUserAgent
|
|
338
284
|
? { ...scraperOpts.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
|
|
339
285
|
: scraperOpts.cloak;
|
|
286
|
+
const pageService = await this.newPage();
|
|
340
287
|
let page;
|
|
341
288
|
try {
|
|
342
|
-
page = await
|
|
289
|
+
page = await pageService.navigateTo(url, this.buildNavOptions(scraperOpts), cloak, scraperOpts.interceptResource);
|
|
343
290
|
}
|
|
344
291
|
catch (err) {
|
|
345
|
-
await
|
|
292
|
+
await pageService.closePage();
|
|
346
293
|
throw err;
|
|
347
294
|
}
|
|
348
295
|
try {
|
|
@@ -370,271 +317,29 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
370
317
|
});
|
|
371
318
|
};
|
|
372
319
|
if (pagination.type === 'click-next') {
|
|
373
|
-
const { items, pages } = await this.paginateClickNext(page, containerFn, pagination);
|
|
320
|
+
const { items, pages } = await this.pagination.paginateClickNext(page, containerFn, pagination);
|
|
374
321
|
return { results: items, totalPages: pages };
|
|
375
322
|
}
|
|
376
323
|
else if (pagination.type === 'load-more') {
|
|
377
|
-
const { items, pages } = await this.paginateLoadMore(page, containerFn, pagination);
|
|
324
|
+
const { items, pages } = await this.pagination.paginateLoadMore(page, containerFn, pagination);
|
|
378
325
|
return { results: items, totalPages: pages };
|
|
379
326
|
}
|
|
380
327
|
else {
|
|
381
|
-
const { items, pages } = await this.paginateInfiniteScroll(page, containerFn, pagination);
|
|
328
|
+
const { items, pages } = await this.pagination.paginateInfiniteScroll(page, containerFn, pagination);
|
|
382
329
|
return { results: items, totalPages: pages };
|
|
383
330
|
}
|
|
384
331
|
}
|
|
385
332
|
finally {
|
|
386
|
-
await
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
async findPaginationElement(page, selector) {
|
|
390
|
-
if (!(0, dom_util_1.isXPathSelector)(selector)) {
|
|
391
|
-
return page.$(selector);
|
|
392
|
-
}
|
|
393
|
-
const pageWithX = page;
|
|
394
|
-
if (typeof pageWithX.$x === 'function') {
|
|
395
|
-
const results = await pageWithX.$x(selector);
|
|
396
|
-
return results[0] ?? null;
|
|
397
|
-
}
|
|
398
|
-
const handle = await page
|
|
399
|
-
.evaluateHandle((sel) => {
|
|
400
|
-
const res = document.evaluate(sel, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
|
|
401
|
-
return res.singleNodeValue;
|
|
402
|
-
}, selector)
|
|
403
|
-
.then((h) => h.asElement());
|
|
404
|
-
return handle;
|
|
405
|
-
}
|
|
406
|
-
async paginateClickNext(page, containerFn, opts) {
|
|
407
|
-
const max = opts.maxPages ?? 10;
|
|
408
|
-
const wait = opts.waitAfter ?? 800;
|
|
409
|
-
const all = [];
|
|
410
|
-
let pages = 0;
|
|
411
|
-
for (let i = 0; i < max; i++) {
|
|
412
|
-
try {
|
|
413
|
-
const items = await containerFn(page);
|
|
414
|
-
all.push(...items);
|
|
415
|
-
pages++;
|
|
416
|
-
if (!opts.selector)
|
|
417
|
-
break;
|
|
418
|
-
const btn = await this.findPaginationElement(page, opts.selector);
|
|
419
|
-
if (!btn)
|
|
420
|
-
break;
|
|
421
|
-
await Promise.all([
|
|
422
|
-
page.waitForNavigation({ waitUntil: 'domcontentloaded' }),
|
|
423
|
-
btn.click(),
|
|
424
|
-
]);
|
|
425
|
-
await (0, delay_util_1.delay)(wait);
|
|
426
|
-
}
|
|
427
|
-
catch (err) {
|
|
428
|
-
if (isTargetClosed(err))
|
|
429
|
-
break;
|
|
430
|
-
if (err instanceof Error && err.message.includes('TimeoutError'))
|
|
431
|
-
break;
|
|
432
|
-
throw err;
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
return { items: all, pages };
|
|
436
|
-
}
|
|
437
|
-
async paginateLoadMore(page, containerFn, opts) {
|
|
438
|
-
const max = opts.maxPages ?? 10;
|
|
439
|
-
const wait = opts.waitAfter ?? 800;
|
|
440
|
-
let previousCount = 0;
|
|
441
|
-
let clicks = 0;
|
|
442
|
-
let lastItems = [];
|
|
443
|
-
for (let i = 0; i < max; i++) {
|
|
444
|
-
try {
|
|
445
|
-
const items = await containerFn(page);
|
|
446
|
-
if (items.length === previousCount) {
|
|
447
|
-
return { items, pages: clicks + 1 };
|
|
448
|
-
}
|
|
449
|
-
lastItems = items;
|
|
450
|
-
previousCount = items.length;
|
|
451
|
-
if (!opts.selector)
|
|
452
|
-
break;
|
|
453
|
-
const btn = await this.findPaginationElement(page, opts.selector);
|
|
454
|
-
if (!btn)
|
|
455
|
-
break;
|
|
456
|
-
await btn.click();
|
|
457
|
-
clicks++;
|
|
458
|
-
await (0, delay_util_1.delay)(wait);
|
|
459
|
-
}
|
|
460
|
-
catch (err) {
|
|
461
|
-
if (isTargetClosed(err))
|
|
462
|
-
break;
|
|
463
|
-
throw err;
|
|
464
|
-
}
|
|
465
|
-
}
|
|
466
|
-
try {
|
|
467
|
-
const finalItems = await containerFn(page);
|
|
468
|
-
return { items: finalItems, pages: clicks + 1 };
|
|
469
|
-
}
|
|
470
|
-
catch (err) {
|
|
471
|
-
if (isTargetClosed(err))
|
|
472
|
-
return { items: lastItems, pages: clicks + 1 };
|
|
473
|
-
throw err;
|
|
474
|
-
}
|
|
475
|
-
}
|
|
476
|
-
async paginateInfiniteScroll(page, containerFn, opts) {
|
|
477
|
-
const max = opts.maxPages ?? 10;
|
|
478
|
-
const wait = opts.waitAfter ?? 800;
|
|
479
|
-
let previousCount = 0;
|
|
480
|
-
let scrolls = 0;
|
|
481
|
-
let lastItems = [];
|
|
482
|
-
for (let i = 0; i < max; i++) {
|
|
483
|
-
try {
|
|
484
|
-
const items = await containerFn(page);
|
|
485
|
-
if (opts.endSelector) {
|
|
486
|
-
const endEl = await this.findPaginationElement(page, opts.endSelector);
|
|
487
|
-
if (endEl)
|
|
488
|
-
return { items, pages: scrolls + 1 };
|
|
489
|
-
}
|
|
490
|
-
if (items.length === previousCount)
|
|
491
|
-
return { items, pages: scrolls + 1 };
|
|
492
|
-
lastItems = items;
|
|
493
|
-
previousCount = items.length;
|
|
494
|
-
if (opts.selector) {
|
|
495
|
-
const sentinel = await this.findPaginationElement(page, opts.selector);
|
|
496
|
-
if (sentinel) {
|
|
497
|
-
await page.evaluate((el) => el.scrollIntoView(), sentinel);
|
|
498
|
-
}
|
|
499
|
-
else {
|
|
500
|
-
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
else {
|
|
504
|
-
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
|
505
|
-
}
|
|
506
|
-
scrolls++;
|
|
507
|
-
await (0, delay_util_1.delay)(wait);
|
|
508
|
-
}
|
|
509
|
-
catch (err) {
|
|
510
|
-
if (isTargetClosed(err))
|
|
511
|
-
break;
|
|
512
|
-
throw err;
|
|
513
|
-
}
|
|
514
|
-
}
|
|
515
|
-
try {
|
|
516
|
-
const finalItems = await containerFn(page);
|
|
517
|
-
return { items: finalItems, pages: scrolls + 1 };
|
|
518
|
-
}
|
|
519
|
-
catch (err) {
|
|
520
|
-
if (isTargetClosed(err))
|
|
521
|
-
return { items: lastItems, pages: scrolls + 1 };
|
|
522
|
-
throw err;
|
|
523
|
-
}
|
|
524
|
-
}
|
|
525
|
-
async paginateUrlIncrement(urlTemplate, containerFn, opts, startPage) {
|
|
526
|
-
const max = opts.maxPages ?? 10;
|
|
527
|
-
const wait = opts.waitAfter ?? 800;
|
|
528
|
-
const all = [];
|
|
529
|
-
let pagesScraped = 0;
|
|
530
|
-
for (let p = startPage; p < startPage + max; p++) {
|
|
531
|
-
try {
|
|
532
|
-
const url = urlTemplate.replace('{page}', String(p));
|
|
533
|
-
const items = await containerFn(url);
|
|
534
|
-
if (items.length === 0)
|
|
535
|
-
break;
|
|
536
|
-
all.push(...items);
|
|
537
|
-
pagesScraped++;
|
|
538
|
-
if (p < startPage + max - 1)
|
|
539
|
-
await (0, delay_util_1.delay)(wait);
|
|
540
|
-
}
|
|
541
|
-
catch (err) {
|
|
542
|
-
if (isTargetClosed(err))
|
|
543
|
-
break;
|
|
544
|
-
throw err;
|
|
545
|
-
}
|
|
333
|
+
await pageService.closePage();
|
|
546
334
|
}
|
|
547
|
-
return { items: all, pages: pagesScraped };
|
|
548
|
-
}
|
|
549
|
-
async extractAllData(page, target, as = 'text', attribute) {
|
|
550
|
-
if (target.shadowHost && !target.value) {
|
|
551
|
-
const hosts = await page.$$(target.shadowHost);
|
|
552
|
-
const results = await Promise.all(hosts.map((host) => host.evaluate((el, extractAs, attr) => {
|
|
553
|
-
const sr = el.shadowRoot;
|
|
554
|
-
if (!sr)
|
|
555
|
-
return '';
|
|
556
|
-
if (extractAs === 'html' || extractAs === 'outerHtml')
|
|
557
|
-
return sr.innerHTML;
|
|
558
|
-
if (extractAs === 'attribute')
|
|
559
|
-
return el.getAttribute(attr || '') || '';
|
|
560
|
-
return sr.textContent?.trim() || '';
|
|
561
|
-
}, as, attribute)));
|
|
562
|
-
return results;
|
|
563
|
-
}
|
|
564
|
-
if (target.shadowHost) {
|
|
565
|
-
return await this.extractAllFromShadowRoot(page, target, as, attribute);
|
|
566
|
-
}
|
|
567
|
-
if (target.type === 'css') {
|
|
568
|
-
return await page.$$eval(target.value, (elements, extractAs, attr) => elements.map((el) => {
|
|
569
|
-
if (extractAs === 'html')
|
|
570
|
-
return el.innerHTML;
|
|
571
|
-
if (extractAs === 'outerHtml')
|
|
572
|
-
return el.outerHTML;
|
|
573
|
-
if (extractAs === 'attribute')
|
|
574
|
-
return el.getAttribute(attr || '') || '';
|
|
575
|
-
return el.textContent?.trim() || '';
|
|
576
|
-
}), as, attribute);
|
|
577
|
-
}
|
|
578
|
-
return await page.evaluate((selector, extractAs, attr) => {
|
|
579
|
-
const results = document.evaluate(selector, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
|
580
|
-
const values = [];
|
|
581
|
-
for (let i = 0; i < results.snapshotLength; i++) {
|
|
582
|
-
const node = results.snapshotItem(i);
|
|
583
|
-
if (!(node instanceof Element)) {
|
|
584
|
-
values.push(node?.textContent?.trim() || '');
|
|
585
|
-
continue;
|
|
586
|
-
}
|
|
587
|
-
if (extractAs === 'html')
|
|
588
|
-
values.push(node.innerHTML);
|
|
589
|
-
else if (extractAs === 'outerHtml')
|
|
590
|
-
values.push(node.outerHTML);
|
|
591
|
-
else if (extractAs === 'attribute')
|
|
592
|
-
values.push(node.getAttribute(attr || '') || '');
|
|
593
|
-
else
|
|
594
|
-
values.push(node.textContent?.trim() || '');
|
|
595
|
-
}
|
|
596
|
-
return values;
|
|
597
|
-
}, target.value, as, attribute);
|
|
598
|
-
}
|
|
599
|
-
async extractAllFromShadowRoot(page, target, as = 'text', attribute) {
|
|
600
|
-
const hosts = await page.$$(target.shadowHost);
|
|
601
|
-
const allResults = await Promise.all(hosts.map(async (host) => {
|
|
602
|
-
return await host.evaluate((el, selector, targetType, extractAs, attr) => {
|
|
603
|
-
const shadowRoot = el.shadowRoot;
|
|
604
|
-
if (!shadowRoot)
|
|
605
|
-
return [];
|
|
606
|
-
const getVal = (e) => {
|
|
607
|
-
if (extractAs === 'html')
|
|
608
|
-
return e.innerHTML;
|
|
609
|
-
if (extractAs === 'outerHtml')
|
|
610
|
-
return e.outerHTML;
|
|
611
|
-
if (extractAs === 'attribute')
|
|
612
|
-
return e.getAttribute(attr || '') || '';
|
|
613
|
-
return e.textContent?.trim() || '';
|
|
614
|
-
};
|
|
615
|
-
if (targetType === 'css') {
|
|
616
|
-
const elements = shadowRoot.querySelectorAll(selector);
|
|
617
|
-
return Array.from(elements).map(getVal);
|
|
618
|
-
}
|
|
619
|
-
const results = document.evaluate(selector, shadowRoot, null, XPathResult.UNORDERED_NODE_ITERATOR_TYPE, null);
|
|
620
|
-
const values = [];
|
|
621
|
-
let node;
|
|
622
|
-
while ((node = results.iterateNext())) {
|
|
623
|
-
values.push(node instanceof Element
|
|
624
|
-
? getVal(node)
|
|
625
|
-
: node?.textContent?.trim() || '');
|
|
626
|
-
}
|
|
627
|
-
return values;
|
|
628
|
-
}, target.value, target.type, as, attribute);
|
|
629
|
-
}));
|
|
630
|
-
return allResults.flat();
|
|
631
335
|
}
|
|
632
336
|
async waitForSelector(url, selector, timeout, scraperOptions) {
|
|
633
337
|
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Waiting for ${selector} on ${url}`));
|
|
634
338
|
const cloak = scraperOptions?.useRandomUserAgent
|
|
635
339
|
? { ...scraperOptions?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
|
|
636
340
|
: scraperOptions?.cloak;
|
|
637
|
-
const
|
|
341
|
+
const pageService = await this.newPage();
|
|
342
|
+
const page = await pageService.navigateTo(url, this.buildNavOptions(scraperOptions), cloak, scraperOptions?.interceptResource);
|
|
638
343
|
await page.waitForSelector(selector, { timeout });
|
|
639
344
|
return page;
|
|
640
345
|
}
|
|
@@ -643,11 +348,12 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
643
348
|
const cloak = options?.useRandomUserAgent
|
|
644
349
|
? { ...options?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
|
|
645
350
|
: options?.cloak;
|
|
646
|
-
const
|
|
351
|
+
const pageService = await this.newPage();
|
|
352
|
+
const page = await pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
|
|
647
353
|
const result = typeof script === 'function'
|
|
648
354
|
? await page.evaluate(script)
|
|
649
355
|
: await page.evaluate(script);
|
|
650
|
-
await
|
|
356
|
+
await pageService.closePage();
|
|
651
357
|
return result;
|
|
652
358
|
}
|
|
653
359
|
async scrapeWithWorkflow(url, workflow, variables) {
|
|
@@ -657,7 +363,8 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
657
363
|
this.moduleOptions?.debugLogMaxLength ??
|
|
658
364
|
browser_action_constants_1.DEFAULT_DEBUG_LOG_MAX_LENGTH;
|
|
659
365
|
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Starting workflow execution for ${url}`));
|
|
660
|
-
const
|
|
366
|
+
const pageService = await this.newPage();
|
|
367
|
+
const page = await pageService.navigateTo(url, undefined, workflow.cloak, workflow.interceptResource);
|
|
661
368
|
const result = {
|
|
662
369
|
success: false,
|
|
663
370
|
data: {},
|
|
@@ -669,7 +376,7 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
669
376
|
const errorConfig = workflow.onError || {};
|
|
670
377
|
for (const action of workflow.actions) {
|
|
671
378
|
try {
|
|
672
|
-
await this.executeAction(page, action, context);
|
|
379
|
+
await this.workflow.executeAction(page, action, context, this.activeDebugLogMaxLength);
|
|
673
380
|
}
|
|
674
381
|
catch (error) {
|
|
675
382
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -685,7 +392,7 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
685
392
|
}
|
|
686
393
|
if (!errorConfig.continue) {
|
|
687
394
|
result.errors.push(errorMessage);
|
|
688
|
-
await
|
|
395
|
+
await pageService.closePage();
|
|
689
396
|
return result;
|
|
690
397
|
}
|
|
691
398
|
}
|
|
@@ -699,666 +406,36 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
|
|
|
699
406
|
this.logger.error(`Workflow execution failed: ${errorMessage}`);
|
|
700
407
|
}
|
|
701
408
|
finally {
|
|
702
|
-
await
|
|
409
|
+
await pageService.closePage();
|
|
703
410
|
}
|
|
704
411
|
return result;
|
|
705
412
|
}
|
|
706
413
|
async scrapeAllWithWorkflow(url, workflow, variables) {
|
|
707
414
|
return await this.scrapeWithWorkflow(url, workflow, variables);
|
|
708
415
|
}
|
|
416
|
+
async executeContainerExtraction(page, descriptor, currentPage = 1) {
|
|
417
|
+
return this.container.executeContainerExtraction(page, descriptor, currentPage);
|
|
418
|
+
}
|
|
709
419
|
async executeAction(page, action, context) {
|
|
710
420
|
if (action.condition) {
|
|
711
421
|
const shouldExecute = await this.evaluateCondition(page, action.condition);
|
|
712
|
-
if (!shouldExecute)
|
|
713
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Skipping action due to condition: ${action.action}`));
|
|
422
|
+
if (!shouldExecute)
|
|
714
423
|
return;
|
|
715
|
-
}
|
|
716
|
-
}
|
|
717
|
-
const value = this.interpolateValue(String(action.value || ''), context);
|
|
718
|
-
const actionLabel = action.id
|
|
719
|
-
? `[${action.action}] id="${action.id}"`
|
|
720
|
-
: `[${action.action}]`;
|
|
721
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Executing action: ${actionLabel}`));
|
|
722
|
-
const maxRetries = Math.min(action.options?.retry ?? 0, 100);
|
|
723
|
-
const retryDelay = Math.min(action.options?.retryDelay ?? 0, 300_000);
|
|
724
|
-
for (let attempt = 0;; attempt++) {
|
|
725
|
-
try {
|
|
726
|
-
await this.dispatchAction(page, action, value, context);
|
|
727
|
-
return;
|
|
728
|
-
}
|
|
729
|
-
catch (err) {
|
|
730
|
-
if (attempt >= maxRetries)
|
|
731
|
-
throw err;
|
|
732
|
-
this.logger.warn((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Action ${actionLabel} failed (attempt ${attempt + 1}/${maxRetries + 1}), retrying: ${err instanceof Error ? err.message : String(err)}`));
|
|
733
|
-
if (retryDelay > 0)
|
|
734
|
-
await (0, delay_util_1.delay)(retryDelay);
|
|
735
|
-
}
|
|
736
|
-
}
|
|
737
|
-
}
|
|
738
|
-
async dispatchAction(page, action, value, context) {
|
|
739
|
-
switch (action.action) {
|
|
740
|
-
case 'navigate': {
|
|
741
|
-
const navOptions = {};
|
|
742
|
-
if (action.options?.waitUntil)
|
|
743
|
-
navOptions.waitUntil = action.options.waitUntil;
|
|
744
|
-
if (action.options?.timeout)
|
|
745
|
-
navOptions.timeout = action.options.timeout;
|
|
746
|
-
if (!(0, class_validator_1.isURL)(value, {
|
|
747
|
-
require_protocol: true,
|
|
748
|
-
protocols: ['http', 'https'],
|
|
749
|
-
})) {
|
|
750
|
-
throw new Error(`Invalid or disallowed URL: ${value}`);
|
|
751
|
-
}
|
|
752
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` navigate → ${value}`));
|
|
753
|
-
await page.goto(value, Object.keys(navOptions).length ? navOptions : undefined);
|
|
754
|
-
break;
|
|
755
|
-
}
|
|
756
|
-
case 'wait':
|
|
757
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` wait ${Number(action.value) || 0}ms`));
|
|
758
|
-
await (0, delay_util_1.delay)(Math.min(Number(action.value) || 0, 300_000));
|
|
759
|
-
break;
|
|
760
|
-
case 'waitFor':
|
|
761
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` waitFor target: ${this.describeTarget(action.target)}`));
|
|
762
|
-
await this.waitForTarget(page, action.target, action.options);
|
|
763
|
-
break;
|
|
764
|
-
case 'click':
|
|
765
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` click target: ${this.describeTarget(action.target)}`));
|
|
766
|
-
await this.clickElement(page, action.target, action.options);
|
|
767
|
-
break;
|
|
768
|
-
case 'type':
|
|
769
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` type target: ${this.describeTarget(action.target)} value: "${value}"`));
|
|
770
|
-
await this.typeText(page, action.target, value, action.options);
|
|
771
|
-
break;
|
|
772
|
-
case 'select':
|
|
773
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` select target: ${this.describeTarget(action.target)} value: "${value}"`));
|
|
774
|
-
await this.selectOption(page, action.target, value, action.options);
|
|
775
|
-
break;
|
|
776
|
-
case 'scroll':
|
|
777
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` scroll target: ${this.describeTarget(action.target)}`));
|
|
778
|
-
await this.scrollToElement(page, action.target);
|
|
779
|
-
break;
|
|
780
|
-
case 'extract': {
|
|
781
|
-
const extractOptions = action.options || {};
|
|
782
|
-
const extractAs = extractOptions.as ?? 'text';
|
|
783
|
-
const extractAttr = extractOptions.attribute;
|
|
784
|
-
if (!action.target) {
|
|
785
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extract full page HTML`));
|
|
786
|
-
if (action.id)
|
|
787
|
-
context[action.id] = await page.content();
|
|
788
|
-
break;
|
|
789
|
-
}
|
|
790
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extract target: ${this.describeTarget(action.target)} as="${extractAs}"${extractAttr ? ` attr="${extractAttr}"` : ''}${extractOptions.multiple ? ' multiple=true' : ''}`));
|
|
791
|
-
if (extractOptions.multiple) {
|
|
792
|
-
const allValues = await this.extractAllData(page, action.target, extractAs, extractAttr);
|
|
793
|
-
if (action.id) {
|
|
794
|
-
context[action.id] = allValues;
|
|
795
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extracted ${Array.isArray(allValues) ? allValues.length : 1} item(s) → id="${action.id}"`));
|
|
796
|
-
}
|
|
797
|
-
}
|
|
798
|
-
else {
|
|
799
|
-
const extractedValue = await this.extractData(page, action.target, extractAs, extractAttr);
|
|
800
|
-
if (action.id) {
|
|
801
|
-
context[action.id] = extractedValue;
|
|
802
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extracted → id="${action.id}": ${JSON.stringify(extractedValue)}`));
|
|
803
|
-
}
|
|
804
|
-
}
|
|
805
|
-
break;
|
|
806
|
-
}
|
|
807
|
-
case 'screenshot': {
|
|
808
|
-
const screenshotPath = String(action.value || `${browser_action_constants_1.DEFAULT_SCREENSHOT_FILENAME}-${Date.now()}.png`);
|
|
809
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` screenshot → ${screenshotPath}`));
|
|
810
|
-
await page.screenshot({ path: screenshotPath });
|
|
811
|
-
break;
|
|
812
|
-
}
|
|
813
|
-
case 'evaluate': {
|
|
814
|
-
let evalCode;
|
|
815
|
-
if (value.includes('=>')) {
|
|
816
|
-
evalCode = `(${value})()`;
|
|
817
|
-
}
|
|
818
|
-
else {
|
|
819
|
-
evalCode = value;
|
|
820
|
-
}
|
|
821
|
-
if (evalCode.length > 50_000) {
|
|
822
|
-
throw new Error('evaluate script exceeds maximum length of 50000 characters');
|
|
823
|
-
}
|
|
824
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Evaluating: ${evalCode}`));
|
|
825
|
-
const evalResult = await page.evaluate(evalCode);
|
|
826
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Result: ${JSON.stringify(evalResult)}`));
|
|
827
|
-
if (action.id) {
|
|
828
|
-
context[action.id] = evalResult;
|
|
829
|
-
}
|
|
830
|
-
break;
|
|
831
|
-
}
|
|
832
|
-
case 'cleanse': {
|
|
833
|
-
const pipes = action.options?.pipes;
|
|
834
|
-
if (!pipes) {
|
|
835
|
-
throw new Error('cleanse action requires pipes');
|
|
836
|
-
}
|
|
837
|
-
const valueKey = String(action.value || '');
|
|
838
|
-
const rawValue = this.resolveRawValue(valueKey, context);
|
|
839
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` cleanse raw: ${JSON.stringify(rawValue)}`));
|
|
840
|
-
const cleanedValue = Array.isArray(rawValue)
|
|
841
|
-
? rawValue.map((item) => this.pipeEngine.apply(String(item), pipes))
|
|
842
|
-
: this.pipeEngine.apply(String(rawValue), pipes);
|
|
843
|
-
if (action.id) {
|
|
844
|
-
context[action.id] = cleanedValue;
|
|
845
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Cleansed value for '${action.id}': ${JSON.stringify(cleanedValue)}`));
|
|
846
|
-
}
|
|
847
|
-
break;
|
|
848
|
-
}
|
|
849
|
-
case 'saveCookies': {
|
|
850
|
-
const sessionName = String(value);
|
|
851
|
-
const overwrite = action.options?.overwrite ?? false;
|
|
852
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` saveCookies session="${sessionName}" overwrite=${overwrite}`));
|
|
853
|
-
const metadata = {};
|
|
854
|
-
if (action.options?.metadata) {
|
|
855
|
-
Object.assign(metadata, action.options.metadata);
|
|
856
|
-
}
|
|
857
|
-
await this.cookieService.saveCookies(page, sessionName, {
|
|
858
|
-
overwrite,
|
|
859
|
-
metadata,
|
|
860
|
-
});
|
|
861
|
-
break;
|
|
862
|
-
}
|
|
863
|
-
case 'loadCookies': {
|
|
864
|
-
const sessionName = String(value);
|
|
865
|
-
const throwIfNotExists = action.onError !== 'skip' && action.onError !== 'continue';
|
|
866
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` loadCookies session="${sessionName}"`));
|
|
867
|
-
await this.cookieService.loadCookies(page, sessionName, {
|
|
868
|
-
throwIfNotExists,
|
|
869
|
-
});
|
|
870
|
-
break;
|
|
871
|
-
}
|
|
872
|
-
case 'clearCookies': {
|
|
873
|
-
const sessionName = String(value);
|
|
874
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` clearCookies session="${sessionName}"`));
|
|
875
|
-
await this.cookieService.deleteCookies(sessionName);
|
|
876
|
-
break;
|
|
877
|
-
}
|
|
878
|
-
case 'listCookies': {
|
|
879
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` listCookies`));
|
|
880
|
-
const sessions = await this.cookieService.listCookies();
|
|
881
|
-
if (action.id) {
|
|
882
|
-
context[action.id] = sessions;
|
|
883
|
-
}
|
|
884
|
-
break;
|
|
885
|
-
}
|
|
886
|
-
case 'hover':
|
|
887
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` hover target: ${this.describeTarget(action.target)}`));
|
|
888
|
-
await this.hoverElement(page, action.target, action.options);
|
|
889
|
-
break;
|
|
890
|
-
case 'keyPress':
|
|
891
|
-
if (!value) {
|
|
892
|
-
throw new Error('keyPress action requires a key value');
|
|
893
|
-
}
|
|
894
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` keyPress key="${value}"`));
|
|
895
|
-
await page.keyboard.press(value);
|
|
896
|
-
break;
|
|
897
|
-
case 'clear':
|
|
898
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` clear target: ${this.describeTarget(action.target)}`));
|
|
899
|
-
await this.clearElement(page, action.target);
|
|
900
|
-
break;
|
|
901
|
-
case 'waitForNetwork':
|
|
902
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` waitForNetwork timeout=${action.options?.timeout ?? browser_action_constants_1.DEFAULT_ACTION_TIMEOUT}ms`));
|
|
903
|
-
await page.waitForNetworkIdle({
|
|
904
|
-
timeout: action.options?.timeout ?? browser_action_constants_1.DEFAULT_ACTION_TIMEOUT,
|
|
905
|
-
});
|
|
906
|
-
break;
|
|
907
|
-
case 'reload':
|
|
908
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` reload waitUntil="${action.options?.waitUntil ?? 'load'}"`));
|
|
909
|
-
await page.reload({
|
|
910
|
-
waitUntil: action.options?.waitUntil ?? 'load',
|
|
911
|
-
timeout: action.options?.timeout ?? browser_action_constants_1.DEFAULT_NAVIGATION_TIMEOUT,
|
|
912
|
-
});
|
|
913
|
-
break;
|
|
914
|
-
case 'scrapeContainer': {
|
|
915
|
-
const o = action.options;
|
|
916
|
-
if (!o?.container || !o?.fields) {
|
|
917
|
-
throw new Error('scrapeContainer requires options.container and options.fields');
|
|
918
|
-
}
|
|
919
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` scrapeContainer container="${o.container}"`));
|
|
920
|
-
const containerResult = await this.executeContainerExtraction(page, {
|
|
921
|
-
container: o.container,
|
|
922
|
-
fields: o.fields,
|
|
923
|
-
pagination: o.pagination,
|
|
924
|
-
}, o.currentPage ?? 1);
|
|
925
|
-
if (action.id) {
|
|
926
|
-
context[action.id] = containerResult.items;
|
|
927
|
-
if (containerResult.pagination) {
|
|
928
|
-
context[`${action.id}_pagination`] = containerResult.pagination;
|
|
929
|
-
}
|
|
930
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` scrapeContainer → id="${action.id}": ${containerResult.items.length} items`));
|
|
931
|
-
}
|
|
932
|
-
break;
|
|
933
|
-
}
|
|
934
|
-
case 'extractPagination': {
|
|
935
|
-
const o = action.options;
|
|
936
|
-
if (!o?.container || !o?.linkSelector || !o?.labelSelector) {
|
|
937
|
-
throw new Error('extractPagination requires options.container, linkSelector, and labelSelector');
|
|
938
|
-
}
|
|
939
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extractPagination container="${o.container}"`));
|
|
940
|
-
const paginationResult = await this.resolvePagination(page, {
|
|
941
|
-
container: o.container,
|
|
942
|
-
linkSelector: o.linkSelector,
|
|
943
|
-
labelSelector: o.labelSelector,
|
|
944
|
-
}, o.currentPage ?? 1);
|
|
945
|
-
if (action.id) {
|
|
946
|
-
context[action.id] = paginationResult;
|
|
947
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extractPagination → id="${action.id}": nextUrl=${paginationResult.nextUrl}`));
|
|
948
|
-
}
|
|
949
|
-
break;
|
|
950
|
-
}
|
|
951
|
-
default: {
|
|
952
|
-
const exhaustiveCheck = action.action;
|
|
953
|
-
throw new Error(`Unknown action type: ${String(exhaustiveCheck)}`);
|
|
954
|
-
}
|
|
955
424
|
}
|
|
425
|
+
await this.workflow.executeAction(page, { ...action, condition: undefined }, context, this.activeDebugLogMaxLength);
|
|
956
426
|
}
|
|
957
427
|
async evaluateCondition(page, condition) {
|
|
958
|
-
|
|
959
|
-
return true;
|
|
960
|
-
if (condition.ifExists) {
|
|
961
|
-
const element = await this.findElement(page, condition.ifExists);
|
|
962
|
-
return element !== null;
|
|
963
|
-
}
|
|
964
|
-
if (condition.unlessExists) {
|
|
965
|
-
const element = await this.findElement(page, condition.unlessExists);
|
|
966
|
-
return element === null;
|
|
967
|
-
}
|
|
968
|
-
return true;
|
|
969
|
-
}
|
|
970
|
-
describeTarget(target) {
|
|
971
|
-
const base = target.value
|
|
972
|
-
? `${target.type}="${target.value}"`
|
|
973
|
-
: `(no selector)`;
|
|
974
|
-
return target.shadowHost
|
|
975
|
-
? `shadowHost="${target.shadowHost}" > ${base}`
|
|
976
|
-
: base;
|
|
977
|
-
}
|
|
978
|
-
async findElement(page, target) {
|
|
979
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` findElement: ${this.describeTarget(target)}`));
|
|
980
|
-
if (target.shadowHost) {
|
|
981
|
-
const el = await this.findElementInShadowRoot(page, target);
|
|
982
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` findElement result: ${el ? 'found' : 'NOT FOUND'}`));
|
|
983
|
-
return el;
|
|
984
|
-
}
|
|
985
|
-
if (target.type === 'css') {
|
|
986
|
-
const el = await page.$(target.value);
|
|
987
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` findElement result: ${el ? 'found' : 'NOT FOUND'}`));
|
|
988
|
-
return el;
|
|
989
|
-
}
|
|
990
|
-
const el = await page
|
|
991
|
-
.evaluateHandle((selector) => {
|
|
992
|
-
const result = document.evaluate(selector, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
|
|
993
|
-
return result.singleNodeValue;
|
|
994
|
-
}, target.value)
|
|
995
|
-
.then((handle) => handle.asElement());
|
|
996
|
-
this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` findElement result: ${el ? 'found' : 'NOT FOUND'}`));
|
|
997
|
-
return el;
|
|
998
|
-
}
|
|
999
|
-
async findElementInShadowRoot(page, target) {
|
|
1000
|
-
const host = await page.$(target.shadowHost);
|
|
1001
|
-
if (!host)
|
|
1002
|
-
return null;
|
|
1003
|
-
return host
|
|
1004
|
-
.evaluateHandle((el, selector, targetType) => {
|
|
1005
|
-
const shadowRoot = el.shadowRoot;
|
|
1006
|
-
if (!shadowRoot)
|
|
1007
|
-
return null;
|
|
1008
|
-
if (targetType === 'css') {
|
|
1009
|
-
return shadowRoot.querySelector(selector);
|
|
1010
|
-
}
|
|
1011
|
-
const result = document.evaluate(selector, shadowRoot, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
|
|
1012
|
-
return result.singleNodeValue;
|
|
1013
|
-
}, target.value, target.type)
|
|
1014
|
-
.then((handle) => handle.asElement());
|
|
1015
|
-
}
|
|
1016
|
-
async waitForTarget(page, target, options) {
|
|
1017
|
-
const timeout = options?.timeout || browser_action_constants_1.DEFAULT_ACTION_TIMEOUT;
|
|
1018
|
-
if (target.type === 'css') {
|
|
1019
|
-
await page.waitForSelector(target.value, { timeout });
|
|
1020
|
-
}
|
|
1021
|
-
else {
|
|
1022
|
-
await page.waitForFunction((selector) => {
|
|
1023
|
-
const result = document.evaluate(selector, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
|
|
1024
|
-
return result.singleNodeValue !== null;
|
|
1025
|
-
}, { timeout }, target.value);
|
|
1026
|
-
}
|
|
1027
|
-
}
|
|
1028
|
-
async clickElement(page, target, options) {
|
|
1029
|
-
const element = await this.findElement(page, target);
|
|
1030
|
-
if (!element) {
|
|
1031
|
-
throw new Error(`Element not found: ${target.value}`);
|
|
1032
|
-
}
|
|
1033
|
-
await this.maybeScrollToElement(element, options?.scrollTo);
|
|
1034
|
-
const elem = element;
|
|
1035
|
-
await elem.click();
|
|
1036
|
-
if (options?.waitForNavigation) {
|
|
1037
|
-
await page.waitForNavigation({
|
|
1038
|
-
timeout: options.navigationTimeout || browser_action_constants_1.DEFAULT_NAVIGATION_TIMEOUT,
|
|
1039
|
-
});
|
|
1040
|
-
}
|
|
1041
|
-
}
|
|
1042
|
-
async typeText(page, target, value, options) {
|
|
1043
|
-
const element = await this.findElement(page, target);
|
|
1044
|
-
if (!element) {
|
|
1045
|
-
throw new Error(`Element not found: ${target.value}`);
|
|
1046
|
-
}
|
|
1047
|
-
await this.maybeScrollToElement(element, options?.scrollTo);
|
|
1048
|
-
const elem = element;
|
|
1049
|
-
await elem.click();
|
|
1050
|
-
const delay = options?.delay || 0;
|
|
1051
|
-
await elem.type(value, { delay });
|
|
1052
|
-
await elem.evaluate((el) => {
|
|
1053
|
-
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
1054
|
-
el.dispatchEvent(new Event('blur', { bubbles: true }));
|
|
1055
|
-
});
|
|
1056
|
-
}
|
|
1057
|
-
async selectOption(page, target, value, options) {
|
|
1058
|
-
const element = await this.findElement(page, target);
|
|
1059
|
-
if (!element) {
|
|
1060
|
-
throw new Error(`Element not found: ${target.value}`);
|
|
1061
|
-
}
|
|
1062
|
-
await this.maybeScrollToElement(element, options?.scrollTo);
|
|
1063
|
-
const elem = element;
|
|
1064
|
-
await elem.select(value);
|
|
1065
|
-
}
|
|
1066
|
-
async scrollToElement(page, target) {
|
|
1067
|
-
const element = await this.findElement(page, target);
|
|
1068
|
-
if (!element) {
|
|
1069
|
-
throw new Error(`Element not found: ${target.value}`);
|
|
1070
|
-
}
|
|
1071
|
-
await this.scrollElementIntoView(element);
|
|
1072
|
-
}
|
|
1073
|
-
async maybeScrollToElement(element, shouldScroll) {
|
|
1074
|
-
if (shouldScroll) {
|
|
1075
|
-
await this.scrollElementIntoView(element);
|
|
1076
|
-
}
|
|
1077
|
-
}
|
|
1078
|
-
async scrollElementIntoView(element) {
|
|
1079
|
-
await element.evaluate((el) => el.scrollIntoView({ behavior: 'smooth', block: 'center' }), {});
|
|
1080
|
-
await (0, delay_util_1.delay)(browser_action_constants_1.DEFAULT_SCROLL_DELAY_MS);
|
|
1081
|
-
}
|
|
1082
|
-
async executeContainerExtraction(page, descriptor, currentPage = 1) {
|
|
1083
|
-
const rawItems = await page.evaluate((containerSel, fieldDefs) => {
|
|
1084
|
-
const isXPath = (s) => s.trim().startsWith('//') ||
|
|
1085
|
-
s.trim().startsWith('(') ||
|
|
1086
|
-
s.trim().startsWith('./');
|
|
1087
|
-
const getContainerNodes = (sel) => {
|
|
1088
|
-
if (isXPath(sel)) {
|
|
1089
|
-
const res = document.evaluate(sel, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
|
1090
|
-
const nodes = [];
|
|
1091
|
-
for (let i = 0; i < res.snapshotLength; i++)
|
|
1092
|
-
nodes.push(res.snapshotItem(i));
|
|
1093
|
-
return nodes;
|
|
1094
|
-
}
|
|
1095
|
-
return Array.from(document.querySelectorAll(sel));
|
|
1096
|
-
};
|
|
1097
|
-
const extractOne = (sel, root, attr, returnType) => {
|
|
1098
|
-
let node = null;
|
|
1099
|
-
if (isXPath(sel)) {
|
|
1100
|
-
const res = document.evaluate(sel, root, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
|
|
1101
|
-
node = res.singleNodeValue;
|
|
1102
|
-
}
|
|
1103
|
-
else {
|
|
1104
|
-
node = root.querySelector(sel);
|
|
1105
|
-
}
|
|
1106
|
-
if (!node)
|
|
1107
|
-
return '';
|
|
1108
|
-
if (attr)
|
|
1109
|
-
return node.getAttribute?.(attr) ?? '';
|
|
1110
|
-
if (returnType === 'html')
|
|
1111
|
-
return node.innerHTML ?? '';
|
|
1112
|
-
return node.textContent?.trim() ?? '';
|
|
1113
|
-
};
|
|
1114
|
-
const extractMany = (sel, root, attr, returnType) => {
|
|
1115
|
-
let nodes = [];
|
|
1116
|
-
if (isXPath(sel)) {
|
|
1117
|
-
const res = document.evaluate(sel, root, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
|
1118
|
-
for (let i = 0; i < res.snapshotLength; i++)
|
|
1119
|
-
nodes.push(res.snapshotItem(i));
|
|
1120
|
-
}
|
|
1121
|
-
else {
|
|
1122
|
-
nodes = Array.from(root.querySelectorAll(sel));
|
|
1123
|
-
}
|
|
1124
|
-
return nodes.map((n) => {
|
|
1125
|
-
if (attr)
|
|
1126
|
-
return n.getAttribute?.(attr) ?? '';
|
|
1127
|
-
if (returnType === 'html')
|
|
1128
|
-
return n.innerHTML ?? '';
|
|
1129
|
-
return n.textContent?.trim() ?? '';
|
|
1130
|
-
});
|
|
1131
|
-
};
|
|
1132
|
-
const extractField = (root, fd) => {
|
|
1133
|
-
const selectors = [fd.selector, ...(fd.fallback ?? [])];
|
|
1134
|
-
for (const sel of selectors) {
|
|
1135
|
-
const val = fd.multiple
|
|
1136
|
-
? extractMany(sel, root, fd.attribute, fd.returnType)
|
|
1137
|
-
: extractOne(sel, root, fd.attribute, fd.returnType);
|
|
1138
|
-
if (Array.isArray(val) ? val.length > 0 : val !== '')
|
|
1139
|
-
return val;
|
|
1140
|
-
}
|
|
1141
|
-
return fd.multiple ? [] : '';
|
|
1142
|
-
};
|
|
1143
|
-
return getContainerNodes(containerSel).map((node) => {
|
|
1144
|
-
const item = {};
|
|
1145
|
-
for (const [key, fd] of Object.entries(fieldDefs))
|
|
1146
|
-
item[key] = extractField(node, fd);
|
|
1147
|
-
return item;
|
|
1148
|
-
});
|
|
1149
|
-
}, descriptor.container, descriptor.fields);
|
|
1150
|
-
const items = rawItems;
|
|
1151
|
-
let pagination;
|
|
1152
|
-
if (descriptor.pagination) {
|
|
1153
|
-
pagination = await this.resolvePagination(page, descriptor.pagination, currentPage);
|
|
1154
|
-
}
|
|
1155
|
-
return { items, pagination };
|
|
1156
|
-
}
|
|
1157
|
-
async resolvePagination(page, descriptor, currentPage = 1) {
|
|
1158
|
-
currentPage = Math.max(1, currentPage);
|
|
1159
|
-
const rawPages = await page.evaluate((containerSel, linkSel, labelSel) => {
|
|
1160
|
-
const getNode = (sel, root) => {
|
|
1161
|
-
if (sel.trim().startsWith('//') ||
|
|
1162
|
-
sel.trim().startsWith('(') ||
|
|
1163
|
-
sel.trim().startsWith('./')) {
|
|
1164
|
-
const res = document.evaluate(sel, root, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
|
|
1165
|
-
return res.singleNodeValue;
|
|
1166
|
-
}
|
|
1167
|
-
return root.querySelector(sel);
|
|
1168
|
-
};
|
|
1169
|
-
const getNodes = (sel, root) => {
|
|
1170
|
-
if (sel.trim().startsWith('//') ||
|
|
1171
|
-
sel.trim().startsWith('(') ||
|
|
1172
|
-
sel.trim().startsWith('./')) {
|
|
1173
|
-
const res = document.evaluate(sel, root, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
|
1174
|
-
const nodes = [];
|
|
1175
|
-
for (let i = 0; i < res.snapshotLength; i++)
|
|
1176
|
-
nodes.push(res.snapshotItem(i));
|
|
1177
|
-
return nodes;
|
|
1178
|
-
}
|
|
1179
|
-
return Array.from(root.querySelectorAll(sel));
|
|
1180
|
-
};
|
|
1181
|
-
const container = getNode(containerSel, document);
|
|
1182
|
-
if (!container)
|
|
1183
|
-
return [];
|
|
1184
|
-
return getNodes(linkSel, container).map((el) => {
|
|
1185
|
-
const isXPath = (s) => s.trim().startsWith('//') ||
|
|
1186
|
-
s.trim().startsWith('(') ||
|
|
1187
|
-
s.trim().startsWith('./');
|
|
1188
|
-
const labelNodes = isXPath(labelSel)
|
|
1189
|
-
? (() => {
|
|
1190
|
-
const r = document.evaluate(labelSel, el, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
|
1191
|
-
const ns = [];
|
|
1192
|
-
for (let i = 0; i < r.snapshotLength; i++)
|
|
1193
|
-
ns.push(r.snapshotItem(i));
|
|
1194
|
-
return ns;
|
|
1195
|
-
})()
|
|
1196
|
-
: Array.from(el.querySelectorAll(labelSel));
|
|
1197
|
-
const label = labelNodes.length > 0
|
|
1198
|
-
? (labelNodes[0].textContent?.trim() ?? '')
|
|
1199
|
-
: (el.textContent?.trim() ?? '');
|
|
1200
|
-
return {
|
|
1201
|
-
label,
|
|
1202
|
-
url: el.getAttribute('href') ?? '',
|
|
1203
|
-
};
|
|
1204
|
-
});
|
|
1205
|
-
}, descriptor.container, descriptor.linkSelector, descriptor.labelSelector);
|
|
1206
|
-
const pages = rawPages.filter((p) => p.url);
|
|
1207
|
-
const numericNext = pages
|
|
1208
|
-
.filter((p) => !Number.isNaN(parseInt(p.label, 10)) &&
|
|
1209
|
-
parseInt(p.label, 10) > currentPage)
|
|
1210
|
-
.sort((a, b) => parseInt(a.label, 10) - parseInt(b.label, 10))[0];
|
|
1211
|
-
const nextUrl = numericNext?.url ??
|
|
1212
|
-
pages.find((p) => ['next', '>'].includes(p.label.toLowerCase()))?.url ??
|
|
1213
|
-
null;
|
|
1214
|
-
return { pages, nextUrl };
|
|
1215
|
-
}
|
|
1216
|
-
validateSelector(key, selector) {
|
|
1217
|
-
if (!selector || selector.trim() === '') {
|
|
1218
|
-
throw new Error(`Selector for '${key}' cannot be empty`);
|
|
1219
|
-
}
|
|
1220
|
-
}
|
|
1221
|
-
async extractData(page, target, as = 'text', attribute) {
|
|
1222
|
-
if (target.shadowHost && !target.value) {
|
|
1223
|
-
const host = await page.$(target.shadowHost);
|
|
1224
|
-
if (!host)
|
|
1225
|
-
throw new Error(`Shadow host not found: ${target.shadowHost}`);
|
|
1226
|
-
return host.evaluate((el, extractAs, attr) => {
|
|
1227
|
-
const sr = el.shadowRoot;
|
|
1228
|
-
if (!sr)
|
|
1229
|
-
return '';
|
|
1230
|
-
if (extractAs === 'html' || extractAs === 'outerHtml')
|
|
1231
|
-
return sr.innerHTML;
|
|
1232
|
-
if (extractAs === 'attribute')
|
|
1233
|
-
return el.getAttribute(attr || '') || '';
|
|
1234
|
-
return sr.textContent?.trim() || '';
|
|
1235
|
-
}, as, attribute);
|
|
1236
|
-
}
|
|
1237
|
-
const element = await this.findElement(page, target);
|
|
1238
|
-
if (!element) {
|
|
1239
|
-
throw new Error(`Element not found: ${target.value}`);
|
|
1240
|
-
}
|
|
1241
|
-
return element.evaluate((el, extractAs, attr) => {
|
|
1242
|
-
if (extractAs === 'html')
|
|
1243
|
-
return el.innerHTML;
|
|
1244
|
-
if (extractAs === 'outerHtml')
|
|
1245
|
-
return el.outerHTML;
|
|
1246
|
-
if (extractAs === 'attribute')
|
|
1247
|
-
return el.getAttribute(attr || '') || '';
|
|
1248
|
-
return el.textContent?.trim() || '';
|
|
1249
|
-
}, as, attribute);
|
|
1250
|
-
}
|
|
1251
|
-
async hoverElement(page, target, options) {
|
|
1252
|
-
const element = await this.findElement(page, target);
|
|
1253
|
-
if (!element) {
|
|
1254
|
-
throw new Error(`Element not found: ${target.value}`);
|
|
1255
|
-
}
|
|
1256
|
-
await this.maybeScrollToElement(element, options?.scrollTo);
|
|
1257
|
-
const elem = element;
|
|
1258
|
-
await elem.hover();
|
|
1259
|
-
}
|
|
1260
|
-
async clearElement(page, target) {
|
|
1261
|
-
const element = await this.findElement(page, target);
|
|
1262
|
-
if (!element) {
|
|
1263
|
-
throw new Error(`Element not found: ${target.value}`);
|
|
1264
|
-
}
|
|
1265
|
-
await element.evaluate((el) => {
|
|
1266
|
-
if ('value' in el) {
|
|
1267
|
-
el.value = '';
|
|
1268
|
-
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
1269
|
-
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
1270
|
-
}
|
|
1271
|
-
});
|
|
1272
|
-
}
|
|
1273
|
-
resolveRawValue(value, context) {
|
|
1274
|
-
if (value.startsWith('${') && value.endsWith('}')) {
|
|
1275
|
-
const path = value.slice(2, -1);
|
|
1276
|
-
if (!path.includes('${') && !path.includes('.') && !path.includes('[')) {
|
|
1277
|
-
return context[path] ?? value;
|
|
1278
|
-
}
|
|
1279
|
-
}
|
|
1280
|
-
return this.resolveValue(value, context);
|
|
1281
|
-
}
|
|
1282
|
-
resolveValue(value, context) {
|
|
1283
|
-
if (value.startsWith('${') && value.endsWith('}')) {
|
|
1284
|
-
const path = value.slice(2, -1);
|
|
1285
|
-
if (!path.includes('${')) {
|
|
1286
|
-
const keys = path.split('.');
|
|
1287
|
-
let result = context;
|
|
1288
|
-
for (const key of keys) {
|
|
1289
|
-
const arrayMatch = key.match(/^(\w+)\[(\d+)\]$/);
|
|
1290
|
-
if (arrayMatch) {
|
|
1291
|
-
const [, arrayKey, indexStr] = arrayMatch;
|
|
1292
|
-
if (!result || typeof result !== 'object') {
|
|
1293
|
-
return value;
|
|
1294
|
-
}
|
|
1295
|
-
const arrayValue = result[arrayKey];
|
|
1296
|
-
const index = parseInt(indexStr, 10);
|
|
1297
|
-
result = Array.isArray(arrayValue) ? arrayValue[index] : undefined;
|
|
1298
|
-
}
|
|
1299
|
-
else {
|
|
1300
|
-
if (!result || typeof result !== 'object') {
|
|
1301
|
-
return value;
|
|
1302
|
-
}
|
|
1303
|
-
result = result[key];
|
|
1304
|
-
}
|
|
1305
|
-
if (result === undefined) {
|
|
1306
|
-
return value;
|
|
1307
|
-
}
|
|
1308
|
-
}
|
|
1309
|
-
if (result === null || typeof result === 'object') {
|
|
1310
|
-
return value;
|
|
1311
|
-
}
|
|
1312
|
-
return result;
|
|
1313
|
-
}
|
|
1314
|
-
return this.interpolateValue(path, context);
|
|
1315
|
-
}
|
|
1316
|
-
return value;
|
|
1317
|
-
}
|
|
1318
|
-
interpolateValue(value, context) {
|
|
1319
|
-
return value.replace(/\$\{([^}]+)\}/g, (_match, path) => {
|
|
1320
|
-
const keys = path.split('.');
|
|
1321
|
-
let result = context;
|
|
1322
|
-
for (const key of keys) {
|
|
1323
|
-
if (key === '__proto__' ||
|
|
1324
|
-
key === 'constructor' ||
|
|
1325
|
-
key === 'prototype') {
|
|
1326
|
-
return '';
|
|
1327
|
-
}
|
|
1328
|
-
const arrayMatch = key.match(/^(\w+)\[(\d+)\]$/);
|
|
1329
|
-
if (arrayMatch) {
|
|
1330
|
-
const [, arrayKey, indexStr] = arrayMatch;
|
|
1331
|
-
if (!result || typeof result !== 'object') {
|
|
1332
|
-
return '';
|
|
1333
|
-
}
|
|
1334
|
-
const arrayValue = result[arrayKey];
|
|
1335
|
-
const index = parseInt(indexStr, 10);
|
|
1336
|
-
result = Array.isArray(arrayValue) ? arrayValue[index] : undefined;
|
|
1337
|
-
}
|
|
1338
|
-
else {
|
|
1339
|
-
if (!result || typeof result !== 'object') {
|
|
1340
|
-
return '';
|
|
1341
|
-
}
|
|
1342
|
-
result = result[key];
|
|
1343
|
-
}
|
|
1344
|
-
if (result === undefined) {
|
|
1345
|
-
return '';
|
|
1346
|
-
}
|
|
1347
|
-
}
|
|
1348
|
-
if (result === null || typeof result === 'object') {
|
|
1349
|
-
return '';
|
|
1350
|
-
}
|
|
1351
|
-
return result;
|
|
1352
|
-
});
|
|
428
|
+
return this.workflow.evaluateCondition(page, condition, this.activeDebugLogMaxLength);
|
|
1353
429
|
}
|
|
1354
430
|
};
|
|
1355
431
|
exports.BrowserActionService = BrowserActionService;
|
|
1356
432
|
exports.BrowserActionService = BrowserActionService = BrowserActionService_1 = __decorate([
|
|
1357
|
-
(0, common_1.Injectable)(
|
|
1358
|
-
__param(
|
|
1359
|
-
__param(
|
|
433
|
+
(0, common_1.Injectable)(),
|
|
434
|
+
__param(4, (0, common_1.Optional)()),
|
|
435
|
+
__param(4, (0, common_1.Inject)(browser_action_constants_1.BROWSER_ACTION_OPTIONS)),
|
|
1360
436
|
__metadata("design:paramtypes", [page_service_1.PageService,
|
|
1361
437
|
cookie_service_1.CookieService,
|
|
1362
|
-
cleansing_service_1.CleansingService,
|
|
438
|
+
cleansing_service_1.CleansingService,
|
|
439
|
+
core_1.ModuleRef, Object])
|
|
1363
440
|
], BrowserActionService);
|
|
1364
441
|
//# sourceMappingURL=browser-action.service.js.map
|