@hanivanrizky/nestjs-browser-action 0.20.1 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +72 -31
  2. package/dist/browser-action.module-definition.d.ts +1 -0
  3. package/dist/browser-action.module.d.ts +3 -0
  4. package/dist/browser-action.module.js +146 -27
  5. package/dist/browser-action.module.js.map +1 -1
  6. package/dist/common/index.d.ts +1 -0
  7. package/dist/common/index.js +18 -0
  8. package/dist/common/index.js.map +1 -0
  9. package/dist/common/tokens.d.ts +7 -0
  10. package/dist/common/tokens.js +36 -0
  11. package/dist/common/tokens.js.map +1 -0
  12. package/dist/decorators/index.d.ts +1 -0
  13. package/dist/decorators/index.js +1 -0
  14. package/dist/decorators/index.js.map +1 -1
  15. package/dist/decorators/inject-browser.decorator.d.ts +2 -1
  16. package/dist/decorators/inject-browser.decorator.js +2 -1
  17. package/dist/decorators/inject-browser.decorator.js.map +1 -1
  18. package/dist/decorators/inject-page-controller.decorator.d.ts +2 -0
  19. package/dist/decorators/inject-page-controller.decorator.js +8 -0
  20. package/dist/decorators/inject-page-controller.decorator.js.map +1 -0
  21. package/dist/decorators/inject-page.decorator.d.ts +2 -1
  22. package/dist/decorators/inject-page.decorator.js +2 -1
  23. package/dist/decorators/inject-page.decorator.js.map +1 -1
  24. package/dist/examples/named-browser-example.d.ts +1 -0
  25. package/dist/examples/named-browser-example.js +183 -0
  26. package/dist/examples/named-browser-example.js.map +1 -0
  27. package/dist/index.d.ts +5 -0
  28. package/dist/index.js +5 -0
  29. package/dist/index.js.map +1 -1
  30. package/dist/interfaces/browser-action-options.d.ts +1 -0
  31. package/dist/operators/capture.operator.d.ts +11 -0
  32. package/dist/operators/capture.operator.js +62 -0
  33. package/dist/operators/capture.operator.js.map +1 -0
  34. package/dist/operators/container.operator.d.ts +13 -0
  35. package/dist/operators/container.operator.js +149 -0
  36. package/dist/operators/container.operator.js.map +1 -0
  37. package/dist/operators/extraction.operator.d.ts +22 -0
  38. package/dist/operators/extraction.operator.js +204 -0
  39. package/dist/operators/extraction.operator.js.map +1 -0
  40. package/dist/operators/index.d.ts +5 -0
  41. package/dist/operators/index.js +14 -0
  42. package/dist/operators/index.js.map +1 -0
  43. package/dist/operators/pagination.operator.d.ts +24 -0
  44. package/dist/operators/pagination.operator.js +182 -0
  45. package/dist/operators/pagination.operator.js.map +1 -0
  46. package/dist/operators/workflow.operator.d.ts +35 -0
  47. package/dist/operators/workflow.operator.js +501 -0
  48. package/dist/operators/workflow.operator.js.map +1 -0
  49. package/dist/services/browser-action.service.d.ts +10 -30
  50. package/dist/services/browser-action.service.js +72 -995
  51. package/dist/services/browser-action.service.js.map +1 -1
  52. package/dist/services/browser-holder.d.ts +7 -0
  53. package/dist/services/browser-holder.js +26 -0
  54. package/dist/services/browser-holder.js.map +1 -0
  55. package/dist/services/browser-manager.service.js +3 -2
  56. package/dist/services/browser-manager.service.js.map +1 -1
  57. package/dist/services/browser-pool.service.d.ts +4 -4
  58. package/dist/services/browser-pool.service.js +45 -104
  59. package/dist/services/browser-pool.service.js.map +1 -1
  60. package/dist/services/named-browser-shutdown.d.ts +9 -0
  61. package/dist/services/named-browser-shutdown.js +47 -0
  62. package/dist/services/named-browser-shutdown.js.map +1 -0
  63. package/dist/services/page-controller.d.ts +39 -0
  64. package/dist/services/page-controller.js +325 -0
  65. package/dist/services/page-controller.js.map +1 -0
  66. package/dist/services/page-session.d.ts +26 -0
  67. package/dist/services/page-session.js +135 -0
  68. package/dist/services/page-session.js.map +1 -0
  69. package/dist/services/page.service.js +14 -3
  70. package/dist/services/page.service.js.map +1 -1
  71. package/dist/tsconfig.build.tsbuildinfo +1 -1
  72. package/dist/utils/browser-launcher.d.ts +6 -0
  73. package/dist/utils/browser-launcher.js +100 -0
  74. package/dist/utils/browser-launcher.js.map +1 -0
  75. package/package.json +2 -2
@@ -15,59 +15,63 @@ var BrowserActionService_1;
15
15
  Object.defineProperty(exports, "__esModule", { value: true });
16
16
  exports.BrowserActionService = void 0;
17
17
  const common_1 = require("@nestjs/common");
18
- const fs_1 = require("fs");
19
- const path_1 = require("path");
20
- const class_validator_1 = require("class-validator");
18
+ const core_1 = require("@nestjs/core");
21
19
  const path_util_1 = require("../utils/path.util");
22
20
  const workflow_validator_1 = require("../validators/workflow.validator");
23
21
  const page_service_1 = require("./page.service");
24
22
  const cookie_service_1 = require("./cookie.service");
25
23
  const cleansing_service_1 = require("./cleansing.service");
26
24
  const pipe_engine_1 = require("../pipes/pipe-engine");
25
+ const operators_1 = require("../operators");
27
26
  const user_agent_util_1 = require("../utils/user-agent.util");
28
27
  const logger_util_1 = require("../utils/logger.util");
29
- const delay_util_1 = require("../utils/delay.util");
30
28
  const dom_util_1 = require("../utils/dom.util");
31
29
  const truncate_log_util_1 = require("../utils/truncate-log.util");
32
30
  const browser_action_constants_1 = require("../constants/browser-action.constants");
33
- const isTargetClosed = (err) => err instanceof Error &&
34
- (err.message.includes('Target closed') ||
35
- err.message.includes('No target with given id') ||
36
- err.message.includes('detached Frame') ||
37
- err.message.includes('Attempted to use detached') ||
38
- err.message.includes('Execution context was destroyed') ||
39
- err.message.includes('context was destroyed') ||
40
- err.message.includes('Session closed') ||
41
- err.message.includes('Navigating frame was detached'));
42
31
  let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
43
32
  pageService;
44
33
  cookieService;
45
34
  cleansingService;
35
+ moduleRef;
46
36
  moduleOptions;
47
37
  logger;
48
38
  pipeEngine = new pipe_engine_1.PipeEngine();
49
39
  activeDebugLogMaxLength;
50
- constructor(pageService, cookieService, cleansingService, moduleOptions) {
40
+ extraction;
41
+ capture;
42
+ container;
43
+ workflow;
44
+ pagination;
45
+ constructor(pageService, cookieService, cleansingService, moduleRef, moduleOptions) {
51
46
  this.pageService = pageService;
52
47
  this.cookieService = cookieService;
53
48
  this.cleansingService = cleansingService;
49
+ this.moduleRef = moduleRef;
54
50
  this.moduleOptions = moduleOptions;
55
51
  this.logger = new logger_util_1.LoggerWithLevel(BrowserActionService_1.name, this.pageService.getLogLevel());
56
52
  this.activeDebugLogMaxLength =
57
53
  moduleOptions?.debugLogMaxLength ?? browser_action_constants_1.DEFAULT_DEBUG_LOG_MAX_LENGTH;
54
+ this.extraction = new operators_1.ExtractionOperator(this.pipeEngine, this.logger);
55
+ this.capture = new operators_1.CaptureOperator(this.logger);
56
+ this.container = new operators_1.ContainerOperator(this.extraction, this.pipeEngine, this.logger);
57
+ this.workflow = new operators_1.WorkflowOperator(this.extraction, this.container, this.pipeEngine, this.cleansingService, this.logger, () => this.cookieService);
58
+ this.pagination = new operators_1.PaginationOperator(this.logger);
59
+ }
60
+ async newPage() {
61
+ return this.moduleRef.resolve(page_service_1.PageService, undefined, { strict: false });
58
62
  }
59
63
  async takeScreenshot(url, path, options, scraperOptions) {
60
64
  this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Taking screenshot of ${url}`));
61
65
  const cloak = scraperOptions?.useRandomUserAgent
62
66
  ? { ...scraperOptions?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
63
67
  : scraperOptions?.cloak;
64
- const page = await this.pageService.navigateTo(url, this.buildNavOptions(scraperOptions), cloak, scraperOptions?.interceptResource);
68
+ const pageService = await this.newPage();
69
+ const page = await pageService.navigateTo(url, this.buildNavOptions(scraperOptions), cloak, scraperOptions?.interceptResource);
65
70
  try {
66
- const result = await page.screenshot({ path, ...options });
67
- return result;
71
+ return await this.capture.screenshot(page, path, options);
68
72
  }
69
73
  finally {
70
- await this.pageService.closePage();
74
+ await pageService.closePage();
71
75
  }
72
76
  }
73
77
  async generatePDF(url, path, options, scraperOptions) {
@@ -75,99 +79,39 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
75
79
  const cloak = scraperOptions?.useRandomUserAgent
76
80
  ? { ...scraperOptions?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
77
81
  : scraperOptions?.cloak;
78
- const page = await this.pageService.navigateTo(url, this.buildNavOptions(scraperOptions), cloak, scraperOptions?.interceptResource);
82
+ const pageService = await this.newPage();
83
+ const page = await pageService.navigateTo(url, this.buildNavOptions(scraperOptions), cloak, scraperOptions?.interceptResource);
79
84
  try {
80
- const pdf = await page.pdf({ path, ...options });
81
- return Buffer.from(pdf);
85
+ return await this.capture.pdf(page, path, options);
82
86
  }
83
87
  finally {
84
- await this.pageService.closePage();
88
+ await pageService.closePage();
85
89
  }
86
90
  }
87
91
  async captureTlsFingerprint(path, url = browser_action_constants_1.TLS_CAPTURE_URL) {
88
92
  this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Capturing TLS fingerprint from ${url}`));
89
- const page = await this.pageService.navigateTo(url);
90
- let rawJson;
93
+ const pageService = await this.newPage();
94
+ const page = await pageService.navigateTo(url);
91
95
  try {
92
- rawJson = await page.evaluate(() => document.body.innerText);
96
+ return await this.capture.tlsFingerprint(page, path);
93
97
  }
94
98
  finally {
95
- await this.pageService.closePage();
99
+ await pageService.closePage();
96
100
  }
97
- const data = JSON.parse(rawJson);
98
- const fingerprint = this.buildTlsFingerprint(data);
99
- await fs_1.promises.mkdir((0, path_1.dirname)(path), { recursive: true });
100
- await fs_1.promises.writeFile(path, JSON.stringify(fingerprint, null, 2), 'utf-8');
101
- this.logger.log(`Saved TLS fingerprint to ${path}`);
102
- return fingerprint;
103
- }
104
- buildTlsFingerprint(data) {
105
- const tls = (data.tls ?? {});
106
- const http2 = (data.http2 ?? {});
107
- const extensions = Array.isArray(tls.extensions)
108
- ? tls.extensions
109
- : [];
110
- const sentFrames = Array.isArray(http2.sent_frames)
111
- ? http2.sent_frames
112
- : [];
113
- const headersFrame = sentFrames.find((frame) => frame.frame_type === 'HEADERS');
114
- return {
115
- capturedAt: new Date().toISOString(),
116
- ip: data.ip ?? '',
117
- httpVersion: data.http_version ?? '',
118
- method: data.method ?? '',
119
- userAgent: data.user_agent ?? '',
120
- ja3: tls.ja3 ?? '',
121
- ja3Hash: tls.ja3_hash ?? '',
122
- ja4: tls.ja4 ?? '',
123
- ja4_r: tls.ja4_r,
124
- peetprint: tls.peetprint ?? '',
125
- peetprintHash: tls.peetprint_hash ?? '',
126
- ciphers: Array.isArray(tls.ciphers) ? tls.ciphers : [],
127
- tlsExtensions: extensions.map((ext) => String(ext.name)),
128
- akamaiFingerprint: http2.akamai_fingerprint ?? '',
129
- akamaiFingerprintHash: http2.akamai_fingerprint_hash ?? '',
130
- headers: Array.isArray(headersFrame?.headers)
131
- ? headersFrame.headers
132
- : [],
133
- raw: data,
134
- };
135
101
  }
136
102
  async scrape(url, selectors, options) {
137
103
  this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Scraping ${url}`));
138
104
  const cloak = options?.useRandomUserAgent
139
105
  ? { ...options?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
140
106
  : options?.cloak;
141
- const page = await this.pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
142
- const result = {};
143
- const evalOne = page['$eval'].bind(page);
107
+ const pageService = await this.newPage();
108
+ const page = await pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
144
109
  try {
145
- await Promise.all(Object.entries(selectors).map(async ([key, rawSelector]) => {
146
- const { selector, attribute } = this.parseSelector(rawSelector);
147
- try {
148
- const value = attribute
149
- ? await evalOne(selector, (el, attr) => el.getAttribute(attr), attribute)
150
- : await evalOne(selector, (el) => el.textContent);
151
- result[key] = options?.pipes?.[key]
152
- ? this.pipeEngine.apply(typeof value === 'string' ? value : String(value ?? ''), options.pipes[key], url)
153
- : value;
154
- }
155
- catch (err) {
156
- this.logger.warn(`Failed to scrape '${key}' (${rawSelector}): ${err instanceof Error ? err.message : String(err)}`);
157
- }
158
- }));
110
+ return await this.extraction.extractSingle(page, url, selectors, options?.pipes);
159
111
  }
160
112
  finally {
161
- await this.pageService.closePage();
162
- }
163
- return result;
164
- }
165
- parseSelector(raw) {
166
- const match = raw.match(/^(.*)@([A-Za-z_][\w-]*)$/);
167
- if (match && match[1].trim() !== '') {
168
- return { selector: match[1], attribute: match[2] };
113
+ await pageService.closePage();
169
114
  }
170
- return { selector: raw };
171
115
  }
172
116
  buildNavOptions(options) {
173
117
  if (!options?.waitUntil && !options?.timeout)
@@ -182,12 +126,13 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
182
126
  const cloak = options?.useRandomUserAgent
183
127
  ? { ...options?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
184
128
  : options?.cloak;
185
- const page = await this.pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
129
+ const pageService = await this.newPage();
130
+ const page = await pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
186
131
  const result = {};
187
132
  const evalAll = page['$$eval'].bind(page);
188
- Object.entries(selectors).forEach(([key, selector]) => this.validateSelector(key, selector));
133
+ Object.entries(selectors).forEach(([key, selector]) => this.extraction.validateSelector(key, selector));
189
134
  await Promise.all(Object.entries(selectors).map(async ([key, rawSelector]) => {
190
- const { selector, attribute } = this.parseSelector(rawSelector);
135
+ const { selector, attribute } = this.extraction.parseSelector(rawSelector);
191
136
  try {
192
137
  const isXPath = (0, dom_util_1.isXPathSelector)(selector);
193
138
  let values;
@@ -232,7 +177,7 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
232
177
  this.logger.warn(`Failed to scrape '${key}' (${rawSelector}): ${err instanceof Error ? err.message : String(err)}`);
233
178
  }
234
179
  }));
235
- await this.pageService.closePage();
180
+ await pageService.closePage();
236
181
  return result;
237
182
  }
238
183
  async scrapeContainerFields(url, descriptor, options) {
@@ -240,12 +185,13 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
240
185
  const cloak = options?.useRandomUserAgent
241
186
  ? { ...options?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
242
187
  : options?.cloak;
188
+ const pageService = await this.newPage();
243
189
  let page;
244
190
  try {
245
- page = await this.pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
191
+ page = await pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
246
192
  }
247
193
  catch (err) {
248
- await this.pageService.closePage();
194
+ await pageService.closePage();
249
195
  throw err;
250
196
  }
251
197
  try {
@@ -273,7 +219,7 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
273
219
  return { items, pagination: raw.pagination };
274
220
  }
275
221
  finally {
276
- await this.pageService.closePage();
222
+ await pageService.closePage();
277
223
  }
278
224
  }
279
225
  async evaluateWebsite(options) {
@@ -325,7 +271,7 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
325
271
  const template = pagination.urlTemplate ?? url;
326
272
  const startPage = pagination.startPage ?? 2;
327
273
  const { items: page1Items } = await this.scrapeContainerFields(url, descriptor, scraperOpts);
328
- const { items: restItems, pages: restPages } = await this.paginateUrlIncrement(template, async (pageUrl) => {
274
+ const { items: restItems, pages: restPages } = await this.pagination.paginateUrlIncrement(template, async (pageUrl) => {
329
275
  const { items } = await this.scrapeContainerFields(pageUrl, descriptor, scraperOpts);
330
276
  return items;
331
277
  }, pagination, startPage);
@@ -337,12 +283,13 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
337
283
  const cloak = scraperOpts.useRandomUserAgent
338
284
  ? { ...scraperOpts.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
339
285
  : scraperOpts.cloak;
286
+ const pageService = await this.newPage();
340
287
  let page;
341
288
  try {
342
- page = await this.pageService.navigateTo(url, this.buildNavOptions(scraperOpts), cloak, scraperOpts.interceptResource);
289
+ page = await pageService.navigateTo(url, this.buildNavOptions(scraperOpts), cloak, scraperOpts.interceptResource);
343
290
  }
344
291
  catch (err) {
345
- await this.pageService.closePage();
292
+ await pageService.closePage();
346
293
  throw err;
347
294
  }
348
295
  try {
@@ -370,271 +317,29 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
370
317
  });
371
318
  };
372
319
  if (pagination.type === 'click-next') {
373
- const { items, pages } = await this.paginateClickNext(page, containerFn, pagination);
320
+ const { items, pages } = await this.pagination.paginateClickNext(page, containerFn, pagination);
374
321
  return { results: items, totalPages: pages };
375
322
  }
376
323
  else if (pagination.type === 'load-more') {
377
- const { items, pages } = await this.paginateLoadMore(page, containerFn, pagination);
324
+ const { items, pages } = await this.pagination.paginateLoadMore(page, containerFn, pagination);
378
325
  return { results: items, totalPages: pages };
379
326
  }
380
327
  else {
381
- const { items, pages } = await this.paginateInfiniteScroll(page, containerFn, pagination);
328
+ const { items, pages } = await this.pagination.paginateInfiniteScroll(page, containerFn, pagination);
382
329
  return { results: items, totalPages: pages };
383
330
  }
384
331
  }
385
332
  finally {
386
- await this.pageService.closePage();
387
- }
388
- }
389
- async findPaginationElement(page, selector) {
390
- if (!(0, dom_util_1.isXPathSelector)(selector)) {
391
- return page.$(selector);
392
- }
393
- const pageWithX = page;
394
- if (typeof pageWithX.$x === 'function') {
395
- const results = await pageWithX.$x(selector);
396
- return results[0] ?? null;
397
- }
398
- const handle = await page
399
- .evaluateHandle((sel) => {
400
- const res = document.evaluate(sel, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
401
- return res.singleNodeValue;
402
- }, selector)
403
- .then((h) => h.asElement());
404
- return handle;
405
- }
406
- async paginateClickNext(page, containerFn, opts) {
407
- const max = opts.maxPages ?? 10;
408
- const wait = opts.waitAfter ?? 800;
409
- const all = [];
410
- let pages = 0;
411
- for (let i = 0; i < max; i++) {
412
- try {
413
- const items = await containerFn(page);
414
- all.push(...items);
415
- pages++;
416
- if (!opts.selector)
417
- break;
418
- const btn = await this.findPaginationElement(page, opts.selector);
419
- if (!btn)
420
- break;
421
- await Promise.all([
422
- page.waitForNavigation({ waitUntil: 'domcontentloaded' }),
423
- btn.click(),
424
- ]);
425
- await (0, delay_util_1.delay)(wait);
426
- }
427
- catch (err) {
428
- if (isTargetClosed(err))
429
- break;
430
- if (err instanceof Error && err.message.includes('TimeoutError'))
431
- break;
432
- throw err;
433
- }
434
- }
435
- return { items: all, pages };
436
- }
437
- async paginateLoadMore(page, containerFn, opts) {
438
- const max = opts.maxPages ?? 10;
439
- const wait = opts.waitAfter ?? 800;
440
- let previousCount = 0;
441
- let clicks = 0;
442
- let lastItems = [];
443
- for (let i = 0; i < max; i++) {
444
- try {
445
- const items = await containerFn(page);
446
- if (items.length === previousCount) {
447
- return { items, pages: clicks + 1 };
448
- }
449
- lastItems = items;
450
- previousCount = items.length;
451
- if (!opts.selector)
452
- break;
453
- const btn = await this.findPaginationElement(page, opts.selector);
454
- if (!btn)
455
- break;
456
- await btn.click();
457
- clicks++;
458
- await (0, delay_util_1.delay)(wait);
459
- }
460
- catch (err) {
461
- if (isTargetClosed(err))
462
- break;
463
- throw err;
464
- }
465
- }
466
- try {
467
- const finalItems = await containerFn(page);
468
- return { items: finalItems, pages: clicks + 1 };
469
- }
470
- catch (err) {
471
- if (isTargetClosed(err))
472
- return { items: lastItems, pages: clicks + 1 };
473
- throw err;
474
- }
475
- }
476
- async paginateInfiniteScroll(page, containerFn, opts) {
477
- const max = opts.maxPages ?? 10;
478
- const wait = opts.waitAfter ?? 800;
479
- let previousCount = 0;
480
- let scrolls = 0;
481
- let lastItems = [];
482
- for (let i = 0; i < max; i++) {
483
- try {
484
- const items = await containerFn(page);
485
- if (opts.endSelector) {
486
- const endEl = await this.findPaginationElement(page, opts.endSelector);
487
- if (endEl)
488
- return { items, pages: scrolls + 1 };
489
- }
490
- if (items.length === previousCount)
491
- return { items, pages: scrolls + 1 };
492
- lastItems = items;
493
- previousCount = items.length;
494
- if (opts.selector) {
495
- const sentinel = await this.findPaginationElement(page, opts.selector);
496
- if (sentinel) {
497
- await page.evaluate((el) => el.scrollIntoView(), sentinel);
498
- }
499
- else {
500
- await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
501
- }
502
- }
503
- else {
504
- await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
505
- }
506
- scrolls++;
507
- await (0, delay_util_1.delay)(wait);
508
- }
509
- catch (err) {
510
- if (isTargetClosed(err))
511
- break;
512
- throw err;
513
- }
514
- }
515
- try {
516
- const finalItems = await containerFn(page);
517
- return { items: finalItems, pages: scrolls + 1 };
518
- }
519
- catch (err) {
520
- if (isTargetClosed(err))
521
- return { items: lastItems, pages: scrolls + 1 };
522
- throw err;
523
- }
524
- }
525
- async paginateUrlIncrement(urlTemplate, containerFn, opts, startPage) {
526
- const max = opts.maxPages ?? 10;
527
- const wait = opts.waitAfter ?? 800;
528
- const all = [];
529
- let pagesScraped = 0;
530
- for (let p = startPage; p < startPage + max; p++) {
531
- try {
532
- const url = urlTemplate.replace('{page}', String(p));
533
- const items = await containerFn(url);
534
- if (items.length === 0)
535
- break;
536
- all.push(...items);
537
- pagesScraped++;
538
- if (p < startPage + max - 1)
539
- await (0, delay_util_1.delay)(wait);
540
- }
541
- catch (err) {
542
- if (isTargetClosed(err))
543
- break;
544
- throw err;
545
- }
333
+ await pageService.closePage();
546
334
  }
547
- return { items: all, pages: pagesScraped };
548
- }
549
- async extractAllData(page, target, as = 'text', attribute) {
550
- if (target.shadowHost && !target.value) {
551
- const hosts = await page.$$(target.shadowHost);
552
- const results = await Promise.all(hosts.map((host) => host.evaluate((el, extractAs, attr) => {
553
- const sr = el.shadowRoot;
554
- if (!sr)
555
- return '';
556
- if (extractAs === 'html' || extractAs === 'outerHtml')
557
- return sr.innerHTML;
558
- if (extractAs === 'attribute')
559
- return el.getAttribute(attr || '') || '';
560
- return sr.textContent?.trim() || '';
561
- }, as, attribute)));
562
- return results;
563
- }
564
- if (target.shadowHost) {
565
- return await this.extractAllFromShadowRoot(page, target, as, attribute);
566
- }
567
- if (target.type === 'css') {
568
- return await page.$$eval(target.value, (elements, extractAs, attr) => elements.map((el) => {
569
- if (extractAs === 'html')
570
- return el.innerHTML;
571
- if (extractAs === 'outerHtml')
572
- return el.outerHTML;
573
- if (extractAs === 'attribute')
574
- return el.getAttribute(attr || '') || '';
575
- return el.textContent?.trim() || '';
576
- }), as, attribute);
577
- }
578
- return await page.evaluate((selector, extractAs, attr) => {
579
- const results = document.evaluate(selector, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
580
- const values = [];
581
- for (let i = 0; i < results.snapshotLength; i++) {
582
- const node = results.snapshotItem(i);
583
- if (!(node instanceof Element)) {
584
- values.push(node?.textContent?.trim() || '');
585
- continue;
586
- }
587
- if (extractAs === 'html')
588
- values.push(node.innerHTML);
589
- else if (extractAs === 'outerHtml')
590
- values.push(node.outerHTML);
591
- else if (extractAs === 'attribute')
592
- values.push(node.getAttribute(attr || '') || '');
593
- else
594
- values.push(node.textContent?.trim() || '');
595
- }
596
- return values;
597
- }, target.value, as, attribute);
598
- }
599
- async extractAllFromShadowRoot(page, target, as = 'text', attribute) {
600
- const hosts = await page.$$(target.shadowHost);
601
- const allResults = await Promise.all(hosts.map(async (host) => {
602
- return await host.evaluate((el, selector, targetType, extractAs, attr) => {
603
- const shadowRoot = el.shadowRoot;
604
- if (!shadowRoot)
605
- return [];
606
- const getVal = (e) => {
607
- if (extractAs === 'html')
608
- return e.innerHTML;
609
- if (extractAs === 'outerHtml')
610
- return e.outerHTML;
611
- if (extractAs === 'attribute')
612
- return e.getAttribute(attr || '') || '';
613
- return e.textContent?.trim() || '';
614
- };
615
- if (targetType === 'css') {
616
- const elements = shadowRoot.querySelectorAll(selector);
617
- return Array.from(elements).map(getVal);
618
- }
619
- const results = document.evaluate(selector, shadowRoot, null, XPathResult.UNORDERED_NODE_ITERATOR_TYPE, null);
620
- const values = [];
621
- let node;
622
- while ((node = results.iterateNext())) {
623
- values.push(node instanceof Element
624
- ? getVal(node)
625
- : node?.textContent?.trim() || '');
626
- }
627
- return values;
628
- }, target.value, target.type, as, attribute);
629
- }));
630
- return allResults.flat();
631
335
  }
632
336
  async waitForSelector(url, selector, timeout, scraperOptions) {
633
337
  this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Waiting for ${selector} on ${url}`));
634
338
  const cloak = scraperOptions?.useRandomUserAgent
635
339
  ? { ...scraperOptions?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
636
340
  : scraperOptions?.cloak;
637
- const page = await this.pageService.navigateTo(url, this.buildNavOptions(scraperOptions), cloak, scraperOptions?.interceptResource);
341
+ const pageService = await this.newPage();
342
+ const page = await pageService.navigateTo(url, this.buildNavOptions(scraperOptions), cloak, scraperOptions?.interceptResource);
638
343
  await page.waitForSelector(selector, { timeout });
639
344
  return page;
640
345
  }
@@ -643,11 +348,12 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
643
348
  const cloak = options?.useRandomUserAgent
644
349
  ? { ...options?.cloak, userAgent: (0, user_agent_util_1.getRandomUserAgent)() }
645
350
  : options?.cloak;
646
- const page = await this.pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
351
+ const pageService = await this.newPage();
352
+ const page = await pageService.navigateTo(url, this.buildNavOptions(options), cloak, options?.interceptResource);
647
353
  const result = typeof script === 'function'
648
354
  ? await page.evaluate(script)
649
355
  : await page.evaluate(script);
650
- await this.pageService.closePage();
356
+ await pageService.closePage();
651
357
  return result;
652
358
  }
653
359
  async scrapeWithWorkflow(url, workflow, variables) {
@@ -657,7 +363,8 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
657
363
  this.moduleOptions?.debugLogMaxLength ??
658
364
  browser_action_constants_1.DEFAULT_DEBUG_LOG_MAX_LENGTH;
659
365
  this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Starting workflow execution for ${url}`));
660
- const page = await this.pageService.navigateTo(url, undefined, workflow.cloak, workflow.interceptResource);
366
+ const pageService = await this.newPage();
367
+ const page = await pageService.navigateTo(url, undefined, workflow.cloak, workflow.interceptResource);
661
368
  const result = {
662
369
  success: false,
663
370
  data: {},
@@ -669,7 +376,7 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
669
376
  const errorConfig = workflow.onError || {};
670
377
  for (const action of workflow.actions) {
671
378
  try {
672
- await this.executeAction(page, action, context);
379
+ await this.workflow.executeAction(page, action, context, this.activeDebugLogMaxLength);
673
380
  }
674
381
  catch (error) {
675
382
  const errorMessage = error instanceof Error ? error.message : String(error);
@@ -685,7 +392,7 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
685
392
  }
686
393
  if (!errorConfig.continue) {
687
394
  result.errors.push(errorMessage);
688
- await this.pageService.closePage();
395
+ await pageService.closePage();
689
396
  return result;
690
397
  }
691
398
  }
@@ -699,666 +406,36 @@ let BrowserActionService = BrowserActionService_1 = class BrowserActionService {
699
406
  this.logger.error(`Workflow execution failed: ${errorMessage}`);
700
407
  }
701
408
  finally {
702
- await this.pageService.closePage();
409
+ await pageService.closePage();
703
410
  }
704
411
  return result;
705
412
  }
706
413
  async scrapeAllWithWorkflow(url, workflow, variables) {
707
414
  return await this.scrapeWithWorkflow(url, workflow, variables);
708
415
  }
416
+ async executeContainerExtraction(page, descriptor, currentPage = 1) {
417
+ return this.container.executeContainerExtraction(page, descriptor, currentPage);
418
+ }
709
419
  async executeAction(page, action, context) {
710
420
  if (action.condition) {
711
421
  const shouldExecute = await this.evaluateCondition(page, action.condition);
712
- if (!shouldExecute) {
713
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Skipping action due to condition: ${action.action}`));
422
+ if (!shouldExecute)
714
423
  return;
715
- }
716
- }
717
- const value = this.interpolateValue(String(action.value || ''), context);
718
- const actionLabel = action.id
719
- ? `[${action.action}] id="${action.id}"`
720
- : `[${action.action}]`;
721
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Executing action: ${actionLabel}`));
722
- const maxRetries = Math.min(action.options?.retry ?? 0, 100);
723
- const retryDelay = Math.min(action.options?.retryDelay ?? 0, 300_000);
724
- for (let attempt = 0;; attempt++) {
725
- try {
726
- await this.dispatchAction(page, action, value, context);
727
- return;
728
- }
729
- catch (err) {
730
- if (attempt >= maxRetries)
731
- throw err;
732
- this.logger.warn((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Action ${actionLabel} failed (attempt ${attempt + 1}/${maxRetries + 1}), retrying: ${err instanceof Error ? err.message : String(err)}`));
733
- if (retryDelay > 0)
734
- await (0, delay_util_1.delay)(retryDelay);
735
- }
736
- }
737
- }
738
- async dispatchAction(page, action, value, context) {
739
- switch (action.action) {
740
- case 'navigate': {
741
- const navOptions = {};
742
- if (action.options?.waitUntil)
743
- navOptions.waitUntil = action.options.waitUntil;
744
- if (action.options?.timeout)
745
- navOptions.timeout = action.options.timeout;
746
- if (!(0, class_validator_1.isURL)(value, {
747
- require_protocol: true,
748
- protocols: ['http', 'https'],
749
- })) {
750
- throw new Error(`Invalid or disallowed URL: ${value}`);
751
- }
752
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` navigate → ${value}`));
753
- await page.goto(value, Object.keys(navOptions).length ? navOptions : undefined);
754
- break;
755
- }
756
- case 'wait':
757
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` wait ${Number(action.value) || 0}ms`));
758
- await (0, delay_util_1.delay)(Math.min(Number(action.value) || 0, 300_000));
759
- break;
760
- case 'waitFor':
761
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` waitFor target: ${this.describeTarget(action.target)}`));
762
- await this.waitForTarget(page, action.target, action.options);
763
- break;
764
- case 'click':
765
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` click target: ${this.describeTarget(action.target)}`));
766
- await this.clickElement(page, action.target, action.options);
767
- break;
768
- case 'type':
769
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` type target: ${this.describeTarget(action.target)} value: "${value}"`));
770
- await this.typeText(page, action.target, value, action.options);
771
- break;
772
- case 'select':
773
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` select target: ${this.describeTarget(action.target)} value: "${value}"`));
774
- await this.selectOption(page, action.target, value, action.options);
775
- break;
776
- case 'scroll':
777
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` scroll target: ${this.describeTarget(action.target)}`));
778
- await this.scrollToElement(page, action.target);
779
- break;
780
- case 'extract': {
781
- const extractOptions = action.options || {};
782
- const extractAs = extractOptions.as ?? 'text';
783
- const extractAttr = extractOptions.attribute;
784
- if (!action.target) {
785
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extract full page HTML`));
786
- if (action.id)
787
- context[action.id] = await page.content();
788
- break;
789
- }
790
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extract target: ${this.describeTarget(action.target)} as="${extractAs}"${extractAttr ? ` attr="${extractAttr}"` : ''}${extractOptions.multiple ? ' multiple=true' : ''}`));
791
- if (extractOptions.multiple) {
792
- const allValues = await this.extractAllData(page, action.target, extractAs, extractAttr);
793
- if (action.id) {
794
- context[action.id] = allValues;
795
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extracted ${Array.isArray(allValues) ? allValues.length : 1} item(s) → id="${action.id}"`));
796
- }
797
- }
798
- else {
799
- const extractedValue = await this.extractData(page, action.target, extractAs, extractAttr);
800
- if (action.id) {
801
- context[action.id] = extractedValue;
802
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extracted → id="${action.id}": ${JSON.stringify(extractedValue)}`));
803
- }
804
- }
805
- break;
806
- }
807
- case 'screenshot': {
808
- const screenshotPath = String(action.value || `${browser_action_constants_1.DEFAULT_SCREENSHOT_FILENAME}-${Date.now()}.png`);
809
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` screenshot → ${screenshotPath}`));
810
- await page.screenshot({ path: screenshotPath });
811
- break;
812
- }
813
- case 'evaluate': {
814
- let evalCode;
815
- if (value.includes('=>')) {
816
- evalCode = `(${value})()`;
817
- }
818
- else {
819
- evalCode = value;
820
- }
821
- if (evalCode.length > 50_000) {
822
- throw new Error('evaluate script exceeds maximum length of 50000 characters');
823
- }
824
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Evaluating: ${evalCode}`));
825
- const evalResult = await page.evaluate(evalCode);
826
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Result: ${JSON.stringify(evalResult)}`));
827
- if (action.id) {
828
- context[action.id] = evalResult;
829
- }
830
- break;
831
- }
832
- case 'cleanse': {
833
- const pipes = action.options?.pipes;
834
- if (!pipes) {
835
- throw new Error('cleanse action requires pipes');
836
- }
837
- const valueKey = String(action.value || '');
838
- const rawValue = this.resolveRawValue(valueKey, context);
839
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` cleanse raw: ${JSON.stringify(rawValue)}`));
840
- const cleanedValue = Array.isArray(rawValue)
841
- ? rawValue.map((item) => this.pipeEngine.apply(String(item), pipes))
842
- : this.pipeEngine.apply(String(rawValue), pipes);
843
- if (action.id) {
844
- context[action.id] = cleanedValue;
845
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, `Cleansed value for '${action.id}': ${JSON.stringify(cleanedValue)}`));
846
- }
847
- break;
848
- }
849
- case 'saveCookies': {
850
- const sessionName = String(value);
851
- const overwrite = action.options?.overwrite ?? false;
852
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` saveCookies session="${sessionName}" overwrite=${overwrite}`));
853
- const metadata = {};
854
- if (action.options?.metadata) {
855
- Object.assign(metadata, action.options.metadata);
856
- }
857
- await this.cookieService.saveCookies(page, sessionName, {
858
- overwrite,
859
- metadata,
860
- });
861
- break;
862
- }
863
- case 'loadCookies': {
864
- const sessionName = String(value);
865
- const throwIfNotExists = action.onError !== 'skip' && action.onError !== 'continue';
866
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` loadCookies session="${sessionName}"`));
867
- await this.cookieService.loadCookies(page, sessionName, {
868
- throwIfNotExists,
869
- });
870
- break;
871
- }
872
- case 'clearCookies': {
873
- const sessionName = String(value);
874
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` clearCookies session="${sessionName}"`));
875
- await this.cookieService.deleteCookies(sessionName);
876
- break;
877
- }
878
- case 'listCookies': {
879
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` listCookies`));
880
- const sessions = await this.cookieService.listCookies();
881
- if (action.id) {
882
- context[action.id] = sessions;
883
- }
884
- break;
885
- }
886
- case 'hover':
887
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` hover target: ${this.describeTarget(action.target)}`));
888
- await this.hoverElement(page, action.target, action.options);
889
- break;
890
- case 'keyPress':
891
- if (!value) {
892
- throw new Error('keyPress action requires a key value');
893
- }
894
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` keyPress key="${value}"`));
895
- await page.keyboard.press(value);
896
- break;
897
- case 'clear':
898
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` clear target: ${this.describeTarget(action.target)}`));
899
- await this.clearElement(page, action.target);
900
- break;
901
- case 'waitForNetwork':
902
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` waitForNetwork timeout=${action.options?.timeout ?? browser_action_constants_1.DEFAULT_ACTION_TIMEOUT}ms`));
903
- await page.waitForNetworkIdle({
904
- timeout: action.options?.timeout ?? browser_action_constants_1.DEFAULT_ACTION_TIMEOUT,
905
- });
906
- break;
907
- case 'reload':
908
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` reload waitUntil="${action.options?.waitUntil ?? 'load'}"`));
909
- await page.reload({
910
- waitUntil: action.options?.waitUntil ?? 'load',
911
- timeout: action.options?.timeout ?? browser_action_constants_1.DEFAULT_NAVIGATION_TIMEOUT,
912
- });
913
- break;
914
- case 'scrapeContainer': {
915
- const o = action.options;
916
- if (!o?.container || !o?.fields) {
917
- throw new Error('scrapeContainer requires options.container and options.fields');
918
- }
919
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` scrapeContainer container="${o.container}"`));
920
- const containerResult = await this.executeContainerExtraction(page, {
921
- container: o.container,
922
- fields: o.fields,
923
- pagination: o.pagination,
924
- }, o.currentPage ?? 1);
925
- if (action.id) {
926
- context[action.id] = containerResult.items;
927
- if (containerResult.pagination) {
928
- context[`${action.id}_pagination`] = containerResult.pagination;
929
- }
930
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` scrapeContainer → id="${action.id}": ${containerResult.items.length} items`));
931
- }
932
- break;
933
- }
934
- case 'extractPagination': {
935
- const o = action.options;
936
- if (!o?.container || !o?.linkSelector || !o?.labelSelector) {
937
- throw new Error('extractPagination requires options.container, linkSelector, and labelSelector');
938
- }
939
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extractPagination container="${o.container}"`));
940
- const paginationResult = await this.resolvePagination(page, {
941
- container: o.container,
942
- linkSelector: o.linkSelector,
943
- labelSelector: o.labelSelector,
944
- }, o.currentPage ?? 1);
945
- if (action.id) {
946
- context[action.id] = paginationResult;
947
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` extractPagination → id="${action.id}": nextUrl=${paginationResult.nextUrl}`));
948
- }
949
- break;
950
- }
951
- default: {
952
- const exhaustiveCheck = action.action;
953
- throw new Error(`Unknown action type: ${String(exhaustiveCheck)}`);
954
- }
955
424
  }
425
+ await this.workflow.executeAction(page, { ...action, condition: undefined }, context, this.activeDebugLogMaxLength);
956
426
  }
957
427
  async evaluateCondition(page, condition) {
958
- if (!condition)
959
- return true;
960
- if (condition.ifExists) {
961
- const element = await this.findElement(page, condition.ifExists);
962
- return element !== null;
963
- }
964
- if (condition.unlessExists) {
965
- const element = await this.findElement(page, condition.unlessExists);
966
- return element === null;
967
- }
968
- return true;
969
- }
970
- describeTarget(target) {
971
- const base = target.value
972
- ? `${target.type}="${target.value}"`
973
- : `(no selector)`;
974
- return target.shadowHost
975
- ? `shadowHost="${target.shadowHost}" > ${base}`
976
- : base;
977
- }
978
- async findElement(page, target) {
979
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` findElement: ${this.describeTarget(target)}`));
980
- if (target.shadowHost) {
981
- const el = await this.findElementInShadowRoot(page, target);
982
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` findElement result: ${el ? 'found' : 'NOT FOUND'}`));
983
- return el;
984
- }
985
- if (target.type === 'css') {
986
- const el = await page.$(target.value);
987
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` findElement result: ${el ? 'found' : 'NOT FOUND'}`));
988
- return el;
989
- }
990
- const el = await page
991
- .evaluateHandle((selector) => {
992
- const result = document.evaluate(selector, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
993
- return result.singleNodeValue;
994
- }, target.value)
995
- .then((handle) => handle.asElement());
996
- this.logger.debug((0, truncate_log_util_1.truncateLog)(this.activeDebugLogMaxLength, ` findElement result: ${el ? 'found' : 'NOT FOUND'}`));
997
- return el;
998
- }
999
- async findElementInShadowRoot(page, target) {
1000
- const host = await page.$(target.shadowHost);
1001
- if (!host)
1002
- return null;
1003
- return host
1004
- .evaluateHandle((el, selector, targetType) => {
1005
- const shadowRoot = el.shadowRoot;
1006
- if (!shadowRoot)
1007
- return null;
1008
- if (targetType === 'css') {
1009
- return shadowRoot.querySelector(selector);
1010
- }
1011
- const result = document.evaluate(selector, shadowRoot, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
1012
- return result.singleNodeValue;
1013
- }, target.value, target.type)
1014
- .then((handle) => handle.asElement());
1015
- }
1016
- async waitForTarget(page, target, options) {
1017
- const timeout = options?.timeout || browser_action_constants_1.DEFAULT_ACTION_TIMEOUT;
1018
- if (target.type === 'css') {
1019
- await page.waitForSelector(target.value, { timeout });
1020
- }
1021
- else {
1022
- await page.waitForFunction((selector) => {
1023
- const result = document.evaluate(selector, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
1024
- return result.singleNodeValue !== null;
1025
- }, { timeout }, target.value);
1026
- }
1027
- }
1028
- async clickElement(page, target, options) {
1029
- const element = await this.findElement(page, target);
1030
- if (!element) {
1031
- throw new Error(`Element not found: ${target.value}`);
1032
- }
1033
- await this.maybeScrollToElement(element, options?.scrollTo);
1034
- const elem = element;
1035
- await elem.click();
1036
- if (options?.waitForNavigation) {
1037
- await page.waitForNavigation({
1038
- timeout: options.navigationTimeout || browser_action_constants_1.DEFAULT_NAVIGATION_TIMEOUT,
1039
- });
1040
- }
1041
- }
1042
- async typeText(page, target, value, options) {
1043
- const element = await this.findElement(page, target);
1044
- if (!element) {
1045
- throw new Error(`Element not found: ${target.value}`);
1046
- }
1047
- await this.maybeScrollToElement(element, options?.scrollTo);
1048
- const elem = element;
1049
- await elem.click();
1050
- const delay = options?.delay || 0;
1051
- await elem.type(value, { delay });
1052
- await elem.evaluate((el) => {
1053
- el.dispatchEvent(new Event('change', { bubbles: true }));
1054
- el.dispatchEvent(new Event('blur', { bubbles: true }));
1055
- });
1056
- }
1057
- async selectOption(page, target, value, options) {
1058
- const element = await this.findElement(page, target);
1059
- if (!element) {
1060
- throw new Error(`Element not found: ${target.value}`);
1061
- }
1062
- await this.maybeScrollToElement(element, options?.scrollTo);
1063
- const elem = element;
1064
- await elem.select(value);
1065
- }
1066
- async scrollToElement(page, target) {
1067
- const element = await this.findElement(page, target);
1068
- if (!element) {
1069
- throw new Error(`Element not found: ${target.value}`);
1070
- }
1071
- await this.scrollElementIntoView(element);
1072
- }
1073
- async maybeScrollToElement(element, shouldScroll) {
1074
- if (shouldScroll) {
1075
- await this.scrollElementIntoView(element);
1076
- }
1077
- }
1078
- async scrollElementIntoView(element) {
1079
- await element.evaluate((el) => el.scrollIntoView({ behavior: 'smooth', block: 'center' }), {});
1080
- await (0, delay_util_1.delay)(browser_action_constants_1.DEFAULT_SCROLL_DELAY_MS);
1081
- }
1082
- async executeContainerExtraction(page, descriptor, currentPage = 1) {
1083
- const rawItems = await page.evaluate((containerSel, fieldDefs) => {
1084
- const isXPath = (s) => s.trim().startsWith('//') ||
1085
- s.trim().startsWith('(') ||
1086
- s.trim().startsWith('./');
1087
- const getContainerNodes = (sel) => {
1088
- if (isXPath(sel)) {
1089
- const res = document.evaluate(sel, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
1090
- const nodes = [];
1091
- for (let i = 0; i < res.snapshotLength; i++)
1092
- nodes.push(res.snapshotItem(i));
1093
- return nodes;
1094
- }
1095
- return Array.from(document.querySelectorAll(sel));
1096
- };
1097
- const extractOne = (sel, root, attr, returnType) => {
1098
- let node = null;
1099
- if (isXPath(sel)) {
1100
- const res = document.evaluate(sel, root, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
1101
- node = res.singleNodeValue;
1102
- }
1103
- else {
1104
- node = root.querySelector(sel);
1105
- }
1106
- if (!node)
1107
- return '';
1108
- if (attr)
1109
- return node.getAttribute?.(attr) ?? '';
1110
- if (returnType === 'html')
1111
- return node.innerHTML ?? '';
1112
- return node.textContent?.trim() ?? '';
1113
- };
1114
- const extractMany = (sel, root, attr, returnType) => {
1115
- let nodes = [];
1116
- if (isXPath(sel)) {
1117
- const res = document.evaluate(sel, root, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
1118
- for (let i = 0; i < res.snapshotLength; i++)
1119
- nodes.push(res.snapshotItem(i));
1120
- }
1121
- else {
1122
- nodes = Array.from(root.querySelectorAll(sel));
1123
- }
1124
- return nodes.map((n) => {
1125
- if (attr)
1126
- return n.getAttribute?.(attr) ?? '';
1127
- if (returnType === 'html')
1128
- return n.innerHTML ?? '';
1129
- return n.textContent?.trim() ?? '';
1130
- });
1131
- };
1132
- const extractField = (root, fd) => {
1133
- const selectors = [fd.selector, ...(fd.fallback ?? [])];
1134
- for (const sel of selectors) {
1135
- const val = fd.multiple
1136
- ? extractMany(sel, root, fd.attribute, fd.returnType)
1137
- : extractOne(sel, root, fd.attribute, fd.returnType);
1138
- if (Array.isArray(val) ? val.length > 0 : val !== '')
1139
- return val;
1140
- }
1141
- return fd.multiple ? [] : '';
1142
- };
1143
- return getContainerNodes(containerSel).map((node) => {
1144
- const item = {};
1145
- for (const [key, fd] of Object.entries(fieldDefs))
1146
- item[key] = extractField(node, fd);
1147
- return item;
1148
- });
1149
- }, descriptor.container, descriptor.fields);
1150
- const items = rawItems;
1151
- let pagination;
1152
- if (descriptor.pagination) {
1153
- pagination = await this.resolvePagination(page, descriptor.pagination, currentPage);
1154
- }
1155
- return { items, pagination };
1156
- }
1157
- async resolvePagination(page, descriptor, currentPage = 1) {
1158
- currentPage = Math.max(1, currentPage);
1159
- const rawPages = await page.evaluate((containerSel, linkSel, labelSel) => {
1160
- const getNode = (sel, root) => {
1161
- if (sel.trim().startsWith('//') ||
1162
- sel.trim().startsWith('(') ||
1163
- sel.trim().startsWith('./')) {
1164
- const res = document.evaluate(sel, root, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
1165
- return res.singleNodeValue;
1166
- }
1167
- return root.querySelector(sel);
1168
- };
1169
- const getNodes = (sel, root) => {
1170
- if (sel.trim().startsWith('//') ||
1171
- sel.trim().startsWith('(') ||
1172
- sel.trim().startsWith('./')) {
1173
- const res = document.evaluate(sel, root, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
1174
- const nodes = [];
1175
- for (let i = 0; i < res.snapshotLength; i++)
1176
- nodes.push(res.snapshotItem(i));
1177
- return nodes;
1178
- }
1179
- return Array.from(root.querySelectorAll(sel));
1180
- };
1181
- const container = getNode(containerSel, document);
1182
- if (!container)
1183
- return [];
1184
- return getNodes(linkSel, container).map((el) => {
1185
- const isXPath = (s) => s.trim().startsWith('//') ||
1186
- s.trim().startsWith('(') ||
1187
- s.trim().startsWith('./');
1188
- const labelNodes = isXPath(labelSel)
1189
- ? (() => {
1190
- const r = document.evaluate(labelSel, el, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
1191
- const ns = [];
1192
- for (let i = 0; i < r.snapshotLength; i++)
1193
- ns.push(r.snapshotItem(i));
1194
- return ns;
1195
- })()
1196
- : Array.from(el.querySelectorAll(labelSel));
1197
- const label = labelNodes.length > 0
1198
- ? (labelNodes[0].textContent?.trim() ?? '')
1199
- : (el.textContent?.trim() ?? '');
1200
- return {
1201
- label,
1202
- url: el.getAttribute('href') ?? '',
1203
- };
1204
- });
1205
- }, descriptor.container, descriptor.linkSelector, descriptor.labelSelector);
1206
- const pages = rawPages.filter((p) => p.url);
1207
- const numericNext = pages
1208
- .filter((p) => !Number.isNaN(parseInt(p.label, 10)) &&
1209
- parseInt(p.label, 10) > currentPage)
1210
- .sort((a, b) => parseInt(a.label, 10) - parseInt(b.label, 10))[0];
1211
- const nextUrl = numericNext?.url ??
1212
- pages.find((p) => ['next', '>'].includes(p.label.toLowerCase()))?.url ??
1213
- null;
1214
- return { pages, nextUrl };
1215
- }
1216
- validateSelector(key, selector) {
1217
- if (!selector || selector.trim() === '') {
1218
- throw new Error(`Selector for '${key}' cannot be empty`);
1219
- }
1220
- }
1221
- async extractData(page, target, as = 'text', attribute) {
1222
- if (target.shadowHost && !target.value) {
1223
- const host = await page.$(target.shadowHost);
1224
- if (!host)
1225
- throw new Error(`Shadow host not found: ${target.shadowHost}`);
1226
- return host.evaluate((el, extractAs, attr) => {
1227
- const sr = el.shadowRoot;
1228
- if (!sr)
1229
- return '';
1230
- if (extractAs === 'html' || extractAs === 'outerHtml')
1231
- return sr.innerHTML;
1232
- if (extractAs === 'attribute')
1233
- return el.getAttribute(attr || '') || '';
1234
- return sr.textContent?.trim() || '';
1235
- }, as, attribute);
1236
- }
1237
- const element = await this.findElement(page, target);
1238
- if (!element) {
1239
- throw new Error(`Element not found: ${target.value}`);
1240
- }
1241
- return element.evaluate((el, extractAs, attr) => {
1242
- if (extractAs === 'html')
1243
- return el.innerHTML;
1244
- if (extractAs === 'outerHtml')
1245
- return el.outerHTML;
1246
- if (extractAs === 'attribute')
1247
- return el.getAttribute(attr || '') || '';
1248
- return el.textContent?.trim() || '';
1249
- }, as, attribute);
1250
- }
1251
- async hoverElement(page, target, options) {
1252
- const element = await this.findElement(page, target);
1253
- if (!element) {
1254
- throw new Error(`Element not found: ${target.value}`);
1255
- }
1256
- await this.maybeScrollToElement(element, options?.scrollTo);
1257
- const elem = element;
1258
- await elem.hover();
1259
- }
1260
- async clearElement(page, target) {
1261
- const element = await this.findElement(page, target);
1262
- if (!element) {
1263
- throw new Error(`Element not found: ${target.value}`);
1264
- }
1265
- await element.evaluate((el) => {
1266
- if ('value' in el) {
1267
- el.value = '';
1268
- el.dispatchEvent(new Event('input', { bubbles: true }));
1269
- el.dispatchEvent(new Event('change', { bubbles: true }));
1270
- }
1271
- });
1272
- }
1273
- resolveRawValue(value, context) {
1274
- if (value.startsWith('${') && value.endsWith('}')) {
1275
- const path = value.slice(2, -1);
1276
- if (!path.includes('${') && !path.includes('.') && !path.includes('[')) {
1277
- return context[path] ?? value;
1278
- }
1279
- }
1280
- return this.resolveValue(value, context);
1281
- }
1282
- resolveValue(value, context) {
1283
- if (value.startsWith('${') && value.endsWith('}')) {
1284
- const path = value.slice(2, -1);
1285
- if (!path.includes('${')) {
1286
- const keys = path.split('.');
1287
- let result = context;
1288
- for (const key of keys) {
1289
- const arrayMatch = key.match(/^(\w+)\[(\d+)\]$/);
1290
- if (arrayMatch) {
1291
- const [, arrayKey, indexStr] = arrayMatch;
1292
- if (!result || typeof result !== 'object') {
1293
- return value;
1294
- }
1295
- const arrayValue = result[arrayKey];
1296
- const index = parseInt(indexStr, 10);
1297
- result = Array.isArray(arrayValue) ? arrayValue[index] : undefined;
1298
- }
1299
- else {
1300
- if (!result || typeof result !== 'object') {
1301
- return value;
1302
- }
1303
- result = result[key];
1304
- }
1305
- if (result === undefined) {
1306
- return value;
1307
- }
1308
- }
1309
- if (result === null || typeof result === 'object') {
1310
- return value;
1311
- }
1312
- return result;
1313
- }
1314
- return this.interpolateValue(path, context);
1315
- }
1316
- return value;
1317
- }
1318
- interpolateValue(value, context) {
1319
- return value.replace(/\$\{([^}]+)\}/g, (_match, path) => {
1320
- const keys = path.split('.');
1321
- let result = context;
1322
- for (const key of keys) {
1323
- if (key === '__proto__' ||
1324
- key === 'constructor' ||
1325
- key === 'prototype') {
1326
- return '';
1327
- }
1328
- const arrayMatch = key.match(/^(\w+)\[(\d+)\]$/);
1329
- if (arrayMatch) {
1330
- const [, arrayKey, indexStr] = arrayMatch;
1331
- if (!result || typeof result !== 'object') {
1332
- return '';
1333
- }
1334
- const arrayValue = result[arrayKey];
1335
- const index = parseInt(indexStr, 10);
1336
- result = Array.isArray(arrayValue) ? arrayValue[index] : undefined;
1337
- }
1338
- else {
1339
- if (!result || typeof result !== 'object') {
1340
- return '';
1341
- }
1342
- result = result[key];
1343
- }
1344
- if (result === undefined) {
1345
- return '';
1346
- }
1347
- }
1348
- if (result === null || typeof result === 'object') {
1349
- return '';
1350
- }
1351
- return result;
1352
- });
428
+ return this.workflow.evaluateCondition(page, condition, this.activeDebugLogMaxLength);
1353
429
  }
1354
430
  };
1355
431
  exports.BrowserActionService = BrowserActionService;
1356
432
  exports.BrowserActionService = BrowserActionService = BrowserActionService_1 = __decorate([
1357
- (0, common_1.Injectable)({ scope: common_1.Scope.TRANSIENT }),
1358
- __param(3, (0, common_1.Optional)()),
1359
- __param(3, (0, common_1.Inject)(browser_action_constants_1.BROWSER_ACTION_OPTIONS)),
433
+ (0, common_1.Injectable)(),
434
+ __param(4, (0, common_1.Optional)()),
435
+ __param(4, (0, common_1.Inject)(browser_action_constants_1.BROWSER_ACTION_OPTIONS)),
1360
436
  __metadata("design:paramtypes", [page_service_1.PageService,
1361
437
  cookie_service_1.CookieService,
1362
- cleansing_service_1.CleansingService, Object])
438
+ cleansing_service_1.CleansingService,
439
+ core_1.ModuleRef, Object])
1363
440
  ], BrowserActionService);
1364
441
  //# sourceMappingURL=browser-action.service.js.map