@rafikidota/scoutee 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. package/dist/modules/playwright/constant/playwright.browser.constant.js +3 -0
  2. package/dist/modules/playwright/constant/playwright.browser.constant.js.map +1 -1
  3. package/dist/modules/playwright/constant/playwright.env.constant.d.ts +1 -0
  4. package/dist/modules/playwright/constant/playwright.env.constant.js +2 -1
  5. package/dist/modules/playwright/constant/playwright.env.constant.js.map +1 -1
  6. package/dist/modules/playwright/env/playwright.env.config.d.ts +3 -2
  7. package/dist/modules/playwright/env/playwright.env.config.js +3 -2
  8. package/dist/modules/playwright/env/playwright.env.config.js.map +1 -1
  9. package/dist/modules/playwright/env/playwright.env.validation.js +4 -0
  10. package/dist/modules/playwright/env/playwright.env.validation.js.map +1 -1
  11. package/dist/modules/playwright/playwright.service.js +3 -3
  12. package/dist/modules/playwright/playwright.service.js.map +1 -1
  13. package/dist/modules/playwright/services/playwright.hook.service.d.ts +4 -0
  14. package/dist/modules/playwright/services/playwright.hook.service.js +17 -6
  15. package/dist/modules/playwright/services/playwright.hook.service.js.map +1 -1
  16. package/package.json +16 -16
package/dist/modules/playwright/constant/playwright.browser.constant.js CHANGED
@@ -9,5 +9,8 @@ exports.BROWSER_ARGS = [
9
9
  '--no-first-run',
10
10
  '--no-zygote',
11
11
  '--disable-gpu',
12
+ '--disable-blink-features=AutomationControlled',
13
+ '--disable-extensions',
14
+ '--disable-background-networking',
12
15
  ];
13
16
  //# sourceMappingURL=playwright.browser.constant.js.map
package/dist/modules/playwright/constant/playwright.browser.constant.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"playwright.browser.constant.js","sourceRoot":"","sources":["../../../../src/modules/playwright/constant/playwright.browser.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,YAAY,GAAG;IAC1B,cAAc;IACd,0BAA0B;IAC1B,yBAAyB;IACzB,iCAAiC;IACjC,gBAAgB;IAChB,aAAa;IACb,eAAe;CAChB,CAAC"}
1
+ {"version":3,"file":"playwright.browser.constant.js","sourceRoot":"","sources":["../../../../src/modules/playwright/constant/playwright.browser.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,YAAY,GAAG;IAC1B,cAAc;IACd,0BAA0B;IAC1B,yBAAyB;IACzB,iCAAiC;IACjC,gBAAgB;IAChB,aAAa;IACb,eAAe;IACf,+CAA+C;IAC/C,sBAAsB;IACtB,iCAAiC;CAClC,CAAC"}
package/dist/modules/playwright/constant/playwright.env.constant.d.ts CHANGED
@@ -5,3 +5,4 @@ export declare const TIMEOUT_SECS = "CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS";
5
5
  export declare const MAX_REQUESTS = "CRAWLEE_PLAYWRIGHT_MAX_REQUESTS";
6
6
  export declare const INITIAL_PAGE = "CRAWLEE_PLAYWRIGHT_INITIAL_PAGE";
7
7
  export declare const HEADLESS = "CRAWLEE_PLAYWRIGHT_HEADLESS";
8
+ export declare const HANDLE_CLOUDFLARE_CHALLENGE = "CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE";
package/dist/modules/playwright/constant/playwright.env.constant.js CHANGED
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.HEADLESS = exports.INITIAL_PAGE = exports.MAX_REQUESTS = exports.TIMEOUT_SECS = exports.MAX_REQUEST_RETRIES = exports.MAX_CONCURRENCY = exports.MIN_CONCURRENCY = void 0;
3
+ exports.HANDLE_CLOUDFLARE_CHALLENGE = exports.HEADLESS = exports.INITIAL_PAGE = exports.MAX_REQUESTS = exports.TIMEOUT_SECS = exports.MAX_REQUEST_RETRIES = exports.MAX_CONCURRENCY = exports.MIN_CONCURRENCY = void 0;
4
4
  exports.MIN_CONCURRENCY = 'CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY';
5
5
  exports.MAX_CONCURRENCY = 'CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY';
6
6
  exports.MAX_REQUEST_RETRIES = 'CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES';
@@ -8,4 +8,5 @@ exports.TIMEOUT_SECS = 'CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS';
8
8
  exports.MAX_REQUESTS = 'CRAWLEE_PLAYWRIGHT_MAX_REQUESTS';
9
9
  exports.INITIAL_PAGE = 'CRAWLEE_PLAYWRIGHT_INITIAL_PAGE';
10
10
  exports.HEADLESS = 'CRAWLEE_PLAYWRIGHT_HEADLESS';
11
+ exports.HANDLE_CLOUDFLARE_CHALLENGE = 'CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE';
11
12
  //# sourceMappingURL=playwright.env.constant.js.map
package/dist/modules/playwright/constant/playwright.env.constant.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"playwright.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/playwright/constant/playwright.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,eAAe,GAAG,oCAAoC,CAAC;AACvD,QAAA,eAAe,GAAG,oCAAoC,CAAC;AACvD,QAAA,mBAAmB,GAAG,wCAAwC,CAAC;AAC/D,QAAA,YAAY,GAAG,iCAAiC,CAAC;AACjD,QAAA,YAAY,GAAG,iCAAiC,CAAC;AACjD,QAAA,YAAY,GAAG,iCAAiC,CAAC;AACjD,QAAA,QAAQ,GAAG,6BAA6B,CAAC"}
1
+ {"version":3,"file":"playwright.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/playwright/constant/playwright.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,eAAe,GAAG,oCAAoC,CAAC;AACvD,QAAA,eAAe,GAAG,oCAAoC,CAAC;AACvD,QAAA,mBAAmB,GAAG,wCAAwC,CAAC;AAC/D,QAAA,YAAY,GAAG,iCAAiC,CAAC;AACjD,QAAA,YAAY,GAAG,iCAAiC,CAAC;AACjD,QAAA,YAAY,GAAG,iCAAiC,CAAC;AACjD,QAAA,QAAQ,GAAG,6BAA6B,CAAC;AACzC,QAAA,2BAA2B,GACtC,gDAAgD,CAAC"}
package/dist/modules/playwright/env/playwright.env.config.d.ts CHANGED
@@ -1,9 +1,10 @@
1
1
  export declare const PlaywrightEnvConfig: () => {
2
- CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY: number;
3
2
  CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY: number;
3
+ CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY: number;
4
4
  CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES: number;
5
5
  CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS: number;
6
6
  CRAWLEE_PLAYWRIGHT_MAX_REQUESTS: number;
7
7
  CRAWLEE_PLAYWRIGHT_INITIAL_PAGE: number;
8
- CRAWLEE_PLAYWRIGHT_HEADLESS: boolean;
8
+ CRAWLEE_PLAYWRIGHT_HEADLESS: any;
9
+ CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE: any;
9
10
  };
package/dist/modules/playwright/env/playwright.env.config.js CHANGED
@@ -37,13 +37,14 @@ exports.PlaywrightEnvConfig = void 0;
37
37
  const dotenv = __importStar(require("dotenv"));
38
38
  dotenv.config();
39
39
  const PlaywrightEnvConfig = () => ({
40
- CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY: Number(process.env.CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY),
41
40
  CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY: Number(process.env.CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY),
41
+ CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY: Number(process.env.CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY),
42
42
  CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES: Number(process.env.CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES),
43
43
  CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS: Number(process.env.CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS),
44
44
  CRAWLEE_PLAYWRIGHT_MAX_REQUESTS: Number(process.env.CRAWLEE_PLAYWRIGHT_MAX_REQUESTS),
45
45
  CRAWLEE_PLAYWRIGHT_INITIAL_PAGE: Number(process.env.CRAWLEE_PLAYWRIGHT_INITIAL_PAGE),
46
- CRAWLEE_PLAYWRIGHT_HEADLESS: Boolean(process.env.CRAWLEE_PLAYWRIGHT_HEADLESS === 'true'),
46
+ CRAWLEE_PLAYWRIGHT_HEADLESS: JSON.parse(process.env.CRAWLEE_PLAYWRIGHT_HEADLESS),
47
+ CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE: JSON.parse(process.env.CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE),
47
48
  });
48
49
  exports.PlaywrightEnvConfig = PlaywrightEnvConfig;
49
50
  //# sourceMappingURL=playwright.env.config.js.map
package/dist/modules/playwright/env/playwright.env.config.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"playwright.env.config.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.config.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AAEjC,MAAM,CAAC,MAAM,EAAE,CAAC;AACT,MAAM,mBAAmB,GAAG,GAAG,EAAE,CAAC,CAAC;IACxC,kCAAkC,EAAE,MAAM,CACxC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAC/C;IACD,kCAAkC,EAAE,MAAM,CACxC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAC/C;IACD,sCAAsC,EAAE,MAAM,CAC5C,OAAO,CAAC,GAAG,CAAC,sCAAsC,CACnD;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,2BAA2B,EAAE,OAAO,CAClC,OAAO,CAAC,GAAG,CAAC,2BAA2B,KAAK,MAAM,CACnD;CACF,CAAC,CAAC;AAtBU,QAAA,mBAAmB,uBAsB7B"}
1
+ {"version":3,"file":"playwright.env.config.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.config.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AAEjC,MAAM,CAAC,MAAM,EAAE,CAAC;AACT,MAAM,mBAAmB,GAAG,GAAG,EAAE,CAAC,CAAC;IACxC,kCAAkC,EAAE,MAAM,CACxC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAC/C;IACD,kCAAkC,EAAE,MAAM,CACxC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAC/C;IACD,sCAAsC,EAAE,MAAM,CAC5C,OAAO,CAAC,GAAG,CAAC,sCAAsC,CACnD;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,2BAA2B,EAAE,IAAI,CAAC,KAAK,CACrC,OAAO,CAAC,GAAG,CAAC,2BAA2B,CACxC;IACD,8CAA8C,EAAE,IAAI,CAAC,KAAK,CACxD,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAC3D;CACF,CAAC,CAAC;AAzBU,QAAA,mBAAmB,uBAyB7B"}
package/dist/modules/playwright/env/playwright.env.validation.js CHANGED
@@ -43,5 +43,9 @@ exports.PlaywrightConfigValidationSchema = joi.object({
43
43
  CRAWLEE_PLAYWRIGHT_MAX_REQUESTS: joi.number().optional().default(100000),
44
44
  CRAWLEE_PLAYWRIGHT_INITIAL_PAGE: joi.number().optional().default(1),
45
45
  CRAWLEE_PLAYWRIGHT_HEADLESS: joi.boolean().optional().default(true),
46
+ CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE: joi
47
+ .boolean()
48
+ .optional()
49
+ .default(false),
46
50
  });
47
51
  //# sourceMappingURL=playwright.env.validation.js.map
package/dist/modules/playwright/env/playwright.env.validation.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"playwright.env.validation.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.validation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yCAA2B;AAEd,QAAA,gCAAgC,GAAG,GAAG,CAAC,MAAM,CAAC;IACzD,kCAAkC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,kCAAkC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,sCAAsC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC3E,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IACrE,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IACxE,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACnE,2BAA2B,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;CACpE,CAAC,CAAC"}
1
+ {"version":3,"file":"playwright.env.validation.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.validation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yCAA2B;AAEd,QAAA,gCAAgC,GAAG,GAAG,CAAC,MAAM,CAAC;IACzD,kCAAkC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,kCAAkC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,sCAAsC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC3E,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IACrE,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IACxE,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACnE,2BAA2B,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACnE,8CAA8C,EAAE,GAAG;SAChD,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,KAAK,CAAC;CAClB,CAAC,CAAC"}
package/dist/modules/playwright/playwright.service.js CHANGED
@@ -38,7 +38,7 @@ let PlaywrightService = PlaywrightService_1 = class PlaywrightService {
38
38
  this.headless = this.config.get(playwright_env_constant_6.HEADLESS);
39
39
  }
40
40
  create(options) {
41
- this.logger.log('🚀 Creating Playwright Crawler instance');
41
+ this.logger.log('🚀 Creating Playwright Crawler');
42
42
  const { preNavigationHooks = [], postNavigationHooks = [] } = options;
43
43
  const defaultPreNavHooks = this.hook.getPreNavigationHooks();
44
44
  const defaultPostNavHooks = this.hook.getPostNavigationHooks();
@@ -55,11 +55,11 @@ let PlaywrightService = PlaywrightService_1 = class PlaywrightService {
55
55
  launchOptions: {
56
56
  args: playwright_browser_constant_1.BROWSER_ARGS,
57
57
  },
58
- useIncognitoPages: true,
59
58
  },
60
59
  browserPoolOptions: {
61
60
  useFingerprints: true,
62
61
  retireBrowserAfterPageCount: 50,
62
+ retireInactiveBrowserAfterSecs: 30,
63
63
  },
64
64
  };
65
65
  const crawler = new playwright_1.PlaywrightCrawler({
@@ -68,7 +68,7 @@ let PlaywrightService = PlaywrightService_1 = class PlaywrightService {
68
68
  preNavigationHooks: [...defaultPreNavHooks, ...preNavigationHooks],
69
69
  postNavigationHooks: [...defaultPostNavHooks, ...postNavigationHooks],
70
70
  });
71
- this.logger.log('✅ Playwright Crawler instance created successfully');
71
+ this.logger.log('✅ Playwright Crawler created successfully');
72
72
  return crawler;
73
73
  }
74
74
  async getBrowser() {
package/dist/modules/playwright/playwright.service.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"playwright.service.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAE/C,oDAAwD;AAGxD,gFAA2E;AAC3E,sFAAiF;AAEjF,uDAAoD;AACpD,gFAAqE;AACrE,gFAAqE;AACrE,gFAAyE;AACzE,gFAAkE;AAClE,gFAAkE;AAClE,wFAAsE;AACtE,gFAA8D;AAGvD,IAAM,iBAAiB,yBAAvB,MAAM,iBAAiB;IAU5B,YACmB,MAAqB,EACrB,IAA2B,EAC3B,OAAiC;QAFjC,WAAM,GAAN,MAAM,CAAe;QACrB,SAAI,GAAJ,IAAI,CAAuB;QAC3B,YAAO,GAAP,OAAO,CAA0B;QAZnC,WAAM,GAAG,IAAI,eAAM,CAAC,mBAAiB,CAAC,IAAI,CAAC,CAAC;QAc3D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,yCAAe,CAAC,CAAC;QAC/D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,yCAAe,CAAC,CAAC;QAC/D,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,6CAAmB,CAAC,CAAC;QACtE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,sCAAY,CAAC,CAAC;QACzD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,sCAAY,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAU,kCAAQ,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,CAAC,OAAiC;QACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;QAC3D,MAAM,EAAE,kBAAkB,GAAG,EAAE,EAAE,mBAAmB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/D,MAAM,cAAc,GAA6B;YAC/C,mBAAmB,EAAE,IAAI,CAAC,WAAW;YACrC,yBAAyB,EAAE,IAAI,CAAC,WAAW;YAC3C,qBAAqB,EAAE,IAAI,CAAC,WAAW;YACvC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE;gBACb,aAAa,EAAE;oBACb,IAAI,EAAE,0CAAY;iBACnB;gBACD,iBAAiB,EAAE,IAAI;aACxB;YACD,kBAAkB,EAAE;gBAClB,eAAe,EAAE,IAAI;gBACrB,2BAA2B,EAAE,EAAE;aAChC;SACF,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,8BAAiB,CAAC;YACpC,GAAG,cAAc;YACjB,GAAG,OAAO;YACV,kBAAkB,EAAE,CAAC,GAAG,kBAAkB,EAAE,GAAG,kBAAkB,CAAC;YAClE,mBAAmB,EAAE,CAAC,GAAG,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;SACtE,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QACtE,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;IACnC,CAAC;CACF,CAAA;AA7DY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAYgB,sBAAa;QACf,+CAAqB;QAClB,qDAAwB;GAbzC,iBAAiB,CA6D7B"}
1
+ {"version":3,"file":"playwright.service.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAE/C,oDAAwD;AAGxD,gFAA2E;AAC3E,sFAAiF;AAEjF,uDAAoD;AACpD,gFAAqE;AACrE,gFAAqE;AACrE,gFAAyE;AACzE,gFAAkE;AAClE,gFAAkE;AAClE,wFAAsE;AACtE,gFAA8D;AAGvD,IAAM,iBAAiB,yBAAvB,MAAM,iBAAiB;IAU5B,YACmB,MAAqB,EACrB,IAA2B,EAC3B,OAAiC;QAFjC,WAAM,GAAN,MAAM,CAAe;QACrB,SAAI,GAAJ,IAAI,CAAuB;QAC3B,YAAO,GAAP,OAAO,CAA0B;QAZnC,WAAM,GAAG,IAAI,eAAM,CAAC,mBAAiB,CAAC,IAAI,CAAC,CAAC;QAc3D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,yCAAe,CAAC,CAAC;QAC/D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,yCAAe,CAAC,CAAC;QAC/D,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,6CAAmB,CAAC,CAAC;QACtE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,sCAAY,CAAC,CAAC;QACzD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,sCAAY,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAU,kCAAQ,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,CAAC,OAAiC;QACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAClD,MAAM,EAAE,kBAAkB,GAAG,EAAE,EAAE,mBAAmB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/D,MAAM,cAAc,GAA6B;YAC/C,mBAAmB,EAAE,IAAI,CAAC,WAAW;YACrC,yBAAyB,EAAE,IAAI,CAAC,WAAW;YAC3C,qBAAqB,EAAE,IAAI,CAAC,WAAW;YACvC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE;gBACb,aAAa,EAAE;oBACb,IAAI,EAAE,0CAAY;iBACnB;aACF;YACD,kBAAkB,EAAE;gBAClB,eAAe,EAAE,IAAI;gBACrB,2BAA2B,EAAE,EAAE;gBAC/B,8BAA8B,EAAE,EAAE;aACnC;SACF,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,8BAAiB,CAAC;YACpC,GAAG,cAAc;YACjB,GAAG,OAAO;YACV,kBAAkB,EAAE,CAAC,GAAG,kBAAkB,EAAE,GAAG,kBAAkB,CAAC;YAClE,mBAAmB,EAAE,CAAC,GAAG,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;SACtE,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;QAC7D,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;IACnC,CAAC;CACF,CAAA;AA7DY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAYgB,sBAAa;QACf,+CAAqB;QAClB,qDAAwB;GAbzC,iBAAiB,CA6D7B"}
package/dist/modules/playwright/services/playwright.hook.service.d.ts CHANGED
@@ -1,6 +1,10 @@
1
+ import { ConfigService } from '@nestjs/config';
1
2
  import { PlaywrightHook } from '@crawlee/playwright';
2
3
  export declare class PlaywrightHookService {
4
+ private readonly config;
3
5
  private readonly logger;
6
+ private readonly shouldHandleCloudflareChallenge;
7
+ constructor(config: ConfigService);
4
8
  getPreNavigationHooks(): PlaywrightHook[];
5
9
  getPostNavigationHooks(): PlaywrightHook[];
6
10
  }
package/dist/modules/playwright/services/playwright.hook.service.js CHANGED
@@ -5,14 +5,22 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
5
5
  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
6
  return c > 3 && r && Object.defineProperty(target, key, r), r;
7
7
  };
8
+ var __metadata = (this && this.__metadata) || function (k, v) {
9
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
+ };
8
11
  Object.defineProperty(exports, "__esModule", { value: true });
9
12
  exports.PlaywrightHookService = void 0;
10
13
  const common_1 = require("@nestjs/common");
14
+ const config_1 = require("@nestjs/config");
11
15
  const logger_1 = require("../../../common/logger/logger");
12
16
  const playwright_service_1 = require("../playwright.service");
17
+ const playwright_env_constant_1 = require("../constant/playwright.env.constant");
13
18
  let PlaywrightHookService = class PlaywrightHookService {
14
- constructor() {
19
+ constructor(config) {
20
+ this.config = config;
15
21
  this.logger = new logger_1.Logger(playwright_service_1.PlaywrightService.name);
22
+ const enabled = this.config.get(playwright_env_constant_1.HANDLE_CLOUDFLARE_CHALLENGE, false);
23
+ this.shouldHandleCloudflareChallenge = enabled;
16
24
  }
17
25
  getPreNavigationHooks() {
18
26
  return [
@@ -24,7 +32,9 @@ let PlaywrightHookService = class PlaywrightHookService {
24
32
  getPostNavigationHooks() {
25
33
  return [
26
34
  async ({ handleCloudflareChallenge }) => {
27
- await handleCloudflareChallenge();
35
+ if (this.shouldHandleCloudflareChallenge) {
36
+ await handleCloudflareChallenge();
37
+ }
28
38
  },
29
39
  ({ request, response }) => {
30
40
  const { url } = request;
@@ -34,13 +44,13 @@ let PlaywrightHookService = class PlaywrightHookService {
34
44
  this.logger.log(`✅ ${statusCode} ${url}`);
35
45
  break;
36
46
  case common_1.HttpStatus.BAD_REQUEST:
37
- this.logger.warn(`⚠️ ${statusCode} ${url}`);
47
+ this.logger.warn(`⚠️ ${statusCode} ${url}`);
38
48
  break;
39
49
  case common_1.HttpStatus.UNAUTHORIZED:
40
- this.logger.warn(`⚠️ ${statusCode} ${url}`);
50
+ this.logger.warn(`⚠️ ${statusCode} ${url}`);
41
51
  break;
42
52
  case common_1.HttpStatus.FORBIDDEN:
43
- this.logger.warn(`⚠️ ${statusCode} ${url}`);
53
+ this.logger.warn(`⚠️ ${statusCode} ${url}`);
44
54
  break;
45
55
  case common_1.HttpStatus.TOO_MANY_REQUESTS:
46
56
  this.logger.warn(`⛔ ${statusCode} ${url}`);
@@ -64,6 +74,7 @@ let PlaywrightHookService = class PlaywrightHookService {
64
74
  };
65
75
  exports.PlaywrightHookService = PlaywrightHookService;
66
76
  exports.PlaywrightHookService = PlaywrightHookService = __decorate([
67
- (0, common_1.Injectable)()
77
+ (0, common_1.Injectable)(),
78
+ __metadata("design:paramtypes", [config_1.ConfigService])
68
79
  ], PlaywrightHookService);
69
80
  //# sourceMappingURL=playwright.hook.service.js.map
package/dist/modules/playwright/services/playwright.hook.service.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"playwright.hook.service.js","sourceRoot":"","sources":["../../../../src/modules/playwright/services/playwright.hook.service.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwD;AAGxD,0DAAuD;AACvD,8DAA0D;AAGnD,IAAM,qBAAqB,GAA3B,MAAM,qBAAqB;IAA3B;QACY,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAmD/D,CAAC;IAjDC,qBAAqB;QACnB,OAAO;YACL,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACvC,CAAC;SACF,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,OAAO;YACL,KAAK,EAAE,EAAE,yBAAyB,EAAE,EAAE,EAAE;gBACtC,MAAM,yBAAyB,EAAE,CAAC;YACpC,CAAC;YACD,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE;gBACxB,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;gBACxB,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAErC,QAAQ,UAAwB,EAAE,CAAC;oBACjC,KAAK,mBAAU,CAAC,EAAE;wBAChB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC1C,MAAM;oBACR,KAAK,mBAAU,CAAC,WAAW;wBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC5C,MAAM;oBACR,KAAK,mBAAU,CAAC,YAAY;wBAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC5C,MAAM;oBACR,KAAK,mBAAU,CAAC,SAAS;wBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC5C,MAAM;oBACR,KAAK,mBAAU,CAAC,iBAAiB;wBAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC3C,MAAM;oBACR,KAAK,mBAAU,CAAC,qBAAqB;wBACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,WAAW;wBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,mBAAmB;wBACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR;wBACE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC3C,MAAM;gBACV,CAAC;YACH,CAAC;SACF,CAAC;IACJ,CAAC;CACF,CAAA;AApDY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,mBAAU,GAAE;GACA,qBAAqB,CAoDjC"}
1
+ {"version":3,"file":"playwright.hook.service.js","sourceRoot":"","sources":["../../../../src/modules/playwright/services/playwright.hook.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAAwD;AACxD,2CAA+C;AAG/C,0DAAuD;AACvD,8DAA0D;AAC1D,iFAAgG;AAGzF,IAAM,qBAAqB,GAA3B,MAAM,qBAAqB;IAIhC,YAA6B,MAAqB;QAArB,WAAM,GAAN,MAAM,CAAe;QAHjC,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;QAI3D,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAU,qDAAU,EAAE,KAAK,CAAC,CAAC;QAC5D,IAAI,CAAC,+BAA+B,GAAG,OAAO,CAAC;IACjD,CAAC;IAED,qBAAqB;QACnB,OAAO;YACL,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACvC,CAAC;SACF,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,OAAO;YACL,KAAK,EAAE,EAAE,yBAAyB,EAAE,EAAE,EAAE;gBACtC,IAAI,IAAI,CAAC,+BAA+B,EAAE,CAAC;oBACzC,MAAM,yBAAyB,EAAE,CAAC;gBACpC,CAAC;YACH,CAAC;YACD,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE;gBACxB,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;gBACxB,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAErC,QAAQ,UAAwB,EAAE,CAAC;oBACjC,KAAK,mBAAU,CAAC,EAAE;wBAChB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC1C,MAAM;oBACR,KAAK,mBAAU,CAAC,WAAW;wBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,YAAY;wBAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,SAAS;wBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,iBAAiB;wBAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC3C,MAAM;oBACR,KAAK,mBAAU,CAAC,qBAAqB;wBACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,WAAW;wBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,mBAAmB;wBACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR;wBACE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC3C,MAAM;gBACV,CAAC;YACH,CAAC;SACF,CAAC;IACJ,CAAC;CACF,CAAA;AA5DY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,mBAAU,GAAE;qCAK0B,sBAAa;GAJvC,qBAAqB,CA4DjC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@rafikidota/scoutee",
3
- "version": "0.6.0",
3
+ "version": "0.8.0",
4
4
  "description": "Sometimes, the best way to solve your own problems is to help someone else.",
5
5
  "repository": {
6
6
  "type": "git",
@@ -15,32 +15,32 @@
15
15
  "module": "./dist/index.js",
16
16
  "types": "./dist/index.d.ts",
17
17
  "devDependencies": {
18
- "@crawlee/playwright": "3.15.3",
18
+ "@crawlee/playwright": "3.16.0",
19
19
  "@eslint/eslintrc": "3.3.3",
20
20
  "@eslint/js": "9.39.1",
21
- "@nestjs/common": "11.1.9",
22
- "@nestjs/config": "4.0.2",
23
- "@nestjs/core": "11.1.9",
24
- "@types/node": "24.10.2",
25
- "@typescript-eslint/eslint-plugin": "8.49.0",
26
- "@typescript-eslint/parser": "8.49.0",
27
- "camoufox-js": "0.8.4",
28
- "crawlee": "3.15.3",
29
- "dotenv": "17.2.3",
21
+ "@nestjs/common": "11.1.13",
22
+ "@nestjs/config": "4.0.3",
23
+ "@nestjs/core": "11.1.13",
24
+ "@types/node": "25.2.3",
25
+ "@typescript-eslint/eslint-plugin": "8.55.0",
26
+ "@typescript-eslint/parser": "8.55.0",
27
+ "camoufox-js": "0.8.5",
28
+ "crawlee": "3.16.0",
29
+ "dotenv": "17.3.1",
30
30
  "eslint": "9.39.1",
31
31
  "eslint-config-prettier": "10.1.8",
32
32
  "eslint-plugin-import": "2.32.0",
33
- "eslint-plugin-prettier": "5.5.4",
34
- "globals": "16.5.0",
33
+ "eslint-plugin-prettier": "5.5.5",
34
+ "globals": "17.3.0",
35
35
  "husky": "9.1.7",
36
36
  "joi": "18.0.2",
37
37
  "lint-staged": "16.2.7",
38
- "playwright": "1.57.0",
39
- "prettier": "3.7.4",
38
+ "playwright": "1.58.2",
39
+ "prettier": "3.8.1",
40
40
  "rimraf": "6.1.2",
41
41
  "rxjs": "7.8.2",
42
42
  "typescript": "5.9.3",
43
- "typescript-eslint": "8.49.0"
43
+ "typescript-eslint": "8.55.0"
44
44
  },
45
45
  "keywords": [
46
46
  "nestjs",