@rafikidota/scoutee 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/exports/camoufox.d.ts +5 -0
- package/dist/exports/camoufox.js +22 -0
- package/dist/exports/camoufox.js.map +1 -0
- package/dist/exports/cheerio.d.ts +5 -0
- package/dist/exports/cheerio.js +22 -0
- package/dist/exports/cheerio.js.map +1 -0
- package/dist/exports/http.d.ts +5 -0
- package/dist/exports/http.js +22 -0
- package/dist/exports/http.js.map +1 -0
- package/dist/exports/playwright.d.ts +5 -0
- package/dist/exports/playwright.js +22 -0
- package/dist/exports/playwright.js.map +1 -0
- package/dist/modules/camoufox/camoufox.module.js +7 -1
- package/dist/modules/camoufox/camoufox.module.js.map +1 -1
- package/dist/modules/camoufox/camoufox.service.d.ts +2 -9
- package/dist/modules/camoufox/camoufox.service.js +11 -23
- package/dist/modules/camoufox/camoufox.service.js.map +1 -1
- package/dist/modules/camoufox/constant/camoufox.env.constant.d.ts +12 -10
- package/dist/modules/camoufox/constant/camoufox.env.constant.js +13 -11
- package/dist/modules/camoufox/constant/camoufox.env.constant.js.map +1 -1
- package/dist/modules/camoufox/services/camoufox.browser.service.d.ts +2 -4
- package/dist/modules/camoufox/services/camoufox.browser.service.js +5 -8
- package/dist/modules/camoufox/services/camoufox.browser.service.js.map +1 -1
- package/dist/modules/camoufox/services/camoufox.config.service.d.ts +18 -0
- package/dist/modules/camoufox/services/camoufox.config.service.js +71 -0
- package/dist/modules/camoufox/services/camoufox.config.service.js.map +1 -0
- package/dist/modules/camoufox/services/camoufox.hook.service.d.ts +2 -3
- package/dist/modules/camoufox/services/camoufox.hook.service.js +3 -6
- package/dist/modules/camoufox/services/camoufox.hook.service.js.map +1 -1
- package/dist/modules/camoufox/types/camoufox.config.d.ts +12 -0
- package/dist/modules/camoufox/types/camoufox.config.js +3 -0
- package/dist/modules/camoufox/types/camoufox.config.js.map +1 -0
- package/dist/modules/cheerio/cheerio.module.js +2 -1
- package/dist/modules/cheerio/cheerio.module.js.map +1 -1
- package/dist/modules/cheerio/cheerio.service.d.ts +2 -7
- package/dist/modules/cheerio/cheerio.service.js +8 -17
- package/dist/modules/cheerio/cheerio.service.js.map +1 -1
- package/dist/modules/cheerio/constant/cheerio.env.constant.d.ts +8 -6
- package/dist/modules/cheerio/constant/cheerio.env.constant.js +9 -7
- package/dist/modules/cheerio/constant/cheerio.env.constant.js.map +1 -1
- package/dist/modules/cheerio/services/cheerio.config.service.d.ts +14 -0
- package/dist/modules/cheerio/services/cheerio.config.service.js +55 -0
- package/dist/modules/cheerio/services/cheerio.config.service.js.map +1 -0
- package/dist/modules/cheerio/types/cheerio.config.d.ts +8 -0
- package/dist/modules/cheerio/types/cheerio.config.js +3 -0
- package/dist/modules/cheerio/types/cheerio.config.js.map +1 -0
- package/dist/modules/http/constant/http.env.constant.d.ts +8 -6
- package/dist/modules/http/constant/http.env.constant.js +9 -7
- package/dist/modules/http/constant/http.env.constant.js.map +1 -1
- package/dist/modules/http/http.module.js +2 -1
- package/dist/modules/http/http.module.js.map +1 -1
- package/dist/modules/http/http.service.d.ts +2 -7
- package/dist/modules/http/http.service.js +9 -18
- package/dist/modules/http/http.service.js.map +1 -1
- package/dist/modules/http/services/http.config.service.d.ts +14 -0
- package/dist/modules/http/services/http.config.service.js +55 -0
- package/dist/modules/http/services/http.config.service.js.map +1 -0
- package/dist/modules/http/types/http.config.d.ts +8 -0
- package/dist/modules/http/types/http.config.js +3 -0
- package/dist/modules/http/types/http.config.js.map +1 -0
- package/dist/modules/playwright/constant/playwright.env.constant.d.ts +12 -10
- package/dist/modules/playwright/constant/playwright.env.constant.js +13 -11
- package/dist/modules/playwright/constant/playwright.env.constant.js.map +1 -1
- package/dist/modules/playwright/playwright.module.js +2 -0
- package/dist/modules/playwright/playwright.module.js.map +1 -1
- package/dist/modules/playwright/playwright.service.d.ts +2 -9
- package/dist/modules/playwright/playwright.service.js +11 -23
- package/dist/modules/playwright/playwright.service.js.map +1 -1
- package/dist/modules/playwright/services/playwright.browser.service.d.ts +2 -4
- package/dist/modules/playwright/services/playwright.browser.service.js +5 -8
- package/dist/modules/playwright/services/playwright.browser.service.js.map +1 -1
- package/dist/modules/playwright/services/playwright.config.service.d.ts +19 -0
- package/dist/modules/playwright/services/playwright.config.service.js +71 -0
- package/dist/modules/playwright/services/playwright.config.service.js.map +1 -0
- package/dist/modules/playwright/services/playwright.hook.service.d.ts +2 -3
- package/dist/modules/playwright/services/playwright.hook.service.js +3 -6
- package/dist/modules/playwright/services/playwright.hook.service.js.map +1 -1
- package/dist/modules/playwright/types/playwright.config.d.ts +13 -0
- package/dist/modules/playwright/types/playwright.config.js +3 -0
- package/dist/modules/playwright/types/playwright.config.js.map +1 -0
- package/package.json +23 -1
|
@@ -12,17 +12,11 @@ var PlaywrightService_1;
|
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
exports.PlaywrightService = void 0;
|
|
14
14
|
const common_1 = require("@nestjs/common");
|
|
15
|
-
const config_1 = require("@nestjs/config");
|
|
16
15
|
const playwright_1 = require("@crawlee/playwright");
|
|
17
16
|
const playwright_hook_service_1 = require("./services/playwright.hook.service");
|
|
18
17
|
const playwright_browser_service_1 = require("./services/playwright.browser.service");
|
|
18
|
+
const playwright_config_service_1 = require("./services/playwright.config.service");
|
|
19
19
|
const logger_1 = require("../../common/logger/logger");
|
|
20
|
-
const playwright_env_constant_1 = require("./constant/playwright.env.constant");
|
|
21
|
-
const playwright_env_constant_2 = require("./constant/playwright.env.constant");
|
|
22
|
-
const playwright_env_constant_3 = require("./constant/playwright.env.constant");
|
|
23
|
-
const playwright_env_constant_4 = require("./constant/playwright.env.constant");
|
|
24
|
-
const playwright_env_constant_5 = require("./constant/playwright.env.constant");
|
|
25
|
-
const playwright_env_constant_6 = require("./constant/playwright.env.constant");
|
|
26
20
|
const playwright_browser_constant_1 = require("./constant/playwright.browser.constant");
|
|
27
21
|
let PlaywrightService = PlaywrightService_1 = class PlaywrightService {
|
|
28
22
|
constructor(config, hook, browser) {
|
|
@@ -30,13 +24,6 @@ let PlaywrightService = PlaywrightService_1 = class PlaywrightService {
|
|
|
30
24
|
this.hook = hook;
|
|
31
25
|
this.browser = browser;
|
|
32
26
|
this.logger = new logger_1.Logger(PlaywrightService_1.name);
|
|
33
|
-
this.maxConcurrency = this.config.get(playwright_env_constant_1.MAX_CONCURRENCY);
|
|
34
|
-
this.minConcurrency = this.config.get(playwright_env_constant_2.MIN_CONCURRENCY);
|
|
35
|
-
this.maxRequestRetries = this.config.get(playwright_env_constant_3.MAX_REQUEST_RETRIES);
|
|
36
|
-
this.timeoutSecs = this.config.get(playwright_env_constant_4.TIMEOUT_SECS);
|
|
37
|
-
this.maxRequests = this.config.get(playwright_env_constant_5.MAX_REQUESTS);
|
|
38
|
-
this.headless = this.config.get(playwright_env_constant_6.HEADLESS);
|
|
39
|
-
this.useIncognitoPages = this.config.get(playwright_env_constant_6.INCOGNITO_PAGES);
|
|
40
27
|
}
|
|
41
28
|
create(options) {
|
|
42
29
|
this.logger.log('🚀 Creating Playwright Crawler');
|
|
@@ -44,21 +31,22 @@ let PlaywrightService = PlaywrightService_1 = class PlaywrightService {
|
|
|
44
31
|
const defaultPreNavHooks = this.hook.getPreNavigationHooks();
|
|
45
32
|
const defaultPostNavHooks = this.hook.getPostNavigationHooks();
|
|
46
33
|
const launcher = this.browser.getBrowserLauncher();
|
|
34
|
+
const { env } = this.config;
|
|
47
35
|
const defaultOptions = {
|
|
48
|
-
maxRequestsPerCrawl:
|
|
49
|
-
requestHandlerTimeoutSecs:
|
|
50
|
-
navigationTimeoutSecs:
|
|
51
|
-
maxConcurrency:
|
|
52
|
-
minConcurrency:
|
|
53
|
-
maxRequestRetries:
|
|
36
|
+
maxRequestsPerCrawl: env.maxRequests,
|
|
37
|
+
requestHandlerTimeoutSecs: env.timeoutSecs,
|
|
38
|
+
navigationTimeoutSecs: env.timeoutSecs,
|
|
39
|
+
maxConcurrency: env.maxConcurrency,
|
|
40
|
+
minConcurrency: env.minConcurrency,
|
|
41
|
+
maxRequestRetries: env.maxRequestRetries,
|
|
54
42
|
useSessionPool: true,
|
|
55
|
-
headless:
|
|
43
|
+
headless: env.headless,
|
|
56
44
|
launchContext: {
|
|
57
45
|
launcher,
|
|
58
46
|
launchOptions: {
|
|
59
47
|
args: playwright_browser_constant_1.BROWSER_ARGS,
|
|
60
48
|
},
|
|
61
|
-
useIncognitoPages:
|
|
49
|
+
useIncognitoPages: env.useIncognitoPages,
|
|
62
50
|
},
|
|
63
51
|
browserPoolOptions: {
|
|
64
52
|
useFingerprints: true,
|
|
@@ -83,7 +71,7 @@ let PlaywrightService = PlaywrightService_1 = class PlaywrightService {
|
|
|
83
71
|
exports.PlaywrightService = PlaywrightService;
|
|
84
72
|
exports.PlaywrightService = PlaywrightService = PlaywrightService_1 = __decorate([
|
|
85
73
|
(0, common_1.Injectable)(),
|
|
86
|
-
__metadata("design:paramtypes", [
|
|
74
|
+
__metadata("design:paramtypes", [playwright_config_service_1.PlaywrightConfigService,
|
|
87
75
|
playwright_hook_service_1.PlaywrightHookService,
|
|
88
76
|
playwright_browser_service_1.PlaywrightBrowserService])
|
|
89
77
|
], PlaywrightService);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.service.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;
|
|
1
|
+
{"version":3,"file":"playwright.service.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAE5C,oDAAwD;AAGxD,gFAA2E;AAC3E,sFAAiF;AACjF,oFAA+E;AAE/E,uDAAoD;AACpD,wFAAsE;AAG/D,IAAM,iBAAiB,yBAAvB,MAAM,iBAAiB;IAG5B,YACmB,MAA+B,EAC/B,IAA2B,EAC3B,OAAiC;QAFjC,WAAM,GAAN,MAAM,CAAyB;QAC/B,SAAI,GAAJ,IAAI,CAAuB;QAC3B,YAAO,GAAP,OAAO,CAA0B;QALnC,WAAM,GAAG,IAAI,eAAM,CAAC,mBAAiB,CAAC,IAAI,CAAC,CAAC;IAM1D,CAAC;IAEJ,MAAM,CAAC,OAAiC;QACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAClD,MAAM,EAAE,kBAAkB,GAAG,EAAE,EAAE,mBAAmB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACnD,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5B,MAAM,cAAc,GAA6B;YAC/C,mBAAmB,EAAE,GAAG,CAAC,WAAW;YACpC,yBAAyB,EAAE,GAAG,CAAC,WAAW;YAC1C,qBAAqB,EAAE,GAAG,CAAC,WAAW;YACtC,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;YACxC,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE;gBACb,QAAQ;gBACR,aAAa,EAAE;oBACb,IAAI,EAAE,0CAAY;iBACnB;gBACD,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;aACzC;YACD,kBAAkB,EAAE;gBAClB,eAAe,EAAE,IAAI;gBACrB,2BAA2B,EAAE,EAAE;gBAC/B,8BAA8B,EAAE,EAAE;aACnC;SACF,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,8BAAiB,CAAC;YACpC,GAAG,cAAc;YACjB,GAAG,OAAO;YACV,kBAAkB,EAAE,CAAC,GAAG,kBAAkB,EAAE,GAAG,kBAAkB,CAAC;YAClE,mBAAmB,EAAE,CAAC,GAAG,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;SACtE,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,0CAA0C,CAAC,CAAC;QACrE,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;IACnC,CAAC;CACF,CAAA;AApDY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAKgB,mDAAuB;QACzB,+CAAqB;QAClB,qDAAwB;GANzC,iBAAiB,CAoD7B"}
|
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
import { ConfigService } from '@nestjs/config';
|
|
2
1
|
import { Browser } from 'playwright';
|
|
3
2
|
import * as playwright from 'playwright';
|
|
4
3
|
import { BrowserName } from '../enum/playwright.browser-name.enum';
|
|
4
|
+
import { PlaywrightConfigService } from './playwright.config.service';
|
|
5
5
|
export declare class PlaywrightBrowserService {
|
|
6
6
|
private readonly config;
|
|
7
7
|
private readonly logger;
|
|
8
|
-
|
|
9
|
-
private readonly type;
|
|
10
|
-
constructor(config: ConfigService);
|
|
8
|
+
constructor(config: PlaywrightConfigService);
|
|
11
9
|
getBrowserLauncher(): playwright.BrowserType;
|
|
12
10
|
getBrowserName(): BrowserName;
|
|
13
11
|
getBrowser(): Promise<Browser>;
|
|
@@ -44,20 +44,17 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.PlaywrightBrowserService = void 0;
|
|
46
46
|
const common_1 = require("@nestjs/common");
|
|
47
|
-
const config_1 = require("@nestjs/config");
|
|
48
47
|
const playwright = __importStar(require("playwright"));
|
|
49
48
|
const playwright_service_1 = require("../playwright.service");
|
|
50
49
|
const playwright_browser_constant_1 = require("../constant/playwright.browser.constant");
|
|
51
|
-
const playwright_env_constant_1 = require("../constant/playwright.env.constant");
|
|
52
50
|
const playwright_browser_type_enum_1 = require("../enum/playwright.browser-type.enum");
|
|
53
51
|
const logger_1 = require("../../../common/logger/logger");
|
|
54
52
|
const playwright_browser_name_enum_1 = require("../enum/playwright.browser-name.enum");
|
|
53
|
+
const playwright_config_service_1 = require("./playwright.config.service");
|
|
55
54
|
let PlaywrightBrowserService = class PlaywrightBrowserService {
|
|
56
55
|
constructor(config) {
|
|
57
56
|
this.config = config;
|
|
58
57
|
this.logger = new logger_1.Logger(playwright_service_1.PlaywrightService.name);
|
|
59
|
-
this.headless = this.config.get(playwright_env_constant_1.HEADLESS);
|
|
60
|
-
this.type = this.config.get(playwright_env_constant_1.BROWSER);
|
|
61
58
|
}
|
|
62
59
|
getBrowserLauncher() {
|
|
63
60
|
const map = {
|
|
@@ -66,7 +63,7 @@ let PlaywrightBrowserService = class PlaywrightBrowserService {
|
|
|
66
63
|
[playwright_browser_type_enum_1.BrowserType.WEBKIT]: playwright.webkit,
|
|
67
64
|
[playwright_browser_type_enum_1.BrowserType.CAMOFOUX]: playwright.firefox,
|
|
68
65
|
};
|
|
69
|
-
return map[this.
|
|
66
|
+
return map[this.config.browser] ?? playwright.chromium;
|
|
70
67
|
}
|
|
71
68
|
getBrowserName() {
|
|
72
69
|
const map = {
|
|
@@ -75,13 +72,13 @@ let PlaywrightBrowserService = class PlaywrightBrowserService {
|
|
|
75
72
|
[playwright_browser_type_enum_1.BrowserType.WEBKIT]: playwright_browser_name_enum_1.BrowserName.WEBKIT,
|
|
76
73
|
[playwright_browser_type_enum_1.BrowserType.CAMOFOUX]: playwright_browser_name_enum_1.BrowserName.CAMOFOUX,
|
|
77
74
|
};
|
|
78
|
-
return map[this.
|
|
75
|
+
return map[this.config.browser] ?? playwright_browser_name_enum_1.BrowserName.CHROMIUM;
|
|
79
76
|
}
|
|
80
77
|
async getBrowser() {
|
|
81
78
|
this.logger.log('🚀 Creating direct Playwright browser instance');
|
|
82
79
|
const launcher = this.getBrowserLauncher();
|
|
83
80
|
const browser = await launcher.launch({
|
|
84
|
-
headless: this.headless,
|
|
81
|
+
headless: this.config.headless,
|
|
85
82
|
args: playwright_browser_constant_1.BROWSER_ARGS,
|
|
86
83
|
});
|
|
87
84
|
const name = this.getBrowserName();
|
|
@@ -92,6 +89,6 @@ let PlaywrightBrowserService = class PlaywrightBrowserService {
|
|
|
92
89
|
exports.PlaywrightBrowserService = PlaywrightBrowserService;
|
|
93
90
|
exports.PlaywrightBrowserService = PlaywrightBrowserService = __decorate([
|
|
94
91
|
(0, common_1.Injectable)(),
|
|
95
|
-
__metadata("design:paramtypes", [
|
|
92
|
+
__metadata("design:paramtypes", [playwright_config_service_1.PlaywrightConfigService])
|
|
96
93
|
], PlaywrightBrowserService);
|
|
97
94
|
//# sourceMappingURL=playwright.browser.service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.browser.service.js","sourceRoot":"","sources":["../../../../src/modules/playwright/services/playwright.browser.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA4C;
|
|
1
|
+
{"version":3,"file":"playwright.browser.service.js","sourceRoot":"","sources":["../../../../src/modules/playwright/services/playwright.browser.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA4C;AAG5C,uDAAyC;AAEzC,8DAA0D;AAC1D,yFAAuE;AACvE,uFAAmE;AACnE,0DAAuD;AACvD,uFAAmE;AACnE,2EAAsE;AAG/D,IAAM,wBAAwB,GAA9B,MAAM,wBAAwB;IAGnC,YAA6B,MAA+B;QAA/B,WAAM,GAAN,MAAM,CAAyB;QAF3C,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAEE,CAAC;IAEhE,kBAAkB;QAChB,MAAM,GAAG,GAAG;YACV,CAAC,0CAAW,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC,QAAQ;YAC3C,CAAC,0CAAW,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC,OAAO;YACzC,CAAC,0CAAW,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC,MAAM;YACvC,CAAC,0CAAW,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC,OAAO;SAC3C,CAAC;QAEF,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC;IACzD,CAAC;IAED,cAAc;QACZ,MAAM,GAAG,GAAG;YACV,CAAC,0CAAW,CAAC,QAAQ,CAAC,EAAE,0CAAW,CAAC,QAAQ;YAC5C,CAAC,0CAAW,CAAC,OAAO,CAAC,EAAE,0CAAW,CAAC,OAAO;YAC1C,CAAC,0CAAW,CAAC,MAAM,CAAC,EAAE,0CAAW,CAAC,MAAM;YACxC,CAAC,0CAAW,CAAC,QAAQ,CAAC,EAAE,0CAAW,CAAC,QAAQ;SAC7C,CAAC;QAEF,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,0CAAW,CAAC,QAAQ,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;YACpC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;YAC9B,IAAI,EAAE,0CAAY;SACnB,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,0CAA0C,CAAC,CAAC;QACrE,OAAO,OAAO,CAAC;IACjB,CAAC;CACF,CAAA;AAtCY,4DAAwB;mCAAxB,wBAAwB;IADpC,IAAA,mBAAU,GAAE;qCAI0B,mDAAuB;GAHjD,wBAAwB,CAsCpC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { ConfigService } from '@nestjs/config';
|
|
2
|
+
import { PlaywrightConfig } from '../types/playwright.config';
|
|
3
|
+
import { BrowserType } from '../enum/playwright.browser-type.enum';
|
|
4
|
+
export declare class PlaywrightConfigService {
|
|
5
|
+
private readonly service;
|
|
6
|
+
private readonly _env;
|
|
7
|
+
constructor(service: ConfigService);
|
|
8
|
+
get env(): PlaywrightConfig;
|
|
9
|
+
get maxRequests(): number;
|
|
10
|
+
get timeoutSecs(): number;
|
|
11
|
+
get maxConcurrency(): number;
|
|
12
|
+
get minConcurrency(): number;
|
|
13
|
+
get maxRequestRetries(): number;
|
|
14
|
+
get initialPage(): number;
|
|
15
|
+
get browser(): BrowserType;
|
|
16
|
+
get headless(): boolean;
|
|
17
|
+
get useIncognitoPages(): boolean;
|
|
18
|
+
get handleCloudflareChallenge(): boolean;
|
|
19
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.PlaywrightConfigService = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const config_1 = require("@nestjs/config");
|
|
15
|
+
const playwright_env_constant_1 = require("../constant/playwright.env.constant");
|
|
16
|
+
let PlaywrightConfigService = class PlaywrightConfigService {
|
|
17
|
+
constructor(service) {
|
|
18
|
+
this.service = service;
|
|
19
|
+
this._env = {
|
|
20
|
+
maxConcurrency: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.MAX_CONCURRENCY),
|
|
21
|
+
minConcurrency: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.MIN_CONCURRENCY),
|
|
22
|
+
maxRequestRetries: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.MAX_REQUEST_RETRIES),
|
|
23
|
+
timeoutSecs: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.TIMEOUT_SECS),
|
|
24
|
+
maxRequests: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.MAX_REQUESTS),
|
|
25
|
+
initialPage: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.INITIAL_PAGE),
|
|
26
|
+
browser: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.BROWSER),
|
|
27
|
+
headless: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.HEADLESS),
|
|
28
|
+
useIncognitoPages: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.INCOGNITO_PAGES),
|
|
29
|
+
handleCloudflareChallenge: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.HANDLE_CLOUDFLARE_CHALLENGE),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
get env() {
|
|
33
|
+
return this._env;
|
|
34
|
+
}
|
|
35
|
+
get maxRequests() {
|
|
36
|
+
return Number(this._env.maxRequests);
|
|
37
|
+
}
|
|
38
|
+
get timeoutSecs() {
|
|
39
|
+
return Number(this._env.timeoutSecs);
|
|
40
|
+
}
|
|
41
|
+
get maxConcurrency() {
|
|
42
|
+
return Number(this._env.maxConcurrency);
|
|
43
|
+
}
|
|
44
|
+
get minConcurrency() {
|
|
45
|
+
return Number(this._env.minConcurrency);
|
|
46
|
+
}
|
|
47
|
+
get maxRequestRetries() {
|
|
48
|
+
return Number(this._env.maxRequestRetries);
|
|
49
|
+
}
|
|
50
|
+
get initialPage() {
|
|
51
|
+
return Number(this._env.initialPage);
|
|
52
|
+
}
|
|
53
|
+
get browser() {
|
|
54
|
+
return this._env.browser;
|
|
55
|
+
}
|
|
56
|
+
get headless() {
|
|
57
|
+
return Boolean(this._env.headless);
|
|
58
|
+
}
|
|
59
|
+
get useIncognitoPages() {
|
|
60
|
+
return Boolean(this._env.useIncognitoPages);
|
|
61
|
+
}
|
|
62
|
+
get handleCloudflareChallenge() {
|
|
63
|
+
return Boolean(this._env.handleCloudflareChallenge);
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
exports.PlaywrightConfigService = PlaywrightConfigService;
|
|
67
|
+
exports.PlaywrightConfigService = PlaywrightConfigService = __decorate([
|
|
68
|
+
(0, common_1.Injectable)(),
|
|
69
|
+
__metadata("design:paramtypes", [config_1.ConfigService])
|
|
70
|
+
], PlaywrightConfigService);
|
|
71
|
+
//# sourceMappingURL=playwright.config.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright.config.service.js","sourceRoot":"","sources":["../../../../src/modules/playwright/services/playwright.config.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAG/C,iFAA4E;AAIrE,IAAM,uBAAuB,GAA7B,MAAM,uBAAuB;IAGlC,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;QACjD,IAAI,CAAC,IAAI,GAAG;YACV,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,eAAe,CAAC;YAC7D,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,eAAe,CAAC;YAC7D,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,mBAAmB,CAAC;YACpE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,YAAY,CAAC;YACvD,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,OAAO,CAAgB;YAC7D,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAU,wCAAG,CAAC,QAAQ,CAAC;YACjD,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAU,wCAAG,CAAC,eAAe,CAAC;YACjE,yBAAyB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CACzC,wCAAG,CAAC,2BAA2B,CAChC;SACF,CAAC;IACJ,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;IAC3B,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,yBAAyB;QAC3B,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACtD,CAAC;CACF,CAAA;AA/DY,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,mBAAU,GAAE;qCAI2B,sBAAa;GAHxC,uBAAuB,CA+DnC"}
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { ConfigService } from '@nestjs/config';
|
|
2
1
|
import { PlaywrightHook } from '@crawlee/playwright';
|
|
2
|
+
import { PlaywrightConfigService } from './playwright.config.service';
|
|
3
3
|
export declare class PlaywrightHookService {
|
|
4
4
|
private readonly config;
|
|
5
5
|
private readonly logger;
|
|
6
|
-
|
|
7
|
-
constructor(config: ConfigService);
|
|
6
|
+
constructor(config: PlaywrightConfigService);
|
|
8
7
|
getPreNavigationHooks(): PlaywrightHook[];
|
|
9
8
|
getPostNavigationHooks(): PlaywrightHook[];
|
|
10
9
|
}
|
|
@@ -11,16 +11,13 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.PlaywrightHookService = void 0;
|
|
13
13
|
const common_1 = require("@nestjs/common");
|
|
14
|
-
const config_1 = require("@nestjs/config");
|
|
15
14
|
const logger_1 = require("../../../common/logger/logger");
|
|
16
15
|
const playwright_service_1 = require("../playwright.service");
|
|
17
|
-
const
|
|
16
|
+
const playwright_config_service_1 = require("./playwright.config.service");
|
|
18
17
|
let PlaywrightHookService = class PlaywrightHookService {
|
|
19
18
|
constructor(config) {
|
|
20
19
|
this.config = config;
|
|
21
20
|
this.logger = new logger_1.Logger(playwright_service_1.PlaywrightService.name);
|
|
22
|
-
const enabled = this.config.get(playwright_env_constant_1.HANDLE_CLOUDFLARE_CHALLENGE, false);
|
|
23
|
-
this.shouldHandleCloudflareChallenge = enabled;
|
|
24
21
|
}
|
|
25
22
|
getPreNavigationHooks() {
|
|
26
23
|
return [
|
|
@@ -32,7 +29,7 @@ let PlaywrightHookService = class PlaywrightHookService {
|
|
|
32
29
|
getPostNavigationHooks() {
|
|
33
30
|
return [
|
|
34
31
|
async ({ handleCloudflareChallenge }) => {
|
|
35
|
-
if (this.
|
|
32
|
+
if (this.config.handleCloudflareChallenge) {
|
|
36
33
|
await handleCloudflareChallenge();
|
|
37
34
|
}
|
|
38
35
|
},
|
|
@@ -75,6 +72,6 @@ let PlaywrightHookService = class PlaywrightHookService {
|
|
|
75
72
|
exports.PlaywrightHookService = PlaywrightHookService;
|
|
76
73
|
exports.PlaywrightHookService = PlaywrightHookService = __decorate([
|
|
77
74
|
(0, common_1.Injectable)(),
|
|
78
|
-
__metadata("design:paramtypes", [
|
|
75
|
+
__metadata("design:paramtypes", [playwright_config_service_1.PlaywrightConfigService])
|
|
79
76
|
], PlaywrightHookService);
|
|
80
77
|
//# sourceMappingURL=playwright.hook.service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.hook.service.js","sourceRoot":"","sources":["../../../../src/modules/playwright/services/playwright.hook.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAAwD;
|
|
1
|
+
{"version":3,"file":"playwright.hook.service.js","sourceRoot":"","sources":["../../../../src/modules/playwright/services/playwright.hook.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAAwD;AAGxD,0DAAuD;AACvD,8DAA0D;AAC1D,2EAAsE;AAG/D,IAAM,qBAAqB,GAA3B,MAAM,qBAAqB;IAGhC,YAA6B,MAA+B;QAA/B,WAAM,GAAN,MAAM,CAAyB;QAF3C,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAEE,CAAC;IAEhE,qBAAqB;QACnB,OAAO;YACL,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACvC,CAAC;SACF,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,OAAO;YACL,KAAK,EAAE,EAAE,yBAAyB,EAAE,EAAE,EAAE;gBACtC,IAAI,IAAI,CAAC,MAAM,CAAC,yBAAyB,EAAE,CAAC;oBAC1C,MAAM,yBAAyB,EAAE,CAAC;gBACpC,CAAC;YACH,CAAC;YACD,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE;gBACxB,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;gBACxB,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAErC,QAAQ,UAAwB,EAAE,CAAC;oBACjC,KAAK,mBAAU,CAAC,EAAE;wBAChB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC1C,MAAM;oBACR,KAAK,mBAAU,CAAC,WAAW;wBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,YAAY;wBAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,SAAS;wBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,iBAAiB;wBAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC3C,MAAM;oBACR,KAAK,mBAAU,CAAC,qBAAqB;wBACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,WAAW;wBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,mBAAmB;wBACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR;wBACE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC3C,MAAM;gBACV,CAAC;YACH,CAAC;SACF,CAAC;IACJ,CAAC;CACF,CAAA;AAxDY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,mBAAU,GAAE;qCAI0B,mDAAuB;GAHjD,qBAAqB,CAwDjC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { BrowserType } from '../enum/playwright.browser-type.enum';
|
|
2
|
+
export type PlaywrightConfig = {
|
|
3
|
+
maxRequests: number;
|
|
4
|
+
timeoutSecs: number;
|
|
5
|
+
maxConcurrency: number;
|
|
6
|
+
minConcurrency: number;
|
|
7
|
+
maxRequestRetries: number;
|
|
8
|
+
initialPage: number;
|
|
9
|
+
browser: BrowserType;
|
|
10
|
+
headless: boolean;
|
|
11
|
+
useIncognitoPages: boolean;
|
|
12
|
+
handleCloudflareChallenge: boolean;
|
|
13
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright.config.js","sourceRoot":"","sources":["../../../../src/modules/playwright/types/playwright.config.ts"],"names":[],"mappings":""}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rafikidota/scoutee",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"description": "Sometimes, the best way to solve your own problems is to help someone else.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -14,6 +14,28 @@
|
|
|
14
14
|
"main": "./dist/index.js",
|
|
15
15
|
"module": "./dist/index.js",
|
|
16
16
|
"types": "./dist/index.d.ts",
|
|
17
|
+
"exports": {
|
|
18
|
+
".": {
|
|
19
|
+
"types": "./dist/index.d.ts",
|
|
20
|
+
"default": "./dist/index.js"
|
|
21
|
+
},
|
|
22
|
+
"./cheerio": {
|
|
23
|
+
"types": "./dist/exports/cheerio.d.ts",
|
|
24
|
+
"default": "./dist/exports/cheerio.js"
|
|
25
|
+
},
|
|
26
|
+
"./http": {
|
|
27
|
+
"types": "./dist/exports/http.d.ts",
|
|
28
|
+
"default": "./dist/exports/http.js"
|
|
29
|
+
},
|
|
30
|
+
"./playwright": {
|
|
31
|
+
"types": "./dist/exports/playwright.d.ts",
|
|
32
|
+
"default": "./dist/exports/playwright.js"
|
|
33
|
+
},
|
|
34
|
+
"./camoufox": {
|
|
35
|
+
"types": "./dist/exports/camoufox.d.ts",
|
|
36
|
+
"default": "./dist/exports/camoufox.js"
|
|
37
|
+
}
|
|
38
|
+
},
|
|
17
39
|
"devDependencies": {
|
|
18
40
|
"@crawlee/playwright": "3.16.0",
|
|
19
41
|
"@eslint/eslintrc": "3.3.3",
|