@rafikidota/scoutee 0.12.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/constant/boolean.constant.d.ts +3 -0
- package/dist/common/constant/boolean.constant.js +17 -0
- package/dist/common/constant/boolean.constant.js.map +1 -0
- package/dist/common/util/boolean.d.ts +1 -0
- package/dist/common/util/{boolean.validator.js → boolean.joi.js} +10 -22
- package/dist/common/util/boolean.joi.js.map +1 -0
- package/dist/common/util/boolean.js +22 -0
- package/dist/common/util/boolean.js.map +1 -0
- package/dist/common/util/boolean.zod.js +3 -14
- package/dist/common/util/boolean.zod.js.map +1 -1
- package/dist/exports/camoufox.d.ts +5 -0
- package/dist/exports/camoufox.js +22 -0
- package/dist/exports/camoufox.js.map +1 -0
- package/dist/exports/cheerio.d.ts +5 -0
- package/dist/exports/cheerio.js +22 -0
- package/dist/exports/cheerio.js.map +1 -0
- package/dist/exports/http.d.ts +5 -0
- package/dist/exports/http.js +22 -0
- package/dist/exports/http.js.map +1 -0
- package/dist/exports/playwright.d.ts +5 -0
- package/dist/exports/playwright.js +22 -0
- package/dist/exports/playwright.js.map +1 -0
- package/dist/modules/camoufox/camoufox.module.js +7 -1
- package/dist/modules/camoufox/camoufox.module.js.map +1 -1
- package/dist/modules/camoufox/camoufox.service.d.ts +2 -9
- package/dist/modules/camoufox/camoufox.service.js +11 -23
- package/dist/modules/camoufox/camoufox.service.js.map +1 -1
- package/dist/modules/camoufox/constant/camoufox.env.constant.d.ts +12 -10
- package/dist/modules/camoufox/constant/camoufox.env.constant.js +13 -11
- package/dist/modules/camoufox/constant/camoufox.env.constant.js.map +1 -1
- package/dist/modules/camoufox/env/camoufox.env.config.d.ts +9 -9
- package/dist/modules/camoufox/env/camoufox.env.config.js +9 -9
- package/dist/modules/camoufox/env/camoufox.env.config.js.map +1 -1
- package/dist/modules/camoufox/env/camoufox.env.validation.js +4 -4
- package/dist/modules/camoufox/env/camoufox.env.validation.js.map +1 -1
- package/dist/modules/camoufox/services/camoufox.browser.service.d.ts +2 -4
- package/dist/modules/camoufox/services/camoufox.browser.service.js +5 -8
- package/dist/modules/camoufox/services/camoufox.browser.service.js.map +1 -1
- package/dist/modules/camoufox/services/camoufox.config.service.d.ts +18 -0
- package/dist/modules/camoufox/services/camoufox.config.service.js +71 -0
- package/dist/modules/camoufox/services/camoufox.config.service.js.map +1 -0
- package/dist/modules/camoufox/services/camoufox.hook.service.d.ts +2 -3
- package/dist/modules/camoufox/services/camoufox.hook.service.js +3 -6
- package/dist/modules/camoufox/services/camoufox.hook.service.js.map +1 -1
- package/dist/modules/camoufox/types/camoufox.config.d.ts +12 -0
- package/dist/modules/camoufox/types/camoufox.config.js +3 -0
- package/dist/modules/camoufox/types/camoufox.config.js.map +1 -0
- package/dist/modules/cheerio/cheerio.module.js +2 -1
- package/dist/modules/cheerio/cheerio.module.js.map +1 -1
- package/dist/modules/cheerio/cheerio.service.d.ts +2 -7
- package/dist/modules/cheerio/cheerio.service.js +8 -17
- package/dist/modules/cheerio/cheerio.service.js.map +1 -1
- package/dist/modules/cheerio/constant/cheerio.env.constant.d.ts +8 -6
- package/dist/modules/cheerio/constant/cheerio.env.constant.js +9 -7
- package/dist/modules/cheerio/constant/cheerio.env.constant.js.map +1 -1
- package/dist/modules/cheerio/services/cheerio.config.service.d.ts +14 -0
- package/dist/modules/cheerio/services/cheerio.config.service.js +55 -0
- package/dist/modules/cheerio/services/cheerio.config.service.js.map +1 -0
- package/dist/modules/cheerio/types/cheerio.config.d.ts +8 -0
- package/dist/modules/cheerio/types/cheerio.config.js +3 -0
- package/dist/modules/cheerio/types/cheerio.config.js.map +1 -0
- package/dist/modules/http/constant/http.env.constant.d.ts +8 -6
- package/dist/modules/http/constant/http.env.constant.js +9 -7
- package/dist/modules/http/constant/http.env.constant.js.map +1 -1
- package/dist/modules/http/http.module.js +2 -1
- package/dist/modules/http/http.module.js.map +1 -1
- package/dist/modules/http/http.service.d.ts +2 -7
- package/dist/modules/http/http.service.js +9 -18
- package/dist/modules/http/http.service.js.map +1 -1
- package/dist/modules/http/services/http.config.service.d.ts +14 -0
- package/dist/modules/http/services/http.config.service.js +55 -0
- package/dist/modules/http/services/http.config.service.js.map +1 -0
- package/dist/modules/http/types/http.config.d.ts +8 -0
- package/dist/modules/http/types/http.config.js +3 -0
- package/dist/modules/http/types/http.config.js.map +1 -0
- package/dist/modules/playwright/constant/playwright.env.constant.d.ts +12 -10
- package/dist/modules/playwright/constant/playwright.env.constant.js +13 -11
- package/dist/modules/playwright/constant/playwright.env.constant.js.map +1 -1
- package/dist/modules/playwright/env/playwright.env.config.d.ts +9 -9
- package/dist/modules/playwright/env/playwright.env.config.js +9 -9
- package/dist/modules/playwright/env/playwright.env.config.js.map +1 -1
- package/dist/modules/playwright/env/playwright.env.validation.js +4 -4
- package/dist/modules/playwright/env/playwright.env.validation.js.map +1 -1
- package/dist/modules/playwright/playwright.module.js +2 -0
- package/dist/modules/playwright/playwright.module.js.map +1 -1
- package/dist/modules/playwright/playwright.service.d.ts +2 -9
- package/dist/modules/playwright/playwright.service.js +11 -23
- package/dist/modules/playwright/playwright.service.js.map +1 -1
- package/dist/modules/playwright/services/playwright.browser.service.d.ts +2 -4
- package/dist/modules/playwright/services/playwright.browser.service.js +5 -8
- package/dist/modules/playwright/services/playwright.browser.service.js.map +1 -1
- package/dist/modules/playwright/services/playwright.config.service.d.ts +19 -0
- package/dist/modules/playwright/services/playwright.config.service.js +71 -0
- package/dist/modules/playwright/services/playwright.config.service.js.map +1 -0
- package/dist/modules/playwright/services/playwright.hook.service.d.ts +2 -3
- package/dist/modules/playwright/services/playwright.hook.service.js +3 -6
- package/dist/modules/playwright/services/playwright.hook.service.js.map +1 -1
- package/dist/modules/playwright/types/playwright.config.d.ts +13 -0
- package/dist/modules/playwright/types/playwright.config.js +3 -0
- package/dist/modules/playwright/types/playwright.config.js.map +1 -0
- package/package.json +23 -1
- package/dist/common/util/boolean.validator.js.map +0 -1
- /package/dist/common/util/{boolean.validator.d.ts → boolean.joi.d.ts} +0 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.service.js","sourceRoot":"","sources":["../../../src/modules/http/http.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;
|
|
1
|
+
{"version":3,"file":"http.service.js","sourceRoot":"","sources":["../../../src/modules/http/http.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAE5C,qCAA0D;AAE1D,oEAA+D;AAC/D,wEAAmE;AACnE,uDAAoD;AAG7C,IAAM,WAAW,mBAAjB,MAAM,WAAW;IAGtB,YACmB,MAAyB,EACzB,IAAqB;QADrB,WAAM,GAAN,MAAM,CAAmB;QACzB,SAAI,GAAJ,IAAI,CAAiB;QAJvB,WAAM,GAAG,IAAI,eAAM,CAAC,aAAW,CAAC,IAAI,CAAC,CAAC;IAKpD,CAAC;IAEJ,MAAM,CAAC,OAA2B;QAChC,MAAM,EAAE,kBAAkB,GAAG,EAAE,EAAE,mBAAmB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/D,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5B,MAAM,cAAc,GAAuB;YACzC,mBAAmB,EAAE,GAAG,CAAC,WAAW;YACpC,yBAAyB,EAAE,GAAG,CAAC,WAAW;YAC1C,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;SACzC,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAClD,OAAO,IAAI,qBAAW,CAAC;YACrB,GAAG,cAAc;YACjB,GAAG,OAAO;YACV,kBAAkB,EAAE,CAAC,GAAG,kBAAkB,EAAE,GAAG,kBAAkB,CAAC;YAClE,mBAAmB,EAAE,CAAC,GAAG,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;SACtE,CAAC,CAAC;IACL,CAAC;CACF,CAAA;AA5BY,kCAAW;sBAAX,WAAW;IADvB,IAAA,mBAAU,GAAE;qCAKgB,uCAAiB;QACnB,mCAAe;GAL7B,WAAW,CA4BvB"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { ConfigService } from '@nestjs/config';
|
|
2
|
+
import { HttpConfig } from '../types/http.config';
|
|
3
|
+
export declare class HttpConfigService {
|
|
4
|
+
private readonly service;
|
|
5
|
+
private readonly _env;
|
|
6
|
+
constructor(service: ConfigService);
|
|
7
|
+
get env(): HttpConfig;
|
|
8
|
+
get maxRequests(): number;
|
|
9
|
+
get timeoutSecs(): number;
|
|
10
|
+
get maxConcurrency(): number;
|
|
11
|
+
get minConcurrency(): number;
|
|
12
|
+
get maxRequestRetries(): number;
|
|
13
|
+
get initialPage(): number;
|
|
14
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.HttpConfigService = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const config_1 = require("@nestjs/config");
|
|
15
|
+
const http_env_constant_1 = require("../constant/http.env.constant");
|
|
16
|
+
let HttpConfigService = class HttpConfigService {
|
|
17
|
+
constructor(service) {
|
|
18
|
+
this.service = service;
|
|
19
|
+
this._env = {
|
|
20
|
+
maxConcurrency: this.service.get(http_env_constant_1.HTTP_ENV.MAX_CONCURRENCY),
|
|
21
|
+
minConcurrency: this.service.get(http_env_constant_1.HTTP_ENV.MIN_CONCURRENCY),
|
|
22
|
+
maxRequestRetries: this.service.get(http_env_constant_1.HTTP_ENV.MAX_REQUEST_RETRIES),
|
|
23
|
+
timeoutSecs: this.service.get(http_env_constant_1.HTTP_ENV.TIMEOUT_SECS),
|
|
24
|
+
maxRequests: this.service.get(http_env_constant_1.HTTP_ENV.MAX_REQUESTS),
|
|
25
|
+
initialPage: this.service.get(http_env_constant_1.HTTP_ENV.INITIAL_PAGE),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
get env() {
|
|
29
|
+
return this._env;
|
|
30
|
+
}
|
|
31
|
+
get maxRequests() {
|
|
32
|
+
return Number(this._env.maxRequests);
|
|
33
|
+
}
|
|
34
|
+
get timeoutSecs() {
|
|
35
|
+
return Number(this._env.timeoutSecs);
|
|
36
|
+
}
|
|
37
|
+
get maxConcurrency() {
|
|
38
|
+
return Number(this._env.maxConcurrency);
|
|
39
|
+
}
|
|
40
|
+
get minConcurrency() {
|
|
41
|
+
return Number(this._env.minConcurrency);
|
|
42
|
+
}
|
|
43
|
+
get maxRequestRetries() {
|
|
44
|
+
return Number(this._env.maxRequestRetries);
|
|
45
|
+
}
|
|
46
|
+
get initialPage() {
|
|
47
|
+
return Number(this._env.initialPage);
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
exports.HttpConfigService = HttpConfigService;
|
|
51
|
+
exports.HttpConfigService = HttpConfigService = __decorate([
|
|
52
|
+
(0, common_1.Injectable)(),
|
|
53
|
+
__metadata("design:paramtypes", [config_1.ConfigService])
|
|
54
|
+
], HttpConfigService);
|
|
55
|
+
//# sourceMappingURL=http.config.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.config.service.js","sourceRoot":"","sources":["../../../../src/modules/http/services/http.config.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAG/C,qEAAgE;AAGzD,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAG5B,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;QACjD,IAAI,CAAC,IAAI,GAAG;YACV,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,eAAe,CAAC;YAC7D,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,eAAe,CAAC;YAC7D,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,mBAAmB,CAAC;YACpE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,YAAY,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;CACF,CAAA;AAzCY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAI2B,sBAAa;GAHxC,iBAAiB,CAyC7B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.config.js","sourceRoot":"","sources":["../../../../src/modules/http/types/http.config.ts"],"names":[],"mappings":""}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
export declare const
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
1
|
+
export declare const PLAYWRIGHT_ENV: {
|
|
2
|
+
readonly BROWSER: "CRAWLEE_PLAYWRIGHT_BROWSER";
|
|
3
|
+
readonly MIN_CONCURRENCY: "CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY";
|
|
4
|
+
readonly MAX_CONCURRENCY: "CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY";
|
|
5
|
+
readonly MAX_REQUEST_RETRIES: "CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES";
|
|
6
|
+
readonly TIMEOUT_SECS: "CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS";
|
|
7
|
+
readonly MAX_REQUESTS: "CRAWLEE_PLAYWRIGHT_MAX_REQUESTS";
|
|
8
|
+
readonly INITIAL_PAGE: "CRAWLEE_PLAYWRIGHT_INITIAL_PAGE";
|
|
9
|
+
readonly HEADLESS: "CRAWLEE_PLAYWRIGHT_HEADLESS";
|
|
10
|
+
readonly INCOGNITO_PAGES: "CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES";
|
|
11
|
+
readonly HANDLE_CLOUDFLARE_CHALLENGE: "CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE";
|
|
12
|
+
};
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
exports.
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
3
|
+
exports.PLAYWRIGHT_ENV = void 0;
|
|
4
|
+
exports.PLAYWRIGHT_ENV = {
|
|
5
|
+
BROWSER: 'CRAWLEE_PLAYWRIGHT_BROWSER',
|
|
6
|
+
MIN_CONCURRENCY: 'CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY',
|
|
7
|
+
MAX_CONCURRENCY: 'CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY',
|
|
8
|
+
MAX_REQUEST_RETRIES: 'CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES',
|
|
9
|
+
TIMEOUT_SECS: 'CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS',
|
|
10
|
+
MAX_REQUESTS: 'CRAWLEE_PLAYWRIGHT_MAX_REQUESTS',
|
|
11
|
+
INITIAL_PAGE: 'CRAWLEE_PLAYWRIGHT_INITIAL_PAGE',
|
|
12
|
+
HEADLESS: 'CRAWLEE_PLAYWRIGHT_HEADLESS',
|
|
13
|
+
INCOGNITO_PAGES: 'CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES',
|
|
14
|
+
HANDLE_CLOUDFLARE_CHALLENGE: 'CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE',
|
|
15
|
+
};
|
|
14
16
|
//# sourceMappingURL=playwright.env.constant.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/playwright/constant/playwright.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,
|
|
1
|
+
{"version":3,"file":"playwright.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/playwright/constant/playwright.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,cAAc,GAAG;IAC5B,OAAO,EAAE,4BAA4B;IACrC,eAAe,EAAE,oCAAoC;IACrD,eAAe,EAAE,oCAAoC;IACrD,mBAAmB,EAAE,wCAAwC;IAC7D,YAAY,EAAE,iCAAiC;IAC/C,YAAY,EAAE,iCAAiC;IAC/C,YAAY,EAAE,iCAAiC;IAC/C,QAAQ,EAAE,6BAA6B;IACvC,eAAe,EAAE,wCAAwC;IACzD,2BAA2B,EAAE,gDAAgD;CACrE,CAAC"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
export declare const PlaywrightEnvConfig: () => {
|
|
2
2
|
CRAWLEE_PLAYWRIGHT_BROWSER: string;
|
|
3
|
-
CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY:
|
|
4
|
-
CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY:
|
|
5
|
-
CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES:
|
|
6
|
-
CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS:
|
|
7
|
-
CRAWLEE_PLAYWRIGHT_MAX_REQUESTS:
|
|
8
|
-
CRAWLEE_PLAYWRIGHT_INITIAL_PAGE:
|
|
9
|
-
CRAWLEE_PLAYWRIGHT_HEADLESS:
|
|
10
|
-
CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE:
|
|
11
|
-
CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES:
|
|
3
|
+
CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY: number;
|
|
4
|
+
CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY: number;
|
|
5
|
+
CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES: number;
|
|
6
|
+
CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS: number;
|
|
7
|
+
CRAWLEE_PLAYWRIGHT_MAX_REQUESTS: number;
|
|
8
|
+
CRAWLEE_PLAYWRIGHT_INITIAL_PAGE: number;
|
|
9
|
+
CRAWLEE_PLAYWRIGHT_HEADLESS: any;
|
|
10
|
+
CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE: any;
|
|
11
|
+
CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES: any;
|
|
12
12
|
};
|
|
@@ -38,15 +38,15 @@ const dotenv = __importStar(require("dotenv"));
|
|
|
38
38
|
dotenv.config();
|
|
39
39
|
const PlaywrightEnvConfig = () => ({
|
|
40
40
|
CRAWLEE_PLAYWRIGHT_BROWSER: process.env.CRAWLEE_PLAYWRIGHT_BROWSER,
|
|
41
|
-
CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY: process.env.CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY,
|
|
42
|
-
CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY: process.env.CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY,
|
|
43
|
-
CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES: process.env.CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES,
|
|
44
|
-
CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS: process.env.CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS,
|
|
45
|
-
CRAWLEE_PLAYWRIGHT_MAX_REQUESTS: process.env.CRAWLEE_PLAYWRIGHT_MAX_REQUESTS,
|
|
46
|
-
CRAWLEE_PLAYWRIGHT_INITIAL_PAGE: process.env.CRAWLEE_PLAYWRIGHT_INITIAL_PAGE,
|
|
47
|
-
CRAWLEE_PLAYWRIGHT_HEADLESS: process.env.CRAWLEE_PLAYWRIGHT_HEADLESS,
|
|
48
|
-
CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE: process.env.CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE,
|
|
49
|
-
CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES: process.env.CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES,
|
|
41
|
+
CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY: Number(process.env.CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY),
|
|
42
|
+
CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY: Number(process.env.CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY),
|
|
43
|
+
CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES: Number(process.env.CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES),
|
|
44
|
+
CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS: Number(process.env.CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS),
|
|
45
|
+
CRAWLEE_PLAYWRIGHT_MAX_REQUESTS: Number(process.env.CRAWLEE_PLAYWRIGHT_MAX_REQUESTS),
|
|
46
|
+
CRAWLEE_PLAYWRIGHT_INITIAL_PAGE: Number(process.env.CRAWLEE_PLAYWRIGHT_INITIAL_PAGE),
|
|
47
|
+
CRAWLEE_PLAYWRIGHT_HEADLESS: JSON.parse(process.env.CRAWLEE_PLAYWRIGHT_HEADLESS),
|
|
48
|
+
CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE: JSON.parse(process.env.CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE),
|
|
49
|
+
CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES: JSON.parse(process.env.CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES),
|
|
50
50
|
});
|
|
51
51
|
exports.PlaywrightEnvConfig = PlaywrightEnvConfig;
|
|
52
52
|
//# sourceMappingURL=playwright.env.config.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.env.config.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.config.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AAEjC,MAAM,CAAC,MAAM,EAAE,CAAC;AACT,MAAM,mBAAmB,GAAG,GAAG,EAAE,CAAC,CAAC;IACxC,0BAA0B,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B;IAClE,kCAAkC,
|
|
1
|
+
{"version":3,"file":"playwright.env.config.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.config.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AAEjC,MAAM,CAAC,MAAM,EAAE,CAAC;AACT,MAAM,mBAAmB,GAAG,GAAG,EAAE,CAAC,CAAC;IACxC,0BAA0B,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B;IAClE,kCAAkC,EAAE,MAAM,CACxC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAC/C;IACD,kCAAkC,EAAE,MAAM,CACxC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAC/C;IACD,sCAAsC,EAAE,MAAM,CAC5C,OAAO,CAAC,GAAG,CAAC,sCAAsC,CACnD;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IAED,2BAA2B,EAAE,IAAI,CAAC,KAAK,CACrC,OAAO,CAAC,GAAG,CAAC,2BAA2B,CACxC;IACD,8CAA8C,EAAE,IAAI,CAAC,KAAK,CACxD,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAC3D;IACD,sCAAsC,EAAE,IAAI,CAAC,KAAK,CAChD,OAAO,CAAC,GAAG,CAAC,sCAAsC,CACnD;CACF,CAAC,CAAC;AA9BU,QAAA,mBAAmB,uBA8B7B"}
|
|
@@ -35,7 +35,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.PlaywrightConfigValidationSchema = void 0;
|
|
37
37
|
const joi = __importStar(require("joi"));
|
|
38
|
-
const
|
|
38
|
+
const boolean_joi_1 = require("../../../common/util/boolean.joi");
|
|
39
39
|
const playwright_browser_type_enum_1 = require("../enum/playwright.browser-type.enum");
|
|
40
40
|
exports.PlaywrightConfigValidationSchema = joi.object({
|
|
41
41
|
CRAWLEE_PLAYWRIGHT_BROWSER: joi
|
|
@@ -49,10 +49,10 @@ exports.PlaywrightConfigValidationSchema = joi.object({
|
|
|
49
49
|
CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS: joi.number().optional().default(180),
|
|
50
50
|
CRAWLEE_PLAYWRIGHT_MAX_REQUESTS: joi.number().optional().default(100000),
|
|
51
51
|
CRAWLEE_PLAYWRIGHT_INITIAL_PAGE: joi.number().optional().default(1),
|
|
52
|
-
CRAWLEE_PLAYWRIGHT_HEADLESS:
|
|
53
|
-
CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE:
|
|
52
|
+
CRAWLEE_PLAYWRIGHT_HEADLESS: boolean_joi_1.bool.optional().default(true),
|
|
53
|
+
CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE: boolean_joi_1.bool
|
|
54
54
|
.optional()
|
|
55
55
|
.default(false),
|
|
56
|
-
CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES:
|
|
56
|
+
CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES: boolean_joi_1.bool.optional().default(true),
|
|
57
57
|
});
|
|
58
58
|
//# sourceMappingURL=playwright.env.validation.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.env.validation.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.validation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yCAA2B;AAC3B,
|
|
1
|
+
{"version":3,"file":"playwright.env.validation.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.validation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yCAA2B;AAC3B,kEAAwD;AAExD,uFAA8E;AAEjE,QAAA,gCAAgC,GAAG,GAAG,CAAC,MAAM,CAAC;IACzD,0BAA0B,EAAE,GAAG;SAC5B,MAAM,EAAE;SACR,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,0CAAO,CAAC,CAAC;SAChC,QAAQ,EAAE;SACV,OAAO,CAAC,0CAAO,CAAC,QAAQ,CAAC;IAC5B,kCAAkC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,kCAAkC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,sCAAsC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC3E,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IACrE,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IACxE,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACnE,2BAA2B,EAAE,kBAAI,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAC1D,8CAA8C,EAAE,kBAAI;SACjD,QAAQ,EAAE;SACV,OAAO,CAAC,KAAK,CAAC;IACjB,sCAAsC,EAAE,kBAAI,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;CACtE,CAAC,CAAC"}
|
|
@@ -9,6 +9,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
9
9
|
exports.PlaywrightModule = void 0;
|
|
10
10
|
const common_1 = require("@nestjs/common");
|
|
11
11
|
const playwright_service_1 = require("./playwright.service");
|
|
12
|
+
const playwright_config_service_1 = require("./services/playwright.config.service");
|
|
12
13
|
const playwright_hook_service_1 = require("./services/playwright.hook.service");
|
|
13
14
|
const playwright_browser_service_1 = require("./services/playwright.browser.service");
|
|
14
15
|
let PlaywrightModule = class PlaywrightModule {
|
|
@@ -18,6 +19,7 @@ exports.PlaywrightModule = PlaywrightModule = __decorate([
|
|
|
18
19
|
(0, common_1.Module)({
|
|
19
20
|
providers: [
|
|
20
21
|
playwright_service_1.PlaywrightService,
|
|
22
|
+
playwright_config_service_1.PlaywrightConfigService,
|
|
21
23
|
playwright_hook_service_1.PlaywrightHookService,
|
|
22
24
|
playwright_browser_service_1.PlaywrightBrowserService,
|
|
23
25
|
],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.module.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,6DAAyD;AACzD,gFAA2E;AAC3E,sFAAiF;
|
|
1
|
+
{"version":3,"file":"playwright.module.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,6DAAyD;AACzD,oFAA+E;AAC/E,gFAA2E;AAC3E,sFAAiF;AAW1E,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;CAAG,CAAA;AAAnB,4CAAgB;2BAAhB,gBAAgB;IAT5B,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,sCAAiB;YACjB,mDAAuB;YACvB,+CAAqB;YACrB,qDAAwB;SACzB;QACD,OAAO,EAAE,CAAC,sCAAiB,CAAC;KAC7B,CAAC;GACW,gBAAgB,CAAG"}
|
|
@@ -1,21 +1,14 @@
|
|
|
1
|
-
import { ConfigService } from '@nestjs/config';
|
|
2
1
|
import { PlaywrightCrawler } from '@crawlee/playwright';
|
|
3
2
|
import { PlaywrightCrawlerOptions } from '@crawlee/playwright';
|
|
4
3
|
import { PlaywrightHookService } from './services/playwright.hook.service';
|
|
5
4
|
import { PlaywrightBrowserService } from './services/playwright.browser.service';
|
|
5
|
+
import { PlaywrightConfigService } from './services/playwright.config.service';
|
|
6
6
|
export declare class PlaywrightService {
|
|
7
7
|
private readonly config;
|
|
8
8
|
private readonly hook;
|
|
9
9
|
private readonly browser;
|
|
10
10
|
private readonly logger;
|
|
11
|
-
|
|
12
|
-
private readonly timeoutSecs;
|
|
13
|
-
private readonly maxConcurrency;
|
|
14
|
-
private readonly minConcurrency;
|
|
15
|
-
private readonly maxRequestRetries;
|
|
16
|
-
private readonly headless;
|
|
17
|
-
private readonly useIncognitoPages;
|
|
18
|
-
constructor(config: ConfigService, hook: PlaywrightHookService, browser: PlaywrightBrowserService);
|
|
11
|
+
constructor(config: PlaywrightConfigService, hook: PlaywrightHookService, browser: PlaywrightBrowserService);
|
|
19
12
|
create(options: PlaywrightCrawlerOptions): PlaywrightCrawler;
|
|
20
13
|
getBrowser(): Promise<import("playwright").Browser>;
|
|
21
14
|
}
|
|
@@ -12,17 +12,11 @@ var PlaywrightService_1;
|
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
exports.PlaywrightService = void 0;
|
|
14
14
|
const common_1 = require("@nestjs/common");
|
|
15
|
-
const config_1 = require("@nestjs/config");
|
|
16
15
|
const playwright_1 = require("@crawlee/playwright");
|
|
17
16
|
const playwright_hook_service_1 = require("./services/playwright.hook.service");
|
|
18
17
|
const playwright_browser_service_1 = require("./services/playwright.browser.service");
|
|
18
|
+
const playwright_config_service_1 = require("./services/playwright.config.service");
|
|
19
19
|
const logger_1 = require("../../common/logger/logger");
|
|
20
|
-
const playwright_env_constant_1 = require("./constant/playwright.env.constant");
|
|
21
|
-
const playwright_env_constant_2 = require("./constant/playwright.env.constant");
|
|
22
|
-
const playwright_env_constant_3 = require("./constant/playwright.env.constant");
|
|
23
|
-
const playwright_env_constant_4 = require("./constant/playwright.env.constant");
|
|
24
|
-
const playwright_env_constant_5 = require("./constant/playwright.env.constant");
|
|
25
|
-
const playwright_env_constant_6 = require("./constant/playwright.env.constant");
|
|
26
20
|
const playwright_browser_constant_1 = require("./constant/playwright.browser.constant");
|
|
27
21
|
let PlaywrightService = PlaywrightService_1 = class PlaywrightService {
|
|
28
22
|
constructor(config, hook, browser) {
|
|
@@ -30,13 +24,6 @@ let PlaywrightService = PlaywrightService_1 = class PlaywrightService {
|
|
|
30
24
|
this.hook = hook;
|
|
31
25
|
this.browser = browser;
|
|
32
26
|
this.logger = new logger_1.Logger(PlaywrightService_1.name);
|
|
33
|
-
this.maxConcurrency = this.config.get(playwright_env_constant_1.MAX_CONCURRENCY);
|
|
34
|
-
this.minConcurrency = this.config.get(playwright_env_constant_2.MIN_CONCURRENCY);
|
|
35
|
-
this.maxRequestRetries = this.config.get(playwright_env_constant_3.MAX_REQUEST_RETRIES);
|
|
36
|
-
this.timeoutSecs = this.config.get(playwright_env_constant_4.TIMEOUT_SECS);
|
|
37
|
-
this.maxRequests = this.config.get(playwright_env_constant_5.MAX_REQUESTS);
|
|
38
|
-
this.headless = this.config.get(playwright_env_constant_6.HEADLESS);
|
|
39
|
-
this.useIncognitoPages = this.config.get(playwright_env_constant_6.INCOGNITO_PAGES);
|
|
40
27
|
}
|
|
41
28
|
create(options) {
|
|
42
29
|
this.logger.log('🚀 Creating Playwright Crawler');
|
|
@@ -44,21 +31,22 @@ let PlaywrightService = PlaywrightService_1 = class PlaywrightService {
|
|
|
44
31
|
const defaultPreNavHooks = this.hook.getPreNavigationHooks();
|
|
45
32
|
const defaultPostNavHooks = this.hook.getPostNavigationHooks();
|
|
46
33
|
const launcher = this.browser.getBrowserLauncher();
|
|
34
|
+
const { env } = this.config;
|
|
47
35
|
const defaultOptions = {
|
|
48
|
-
maxRequestsPerCrawl:
|
|
49
|
-
requestHandlerTimeoutSecs:
|
|
50
|
-
navigationTimeoutSecs:
|
|
51
|
-
maxConcurrency:
|
|
52
|
-
minConcurrency:
|
|
53
|
-
maxRequestRetries:
|
|
36
|
+
maxRequestsPerCrawl: env.maxRequests,
|
|
37
|
+
requestHandlerTimeoutSecs: env.timeoutSecs,
|
|
38
|
+
navigationTimeoutSecs: env.timeoutSecs,
|
|
39
|
+
maxConcurrency: env.maxConcurrency,
|
|
40
|
+
minConcurrency: env.minConcurrency,
|
|
41
|
+
maxRequestRetries: env.maxRequestRetries,
|
|
54
42
|
useSessionPool: true,
|
|
55
|
-
headless:
|
|
43
|
+
headless: env.headless,
|
|
56
44
|
launchContext: {
|
|
57
45
|
launcher,
|
|
58
46
|
launchOptions: {
|
|
59
47
|
args: playwright_browser_constant_1.BROWSER_ARGS,
|
|
60
48
|
},
|
|
61
|
-
useIncognitoPages:
|
|
49
|
+
useIncognitoPages: env.useIncognitoPages,
|
|
62
50
|
},
|
|
63
51
|
browserPoolOptions: {
|
|
64
52
|
useFingerprints: true,
|
|
@@ -83,7 +71,7 @@ let PlaywrightService = PlaywrightService_1 = class PlaywrightService {
|
|
|
83
71
|
exports.PlaywrightService = PlaywrightService;
|
|
84
72
|
exports.PlaywrightService = PlaywrightService = PlaywrightService_1 = __decorate([
|
|
85
73
|
(0, common_1.Injectable)(),
|
|
86
|
-
__metadata("design:paramtypes", [
|
|
74
|
+
__metadata("design:paramtypes", [playwright_config_service_1.PlaywrightConfigService,
|
|
87
75
|
playwright_hook_service_1.PlaywrightHookService,
|
|
88
76
|
playwright_browser_service_1.PlaywrightBrowserService])
|
|
89
77
|
], PlaywrightService);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.service.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;
|
|
1
|
+
{"version":3,"file":"playwright.service.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAE5C,oDAAwD;AAGxD,gFAA2E;AAC3E,sFAAiF;AACjF,oFAA+E;AAE/E,uDAAoD;AACpD,wFAAsE;AAG/D,IAAM,iBAAiB,yBAAvB,MAAM,iBAAiB;IAG5B,YACmB,MAA+B,EAC/B,IAA2B,EAC3B,OAAiC;QAFjC,WAAM,GAAN,MAAM,CAAyB;QAC/B,SAAI,GAAJ,IAAI,CAAuB;QAC3B,YAAO,GAAP,OAAO,CAA0B;QALnC,WAAM,GAAG,IAAI,eAAM,CAAC,mBAAiB,CAAC,IAAI,CAAC,CAAC;IAM1D,CAAC;IAEJ,MAAM,CAAC,OAAiC;QACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAClD,MAAM,EAAE,kBAAkB,GAAG,EAAE,EAAE,mBAAmB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACnD,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5B,MAAM,cAAc,GAA6B;YAC/C,mBAAmB,EAAE,GAAG,CAAC,WAAW;YACpC,yBAAyB,EAAE,GAAG,CAAC,WAAW;YAC1C,qBAAqB,EAAE,GAAG,CAAC,WAAW;YACtC,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;YACxC,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE;gBACb,QAAQ;gBACR,aAAa,EAAE;oBACb,IAAI,EAAE,0CAAY;iBACnB;gBACD,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;aACzC;YACD,kBAAkB,EAAE;gBAClB,eAAe,EAAE,IAAI;gBACrB,2BAA2B,EAAE,EAAE;gBAC/B,8BAA8B,EAAE,EAAE;aACnC;SACF,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,8BAAiB,CAAC;YACpC,GAAG,cAAc;YACjB,GAAG,OAAO;YACV,kBAAkB,EAAE,CAAC,GAAG,kBAAkB,EAAE,GAAG,kBAAkB,CAAC;YAClE,mBAAmB,EAAE,CAAC,GAAG,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;SACtE,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,0CAA0C,CAAC,CAAC;QACrE,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;IACnC,CAAC;CACF,CAAA;AApDY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAKgB,mDAAuB;QACzB,+CAAqB;QAClB,qDAAwB;GANzC,iBAAiB,CAoD7B"}
|
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
import { ConfigService } from '@nestjs/config';
|
|
2
1
|
import { Browser } from 'playwright';
|
|
3
2
|
import * as playwright from 'playwright';
|
|
4
3
|
import { BrowserName } from '../enum/playwright.browser-name.enum';
|
|
4
|
+
import { PlaywrightConfigService } from './playwright.config.service';
|
|
5
5
|
export declare class PlaywrightBrowserService {
|
|
6
6
|
private readonly config;
|
|
7
7
|
private readonly logger;
|
|
8
|
-
|
|
9
|
-
private readonly type;
|
|
10
|
-
constructor(config: ConfigService);
|
|
8
|
+
constructor(config: PlaywrightConfigService);
|
|
11
9
|
getBrowserLauncher(): playwright.BrowserType;
|
|
12
10
|
getBrowserName(): BrowserName;
|
|
13
11
|
getBrowser(): Promise<Browser>;
|
|
@@ -44,20 +44,17 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.PlaywrightBrowserService = void 0;
|
|
46
46
|
const common_1 = require("@nestjs/common");
|
|
47
|
-
const config_1 = require("@nestjs/config");
|
|
48
47
|
const playwright = __importStar(require("playwright"));
|
|
49
48
|
const playwright_service_1 = require("../playwright.service");
|
|
50
49
|
const playwright_browser_constant_1 = require("../constant/playwright.browser.constant");
|
|
51
|
-
const playwright_env_constant_1 = require("../constant/playwright.env.constant");
|
|
52
50
|
const playwright_browser_type_enum_1 = require("../enum/playwright.browser-type.enum");
|
|
53
51
|
const logger_1 = require("../../../common/logger/logger");
|
|
54
52
|
const playwright_browser_name_enum_1 = require("../enum/playwright.browser-name.enum");
|
|
53
|
+
const playwright_config_service_1 = require("./playwright.config.service");
|
|
55
54
|
let PlaywrightBrowserService = class PlaywrightBrowserService {
|
|
56
55
|
constructor(config) {
|
|
57
56
|
this.config = config;
|
|
58
57
|
this.logger = new logger_1.Logger(playwright_service_1.PlaywrightService.name);
|
|
59
|
-
this.headless = this.config.get(playwright_env_constant_1.HEADLESS);
|
|
60
|
-
this.type = this.config.get(playwright_env_constant_1.BROWSER);
|
|
61
58
|
}
|
|
62
59
|
getBrowserLauncher() {
|
|
63
60
|
const map = {
|
|
@@ -66,7 +63,7 @@ let PlaywrightBrowserService = class PlaywrightBrowserService {
|
|
|
66
63
|
[playwright_browser_type_enum_1.BrowserType.WEBKIT]: playwright.webkit,
|
|
67
64
|
[playwright_browser_type_enum_1.BrowserType.CAMOFOUX]: playwright.firefox,
|
|
68
65
|
};
|
|
69
|
-
return map[this.
|
|
66
|
+
return map[this.config.browser] ?? playwright.chromium;
|
|
70
67
|
}
|
|
71
68
|
getBrowserName() {
|
|
72
69
|
const map = {
|
|
@@ -75,13 +72,13 @@ let PlaywrightBrowserService = class PlaywrightBrowserService {
|
|
|
75
72
|
[playwright_browser_type_enum_1.BrowserType.WEBKIT]: playwright_browser_name_enum_1.BrowserName.WEBKIT,
|
|
76
73
|
[playwright_browser_type_enum_1.BrowserType.CAMOFOUX]: playwright_browser_name_enum_1.BrowserName.CAMOFOUX,
|
|
77
74
|
};
|
|
78
|
-
return map[this.
|
|
75
|
+
return map[this.config.browser] ?? playwright_browser_name_enum_1.BrowserName.CHROMIUM;
|
|
79
76
|
}
|
|
80
77
|
async getBrowser() {
|
|
81
78
|
this.logger.log('🚀 Creating direct Playwright browser instance');
|
|
82
79
|
const launcher = this.getBrowserLauncher();
|
|
83
80
|
const browser = await launcher.launch({
|
|
84
|
-
headless: this.headless,
|
|
81
|
+
headless: this.config.headless,
|
|
85
82
|
args: playwright_browser_constant_1.BROWSER_ARGS,
|
|
86
83
|
});
|
|
87
84
|
const name = this.getBrowserName();
|
|
@@ -92,6 +89,6 @@ let PlaywrightBrowserService = class PlaywrightBrowserService {
|
|
|
92
89
|
exports.PlaywrightBrowserService = PlaywrightBrowserService;
|
|
93
90
|
exports.PlaywrightBrowserService = PlaywrightBrowserService = __decorate([
|
|
94
91
|
(0, common_1.Injectable)(),
|
|
95
|
-
__metadata("design:paramtypes", [
|
|
92
|
+
__metadata("design:paramtypes", [playwright_config_service_1.PlaywrightConfigService])
|
|
96
93
|
], PlaywrightBrowserService);
|
|
97
94
|
//# sourceMappingURL=playwright.browser.service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.browser.service.js","sourceRoot":"","sources":["../../../../src/modules/playwright/services/playwright.browser.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA4C;
|
|
1
|
+
{"version":3,"file":"playwright.browser.service.js","sourceRoot":"","sources":["../../../../src/modules/playwright/services/playwright.browser.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA4C;AAG5C,uDAAyC;AAEzC,8DAA0D;AAC1D,yFAAuE;AACvE,uFAAmE;AACnE,0DAAuD;AACvD,uFAAmE;AACnE,2EAAsE;AAG/D,IAAM,wBAAwB,GAA9B,MAAM,wBAAwB;IAGnC,YAA6B,MAA+B;QAA/B,WAAM,GAAN,MAAM,CAAyB;QAF3C,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAEE,CAAC;IAEhE,kBAAkB;QAChB,MAAM,GAAG,GAAG;YACV,CAAC,0CAAW,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC,QAAQ;YAC3C,CAAC,0CAAW,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC,OAAO;YACzC,CAAC,0CAAW,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC,MAAM;YACvC,CAAC,0CAAW,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC,OAAO;SAC3C,CAAC;QAEF,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC;IACzD,CAAC;IAED,cAAc;QACZ,MAAM,GAAG,GAAG;YACV,CAAC,0CAAW,CAAC,QAAQ,CAAC,EAAE,0CAAW,CAAC,QAAQ;YAC5C,CAAC,0CAAW,CAAC,OAAO,CAAC,EAAE,0CAAW,CAAC,OAAO;YAC1C,CAAC,0CAAW,CAAC,MAAM,CAAC,EAAE,0CAAW,CAAC,MAAM;YACxC,CAAC,0CAAW,CAAC,QAAQ,CAAC,EAAE,0CAAW,CAAC,QAAQ;SAC7C,CAAC;QAEF,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,0CAAW,CAAC,QAAQ,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;YACpC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;YAC9B,IAAI,EAAE,0CAAY;SACnB,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,0CAA0C,CAAC,CAAC;QACrE,OAAO,OAAO,CAAC;IACjB,CAAC;CACF,CAAA;AAtCY,4DAAwB;mCAAxB,wBAAwB;IADpC,IAAA,mBAAU,GAAE;qCAI0B,mDAAuB;GAHjD,wBAAwB,CAsCpC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { ConfigService } from '@nestjs/config';
|
|
2
|
+
import { PlaywrightConfig } from '../types/playwright.config';
|
|
3
|
+
import { BrowserType } from '../enum/playwright.browser-type.enum';
|
|
4
|
+
export declare class PlaywrightConfigService {
|
|
5
|
+
private readonly service;
|
|
6
|
+
private readonly _env;
|
|
7
|
+
constructor(service: ConfigService);
|
|
8
|
+
get env(): PlaywrightConfig;
|
|
9
|
+
get maxRequests(): number;
|
|
10
|
+
get timeoutSecs(): number;
|
|
11
|
+
get maxConcurrency(): number;
|
|
12
|
+
get minConcurrency(): number;
|
|
13
|
+
get maxRequestRetries(): number;
|
|
14
|
+
get initialPage(): number;
|
|
15
|
+
get browser(): BrowserType;
|
|
16
|
+
get headless(): boolean;
|
|
17
|
+
get useIncognitoPages(): boolean;
|
|
18
|
+
get handleCloudflareChallenge(): boolean;
|
|
19
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.PlaywrightConfigService = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const config_1 = require("@nestjs/config");
|
|
15
|
+
const playwright_env_constant_1 = require("../constant/playwright.env.constant");
|
|
16
|
+
let PlaywrightConfigService = class PlaywrightConfigService {
|
|
17
|
+
constructor(service) {
|
|
18
|
+
this.service = service;
|
|
19
|
+
this._env = {
|
|
20
|
+
maxConcurrency: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.MAX_CONCURRENCY),
|
|
21
|
+
minConcurrency: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.MIN_CONCURRENCY),
|
|
22
|
+
maxRequestRetries: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.MAX_REQUEST_RETRIES),
|
|
23
|
+
timeoutSecs: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.TIMEOUT_SECS),
|
|
24
|
+
maxRequests: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.MAX_REQUESTS),
|
|
25
|
+
initialPage: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.INITIAL_PAGE),
|
|
26
|
+
browser: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.BROWSER),
|
|
27
|
+
headless: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.HEADLESS),
|
|
28
|
+
useIncognitoPages: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.INCOGNITO_PAGES),
|
|
29
|
+
handleCloudflareChallenge: this.service.get(playwright_env_constant_1.PLAYWRIGHT_ENV.HANDLE_CLOUDFLARE_CHALLENGE),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
get env() {
|
|
33
|
+
return this._env;
|
|
34
|
+
}
|
|
35
|
+
get maxRequests() {
|
|
36
|
+
return Number(this._env.maxRequests);
|
|
37
|
+
}
|
|
38
|
+
get timeoutSecs() {
|
|
39
|
+
return Number(this._env.timeoutSecs);
|
|
40
|
+
}
|
|
41
|
+
get maxConcurrency() {
|
|
42
|
+
return Number(this._env.maxConcurrency);
|
|
43
|
+
}
|
|
44
|
+
get minConcurrency() {
|
|
45
|
+
return Number(this._env.minConcurrency);
|
|
46
|
+
}
|
|
47
|
+
get maxRequestRetries() {
|
|
48
|
+
return Number(this._env.maxRequestRetries);
|
|
49
|
+
}
|
|
50
|
+
get initialPage() {
|
|
51
|
+
return Number(this._env.initialPage);
|
|
52
|
+
}
|
|
53
|
+
get browser() {
|
|
54
|
+
return this._env.browser;
|
|
55
|
+
}
|
|
56
|
+
get headless() {
|
|
57
|
+
return Boolean(this._env.headless);
|
|
58
|
+
}
|
|
59
|
+
get useIncognitoPages() {
|
|
60
|
+
return Boolean(this._env.useIncognitoPages);
|
|
61
|
+
}
|
|
62
|
+
get handleCloudflareChallenge() {
|
|
63
|
+
return Boolean(this._env.handleCloudflareChallenge);
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
exports.PlaywrightConfigService = PlaywrightConfigService;
|
|
67
|
+
exports.PlaywrightConfigService = PlaywrightConfigService = __decorate([
|
|
68
|
+
(0, common_1.Injectable)(),
|
|
69
|
+
__metadata("design:paramtypes", [config_1.ConfigService])
|
|
70
|
+
], PlaywrightConfigService);
|
|
71
|
+
//# sourceMappingURL=playwright.config.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright.config.service.js","sourceRoot":"","sources":["../../../../src/modules/playwright/services/playwright.config.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAG/C,iFAA4E;AAIrE,IAAM,uBAAuB,GAA7B,MAAM,uBAAuB;IAGlC,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;QACjD,IAAI,CAAC,IAAI,GAAG;YACV,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,eAAe,CAAC;YAC7D,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,eAAe,CAAC;YAC7D,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,mBAAmB,CAAC;YACpE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,YAAY,CAAC;YACvD,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,wCAAG,CAAC,OAAO,CAAgB;YAC7D,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAU,wCAAG,CAAC,QAAQ,CAAC;YACjD,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAU,wCAAG,CAAC,eAAe,CAAC;YACjE,yBAAyB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CACzC,wCAAG,CAAC,2BAA2B,CAChC;SACF,CAAC;IACJ,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;IAC3B,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,yBAAyB;QAC3B,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACtD,CAAC;CACF,CAAA;AA/DY,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,mBAAU,GAAE;qCAI2B,sBAAa;GAHxC,uBAAuB,CA+DnC"}
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { ConfigService } from '@nestjs/config';
|
|
2
1
|
import { PlaywrightHook } from '@crawlee/playwright';
|
|
2
|
+
import { PlaywrightConfigService } from './playwright.config.service';
|
|
3
3
|
export declare class PlaywrightHookService {
|
|
4
4
|
private readonly config;
|
|
5
5
|
private readonly logger;
|
|
6
|
-
|
|
7
|
-
constructor(config: ConfigService);
|
|
6
|
+
constructor(config: PlaywrightConfigService);
|
|
8
7
|
getPreNavigationHooks(): PlaywrightHook[];
|
|
9
8
|
getPostNavigationHooks(): PlaywrightHook[];
|
|
10
9
|
}
|