@rafikidota/scoutee 0.13.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/exports/camoufox.d.ts +5 -0
- package/dist/exports/camoufox.js +22 -0
- package/dist/exports/camoufox.js.map +1 -0
- package/dist/exports/cheerio.d.ts +5 -0
- package/dist/exports/cheerio.js +22 -0
- package/dist/exports/cheerio.js.map +1 -0
- package/dist/exports/http.d.ts +5 -0
- package/dist/exports/http.js +22 -0
- package/dist/exports/http.js.map +1 -0
- package/dist/exports/playwright.d.ts +5 -0
- package/dist/exports/playwright.js +22 -0
- package/dist/exports/playwright.js.map +1 -0
- package/dist/modules/camoufox/camoufox.module.js +7 -1
- package/dist/modules/camoufox/camoufox.module.js.map +1 -1
- package/dist/modules/camoufox/camoufox.service.d.ts +2 -9
- package/dist/modules/camoufox/camoufox.service.js +11 -23
- package/dist/modules/camoufox/camoufox.service.js.map +1 -1
- package/dist/modules/camoufox/constant/camoufox.env.constant.d.ts +12 -10
- package/dist/modules/camoufox/constant/camoufox.env.constant.js +13 -11
- package/dist/modules/camoufox/constant/camoufox.env.constant.js.map +1 -1
- package/dist/modules/camoufox/services/camoufox.browser.service.d.ts +2 -4
- package/dist/modules/camoufox/services/camoufox.browser.service.js +5 -8
- package/dist/modules/camoufox/services/camoufox.browser.service.js.map +1 -1
- package/dist/modules/camoufox/services/camoufox.config.service.d.ts +18 -0
- package/dist/modules/camoufox/services/camoufox.config.service.js +73 -0
- package/dist/modules/camoufox/services/camoufox.config.service.js.map +1 -0
- package/dist/modules/camoufox/services/camoufox.hook.service.d.ts +2 -3
- package/dist/modules/camoufox/services/camoufox.hook.service.js +3 -6
- package/dist/modules/camoufox/services/camoufox.hook.service.js.map +1 -1
- package/dist/modules/camoufox/types/camoufox.config.d.ts +12 -0
- package/dist/modules/camoufox/types/camoufox.config.js +3 -0
- package/dist/modules/camoufox/types/camoufox.config.js.map +1 -0
- package/dist/modules/cheerio/cheerio.module.js +2 -1
- package/dist/modules/cheerio/cheerio.module.js.map +1 -1
- package/dist/modules/cheerio/cheerio.service.d.ts +2 -7
- package/dist/modules/cheerio/cheerio.service.js +8 -17
- package/dist/modules/cheerio/cheerio.service.js.map +1 -1
- package/dist/modules/cheerio/constant/cheerio.env.constant.d.ts +8 -6
- package/dist/modules/cheerio/constant/cheerio.env.constant.js +9 -7
- package/dist/modules/cheerio/constant/cheerio.env.constant.js.map +1 -1
- package/dist/modules/cheerio/services/cheerio.config.service.d.ts +14 -0
- package/dist/modules/cheerio/services/cheerio.config.service.js +55 -0
- package/dist/modules/cheerio/services/cheerio.config.service.js.map +1 -0
- package/dist/modules/cheerio/types/cheerio.config.d.ts +8 -0
- package/dist/modules/cheerio/types/cheerio.config.js +3 -0
- package/dist/modules/cheerio/types/cheerio.config.js.map +1 -0
- package/dist/modules/http/constant/http.env.constant.d.ts +8 -6
- package/dist/modules/http/constant/http.env.constant.js +9 -7
- package/dist/modules/http/constant/http.env.constant.js.map +1 -1
- package/dist/modules/http/http.module.js +2 -1
- package/dist/modules/http/http.module.js.map +1 -1
- package/dist/modules/http/http.service.d.ts +2 -7
- package/dist/modules/http/http.service.js +9 -18
- package/dist/modules/http/http.service.js.map +1 -1
- package/dist/modules/http/services/http.config.service.d.ts +14 -0
- package/dist/modules/http/services/http.config.service.js +55 -0
- package/dist/modules/http/services/http.config.service.js.map +1 -0
- package/dist/modules/http/types/http.config.d.ts +8 -0
- package/dist/modules/http/types/http.config.js +3 -0
- package/dist/modules/http/types/http.config.js.map +1 -0
- package/dist/modules/playwright/constant/playwright.env.constant.d.ts +12 -10
- package/dist/modules/playwright/constant/playwright.env.constant.js +13 -11
- package/dist/modules/playwright/constant/playwright.env.constant.js.map +1 -1
- package/dist/modules/playwright/playwright.module.js +2 -0
- package/dist/modules/playwright/playwright.module.js.map +1 -1
- package/dist/modules/playwright/playwright.service.d.ts +2 -9
- package/dist/modules/playwright/playwright.service.js +11 -23
- package/dist/modules/playwright/playwright.service.js.map +1 -1
- package/dist/modules/playwright/services/playwright.browser.service.d.ts +2 -4
- package/dist/modules/playwright/services/playwright.browser.service.js +5 -8
- package/dist/modules/playwright/services/playwright.browser.service.js.map +1 -1
- package/dist/modules/playwright/services/playwright.config.service.d.ts +19 -0
- package/dist/modules/playwright/services/playwright.config.service.js +71 -0
- package/dist/modules/playwright/services/playwright.config.service.js.map +1 -0
- package/dist/modules/playwright/services/playwright.hook.service.d.ts +2 -3
- package/dist/modules/playwright/services/playwright.hook.service.js +3 -6
- package/dist/modules/playwright/services/playwright.hook.service.js.map +1 -1
- package/dist/modules/playwright/types/playwright.config.d.ts +13 -0
- package/dist/modules/playwright/types/playwright.config.js +3 -0
- package/dist/modules/playwright/types/playwright.config.js.map +1 -0
- package/package.json +23 -1
|
@@ -1,15 +1,10 @@
|
|
|
1
|
-
import { ConfigService } from '@nestjs/config';
|
|
2
1
|
import { CheerioCrawler, CheerioCrawlerOptions } from 'crawlee';
|
|
3
2
|
import { CheerioHookService } from './services/cheerio.hook.service';
|
|
3
|
+
import { CheerioConfigService } from './services/cheerio.config.service';
|
|
4
4
|
export declare class CheerioService {
|
|
5
5
|
private readonly config;
|
|
6
6
|
private readonly hook;
|
|
7
7
|
private readonly logger;
|
|
8
|
-
|
|
9
|
-
private readonly timeoutSecs;
|
|
10
|
-
private readonly maxConcurrency;
|
|
11
|
-
private readonly minConcurrency;
|
|
12
|
-
private readonly maxRequestRetries;
|
|
13
|
-
constructor(config: ConfigService, hook: CheerioHookService);
|
|
8
|
+
constructor(config: CheerioConfigService, hook: CheerioHookService);
|
|
14
9
|
create(options: CheerioCrawlerOptions): CheerioCrawler;
|
|
15
10
|
}
|
|
@@ -12,36 +12,27 @@ var CheerioService_1;
|
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
exports.CheerioService = void 0;
|
|
14
14
|
const common_1 = require("@nestjs/common");
|
|
15
|
-
const config_1 = require("@nestjs/config");
|
|
16
15
|
const crawlee_1 = require("crawlee");
|
|
17
16
|
const cheerio_hook_service_1 = require("./services/cheerio.hook.service");
|
|
17
|
+
const cheerio_config_service_1 = require("./services/cheerio.config.service");
|
|
18
18
|
const logger_1 = require("../../common/logger/logger");
|
|
19
|
-
const cheerio_env_constant_1 = require("./constant/cheerio.env.constant");
|
|
20
|
-
const cheerio_env_constant_2 = require("./constant/cheerio.env.constant");
|
|
21
|
-
const cheerio_env_constant_3 = require("./constant/cheerio.env.constant");
|
|
22
|
-
const cheerio_env_constant_4 = require("./constant/cheerio.env.constant");
|
|
23
|
-
const cheerio_env_constant_5 = require("./constant/cheerio.env.constant");
|
|
24
19
|
let CheerioService = CheerioService_1 = class CheerioService {
|
|
25
20
|
constructor(config, hook) {
|
|
26
21
|
this.config = config;
|
|
27
22
|
this.hook = hook;
|
|
28
23
|
this.logger = new logger_1.Logger(CheerioService_1.name);
|
|
29
|
-
this.maxConcurrency = this.config.get(cheerio_env_constant_1.MAX_CONCURRENCY);
|
|
30
|
-
this.minConcurrency = this.config.get(cheerio_env_constant_2.MIN_CONCURRENCY);
|
|
31
|
-
this.maxRequestRetries = this.config.get(cheerio_env_constant_3.MAX_REQUEST_RETRIES);
|
|
32
|
-
this.timeoutSecs = this.config.get(cheerio_env_constant_4.TIMEOUT_SECS);
|
|
33
|
-
this.maxRequests = this.config.get(cheerio_env_constant_5.MAX_REQUESTS);
|
|
34
24
|
}
|
|
35
25
|
create(options) {
|
|
36
26
|
const { preNavigationHooks = [], postNavigationHooks = [] } = options;
|
|
37
27
|
const defaultPreNavHooks = this.hook.getPreNavigationHooks();
|
|
38
28
|
const defaultPostNavHooks = this.hook.getPostNavigationHooks();
|
|
29
|
+
const { env } = this.config;
|
|
39
30
|
const defaultOptions = {
|
|
40
|
-
maxRequestsPerCrawl:
|
|
41
|
-
requestHandlerTimeoutSecs:
|
|
42
|
-
maxConcurrency:
|
|
43
|
-
minConcurrency:
|
|
44
|
-
maxRequestRetries:
|
|
31
|
+
maxRequestsPerCrawl: env.maxRequests,
|
|
32
|
+
requestHandlerTimeoutSecs: env.timeoutSecs,
|
|
33
|
+
maxConcurrency: env.maxConcurrency,
|
|
34
|
+
minConcurrency: env.minConcurrency,
|
|
35
|
+
maxRequestRetries: env.maxRequestRetries,
|
|
45
36
|
};
|
|
46
37
|
this.logger.log('Creating Cheerio Crawler instance');
|
|
47
38
|
return new crawlee_1.CheerioCrawler({
|
|
@@ -55,7 +46,7 @@ let CheerioService = CheerioService_1 = class CheerioService {
|
|
|
55
46
|
exports.CheerioService = CheerioService;
|
|
56
47
|
exports.CheerioService = CheerioService = CheerioService_1 = __decorate([
|
|
57
48
|
(0, common_1.Injectable)(),
|
|
58
|
-
__metadata("design:paramtypes", [
|
|
49
|
+
__metadata("design:paramtypes", [cheerio_config_service_1.CheerioConfigService,
|
|
59
50
|
cheerio_hook_service_1.CheerioHookService])
|
|
60
51
|
], CheerioService);
|
|
61
52
|
//# sourceMappingURL=cheerio.service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.service.js","sourceRoot":"","sources":["../../../src/modules/cheerio/cheerio.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;
|
|
1
|
+
{"version":3,"file":"cheerio.service.js","sourceRoot":"","sources":["../../../src/modules/cheerio/cheerio.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAE5C,qCAAgE;AAChE,0EAAqE;AACrE,8EAAyE;AACzE,uDAAoD;AAG7C,IAAM,cAAc,sBAApB,MAAM,cAAc;IAGzB,YACmB,MAA4B,EAC5B,IAAwB;QADxB,WAAM,GAAN,MAAM,CAAsB;QAC5B,SAAI,GAAJ,IAAI,CAAoB;QAJ1B,WAAM,GAAG,IAAI,eAAM,CAAC,gBAAc,CAAC,IAAI,CAAC,CAAC;IAKvD,CAAC;IAEJ,MAAM,CAAC,OAA8B;QACnC,MAAM,EAAE,kBAAkB,GAAG,EAAE,EAAE,mBAAmB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/D,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5B,MAAM,cAAc,GAA0B;YAC5C,mBAAmB,EAAE,GAAG,CAAC,WAAW;YACpC,yBAAyB,EAAE,GAAG,CAAC,WAAW;YAC1C,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;SACzC,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACrD,OAAO,IAAI,wBAAc,CAAC;YACxB,GAAG,cAAc;YACjB,GAAG,OAAO;YACV,kBAAkB,EAAE,CAAC,GAAG,kBAAkB,EAAE,GAAG,kBAAkB,CAAC;YAClE,mBAAmB,EAAE,CAAC,GAAG,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;SACtE,CAAC,CAAC;IACL,CAAC;CACF,CAAA;AA5BY,wCAAc;yBAAd,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAKgB,6CAAoB;QACtB,yCAAkB;GALhC,cAAc,CA4B1B"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
export declare const
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
1
|
+
export declare const CHEERIO_ENV: {
|
|
2
|
+
readonly MIN_CONCURRENCY: "CRAWLEE_CHEERIO_MIN_CONCURRENCY";
|
|
3
|
+
readonly MAX_CONCURRENCY: "CRAWLEE_CHEERIO_MAX_CONCURRENCY";
|
|
4
|
+
readonly MAX_REQUEST_RETRIES: "CRAWLEE_CHEERIO_MAX_REQUEST_RETRIES";
|
|
5
|
+
readonly TIMEOUT_SECS: "CRAWLEE_CHEERIO_TIMEOUT_SECS";
|
|
6
|
+
readonly MAX_REQUESTS: "CRAWLEE_CHEERIO_MAX_REQUESTS";
|
|
7
|
+
readonly INITIAL_PAGE: "CRAWLEE_CHEERIO_INITIAL_PAGE";
|
|
8
|
+
};
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
exports.
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
3
|
+
exports.CHEERIO_ENV = void 0;
|
|
4
|
+
exports.CHEERIO_ENV = {
|
|
5
|
+
MIN_CONCURRENCY: 'CRAWLEE_CHEERIO_MIN_CONCURRENCY',
|
|
6
|
+
MAX_CONCURRENCY: 'CRAWLEE_CHEERIO_MAX_CONCURRENCY',
|
|
7
|
+
MAX_REQUEST_RETRIES: 'CRAWLEE_CHEERIO_MAX_REQUEST_RETRIES',
|
|
8
|
+
TIMEOUT_SECS: 'CRAWLEE_CHEERIO_TIMEOUT_SECS',
|
|
9
|
+
MAX_REQUESTS: 'CRAWLEE_CHEERIO_MAX_REQUESTS',
|
|
10
|
+
INITIAL_PAGE: 'CRAWLEE_CHEERIO_INITIAL_PAGE',
|
|
11
|
+
};
|
|
10
12
|
//# sourceMappingURL=cheerio.env.constant.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/cheerio/constant/cheerio.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,
|
|
1
|
+
{"version":3,"file":"cheerio.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/cheerio/constant/cheerio.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,WAAW,GAAG;IACzB,eAAe,EAAE,iCAAiC;IAClD,eAAe,EAAE,iCAAiC;IAClD,mBAAmB,EAAE,qCAAqC;IAC1D,YAAY,EAAE,8BAA8B;IAC5C,YAAY,EAAE,8BAA8B;IAC5C,YAAY,EAAE,8BAA8B;CACpC,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { ConfigService } from '@nestjs/config';
|
|
2
|
+
import { CheerioConfig } from '../types/cheerio.config';
|
|
3
|
+
export declare class CheerioConfigService {
|
|
4
|
+
private readonly service;
|
|
5
|
+
private readonly _env;
|
|
6
|
+
constructor(service: ConfigService);
|
|
7
|
+
get env(): CheerioConfig;
|
|
8
|
+
get maxRequests(): number;
|
|
9
|
+
get timeoutSecs(): number;
|
|
10
|
+
get maxConcurrency(): number;
|
|
11
|
+
get minConcurrency(): number;
|
|
12
|
+
get maxRequestRetries(): number;
|
|
13
|
+
get initialPage(): number;
|
|
14
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.CheerioConfigService = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const config_1 = require("@nestjs/config");
|
|
15
|
+
const cheerio_env_constant_1 = require("../constant/cheerio.env.constant");
|
|
16
|
+
let CheerioConfigService = class CheerioConfigService {
|
|
17
|
+
constructor(service) {
|
|
18
|
+
this.service = service;
|
|
19
|
+
this._env = {
|
|
20
|
+
maxConcurrency: this.service.get(cheerio_env_constant_1.CHEERIO_ENV.MAX_CONCURRENCY),
|
|
21
|
+
minConcurrency: this.service.get(cheerio_env_constant_1.CHEERIO_ENV.MIN_CONCURRENCY),
|
|
22
|
+
maxRequestRetries: this.service.get(cheerio_env_constant_1.CHEERIO_ENV.MAX_REQUEST_RETRIES),
|
|
23
|
+
timeoutSecs: this.service.get(cheerio_env_constant_1.CHEERIO_ENV.TIMEOUT_SECS),
|
|
24
|
+
maxRequests: this.service.get(cheerio_env_constant_1.CHEERIO_ENV.MAX_REQUESTS),
|
|
25
|
+
initialPage: this.service.get(cheerio_env_constant_1.CHEERIO_ENV.INITIAL_PAGE),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
get env() {
|
|
29
|
+
return this._env;
|
|
30
|
+
}
|
|
31
|
+
get maxRequests() {
|
|
32
|
+
return Number(this._env.maxRequests);
|
|
33
|
+
}
|
|
34
|
+
get timeoutSecs() {
|
|
35
|
+
return Number(this._env.timeoutSecs);
|
|
36
|
+
}
|
|
37
|
+
get maxConcurrency() {
|
|
38
|
+
return Number(this._env.maxConcurrency);
|
|
39
|
+
}
|
|
40
|
+
get minConcurrency() {
|
|
41
|
+
return Number(this._env.minConcurrency);
|
|
42
|
+
}
|
|
43
|
+
get maxRequestRetries() {
|
|
44
|
+
return Number(this._env.maxRequestRetries);
|
|
45
|
+
}
|
|
46
|
+
get initialPage() {
|
|
47
|
+
return Number(this._env.initialPage);
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
exports.CheerioConfigService = CheerioConfigService;
|
|
51
|
+
exports.CheerioConfigService = CheerioConfigService = __decorate([
|
|
52
|
+
(0, common_1.Injectable)(),
|
|
53
|
+
__metadata("design:paramtypes", [config_1.ConfigService])
|
|
54
|
+
], CheerioConfigService);
|
|
55
|
+
//# sourceMappingURL=cheerio.config.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cheerio.config.service.js","sourceRoot":"","sources":["../../../../src/modules/cheerio/services/cheerio.config.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAG/C,2EAAsE;AAG/D,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;IAG/B,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;QACjD,IAAI,CAAC,IAAI,GAAG;YACV,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,kCAAG,CAAC,eAAe,CAAC;YAC7D,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,kCAAG,CAAC,eAAe,CAAC;YAC7D,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,kCAAG,CAAC,mBAAmB,CAAC;YACpE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,kCAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,kCAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,kCAAG,CAAC,YAAY,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;CACF,CAAA;AAzCY,oDAAoB;+BAApB,oBAAoB;IADhC,IAAA,mBAAU,GAAE;qCAI2B,sBAAa;GAHxC,oBAAoB,CAyChC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cheerio.config.js","sourceRoot":"","sources":["../../../../src/modules/cheerio/types/cheerio.config.ts"],"names":[],"mappings":""}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
export declare const
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
1
|
+
export declare const HTTP_ENV: {
|
|
2
|
+
readonly MAX_CONCURRENCY: "CRAWLEE_HTTP_MAX_CONCURRENCY";
|
|
3
|
+
readonly MIN_CONCURRENCY: "CRAWLEE_HTTP_MIN_CONCURRENCY";
|
|
4
|
+
readonly MAX_REQUEST_RETRIES: "CRAWLEE_HTTP_MAX_REQUEST_RETRIES";
|
|
5
|
+
readonly TIMEOUT_SECS: "CRAWLEE_HTTP_TIMEOUT_SECS";
|
|
6
|
+
readonly MAX_REQUESTS: "CRAWLEE_HTTP_MAX_REQUESTS";
|
|
7
|
+
readonly INITIAL_PAGE: "CRAWLEE_HTTP_INITIAL_PAGE";
|
|
8
|
+
};
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
exports.
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
3
|
+
exports.HTTP_ENV = void 0;
|
|
4
|
+
exports.HTTP_ENV = {
|
|
5
|
+
MAX_CONCURRENCY: 'CRAWLEE_HTTP_MAX_CONCURRENCY',
|
|
6
|
+
MIN_CONCURRENCY: 'CRAWLEE_HTTP_MIN_CONCURRENCY',
|
|
7
|
+
MAX_REQUEST_RETRIES: 'CRAWLEE_HTTP_MAX_REQUEST_RETRIES',
|
|
8
|
+
TIMEOUT_SECS: 'CRAWLEE_HTTP_TIMEOUT_SECS',
|
|
9
|
+
MAX_REQUESTS: 'CRAWLEE_HTTP_MAX_REQUESTS',
|
|
10
|
+
INITIAL_PAGE: 'CRAWLEE_HTTP_INITIAL_PAGE',
|
|
11
|
+
};
|
|
10
12
|
//# sourceMappingURL=http.env.constant.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/http/constant/http.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,
|
|
1
|
+
{"version":3,"file":"http.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/http/constant/http.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,QAAQ,GAAG;IACtB,eAAe,EAAE,8BAA8B;IAC/C,eAAe,EAAE,8BAA8B;IAC/C,mBAAmB,EAAE,kCAAkC;IACvD,YAAY,EAAE,2BAA2B;IACzC,YAAY,EAAE,2BAA2B;IACzC,YAAY,EAAE,2BAA2B;CACjC,CAAC"}
|
|
@@ -9,13 +9,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
9
9
|
exports.HttpModule = void 0;
|
|
10
10
|
const common_1 = require("@nestjs/common");
|
|
11
11
|
const http_service_1 = require("./http.service");
|
|
12
|
+
const http_config_service_1 = require("./services/http.config.service");
|
|
12
13
|
const http_hook_service_1 = require("./services/http.hook.service");
|
|
13
14
|
let HttpModule = class HttpModule {
|
|
14
15
|
};
|
|
15
16
|
exports.HttpModule = HttpModule;
|
|
16
17
|
exports.HttpModule = HttpModule = __decorate([
|
|
17
18
|
(0, common_1.Module)({
|
|
18
|
-
providers: [http_service_1.HttpService, http_hook_service_1.HttpHookService],
|
|
19
|
+
providers: [http_service_1.HttpService, http_config_service_1.HttpConfigService, http_hook_service_1.HttpHookService],
|
|
19
20
|
exports: [http_service_1.HttpService],
|
|
20
21
|
})
|
|
21
22
|
], HttpModule);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.module.js","sourceRoot":"","sources":["../../../src/modules/http/http.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,iDAA6C;AAC7C,oEAA+D;AAMxD,IAAM,UAAU,GAAhB,MAAM,UAAU;CAAG,CAAA;AAAb,gCAAU;qBAAV,UAAU;IAJtB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE,CAAC,0BAAW,EAAE,mCAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"http.module.js","sourceRoot":"","sources":["../../../src/modules/http/http.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,iDAA6C;AAC7C,wEAAmE;AACnE,oEAA+D;AAMxD,IAAM,UAAU,GAAhB,MAAM,UAAU;CAAG,CAAA;AAAb,gCAAU;qBAAV,UAAU;IAJtB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE,CAAC,0BAAW,EAAE,uCAAiB,EAAE,mCAAe,CAAC;QAC5D,OAAO,EAAE,CAAC,0BAAW,CAAC;KACvB,CAAC;GACW,UAAU,CAAG"}
|
|
@@ -1,15 +1,10 @@
|
|
|
1
|
-
import { ConfigService } from '@nestjs/config';
|
|
2
1
|
import { HttpCrawler, HttpCrawlerOptions } from 'crawlee';
|
|
3
2
|
import { HttpHookService } from './services/http.hook.service';
|
|
3
|
+
import { HttpConfigService } from './services/http.config.service';
|
|
4
4
|
export declare class HttpService {
|
|
5
5
|
private readonly config;
|
|
6
6
|
private readonly hook;
|
|
7
7
|
private readonly logger;
|
|
8
|
-
|
|
9
|
-
private readonly timeoutSecs;
|
|
10
|
-
private readonly maxConcurrency;
|
|
11
|
-
private readonly minConcurrency;
|
|
12
|
-
private readonly maxRequestRetries;
|
|
13
|
-
constructor(config: ConfigService, hook: HttpHookService);
|
|
8
|
+
constructor(config: HttpConfigService, hook: HttpHookService);
|
|
14
9
|
create(options: HttpCrawlerOptions): HttpCrawler<import("crawlee").InternalHttpCrawlingContext<any, any, HttpCrawler<any>>>;
|
|
15
10
|
}
|
|
@@ -12,38 +12,29 @@ var HttpService_1;
|
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
exports.HttpService = void 0;
|
|
14
14
|
const common_1 = require("@nestjs/common");
|
|
15
|
-
const config_1 = require("@nestjs/config");
|
|
16
15
|
const crawlee_1 = require("crawlee");
|
|
17
16
|
const http_hook_service_1 = require("./services/http.hook.service");
|
|
17
|
+
const http_config_service_1 = require("./services/http.config.service");
|
|
18
18
|
const logger_1 = require("../../common/logger/logger");
|
|
19
|
-
const http_env_constant_1 = require("./constant/http.env.constant");
|
|
20
|
-
const http_env_constant_2 = require("./constant/http.env.constant");
|
|
21
|
-
const http_env_constant_3 = require("./constant/http.env.constant");
|
|
22
|
-
const http_env_constant_4 = require("./constant/http.env.constant");
|
|
23
|
-
const http_env_constant_5 = require("./constant/http.env.constant");
|
|
24
19
|
let HttpService = HttpService_1 = class HttpService {
|
|
25
20
|
constructor(config, hook) {
|
|
26
21
|
this.config = config;
|
|
27
22
|
this.hook = hook;
|
|
28
23
|
this.logger = new logger_1.Logger(HttpService_1.name);
|
|
29
|
-
this.maxConcurrency = this.config.get(http_env_constant_1.MAX_CONCURRENCY);
|
|
30
|
-
this.minConcurrency = this.config.get(http_env_constant_2.MIN_CONCURRENCY);
|
|
31
|
-
this.maxRequestRetries = this.config.get(http_env_constant_3.MAX_REQUEST_RETRIES);
|
|
32
|
-
this.timeoutSecs = this.config.get(http_env_constant_4.TIMEOUT_SECS);
|
|
33
|
-
this.maxRequests = this.config.get(http_env_constant_5.MAX_REQUESTS);
|
|
34
24
|
}
|
|
35
25
|
create(options) {
|
|
36
26
|
const { preNavigationHooks = [], postNavigationHooks = [] } = options;
|
|
37
27
|
const defaultPreNavHooks = this.hook.getPreNavigationHooks();
|
|
38
28
|
const defaultPostNavHooks = this.hook.getPostNavigationHooks();
|
|
29
|
+
const { env } = this.config;
|
|
39
30
|
const defaultOptions = {
|
|
40
|
-
maxRequestsPerCrawl:
|
|
41
|
-
requestHandlerTimeoutSecs:
|
|
42
|
-
maxConcurrency:
|
|
43
|
-
minConcurrency:
|
|
44
|
-
maxRequestRetries:
|
|
31
|
+
maxRequestsPerCrawl: env.maxRequests,
|
|
32
|
+
requestHandlerTimeoutSecs: env.timeoutSecs,
|
|
33
|
+
maxConcurrency: env.maxConcurrency,
|
|
34
|
+
minConcurrency: env.minConcurrency,
|
|
35
|
+
maxRequestRetries: env.maxRequestRetries,
|
|
45
36
|
};
|
|
46
|
-
this.logger.log('Creating
|
|
37
|
+
this.logger.log('Creating Http Crawler instance');
|
|
47
38
|
return new crawlee_1.HttpCrawler({
|
|
48
39
|
...defaultOptions,
|
|
49
40
|
...options,
|
|
@@ -55,7 +46,7 @@ let HttpService = HttpService_1 = class HttpService {
|
|
|
55
46
|
exports.HttpService = HttpService;
|
|
56
47
|
exports.HttpService = HttpService = HttpService_1 = __decorate([
|
|
57
48
|
(0, common_1.Injectable)(),
|
|
58
|
-
__metadata("design:paramtypes", [
|
|
49
|
+
__metadata("design:paramtypes", [http_config_service_1.HttpConfigService,
|
|
59
50
|
http_hook_service_1.HttpHookService])
|
|
60
51
|
], HttpService);
|
|
61
52
|
//# sourceMappingURL=http.service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.service.js","sourceRoot":"","sources":["../../../src/modules/http/http.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;
|
|
1
|
+
{"version":3,"file":"http.service.js","sourceRoot":"","sources":["../../../src/modules/http/http.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAE5C,qCAA0D;AAE1D,oEAA+D;AAC/D,wEAAmE;AACnE,uDAAoD;AAG7C,IAAM,WAAW,mBAAjB,MAAM,WAAW;IAGtB,YACmB,MAAyB,EACzB,IAAqB;QADrB,WAAM,GAAN,MAAM,CAAmB;QACzB,SAAI,GAAJ,IAAI,CAAiB;QAJvB,WAAM,GAAG,IAAI,eAAM,CAAC,aAAW,CAAC,IAAI,CAAC,CAAC;IAKpD,CAAC;IAEJ,MAAM,CAAC,OAA2B;QAChC,MAAM,EAAE,kBAAkB,GAAG,EAAE,EAAE,mBAAmB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/D,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5B,MAAM,cAAc,GAAuB;YACzC,mBAAmB,EAAE,GAAG,CAAC,WAAW;YACpC,yBAAyB,EAAE,GAAG,CAAC,WAAW;YAC1C,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;SACzC,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAClD,OAAO,IAAI,qBAAW,CAAC;YACrB,GAAG,cAAc;YACjB,GAAG,OAAO;YACV,kBAAkB,EAAE,CAAC,GAAG,kBAAkB,EAAE,GAAG,kBAAkB,CAAC;YAClE,mBAAmB,EAAE,CAAC,GAAG,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;SACtE,CAAC,CAAC;IACL,CAAC;CACF,CAAA;AA5BY,kCAAW;sBAAX,WAAW;IADvB,IAAA,mBAAU,GAAE;qCAKgB,uCAAiB;QACnB,mCAAe;GAL7B,WAAW,CA4BvB"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { ConfigService } from '@nestjs/config';
|
|
2
|
+
import { HttpConfig } from '../types/http.config';
|
|
3
|
+
export declare class HttpConfigService {
|
|
4
|
+
private readonly service;
|
|
5
|
+
private readonly _env;
|
|
6
|
+
constructor(service: ConfigService);
|
|
7
|
+
get env(): HttpConfig;
|
|
8
|
+
get maxRequests(): number;
|
|
9
|
+
get timeoutSecs(): number;
|
|
10
|
+
get maxConcurrency(): number;
|
|
11
|
+
get minConcurrency(): number;
|
|
12
|
+
get maxRequestRetries(): number;
|
|
13
|
+
get initialPage(): number;
|
|
14
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.HttpConfigService = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const config_1 = require("@nestjs/config");
|
|
15
|
+
const http_env_constant_1 = require("../constant/http.env.constant");
|
|
16
|
+
let HttpConfigService = class HttpConfigService {
|
|
17
|
+
constructor(service) {
|
|
18
|
+
this.service = service;
|
|
19
|
+
this._env = {
|
|
20
|
+
maxConcurrency: this.service.get(http_env_constant_1.HTTP_ENV.MAX_CONCURRENCY),
|
|
21
|
+
minConcurrency: this.service.get(http_env_constant_1.HTTP_ENV.MIN_CONCURRENCY),
|
|
22
|
+
maxRequestRetries: this.service.get(http_env_constant_1.HTTP_ENV.MAX_REQUEST_RETRIES),
|
|
23
|
+
timeoutSecs: this.service.get(http_env_constant_1.HTTP_ENV.TIMEOUT_SECS),
|
|
24
|
+
maxRequests: this.service.get(http_env_constant_1.HTTP_ENV.MAX_REQUESTS),
|
|
25
|
+
initialPage: this.service.get(http_env_constant_1.HTTP_ENV.INITIAL_PAGE),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
get env() {
|
|
29
|
+
return this._env;
|
|
30
|
+
}
|
|
31
|
+
get maxRequests() {
|
|
32
|
+
return Number(this._env.maxRequests);
|
|
33
|
+
}
|
|
34
|
+
get timeoutSecs() {
|
|
35
|
+
return Number(this._env.timeoutSecs);
|
|
36
|
+
}
|
|
37
|
+
get maxConcurrency() {
|
|
38
|
+
return Number(this._env.maxConcurrency);
|
|
39
|
+
}
|
|
40
|
+
get minConcurrency() {
|
|
41
|
+
return Number(this._env.minConcurrency);
|
|
42
|
+
}
|
|
43
|
+
get maxRequestRetries() {
|
|
44
|
+
return Number(this._env.maxRequestRetries);
|
|
45
|
+
}
|
|
46
|
+
get initialPage() {
|
|
47
|
+
return Number(this._env.initialPage);
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
exports.HttpConfigService = HttpConfigService;
|
|
51
|
+
exports.HttpConfigService = HttpConfigService = __decorate([
|
|
52
|
+
(0, common_1.Injectable)(),
|
|
53
|
+
__metadata("design:paramtypes", [config_1.ConfigService])
|
|
54
|
+
], HttpConfigService);
|
|
55
|
+
//# sourceMappingURL=http.config.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.config.service.js","sourceRoot":"","sources":["../../../../src/modules/http/services/http.config.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAG/C,qEAAgE;AAGzD,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAG5B,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;QACjD,IAAI,CAAC,IAAI,GAAG;YACV,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,eAAe,CAAC;YAC7D,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,eAAe,CAAC;YAC7D,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,mBAAmB,CAAC;YACpE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,4BAAG,CAAC,YAAY,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;CACF,CAAA;AAzCY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAI2B,sBAAa;GAHxC,iBAAiB,CAyC7B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.config.js","sourceRoot":"","sources":["../../../../src/modules/http/types/http.config.ts"],"names":[],"mappings":""}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
export declare const
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
1
|
+
export declare const PLAYWRIGHT_ENV: {
|
|
2
|
+
readonly BROWSER: "CRAWLEE_PLAYWRIGHT_BROWSER";
|
|
3
|
+
readonly MIN_CONCURRENCY: "CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY";
|
|
4
|
+
readonly MAX_CONCURRENCY: "CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY";
|
|
5
|
+
readonly MAX_REQUEST_RETRIES: "CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES";
|
|
6
|
+
readonly TIMEOUT_SECS: "CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS";
|
|
7
|
+
readonly MAX_REQUESTS: "CRAWLEE_PLAYWRIGHT_MAX_REQUESTS";
|
|
8
|
+
readonly INITIAL_PAGE: "CRAWLEE_PLAYWRIGHT_INITIAL_PAGE";
|
|
9
|
+
readonly HEADLESS: "CRAWLEE_PLAYWRIGHT_HEADLESS";
|
|
10
|
+
readonly INCOGNITO_PAGES: "CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES";
|
|
11
|
+
readonly HANDLE_CLOUDFLARE_CHALLENGE: "CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE";
|
|
12
|
+
};
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
exports.
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
3
|
+
exports.PLAYWRIGHT_ENV = void 0;
|
|
4
|
+
exports.PLAYWRIGHT_ENV = {
|
|
5
|
+
BROWSER: 'CRAWLEE_PLAYWRIGHT_BROWSER',
|
|
6
|
+
MIN_CONCURRENCY: 'CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY',
|
|
7
|
+
MAX_CONCURRENCY: 'CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY',
|
|
8
|
+
MAX_REQUEST_RETRIES: 'CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES',
|
|
9
|
+
TIMEOUT_SECS: 'CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS',
|
|
10
|
+
MAX_REQUESTS: 'CRAWLEE_PLAYWRIGHT_MAX_REQUESTS',
|
|
11
|
+
INITIAL_PAGE: 'CRAWLEE_PLAYWRIGHT_INITIAL_PAGE',
|
|
12
|
+
HEADLESS: 'CRAWLEE_PLAYWRIGHT_HEADLESS',
|
|
13
|
+
INCOGNITO_PAGES: 'CRAWLEE_PLAYWRIGHT_USE_INCOGNITO_PAGES',
|
|
14
|
+
HANDLE_CLOUDFLARE_CHALLENGE: 'CRAWLEE_PLAYWRIGHT_HANDLE_CLOUDFLARE_CHALLENGE',
|
|
15
|
+
};
|
|
14
16
|
//# sourceMappingURL=playwright.env.constant.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/playwright/constant/playwright.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,
|
|
1
|
+
{"version":3,"file":"playwright.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/playwright/constant/playwright.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,cAAc,GAAG;IAC5B,OAAO,EAAE,4BAA4B;IACrC,eAAe,EAAE,oCAAoC;IACrD,eAAe,EAAE,oCAAoC;IACrD,mBAAmB,EAAE,wCAAwC;IAC7D,YAAY,EAAE,iCAAiC;IAC/C,YAAY,EAAE,iCAAiC;IAC/C,YAAY,EAAE,iCAAiC;IAC/C,QAAQ,EAAE,6BAA6B;IACvC,eAAe,EAAE,wCAAwC;IACzD,2BAA2B,EAAE,gDAAgD;CACrE,CAAC"}
|
|
@@ -9,6 +9,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
9
9
|
exports.PlaywrightModule = void 0;
|
|
10
10
|
const common_1 = require("@nestjs/common");
|
|
11
11
|
const playwright_service_1 = require("./playwright.service");
|
|
12
|
+
const playwright_config_service_1 = require("./services/playwright.config.service");
|
|
12
13
|
const playwright_hook_service_1 = require("./services/playwright.hook.service");
|
|
13
14
|
const playwright_browser_service_1 = require("./services/playwright.browser.service");
|
|
14
15
|
let PlaywrightModule = class PlaywrightModule {
|
|
@@ -18,6 +19,7 @@ exports.PlaywrightModule = PlaywrightModule = __decorate([
|
|
|
18
19
|
(0, common_1.Module)({
|
|
19
20
|
providers: [
|
|
20
21
|
playwright_service_1.PlaywrightService,
|
|
22
|
+
playwright_config_service_1.PlaywrightConfigService,
|
|
21
23
|
playwright_hook_service_1.PlaywrightHookService,
|
|
22
24
|
playwright_browser_service_1.PlaywrightBrowserService,
|
|
23
25
|
],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.module.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,6DAAyD;AACzD,gFAA2E;AAC3E,sFAAiF;
|
|
1
|
+
{"version":3,"file":"playwright.module.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,6DAAyD;AACzD,oFAA+E;AAC/E,gFAA2E;AAC3E,sFAAiF;AAW1E,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;CAAG,CAAA;AAAnB,4CAAgB;2BAAhB,gBAAgB;IAT5B,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,sCAAiB;YACjB,mDAAuB;YACvB,+CAAqB;YACrB,qDAAwB;SACzB;QACD,OAAO,EAAE,CAAC,sCAAiB,CAAC;KAC7B,CAAC;GACW,gBAAgB,CAAG"}
|
|
@@ -1,21 +1,14 @@
|
|
|
1
|
-
import { ConfigService } from '@nestjs/config';
|
|
2
1
|
import { PlaywrightCrawler } from '@crawlee/playwright';
|
|
3
2
|
import { PlaywrightCrawlerOptions } from '@crawlee/playwright';
|
|
4
3
|
import { PlaywrightHookService } from './services/playwright.hook.service';
|
|
5
4
|
import { PlaywrightBrowserService } from './services/playwright.browser.service';
|
|
5
|
+
import { PlaywrightConfigService } from './services/playwright.config.service';
|
|
6
6
|
export declare class PlaywrightService {
|
|
7
7
|
private readonly config;
|
|
8
8
|
private readonly hook;
|
|
9
9
|
private readonly browser;
|
|
10
10
|
private readonly logger;
|
|
11
|
-
|
|
12
|
-
private readonly timeoutSecs;
|
|
13
|
-
private readonly maxConcurrency;
|
|
14
|
-
private readonly minConcurrency;
|
|
15
|
-
private readonly maxRequestRetries;
|
|
16
|
-
private readonly headless;
|
|
17
|
-
private readonly useIncognitoPages;
|
|
18
|
-
constructor(config: ConfigService, hook: PlaywrightHookService, browser: PlaywrightBrowserService);
|
|
11
|
+
constructor(config: PlaywrightConfigService, hook: PlaywrightHookService, browser: PlaywrightBrowserService);
|
|
19
12
|
create(options: PlaywrightCrawlerOptions): PlaywrightCrawler;
|
|
20
13
|
getBrowser(): Promise<import("playwright").Browser>;
|
|
21
14
|
}
|