@rafikidota/scoutee 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/exports/camoufox.d.ts +5 -0
- package/dist/exports/camoufox.js +22 -0
- package/dist/exports/camoufox.js.map +1 -0
- package/dist/exports/cheerio.d.ts +5 -0
- package/dist/exports/cheerio.js +22 -0
- package/dist/exports/cheerio.js.map +1 -0
- package/dist/exports/http.d.ts +5 -0
- package/dist/exports/http.js +22 -0
- package/dist/exports/http.js.map +1 -0
- package/dist/exports/playwright.d.ts +5 -0
- package/dist/exports/playwright.js +22 -0
- package/dist/exports/playwright.js.map +1 -0
- package/dist/modules/camoufox/camoufox.module.js +7 -1
- package/dist/modules/camoufox/camoufox.module.js.map +1 -1
- package/dist/modules/camoufox/camoufox.service.d.ts +2 -9
- package/dist/modules/camoufox/camoufox.service.js +11 -23
- package/dist/modules/camoufox/camoufox.service.js.map +1 -1
- package/dist/modules/camoufox/constant/camoufox.env.constant.d.ts +12 -10
- package/dist/modules/camoufox/constant/camoufox.env.constant.js +13 -11
- package/dist/modules/camoufox/constant/camoufox.env.constant.js.map +1 -1
- package/dist/modules/camoufox/services/camoufox.browser.service.d.ts +2 -4
- package/dist/modules/camoufox/services/camoufox.browser.service.js +5 -8
- package/dist/modules/camoufox/services/camoufox.browser.service.js.map +1 -1
- package/dist/modules/camoufox/services/camoufox.config.service.d.ts +18 -0
- package/dist/modules/camoufox/services/camoufox.config.service.js +71 -0
- package/dist/modules/camoufox/services/camoufox.config.service.js.map +1 -0
- package/dist/modules/camoufox/services/camoufox.hook.service.d.ts +2 -3
- package/dist/modules/camoufox/services/camoufox.hook.service.js +3 -6
- package/dist/modules/camoufox/services/camoufox.hook.service.js.map +1 -1
- package/dist/modules/camoufox/types/camoufox.config.d.ts +12 -0
- package/dist/modules/camoufox/types/camoufox.config.js +3 -0
- package/dist/modules/camoufox/types/camoufox.config.js.map +1 -0
- package/dist/modules/cheerio/cheerio.module.js +2 -1
- package/dist/modules/cheerio/cheerio.module.js.map +1 -1
- package/dist/modules/cheerio/cheerio.service.d.ts +2 -7
- package/dist/modules/cheerio/cheerio.service.js +8 -17
- package/dist/modules/cheerio/cheerio.service.js.map +1 -1
- package/dist/modules/cheerio/constant/cheerio.env.constant.d.ts +8 -6
- package/dist/modules/cheerio/constant/cheerio.env.constant.js +9 -7
- package/dist/modules/cheerio/constant/cheerio.env.constant.js.map +1 -1
- package/dist/modules/cheerio/services/cheerio.config.service.d.ts +14 -0
- package/dist/modules/cheerio/services/cheerio.config.service.js +55 -0
- package/dist/modules/cheerio/services/cheerio.config.service.js.map +1 -0
- package/dist/modules/cheerio/types/cheerio.config.d.ts +8 -0
- package/dist/modules/cheerio/types/cheerio.config.js +3 -0
- package/dist/modules/cheerio/types/cheerio.config.js.map +1 -0
- package/dist/modules/http/constant/http.env.constant.d.ts +8 -6
- package/dist/modules/http/constant/http.env.constant.js +9 -7
- package/dist/modules/http/constant/http.env.constant.js.map +1 -1
- package/dist/modules/http/http.module.js +2 -1
- package/dist/modules/http/http.module.js.map +1 -1
- package/dist/modules/http/http.service.d.ts +2 -7
- package/dist/modules/http/http.service.js +9 -18
- package/dist/modules/http/http.service.js.map +1 -1
- package/dist/modules/http/services/http.config.service.d.ts +14 -0
- package/dist/modules/http/services/http.config.service.js +55 -0
- package/dist/modules/http/services/http.config.service.js.map +1 -0
- package/dist/modules/http/types/http.config.d.ts +8 -0
- package/dist/modules/http/types/http.config.js +3 -0
- package/dist/modules/http/types/http.config.js.map +1 -0
- package/dist/modules/playwright/constant/playwright.env.constant.d.ts +12 -10
- package/dist/modules/playwright/constant/playwright.env.constant.js +13 -11
- package/dist/modules/playwright/constant/playwright.env.constant.js.map +1 -1
- package/dist/modules/playwright/playwright.module.js +2 -0
- package/dist/modules/playwright/playwright.module.js.map +1 -1
- package/dist/modules/playwright/playwright.service.d.ts +2 -9
- package/dist/modules/playwright/playwright.service.js +11 -23
- package/dist/modules/playwright/playwright.service.js.map +1 -1
- package/dist/modules/playwright/services/playwright.browser.service.d.ts +2 -4
- package/dist/modules/playwright/services/playwright.browser.service.js +5 -8
- package/dist/modules/playwright/services/playwright.browser.service.js.map +1 -1
- package/dist/modules/playwright/services/playwright.config.service.d.ts +19 -0
- package/dist/modules/playwright/services/playwright.config.service.js +71 -0
- package/dist/modules/playwright/services/playwright.config.service.js.map +1 -0
- package/dist/modules/playwright/services/playwright.hook.service.d.ts +2 -3
- package/dist/modules/playwright/services/playwright.hook.service.js +3 -6
- package/dist/modules/playwright/services/playwright.hook.service.js.map +1 -1
- package/dist/modules/playwright/types/playwright.config.d.ts +13 -0
- package/dist/modules/playwright/types/playwright.config.js +3 -0
- package/dist/modules/playwright/types/playwright.config.js.map +1 -0
- package/package.json +23 -1
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export * from '../modules/camoufox/camoufox.module';
|
|
2
|
+
export * from '../modules/camoufox/camoufox.service';
|
|
3
|
+
export * from '../modules/camoufox/env/camoufox.env.config';
|
|
4
|
+
export * from '../modules/camoufox/env/camoufox.env.validation';
|
|
5
|
+
export * from '../modules/camoufox/env/camoufox.env.zod';
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("../modules/camoufox/camoufox.module"), exports);
|
|
18
|
+
__exportStar(require("../modules/camoufox/camoufox.service"), exports);
|
|
19
|
+
__exportStar(require("../modules/camoufox/env/camoufox.env.config"), exports);
|
|
20
|
+
__exportStar(require("../modules/camoufox/env/camoufox.env.validation"), exports);
|
|
21
|
+
__exportStar(require("../modules/camoufox/env/camoufox.env.zod"), exports);
|
|
22
|
+
//# sourceMappingURL=camoufox.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"camoufox.js","sourceRoot":"","sources":["../../src/exports/camoufox.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,sEAAoD;AACpD,uEAAqD;AACrD,8EAA4D;AAC5D,kFAAgE;AAChE,2EAAyD"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export * from '../modules/cheerio/cheerio.module';
|
|
2
|
+
export * from '../modules/cheerio/cheerio.service';
|
|
3
|
+
export * from '../modules/cheerio/env/cheerio.env.config';
|
|
4
|
+
export * from '../modules/cheerio/env/cheerio.env.validation';
|
|
5
|
+
export * from '../modules/cheerio/env/cheerio.env.zod';
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("../modules/cheerio/cheerio.module"), exports);
|
|
18
|
+
__exportStar(require("../modules/cheerio/cheerio.service"), exports);
|
|
19
|
+
__exportStar(require("../modules/cheerio/env/cheerio.env.config"), exports);
|
|
20
|
+
__exportStar(require("../modules/cheerio/env/cheerio.env.validation"), exports);
|
|
21
|
+
__exportStar(require("../modules/cheerio/env/cheerio.env.zod"), exports);
|
|
22
|
+
//# sourceMappingURL=cheerio.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/exports/cheerio.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,oEAAkD;AAClD,qEAAmD;AACnD,4EAA0D;AAC1D,gFAA8D;AAC9D,yEAAuD"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("../modules/http/http.module"), exports);
|
|
18
|
+
__exportStar(require("../modules/http/http.service"), exports);
|
|
19
|
+
__exportStar(require("../modules/http/env/http.env.config"), exports);
|
|
20
|
+
__exportStar(require("../modules/http/env/http.env.validation"), exports);
|
|
21
|
+
__exportStar(require("../modules/http/env/http.env.zod"), exports);
|
|
22
|
+
//# sourceMappingURL=http.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.js","sourceRoot":"","sources":["../../src/exports/http.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,8DAA4C;AAC5C,+DAA6C;AAC7C,sEAAoD;AACpD,0EAAwD;AACxD,mEAAiD"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export * from '../modules/playwright/playwright.module';
|
|
2
|
+
export * from '../modules/playwright/playwright.service';
|
|
3
|
+
export * from '../modules/playwright/env/playwright.env.config';
|
|
4
|
+
export * from '../modules/playwright/env/playwright.env.validation';
|
|
5
|
+
export * from '../modules/playwright/env/playwright.env.zod';
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("../modules/playwright/playwright.module"), exports);
|
|
18
|
+
__exportStar(require("../modules/playwright/playwright.service"), exports);
|
|
19
|
+
__exportStar(require("../modules/playwright/env/playwright.env.config"), exports);
|
|
20
|
+
__exportStar(require("../modules/playwright/env/playwright.env.validation"), exports);
|
|
21
|
+
__exportStar(require("../modules/playwright/env/playwright.env.zod"), exports);
|
|
22
|
+
//# sourceMappingURL=playwright.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../src/exports/playwright.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0EAAwD;AACxD,2EAAyD;AACzD,kFAAgE;AAChE,sFAAoE;AACpE,+EAA6D"}
|
|
@@ -9,6 +9,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
9
9
|
exports.CamoufoxModule = void 0;
|
|
10
10
|
const common_1 = require("@nestjs/common");
|
|
11
11
|
const camoufox_service_1 = require("./camoufox.service");
|
|
12
|
+
const camoufox_config_service_1 = require("./services/camoufox.config.service");
|
|
12
13
|
const camoufox_hook_service_1 = require("./services/camoufox.hook.service");
|
|
13
14
|
const camoufox_browser_service_1 = require("./services/camoufox.browser.service");
|
|
14
15
|
let CamoufoxModule = class CamoufoxModule {
|
|
@@ -16,7 +17,12 @@ let CamoufoxModule = class CamoufoxModule {
|
|
|
16
17
|
exports.CamoufoxModule = CamoufoxModule;
|
|
17
18
|
exports.CamoufoxModule = CamoufoxModule = __decorate([
|
|
18
19
|
(0, common_1.Module)({
|
|
19
|
-
providers: [
|
|
20
|
+
providers: [
|
|
21
|
+
camoufox_service_1.CamoufoxService,
|
|
22
|
+
camoufox_config_service_1.CamoufoxConfigService,
|
|
23
|
+
camoufox_hook_service_1.CamoufoxHookService,
|
|
24
|
+
camoufox_browser_service_1.CamoufoxBrowserService,
|
|
25
|
+
],
|
|
20
26
|
exports: [camoufox_service_1.CamoufoxService],
|
|
21
27
|
})
|
|
22
28
|
], CamoufoxModule);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"camoufox.module.js","sourceRoot":"","sources":["../../../src/modules/camoufox/camoufox.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,yDAAqD;AACrD,4EAAuE;AACvE,kFAA6E;
|
|
1
|
+
{"version":3,"file":"camoufox.module.js","sourceRoot":"","sources":["../../../src/modules/camoufox/camoufox.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,yDAAqD;AACrD,gFAA2E;AAC3E,4EAAuE;AACvE,kFAA6E;AAWtE,IAAM,cAAc,GAApB,MAAM,cAAc;CAAG,CAAA;AAAjB,wCAAc;yBAAd,cAAc;IAT1B,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,kCAAe;YACf,+CAAqB;YACrB,2CAAmB;YACnB,iDAAsB;SACvB;QACD,OAAO,EAAE,CAAC,kCAAe,CAAC;KAC3B,CAAC;GACW,cAAc,CAAG"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { ConfigService } from '@nestjs/config';
|
|
2
1
|
import { PlaywrightCrawler } from '@crawlee/playwright';
|
|
3
2
|
import { PlaywrightCrawlerOptions } from '@crawlee/playwright';
|
|
3
|
+
import { CamoufoxConfigService } from './services/camoufox.config.service';
|
|
4
4
|
import { CamoufoxHookService } from './services/camoufox.hook.service';
|
|
5
5
|
import { CamoufoxBrowserService } from './services/camoufox.browser.service';
|
|
6
6
|
export declare class CamoufoxService {
|
|
@@ -8,14 +8,7 @@ export declare class CamoufoxService {
|
|
|
8
8
|
private readonly hook;
|
|
9
9
|
private readonly browser;
|
|
10
10
|
private readonly logger;
|
|
11
|
-
|
|
12
|
-
private readonly timeoutSecs;
|
|
13
|
-
private readonly maxConcurrency;
|
|
14
|
-
private readonly minConcurrency;
|
|
15
|
-
private readonly maxRequestRetries;
|
|
16
|
-
private readonly headless;
|
|
17
|
-
private readonly useIncognitoPages;
|
|
18
|
-
constructor(config: ConfigService, hook: CamoufoxHookService, browser: CamoufoxBrowserService);
|
|
11
|
+
constructor(config: CamoufoxConfigService, hook: CamoufoxHookService, browser: CamoufoxBrowserService);
|
|
19
12
|
create(options: PlaywrightCrawlerOptions): Promise<PlaywrightCrawler>;
|
|
20
13
|
getBrowser(): Promise<import("playwright").Browser>;
|
|
21
14
|
}
|
|
@@ -12,31 +12,18 @@ var CamoufoxService_1;
|
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
exports.CamoufoxService = void 0;
|
|
14
14
|
const common_1 = require("@nestjs/common");
|
|
15
|
-
const config_1 = require("@nestjs/config");
|
|
16
15
|
const playwright_1 = require("@crawlee/playwright");
|
|
17
16
|
const playwright_2 = require("playwright");
|
|
17
|
+
const camoufox_config_service_1 = require("./services/camoufox.config.service");
|
|
18
18
|
const camoufox_hook_service_1 = require("./services/camoufox.hook.service");
|
|
19
19
|
const camoufox_browser_service_1 = require("./services/camoufox.browser.service");
|
|
20
20
|
const logger_1 = require("../../common/logger/logger");
|
|
21
|
-
const camoufox_env_constant_1 = require("./constant/camoufox.env.constant");
|
|
22
|
-
const camoufox_env_constant_2 = require("./constant/camoufox.env.constant");
|
|
23
|
-
const camoufox_env_constant_3 = require("./constant/camoufox.env.constant");
|
|
24
|
-
const camoufox_env_constant_4 = require("./constant/camoufox.env.constant");
|
|
25
|
-
const camoufox_env_constant_5 = require("./constant/camoufox.env.constant");
|
|
26
|
-
const camoufox_env_constant_6 = require("./constant/camoufox.env.constant");
|
|
27
21
|
let CamoufoxService = CamoufoxService_1 = class CamoufoxService {
|
|
28
22
|
constructor(config, hook, browser) {
|
|
29
23
|
this.config = config;
|
|
30
24
|
this.hook = hook;
|
|
31
25
|
this.browser = browser;
|
|
32
26
|
this.logger = new logger_1.Logger(CamoufoxService_1.name);
|
|
33
|
-
this.maxConcurrency = this.config.get(camoufox_env_constant_1.MAX_CONCURRENCY);
|
|
34
|
-
this.minConcurrency = this.config.get(camoufox_env_constant_2.MIN_CONCURRENCY);
|
|
35
|
-
this.maxRequestRetries = this.config.get(camoufox_env_constant_3.MAX_REQUEST_RETRIES);
|
|
36
|
-
this.timeoutSecs = this.config.get(camoufox_env_constant_4.TIMEOUT_SECS);
|
|
37
|
-
this.maxRequests = this.config.get(camoufox_env_constant_5.MAX_REQUESTS);
|
|
38
|
-
this.headless = this.config.get(camoufox_env_constant_6.HEADLESS);
|
|
39
|
-
this.useIncognitoPages = this.config.get(camoufox_env_constant_6.INCOGNITO_PAGES);
|
|
40
27
|
}
|
|
41
28
|
async create(options) {
|
|
42
29
|
this.logger.log('🚀 Creating Camoufox Crawler instance');
|
|
@@ -44,19 +31,20 @@ let CamoufoxService = CamoufoxService_1 = class CamoufoxService {
|
|
|
44
31
|
const defaultPreNavHooks = this.hook.getPreNavigationHooks();
|
|
45
32
|
const defaultPostNavHooks = this.hook.getPostNavigationHooks();
|
|
46
33
|
const launchOptions = await this.browser.getLaunchOptions();
|
|
34
|
+
const { env } = this.config;
|
|
47
35
|
const defaultOptions = {
|
|
48
|
-
maxRequestsPerCrawl:
|
|
49
|
-
requestHandlerTimeoutSecs:
|
|
50
|
-
navigationTimeoutSecs:
|
|
51
|
-
maxConcurrency:
|
|
52
|
-
minConcurrency:
|
|
53
|
-
maxRequestRetries:
|
|
36
|
+
maxRequestsPerCrawl: env.maxRequests,
|
|
37
|
+
requestHandlerTimeoutSecs: env.timeoutSecs,
|
|
38
|
+
navigationTimeoutSecs: env.timeoutSecs,
|
|
39
|
+
maxConcurrency: env.maxConcurrency,
|
|
40
|
+
minConcurrency: env.minConcurrency,
|
|
41
|
+
maxRequestRetries: env.maxRequestRetries,
|
|
54
42
|
useSessionPool: true,
|
|
55
|
-
headless:
|
|
43
|
+
headless: env.headless,
|
|
56
44
|
launchContext: {
|
|
57
45
|
launcher: playwright_2.firefox,
|
|
58
46
|
launchOptions,
|
|
59
|
-
useIncognitoPages:
|
|
47
|
+
useIncognitoPages: env.useIncognitoPages,
|
|
60
48
|
},
|
|
61
49
|
browserPoolOptions: {
|
|
62
50
|
useFingerprints: false,
|
|
@@ -80,7 +68,7 @@ let CamoufoxService = CamoufoxService_1 = class CamoufoxService {
|
|
|
80
68
|
exports.CamoufoxService = CamoufoxService;
|
|
81
69
|
exports.CamoufoxService = CamoufoxService = CamoufoxService_1 = __decorate([
|
|
82
70
|
(0, common_1.Injectable)(),
|
|
83
|
-
__metadata("design:paramtypes", [
|
|
71
|
+
__metadata("design:paramtypes", [camoufox_config_service_1.CamoufoxConfigService,
|
|
84
72
|
camoufox_hook_service_1.CamoufoxHookService,
|
|
85
73
|
camoufox_browser_service_1.CamoufoxBrowserService])
|
|
86
74
|
], CamoufoxService);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"camoufox.service.js","sourceRoot":"","sources":["../../../src/modules/camoufox/camoufox.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;
|
|
1
|
+
{"version":3,"file":"camoufox.service.js","sourceRoot":"","sources":["../../../src/modules/camoufox/camoufox.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAE5C,oDAAwD;AAExD,2CAAqC;AAErC,gFAA2E;AAC3E,4EAAuE;AACvE,kFAA6E;AAE7E,uDAAoD;AAG7C,IAAM,eAAe,uBAArB,MAAM,eAAe;IAG1B,YACmB,MAA6B,EAC7B,IAAyB,EACzB,OAA+B;QAF/B,WAAM,GAAN,MAAM,CAAuB;QAC7B,SAAI,GAAJ,IAAI,CAAqB;QACzB,YAAO,GAAP,OAAO,CAAwB;QALjC,WAAM,GAAG,IAAI,eAAM,CAAC,iBAAe,CAAC,IAAI,CAAC,CAAC;IAMxD,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,OAAiC;QAC5C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QACzD,MAAM,EAAE,kBAAkB,GAAG,EAAE,EAAE,mBAAmB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/D,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC5D,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5B,MAAM,cAAc,GAA6B;YAC/C,mBAAmB,EAAE,GAAG,CAAC,WAAW;YACpC,yBAAyB,EAAE,GAAG,CAAC,WAAW;YAC1C,qBAAqB,EAAE,GAAG,CAAC,WAAW;YACtC,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;YACxC,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE;gBACb,QAAQ,EAAE,oBAAO;gBACjB,aAAa;gBACb,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;aACzC;YACD,kBAAkB,EAAE;gBAClB,eAAe,EAAE,KAAK;gBACtB,2BAA2B,EAAE,EAAE;gBAC/B,8BAA8B,EAAE,EAAE;aACnC;SACF,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,8BAAiB,CAAC;YACpC,GAAG,cAAc;YACjB,GAAG,OAAO;YACV,kBAAkB,EAAE,CAAC,GAAG,kBAAkB,EAAE,GAAG,kBAAkB,CAAC;YAClE,mBAAmB,EAAE,CAAC,GAAG,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;SACtE,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QACtE,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;IACnC,CAAC;CACF,CAAA;AAjDY,0CAAe;0BAAf,eAAe;IAD3B,IAAA,mBAAU,GAAE;qCAKgB,+CAAqB;QACvB,2CAAmB;QAChB,iDAAsB;GANvC,eAAe,CAiD3B"}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
export declare const
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
1
|
+
export declare const CAMOUFOX_ENV: {
|
|
2
|
+
readonly MIN_CONCURRENCY: "CRAWLEE_CAMOUFOX_MIN_CONCURRENCY";
|
|
3
|
+
readonly MAX_CONCURRENCY: "CRAWLEE_CAMOUFOX_MAX_CONCURRENCY";
|
|
4
|
+
readonly MAX_REQUEST_RETRIES: "CRAWLEE_CAMOUFOX_MAX_REQUEST_RETRIES";
|
|
5
|
+
readonly TIMEOUT_SECS: "CRAWLEE_CAMOUFOX_TIMEOUT_SECS";
|
|
6
|
+
readonly MAX_REQUESTS: "CRAWLEE_CAMOUFOX_MAX_REQUESTS";
|
|
7
|
+
readonly INITIAL_PAGE: "CRAWLEE_CAMOUFOX_INITIAL_PAGE";
|
|
8
|
+
readonly HEADLESS: "CRAWLEE_CAMOUFOX_HEADLESS";
|
|
9
|
+
readonly EXECUTABLE_PATH: "CRAWLEE_CAMOUFOX_EXECUTABLE_PATH";
|
|
10
|
+
readonly HANDLE_CLOUDFLARE_CHALLENGE: "CRAWLEE_CAMOUFOX_HANDLE_CLOUDFLARE_CHALLENGE";
|
|
11
|
+
readonly INCOGNITO_PAGES: "CRAWLEE_CAMOUFOX_USE_INCOGNITO_PAGES";
|
|
12
|
+
};
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
exports.
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
3
|
+
exports.CAMOUFOX_ENV = void 0;
|
|
4
|
+
exports.CAMOUFOX_ENV = {
|
|
5
|
+
MIN_CONCURRENCY: 'CRAWLEE_CAMOUFOX_MIN_CONCURRENCY',
|
|
6
|
+
MAX_CONCURRENCY: 'CRAWLEE_CAMOUFOX_MAX_CONCURRENCY',
|
|
7
|
+
MAX_REQUEST_RETRIES: 'CRAWLEE_CAMOUFOX_MAX_REQUEST_RETRIES',
|
|
8
|
+
TIMEOUT_SECS: 'CRAWLEE_CAMOUFOX_TIMEOUT_SECS',
|
|
9
|
+
MAX_REQUESTS: 'CRAWLEE_CAMOUFOX_MAX_REQUESTS',
|
|
10
|
+
INITIAL_PAGE: 'CRAWLEE_CAMOUFOX_INITIAL_PAGE',
|
|
11
|
+
HEADLESS: 'CRAWLEE_CAMOUFOX_HEADLESS',
|
|
12
|
+
EXECUTABLE_PATH: 'CRAWLEE_CAMOUFOX_EXECUTABLE_PATH',
|
|
13
|
+
HANDLE_CLOUDFLARE_CHALLENGE: 'CRAWLEE_CAMOUFOX_HANDLE_CLOUDFLARE_CHALLENGE',
|
|
14
|
+
INCOGNITO_PAGES: 'CRAWLEE_CAMOUFOX_USE_INCOGNITO_PAGES',
|
|
15
|
+
};
|
|
14
16
|
//# sourceMappingURL=camoufox.env.constant.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"camoufox.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/constant/camoufox.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,
|
|
1
|
+
{"version":3,"file":"camoufox.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/constant/camoufox.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,YAAY,GAAG;IAC1B,eAAe,EAAE,kCAAkC;IACnD,eAAe,EAAE,kCAAkC;IACnD,mBAAmB,EAAE,sCAAsC;IAC3D,YAAY,EAAE,+BAA+B;IAC7C,YAAY,EAAE,+BAA+B;IAC7C,YAAY,EAAE,+BAA+B;IAC7C,QAAQ,EAAE,2BAA2B;IACrC,eAAe,EAAE,kCAAkC;IACnD,2BAA2B,EAAE,8CAA8C;IAC3E,eAAe,EAAE,sCAAsC;CAC/C,CAAC"}
|
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import { ConfigService } from '@nestjs/config';
|
|
2
1
|
import { Browser } from 'playwright';
|
|
2
|
+
import { CamoufoxConfigService } from './camoufox.config.service';
|
|
3
3
|
export declare class CamoufoxBrowserService {
|
|
4
4
|
private readonly config;
|
|
5
5
|
private readonly logger;
|
|
6
|
-
|
|
7
|
-
private readonly path?;
|
|
8
|
-
constructor(config: ConfigService);
|
|
6
|
+
constructor(config: CamoufoxConfigService);
|
|
9
7
|
getLaunchOptions(): Promise<Record<string, any>>;
|
|
10
8
|
getBrowser(): Promise<Browser>;
|
|
11
9
|
}
|
|
@@ -11,25 +11,22 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.CamoufoxBrowserService = void 0;
|
|
13
13
|
const common_1 = require("@nestjs/common");
|
|
14
|
-
const config_1 = require("@nestjs/config");
|
|
15
14
|
const playwright_1 = require("playwright");
|
|
16
15
|
const camoufox_js_1 = require("camoufox-js");
|
|
17
16
|
const camoufox_service_1 = require("../camoufox.service");
|
|
18
17
|
const logger_1 = require("../../../common/logger/logger");
|
|
19
|
-
const
|
|
18
|
+
const camoufox_config_service_1 = require("./camoufox.config.service");
|
|
20
19
|
let CamoufoxBrowserService = class CamoufoxBrowserService {
|
|
21
20
|
constructor(config) {
|
|
22
21
|
this.config = config;
|
|
23
22
|
this.logger = new logger_1.Logger(camoufox_service_1.CamoufoxService.name);
|
|
24
|
-
this.headless = this.config.get(camoufox_env_constant_1.HEADLESS);
|
|
25
|
-
this.path = this.config.get(camoufox_env_constant_1.EXECUTABLE_PATH);
|
|
26
23
|
}
|
|
27
24
|
async getLaunchOptions() {
|
|
28
25
|
const options = await (0, camoufox_js_1.launchOptions)({
|
|
29
|
-
headless: this.headless,
|
|
26
|
+
headless: this.config.headless,
|
|
30
27
|
});
|
|
31
|
-
if (this.
|
|
32
|
-
options.executablePath = this.
|
|
28
|
+
if (this.config.executablePath) {
|
|
29
|
+
options.executablePath = this.config.executablePath;
|
|
33
30
|
}
|
|
34
31
|
return options;
|
|
35
32
|
}
|
|
@@ -44,6 +41,6 @@ let CamoufoxBrowserService = class CamoufoxBrowserService {
|
|
|
44
41
|
exports.CamoufoxBrowserService = CamoufoxBrowserService;
|
|
45
42
|
exports.CamoufoxBrowserService = CamoufoxBrowserService = __decorate([
|
|
46
43
|
(0, common_1.Injectable)(),
|
|
47
|
-
__metadata("design:paramtypes", [
|
|
44
|
+
__metadata("design:paramtypes", [camoufox_config_service_1.CamoufoxConfigService])
|
|
48
45
|
], CamoufoxBrowserService);
|
|
49
46
|
//# sourceMappingURL=camoufox.browser.service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"camoufox.browser.service.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/services/camoufox.browser.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;
|
|
1
|
+
{"version":3,"file":"camoufox.browser.service.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/services/camoufox.browser.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAE5C,2CAA8C;AAC9C,6CAA4C;AAE5C,0DAAsD;AAEtD,0DAAuD;AACvD,uEAAkE;AAG3D,IAAM,sBAAsB,GAA5B,MAAM,sBAAsB;IAGjC,YAA6B,MAA6B;QAA7B,WAAM,GAAN,MAAM,CAAuB;QAFzC,WAAM,GAAG,IAAI,eAAM,CAAC,kCAAe,CAAC,IAAI,CAAC,CAAC;IAEE,CAAC;IAE9D,KAAK,CAAC,gBAAgB;QACpB,MAAM,OAAO,GAAG,MAAM,IAAA,2BAAa,EAAC;YAClC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;SAC/B,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAC/B,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;QACtD,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,oBAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QACtE,OAAO,OAAO,CAAC;IACjB,CAAC;CACF,CAAA;AAtBY,wDAAsB;iCAAtB,sBAAsB;IADlC,IAAA,mBAAU,GAAE;qCAI0B,+CAAqB;GAH/C,sBAAsB,CAsBlC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { ConfigService } from '@nestjs/config';
|
|
2
|
+
import { CamoufoxConfig } from '../types/camoufox.config';
|
|
3
|
+
export declare class CamoufoxConfigService {
|
|
4
|
+
private readonly service;
|
|
5
|
+
private readonly _env;
|
|
6
|
+
constructor(service: ConfigService);
|
|
7
|
+
get env(): CamoufoxConfig;
|
|
8
|
+
get maxRequests(): number;
|
|
9
|
+
get timeoutSecs(): number;
|
|
10
|
+
get maxConcurrency(): number;
|
|
11
|
+
get minConcurrency(): number;
|
|
12
|
+
get maxRequestRetries(): number;
|
|
13
|
+
get headless(): boolean;
|
|
14
|
+
get useIncognitoPages(): boolean;
|
|
15
|
+
get initialPage(): number;
|
|
16
|
+
get executablePath(): string;
|
|
17
|
+
get handleCloudflareChallenge(): boolean;
|
|
18
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.CamoufoxConfigService = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const config_1 = require("@nestjs/config");
|
|
15
|
+
const camoufox_env_constant_1 = require("./../constant/camoufox.env.constant");
|
|
16
|
+
let CamoufoxConfigService = class CamoufoxConfigService {
|
|
17
|
+
constructor(service) {
|
|
18
|
+
this.service = service;
|
|
19
|
+
this._env = {
|
|
20
|
+
maxConcurrency: this.service.get(camoufox_env_constant_1.CAMOUFOX_ENV.MAX_CONCURRENCY),
|
|
21
|
+
minConcurrency: this.service.get(camoufox_env_constant_1.CAMOUFOX_ENV.MIN_CONCURRENCY),
|
|
22
|
+
maxRequestRetries: this.service.get(camoufox_env_constant_1.CAMOUFOX_ENV.MAX_REQUEST_RETRIES),
|
|
23
|
+
timeoutSecs: this.service.get(camoufox_env_constant_1.CAMOUFOX_ENV.TIMEOUT_SECS),
|
|
24
|
+
maxRequests: this.service.get(camoufox_env_constant_1.CAMOUFOX_ENV.MAX_REQUESTS),
|
|
25
|
+
initialPage: this.service.get(camoufox_env_constant_1.CAMOUFOX_ENV.INITIAL_PAGE),
|
|
26
|
+
headless: this.service.get(camoufox_env_constant_1.CAMOUFOX_ENV.HEADLESS),
|
|
27
|
+
executablePath: this.service.get(camoufox_env_constant_1.CAMOUFOX_ENV.EXECUTABLE_PATH),
|
|
28
|
+
handleCloudflareChallenge: this.service.get(camoufox_env_constant_1.CAMOUFOX_ENV.HANDLE_CLOUDFLARE_CHALLENGE),
|
|
29
|
+
useIncognitoPages: this.service.get(camoufox_env_constant_1.CAMOUFOX_ENV.INCOGNITO_PAGES),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
get env() {
|
|
33
|
+
return this._env;
|
|
34
|
+
}
|
|
35
|
+
get maxRequests() {
|
|
36
|
+
return Number(this._env.maxRequests);
|
|
37
|
+
}
|
|
38
|
+
get timeoutSecs() {
|
|
39
|
+
return Number(this._env.timeoutSecs);
|
|
40
|
+
}
|
|
41
|
+
get maxConcurrency() {
|
|
42
|
+
return Number(this._env.maxConcurrency);
|
|
43
|
+
}
|
|
44
|
+
get minConcurrency() {
|
|
45
|
+
return Number(this._env.minConcurrency);
|
|
46
|
+
}
|
|
47
|
+
get maxRequestRetries() {
|
|
48
|
+
return Number(this._env.maxRequestRetries);
|
|
49
|
+
}
|
|
50
|
+
get headless() {
|
|
51
|
+
return Boolean(this._env.headless);
|
|
52
|
+
}
|
|
53
|
+
get useIncognitoPages() {
|
|
54
|
+
return Boolean(this._env.useIncognitoPages);
|
|
55
|
+
}
|
|
56
|
+
get initialPage() {
|
|
57
|
+
return Number(this._env.initialPage);
|
|
58
|
+
}
|
|
59
|
+
get executablePath() {
|
|
60
|
+
return String(this._env.executablePath);
|
|
61
|
+
}
|
|
62
|
+
get handleCloudflareChallenge() {
|
|
63
|
+
return Boolean(this._env.handleCloudflareChallenge);
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
exports.CamoufoxConfigService = CamoufoxConfigService;
|
|
67
|
+
exports.CamoufoxConfigService = CamoufoxConfigService = __decorate([
|
|
68
|
+
(0, common_1.Injectable)(),
|
|
69
|
+
__metadata("design:paramtypes", [config_1.ConfigService])
|
|
70
|
+
], CamoufoxConfigService);
|
|
71
|
+
//# sourceMappingURL=camoufox.config.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"camoufox.config.service.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/services/camoufox.config.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAG/C,+EAA0E;AAGnE,IAAM,qBAAqB,GAA3B,MAAM,qBAAqB;IAGhC,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;QACjD,IAAI,CAAC,IAAI,GAAG;YACV,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,oCAAG,CAAC,eAAe,CAAC;YAC7D,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,oCAAG,CAAC,eAAe,CAAC;YAC7D,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,oCAAG,CAAC,mBAAmB,CAAC;YACpE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,oCAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,oCAAG,CAAC,YAAY,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,oCAAG,CAAC,YAAY,CAAC;YACvD,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAU,oCAAG,CAAC,QAAQ,CAAC;YACjD,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAS,oCAAG,CAAC,eAAe,CAAC;YAC7D,yBAAyB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CACzC,oCAAG,CAAC,2BAA2B,CAChC;YACD,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAU,oCAAG,CAAC,eAAe,CAAC;SAClE,CAAC;IACJ,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,WAAW;QACb,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,yBAAyB;QAC3B,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACtD,CAAC;CACF,CAAA;AA/DY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,mBAAU,GAAE;qCAI2B,sBAAa;GAHxC,qBAAqB,CA+DjC"}
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { ConfigService } from '@nestjs/config';
|
|
2
1
|
import { PlaywrightHook } from '@crawlee/playwright';
|
|
2
|
+
import { CamoufoxConfigService } from './camoufox.config.service';
|
|
3
3
|
export declare class CamoufoxHookService {
|
|
4
4
|
private readonly config;
|
|
5
5
|
private readonly logger;
|
|
6
|
-
|
|
7
|
-
constructor(config: ConfigService);
|
|
6
|
+
constructor(config: CamoufoxConfigService);
|
|
8
7
|
getPreNavigationHooks(): PlaywrightHook[];
|
|
9
8
|
getPostNavigationHooks(): PlaywrightHook[];
|
|
10
9
|
}
|
|
@@ -11,16 +11,13 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.CamoufoxHookService = void 0;
|
|
13
13
|
const common_1 = require("@nestjs/common");
|
|
14
|
-
const config_1 = require("@nestjs/config");
|
|
15
14
|
const logger_1 = require("../../../common/logger/logger");
|
|
16
15
|
const camoufox_service_1 = require("../camoufox.service");
|
|
17
|
-
const
|
|
16
|
+
const camoufox_config_service_1 = require("./camoufox.config.service");
|
|
18
17
|
let CamoufoxHookService = class CamoufoxHookService {
|
|
19
18
|
constructor(config) {
|
|
20
19
|
this.config = config;
|
|
21
20
|
this.logger = new logger_1.Logger(camoufox_service_1.CamoufoxService.name);
|
|
22
|
-
const enabled = this.config.get(camoufox_env_constant_1.HANDLE_CLOUDFLARE_CHALLENGE, false);
|
|
23
|
-
this.shouldHandleCloudflareChallenge = enabled;
|
|
24
21
|
}
|
|
25
22
|
getPreNavigationHooks() {
|
|
26
23
|
return [
|
|
@@ -32,7 +29,7 @@ let CamoufoxHookService = class CamoufoxHookService {
|
|
|
32
29
|
getPostNavigationHooks() {
|
|
33
30
|
return [
|
|
34
31
|
async ({ handleCloudflareChallenge }) => {
|
|
35
|
-
if (this.
|
|
32
|
+
if (this.config.handleCloudflareChallenge) {
|
|
36
33
|
await handleCloudflareChallenge();
|
|
37
34
|
}
|
|
38
35
|
},
|
|
@@ -75,6 +72,6 @@ let CamoufoxHookService = class CamoufoxHookService {
|
|
|
75
72
|
exports.CamoufoxHookService = CamoufoxHookService;
|
|
76
73
|
exports.CamoufoxHookService = CamoufoxHookService = __decorate([
|
|
77
74
|
(0, common_1.Injectable)(),
|
|
78
|
-
__metadata("design:paramtypes", [
|
|
75
|
+
__metadata("design:paramtypes", [camoufox_config_service_1.CamoufoxConfigService])
|
|
79
76
|
], CamoufoxHookService);
|
|
80
77
|
//# sourceMappingURL=camoufox.hook.service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"camoufox.hook.service.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/services/camoufox.hook.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAAwD;
|
|
1
|
+
{"version":3,"file":"camoufox.hook.service.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/services/camoufox.hook.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAAwD;AAIxD,0DAAuD;AACvD,0DAAsD;AACtD,uEAAkE;AAG3D,IAAM,mBAAmB,GAAzB,MAAM,mBAAmB;IAG9B,YAA6B,MAA6B;QAA7B,WAAM,GAAN,MAAM,CAAuB;QAFzC,WAAM,GAAG,IAAI,eAAM,CAAC,kCAAe,CAAC,IAAI,CAAC,CAAC;IAEE,CAAC;IAE9D,qBAAqB;QACnB,OAAO;YACL,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACvC,CAAC;SACF,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,OAAO;YACL,KAAK,EAAE,EAAE,yBAAyB,EAAE,EAAE,EAAE;gBACtC,IAAI,IAAI,CAAC,MAAM,CAAC,yBAAyB,EAAE,CAAC;oBAC1C,MAAM,yBAAyB,EAAE,CAAC;gBACpC,CAAC;YACH,CAAC;YACD,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE;gBACxB,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;gBACxB,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAErC,QAAQ,UAAwB,EAAE,CAAC;oBACjC,KAAK,mBAAU,CAAC,EAAE;wBAChB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC1C,MAAM;oBACR,KAAK,mBAAU,CAAC,WAAW;wBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,YAAY;wBAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,SAAS;wBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,iBAAiB;wBAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC3C,MAAM;oBACR,KAAK,mBAAU,CAAC,qBAAqB;wBACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,WAAW;wBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,mBAAmB;wBACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR;wBACE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC3C,MAAM;gBACV,CAAC;YACH,CAAC;SACF,CAAC;IACJ,CAAC;CACF,CAAA;AAxDY,kDAAmB;8BAAnB,mBAAmB;IAD/B,IAAA,mBAAU,GAAE;qCAI0B,+CAAqB;GAH/C,mBAAmB,CAwD/B"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export type CamoufoxConfig = {
|
|
2
|
+
maxRequests: number;
|
|
3
|
+
timeoutSecs: number;
|
|
4
|
+
maxConcurrency: number;
|
|
5
|
+
minConcurrency: number;
|
|
6
|
+
maxRequestRetries: number;
|
|
7
|
+
initialPage: number;
|
|
8
|
+
headless: boolean;
|
|
9
|
+
executablePath: string;
|
|
10
|
+
handleCloudflareChallenge: boolean;
|
|
11
|
+
useIncognitoPages: boolean;
|
|
12
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"camoufox.config.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/types/camoufox.config.ts"],"names":[],"mappings":""}
|
|
@@ -9,13 +9,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
9
9
|
exports.CheerioModule = void 0;
|
|
10
10
|
const common_1 = require("@nestjs/common");
|
|
11
11
|
const cheerio_service_1 = require("./cheerio.service");
|
|
12
|
+
const cheerio_config_service_1 = require("./services/cheerio.config.service");
|
|
12
13
|
const cheerio_hook_service_1 = require("./services/cheerio.hook.service");
|
|
13
14
|
let CheerioModule = class CheerioModule {
|
|
14
15
|
};
|
|
15
16
|
exports.CheerioModule = CheerioModule;
|
|
16
17
|
exports.CheerioModule = CheerioModule = __decorate([
|
|
17
18
|
(0, common_1.Module)({
|
|
18
|
-
providers: [cheerio_service_1.CheerioService, cheerio_hook_service_1.CheerioHookService],
|
|
19
|
+
providers: [cheerio_service_1.CheerioService, cheerio_config_service_1.CheerioConfigService, cheerio_hook_service_1.CheerioHookService],
|
|
19
20
|
exports: [cheerio_service_1.CheerioService],
|
|
20
21
|
})
|
|
21
22
|
], CheerioModule);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.module.js","sourceRoot":"","sources":["../../../src/modules/cheerio/cheerio.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,uDAAmD;AACnD,0EAAqE;AAM9D,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,sCAAa;wBAAb,aAAa;IAJzB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE,CAAC,gCAAc,EAAE,yCAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"cheerio.module.js","sourceRoot":"","sources":["../../../src/modules/cheerio/cheerio.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,uDAAmD;AACnD,8EAAyE;AACzE,0EAAqE;AAM9D,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,sCAAa;wBAAb,aAAa;IAJzB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE,CAAC,gCAAc,EAAE,6CAAoB,EAAE,yCAAkB,CAAC;QACrE,OAAO,EAAE,CAAC,gCAAc,CAAC;KAC1B,CAAC;GACW,aAAa,CAAG"}
|