@rafikidota/scoutee 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/camoufox/camoufox.module.d.ts +2 -0
- package/dist/modules/camoufox/camoufox.module.js +32 -0
- package/dist/modules/camoufox/camoufox.module.js.map +1 -0
- package/dist/modules/camoufox/camoufox.service.d.ts +19 -0
- package/dist/modules/camoufox/camoufox.service.js +74 -0
- package/dist/modules/camoufox/camoufox.service.js.map +1 -0
- package/dist/modules/camoufox/constant/camoufox.env.constant.d.ts +7 -0
- package/dist/modules/camoufox/constant/camoufox.env.constant.js +11 -0
- package/dist/modules/camoufox/constant/camoufox.env.constant.js.map +1 -0
- package/dist/modules/camoufox/env/camoufox.env.config.d.ts +9 -0
- package/dist/modules/camoufox/env/camoufox.env.config.js +49 -0
- package/dist/modules/camoufox/env/camoufox.env.config.js.map +1 -0
- package/dist/modules/camoufox/env/camoufox.env.validation.d.ts +2 -0
- package/dist/modules/camoufox/env/camoufox.env.validation.js +47 -0
- package/dist/modules/camoufox/env/camoufox.env.validation.js.map +1 -0
- package/dist/modules/camoufox/services/camoufox.browser.service.d.ts +10 -0
- package/dist/modules/camoufox/services/camoufox.browser.service.js +40 -0
- package/dist/modules/camoufox/services/camoufox.browser.service.js.map +1 -0
- package/dist/modules/camoufox/services/camoufox.hook.service.d.ts +8 -0
- package/dist/modules/camoufox/services/camoufox.hook.service.js +72 -0
- package/dist/modules/camoufox/services/camoufox.hook.service.js.map +1 -0
- package/dist/modules/camoufox/services/camoufox.logger.service.d.ts +11 -0
- package/dist/modules/camoufox/services/camoufox.logger.service.js +44 -0
- package/dist/modules/camoufox/services/camoufox.logger.service.js.map +1 -0
- package/dist/modules/playwright/constant/playwright.browser.constant.d.ts +1 -0
- package/dist/modules/playwright/constant/playwright.browser.constant.js +13 -0
- package/dist/modules/playwright/constant/playwright.browser.constant.js.map +1 -0
- package/dist/modules/playwright/constant/playwright.env.constant.d.ts +1 -0
- package/dist/modules/playwright/constant/playwright.env.constant.js +2 -1
- package/dist/modules/playwright/constant/playwright.env.constant.js.map +1 -1
- package/dist/modules/playwright/env/playwright.env.config.d.ts +1 -0
- package/dist/modules/playwright/env/playwright.env.config.js +1 -0
- package/dist/modules/playwright/env/playwright.env.config.js.map +1 -1
- package/dist/modules/playwright/env/playwright.env.validation.js +1 -0
- package/dist/modules/playwright/env/playwright.env.validation.js.map +1 -1
- package/dist/modules/playwright/playwright.module.js +2 -0
- package/dist/modules/playwright/playwright.module.js.map +1 -1
- package/dist/modules/playwright/playwright.service.d.ts +5 -4
- package/dist/modules/playwright/playwright.service.js +11 -21
- package/dist/modules/playwright/playwright.service.js.map +1 -1
- package/dist/modules/playwright/services/playwright.browser.service.d.ts +10 -0
- package/dist/modules/playwright/services/playwright.browser.service.js +41 -0
- package/dist/modules/playwright/services/playwright.browser.service.js.map +1 -0
- package/package.json +3 -1
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.CamoufoxModule = void 0;
|
|
10
|
+
const common_1 = require("@nestjs/common");
|
|
11
|
+
const config_1 = require("@nestjs/config");
|
|
12
|
+
const camoufox_service_1 = require("./camoufox.service");
|
|
13
|
+
const camoufox_env_config_1 = require("./env/camoufox.env.config");
|
|
14
|
+
const camoufox_logger_service_1 = require("./services/camoufox.logger.service");
|
|
15
|
+
const camoufox_hook_service_1 = require("./services/camoufox.hook.service");
|
|
16
|
+
const camoufox_browser_service_1 = require("./services/camoufox.browser.service");
|
|
17
|
+
let CamoufoxModule = class CamoufoxModule {
|
|
18
|
+
};
|
|
19
|
+
exports.CamoufoxModule = CamoufoxModule;
|
|
20
|
+
exports.CamoufoxModule = CamoufoxModule = __decorate([
|
|
21
|
+
(0, common_1.Module)({
|
|
22
|
+
imports: [config_1.ConfigModule.forFeature(camoufox_env_config_1.CamoufoxEnvConfig)],
|
|
23
|
+
providers: [
|
|
24
|
+
camoufox_service_1.CamoufoxService,
|
|
25
|
+
camoufox_logger_service_1.CamoufoxLoggerService,
|
|
26
|
+
camoufox_hook_service_1.CamoufoxHookService,
|
|
27
|
+
camoufox_browser_service_1.CamoufoxBrowserService,
|
|
28
|
+
],
|
|
29
|
+
exports: [camoufox_service_1.CamoufoxService],
|
|
30
|
+
})
|
|
31
|
+
], CamoufoxModule);
|
|
32
|
+
//# sourceMappingURL=camoufox.module.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"camoufox.module.js","sourceRoot":"","sources":["../../../src/modules/camoufox/camoufox.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AACxC,2CAA8C;AAE9C,yDAAqD;AACrD,mEAA8D;AAC9D,gFAA2E;AAC3E,4EAAuE;AACvE,kFAA6E;AAYtE,IAAM,cAAc,GAApB,MAAM,cAAc;CAAG,CAAA;AAAjB,wCAAc;yBAAd,cAAc;IAV1B,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,qBAAY,CAAC,UAAU,CAAC,uCAAiB,CAAC,CAAC;QACrD,SAAS,EAAE;YACT,kCAAe;YACf,+CAAqB;YACrB,2CAAmB;YACnB,iDAAsB;SACvB;QACD,OAAO,EAAE,CAAC,kCAAe,CAAC;KAC3B,CAAC;GACW,cAAc,CAAG"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { ConfigService } from '@nestjs/config';
|
|
2
|
+
import { PlaywrightCrawler } from '@crawlee/playwright';
|
|
3
|
+
import { PlaywrightCrawlerOptions } from '@crawlee/playwright';
|
|
4
|
+
import { CamoufoxLoggerService } from './services/camoufox.logger.service';
|
|
5
|
+
import { CamoufoxBrowserService } from './services/camoufox.browser.service';
|
|
6
|
+
export declare class CamoufoxService {
|
|
7
|
+
private readonly config;
|
|
8
|
+
private readonly logger;
|
|
9
|
+
private readonly browser;
|
|
10
|
+
private readonly maxRequests;
|
|
11
|
+
private readonly timeoutSecs;
|
|
12
|
+
private readonly maxConcurrency;
|
|
13
|
+
private readonly minConcurrency;
|
|
14
|
+
private readonly maxRequestRetries;
|
|
15
|
+
private readonly headless;
|
|
16
|
+
constructor(config: ConfigService, logger: CamoufoxLoggerService, browser: CamoufoxBrowserService);
|
|
17
|
+
create(options: PlaywrightCrawlerOptions): Promise<PlaywrightCrawler>;
|
|
18
|
+
getBrowser(): Promise<import("playwright").Browser>;
|
|
19
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.CamoufoxService = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const config_1 = require("@nestjs/config");
|
|
15
|
+
const playwright_1 = require("@crawlee/playwright");
|
|
16
|
+
const camoufox_js_1 = require("camoufox-js");
|
|
17
|
+
const playwright_2 = require("playwright");
|
|
18
|
+
const camoufox_logger_service_1 = require("./services/camoufox.logger.service");
|
|
19
|
+
const camoufox_browser_service_1 = require("./services/camoufox.browser.service");
|
|
20
|
+
const camoufox_env_constant_1 = require("./constant/camoufox.env.constant");
|
|
21
|
+
const camoufox_env_constant_2 = require("./constant/camoufox.env.constant");
|
|
22
|
+
const camoufox_env_constant_3 = require("./constant/camoufox.env.constant");
|
|
23
|
+
const camoufox_env_constant_4 = require("./constant/camoufox.env.constant");
|
|
24
|
+
const camoufox_env_constant_5 = require("./constant/camoufox.env.constant");
|
|
25
|
+
const camoufox_env_constant_6 = require("./constant/camoufox.env.constant");
|
|
26
|
+
let CamoufoxService = class CamoufoxService {
|
|
27
|
+
constructor(config, logger, browser) {
|
|
28
|
+
this.config = config;
|
|
29
|
+
this.logger = logger;
|
|
30
|
+
this.browser = browser;
|
|
31
|
+
this.maxConcurrency = this.config.get(camoufox_env_constant_1.MAX_CONCURRENCY);
|
|
32
|
+
this.minConcurrency = this.config.get(camoufox_env_constant_2.MIN_CONCURRENCY);
|
|
33
|
+
this.maxRequestRetries = this.config.get(camoufox_env_constant_3.MAX_REQUEST_RETRIES);
|
|
34
|
+
this.timeoutSecs = this.config.get(camoufox_env_constant_4.TIMEOUT_SECS);
|
|
35
|
+
this.maxRequests = this.config.get(camoufox_env_constant_5.MAX_REQUESTS);
|
|
36
|
+
this.headless = this.config.get(camoufox_env_constant_6.HEADLESS);
|
|
37
|
+
}
|
|
38
|
+
async create(options) {
|
|
39
|
+
this.logger.log('🚀 Creating Camoufox Crawler instance');
|
|
40
|
+
const defaultOptions = {
|
|
41
|
+
maxRequestsPerCrawl: this.maxRequests,
|
|
42
|
+
requestHandlerTimeoutSecs: this.timeoutSecs,
|
|
43
|
+
navigationTimeoutSecs: this.timeoutSecs,
|
|
44
|
+
maxConcurrency: this.maxConcurrency,
|
|
45
|
+
minConcurrency: this.minConcurrency,
|
|
46
|
+
maxRequestRetries: this.maxRequestRetries,
|
|
47
|
+
useSessionPool: true,
|
|
48
|
+
headless: this.headless,
|
|
49
|
+
launchContext: {
|
|
50
|
+
launcher: playwright_2.firefox,
|
|
51
|
+
launchOptions: await (0, camoufox_js_1.launchOptions)({
|
|
52
|
+
headless: this.headless,
|
|
53
|
+
}),
|
|
54
|
+
},
|
|
55
|
+
browserPoolOptions: {
|
|
56
|
+
useFingerprints: false,
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
const crawler = new playwright_1.PlaywrightCrawler({ ...defaultOptions, ...options });
|
|
60
|
+
this.logger.log('✅ Camoufox Crawler instance created successfully');
|
|
61
|
+
return crawler;
|
|
62
|
+
}
|
|
63
|
+
async getBrowser() {
|
|
64
|
+
return this.browser.getBrowser();
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
exports.CamoufoxService = CamoufoxService;
|
|
68
|
+
exports.CamoufoxService = CamoufoxService = __decorate([
|
|
69
|
+
(0, common_1.Injectable)(),
|
|
70
|
+
__metadata("design:paramtypes", [config_1.ConfigService,
|
|
71
|
+
camoufox_logger_service_1.CamoufoxLoggerService,
|
|
72
|
+
camoufox_browser_service_1.CamoufoxBrowserService])
|
|
73
|
+
], CamoufoxService);
|
|
74
|
+
//# sourceMappingURL=camoufox.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"camoufox.service.js","sourceRoot":"","sources":["../../../src/modules/camoufox/camoufox.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAE/C,oDAAwD;AAExD,6CAA4C;AAC5C,2CAAqC;AAErC,gFAA2E;AAC3E,kFAA6E;AAE7E,4EAAmE;AACnE,4EAAmE;AACnE,4EAAuE;AACvE,4EAAgE;AAChE,4EAAgE;AAChE,4EAA4D;AAGrD,IAAM,eAAe,GAArB,MAAM,eAAe;IAQ1B,YACmB,MAAqB,EACrB,MAA6B,EAC7B,OAA+B;QAF/B,WAAM,GAAN,MAAM,CAAe;QACrB,WAAM,GAAN,MAAM,CAAuB;QAC7B,YAAO,GAAP,OAAO,CAAwB;QAEhD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,uCAAe,CAAC,CAAC;QAC/D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,uCAAe,CAAC,CAAC;QAC/D,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,2CAAmB,CAAC,CAAC;QACtE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,oCAAY,CAAC,CAAC;QACzD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,oCAAY,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAU,gCAAQ,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAiC;QAC5C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QACzD,MAAM,cAAc,GAA6B;YAC/C,mBAAmB,EAAE,IAAI,CAAC,WAAW;YACrC,yBAAyB,EAAE,IAAI,CAAC,WAAW;YAC3C,qBAAqB,EAAE,IAAI,CAAC,WAAW;YACvC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE;gBACb,QAAQ,EAAE,oBAAO;gBACjB,aAAa,EAAE,MAAM,IAAA,2BAAa,EAAC;oBACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACxB,CAAC;aACH;YACD,kBAAkB,EAAE;gBAClB,eAAe,EAAE,KAAK;aACvB;SACF,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,8BAAiB,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;QACpE,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;IACnC,CAAC;CACF,CAAA;AAlDY,0CAAe;0BAAf,eAAe;IAD3B,IAAA,mBAAU,GAAE;qCAUgB,sBAAa;QACb,+CAAqB;QACpB,iDAAsB;GAXvC,eAAe,CAkD3B"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export declare const MIN_CONCURRENCY = "CRAWLEE_CAMOUFOX_MIN_CONCURRENCY";
|
|
2
|
+
export declare const MAX_CONCURRENCY = "CRAWLEE_CAMOUFOX_MAX_CONCURRENCY";
|
|
3
|
+
export declare const MAX_REQUEST_RETRIES = "CRAWLEE_CAMOUFOX_MAX_REQUEST_RETRIES";
|
|
4
|
+
export declare const TIMEOUT_SECS = "CRAWLEE_CAMOUFOX_TIMEOUT_SECS";
|
|
5
|
+
export declare const MAX_REQUESTS = "CRAWLEE_CAMOUFOX_MAX_REQUESTS";
|
|
6
|
+
export declare const INITIAL_PAGE = "CRAWLEE_CAMOUFOX_INITIAL_PAGE";
|
|
7
|
+
export declare const HEADLESS = "CRAWLEE_CAMOUFOX_HEADLESS";
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HEADLESS = exports.INITIAL_PAGE = exports.MAX_REQUESTS = exports.TIMEOUT_SECS = exports.MAX_REQUEST_RETRIES = exports.MAX_CONCURRENCY = exports.MIN_CONCURRENCY = void 0;
|
|
4
|
+
exports.MIN_CONCURRENCY = 'CRAWLEE_CAMOUFOX_MIN_CONCURRENCY';
|
|
5
|
+
exports.MAX_CONCURRENCY = 'CRAWLEE_CAMOUFOX_MAX_CONCURRENCY';
|
|
6
|
+
exports.MAX_REQUEST_RETRIES = 'CRAWLEE_CAMOUFOX_MAX_REQUEST_RETRIES';
|
|
7
|
+
exports.TIMEOUT_SECS = 'CRAWLEE_CAMOUFOX_TIMEOUT_SECS';
|
|
8
|
+
exports.MAX_REQUESTS = 'CRAWLEE_CAMOUFOX_MAX_REQUESTS';
|
|
9
|
+
exports.INITIAL_PAGE = 'CRAWLEE_CAMOUFOX_INITIAL_PAGE';
|
|
10
|
+
exports.HEADLESS = 'CRAWLEE_CAMOUFOX_HEADLESS';
|
|
11
|
+
//# sourceMappingURL=camoufox.env.constant.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"camoufox.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/constant/camoufox.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,eAAe,GAAG,kCAAkC,CAAC;AACrD,QAAA,eAAe,GAAG,kCAAkC,CAAC;AACrD,QAAA,mBAAmB,GAAG,sCAAsC,CAAC;AAC7D,QAAA,YAAY,GAAG,+BAA+B,CAAC;AAC/C,QAAA,YAAY,GAAG,+BAA+B,CAAC;AAC/C,QAAA,YAAY,GAAG,+BAA+B,CAAC;AAC/C,QAAA,QAAQ,GAAG,2BAA2B,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export declare const CamoufoxEnvConfig: () => {
|
|
2
|
+
CRAWLEE_CAMOUFOX_MAX_CONCURRENCY: number;
|
|
3
|
+
CRAWLEE_CAMOUFOX_MIN_CONCURRENCY: number;
|
|
4
|
+
CRAWLEE_CAMOUFOX_MAX_REQUEST_RETRIES: number;
|
|
5
|
+
CRAWLEE_CAMOUFOX_TIMEOUT_SECS: number;
|
|
6
|
+
CRAWLEE_CAMOUFOX_MAX_REQUESTS: number;
|
|
7
|
+
CRAWLEE_CAMOUFOX_INITIAL_PAGE: number;
|
|
8
|
+
CRAWLEE_CAMOUFOX_HEADLESS: boolean;
|
|
9
|
+
};
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.CamoufoxEnvConfig = void 0;
|
|
37
|
+
const dotenv = __importStar(require("dotenv"));
|
|
38
|
+
dotenv.config();
|
|
39
|
+
const CamoufoxEnvConfig = () => ({
|
|
40
|
+
CRAWLEE_CAMOUFOX_MAX_CONCURRENCY: Number(process.env.CRAWLEE_CAMOUFOX_MAX_CONCURRENCY),
|
|
41
|
+
CRAWLEE_CAMOUFOX_MIN_CONCURRENCY: Number(process.env.CRAWLEE_CAMOUFOX_MIN_CONCURRENCY),
|
|
42
|
+
CRAWLEE_CAMOUFOX_MAX_REQUEST_RETRIES: Number(process.env.CRAWLEE_CAMOUFOX_MAX_REQUEST_RETRIES),
|
|
43
|
+
CRAWLEE_CAMOUFOX_TIMEOUT_SECS: Number(process.env.CRAWLEE_CAMOUFOX_TIMEOUT_SECS),
|
|
44
|
+
CRAWLEE_CAMOUFOX_MAX_REQUESTS: Number(process.env.CRAWLEE_CAMOUFOX_MAX_REQUESTS),
|
|
45
|
+
CRAWLEE_CAMOUFOX_INITIAL_PAGE: Number(process.env.CRAWLEE_CAMOUFOX_INITIAL_PAGE),
|
|
46
|
+
CRAWLEE_CAMOUFOX_HEADLESS: Boolean(process.env.CRAWLEE_CAMOUFOX_HEADLESS === 'true'),
|
|
47
|
+
});
|
|
48
|
+
exports.CamoufoxEnvConfig = CamoufoxEnvConfig;
|
|
49
|
+
//# sourceMappingURL=camoufox.env.config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"camoufox.env.config.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/env/camoufox.env.config.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AAEjC,MAAM,CAAC,MAAM,EAAE,CAAC;AACT,MAAM,iBAAiB,GAAG,GAAG,EAAE,CAAC,CAAC;IACtC,gCAAgC,EAAE,MAAM,CACtC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAC7C;IACD,gCAAgC,EAAE,MAAM,CACtC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAC7C;IACD,oCAAoC,EAAE,MAAM,CAC1C,OAAO,CAAC,GAAG,CAAC,oCAAoC,CACjD;IACD,6BAA6B,EAAE,MAAM,CACnC,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAC1C;IACD,6BAA6B,EAAE,MAAM,CACnC,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAC1C;IACD,6BAA6B,EAAE,MAAM,CACnC,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAC1C;IACD,yBAAyB,EAAE,OAAO,CAChC,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,MAAM,CACjD;CACF,CAAC,CAAC;AAtBU,QAAA,iBAAiB,qBAsB3B"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.CamoufoxConfigValidationSchema = void 0;
|
|
37
|
+
const joi = __importStar(require("joi"));
|
|
38
|
+
exports.CamoufoxConfigValidationSchema = joi.object({
|
|
39
|
+
CRAWLEE_CAMOUFOX_MIN_CONCURRENCY: joi.number().optional().default(1),
|
|
40
|
+
CRAWLEE_CAMOUFOX_MAX_CONCURRENCY: joi.number().optional().default(5),
|
|
41
|
+
CRAWLEE_CAMOUFOX_MAX_REQUEST_RETRIES: joi.number().optional().default(10),
|
|
42
|
+
CRAWLEE_CAMOUFOX_TIMEOUT_SECS: joi.number().optional().default(180),
|
|
43
|
+
CRAWLEE_CAMOUFOX_MAX_REQUESTS: joi.number().optional().default(100000),
|
|
44
|
+
CRAWLEE_CAMOUFOX_INITIAL_PAGE: joi.number().optional().default(1),
|
|
45
|
+
CRAWLEE_CAMOUFOX_HEADLESS: joi.boolean().optional().default(true),
|
|
46
|
+
});
|
|
47
|
+
//# sourceMappingURL=camoufox.env.validation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"camoufox.env.validation.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/env/camoufox.env.validation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yCAA2B;AAEd,QAAA,8BAA8B,GAAG,GAAG,CAAC,MAAM,CAAC;IACvD,gCAAgC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACpE,gCAAgC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACpE,oCAAoC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACzE,6BAA6B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IACnE,6BAA6B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IACtE,6BAA6B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACjE,yBAAyB,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;CAClE,CAAC,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { ConfigService } from '@nestjs/config';
|
|
2
|
+
import { Browser } from 'playwright';
|
|
3
|
+
import { CamoufoxLoggerService } from './camoufox.logger.service';
|
|
4
|
+
export declare class CamoufoxBrowserService {
|
|
5
|
+
private readonly config;
|
|
6
|
+
private readonly logger;
|
|
7
|
+
private readonly headless;
|
|
8
|
+
constructor(config: ConfigService, logger: CamoufoxLoggerService);
|
|
9
|
+
getBrowser(): Promise<Browser>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.CamoufoxBrowserService = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const config_1 = require("@nestjs/config");
|
|
15
|
+
const playwright_1 = require("playwright");
|
|
16
|
+
const camoufox_js_1 = require("camoufox-js");
|
|
17
|
+
const camoufox_logger_service_1 = require("./camoufox.logger.service");
|
|
18
|
+
const camoufox_env_constant_1 = require("../constant/camoufox.env.constant");
|
|
19
|
+
let CamoufoxBrowserService = class CamoufoxBrowserService {
|
|
20
|
+
constructor(config, logger) {
|
|
21
|
+
this.config = config;
|
|
22
|
+
this.logger = logger;
|
|
23
|
+
this.headless = this.config.get(camoufox_env_constant_1.HEADLESS);
|
|
24
|
+
}
|
|
25
|
+
async getBrowser() {
|
|
26
|
+
this.logger.log('🚀 Creating Camoufox browser instance');
|
|
27
|
+
const browser = await playwright_1.firefox.launch(await (0, camoufox_js_1.launchOptions)({
|
|
28
|
+
headless: this.headless,
|
|
29
|
+
}));
|
|
30
|
+
this.logger.log('✅ Camoufox Browser instance created successfully');
|
|
31
|
+
return browser;
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
exports.CamoufoxBrowserService = CamoufoxBrowserService;
|
|
35
|
+
exports.CamoufoxBrowserService = CamoufoxBrowserService = __decorate([
|
|
36
|
+
(0, common_1.Injectable)(),
|
|
37
|
+
__metadata("design:paramtypes", [config_1.ConfigService,
|
|
38
|
+
camoufox_logger_service_1.CamoufoxLoggerService])
|
|
39
|
+
], CamoufoxBrowserService);
|
|
40
|
+
//# sourceMappingURL=camoufox.browser.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"camoufox.browser.service.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/services/camoufox.browser.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAE/C,2CAA8C;AAC9C,6CAA4C;AAE5C,uEAAkE;AAElE,6EAA6D;AAGtD,IAAM,sBAAsB,GAA5B,MAAM,sBAAsB;IAGjC,YACmB,MAAqB,EACrB,MAA6B;QAD7B,WAAM,GAAN,MAAM,CAAe;QACrB,WAAM,GAAN,MAAM,CAAuB;QAE9C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAU,gCAAQ,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,MAAM,oBAAO,CAAC,MAAM,CAClC,MAAM,IAAA,2BAAa,EAAC;YAClB,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,CAAC,CACH,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;QACpE,OAAO,OAAO,CAAC;IACjB,CAAC;CACF,CAAA;AApBY,wDAAsB;iCAAtB,sBAAsB;IADlC,IAAA,mBAAU,GAAE;qCAKgB,sBAAa;QACb,+CAAqB;GALrC,sBAAsB,CAoBlC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { CamoufoxLoggerService } from './camoufox.logger.service';
|
|
2
|
+
import { PlaywrightHook } from '@crawlee/playwright';
|
|
3
|
+
export declare class CamoufoxHookService {
|
|
4
|
+
private readonly logger;
|
|
5
|
+
constructor(logger: CamoufoxLoggerService);
|
|
6
|
+
getPreNavigationHooks(): PlaywrightHook[];
|
|
7
|
+
getPostNavigationHooks(): PlaywrightHook[];
|
|
8
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.CamoufoxHookService = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const camoufox_logger_service_1 = require("./camoufox.logger.service");
|
|
15
|
+
let CamoufoxHookService = class CamoufoxHookService {
|
|
16
|
+
constructor(logger) {
|
|
17
|
+
this.logger = logger;
|
|
18
|
+
}
|
|
19
|
+
getPreNavigationHooks() {
|
|
20
|
+
return [
|
|
21
|
+
({ request }) => {
|
|
22
|
+
this.logger.log(`🔎 ${request.url}`);
|
|
23
|
+
},
|
|
24
|
+
];
|
|
25
|
+
}
|
|
26
|
+
getPostNavigationHooks() {
|
|
27
|
+
return [
|
|
28
|
+
async ({ handleCloudflareChallenge }) => {
|
|
29
|
+
await handleCloudflareChallenge();
|
|
30
|
+
},
|
|
31
|
+
({ request, response }) => {
|
|
32
|
+
const { url } = request;
|
|
33
|
+
const statusCode = response.status();
|
|
34
|
+
switch (statusCode) {
|
|
35
|
+
case common_1.HttpStatus.OK:
|
|
36
|
+
this.logger.log(`✅ ${statusCode} ${url}`);
|
|
37
|
+
break;
|
|
38
|
+
case common_1.HttpStatus.BAD_REQUEST:
|
|
39
|
+
this.logger.warn(`⚠️ ${statusCode} ${url}`);
|
|
40
|
+
break;
|
|
41
|
+
case common_1.HttpStatus.UNAUTHORIZED:
|
|
42
|
+
this.logger.warn(`⚠️ ${statusCode} ${url}`);
|
|
43
|
+
break;
|
|
44
|
+
case common_1.HttpStatus.FORBIDDEN:
|
|
45
|
+
this.logger.warn(`⚠️ ${statusCode} ${url}`);
|
|
46
|
+
break;
|
|
47
|
+
case common_1.HttpStatus.TOO_MANY_REQUESTS:
|
|
48
|
+
this.logger.warn(`⛔ ${statusCode} ${url}`);
|
|
49
|
+
break;
|
|
50
|
+
case common_1.HttpStatus.INTERNAL_SERVER_ERROR:
|
|
51
|
+
this.logger.error(`💥 ${statusCode} ${url}`);
|
|
52
|
+
break;
|
|
53
|
+
case common_1.HttpStatus.BAD_GATEWAY:
|
|
54
|
+
this.logger.error(`🚧 ${statusCode} ${url}`);
|
|
55
|
+
break;
|
|
56
|
+
case common_1.HttpStatus.SERVICE_UNAVAILABLE:
|
|
57
|
+
this.logger.error(`🚧 ${statusCode} ${url}`);
|
|
58
|
+
break;
|
|
59
|
+
default:
|
|
60
|
+
this.logger.log(`🤔 ${statusCode} ${url}`);
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
];
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
exports.CamoufoxHookService = CamoufoxHookService;
|
|
68
|
+
exports.CamoufoxHookService = CamoufoxHookService = __decorate([
|
|
69
|
+
(0, common_1.Injectable)(),
|
|
70
|
+
__metadata("design:paramtypes", [camoufox_logger_service_1.CamoufoxLoggerService])
|
|
71
|
+
], CamoufoxHookService);
|
|
72
|
+
//# sourceMappingURL=camoufox.hook.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"camoufox.hook.service.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/services/camoufox.hook.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAAwD;AAExD,uEAAkE;AAI3D,IAAM,mBAAmB,GAAzB,MAAM,mBAAmB;IAC9B,YAA6B,MAA6B;QAA7B,WAAM,GAAN,MAAM,CAAuB;IAAG,CAAC;IAE9D,qBAAqB;QACnB,OAAO;YACL,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACvC,CAAC;SACF,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,OAAO;YACL,KAAK,EAAE,EAAE,yBAAyB,EAAE,EAAE,EAAE;gBACtC,MAAM,yBAAyB,EAAE,CAAC;YACpC,CAAC;YACD,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE;gBACxB,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;gBACxB,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAErC,QAAQ,UAAwB,EAAE,CAAC;oBACjC,KAAK,mBAAU,CAAC,EAAE;wBAChB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC1C,MAAM;oBACR,KAAK,mBAAU,CAAC,WAAW;wBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC5C,MAAM;oBACR,KAAK,mBAAU,CAAC,YAAY;wBAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC5C,MAAM;oBACR,KAAK,mBAAU,CAAC,SAAS;wBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC5C,MAAM;oBACR,KAAK,mBAAU,CAAC,iBAAiB;wBAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC3C,MAAM;oBACR,KAAK,mBAAU,CAAC,qBAAqB;wBACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,WAAW;wBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR,KAAK,mBAAU,CAAC,mBAAmB;wBACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC7C,MAAM;oBACR;wBACE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;wBAC3C,MAAM;gBACV,CAAC;YACH,CAAC;SACF,CAAC;IACJ,CAAC;CACF,CAAA;AApDY,kDAAmB;8BAAnB,mBAAmB;IAD/B,IAAA,mBAAU,GAAE;qCAE0B,+CAAqB;GAD/C,mBAAmB,CAoD/B"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { LoggerService } from '@nestjs/common';
|
|
2
|
+
export declare class CamoufoxLoggerService implements LoggerService {
|
|
3
|
+
private readonly logger;
|
|
4
|
+
constructor();
|
|
5
|
+
log(message: any, ...optionalParams: any[]): any;
|
|
6
|
+
debug(message: any, ...optionalParams: any[]): any;
|
|
7
|
+
verbose(message: any, ...optionalParams: any[]): any;
|
|
8
|
+
warn(message: any, ...optionalParams: any[]): any;
|
|
9
|
+
error(message: any, ...optionalParams: any[]): any;
|
|
10
|
+
fatal?(message: any, ...optionalParams: any[]): any;
|
|
11
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.CamoufoxLoggerService = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const camoufox_service_1 = require("../camoufox.service");
|
|
15
|
+
let CamoufoxLoggerService = class CamoufoxLoggerService {
|
|
16
|
+
constructor() {
|
|
17
|
+
const options = { timestamp: true };
|
|
18
|
+
this.logger = new common_1.Logger(camoufox_service_1.CamoufoxService.name, options);
|
|
19
|
+
}
|
|
20
|
+
log(message, ...optionalParams) {
|
|
21
|
+
this.logger.log(message, ...optionalParams);
|
|
22
|
+
}
|
|
23
|
+
debug(message, ...optionalParams) {
|
|
24
|
+
this.logger.debug(message, ...optionalParams);
|
|
25
|
+
}
|
|
26
|
+
verbose(message, ...optionalParams) {
|
|
27
|
+
this.logger.verbose(message, ...optionalParams);
|
|
28
|
+
}
|
|
29
|
+
warn(message, ...optionalParams) {
|
|
30
|
+
this.logger.warn(message, ...optionalParams);
|
|
31
|
+
}
|
|
32
|
+
error(message, ...optionalParams) {
|
|
33
|
+
this.logger.error(message, ...optionalParams);
|
|
34
|
+
}
|
|
35
|
+
fatal(message, ...optionalParams) {
|
|
36
|
+
this.logger.fatal(message, ...optionalParams);
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
exports.CamoufoxLoggerService = CamoufoxLoggerService;
|
|
40
|
+
exports.CamoufoxLoggerService = CamoufoxLoggerService = __decorate([
|
|
41
|
+
(0, common_1.Injectable)(),
|
|
42
|
+
__metadata("design:paramtypes", [])
|
|
43
|
+
], CamoufoxLoggerService);
|
|
44
|
+
//# sourceMappingURL=camoufox.logger.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"camoufox.logger.service.js","sourceRoot":"","sources":["../../../../src/modules/camoufox/services/camoufox.logger.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAAmE;AAEnE,0DAAsD;AAG/C,IAAM,qBAAqB,GAA3B,MAAM,qBAAqB;IAEhC;QACE,MAAM,OAAO,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,eAAM,CAAC,kCAAe,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IAED,GAAG,CAAC,OAAY,EAAE,GAAG,cAAqB;QACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,OAAY,EAAE,GAAG,cAAqB;QAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,CAAC,OAAY,EAAE,GAAG,cAAqB;QAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC,CAAC;IAClD,CAAC;IAED,IAAI,CAAC,OAAY,EAAE,GAAG,cAAqB;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,OAAY,EAAE,GAAG,cAAqB;QAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAE,OAAY,EAAE,GAAG,cAAqB;QAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC,CAAC;IAChD,CAAC;CACF,CAAA;AA9BY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,mBAAU,GAAE;;GACA,qBAAqB,CA8BjC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const BROWSER_ARGS: string[];
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BROWSER_ARGS = void 0;
|
|
4
|
+
exports.BROWSER_ARGS = [
|
|
5
|
+
'--no-sandbox',
|
|
6
|
+
'--disable-setuid-sandbox',
|
|
7
|
+
'--disable-dev-shm-usage',
|
|
8
|
+
'--disable-accelerated-2d-canvas',
|
|
9
|
+
'--no-first-run',
|
|
10
|
+
'--no-zygote',
|
|
11
|
+
'--disable-gpu',
|
|
12
|
+
];
|
|
13
|
+
//# sourceMappingURL=playwright.browser.constant.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright.browser.constant.js","sourceRoot":"","sources":["../../../../src/modules/playwright/constant/playwright.browser.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,YAAY,GAAG;IAC1B,cAAc;IACd,0BAA0B;IAC1B,yBAAyB;IACzB,iCAAiC;IACjC,gBAAgB;IAChB,aAAa;IACb,eAAe;CAChB,CAAC"}
|
|
@@ -4,3 +4,4 @@ export declare const MAX_REQUEST_RETRIES = "CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRI
|
|
|
4
4
|
export declare const TIMEOUT_SECS = "CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS";
|
|
5
5
|
export declare const MAX_REQUESTS = "CRAWLEE_PLAYWRIGHT_MAX_REQUESTS";
|
|
6
6
|
export declare const INITIAL_PAGE = "CRAWLEE_PLAYWRIGHT_INITIAL_PAGE";
|
|
7
|
+
export declare const HEADLESS = "CRAWLEE_PLAYWRIGHT_HEADLESS";
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.INITIAL_PAGE = exports.MAX_REQUESTS = exports.TIMEOUT_SECS = exports.MAX_REQUEST_RETRIES = exports.MAX_CONCURRENCY = exports.MIN_CONCURRENCY = void 0;
|
|
3
|
+
exports.HEADLESS = exports.INITIAL_PAGE = exports.MAX_REQUESTS = exports.TIMEOUT_SECS = exports.MAX_REQUEST_RETRIES = exports.MAX_CONCURRENCY = exports.MIN_CONCURRENCY = void 0;
|
|
4
4
|
exports.MIN_CONCURRENCY = 'CRAWLEE_PLAYWRIGHT_MIN_CONCURRENCY';
|
|
5
5
|
exports.MAX_CONCURRENCY = 'CRAWLEE_PLAYWRIGHT_MAX_CONCURRENCY';
|
|
6
6
|
exports.MAX_REQUEST_RETRIES = 'CRAWLEE_PLAYWRIGHT_MAX_REQUEST_RETRIES';
|
|
7
7
|
exports.TIMEOUT_SECS = 'CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS';
|
|
8
8
|
exports.MAX_REQUESTS = 'CRAWLEE_PLAYWRIGHT_MAX_REQUESTS';
|
|
9
9
|
exports.INITIAL_PAGE = 'CRAWLEE_PLAYWRIGHT_INITIAL_PAGE';
|
|
10
|
+
exports.HEADLESS = 'CRAWLEE_PLAYWRIGHT_HEADLESS';
|
|
10
11
|
//# sourceMappingURL=playwright.env.constant.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/playwright/constant/playwright.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,eAAe,GAAG,oCAAoC,CAAC;AACvD,QAAA,eAAe,GAAG,oCAAoC,CAAC;AACvD,QAAA,mBAAmB,GAAG,wCAAwC,CAAC;AAC/D,QAAA,YAAY,GAAG,iCAAiC,CAAC;AACjD,QAAA,YAAY,GAAG,iCAAiC,CAAC;AACjD,QAAA,YAAY,GAAG,iCAAiC,CAAC"}
|
|
1
|
+
{"version":3,"file":"playwright.env.constant.js","sourceRoot":"","sources":["../../../../src/modules/playwright/constant/playwright.env.constant.ts"],"names":[],"mappings":";;;AAAa,QAAA,eAAe,GAAG,oCAAoC,CAAC;AACvD,QAAA,eAAe,GAAG,oCAAoC,CAAC;AACvD,QAAA,mBAAmB,GAAG,wCAAwC,CAAC;AAC/D,QAAA,YAAY,GAAG,iCAAiC,CAAC;AACjD,QAAA,YAAY,GAAG,iCAAiC,CAAC;AACjD,QAAA,YAAY,GAAG,iCAAiC,CAAC;AACjD,QAAA,QAAQ,GAAG,6BAA6B,CAAC"}
|
|
@@ -43,6 +43,7 @@ const PlaywrightEnvConfig = () => ({
|
|
|
43
43
|
CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS: Number(process.env.CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS),
|
|
44
44
|
CRAWLEE_PLAYWRIGHT_MAX_REQUESTS: Number(process.env.CRAWLEE_PLAYWRIGHT_MAX_REQUESTS),
|
|
45
45
|
CRAWLEE_PLAYWRIGHT_INITIAL_PAGE: Number(process.env.CRAWLEE_PLAYWRIGHT_INITIAL_PAGE),
|
|
46
|
+
CRAWLEE_PLAYWRIGHT_HEADLESS: Boolean(process.env.CRAWLEE_PLAYWRIGHT_HEADLESS === 'true'),
|
|
46
47
|
});
|
|
47
48
|
exports.PlaywrightEnvConfig = PlaywrightEnvConfig;
|
|
48
49
|
//# sourceMappingURL=playwright.env.config.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.env.config.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.config.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AAEjC,MAAM,CAAC,MAAM,EAAE,CAAC;AACT,MAAM,mBAAmB,GAAG,GAAG,EAAE,CAAC,CAAC;IACxC,kCAAkC,EAAE,MAAM,CACxC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAC/C;IACD,kCAAkC,EAAE,MAAM,CACxC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAC/C;IACD,sCAAsC,EAAE,MAAM,CAC5C,OAAO,CAAC,GAAG,CAAC,sCAAsC,CACnD;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;CACF,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"playwright.env.config.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.config.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AAEjC,MAAM,CAAC,MAAM,EAAE,CAAC;AACT,MAAM,mBAAmB,GAAG,GAAG,EAAE,CAAC,CAAC;IACxC,kCAAkC,EAAE,MAAM,CACxC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAC/C;IACD,kCAAkC,EAAE,MAAM,CACxC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAC/C;IACD,sCAAsC,EAAE,MAAM,CAC5C,OAAO,CAAC,GAAG,CAAC,sCAAsC,CACnD;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,+BAA+B,EAAE,MAAM,CACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAC5C;IACD,2BAA2B,EAAE,OAAO,CAClC,OAAO,CAAC,GAAG,CAAC,2BAA2B,KAAK,MAAM,CACnD;CACF,CAAC,CAAC;AAtBU,QAAA,mBAAmB,uBAsB7B"}
|
|
@@ -42,5 +42,6 @@ exports.PlaywrightConfigValidationSchema = joi.object({
|
|
|
42
42
|
CRAWLEE_PLAYWRIGHT_TIMEOUT_SECS: joi.number().optional().default(180),
|
|
43
43
|
CRAWLEE_PLAYWRIGHT_MAX_REQUESTS: joi.number().optional().default(100000),
|
|
44
44
|
CRAWLEE_PLAYWRIGHT_INITIAL_PAGE: joi.number().optional().default(1),
|
|
45
|
+
CRAWLEE_PLAYWRIGHT_HEADLESS: joi.boolean().optional().default(true),
|
|
45
46
|
});
|
|
46
47
|
//# sourceMappingURL=playwright.env.validation.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.env.validation.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.validation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yCAA2B;AAEd,QAAA,gCAAgC,GAAG,GAAG,CAAC,MAAM,CAAC;IACzD,kCAAkC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,kCAAkC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,sCAAsC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC3E,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IACrE,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IACxE,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;CACpE,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"playwright.env.validation.js","sourceRoot":"","sources":["../../../../src/modules/playwright/env/playwright.env.validation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yCAA2B;AAEd,QAAA,gCAAgC,GAAG,GAAG,CAAC,MAAM,CAAC;IACzD,kCAAkC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,kCAAkC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,sCAAsC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC3E,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IACrE,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IACxE,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACnE,2BAA2B,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;CACpE,CAAC,CAAC"}
|
|
@@ -13,6 +13,7 @@ const playwright_service_1 = require("./playwright.service");
|
|
|
13
13
|
const playwright_env_config_1 = require("./env/playwright.env.config");
|
|
14
14
|
const playwright_logger_service_1 = require("./services/playwright.logger.service");
|
|
15
15
|
const playwright_hook_service_1 = require("./services/playwright.hook.service");
|
|
16
|
+
const playwright_browser_service_1 = require("./services/playwright.browser.service");
|
|
16
17
|
let PlaywrightModule = class PlaywrightModule {
|
|
17
18
|
};
|
|
18
19
|
exports.PlaywrightModule = PlaywrightModule;
|
|
@@ -23,6 +24,7 @@ exports.PlaywrightModule = PlaywrightModule = __decorate([
|
|
|
23
24
|
playwright_service_1.PlaywrightService,
|
|
24
25
|
playwright_logger_service_1.PlaywrightLoggerService,
|
|
25
26
|
playwright_hook_service_1.PlaywrightHookService,
|
|
27
|
+
playwright_browser_service_1.PlaywrightBrowserService,
|
|
26
28
|
],
|
|
27
29
|
exports: [playwright_service_1.PlaywrightService],
|
|
28
30
|
})
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.module.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AACxC,2CAA8C;AAE9C,6DAAyD;AACzD,uEAAkE;AAClE,oFAA+E;AAC/E,gFAA2E;
|
|
1
|
+
{"version":3,"file":"playwright.module.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AACxC,2CAA8C;AAE9C,6DAAyD;AACzD,uEAAkE;AAClE,oFAA+E;AAC/E,gFAA2E;AAC3E,sFAAiF;AAY1E,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;CAAG,CAAA;AAAnB,4CAAgB;2BAAhB,gBAAgB;IAV5B,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,qBAAY,CAAC,UAAU,CAAC,2CAAmB,CAAC,CAAC;QACvD,SAAS,EAAE;YACT,sCAAiB;YACjB,mDAAuB;YACvB,+CAAqB;YACrB,qDAAwB;SACzB;QACD,OAAO,EAAE,CAAC,sCAAiB,CAAC;KAC7B,CAAC;GACW,gBAAgB,CAAG"}
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
import { ConfigService } from '@nestjs/config';
|
|
2
2
|
import { PlaywrightCrawler } from '@crawlee/playwright';
|
|
3
3
|
import { PlaywrightCrawlerOptions } from '@crawlee/playwright';
|
|
4
|
-
import { Browser } from 'playwright';
|
|
5
4
|
import { PlaywrightLoggerService } from './services/playwright.logger.service';
|
|
5
|
+
import { PlaywrightBrowserService } from './services/playwright.browser.service';
|
|
6
6
|
export declare class PlaywrightService {
|
|
7
7
|
private readonly config;
|
|
8
8
|
private readonly logger;
|
|
9
|
+
private readonly browser;
|
|
9
10
|
private readonly maxRequests;
|
|
10
11
|
private readonly timeoutSecs;
|
|
11
12
|
private readonly maxConcurrency;
|
|
12
13
|
private readonly minConcurrency;
|
|
13
14
|
private readonly maxRequestRetries;
|
|
14
|
-
private readonly
|
|
15
|
-
constructor(config: ConfigService, logger: PlaywrightLoggerService);
|
|
15
|
+
private readonly headless;
|
|
16
|
+
constructor(config: ConfigService, logger: PlaywrightLoggerService, browser: PlaywrightBrowserService);
|
|
16
17
|
create(options: PlaywrightCrawlerOptions): PlaywrightCrawler;
|
|
17
|
-
getBrowser(): Promise<Browser>;
|
|
18
|
+
getBrowser(): Promise<import("playwright").Browser>;
|
|
18
19
|
}
|
|
@@ -13,31 +13,26 @@ exports.PlaywrightService = void 0;
|
|
|
13
13
|
const common_1 = require("@nestjs/common");
|
|
14
14
|
const config_1 = require("@nestjs/config");
|
|
15
15
|
const playwright_1 = require("@crawlee/playwright");
|
|
16
|
-
const playwright_2 = require("playwright");
|
|
17
16
|
const playwright_logger_service_1 = require("./services/playwright.logger.service");
|
|
17
|
+
const playwright_browser_service_1 = require("./services/playwright.browser.service");
|
|
18
18
|
const playwright_env_constant_1 = require("./constant/playwright.env.constant");
|
|
19
19
|
const playwright_env_constant_2 = require("./constant/playwright.env.constant");
|
|
20
20
|
const playwright_env_constant_3 = require("./constant/playwright.env.constant");
|
|
21
21
|
const playwright_env_constant_4 = require("./constant/playwright.env.constant");
|
|
22
22
|
const playwright_env_constant_5 = require("./constant/playwright.env.constant");
|
|
23
|
+
const playwright_browser_constant_1 = require("./constant/playwright.browser.constant");
|
|
24
|
+
const playwright_env_constant_6 = require("./constant/playwright.env.constant");
|
|
23
25
|
let PlaywrightService = class PlaywrightService {
|
|
24
|
-
constructor(config, logger) {
|
|
26
|
+
constructor(config, logger, browser) {
|
|
25
27
|
this.config = config;
|
|
26
28
|
this.logger = logger;
|
|
27
|
-
this.
|
|
28
|
-
'--no-sandbox',
|
|
29
|
-
'--disable-setuid-sandbox',
|
|
30
|
-
'--disable-dev-shm-usage',
|
|
31
|
-
'--disable-accelerated-2d-canvas',
|
|
32
|
-
'--no-first-run',
|
|
33
|
-
'--no-zygote',
|
|
34
|
-
'--disable-gpu',
|
|
35
|
-
];
|
|
29
|
+
this.browser = browser;
|
|
36
30
|
this.maxConcurrency = this.config.get(playwright_env_constant_1.MAX_CONCURRENCY);
|
|
37
31
|
this.minConcurrency = this.config.get(playwright_env_constant_2.MIN_CONCURRENCY);
|
|
38
32
|
this.maxRequestRetries = this.config.get(playwright_env_constant_3.MAX_REQUEST_RETRIES);
|
|
39
33
|
this.timeoutSecs = this.config.get(playwright_env_constant_4.TIMEOUT_SECS);
|
|
40
34
|
this.maxRequests = this.config.get(playwright_env_constant_5.MAX_REQUESTS);
|
|
35
|
+
this.headless = this.config.get(playwright_env_constant_6.HEADLESS);
|
|
41
36
|
}
|
|
42
37
|
create(options) {
|
|
43
38
|
this.logger.log('🚀 Creating Playwright Crawler instance');
|
|
@@ -49,10 +44,10 @@ let PlaywrightService = class PlaywrightService {
|
|
|
49
44
|
minConcurrency: this.minConcurrency,
|
|
50
45
|
maxRequestRetries: this.maxRequestRetries,
|
|
51
46
|
useSessionPool: true,
|
|
52
|
-
headless:
|
|
47
|
+
headless: this.headless,
|
|
53
48
|
launchContext: {
|
|
54
49
|
launchOptions: {
|
|
55
|
-
args:
|
|
50
|
+
args: playwright_browser_constant_1.BROWSER_ARGS,
|
|
56
51
|
},
|
|
57
52
|
useIncognitoPages: true,
|
|
58
53
|
},
|
|
@@ -66,19 +61,14 @@ let PlaywrightService = class PlaywrightService {
|
|
|
66
61
|
return crawler;
|
|
67
62
|
}
|
|
68
63
|
async getBrowser() {
|
|
69
|
-
this.
|
|
70
|
-
const browser = await playwright_2.chromium.launch({
|
|
71
|
-
headless: true,
|
|
72
|
-
args: this.browserArgs,
|
|
73
|
-
});
|
|
74
|
-
this.logger.log('✅ Playwright Browser instance created successfully');
|
|
75
|
-
return browser;
|
|
64
|
+
return this.browser.getBrowser();
|
|
76
65
|
}
|
|
77
66
|
};
|
|
78
67
|
exports.PlaywrightService = PlaywrightService;
|
|
79
68
|
exports.PlaywrightService = PlaywrightService = __decorate([
|
|
80
69
|
(0, common_1.Injectable)(),
|
|
81
70
|
__metadata("design:paramtypes", [config_1.ConfigService,
|
|
82
|
-
playwright_logger_service_1.PlaywrightLoggerService
|
|
71
|
+
playwright_logger_service_1.PlaywrightLoggerService,
|
|
72
|
+
playwright_browser_service_1.PlaywrightBrowserService])
|
|
83
73
|
], PlaywrightService);
|
|
84
74
|
//# sourceMappingURL=playwright.service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.service.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAE/C,oDAAwD;
|
|
1
|
+
{"version":3,"file":"playwright.service.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAE/C,oDAAwD;AAGxD,oFAA+E;AAC/E,sFAAiF;AAEjF,gFAAqE;AACrE,gFAAqE;AACrE,gFAAyE;AACzE,gFAAkE;AAClE,gFAAkE;AAClE,wFAAsE;AACtE,gFAA8D;AAGvD,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAQ5B,YACmB,MAAqB,EACrB,MAA+B,EAC/B,OAAiC;QAFjC,WAAM,GAAN,MAAM,CAAe;QACrB,WAAM,GAAN,MAAM,CAAyB;QAC/B,YAAO,GAAP,OAAO,CAA0B;QAElD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,yCAAe,CAAC,CAAC;QAC/D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,yCAAe,CAAC,CAAC;QAC/D,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,6CAAmB,CAAC,CAAC;QACtE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,sCAAY,CAAC,CAAC;QACzD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,sCAAY,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAU,kCAAQ,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,CAAC,OAAiC;QACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;QAC3D,MAAM,cAAc,GAA6B;YAC/C,mBAAmB,EAAE,IAAI,CAAC,WAAW;YACrC,yBAAyB,EAAE,IAAI,CAAC,WAAW;YAC3C,qBAAqB,EAAE,IAAI,CAAC,WAAW;YACvC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE;gBACb,aAAa,EAAE;oBACb,IAAI,EAAE,0CAAY;iBACnB;gBACD,iBAAiB,EAAE,IAAI;aACxB;YACD,kBAAkB,EAAE;gBAClB,eAAe,EAAE,IAAI;gBACrB,2BAA2B,EAAE,EAAE;aAChC;SACF,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,8BAAiB,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QACtE,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;IACnC,CAAC;CACF,CAAA;AAnDY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAUgB,sBAAa;QACb,mDAAuB;QACtB,qDAAwB;GAXzC,iBAAiB,CAmD7B"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { ConfigService } from '@nestjs/config';
|
|
2
|
+
import { Browser } from 'playwright';
|
|
3
|
+
import { PlaywrightLoggerService } from './playwright.logger.service';
|
|
4
|
+
export declare class PlaywrightBrowserService {
|
|
5
|
+
private readonly config;
|
|
6
|
+
private readonly logger;
|
|
7
|
+
private readonly headless;
|
|
8
|
+
constructor(config: ConfigService, logger: PlaywrightLoggerService);
|
|
9
|
+
getBrowser(): Promise<Browser>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.PlaywrightBrowserService = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const config_1 = require("@nestjs/config");
|
|
15
|
+
const playwright_1 = require("playwright");
|
|
16
|
+
const playwright_logger_service_1 = require("./playwright.logger.service");
|
|
17
|
+
const playwright_browser_constant_1 = require("../constant/playwright.browser.constant");
|
|
18
|
+
const playwright_env_constant_1 = require("../constant/playwright.env.constant");
|
|
19
|
+
let PlaywrightBrowserService = class PlaywrightBrowserService {
|
|
20
|
+
constructor(config, logger) {
|
|
21
|
+
this.config = config;
|
|
22
|
+
this.logger = logger;
|
|
23
|
+
this.headless = this.config.get(playwright_env_constant_1.HEADLESS);
|
|
24
|
+
}
|
|
25
|
+
async getBrowser() {
|
|
26
|
+
this.logger.log('🚀 Creating direct Playwright browser instance');
|
|
27
|
+
const browser = await playwright_1.chromium.launch({
|
|
28
|
+
headless: this.headless,
|
|
29
|
+
args: playwright_browser_constant_1.BROWSER_ARGS,
|
|
30
|
+
});
|
|
31
|
+
this.logger.log('✅ Playwright Browser instance created successfully');
|
|
32
|
+
return browser;
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
exports.PlaywrightBrowserService = PlaywrightBrowserService;
|
|
36
|
+
exports.PlaywrightBrowserService = PlaywrightBrowserService = __decorate([
|
|
37
|
+
(0, common_1.Injectable)(),
|
|
38
|
+
__metadata("design:paramtypes", [config_1.ConfigService,
|
|
39
|
+
playwright_logger_service_1.PlaywrightLoggerService])
|
|
40
|
+
], PlaywrightBrowserService);
|
|
41
|
+
//# sourceMappingURL=playwright.browser.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright.browser.service.js","sourceRoot":"","sources":["../../../../src/modules/playwright/services/playwright.browser.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAE/C,2CAA+C;AAE/C,2EAAsE;AAEtE,yFAAuE;AACvE,iFAA+D;AAGxD,IAAM,wBAAwB,GAA9B,MAAM,wBAAwB;IAGnC,YACmB,MAAqB,EACrB,MAA+B;QAD/B,WAAM,GAAN,MAAM,CAAe;QACrB,WAAM,GAAN,MAAM,CAAyB;QAEhD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAU,kCAAQ,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,MAAM,qBAAQ,CAAC,MAAM,CAAC;YACpC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,0CAAY;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QACtE,OAAO,OAAO,CAAC;IACjB,CAAC;CACF,CAAA;AAnBY,4DAAwB;mCAAxB,wBAAwB;IADpC,IAAA,mBAAU,GAAE;qCAKgB,sBAAa;QACb,mDAAuB;GALvC,wBAAwB,CAmBpC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rafikidota/scoutee",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Sometimes, the best way to solve your own problems is to help someone else.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
"@types/node": "24.10.2",
|
|
25
25
|
"@typescript-eslint/eslint-plugin": "8.49.0",
|
|
26
26
|
"@typescript-eslint/parser": "8.49.0",
|
|
27
|
+
"camoufox-js": "0.8.4",
|
|
27
28
|
"crawlee": "3.15.3",
|
|
28
29
|
"dotenv": "17.2.3",
|
|
29
30
|
"eslint": "9.39.1",
|
|
@@ -46,6 +47,7 @@
|
|
|
46
47
|
"crawlee",
|
|
47
48
|
"cheerio",
|
|
48
49
|
"playwright",
|
|
50
|
+
"camoufox",
|
|
49
51
|
"scraping"
|
|
50
52
|
],
|
|
51
53
|
"scripts": {
|