@rafikidota/scoutee 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +8 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -1
- package/dist/modules/camoufox/camoufox.module.js +0 -3
- package/dist/modules/camoufox/camoufox.module.js.map +1 -1
- package/dist/modules/camoufox/camoufox.service.d.ts +3 -1
- package/dist/modules/camoufox/camoufox.service.js +13 -2
- package/dist/modules/camoufox/camoufox.service.js.map +1 -1
- package/dist/modules/cheerio/cheerio.module.js +0 -3
- package/dist/modules/cheerio/cheerio.module.js.map +1 -1
- package/dist/modules/playwright/playwright.module.js +0 -3
- package/dist/modules/playwright/playwright.module.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,12 @@
|
|
|
1
1
|
export * from './modules/cheerio/cheerio.module';
|
|
2
2
|
export * from './modules/cheerio/cheerio.service';
|
|
3
|
+
export * from './modules/cheerio/env/cheerio.env.config';
|
|
4
|
+
export * from './modules/cheerio/env/cheerio.env.validation';
|
|
3
5
|
export * from './modules/playwright/playwright.module';
|
|
4
6
|
export * from './modules/playwright/playwright.service';
|
|
7
|
+
export * from './modules/playwright/env/playwright.env.config';
|
|
8
|
+
export * from './modules/playwright/env/playwright.env.validation';
|
|
9
|
+
export * from './modules/camoufox/camoufox.module';
|
|
10
|
+
export * from './modules/camoufox/camoufox.service';
|
|
11
|
+
export * from './modules/camoufox/env/camoufox.env.config';
|
|
12
|
+
export * from './modules/camoufox/env/camoufox.env.validation';
|
package/dist/index.js
CHANGED
|
@@ -16,6 +16,14 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
__exportStar(require("./modules/cheerio/cheerio.module"), exports);
|
|
18
18
|
__exportStar(require("./modules/cheerio/cheerio.service"), exports);
|
|
19
|
+
__exportStar(require("./modules/cheerio/env/cheerio.env.config"), exports);
|
|
20
|
+
__exportStar(require("./modules/cheerio/env/cheerio.env.validation"), exports);
|
|
19
21
|
__exportStar(require("./modules/playwright/playwright.module"), exports);
|
|
20
22
|
__exportStar(require("./modules/playwright/playwright.service"), exports);
|
|
23
|
+
__exportStar(require("./modules/playwright/env/playwright.env.config"), exports);
|
|
24
|
+
__exportStar(require("./modules/playwright/env/playwright.env.validation"), exports);
|
|
25
|
+
__exportStar(require("./modules/camoufox/camoufox.module"), exports);
|
|
26
|
+
__exportStar(require("./modules/camoufox/camoufox.service"), exports);
|
|
27
|
+
__exportStar(require("./modules/camoufox/env/camoufox.env.config"), exports);
|
|
28
|
+
__exportStar(require("./modules/camoufox/env/camoufox.env.validation"), exports);
|
|
21
29
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,mEAAiD;AACjD,oEAAkD;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,mEAAiD;AACjD,oEAAkD;AAClD,2EAAyD;AACzD,+EAA6D;AAE7D,yEAAuD;AACvD,0EAAwD;AACxD,iFAA+D;AAC/D,qFAAmE;AAEnE,qEAAmD;AACnD,sEAAoD;AACpD,6EAA2D;AAC3D,iFAA+D"}
|
|
@@ -8,9 +8,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
9
|
exports.CamoufoxModule = void 0;
|
|
10
10
|
const common_1 = require("@nestjs/common");
|
|
11
|
-
const config_1 = require("@nestjs/config");
|
|
12
11
|
const camoufox_service_1 = require("./camoufox.service");
|
|
13
|
-
const camoufox_env_config_1 = require("./env/camoufox.env.config");
|
|
14
12
|
const camoufox_logger_service_1 = require("./services/camoufox.logger.service");
|
|
15
13
|
const camoufox_hook_service_1 = require("./services/camoufox.hook.service");
|
|
16
14
|
const camoufox_browser_service_1 = require("./services/camoufox.browser.service");
|
|
@@ -19,7 +17,6 @@ let CamoufoxModule = class CamoufoxModule {
|
|
|
19
17
|
exports.CamoufoxModule = CamoufoxModule;
|
|
20
18
|
exports.CamoufoxModule = CamoufoxModule = __decorate([
|
|
21
19
|
(0, common_1.Module)({
|
|
22
|
-
imports: [config_1.ConfigModule.forFeature(camoufox_env_config_1.CamoufoxEnvConfig)],
|
|
23
20
|
providers: [
|
|
24
21
|
camoufox_service_1.CamoufoxService,
|
|
25
22
|
camoufox_logger_service_1.CamoufoxLoggerService,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"camoufox.module.js","sourceRoot":"","sources":["../../../src/modules/camoufox/camoufox.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;
|
|
1
|
+
{"version":3,"file":"camoufox.module.js","sourceRoot":"","sources":["../../../src/modules/camoufox/camoufox.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,yDAAqD;AACrD,gFAA2E;AAC3E,4EAAuE;AACvE,kFAA6E;AAWtE,IAAM,cAAc,GAApB,MAAM,cAAc;CAAG,CAAA;AAAjB,wCAAc;yBAAd,cAAc;IAT1B,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,kCAAe;YACf,+CAAqB;YACrB,2CAAmB;YACnB,iDAAsB;SACvB;QACD,OAAO,EAAE,CAAC,kCAAe,CAAC;KAC3B,CAAC;GACW,cAAc,CAAG"}
|
|
@@ -2,10 +2,12 @@ import { ConfigService } from '@nestjs/config';
|
|
|
2
2
|
import { PlaywrightCrawler } from '@crawlee/playwright';
|
|
3
3
|
import { PlaywrightCrawlerOptions } from '@crawlee/playwright';
|
|
4
4
|
import { CamoufoxLoggerService } from './services/camoufox.logger.service';
|
|
5
|
+
import { CamoufoxHookService } from './services/camoufox.hook.service';
|
|
5
6
|
import { CamoufoxBrowserService } from './services/camoufox.browser.service';
|
|
6
7
|
export declare class CamoufoxService {
|
|
7
8
|
private readonly config;
|
|
8
9
|
private readonly logger;
|
|
10
|
+
private readonly hook;
|
|
9
11
|
private readonly browser;
|
|
10
12
|
private readonly maxRequests;
|
|
11
13
|
private readonly timeoutSecs;
|
|
@@ -13,7 +15,7 @@ export declare class CamoufoxService {
|
|
|
13
15
|
private readonly minConcurrency;
|
|
14
16
|
private readonly maxRequestRetries;
|
|
15
17
|
private readonly headless;
|
|
16
|
-
constructor(config: ConfigService, logger: CamoufoxLoggerService, browser: CamoufoxBrowserService);
|
|
18
|
+
constructor(config: ConfigService, logger: CamoufoxLoggerService, hook: CamoufoxHookService, browser: CamoufoxBrowserService);
|
|
17
19
|
create(options: PlaywrightCrawlerOptions): Promise<PlaywrightCrawler>;
|
|
18
20
|
getBrowser(): Promise<import("playwright").Browser>;
|
|
19
21
|
}
|
|
@@ -16,6 +16,7 @@ const playwright_1 = require("@crawlee/playwright");
|
|
|
16
16
|
const camoufox_js_1 = require("camoufox-js");
|
|
17
17
|
const playwright_2 = require("playwright");
|
|
18
18
|
const camoufox_logger_service_1 = require("./services/camoufox.logger.service");
|
|
19
|
+
const camoufox_hook_service_1 = require("./services/camoufox.hook.service");
|
|
19
20
|
const camoufox_browser_service_1 = require("./services/camoufox.browser.service");
|
|
20
21
|
const camoufox_env_constant_1 = require("./constant/camoufox.env.constant");
|
|
21
22
|
const camoufox_env_constant_2 = require("./constant/camoufox.env.constant");
|
|
@@ -24,9 +25,10 @@ const camoufox_env_constant_4 = require("./constant/camoufox.env.constant");
|
|
|
24
25
|
const camoufox_env_constant_5 = require("./constant/camoufox.env.constant");
|
|
25
26
|
const camoufox_env_constant_6 = require("./constant/camoufox.env.constant");
|
|
26
27
|
let CamoufoxService = class CamoufoxService {
|
|
27
|
-
constructor(config, logger, browser) {
|
|
28
|
+
constructor(config, logger, hook, browser) {
|
|
28
29
|
this.config = config;
|
|
29
30
|
this.logger = logger;
|
|
31
|
+
this.hook = hook;
|
|
30
32
|
this.browser = browser;
|
|
31
33
|
this.maxConcurrency = this.config.get(camoufox_env_constant_1.MAX_CONCURRENCY);
|
|
32
34
|
this.minConcurrency = this.config.get(camoufox_env_constant_2.MIN_CONCURRENCY);
|
|
@@ -37,6 +39,9 @@ let CamoufoxService = class CamoufoxService {
|
|
|
37
39
|
}
|
|
38
40
|
async create(options) {
|
|
39
41
|
this.logger.log('🚀 Creating Camoufox Crawler instance');
|
|
42
|
+
const { preNavigationHooks = [], postNavigationHooks = [] } = options;
|
|
43
|
+
const defaultPreNavHooks = this.hook.getPreNavigationHooks();
|
|
44
|
+
const defaultPostNavHooks = this.hook.getPostNavigationHooks();
|
|
40
45
|
const defaultOptions = {
|
|
41
46
|
maxRequestsPerCrawl: this.maxRequests,
|
|
42
47
|
requestHandlerTimeoutSecs: this.timeoutSecs,
|
|
@@ -56,7 +61,12 @@ let CamoufoxService = class CamoufoxService {
|
|
|
56
61
|
useFingerprints: false,
|
|
57
62
|
},
|
|
58
63
|
};
|
|
59
|
-
const crawler = new playwright_1.PlaywrightCrawler({
|
|
64
|
+
const crawler = new playwright_1.PlaywrightCrawler({
|
|
65
|
+
...defaultOptions,
|
|
66
|
+
...options,
|
|
67
|
+
preNavigationHooks: [...defaultPreNavHooks, ...preNavigationHooks],
|
|
68
|
+
postNavigationHooks: [...defaultPostNavHooks, ...postNavigationHooks],
|
|
69
|
+
});
|
|
60
70
|
this.logger.log('✅ Camoufox Crawler instance created successfully');
|
|
61
71
|
return crawler;
|
|
62
72
|
}
|
|
@@ -69,6 +79,7 @@ exports.CamoufoxService = CamoufoxService = __decorate([
|
|
|
69
79
|
(0, common_1.Injectable)(),
|
|
70
80
|
__metadata("design:paramtypes", [config_1.ConfigService,
|
|
71
81
|
camoufox_logger_service_1.CamoufoxLoggerService,
|
|
82
|
+
camoufox_hook_service_1.CamoufoxHookService,
|
|
72
83
|
camoufox_browser_service_1.CamoufoxBrowserService])
|
|
73
84
|
], CamoufoxService);
|
|
74
85
|
//# sourceMappingURL=camoufox.service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"camoufox.service.js","sourceRoot":"","sources":["../../../src/modules/camoufox/camoufox.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAE/C,oDAAwD;AAExD,6CAA4C;AAC5C,2CAAqC;AAErC,gFAA2E;AAC3E,kFAA6E;AAE7E,4EAAmE;AACnE,4EAAmE;AACnE,4EAAuE;AACvE,4EAAgE;AAChE,4EAAgE;AAChE,4EAA4D;AAGrD,IAAM,eAAe,GAArB,MAAM,eAAe;IAQ1B,YACmB,MAAqB,EACrB,MAA6B,EAC7B,OAA+B;
|
|
1
|
+
{"version":3,"file":"camoufox.service.js","sourceRoot":"","sources":["../../../src/modules/camoufox/camoufox.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2CAA+C;AAE/C,oDAAwD;AAExD,6CAA4C;AAC5C,2CAAqC;AAErC,gFAA2E;AAC3E,4EAAuE;AACvE,kFAA6E;AAE7E,4EAAmE;AACnE,4EAAmE;AACnE,4EAAuE;AACvE,4EAAgE;AAChE,4EAAgE;AAChE,4EAA4D;AAGrD,IAAM,eAAe,GAArB,MAAM,eAAe;IAQ1B,YACmB,MAAqB,EACrB,MAA6B,EAC7B,IAAyB,EACzB,OAA+B;QAH/B,WAAM,GAAN,MAAM,CAAe;QACrB,WAAM,GAAN,MAAM,CAAuB;QAC7B,SAAI,GAAJ,IAAI,CAAqB;QACzB,YAAO,GAAP,OAAO,CAAwB;QAEhD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,uCAAe,CAAC,CAAC;QAC/D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,uCAAe,CAAC,CAAC;QAC/D,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,2CAAmB,CAAC,CAAC;QACtE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,oCAAY,CAAC,CAAC;QACzD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,oCAAY,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAU,gCAAQ,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAiC;QAC5C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QACzD,MAAM,EAAE,kBAAkB,GAAG,EAAE,EAAE,mBAAmB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/D,MAAM,cAAc,GAA6B;YAC/C,mBAAmB,EAAE,IAAI,CAAC,WAAW;YACrC,yBAAyB,EAAE,IAAI,CAAC,WAAW;YAC3C,qBAAqB,EAAE,IAAI,CAAC,WAAW;YACvC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE;gBACb,QAAQ,EAAE,oBAAO;gBACjB,aAAa,EAAE,MAAM,IAAA,2BAAa,EAAC;oBACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACxB,CAAC;aACH;YACD,kBAAkB,EAAE;gBAClB,eAAe,EAAE,KAAK;aACvB;SACF,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,8BAAiB,CAAC;YACpC,GAAG,cAAc;YACjB,GAAG,OAAO;YACV,kBAAkB,EAAE,CAAC,GAAG,kBAAkB,EAAE,GAAG,kBAAkB,CAAC;YAClE,mBAAmB,EAAE,CAAC,GAAG,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;SACtE,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;QACpE,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;IACnC,CAAC;CACF,CAAA;AA3DY,0CAAe;0BAAf,eAAe;IAD3B,IAAA,mBAAU,GAAE;qCAUgB,sBAAa;QACb,+CAAqB;QACvB,2CAAmB;QAChB,iDAAsB;GAZvC,eAAe,CA2D3B"}
|
|
@@ -8,9 +8,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
9
|
exports.CheerioModule = void 0;
|
|
10
10
|
const common_1 = require("@nestjs/common");
|
|
11
|
-
const config_1 = require("@nestjs/config");
|
|
12
11
|
const cheerio_service_1 = require("./cheerio.service");
|
|
13
|
-
const cheerio_env_config_1 = require("./env/cheerio.env.config");
|
|
14
12
|
const cheerio_logger_service_1 = require("./services/cheerio.logger.service");
|
|
15
13
|
const cheerio_hook_service_1 = require("./services/cheerio.hook.service");
|
|
16
14
|
let CheerioModule = class CheerioModule {
|
|
@@ -18,7 +16,6 @@ let CheerioModule = class CheerioModule {
|
|
|
18
16
|
exports.CheerioModule = CheerioModule;
|
|
19
17
|
exports.CheerioModule = CheerioModule = __decorate([
|
|
20
18
|
(0, common_1.Module)({
|
|
21
|
-
imports: [config_1.ConfigModule.forFeature(cheerio_env_config_1.CheerioEnvConfig)],
|
|
22
19
|
providers: [cheerio_service_1.CheerioService, cheerio_logger_service_1.CheerioLoggerService, cheerio_hook_service_1.CheerioHookService],
|
|
23
20
|
exports: [cheerio_service_1.CheerioService],
|
|
24
21
|
})
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.module.js","sourceRoot":"","sources":["../../../src/modules/cheerio/cheerio.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;
|
|
1
|
+
{"version":3,"file":"cheerio.module.js","sourceRoot":"","sources":["../../../src/modules/cheerio/cheerio.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,uDAAmD;AACnD,8EAAyE;AACzE,0EAAqE;AAM9D,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,sCAAa;wBAAb,aAAa;IAJzB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE,CAAC,gCAAc,EAAE,6CAAoB,EAAE,yCAAkB,CAAC;QACrE,OAAO,EAAE,CAAC,gCAAc,CAAC;KAC1B,CAAC;GACW,aAAa,CAAG"}
|
|
@@ -8,9 +8,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
9
|
exports.PlaywrightModule = void 0;
|
|
10
10
|
const common_1 = require("@nestjs/common");
|
|
11
|
-
const config_1 = require("@nestjs/config");
|
|
12
11
|
const playwright_service_1 = require("./playwright.service");
|
|
13
|
-
const playwright_env_config_1 = require("./env/playwright.env.config");
|
|
14
12
|
const playwright_logger_service_1 = require("./services/playwright.logger.service");
|
|
15
13
|
const playwright_hook_service_1 = require("./services/playwright.hook.service");
|
|
16
14
|
const playwright_browser_service_1 = require("./services/playwright.browser.service");
|
|
@@ -19,7 +17,6 @@ let PlaywrightModule = class PlaywrightModule {
|
|
|
19
17
|
exports.PlaywrightModule = PlaywrightModule;
|
|
20
18
|
exports.PlaywrightModule = PlaywrightModule = __decorate([
|
|
21
19
|
(0, common_1.Module)({
|
|
22
|
-
imports: [config_1.ConfigModule.forFeature(playwright_env_config_1.PlaywrightEnvConfig)],
|
|
23
20
|
providers: [
|
|
24
21
|
playwright_service_1.PlaywrightService,
|
|
25
22
|
playwright_logger_service_1.PlaywrightLoggerService,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.module.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;
|
|
1
|
+
{"version":3,"file":"playwright.module.js","sourceRoot":"","sources":["../../../src/modules/playwright/playwright.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AAExC,6DAAyD;AACzD,oFAA+E;AAC/E,gFAA2E;AAC3E,sFAAiF;AAW1E,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;CAAG,CAAA;AAAnB,4CAAgB;2BAAhB,gBAAgB;IAT5B,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,sCAAiB;YACjB,mDAAuB;YACvB,+CAAqB;YACrB,qDAAwB;SACzB;QACD,OAAO,EAAE,CAAC,sCAAiB,CAAC;KAC7B,CAAC;GACW,gBAAgB,CAAG"}
|