@rafikidota/cuba-ofertas-scraping 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/cloudflare/cloudflare.service.d.ts +2 -1
- package/dist/modules/cloudflare/cloudflare.service.js +5 -2
- package/dist/modules/cloudflare/cloudflare.service.js.map +1 -1
- package/dist/modules/cloudflare/types/handle-request.types.d.ts +18 -0
- package/dist/modules/cloudflare/types/handle-request.types.js +21 -0
- package/dist/modules/cloudflare/types/handle-request.types.js.map +1 -0
- package/package.json +1 -1
|
@@ -5,6 +5,7 @@ import { ReportCloudflareUseCase } from './use-cases/report-cloudflare.use-case'
|
|
|
5
5
|
import { CloudflareRepositoryService } from './services/cloudflare-repository.service';
|
|
6
6
|
import { NormalizeCloudflareResponseUseCase } from './use-cases/normalize-cloudflare-response.use-case';
|
|
7
7
|
import { CrawleeResponse } from './types/normalize-response.types';
|
|
8
|
+
import { IHandleBlockOption } from './types/handle-request.types';
|
|
8
9
|
export declare class CloudflareService {
|
|
9
10
|
private readonly delayer;
|
|
10
11
|
private readonly checker;
|
|
@@ -15,7 +16,7 @@ export declare class CloudflareService {
|
|
|
15
16
|
constructor(delayer: DelayerService, checker: CheckCloudflareBlockUseCase, reporter: ReportCloudflareUseCase, repository: CloudflareRepositoryService, normalizer: NormalizeCloudflareResponseUseCase);
|
|
16
17
|
check(response: CrawleeResponse): boolean;
|
|
17
18
|
get blocks(): import("./types/block-info").BlockInfo[];
|
|
18
|
-
handle(response: CrawleeResponse,
|
|
19
|
+
handle(response: CrawleeResponse, blockOptions?: IHandleBlockOption): Promise<void>;
|
|
19
20
|
report(stats: FinalStatistics): {
|
|
20
21
|
blocks: number;
|
|
21
22
|
total: number;
|
|
@@ -18,6 +18,7 @@ const check_cloudflare_block_use_case_1 = require("./use-cases/check-cloudflare-
|
|
|
18
18
|
const report_cloudflare_use_case_1 = require("./use-cases/report-cloudflare.use-case");
|
|
19
19
|
const cloudflare_repository_service_1 = require("./services/cloudflare-repository.service");
|
|
20
20
|
const normalize_cloudflare_response_use_case_1 = require("./use-cases/normalize-cloudflare-response.use-case");
|
|
21
|
+
const handle_request_types_1 = require("./types/handle-request.types");
|
|
21
22
|
let CloudflareService = CloudflareService_1 = class CloudflareService {
|
|
22
23
|
constructor(delayer, checker, reporter, repository, normalizer) {
|
|
23
24
|
this.delayer = delayer;
|
|
@@ -33,12 +34,14 @@ let CloudflareService = CloudflareService_1 = class CloudflareService {
|
|
|
33
34
|
get blocks() {
|
|
34
35
|
return this.repository.blocks;
|
|
35
36
|
}
|
|
36
|
-
async handle(response,
|
|
37
|
+
async handle(response, blockOptions) {
|
|
37
38
|
const normalized = this.normalizer.execute(response);
|
|
39
|
+
const { options } = new handle_request_types_1.HandleBlockOption(blockOptions);
|
|
38
40
|
const { statusCode, url } = normalized;
|
|
41
|
+
const { method, body, delay } = options;
|
|
39
42
|
this.repository.add({ url, method, body });
|
|
40
43
|
this.logger.warn(`⚠️ ${statusCode} ${url}`);
|
|
41
|
-
return await this.delayer.wait(url);
|
|
44
|
+
return await this.delayer.wait(url, delay.min, delay.max);
|
|
42
45
|
}
|
|
43
46
|
report(stats) {
|
|
44
47
|
return this.reporter.execute(stats);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cloudflare.service.js","sourceRoot":"","sources":["../../../src/modules/cloudflare/cloudflare.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAG5C,gEAA4D;AAC5D,uDAAoD;AACpD,iGAA0F;AAC1F,uFAAiF;AACjF,4FAAuF;AACvF,+GAAwG;
|
|
1
|
+
{"version":3,"file":"cloudflare.service.js","sourceRoot":"","sources":["../../../src/modules/cloudflare/cloudflare.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAG5C,gEAA4D;AAC5D,uDAAoD;AACpD,iGAA0F;AAC1F,uFAAiF;AACjF,4FAAuF;AACvF,+GAAwG;AAKxG,uEAAiE;AAG1D,IAAM,iBAAiB,yBAAvB,MAAM,iBAAiB;IAG5B,YACmB,OAAuB,EACvB,OAAoC,EACpC,QAAiC,EACjC,UAAuC,EACvC,UAA8C;QAJ9C,YAAO,GAAP,OAAO,CAAgB;QACvB,YAAO,GAAP,OAAO,CAA6B;QACpC,aAAQ,GAAR,QAAQ,CAAyB;QACjC,eAAU,GAAV,UAAU,CAA6B;QACvC,eAAU,GAAV,UAAU,CAAoC;QAPzD,WAAM,GAAG,IAAI,eAAM,CAAC,mBAAiB,CAAC,IAAI,CAAC,CAAC;IAQjD,CAAC;IAEJ,KAAK,CAAC,QAAyB;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAyB,EAAE,YAAiC;QACvE,MAAM,UAAU,GAAuB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,wCAAiB,CAAC,YAAY,CAAC,CAAC;QACxD,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACxC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;QAC7C,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,CAAC,KAAsB;QAC3B,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACF,CAAA;AApCY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAKiB,gCAAc;QACd,6DAA2B;QAC1B,oDAAuB;QACrB,2DAA2B;QAC3B,2EAAkC;GARtD,iBAAiB,CAoC7B"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { HttpMethod } from '../../../common/enum/request.enum';
|
|
2
|
+
import { DelayRange } from '../../../common/types/delay-range';
|
|
3
|
+
export type IHandleBlockOption = {
|
|
4
|
+
method: HttpMethod;
|
|
5
|
+
body?: any;
|
|
6
|
+
delay?: DelayRange;
|
|
7
|
+
};
|
|
8
|
+
export declare class HandleBlockOption {
|
|
9
|
+
private readonly method;
|
|
10
|
+
private body;
|
|
11
|
+
private delay;
|
|
12
|
+
constructor(options: IHandleBlockOption);
|
|
13
|
+
get options(): {
|
|
14
|
+
method: HttpMethod;
|
|
15
|
+
body: any;
|
|
16
|
+
delay: DelayRange;
|
|
17
|
+
};
|
|
18
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HandleBlockOption = void 0;
|
|
4
|
+
const request_enum_1 = require("../../../common/enum/request.enum");
|
|
5
|
+
class HandleBlockOption {
|
|
6
|
+
constructor(options) {
|
|
7
|
+
this.method = request_enum_1.HttpMethod.GET;
|
|
8
|
+
this.method = options.method ?? this.method;
|
|
9
|
+
this.body = options.body ?? this.body;
|
|
10
|
+
this.delay = options.delay ?? this.delay;
|
|
11
|
+
}
|
|
12
|
+
get options() {
|
|
13
|
+
return {
|
|
14
|
+
method: this.method,
|
|
15
|
+
body: this.body,
|
|
16
|
+
delay: this.delay,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
exports.HandleBlockOption = HandleBlockOption;
|
|
21
|
+
//# sourceMappingURL=handle-request.types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"handle-request.types.js","sourceRoot":"","sources":["../../../../src/modules/cloudflare/types/handle-request.types.ts"],"names":[],"mappings":";;;AAAA,oEAA+D;AAS/D,MAAa,iBAAiB;IAK5B,YAAY,OAA2B;QAJtB,WAAM,GAAe,yBAAU,CAAC,GAAG,CAAC;QAKnD,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;QAC5C,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;IAC3C,CAAC;IAED,IAAI,OAAO;QACT,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC;IACJ,CAAC;CACF;AAlBD,8CAkBC"}
|