@rafikidota/cuba-ofertas-scraping 0.27.0 → 0.27.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/cloudflare/cloudflare.service.js +2 -2
- package/dist/modules/cloudflare/cloudflare.service.js.map +1 -1
- package/dist/modules/delayer/use-cases/delayer-mapper.service.d.ts +0 -2
- package/dist/modules/delayer/use-cases/delayer-mapper.service.js +6 -18
- package/dist/modules/delayer/use-cases/delayer-mapper.service.js.map +1 -1
- package/package.json +1 -1
|
@@ -39,10 +39,10 @@ let CloudflareService = CloudflareService_1 = class CloudflareService {
|
|
|
39
39
|
async handle(response, blockOptions) {
|
|
40
40
|
const normalized = this.normalizer.execute(response);
|
|
41
41
|
const { options } = new handle_request_types_1.HandleBlockOption(blockOptions);
|
|
42
|
-
const {
|
|
42
|
+
const { url } = normalized;
|
|
43
43
|
const { method, body, delay } = options;
|
|
44
44
|
this.repository.add({ url, method, body });
|
|
45
|
-
this.logger.
|
|
45
|
+
this.logger.log(`🚧 ${url}`);
|
|
46
46
|
return await this.delayer.wait(url, delay?.min, delay?.max);
|
|
47
47
|
}
|
|
48
48
|
report(stats) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cloudflare.service.js","sourceRoot":"","sources":["../../../src/modules/cloudflare/cloudflare.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAGhD,gEAA4D;AAC5D,uDAAoD;AACpD,iGAA0F;AAC1F,uFAAiF;AACjF,4FAAuF;AACvF,+GAAwG;AAKxG,uEAAiE;AACjE,6DAA6D;AAGtD,IAAM,iBAAiB,yBAAvB,MAAM,iBAAiB;IAG5B,YACmB,OAAuB,EACvB,OAAoC,EACpC,QAAiC,EACjC,UAAuC,EACvC,UAA8C;QAJ9C,YAAO,GAAP,OAAO,CAAgB;QACvB,YAAO,GAAP,OAAO,CAA6B;QACpC,aAAQ,GAAR,QAAQ,CAAyB;QACjC,eAAU,GAAV,UAAU,CAA6B;QACvC,eAAU,GAAV,UAAU,CAAoC;QAPzD,WAAM,GAAG,IAAI,eAAM,CAAC,mBAAiB,CAAC,IAAI,CAAC,CAAC;IAQjD,CAAC;IAEJ,KAAK,CAAC,QAAyB;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAyB,EAAE,YAAiC;QACvE,MAAM,UAAU,GAAuB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,wCAAiB,CAAC,YAAY,CAAC,CAAC;QACxD,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"cloudflare.service.js","sourceRoot":"","sources":["../../../src/modules/cloudflare/cloudflare.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAGhD,gEAA4D;AAC5D,uDAAoD;AACpD,iGAA0F;AAC1F,uFAAiF;AACjF,4FAAuF;AACvF,+GAAwG;AAKxG,uEAAiE;AACjE,6DAA6D;AAGtD,IAAM,iBAAiB,yBAAvB,MAAM,iBAAiB;IAG5B,YACmB,OAAuB,EACvB,OAAoC,EACpC,QAAiC,EACjC,UAAuC,EACvC,UAA8C;QAJ9C,YAAO,GAAP,OAAO,CAAgB;QACvB,YAAO,GAAP,OAAO,CAA6B;QACpC,aAAQ,GAAR,QAAQ,CAAyB;QACjC,eAAU,GAAV,UAAU,CAA6B;QACvC,eAAU,GAAV,UAAU,CAAoC;QAPzD,WAAM,GAAG,IAAI,eAAM,CAAC,mBAAiB,CAAC,IAAI,CAAC,CAAC;IAQjD,CAAC;IAEJ,KAAK,CAAC,QAAyB;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAyB,EAAE,YAAiC;QACvE,MAAM,UAAU,GAAuB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,wCAAiB,CAAC,YAAY,CAAC,CAAC;QACxD,MAAM,EAAE,GAAG,EAAE,GAAG,UAAU,CAAC;QAC3B,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACxC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QAE7B,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,CAAC,KAAsB;QAC3B,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IAGD,KAAK;QACH,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;CACF,CAAA;AAtCY,8CAAiB;AAmC5B;IADC,IAAA,uBAAO,EAAC,0BAAa,CAAC,KAAK,CAAC;;;;8CAG5B;4BArCU,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAKiB,gCAAc;QACd,6DAA2B;QAC1B,oDAAuB;QACrB,2DAA2B;QAC3B,2EAAkC;GARtD,iBAAiB,CAsC7B"}
|
|
@@ -55,30 +55,18 @@ let DelayerMapService = class DelayerMapService {
|
|
|
55
55
|
if (!current) {
|
|
56
56
|
return { min, max };
|
|
57
57
|
}
|
|
58
|
-
if (this.check(current, mapConfig.max)) {
|
|
59
|
-
return this.build(url);
|
|
60
|
-
}
|
|
61
58
|
return this.increase(url, current, mapConfig);
|
|
62
59
|
}
|
|
63
|
-
|
|
64
|
-
return range.min >= max && range.max >= max;
|
|
65
|
-
}
|
|
66
|
-
build(url) {
|
|
60
|
+
increase(url, range, config) {
|
|
67
61
|
const retries = this.retries.get(url) ?? 0;
|
|
68
62
|
const count = retries + 1;
|
|
69
63
|
this.retries.set(url, count);
|
|
70
|
-
const
|
|
71
|
-
const
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
this.logger.log(`🔄 Backoff exponential: ${delay}ms (retry #${count})`);
|
|
75
|
-
return { min: delay, max: delay };
|
|
76
|
-
}
|
|
77
|
-
increase(url, range, config) {
|
|
78
|
-
const min = Math.min(config.max, range.min + config.step);
|
|
79
|
-
const max = Math.min(config.max, range.max + config.step);
|
|
64
|
+
const multiplier = Math.min(1 + count * 0.5, 10);
|
|
65
|
+
const step = config.step * multiplier;
|
|
66
|
+
const min = Math.min(config.max, range.min + step);
|
|
67
|
+
const max = Math.min(config.max, range.max + step);
|
|
80
68
|
this.logger.log(`🔇 ${url}`);
|
|
81
|
-
this.logger.log(`🔂
|
|
69
|
+
this.logger.log(`🔂 ${count}. New range: ${min}ms - ${max}ms`);
|
|
82
70
|
return { min, max };
|
|
83
71
|
}
|
|
84
72
|
async sleep(ms) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"delayer-mapper.service.js","sourceRoot":"","sources":["../../../../src/modules/delayer/use-cases/delayer-mapper.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAE5C,0DAAuD;AACvD,wDAAoD;AACpD,qEAAgE;AAKzD,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAK5B,YAA6B,MAA4B;QAA5B,WAAM,GAAN,MAAM,CAAsB;QAJxC,WAAM,GAAG,IAAI,eAAM,CAAC,gCAAc,CAAC,IAAI,CAAC,CAAC;QACzC,WAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;QACvC,YAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEO,CAAC;IAE7D,KAAK,CAAC,IAAI,CAAC,GAAY,EAAE,GAAY,EAAE,GAAY;QACjD,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC5B,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;YAChB,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QAClB,CAAC;QAED,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACrE,MAAM,QAAQ,GAAG,GAAG,IAAI,UAAU,CAAC;QACnC,MAAM,QAAQ,GAAG,GAAG,IAAI,UAAU,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAExD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,GAAY;QAChB,IAAI,GAAG,EAAE,CAAC;YACR,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACxB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IAED,SAAS;QACP,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAEO,OAAO,CAAC,GAAW,EAAE,GAAY,EAAE,GAAY;QACrD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAE7C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;QACtB,CAAC;QAED,
|
|
1
|
+
{"version":3,"file":"delayer-mapper.service.js","sourceRoot":"","sources":["../../../../src/modules/delayer/use-cases/delayer-mapper.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAE5C,0DAAuD;AACvD,wDAAoD;AACpD,qEAAgE;AAKzD,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAK5B,YAA6B,MAA4B;QAA5B,WAAM,GAAN,MAAM,CAAsB;QAJxC,WAAM,GAAG,IAAI,eAAM,CAAC,gCAAc,CAAC,IAAI,CAAC,CAAC;QACzC,WAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;QACvC,YAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEO,CAAC;IAE7D,KAAK,CAAC,IAAI,CAAC,GAAY,EAAE,GAAY,EAAE,GAAY;QACjD,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC5B,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;YAChB,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QAClB,CAAC;QAED,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACrE,MAAM,QAAQ,GAAG,GAAG,IAAI,UAAU,CAAC;QACnC,MAAM,QAAQ,GAAG,GAAG,IAAI,UAAU,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAExD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,GAAY;QAChB,IAAI,GAAG,EAAE,CAAC;YACR,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACxB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IAED,SAAS;QACP,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAEO,OAAO,CAAC,GAAW,EAAE,GAAY,EAAE,GAAY;QACrD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAE7C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;QACtB,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAChD,CAAC;IAEO,QAAQ,CAAC,GAAW,EAAE,KAAiB,EAAE,MAAwB;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,OAAO,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAE7B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,GAAG,EAAE,EAAE,CAAC,CAAC;QACjD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,GAAG,UAAU,CAAC;QAEtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;QAEnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,KAAK,gBAAgB,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC;QAE/D,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,EAAU;QAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;CACF,CAAA;AAxEY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAM0B,6CAAoB;GAL9C,iBAAiB,CAwE7B"}
|