@rafikidota/cuba-ofertas-scraping 0.40.0 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/publisher/publisher.service.js +1 -1
- package/dist/modules/publisher/publisher.service.js.map +1 -1
- package/dist/modules/repository/services/category.repository.service.js +17 -7
- package/dist/modules/repository/services/category.repository.service.js.map +1 -1
- package/dist/modules/repository/services/image.repository.service.js +17 -7
- package/dist/modules/repository/services/image.repository.service.js.map +1 -1
- package/dist/modules/repository/services/provider.repository.service.js +17 -7
- package/dist/modules/repository/services/provider.repository.service.js.map +1 -1
- package/dist/modules/repository/services/province.repository.service.js +17 -7
- package/dist/modules/repository/services/province.repository.service.js.map +1 -1
- package/package.json +4 -4
|
@@ -41,7 +41,7 @@ let PublisherService = PublisherService_1 = class PublisherService {
|
|
|
41
41
|
const stringified = JSON.stringify(result, null, 2);
|
|
42
42
|
const key = await this._upload({
|
|
43
43
|
buffer: Buffer.from(stringified),
|
|
44
|
-
fileName: slug
|
|
44
|
+
fileName: `data/${slug}`,
|
|
45
45
|
extension: 'json',
|
|
46
46
|
mimeType: 'application/json',
|
|
47
47
|
message: '✅ Scraping results uploaded to MinIO',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"publisher.service.js","sourceRoot":"","sources":["../../../src/modules/publisher/publisher.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAChD,2BAAoC;AAEpC,uDAAoD;AACpD,sEAAkE;AAClE,gEAA4D;AAC5D,0DAAsD;AACtD,6DAA6D;AAKtD,IAAM,gBAAgB,wBAAtB,MAAM,gBAAgB;IAE3B,YACmB,SAA2B,EAC3B,KAAmB,EACnB,OAAuB;QAFvB,cAAS,GAAT,SAAS,CAAkB;QAC3B,UAAK,GAAL,KAAK,CAAc;QACnB,YAAO,GAAP,OAAO,CAAgB;QAJzB,WAAM,GAAG,IAAI,eAAM,CAAC,kBAAgB,CAAC,IAAI,CAAC,CAAC;IAKzD,CAAC;IAEI,KAAK,CAAC,IAAI,CAAC,IAAY;QAC7B,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAA0B;QAC9C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,MAAuB;QAC1C,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QACxB,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAEpD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAC7B,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;YAChC,QAAQ,EAAE,IAAI;
|
|
1
|
+
{"version":3,"file":"publisher.service.js","sourceRoot":"","sources":["../../../src/modules/publisher/publisher.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAChD,2BAAoC;AAEpC,uDAAoD;AACpD,sEAAkE;AAClE,gEAA4D;AAC5D,0DAAsD;AACtD,6DAA6D;AAKtD,IAAM,gBAAgB,wBAAtB,MAAM,gBAAgB;IAE3B,YACmB,SAA2B,EAC3B,KAAmB,EACnB,OAAuB;QAFvB,cAAS,GAAT,SAAS,CAAkB;QAC3B,UAAK,GAAL,KAAK,CAAc;QACnB,YAAO,GAAP,OAAO,CAAgB;QAJzB,WAAM,GAAG,IAAI,eAAM,CAAC,kBAAgB,CAAC,IAAI,CAAC,CAAC;IAKzD,CAAC;IAEI,KAAK,CAAC,IAAI,CAAC,IAAY;QAC7B,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAA0B;QAC9C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,MAAuB;QAC1C,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QACxB,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAEpD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAC7B,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;YAChC,QAAQ,EAAE,QAAQ,IAAI,EAAE;YACxB,SAAS,EAAE,MAAM;YACjB,QAAQ,EAAE,kBAAkB;YAC5B,OAAO,EAAE,sCAAsC;SAChD,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,OAAO,CAAC;YACjB,MAAM,EAAE,MAAM,aAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC;YAC/C,QAAQ,EAAE,aAAa,IAAI,EAAE;YAC7B,SAAS,EAAE,IAAI;YACf,QAAQ,EAAE,uBAAuB;YACjC,OAAO,EAAE,8BAA8B;SACxC,CAAC,CAAC;QAEH,OAAO,GAAG,CAAC;IACb,CAAC;IAGK,AAAN,KAAK,CAAC,OAAO,CAAC,IAAY;QACxB,IAAI,CAAC;YACH,MAAM,MAAM,GAAoB,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEtD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;gBACrD,OAAO;YACT,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,IAAI,EAAE,CAAC,CAAC;gBACzD,OAAO;YACT,CAAC;YAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAE3C,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC;YAEpD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACnC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,EAAE,OAAO,EAAE,GAAG,KAAc,CAAC;YACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;CACF,CAAA;AAtEY,4CAAgB;AA4CrB;IADL,IAAA,uBAAO,EAAC,0BAAa,CAAC,kBAAkB,CAAC;;;;+CA0BzC;2BArEU,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;qCAImB,oCAAgB;QACpB,4BAAY;QACV,gCAAc;GAL/B,gBAAgB,CAsE5B"}
|
|
@@ -35,22 +35,32 @@ let CategoryRepositoryService = class CategoryRepositoryService {
|
|
|
35
35
|
return this.repository.count();
|
|
36
36
|
}
|
|
37
37
|
async save(categories) {
|
|
38
|
+
const sizes = { previous: 0, current: 0 };
|
|
39
|
+
const multiples = { previous: 0, current: 0 };
|
|
40
|
+
sizes.previous = await this.count();
|
|
38
41
|
const entities = [];
|
|
39
42
|
for (const category of categories) {
|
|
40
43
|
const data = category.getData();
|
|
41
44
|
const { id, name } = data;
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
if (id) {
|
|
46
|
+
const exists = await this.repository.existsBy({ id });
|
|
47
|
+
if (!exists) {
|
|
48
|
+
const entity = this.repository.create(data);
|
|
49
|
+
entities.push(entity);
|
|
50
|
+
this.logger.debug(`✅ ${name}`);
|
|
51
|
+
}
|
|
47
52
|
}
|
|
48
53
|
}
|
|
49
54
|
if (entities.length > 0) {
|
|
50
55
|
await this.repository.save(entities);
|
|
51
56
|
}
|
|
52
|
-
|
|
53
|
-
|
|
57
|
+
sizes.current = await this.repository.count();
|
|
58
|
+
multiples.previous = Math.floor(sizes.previous / 10);
|
|
59
|
+
multiples.current = Math.floor(sizes.current / 10);
|
|
60
|
+
if (multiples.current > multiples.previous) {
|
|
61
|
+
const message = `✅ ${sizes.current} categories has been saved successfully`;
|
|
62
|
+
this.logger.verbose(message);
|
|
63
|
+
}
|
|
54
64
|
}
|
|
55
65
|
async reset() {
|
|
56
66
|
await this.repository.clear();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"category.repository.service.js","sourceRoot":"","sources":["../../../../src/modules/repository/services/category.repository.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAChD,6CAAmD;AACnD,qCAAqC;AAErC,8DAA0D;AAC1D,0DAAuD;AACvD,6DAA0D;AAC1D,iEAA6D;AAC7D,gEAAgE;AAGzD,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IAGpC,YAEE,UAAuD;QAAtC,eAAU,GAAV,UAAU,CAA4B;QAJjD,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAKjD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC9C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,mBAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAGK,AAAN,KAAK,CAAC,IAAI,CAAC,UAAsB;QAC/B,MAAM,QAAQ,GAAqB,EAAE,CAAC;QAEtC,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;YAE1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"category.repository.service.js","sourceRoot":"","sources":["../../../../src/modules/repository/services/category.repository.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAChD,6CAAmD;AACnD,qCAAqC;AAErC,8DAA0D;AAC1D,0DAAuD;AACvD,6DAA0D;AAC1D,iEAA6D;AAC7D,gEAAgE;AAGzD,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IAGpC,YAEE,UAAuD;QAAtC,eAAU,GAAV,UAAU,CAA4B;QAJjD,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAKjD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC9C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,mBAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAGK,AAAN,KAAK,CAAC,IAAI,CAAC,UAAsB;QAC/B,MAAM,KAAK,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC9C,KAAK,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAEpC,MAAM,QAAQ,GAAqB,EAAE,CAAC;QAEtC,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;YAE1B,IAAI,EAAE,EAAE,CAAC;gBACP,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC5C,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;QAED,KAAK,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC9C,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,EAAE,CAAC,CAAC;QACrD,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QAEnD,IAAI,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,KAAK,CAAC,OAAO,yCAAyC,CAAC;YAC5E,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;CACF,CAAA;AAxDY,8DAAyB;AAkB9B;IADL,IAAA,uBAAO,EAAC,0BAAa,CAAC,eAAe,CAAC;;;;qDAkCtC;oCAnDU,yBAAyB;IADrC,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,0BAAgB,EAAC,gCAAc,CAAC,CAAA;qCACJ,oBAAU;GAL9B,yBAAyB,CAwDrC"}
|
|
@@ -35,22 +35,32 @@ let ImageRepositoryService = class ImageRepositoryService {
|
|
|
35
35
|
return this.repository.count();
|
|
36
36
|
}
|
|
37
37
|
async save(images) {
|
|
38
|
+
const sizes = { previous: 0, current: 0 };
|
|
39
|
+
const multiples = { previous: 0, current: 0 };
|
|
40
|
+
sizes.previous = await this.count();
|
|
38
41
|
const entities = [];
|
|
39
42
|
for (const image of images) {
|
|
40
43
|
const data = image.getData();
|
|
41
44
|
const { id, name } = data;
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
if (id) {
|
|
46
|
+
const exists = await this.repository.existsBy({ id });
|
|
47
|
+
if (!exists) {
|
|
48
|
+
const entity = this.repository.create(data);
|
|
49
|
+
entities.push(entity);
|
|
50
|
+
this.logger.debug(`✅ ${name}`);
|
|
51
|
+
}
|
|
47
52
|
}
|
|
48
53
|
}
|
|
49
54
|
if (entities.length > 0) {
|
|
50
55
|
await this.repository.save(entities);
|
|
51
56
|
}
|
|
52
|
-
|
|
53
|
-
|
|
57
|
+
sizes.current = await this.repository.count();
|
|
58
|
+
multiples.previous = Math.floor(sizes.previous / 10);
|
|
59
|
+
multiples.current = Math.floor(sizes.current / 10);
|
|
60
|
+
if (multiples.current > multiples.previous) {
|
|
61
|
+
const message = `✅ ${sizes.current} images has been saved successfully`;
|
|
62
|
+
this.logger.verbose(message);
|
|
63
|
+
}
|
|
54
64
|
}
|
|
55
65
|
async reset() {
|
|
56
66
|
await this.repository.clear();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"image.repository.service.js","sourceRoot":"","sources":["../../../../src/modules/repository/services/image.repository.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAChD,6CAAmD;AACnD,qCAAqC;AAErC,8DAA0D;AAC1D,0DAAuD;AACvD,uDAAoD;AACpD,2DAAuD;AACvD,gEAAgE;AAGzD,IAAM,sBAAsB,GAA5B,MAAM,sBAAsB;IAGjC,YAEE,UAAoD;QAAnC,eAAU,GAAV,UAAU,CAAyB;QAJ9C,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAKjD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC9C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,aAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAGK,AAAN,KAAK,CAAC,IAAI,CAAC,MAAe;QACxB,MAAM,QAAQ,GAAkB,EAAE,CAAC;QAEnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;YAC7B,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;YAE1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"image.repository.service.js","sourceRoot":"","sources":["../../../../src/modules/repository/services/image.repository.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAChD,6CAAmD;AACnD,qCAAqC;AAErC,8DAA0D;AAC1D,0DAAuD;AACvD,uDAAoD;AACpD,2DAAuD;AACvD,gEAAgE;AAGzD,IAAM,sBAAsB,GAA5B,MAAM,sBAAsB;IAGjC,YAEE,UAAoD;QAAnC,eAAU,GAAV,UAAU,CAAyB;QAJ9C,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAKjD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC9C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,aAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAGK,AAAN,KAAK,CAAC,IAAI,CAAC,MAAe;QACxB,MAAM,KAAK,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC9C,KAAK,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAEpC,MAAM,QAAQ,GAAkB,EAAE,CAAC;QAEnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;YAC7B,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;YAE1B,IAAI,EAAE,EAAE,CAAC;gBACP,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC5C,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;QAED,KAAK,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC9C,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,EAAE,CAAC,CAAC;QACrD,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QAEnD,IAAI,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,KAAK,CAAC,OAAO,qCAAqC,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;CACF,CAAA;AAxDY,wDAAsB;AAkB3B;IADL,IAAA,uBAAO,EAAC,0BAAa,CAAC,WAAW,CAAC;;;;kDAkClC;iCAnDU,sBAAsB;IADlC,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,0BAAgB,EAAC,0BAAW,CAAC,CAAA;qCACD,oBAAU;GAL9B,sBAAsB,CAwDlC"}
|
|
@@ -35,22 +35,32 @@ let ProviderRepositoryService = class ProviderRepositoryService {
|
|
|
35
35
|
return this.repository.count();
|
|
36
36
|
}
|
|
37
37
|
async save(providers) {
|
|
38
|
+
const sizes = { previous: 0, current: 0 };
|
|
39
|
+
const multiples = { previous: 0, current: 0 };
|
|
40
|
+
sizes.previous = await this.count();
|
|
38
41
|
const entities = [];
|
|
39
42
|
for (const provider of providers) {
|
|
40
43
|
const data = provider.getData();
|
|
41
44
|
const { id, name } = data;
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
if (id) {
|
|
46
|
+
const exists = await this.repository.existsBy({ id });
|
|
47
|
+
if (!exists) {
|
|
48
|
+
const entity = this.repository.create(data);
|
|
49
|
+
entities.push(entity);
|
|
50
|
+
this.logger.debug(`✅ ${name}`);
|
|
51
|
+
}
|
|
47
52
|
}
|
|
48
53
|
}
|
|
49
54
|
if (entities.length > 0) {
|
|
50
55
|
await this.repository.save(entities);
|
|
51
56
|
}
|
|
52
|
-
|
|
53
|
-
|
|
57
|
+
sizes.current = await this.repository.count();
|
|
58
|
+
multiples.previous = Math.floor(sizes.previous / 10);
|
|
59
|
+
multiples.current = Math.floor(sizes.current / 10);
|
|
60
|
+
if (multiples.current > multiples.previous) {
|
|
61
|
+
const message = `✅ ${sizes.current} providers has been saved successfully`;
|
|
62
|
+
this.logger.verbose(message);
|
|
63
|
+
}
|
|
54
64
|
}
|
|
55
65
|
async reset() {
|
|
56
66
|
await this.repository.clear();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider.repository.service.js","sourceRoot":"","sources":["../../../../src/modules/repository/services/provider.repository.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAChD,6CAAmD;AACnD,qCAAqC;AAErC,8DAA0D;AAC1D,0DAAuD;AACvD,6DAA0D;AAC1D,iEAA6D;AAC7D,gEAAgE;AAGzD,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IAGpC,YAEE,UAAuD;QAAtC,eAAU,GAAV,UAAU,CAA4B;QAJjD,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAKjD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC9C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,mBAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAGK,AAAN,KAAK,CAAC,IAAI,CAAC,SAAqB;QAC9B,MAAM,QAAQ,GAAqB,EAAE,CAAC;QAEtC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;YAE1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"provider.repository.service.js","sourceRoot":"","sources":["../../../../src/modules/repository/services/provider.repository.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAChD,6CAAmD;AACnD,qCAAqC;AAErC,8DAA0D;AAC1D,0DAAuD;AACvD,6DAA0D;AAC1D,iEAA6D;AAC7D,gEAAgE;AAGzD,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IAGpC,YAEE,UAAuD;QAAtC,eAAU,GAAV,UAAU,CAA4B;QAJjD,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAKjD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC9C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,mBAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAGK,AAAN,KAAK,CAAC,IAAI,CAAC,SAAqB;QAC9B,MAAM,KAAK,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC9C,KAAK,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAEpC,MAAM,QAAQ,GAAqB,EAAE,CAAC;QAEtC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;YAE1B,IAAI,EAAE,EAAE,CAAC;gBACP,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC5C,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;QAED,KAAK,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC9C,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,EAAE,CAAC,CAAC;QACrD,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QAEnD,IAAI,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,KAAK,CAAC,OAAO,wCAAwC,CAAC;YAC3E,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;CACF,CAAA;AAxDY,8DAAyB;AAkB9B;IADL,IAAA,uBAAO,EAAC,0BAAa,CAAC,cAAc,CAAC;;;;qDAkCrC;oCAnDU,yBAAyB;IADrC,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,0BAAgB,EAAC,gCAAc,CAAC,CAAA;qCACJ,oBAAU;GAL9B,yBAAyB,CAwDrC"}
|
|
@@ -35,22 +35,32 @@ let ProvinceRepositoryService = class ProvinceRepositoryService {
|
|
|
35
35
|
return this.repository.count();
|
|
36
36
|
}
|
|
37
37
|
async save(provinces) {
|
|
38
|
+
const sizes = { previous: 0, current: 0 };
|
|
39
|
+
const multiples = { previous: 0, current: 0 };
|
|
40
|
+
sizes.previous = await this.count();
|
|
38
41
|
const entities = [];
|
|
39
42
|
for (const province of provinces) {
|
|
40
43
|
const data = province.getData();
|
|
41
44
|
const { id, name } = data;
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
if (id) {
|
|
46
|
+
const exists = await this.repository.existsBy({ id });
|
|
47
|
+
if (!exists) {
|
|
48
|
+
const entity = this.repository.create(data);
|
|
49
|
+
entities.push(entity);
|
|
50
|
+
this.logger.debug(`✅ ${name}`);
|
|
51
|
+
}
|
|
47
52
|
}
|
|
48
53
|
}
|
|
49
54
|
if (entities.length > 0) {
|
|
50
55
|
await this.repository.save(entities);
|
|
51
56
|
}
|
|
52
|
-
|
|
53
|
-
|
|
57
|
+
sizes.current = await this.repository.count();
|
|
58
|
+
multiples.previous = Math.floor(sizes.previous / 10);
|
|
59
|
+
multiples.current = Math.floor(sizes.current / 10);
|
|
60
|
+
if (multiples.current > multiples.previous) {
|
|
61
|
+
const message = `✅ ${sizes.current} provinces has been saved successfully`;
|
|
62
|
+
this.logger.verbose(message);
|
|
63
|
+
}
|
|
54
64
|
}
|
|
55
65
|
async reset() {
|
|
56
66
|
await this.repository.clear();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"province.repository.service.js","sourceRoot":"","sources":["../../../../src/modules/repository/services/province.repository.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAChD,6CAAmD;AACnD,qCAAqC;AAErC,0DAAuD;AACvD,gEAA4D;AAC5D,6DAA0D;AAC1D,iEAA6D;AAC7D,gEAAgE;AAGzD,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IAGpC,YAEE,UAAuD;QAAtC,eAAU,GAAV,UAAU,CAA4B;QAJjD,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAKjD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC9C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,mBAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAGK,AAAN,KAAK,CAAC,IAAI,CAAC,SAAqB;QAC9B,MAAM,QAAQ,GAAqB,EAAE,CAAC;QAEtC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;YAE1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"province.repository.service.js","sourceRoot":"","sources":["../../../../src/modules/repository/services/province.repository.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,yDAAgD;AAChD,6CAAmD;AACnD,qCAAqC;AAErC,0DAAuD;AACvD,gEAA4D;AAC5D,6DAA0D;AAC1D,iEAA6D;AAC7D,gEAAgE;AAGzD,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IAGpC,YAEE,UAAuD;QAAtC,eAAU,GAAV,UAAU,CAA4B;QAJjD,WAAM,GAAG,IAAI,eAAM,CAAC,sCAAiB,CAAC,IAAI,CAAC,CAAC;IAKjD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC9C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,mBAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAGK,AAAN,KAAK,CAAC,IAAI,CAAC,SAAqB;QAC9B,MAAM,KAAK,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC9C,KAAK,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAEpC,MAAM,QAAQ,GAAqB,EAAE,CAAC;QAEtC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;YAE1B,IAAI,EAAE,EAAE,CAAC;gBACP,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC5C,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;QAED,KAAK,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC9C,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,EAAE,CAAC,CAAC;QACrD,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QAEnD,IAAI,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,KAAK,CAAC,OAAO,wCAAwC,CAAC;YAC3E,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;CACF,CAAA;AAxDY,8DAAyB;AAkB9B;IADL,IAAA,uBAAO,EAAC,0BAAa,CAAC,cAAc,CAAC;;;;qDAkCrC;oCAnDU,yBAAyB;IADrC,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,0BAAgB,EAAC,gCAAc,CAAC,CAAA;qCACJ,oBAAU;GAL9B,yBAAyB,CAwDrC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rafikidota/cuba-ofertas-scraping",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.41.0",
|
|
4
4
|
"description": "Sometimes, the best way to solve your own problems is to help someone else.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
"module": "./dist/index.js",
|
|
16
16
|
"types": "./dist/index.d.ts",
|
|
17
17
|
"devDependencies": {
|
|
18
|
-
"@aws-sdk/client-s3": "3.
|
|
19
|
-
"@aws-sdk/s3-request-presigner": "3.
|
|
18
|
+
"@aws-sdk/client-s3": "3.1021.0",
|
|
19
|
+
"@aws-sdk/s3-request-presigner": "3.1021.0",
|
|
20
20
|
"@crawlee/cheerio": "3.16.0",
|
|
21
21
|
"@crawlee/playwright": "3.16.0",
|
|
22
22
|
"@eslint/eslintrc": "3.3.5",
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
"husky": "9.1.7",
|
|
53
53
|
"joi": "18.1.2",
|
|
54
54
|
"lint-staged": "16.4.0",
|
|
55
|
-
"playwright": "1.
|
|
55
|
+
"playwright": "1.59.0",
|
|
56
56
|
"prettier": "3.8.1",
|
|
57
57
|
"rimraf": "6.1.3",
|
|
58
58
|
"rxjs": "7.8.2",
|