@bytebrand/export-cars 1.5.2 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/alleskralle.config.js +3 -5
- package/dist/config/alleskralle.config.js.map +1 -1
- package/dist/config/as24.config.js +3 -1
- package/dist/config/as24.config.js.map +1 -1
- package/dist/config/auto24.config.js +3 -3
- package/dist/config/auto24.config.js.map +1 -1
- package/dist/config/autoboerse.config.js +5 -5
- package/dist/config/autoboerse.config.js.map +1 -1
- package/dist/config/autouncle.config.js +3 -2
- package/dist/config/autouncle.config.js.map +1 -1
- package/dist/config/by-owner.config.js +0 -1
- package/dist/config/by-owner.config.js.map +1 -1
- package/dist/config/config.d.ts +8 -12
- package/dist/config/config.js +14 -20
- package/dist/config/config.js.map +1 -1
- package/dist/config/criteo.config.js +3 -1
- package/dist/config/criteo.config.js.map +1 -1
- package/dist/config/dasparking.config.js +2 -4
- package/dist/config/dasparking.config.js.map +1 -1
- package/dist/config/eauto.config.js +4 -3
- package/dist/config/eauto.config.js.map +1 -1
- package/dist/config/fb.config.js +2 -1
- package/dist/config/fb.config.js.map +1 -1
- package/dist/config/google.config.js +2 -1
- package/dist/config/google.config.js.map +1 -1
- package/dist/config/index.d.ts +16 -1
- package/dist/config/index.js +16 -1
- package/dist/config/index.js.map +1 -1
- package/dist/config/{kleinanzeigeFocus.config.js → kleinanzeigefocus.config.js} +2 -3
- package/dist/config/kleinanzeigefocus.config.js.map +1 -0
- package/dist/config/marktde.config.js +4 -3
- package/dist/config/marktde.config.js.map +1 -1
- package/dist/config/meinstadt.config.js +2 -3
- package/dist/config/meinstadt.config.js.map +1 -1
- package/dist/config/ooyyo.config.js +2 -3
- package/dist/config/ooyyo.config.js.map +1 -1
- package/dist/config/yescar.config.js +4 -3
- package/dist/config/yescar.config.js.map +1 -1
- package/dist/enums/export-type.enum.d.ts +0 -1
- package/dist/enums/export-type.enum.js +0 -1
- package/dist/enums/export-type.enum.js.map +1 -1
- package/dist/enums/file-types.enum.d.ts +7 -7
- package/dist/enums/file-types.enum.js +11 -10
- package/dist/enums/file-types.enum.js.map +1 -1
- package/dist/export.schema.d.ts +4 -33
- package/dist/export.schema.js +32 -61
- package/dist/export.schema.js.map +1 -1
- package/dist/exporters/as24.exporter.d.ts +3 -5
- package/dist/exporters/as24.exporter.js +20 -22
- package/dist/exporters/as24.exporter.js.map +1 -1
- package/dist/exporters/auto24.exporter.d.ts +1 -1
- package/dist/exporters/auto24.exporter.js +8 -17
- package/dist/exporters/auto24.exporter.js.map +1 -1
- package/dist/exporters/autouncle.exporter.d.ts +3 -5
- package/dist/exporters/autouncle.exporter.js +19 -35
- package/dist/exporters/autouncle.exporter.js.map +1 -1
- package/dist/exporters/by-owner.exporter.d.ts +2 -3
- package/dist/exporters/by-owner.exporter.js +4 -7
- package/dist/exporters/by-owner.exporter.js.map +1 -1
- package/dist/exporters/fb.exporter.d.ts +2 -3
- package/dist/exporters/fb.exporter.js +10 -16
- package/dist/exporters/fb.exporter.js.map +1 -1
- package/dist/exporters/google.exporter.d.ts +3 -4
- package/dist/exporters/google.exporter.js +13 -19
- package/dist/exporters/google.exporter.js.map +1 -1
- package/dist/exporters/index.d.ts +10 -1
- package/dist/exporters/index.js +10 -1
- package/dist/exporters/index.js.map +1 -1
- package/dist/exporters/meinstadt.exporter.d.ts +1 -1
- package/dist/exporters/meinstadt.exporter.js +18 -34
- package/dist/exporters/meinstadt.exporter.js.map +1 -1
- package/dist/exporters/mobile.exporter.d.ts +2 -5
- package/dist/exporters/mobile.exporter.js +14 -44
- package/dist/exporters/mobile.exporter.js.map +1 -1
- package/dist/exporters/modix.exporter.d.ts +2 -2
- package/dist/exporters/modix.exporter.js +7 -11
- package/dist/exporters/modix.exporter.js.map +1 -1
- package/dist/exporters/ooyyo.exporter.js +6 -9
- package/dist/exporters/ooyyo.exporter.js.map +1 -1
- package/dist/exporters/rubrikk.exporter.js +8 -14
- package/dist/exporters/rubrikk.exporter.js.map +1 -1
- package/dist/main.js +1 -1
- package/dist/main.js.map +1 -1
- package/dist/services/car.service.js +1 -4
- package/dist/services/car.service.js.map +1 -1
- package/dist/services/user.service.d.ts +0 -2
- package/dist/services/user.service.js +0 -3
- package/dist/services/user.service.js.map +1 -1
- package/dist/transformers/as24.transformer.d.ts +4 -2
- package/dist/transformers/as24.transformer.js +25 -3
- package/dist/transformers/as24.transformer.js.map +1 -1
- package/dist/transformers/auto24.transfomer.js +1 -1
- package/dist/transformers/auto24.transfomer.js.map +1 -1
- package/dist/transformers/autouncle.transformer.js +1 -1
- package/dist/transformers/autouncle.transformer.js.map +1 -1
- package/dist/transformers/fb.transformer.js +2 -2
- package/dist/transformers/fb.transformer.js.map +1 -1
- package/dist/transformers/google.transformer.js +3 -3
- package/dist/transformers/google.transformer.js.map +1 -1
- package/dist/transformers/meinstadt.transformer.js +1 -1
- package/dist/transformers/meinstadt.transformer.js.map +1 -1
- package/dist/transformers/mobile.transformer.js +1 -1
- package/dist/transformers/mobile.transformer.js.map +1 -1
- package/dist/transformers/modix.transformer.d.ts +1 -3
- package/dist/transformers/modix.transformer.js +4 -12
- package/dist/transformers/modix.transformer.js.map +1 -1
- package/dist/transformers/ooyyo.transformer.js +2 -2
- package/dist/transformers/ooyyo.transformer.js.map +1 -1
- package/dist/transformers/rubrikk.transformer.js +1 -1
- package/dist/transformers/rubrikk.transformer.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/package.json +11 -11
- package/src/config/alleskralle.config.ts +4 -6
- package/src/config/as24.config.ts +3 -1
- package/src/config/auto24.config.ts +4 -4
- package/src/config/autoboerse.config.ts +5 -5
- package/src/config/autouncle.config.ts +3 -2
- package/src/config/by-owner.config.ts +0 -1
- package/src/config/config.ts +14 -21
- package/src/config/criteo.config.ts +3 -1
- package/src/config/dasparking.config.ts +3 -5
- package/src/config/eauto.config.ts +4 -3
- package/src/config/fb.config.ts +2 -1
- package/src/config/google.config.ts +2 -1
- package/src/config/index.ts +16 -1
- package/src/config/kleinanzeigeFocus.config.ts +2 -3
- package/src/config/marktde.config.ts +4 -3
- package/src/config/meinstadt.config.ts +3 -4
- package/src/config/ooyyo.config.ts +3 -4
- package/src/config/yescar.config.ts +7 -3
- package/src/enums/export-type.enum.ts +12 -1
- package/src/enums/file-types.enum.ts +8 -8
- package/src/export.schema.ts +35 -33
- package/src/exporters/as24.exporter.ts +27 -30
- package/src/exporters/auto24.exporter.ts +15 -23
- package/src/exporters/autouncle.exporter.ts +27 -38
- package/src/exporters/by-owner.exporter.ts +7 -5
- package/src/exporters/fb.exporter.ts +11 -13
- package/src/exporters/google.exporter.ts +16 -17
- package/src/exporters/index.ts +10 -1
- package/src/exporters/meinstadt.exporter.ts +24 -41
- package/src/exporters/mobile.exporter.ts +21 -63
- package/src/exporters/modix.exporter.ts +15 -14
- package/src/exporters/ooyyo.exporter.ts +10 -11
- package/src/exporters/rubrikk.exporter.ts +11 -16
- package/src/main.ts +1 -1
- package/src/services/car.service.ts +3 -5
- package/src/services/user.service.ts +1 -5
- package/src/transformers/as24.transformer.ts +29 -5
- package/src/transformers/auto24.transfomer.ts +1 -1
- package/src/transformers/autouncle.transformer.ts +1 -1
- package/src/transformers/fb.transformer.ts +2 -2
- package/src/transformers/google.transformer.ts +3 -3
- package/src/transformers/meinstadt.transformer.ts +1 -1
- package/src/transformers/mobile.transformer.ts +1 -1
- package/src/transformers/modix.transformer.ts +8 -17
- package/src/transformers/ooyyo.transformer.ts +2 -2
- package/src/transformers/rubrikk.transformer.ts +1 -1
- package/tsconfig.json +1 -1
- package/dist/config/autoborse.config.d.ts +0 -5
- package/dist/config/autoborse.config.js +0 -15
- package/dist/config/autoborse.config.js.map +0 -1
- package/dist/config/kleinanzeigeFocus.config.js.map +0 -1
- package/src/config/autoborse.config.ts +0 -12
- /package/dist/config/{kleinanzeigeFocus.config.d.ts → kleinanzeigefocus.config.d.ts} +0 -0
|
@@ -1,37 +1,37 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
1
|
+
import { writeFile } from 'fs/promises';
|
|
2
|
+
import { get, union } from 'lodash';
|
|
3
|
+
import { normalize } from 'path';
|
|
4
|
+
|
|
5
|
+
|
|
5
6
|
import { Injectable } from '@nestjs/common';
|
|
6
7
|
import { ModuleRef } from '@nestjs/core';
|
|
8
|
+
|
|
9
|
+
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
10
|
+
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
11
|
+
import { Zipper } from '@bytebrand/zipper';
|
|
12
|
+
|
|
13
|
+
import { ConfigService, LoaderService, RulesService, CarService } from '../services';
|
|
7
14
|
import { As24Transformer } from '../transformers/as24.transformer';
|
|
8
|
-
import {
|
|
15
|
+
import { Exporter } from '../interfaces/exporter.interface';
|
|
9
16
|
import { convertToCSV } from '../utils/converter.utils';
|
|
10
|
-
import {
|
|
11
|
-
import { normalize } from 'path';
|
|
12
|
-
import { Zipper } from '@bytebrand/zipper';
|
|
13
|
-
import { FileTypes } from '../enums';
|
|
14
|
-
import { RulesService, CarService } from '../services';
|
|
17
|
+
import { FileType } from '../enums';
|
|
15
18
|
|
|
16
19
|
@Injectable()
|
|
17
20
|
export class As24Exporter implements Exporter {
|
|
18
21
|
constructor(
|
|
19
22
|
private configService: ConfigService,
|
|
20
23
|
private loaderService: LoaderService,
|
|
21
|
-
private userService: UserService,
|
|
22
24
|
private moduleRef: ModuleRef,
|
|
23
25
|
private carService: CarService,
|
|
24
26
|
private readonly rulesService: RulesService,
|
|
25
|
-
) {}
|
|
27
|
+
) { }
|
|
26
28
|
|
|
27
29
|
async export(): Promise<any> {
|
|
28
30
|
await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
|
|
29
31
|
await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
|
|
30
32
|
|
|
31
33
|
const as24Transformer = await this.moduleRef.create(As24Transformer);
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
const processedDealers = [];
|
|
34
|
+
const processedDealers = new Set();
|
|
35
35
|
|
|
36
36
|
await this.loaderService.process(async (chunk: FsChunkInterface) => {
|
|
37
37
|
const chunkData = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
|
|
@@ -39,35 +39,32 @@ export class As24Exporter implements Exporter {
|
|
|
39
39
|
if (chunk && chunkData) {
|
|
40
40
|
for (let i = 0; i < chunk.data.length; i += 1) {
|
|
41
41
|
const data = chunk.data[i];
|
|
42
|
-
const ownerId = get(data.car, 'metaData.ownerId'
|
|
42
|
+
const ownerId = get(data.car, 'metaData.ownerId');
|
|
43
43
|
|
|
44
|
-
const transformedData = await as24Transformer.transform(data.car
|
|
44
|
+
const transformedData = await as24Transformer.transform(data.car);
|
|
45
45
|
const transformedDataCSV = `${convertToCSV(transformedData)}\n`;
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
Buffer.from(transformedDataCSV),
|
|
50
|
-
{ flag: 'a' },
|
|
51
|
-
);
|
|
47
|
+
const filename = `${this.configService.config.pathesConfig.localConvertedDirectory}/${ownerId}.${FileType.CSV}`
|
|
48
|
+
await writeFile(filename, Buffer.from(transformedDataCSV), { flag: 'a' },);
|
|
52
49
|
|
|
53
|
-
|
|
50
|
+
this.carService.counter.countCar(data.images);
|
|
51
|
+
processedDealers.add(ownerId);
|
|
54
52
|
}
|
|
55
|
-
|
|
53
|
+
|
|
56
54
|
}
|
|
57
55
|
});
|
|
58
56
|
|
|
59
57
|
for (const dealer of processedDealers) {
|
|
60
58
|
const now = new Date();
|
|
61
59
|
const d = `${now.getFullYear()}${now.getMonth() + 1}${now.getDate()}`;
|
|
62
|
-
const fileCSV = normalize(
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
const fileZIP = normalize(
|
|
66
|
-
`${this.configService.config.pathesConfig.localConvertedDirectory}/${d}_${dealer}.${FileTypes.zip}`,
|
|
67
|
-
);
|
|
60
|
+
const fileCSV = normalize(`${this.configService.config.pathesConfig.localConvertedDirectory}/${dealer}.${FileType.CSV}`);
|
|
61
|
+
const fileZIP = normalize(`${this.configService.config.pathesConfig.localConvertedDirectory}/${d}_${dealer}.${FileType.ZIP}`);
|
|
62
|
+
|
|
68
63
|
if (await CronicleUtils.exists(fileCSV)) {
|
|
69
64
|
await Zipper.zip({ fileList: [fileCSV], dest: fileZIP });
|
|
65
|
+
await CronicleUtils.unlinkIfExists(fileCSV);
|
|
70
66
|
}
|
|
71
67
|
}
|
|
68
|
+
console.log(this.carService.counter.getCountedCars());
|
|
72
69
|
}
|
|
73
70
|
}
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { Injectable } from '@nestjs/common';
|
|
2
|
-
import { CarService, RulesService, ConfigService, LoaderService } from '../services';
|
|
3
|
-
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
4
|
-
import { Exporter } from '../interfaces/exporter.interface';
|
|
5
|
-
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
6
|
-
import { toNumber } from 'lodash';
|
|
7
2
|
import { ModuleRef } from '@nestjs/core';
|
|
8
|
-
|
|
9
|
-
import {
|
|
3
|
+
|
|
4
|
+
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
5
|
+
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
10
6
|
import { Zipper } from '@bytebrand/zipper';
|
|
11
7
|
|
|
8
|
+
import { CarService, RulesService, ConfigService, LoaderService } from '../services';
|
|
9
|
+
import { Auto24Transformer } from '../transformers/auto24.transfomer';
|
|
10
|
+
import { Auto24XmlBuilder } from '../formats/auto24-xml.builder';
|
|
11
|
+
import { Exporter } from '../interfaces/exporter.interface';
|
|
12
|
+
|
|
12
13
|
@Injectable()
|
|
13
14
|
export class Auto24Exporter implements Exporter {
|
|
14
15
|
constructor(
|
|
@@ -17,17 +18,14 @@ export class Auto24Exporter implements Exporter {
|
|
|
17
18
|
private loaderService: LoaderService,
|
|
18
19
|
private moduleRef: ModuleRef,
|
|
19
20
|
private rulesService: RulesService,
|
|
20
|
-
) {}
|
|
21
|
+
) { }
|
|
21
22
|
|
|
22
23
|
async export(): Promise<void> {
|
|
23
|
-
for (const file of this.configService.config.outputFiles) {
|
|
24
|
-
const exists = await CronicleUtils.exists(file);
|
|
25
|
-
exists && (await CronicleUtils.unlink(file));
|
|
26
|
-
}
|
|
27
|
-
|
|
28
24
|
const [fileXml, fileGZ] = this.configService.config.outputFiles;
|
|
25
|
+
await Promise.all([await CronicleUtils.unlinkIfExists(fileXml), await CronicleUtils.unlinkIfExists(fileGZ)]);
|
|
29
26
|
|
|
30
27
|
const xmlBuilder = new Auto24XmlBuilder(fileXml);
|
|
28
|
+
await xmlBuilder.buildStart();
|
|
31
29
|
|
|
32
30
|
const auto24Transformer = await this.moduleRef.create(Auto24Transformer);
|
|
33
31
|
|
|
@@ -35,25 +33,19 @@ export class Auto24Exporter implements Exporter {
|
|
|
35
33
|
const chunkData = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
|
|
36
34
|
|
|
37
35
|
if (chunk && chunkData) {
|
|
38
|
-
if (toNumber(chunk.id) === 0) {
|
|
39
|
-
await xmlBuilder.buildStart(chunk.total);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
36
|
for (let i = 0; i < chunk.data.length; i += 1) {
|
|
43
37
|
const data = chunk.data[i];
|
|
44
38
|
const { car, images } = data;
|
|
45
|
-
await xmlBuilder.addItem(auto24Transformer.transform(car, images),
|
|
39
|
+
await xmlBuilder.addItem(auto24Transformer.transform(car, images), 'Deutschland');
|
|
46
40
|
this.carService.counter.countCar(images);
|
|
47
41
|
}
|
|
48
|
-
|
|
42
|
+
|
|
49
43
|
}
|
|
50
44
|
});
|
|
51
45
|
|
|
52
46
|
await xmlBuilder.buildEnd();
|
|
53
|
-
console.log(this.carService.counter.getCountedCars());
|
|
54
47
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
}
|
|
48
|
+
fileGZ && await Zipper.tar({ dest: fileGZ, fileList: [fileXml] });
|
|
49
|
+
console.log(this.carService.counter.getCountedCars());
|
|
58
50
|
}
|
|
59
51
|
}
|
|
@@ -1,14 +1,15 @@
|
|
|
1
|
+
import { get } from 'lodash';
|
|
2
|
+
|
|
1
3
|
import { Injectable } from '@nestjs/common';
|
|
2
|
-
import { CarService, RulesService, ConfigService, LoaderService } from '../services';
|
|
3
|
-
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
4
|
-
import { Exporter } from '../interfaces/exporter.interface';
|
|
5
|
-
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
6
4
|
import { ModuleRef } from '@nestjs/core';
|
|
7
|
-
|
|
5
|
+
|
|
6
|
+
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
7
|
+
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
8
|
+
|
|
8
9
|
import { AutouncleTransformer } from '../transformers/autouncle.transformer';
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
10
|
+
import { AutouncleJsonBuilder } from '../formats/autouncle-json.builder';
|
|
11
|
+
import { CarService, RulesService, LoaderService, ConfigService } from '../services';
|
|
12
|
+
import { Exporter } from '../interfaces/exporter.interface';
|
|
12
13
|
|
|
13
14
|
@Injectable()
|
|
14
15
|
export class AutouncleExporter implements Exporter {
|
|
@@ -18,47 +19,35 @@ export class AutouncleExporter implements Exporter {
|
|
|
18
19
|
private loaderService: LoaderService,
|
|
19
20
|
private rulesService: RulesService,
|
|
20
21
|
private moduleRef: ModuleRef,
|
|
21
|
-
|
|
22
|
-
) {}
|
|
22
|
+
) { }
|
|
23
23
|
|
|
24
24
|
async export(): Promise<void> {
|
|
25
|
-
const fileName = this.
|
|
26
|
-
|
|
27
|
-
const exists = await CronicleUtils.exists(fileName);
|
|
28
|
-
exists && (await CronicleUtils.unlink(fileName));
|
|
25
|
+
const [fileName] = this.configService.config.outputFiles;
|
|
26
|
+
await CronicleUtils.unlinkIfExists(fileName);
|
|
29
27
|
|
|
30
28
|
const autouncleJsonBuilder = new AutouncleJsonBuilder(fileName);
|
|
31
|
-
const autouncleTransformer = await this.moduleRef.create(AutouncleTransformer);
|
|
32
|
-
|
|
33
29
|
await autouncleJsonBuilder.buildStart();
|
|
30
|
+
|
|
31
|
+
const autouncleTransformer = await this.moduleRef.create(AutouncleTransformer);
|
|
34
32
|
|
|
35
33
|
let firstCar = true;
|
|
36
34
|
await this.loaderService.process(async (chunk: FsChunkInterface) => {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
);
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if (
|
|
49
|
-
const owner = await this.carService.getOwner(ownerId);
|
|
50
|
-
if (owner) {
|
|
51
|
-
const car = await autouncleTransformer.transform({ owner, ...data });
|
|
52
|
-
await autouncleJsonBuilder.addItem(car, firstCar);
|
|
53
|
-
this.carService.counter.countCar(data.images);
|
|
54
|
-
}
|
|
55
|
-
if (firstCar) {
|
|
56
|
-
firstCar = false;
|
|
57
|
-
}
|
|
58
|
-
}
|
|
35
|
+
chunk.data = chunk.data.filter(e => this.carService.filterCar(e, this.rulesService.rules));
|
|
36
|
+
|
|
37
|
+
for (const data of chunk.data) {
|
|
38
|
+
const ownerId = get(data.car, 'metaData.ownerId');
|
|
39
|
+
const owner = await this.carService.getOwner(ownerId);
|
|
40
|
+
const car = await autouncleTransformer.transform({ owner, ...data });
|
|
41
|
+
|
|
42
|
+
await autouncleJsonBuilder.addItem(car, firstCar);
|
|
43
|
+
|
|
44
|
+
this.carService.counter.countCar(data.images);
|
|
45
|
+
|
|
46
|
+
if (firstCar) { firstCar = false; }
|
|
59
47
|
}
|
|
60
48
|
});
|
|
61
49
|
|
|
62
50
|
await autouncleJsonBuilder.buildEnd();
|
|
51
|
+
console.log(this.carService.counter.getCountedCars());
|
|
63
52
|
}
|
|
64
53
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ConfigService, LoaderService, CarService, RulesService
|
|
1
|
+
import { ConfigService, LoaderService, CarService, RulesService } from '../services';
|
|
2
2
|
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
3
3
|
import { Exporter } from '../interfaces/exporter.interface';
|
|
4
4
|
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
@@ -6,23 +6,24 @@ import { Injectable } from '@nestjs/common';
|
|
|
6
6
|
import { ModuleRef } from '@nestjs/core';
|
|
7
7
|
import { ByOwnerTransformer } from '../transformers/by-owner.transformer';
|
|
8
8
|
import { createWriteStream } from 'fs';
|
|
9
|
-
import { ExportType, FileTypes } from '../enums';
|
|
10
9
|
|
|
11
10
|
const createCsvWriter = require('csv-write-stream');
|
|
12
11
|
|
|
12
|
+
/**
|
|
13
|
+
* @deprecated
|
|
14
|
+
*/
|
|
13
15
|
@Injectable()
|
|
14
16
|
export class ByOwnerExporter implements Exporter {
|
|
15
17
|
constructor(
|
|
16
18
|
private configService: ConfigService,
|
|
17
19
|
private loaderService: LoaderService,
|
|
18
20
|
private moduleRef: ModuleRef,
|
|
19
|
-
private userService: UserService,
|
|
20
21
|
private carService: CarService,
|
|
21
22
|
private rulesService: RulesService,
|
|
22
23
|
) {}
|
|
23
24
|
|
|
24
25
|
async export(): Promise<void> {
|
|
25
|
-
const fileName = this.
|
|
26
|
+
const [fileName] = this.configService.config.outputFiles;
|
|
26
27
|
|
|
27
28
|
await CronicleUtils.unlinkIfExists(fileName);
|
|
28
29
|
|
|
@@ -37,6 +38,7 @@ export class ByOwnerExporter implements Exporter {
|
|
|
37
38
|
chunk.data = null;
|
|
38
39
|
});
|
|
39
40
|
|
|
40
|
-
csvWriter.end();
|
|
41
|
+
await csvWriter.end();
|
|
42
|
+
console.log(this.carService.counter.getCountedCars());
|
|
41
43
|
}
|
|
42
44
|
}
|
|
@@ -7,7 +7,6 @@ import { get } from 'lodash';
|
|
|
7
7
|
import { FbXmlBuilder } from '../formats/fb-xml.builder';
|
|
8
8
|
import { ModuleRef } from '@nestjs/core';
|
|
9
9
|
import { FbTransformer } from '../transformers/fb.transformer';
|
|
10
|
-
import { ExportType, FileTypes } from '../enums';
|
|
11
10
|
import { FbXmlItem } from '../interfaces';
|
|
12
11
|
|
|
13
12
|
@Injectable()
|
|
@@ -18,18 +17,17 @@ export class FbExporter implements Exporter {
|
|
|
18
17
|
private rulesService: RulesService,
|
|
19
18
|
private loaderService: LoaderService,
|
|
20
19
|
private moduleRef: ModuleRef,
|
|
21
|
-
|
|
22
|
-
) {}
|
|
20
|
+
) { }
|
|
23
21
|
|
|
24
22
|
async export(): Promise<void> {
|
|
25
|
-
const fileName = this.
|
|
23
|
+
const [fileName] = this.configService.config.outputFiles;
|
|
24
|
+
await CronicleUtils.unlinkIfExists(fileName);
|
|
26
25
|
|
|
27
|
-
const exists = await CronicleUtils.exists(fileName);
|
|
28
|
-
exists && (await CronicleUtils.unlink(fileName));
|
|
29
26
|
|
|
30
|
-
const fbTransformer = await this.moduleRef.create(FbTransformer);
|
|
31
27
|
const fbXmlBuilder = new FbXmlBuilder(fileName);
|
|
32
|
-
await fbXmlBuilder.buildStart(this.configService.config.
|
|
28
|
+
await fbXmlBuilder.buildStart(this.configService.config.domain);
|
|
29
|
+
|
|
30
|
+
const fbTransformer = await this.moduleRef.create(FbTransformer);
|
|
33
31
|
|
|
34
32
|
await this.loaderService.process(async (chunk: FsChunkInterface) => {
|
|
35
33
|
const chunkData = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
|
|
@@ -37,14 +35,14 @@ export class FbExporter implements Exporter {
|
|
|
37
35
|
for (const data of chunkData) {
|
|
38
36
|
const ownerId = get(data.car, 'metaData.ownerId', null);
|
|
39
37
|
const owner = await this.carService.getOwner(ownerId);
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
}
|
|
38
|
+
const car = await fbTransformer.transform({ owner, ...data });
|
|
39
|
+
|
|
40
|
+
await fbXmlBuilder.addItem(car as FbXmlItem);
|
|
41
|
+
this.carService.counter.countCar(data.images);
|
|
45
42
|
}
|
|
46
43
|
});
|
|
47
44
|
|
|
48
45
|
await fbXmlBuilder.buildEnd();
|
|
46
|
+
console.log(this.carService.counter.getCountedCars());
|
|
49
47
|
}
|
|
50
48
|
}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { Injectable } from '@nestjs/common';
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
import { Exporter } from '../interfaces/exporter.interface';
|
|
2
|
+
import { ModuleRef } from '@nestjs/core';
|
|
3
|
+
|
|
5
4
|
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
6
|
-
import {
|
|
5
|
+
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
6
|
+
|
|
7
|
+
import { CarService, ConfigService, LoaderService, RulesService } from '../services';
|
|
7
8
|
import { GoogleTransformer } from '../transformers/google.transformer';
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
9
|
+
import { GoogleXmlBuilder } from '../formats/google-xml.builder';
|
|
10
|
+
import { Exporter } from '../interfaces/exporter.interface';
|
|
10
11
|
|
|
11
12
|
@Injectable()
|
|
12
13
|
export class GoogleExporter implements Exporter {
|
|
@@ -16,30 +17,28 @@ export class GoogleExporter implements Exporter {
|
|
|
16
17
|
private rulesService: RulesService,
|
|
17
18
|
private loaderService: LoaderService,
|
|
18
19
|
private moduleRef: ModuleRef,
|
|
19
|
-
private userService: UserService,
|
|
20
20
|
) {}
|
|
21
21
|
|
|
22
22
|
async export(): Promise<void> {
|
|
23
|
-
const
|
|
24
|
-
|
|
23
|
+
const [fileName] = this.configService.config.outputFiles;
|
|
24
|
+
await CronicleUtils.unlinkIfExists(fileName);
|
|
25
25
|
|
|
26
|
-
const
|
|
27
|
-
|
|
26
|
+
const googleXmlBuilder = new GoogleXmlBuilder(fileName);
|
|
27
|
+
await googleXmlBuilder.buildStart(this.configService.config.domain);
|
|
28
28
|
|
|
29
29
|
const googleTransformer = await this.moduleRef.create(GoogleTransformer);
|
|
30
|
-
const googleXmlBuilder = new GoogleXmlBuilder(fileName);
|
|
31
|
-
await googleXmlBuilder.buildStart(this.configService.config.url);
|
|
32
30
|
|
|
33
31
|
await this.loaderService.process(async (chunk: FsChunkInterface) => {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
for (const data of
|
|
37
|
-
const car = await googleTransformer.transform(
|
|
32
|
+
chunk.data = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
|
|
33
|
+
|
|
34
|
+
for (const data of chunk.data) {
|
|
35
|
+
const car = await googleTransformer.transform(data);
|
|
38
36
|
await googleXmlBuilder.addItem(car);
|
|
39
37
|
this.carService.counter.countCar(data.images);
|
|
40
38
|
}
|
|
41
39
|
});
|
|
42
40
|
|
|
43
41
|
await googleXmlBuilder.buildEnd();
|
|
42
|
+
console.log(this.carService.counter.getCountedCars());
|
|
44
43
|
}
|
|
45
44
|
}
|
package/src/exporters/index.ts
CHANGED
|
@@ -1,2 +1,11 @@
|
|
|
1
|
-
export * from './
|
|
1
|
+
export * from './as24.exporter';
|
|
2
|
+
export * from './auto24.exporter';
|
|
3
|
+
export * from './autouncle.exporter';
|
|
4
|
+
export * from './by-owner.exporter';
|
|
5
|
+
export * from './fb.exporter';
|
|
6
|
+
export * from './google.exporter';
|
|
7
|
+
export * from './meinstadt.exporter';
|
|
8
|
+
export * from './mobile.exporter';
|
|
2
9
|
export * from './modix.exporter';
|
|
10
|
+
export * from './ooyyo.exporter';
|
|
11
|
+
export * from './rubrikk.exporter';
|
|
@@ -1,14 +1,15 @@
|
|
|
1
|
-
import { CarService, RulesService, LoaderService, ConfigService } from '../services';
|
|
2
|
-
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
3
|
-
import { Exporter } from '../interfaces/exporter.interface';
|
|
4
|
-
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
5
|
-
import { Zipper } from '@bytebrand/zipper';
|
|
6
1
|
import { Injectable } from '@nestjs/common';
|
|
7
2
|
import { ModuleRef } from '@nestjs/core';
|
|
8
|
-
|
|
9
|
-
import {
|
|
3
|
+
|
|
4
|
+
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
5
|
+
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
6
|
+
import { Zipper } from '@bytebrand/zipper';
|
|
7
|
+
|
|
8
|
+
import { CarService, RulesService, LoaderService, ConfigService } from '../services';
|
|
10
9
|
import { MeinstadtTransformer } from '../transformers/meinstadt.transformer';
|
|
10
|
+
import { MeinstadtJsonBuilder } from '../formats/meinstadt-json.builder';
|
|
11
11
|
import { MeinstadtJsonItem } from '../interfaces/export-item.interface';
|
|
12
|
+
import { Exporter } from '../interfaces/exporter.interface';
|
|
12
13
|
|
|
13
14
|
@Injectable()
|
|
14
15
|
export class MeinstadtExporter implements Exporter {
|
|
@@ -18,50 +19,32 @@ export class MeinstadtExporter implements Exporter {
|
|
|
18
19
|
private rulesService: RulesService,
|
|
19
20
|
private loaderService: LoaderService,
|
|
20
21
|
private moduleRef: ModuleRef,
|
|
21
|
-
) {}
|
|
22
|
+
) { }
|
|
22
23
|
|
|
23
24
|
async export(): Promise<void> {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
exists && (await CronicleUtils.unlink(file));
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
const [fileCarJson, fileGZ] = this.configService.config.outputFiles;
|
|
25
|
+
const [fileJson, fileGZ] = this.configService.config.outputFiles;
|
|
26
|
+
await Promise.all([CronicleUtils.unlinkIfExists(fileJson), CronicleUtils.unlinkIfExists(fileGZ)])
|
|
30
27
|
|
|
31
|
-
const meinstadtJsonBuilder = new MeinstadtJsonBuilder(
|
|
32
|
-
meinstadtJsonBuilder.buildStart();
|
|
28
|
+
const meinstadtJsonBuilder = new MeinstadtJsonBuilder(fileJson);
|
|
29
|
+
await meinstadtJsonBuilder.buildStart();
|
|
33
30
|
|
|
34
31
|
const meinstadtTransformer = await this.moduleRef.create(MeinstadtTransformer);
|
|
35
32
|
|
|
36
33
|
let firstCar = true;
|
|
37
34
|
await this.loaderService.process(async (chunk: FsChunkInterface) => {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
);
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
chunk.data = null;
|
|
47
|
-
for (const data of chunkData) {
|
|
48
|
-
const ownerId = get(data.car, 'metaData.ownerId', null);
|
|
49
|
-
if (!isNil(ownerId)) {
|
|
50
|
-
const owner = await this.carService.getOwner(ownerId);
|
|
51
|
-
if (owner) {
|
|
52
|
-
const car = await meinstadtTransformer.transform({ owner, ...data });
|
|
53
|
-
await meinstadtJsonBuilder.addItem(car as MeinstadtJsonItem, firstCar);
|
|
54
|
-
this.carService.counter.countCar(data.images);
|
|
55
|
-
}
|
|
56
|
-
if (firstCar) {
|
|
57
|
-
firstCar = false;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
35
|
+
chunk.data = chunk.data.filter(e => this.carService.filterCar(e, this.rulesService.rules));
|
|
36
|
+
|
|
37
|
+
for (const data of chunk.data) {
|
|
38
|
+
const car = await meinstadtTransformer.transform({ ...data });
|
|
39
|
+
await meinstadtJsonBuilder.addItem(car as MeinstadtJsonItem, firstCar);
|
|
40
|
+
this.carService.counter.countCar(data.images);
|
|
41
|
+
|
|
42
|
+
if (firstCar) { firstCar = false; }
|
|
60
43
|
}
|
|
61
44
|
});
|
|
62
45
|
|
|
63
|
-
meinstadtJsonBuilder.buildEnd();
|
|
64
|
-
|
|
65
|
-
|
|
46
|
+
await meinstadtJsonBuilder.buildEnd();
|
|
47
|
+
await Zipper.zip({ dest: fileGZ, fileList: [fileJson] });
|
|
48
|
+
console.log(this.carService.counter.getCountedCars());
|
|
66
49
|
}
|
|
67
50
|
}
|
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
import { CarService, RulesService, LoaderService, ConfigService } from '../services';
|
|
2
|
-
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
3
1
|
import { writeFile } from 'fs/promises';
|
|
2
|
+
import { basename, normalize } from 'path';
|
|
3
|
+
|
|
4
|
+
import { Injectable } from '@nestjs/common';
|
|
5
|
+
import { ModuleRef } from '@nestjs/core';
|
|
6
|
+
|
|
7
|
+
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
8
|
+
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
9
|
+
import { Zipper } from '@bytebrand/zipper';
|
|
10
|
+
|
|
11
|
+
import { CarService, RulesService, LoaderService, ConfigService } from '../services';
|
|
4
12
|
import { MobileTransformer } from '../transformers/mobile.transformer';
|
|
5
13
|
import { convertToCSV } from '../utils/converter.utils';
|
|
6
14
|
import { Exporter } from '../interfaces/exporter.interface';
|
|
7
|
-
import {
|
|
8
|
-
import { Zipper } from '@bytebrand/zipper';
|
|
9
|
-
import { normalize, join, extname } from 'path';
|
|
10
|
-
import { Injectable } from '@nestjs/common';
|
|
11
|
-
import { ModuleRef } from '@nestjs/core';
|
|
12
|
-
import { HttpService } from '@nestjs/axios';
|
|
13
|
-
import { lastValueFrom } from 'rxjs';
|
|
14
|
-
import { get } from 'lodash';
|
|
15
|
-
import { FileTypes } from '../enums';
|
|
15
|
+
import { FileType } from '../enums';
|
|
16
16
|
|
|
17
17
|
@Injectable()
|
|
18
18
|
export class MobileExporter implements Exporter {
|
|
@@ -22,83 +22,41 @@ export class MobileExporter implements Exporter {
|
|
|
22
22
|
private rulesService: RulesService,
|
|
23
23
|
private loaderService: LoaderService,
|
|
24
24
|
private moduleRef: ModuleRef,
|
|
25
|
-
|
|
26
|
-
) {}
|
|
25
|
+
) { }
|
|
27
26
|
|
|
28
27
|
async export(): Promise<void> {
|
|
29
|
-
const fileName =
|
|
28
|
+
const [fileName] = this.configService.config.outputFiles;
|
|
30
29
|
|
|
31
30
|
await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
|
|
32
31
|
await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
|
|
33
|
-
if (this.configService.config.downloadImages) {
|
|
34
|
-
await CronicleUtils.ensureDir(join(this.configService.config.pathesConfig.localConvertedDirectory, 'images'));
|
|
35
|
-
}
|
|
36
32
|
|
|
37
33
|
const mobileTransformer = await this.moduleRef.create(MobileTransformer);
|
|
38
34
|
|
|
39
35
|
await this.loaderService.process(async (chunk: FsChunkInterface) => {
|
|
40
|
-
|
|
41
|
-
chunk.data = chunk.data.filter(
|
|
42
|
-
e => this.carService.filterCar(e, this.rulesService.rules) && this.carService.isCarNew(e),
|
|
43
|
-
);
|
|
44
|
-
} else {
|
|
45
|
-
chunk.data = chunk.data.filter(e => this.carService.filterCar(e, this.rulesService.rules));
|
|
46
|
-
}
|
|
36
|
+
chunk.data = chunk.data.filter(e => this.carService.filterCar(e, this.rulesService.rules));
|
|
47
37
|
|
|
48
38
|
if (chunk && chunk.data) {
|
|
49
39
|
for (let i = 0; i < chunk.data.length; i += 1) {
|
|
50
40
|
const data = chunk.data[i];
|
|
51
41
|
const { car, images } = data;
|
|
52
42
|
const transformedData = await mobileTransformer.transform(car, images);
|
|
53
|
-
if (this.configService.config.downloadImages) {
|
|
54
|
-
await this.downloadImages(get(transformedData, '1'), get(transformedData, '26', '').split(';'));
|
|
55
|
-
}
|
|
56
|
-
const transformedDataCSV = `${convertToCSV(transformedData)}\n`;
|
|
57
43
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
{ flag: 'a' },
|
|
62
|
-
);
|
|
44
|
+
const file = `${this.configService.config.pathesConfig.localConvertedDirectory}/${fileName}`;
|
|
45
|
+
await writeFile(file, Buffer.from(`${convertToCSV(transformedData)}\n`), { flag: 'a' });
|
|
46
|
+
|
|
63
47
|
this.carService.counter.countCar(data.images);
|
|
64
48
|
}
|
|
65
|
-
|
|
49
|
+
|
|
66
50
|
}
|
|
67
51
|
});
|
|
68
52
|
|
|
69
|
-
const fileZIP = normalize(
|
|
70
|
-
|
|
71
|
-
);
|
|
72
|
-
const fileCSV = normalize(
|
|
73
|
-
`${this.configService.config.pathesConfig.localConvertedDirectory}/${fileName}.${FileTypes.csv}`,
|
|
74
|
-
);
|
|
53
|
+
const fileZIP = normalize(`${this.configService.config.pathesConfig.localConvertedDirectory}/${basename(fileName)}.${FileType.ZIP}`);
|
|
54
|
+
const fileCSV = normalize(`${this.configService.config.pathesConfig.localConvertedDirectory}/${fileName}`);
|
|
75
55
|
const fileList = [fileCSV];
|
|
76
|
-
if (this.configService.config.downloadImages) {
|
|
77
|
-
const fileImages = normalize(join(this.configService.config.pathesConfig.localConvertedDirectory, 'images'));
|
|
78
|
-
fileList.push(fileImages);
|
|
79
|
-
}
|
|
80
56
|
|
|
81
57
|
if (await CronicleUtils.exists(fileCSV)) {
|
|
82
58
|
await Zipper.zip({ fileList, dest: fileZIP });
|
|
83
59
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
private downloadImages(vehicleId: string, imageUrls: string[]): Promise<PromiseSettledResult<unknown>[]> {
|
|
87
|
-
return Promise.allSettled(
|
|
88
|
-
imageUrls.map(async (imageUrl, index) => {
|
|
89
|
-
const response = await lastValueFrom(
|
|
90
|
-
this.httpService.get(imageUrl, {
|
|
91
|
-
responseType: 'arraybuffer',
|
|
92
|
-
}),
|
|
93
|
-
);
|
|
94
|
-
const imageFile = join(
|
|
95
|
-
this.configService.config.pathesConfig.localConvertedDirectory,
|
|
96
|
-
'images',
|
|
97
|
-
`${vehicleId}_${index + 1}${extname(imageUrl) || FileTypes.jpg}`,
|
|
98
|
-
);
|
|
99
|
-
|
|
100
|
-
return writeFile(imageFile, response.data, { flag: 'w' });
|
|
101
|
-
}),
|
|
102
|
-
);
|
|
60
|
+
console.log(this.carService.counter.getCountedCars());
|
|
103
61
|
}
|
|
104
62
|
}
|