@bytebrand/export-cars 1.26.47 → 1.26.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/configs/ooyyo.config.d.ts +14 -0
- package/dist/configs/ooyyo.config.js +40 -6
- package/dist/configs/ooyyo.config.js.map +1 -1
- package/dist/exporters/ooyyo.exporter.d.ts +5 -4
- package/dist/exporters/ooyyo.exporter.js +45 -25
- package/dist/exporters/ooyyo.exporter.js.map +1 -1
- package/dist/formats/ooyyo-xml.builder.d.ts +7 -2
- package/dist/formats/ooyyo-xml.builder.js +59 -36
- package/dist/formats/ooyyo-xml.builder.js.map +1 -1
- package/dist/interfaces/export-item.interface.d.ts +30 -2
- package/dist/interfaces/params.interface.d.ts +3 -0
- package/dist/services/ooyyo-feed-run.d.ts +81 -0
- package/dist/services/ooyyo-feed-run.js +183 -0
- package/dist/services/ooyyo-feed-run.js.map +1 -0
- package/dist/services/ooyyo-feed-verify.d.ts +13 -0
- package/dist/services/ooyyo-feed-verify.js +104 -0
- package/dist/services/ooyyo-feed-verify.js.map +1 -0
- package/dist/transformers/ooyyo.transformer.d.ts +51 -11
- package/dist/transformers/ooyyo.transformer.js +268 -95
- package/dist/transformers/ooyyo.transformer.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/marktde/AUDIT.md +84 -84
- package/docs/marktde/PARTNER-QUESTIONS.md +49 -49
- package/docs/marktde/RUNBOOK.md +58 -58
- package/docs/marktde/TRACKING.md +75 -69
- package/docs/meinestadt/PARTNER-QUESTIONS.md +17 -0
- package/docs/meinestadt/TRACKING.md +107 -0
- package/docs/ooyyo/AUDIT.md +81 -0
- package/docs/ooyyo/PARTNER-QUESTIONS.md +32 -0
- package/docs/ooyyo/RUNBOOK.md +67 -0
- package/docs/ooyyo/TRACKING.md +54 -0
- package/package.json +66 -65
- package/src/configs/marktde.config.ts +70 -70
- package/src/configs/ooyyo.config.ts +65 -15
- package/src/exporters/marktde.exporter.ts +78 -78
- package/src/exporters/ooyyo.exporter.ts +75 -48
- package/src/formats/ooyyo-xml.builder.ts +87 -48
- package/src/interfaces/export-item.interface.ts +43 -2
- package/src/interfaces/params.interface.ts +5 -0
- package/src/services/marktde-feed-run.ts +233 -233
- package/src/services/ooyyo-feed-run.ts +265 -0
- package/src/services/ooyyo-feed-verify.ts +131 -0
- package/src/transformers/marktde.transformer.ts +255 -255
- package/src/transformers/ooyyo.transformer.ts +378 -109
- package/test/fixtures/ooyyo/OoyyoFormat.xml +125 -0
- package/test/fixtures/ooyyo/OoyyoFormat.xsd +116 -0
- package/test/fixtures/ooyyo.fixture.ts +53 -0
- package/test/mapping-csv.test.ts +65 -65
- package/test/marktde-feed-run.test.ts +263 -263
- package/test/marktde-mapping.test.ts +280 -280
- package/test/ooyyo-feed-run.test.ts +293 -0
- package/test/ooyyo-mapping.test.ts +276 -0
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
import { Params } from '../interfaces/params.interface';
|
|
2
|
+
import { FeedGuardOptions } from '../services/feed-guard';
|
|
2
3
|
import { Config } from './config';
|
|
4
|
+
export declare const OOYYO_DEFAULTS: {
|
|
5
|
+
imageConcurrency: number;
|
|
6
|
+
minSourceItems: number;
|
|
7
|
+
minPublishedItems: number;
|
|
8
|
+
maxDropPercent: number;
|
|
9
|
+
maxSourceAgeHours: number;
|
|
10
|
+
maxStaleHours: number;
|
|
11
|
+
};
|
|
3
12
|
export declare class OoyyoConfig extends Config {
|
|
13
|
+
dryRun: boolean;
|
|
14
|
+
imageConcurrency: number;
|
|
15
|
+
maxImages: number;
|
|
16
|
+
stateDir: string;
|
|
17
|
+
feedGuard: FeedGuardOptions;
|
|
4
18
|
constructor(params: Params);
|
|
5
19
|
}
|
|
@@ -1,16 +1,50 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.OoyyoConfig = void 0;
|
|
4
|
-
const
|
|
3
|
+
exports.OoyyoConfig = exports.OOYYO_DEFAULTS = void 0;
|
|
4
|
+
const lodash_1 = require("lodash");
|
|
5
5
|
const enums_1 = require("../enums");
|
|
6
|
+
const config_1 = require("./config");
|
|
7
|
+
const text = (value) => (typeof value === 'string' ? value.trim() : '');
|
|
8
|
+
const positiveNumber = (value, fallback) => {
|
|
9
|
+
const parsed = (0, lodash_1.toNumber)(text(value));
|
|
10
|
+
return text(value) && Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
11
|
+
};
|
|
12
|
+
exports.OOYYO_DEFAULTS = {
|
|
13
|
+
imageConcurrency: 16,
|
|
14
|
+
minSourceItems: 1000,
|
|
15
|
+
minPublishedItems: 1,
|
|
16
|
+
maxDropPercent: 20,
|
|
17
|
+
maxSourceAgeHours: 26,
|
|
18
|
+
maxStaleHours: 48,
|
|
19
|
+
};
|
|
6
20
|
class OoyyoConfig extends config_1.Config {
|
|
21
|
+
dryRun;
|
|
22
|
+
imageConcurrency;
|
|
23
|
+
maxImages;
|
|
24
|
+
stateDir;
|
|
25
|
+
feedGuard;
|
|
7
26
|
constructor(params) {
|
|
8
27
|
super(params);
|
|
9
|
-
this.
|
|
10
|
-
this.
|
|
28
|
+
this.campaign = enums_1.ExportType.OOYYO;
|
|
29
|
+
this.dryRun = text(params.ooyyoDryRun).toLowerCase() !== 'false';
|
|
30
|
+
const mode = this.dryRun ? 'dryrun' : 'export';
|
|
31
|
+
this.pathesConfig.localConvertedDirectory = `resources/${enums_1.ExportType.OOYYO}-${mode}`;
|
|
32
|
+
this.stateDir = `resources/${enums_1.ExportType.OOYYO}-state/${mode}`;
|
|
33
|
+
this.imageType = enums_1.ImageType.clean;
|
|
34
|
+
this.imageConcurrency = positiveNumber(params.ooyyoImageConcurrency, exports.OOYYO_DEFAULTS.imageConcurrency);
|
|
35
|
+
this.maxImages = Math.floor(positiveNumber(params.ooyyoMaxImages, 0));
|
|
36
|
+
this.feedGuard = {
|
|
37
|
+
stateDir: this.stateDir,
|
|
38
|
+
minSourceItems: positiveNumber(params.feedGuardMinSourceItems, exports.OOYYO_DEFAULTS.minSourceItems),
|
|
39
|
+
minPublishedItems: positiveNumber(params.feedGuardMinPublishedItems, exports.OOYYO_DEFAULTS.minPublishedItems),
|
|
40
|
+
maxDropPercent: positiveNumber(params.feedGuardMaxDropPercent, exports.OOYYO_DEFAULTS.maxDropPercent),
|
|
41
|
+
maxSourceAgeHours: positiveNumber(params.feedGuardMaxSourceAgeHours, exports.OOYYO_DEFAULTS.maxSourceAgeHours),
|
|
42
|
+
maxStaleHours: positiveNumber(params.feedGuardMaxStaleHours, exports.OOYYO_DEFAULTS.maxStaleHours),
|
|
43
|
+
acceptDrop: text(params.feedGuardAcceptDrop).toLowerCase() === 'true',
|
|
44
|
+
};
|
|
11
45
|
this.outputFiles = [
|
|
12
|
-
`${this.pathesConfig.localConvertedDirectory}
|
|
13
|
-
`${this.pathesConfig.localConvertedDirectory}
|
|
46
|
+
`${this.pathesConfig.localConvertedDirectory}/${enums_1.ExportType.OOYYO}.xml`,
|
|
47
|
+
`${this.pathesConfig.localConvertedDirectory}/${enums_1.ExportType.OOYYO}.xml.gz`,
|
|
14
48
|
];
|
|
15
49
|
}
|
|
16
50
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ooyyo.config.js","sourceRoot":"","sources":["../../src/configs/ooyyo.config.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"ooyyo.config.js","sourceRoot":"","sources":["../../src/configs/ooyyo.config.ts"],"names":[],"mappings":";;;AAAA,mCAAkC;AAElC,oCAAiD;AAGjD,qCAAkC;AAElC,MAAM,IAAI,GAAG,CAAC,KAAc,EAAU,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAEzF,MAAM,cAAc,GAAG,CAAC,KAAc,EAAE,QAAgB,EAAU,EAAE;IAClE,MAAM,MAAM,GAAG,IAAA,iBAAQ,EAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACrC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClF,CAAC,CAAC;AAIW,QAAA,cAAc,GAAG;IAC5B,gBAAgB,EAAE,EAAE;IACpB,cAAc,EAAE,IAAI;IACpB,iBAAiB,EAAE,CAAC;IACpB,cAAc,EAAE,EAAE;IAClB,iBAAiB,EAAE,EAAE;IACrB,aAAa,EAAE,EAAE;CAClB,CAAC;AASF,MAAa,WAAY,SAAQ,eAAM;IACrC,MAAM,CAAU;IAChB,gBAAgB,CAAS;IAEzB,SAAS,CAAS;IAClB,QAAQ,CAAS;IACjB,SAAS,CAAmB;IAE5B,YAAY,MAAc;QACxB,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,QAAQ,GAAG,kBAAU,CAAC,KAAK,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC;QACjE,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC/C,IAAI,CAAC,YAAY,CAAC,uBAAuB,GAAG,aAAa,kBAAU,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;QACpF,IAAI,CAAC,QAAQ,GAAG,aAAa,kBAAU,CAAC,KAAK,UAAU,IAAI,EAAE,CAAC;QAC9D,IAAI,CAAC,SAAS,GAAG,iBAAS,CAAC,KAAK,CAAC;QACjC,IAAI,CAAC,gBAAgB,GAAG,cAAc,CAAC,MAAM,CAAC,qBAAqB,EAAE,sBAAc,CAAC,gBAAgB,CAAC,CAAC;QACtG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;QACtE,IAAI,CAAC,SAAS,GAAG;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,sBAAc,CAAC,cAAc,CAAC;YAC7F,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,sBAAc,CAAC,iBAAiB,CAAC;YACtG,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,sBAAc,CAAC,cAAc,CAAC;YAC7F,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,sBAAc,CAAC,iBAAiB,CAAC;YACtG,aAAa,EAAE,cAAc,CAAC,MAAM,CAAC,sBAAsB,EAAE,sBAAc,CAAC,aAAa,CAAC;YAC1F,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM;SACtE,CAAC;QACF,IAAI,CAAC,WAAW,GAAG;YACjB,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,IAAI,kBAAU,CAAC,KAAK,MAAM;YACtE,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,IAAI,kBAAU,CAAC,KAAK,SAAS;SAC1E,CAAC;IACJ,CAAC;CACF;AAhCD,kCAgCC"}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import { CarService, LoaderService, ConfigService, RulesService } from '../services';
|
|
2
|
-
import { Exporter } from '../interfaces/exporter.interface';
|
|
3
1
|
import { ModuleRef } from '@nestjs/core';
|
|
2
|
+
import { CarService, RulesService, LoaderService, ConfigService, ImagesService } from '../services';
|
|
3
|
+
import { Exporter } from '../interfaces/exporter.interface';
|
|
4
4
|
export declare class OoyyoExporter implements Exporter {
|
|
5
5
|
private carService;
|
|
6
6
|
private configService;
|
|
7
7
|
private loaderService;
|
|
8
|
-
private moduleRef;
|
|
9
8
|
private rulesService;
|
|
10
|
-
|
|
9
|
+
private imagesService;
|
|
10
|
+
private moduleRef;
|
|
11
|
+
constructor(carService: CarService, configService: ConfigService, loaderService: LoaderService, rulesService: RulesService, imagesService: ImagesService, moduleRef: ModuleRef);
|
|
11
12
|
export(): Promise<void>;
|
|
12
13
|
}
|
|
@@ -11,43 +11,62 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.OoyyoExporter = void 0;
|
|
13
13
|
const common_1 = require("@nestjs/common");
|
|
14
|
-
const services_1 = require("../services");
|
|
15
|
-
const cronicle_helper_1 = require("@bytebrand/cronicle-helper");
|
|
16
14
|
const core_1 = require("@nestjs/core");
|
|
17
|
-
const
|
|
18
|
-
const
|
|
15
|
+
const cronicle_helper_1 = require("@bytebrand/cronicle-helper");
|
|
16
|
+
const services_1 = require("../services");
|
|
17
|
+
const image_url_validator_service_1 = require("../services/image-url-validator.service");
|
|
18
|
+
const feed_guard_1 = require("../services/feed-guard");
|
|
19
|
+
const ooyyo_feed_run_1 = require("../services/ooyyo-feed-run");
|
|
19
20
|
const ooyyo_transformer_1 = require("../transformers/ooyyo.transformer");
|
|
21
|
+
const openai_image_fields_1 = require("../transformers/openai-image-fields");
|
|
20
22
|
let OoyyoExporter = class OoyyoExporter {
|
|
21
23
|
carService;
|
|
22
24
|
configService;
|
|
23
25
|
loaderService;
|
|
24
|
-
moduleRef;
|
|
25
26
|
rulesService;
|
|
26
|
-
|
|
27
|
+
imagesService;
|
|
28
|
+
moduleRef;
|
|
29
|
+
constructor(carService, configService, loaderService, rulesService, imagesService, moduleRef) {
|
|
27
30
|
this.carService = carService;
|
|
28
31
|
this.configService = configService;
|
|
29
32
|
this.loaderService = loaderService;
|
|
30
|
-
this.moduleRef = moduleRef;
|
|
31
33
|
this.rulesService = rulesService;
|
|
34
|
+
this.imagesService = imagesService;
|
|
35
|
+
this.moduleRef = moduleRef;
|
|
32
36
|
}
|
|
33
37
|
async export() {
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
38
|
+
const config = this.configService.config;
|
|
39
|
+
const [xmlFile, gzFile] = config.outputFiles;
|
|
40
|
+
const validator = new image_url_validator_service_1.ImageUrlValidator({ concurrency: config.imageConcurrency });
|
|
41
|
+
const transformer = await this.moduleRef.create(ooyyo_transformer_1.OoyyoTransformer);
|
|
42
|
+
const run = new ooyyo_feed_run_1.OoyyoFeedRun({
|
|
43
|
+
forEachChunk: fn => this.loaderService.process((chunk) => fn(chunk.data)),
|
|
44
|
+
filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
|
|
45
|
+
resolveImages: async ({ car, images }) => (0, ooyyo_transformer_1.toOoyyoImages)(await (0, openai_image_fields_1.resolveOpenAiImageFields)(this.imagesService.getCleanImages(car._id, images), validator), config.maxImages),
|
|
46
|
+
translate: selector => transformer.translate(selector),
|
|
47
|
+
region: (postcode, country) => transformer.region(postcode, country),
|
|
48
|
+
imageStats: () => validator.stats,
|
|
49
|
+
guard: new feed_guard_1.FeedGuard(config.feedGuard),
|
|
50
|
+
}, {
|
|
51
|
+
runId: config.runId,
|
|
52
|
+
dryRun: config.dryRun,
|
|
53
|
+
domain: config.domain,
|
|
54
|
+
sourceDirectory: config.pathesConfig.localDirectory,
|
|
55
|
+
sourceBaseFileName: config.pathesConfig.baseFileName,
|
|
56
|
+
xmlFile,
|
|
57
|
+
gzFile,
|
|
58
|
+
stateDir: config.stateDir,
|
|
47
59
|
});
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
60
|
+
try {
|
|
61
|
+
const report = await run.run();
|
|
62
|
+
if (report.stale) {
|
|
63
|
+
console.log(`ooyyo feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
cronicle_helper_1.LoggerService.exitError(`ooyyo feed not published: ${error?.message ?? error}`);
|
|
68
|
+
process.exit(1);
|
|
69
|
+
}
|
|
51
70
|
}
|
|
52
71
|
};
|
|
53
72
|
exports.OoyyoExporter = OoyyoExporter;
|
|
@@ -56,7 +75,8 @@ exports.OoyyoExporter = OoyyoExporter = __decorate([
|
|
|
56
75
|
__metadata("design:paramtypes", [services_1.CarService,
|
|
57
76
|
services_1.ConfigService,
|
|
58
77
|
services_1.LoaderService,
|
|
59
|
-
|
|
60
|
-
services_1.
|
|
78
|
+
services_1.RulesService,
|
|
79
|
+
services_1.ImagesService,
|
|
80
|
+
core_1.ModuleRef])
|
|
61
81
|
], OoyyoExporter);
|
|
62
82
|
//# sourceMappingURL=ooyyo.exporter.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ooyyo.exporter.js","sourceRoot":"","sources":["../../src/exporters/ooyyo.exporter.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,
|
|
1
|
+
{"version":3,"file":"ooyyo.exporter.js","sourceRoot":"","sources":["../../src/exporters/ooyyo.exporter.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,uCAAyC;AAGzC,gEAA2D;AAE3D,0CAAoG;AACpG,yFAA4E;AAC5E,uDAAmD;AACnD,+DAA0E;AAC1E,yEAAoF;AACpF,6EAA+E;AAUxE,IAAM,aAAa,GAAnB,MAAM,aAAa;IAEd;IACA;IACA;IACA;IACA;IACA;IANV,YACU,UAAsB,EACtB,aAA4B,EAC5B,aAA4B,EAC5B,YAA0B,EAC1B,aAA4B,EAC5B,SAAoB;QALpB,eAAU,GAAV,UAAU,CAAY;QACtB,kBAAa,GAAb,aAAa,CAAe;QAC5B,kBAAa,GAAb,aAAa,CAAe;QAC5B,iBAAY,GAAZ,YAAY,CAAc;QAC1B,kBAAa,GAAb,aAAa,CAAe;QAC5B,cAAS,GAAT,SAAS,CAAW;IAC3B,CAAC;IAEJ,KAAK,CAAC,MAAM;QACV,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,MAAqB,CAAC;QACxD,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QAC7C,MAAM,SAAS,GAAG,IAAI,+CAAiB,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,oCAAgB,CAAC,CAAC;QAElE,MAAM,GAAG,GAAG,IAAI,6BAAY,CAC1B;YACE,YAAY,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,KAAuB,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,IAAwB,CAAC,CAAC;YAC/G,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;YAC/E,aAAa,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,IAAA,iCAAa,EACrD,MAAM,IAAA,8CAAwB,EAAC,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,SAAS,CAAC,EAC7F,MAAM,CAAC,SAAS,CACjB;YACD,SAAS,EAAE,QAAQ,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,CAAC;YACtD,MAAM,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC;YACpE,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK;YACjC,KAAK,EAAE,IAAI,sBAAS,CAAC,MAAM,CAAC,SAAS,CAAC;SACvC,EACD;YACE,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,cAAc;YACnD,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,YAAY;YACpD,OAAO;YACP,MAAM;YACN,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CACF,CAAC;QAEF,IAAI;YACF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;YAC/B,IAAI,MAAM,CAAC,KAAK,EAAE;gBAChB,OAAO,CAAC,GAAG,CAAC,oCAAoC,MAAM,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;aAC9F;SACF;QAAC,OAAO,KAAK,EAAE;YAGd,+BAAa,CAAC,SAAS,CAAC,6BAA6B,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC;YAChF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;CACF,CAAA;AArDY,sCAAa;wBAAb,aAAa;IADzB,IAAA,mBAAU,GAAE;qCAGW,qBAAU;QACP,wBAAa;QACb,wBAAa;QACd,uBAAY;QACX,wBAAa;QACjB,gBAAS;GAPnB,aAAa,CAqDzB"}
|
|
@@ -1,7 +1,12 @@
|
|
|
1
|
-
import { XMLBuilderBase } from '@bytebrand/cronicle-helper';
|
|
2
1
|
import { OoyyoXmlItem } from '../interfaces';
|
|
3
|
-
export declare
|
|
2
|
+
export declare const stripInvalidXmlChars: (value: string) => string;
|
|
3
|
+
export declare const cdata: (value: string) => string;
|
|
4
|
+
export declare class OoyyoXmlBuilder {
|
|
5
|
+
private stream;
|
|
6
|
+
constructor(fileName: string);
|
|
7
|
+
private write;
|
|
4
8
|
buildStart(): Promise<void>;
|
|
5
9
|
addItem(car: OoyyoXmlItem): Promise<void>;
|
|
10
|
+
static toXml(car: OoyyoXmlItem): string;
|
|
6
11
|
buildEnd(): Promise<void>;
|
|
7
12
|
}
|
|
@@ -1,46 +1,69 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.OoyyoXmlBuilder = void 0;
|
|
4
|
-
const
|
|
5
|
-
|
|
3
|
+
exports.OoyyoXmlBuilder = exports.cdata = exports.stripInvalidXmlChars = void 0;
|
|
4
|
+
const fs_1 = require("fs");
|
|
5
|
+
const INVALID_XML_CHARS = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;
|
|
6
|
+
const stripInvalidXmlChars = (value) => value.replace(INVALID_XML_CHARS, '');
|
|
7
|
+
exports.stripInvalidXmlChars = stripInvalidXmlChars;
|
|
8
|
+
const cdata = (value) => `<![CDATA[${(0, exports.stripInvalidXmlChars)(value).replace(/]]>/g, ']]]]><![CDATA[>')}]]>`;
|
|
9
|
+
exports.cdata = cdata;
|
|
10
|
+
class OoyyoXmlBuilder {
|
|
11
|
+
stream;
|
|
12
|
+
constructor(fileName) {
|
|
13
|
+
this.stream = (0, fs_1.createWriteStream)(fileName, { flags: 'w', encoding: 'utf8' });
|
|
14
|
+
}
|
|
15
|
+
write(chunk) {
|
|
16
|
+
return new Promise((resolve, reject) => this.stream.write(chunk, error => (error ? reject(error) : resolve())));
|
|
17
|
+
}
|
|
6
18
|
async buildStart() {
|
|
7
|
-
await this.write(
|
|
8
|
-
|
|
19
|
+
await this.write('<?xml version="1.0" encoding="UTF-8"?>\n'
|
|
20
|
+
+ '<ooyyo xsi:noNamespaceSchemaLocation="OoyyoFormat.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">\n');
|
|
9
21
|
}
|
|
10
22
|
async addItem(car) {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
23
|
+
await this.write(OoyyoXmlBuilder.toXml(car));
|
|
24
|
+
}
|
|
25
|
+
static toXml(car) {
|
|
26
|
+
const el = (tag, value, unit, indent = ' ') => (value ? `${indent}<${tag}${unit ? ` unit="${unit}"` : ''}>${(0, exports.cdata)(value)}</${tag}>\n` : '');
|
|
27
|
+
const location = [
|
|
28
|
+
el('city', car.location?.city, undefined, ' '),
|
|
29
|
+
el('postcode', car.location?.postcode, undefined, ' '),
|
|
30
|
+
el('region', car.location?.region, undefined, ' '),
|
|
31
|
+
].join('');
|
|
32
|
+
const pictures = (car.pictures ?? []).map(url => el('picture', url, undefined, ' ')).join('');
|
|
33
|
+
return ' <ad>\n'
|
|
34
|
+
+ el('url', car.url)
|
|
35
|
+
+ el('title', car.title)
|
|
36
|
+
+ el('content', car.content)
|
|
37
|
+
+ el('price', car.price, car.priceUnit)
|
|
38
|
+
+ el('make', car.make)
|
|
39
|
+
+ el('model', car.model)
|
|
40
|
+
+ el('engine_size', car.engine_size, 'cc')
|
|
41
|
+
+ el('trim', car.trim)
|
|
42
|
+
+ el('body_type', car.body_type)
|
|
43
|
+
+ el('color', car.color)
|
|
44
|
+
+ el('year', car.year)
|
|
45
|
+
+ el('mileage', car.mileage, 'km')
|
|
46
|
+
+ el('fuel_type', car.fuel_type)
|
|
47
|
+
+ el('transmission', car.transmission)
|
|
48
|
+
+ el('euronorm', car.euronorm)
|
|
49
|
+
+ el('power', car.power, 'kw')
|
|
50
|
+
+ el('doors', car.doors)
|
|
51
|
+
+ el('seats', car.seats)
|
|
52
|
+
+ el('gears', car.gears)
|
|
53
|
+
+ (location ? ` <location>\n${location} </location>\n` : '')
|
|
54
|
+
+ (pictures ? ` <pictures>\n${pictures} </pictures>\n` : '')
|
|
55
|
+
+ el('vin', car.vin)
|
|
56
|
+
+ el('is_new', car.is_new)
|
|
57
|
+
+ el('date', car.date)
|
|
58
|
+
+ el('time', car.time)
|
|
59
|
+
+ ' </ad>\n';
|
|
40
60
|
}
|
|
41
61
|
async buildEnd() {
|
|
42
|
-
await this.write('
|
|
43
|
-
await
|
|
62
|
+
await this.write('</ooyyo>\n');
|
|
63
|
+
await new Promise((resolve, reject) => {
|
|
64
|
+
this.stream.on('error', reject);
|
|
65
|
+
this.stream.end(() => resolve());
|
|
66
|
+
});
|
|
44
67
|
}
|
|
45
68
|
}
|
|
46
69
|
exports.OoyyoXmlBuilder = OoyyoXmlBuilder;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ooyyo-xml.builder.js","sourceRoot":"","sources":["../../src/formats/ooyyo-xml.builder.ts"],"names":[],"mappings":";;;AAAA,
|
|
1
|
+
{"version":3,"file":"ooyyo-xml.builder.js","sourceRoot":"","sources":["../../src/formats/ooyyo-xml.builder.ts"],"names":[],"mappings":";;;AAAA,2BAAoD;AAKpD,MAAM,iBAAiB,GAAG,oHAAoH,CAAC;AAExI,MAAM,oBAAoB,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AAAvF,QAAA,oBAAoB,wBAAmE;AAI7F,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAC7C,YAAY,IAAA,4BAAoB,EAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,iBAAiB,CAAC,KAAK,CAAC;AADrE,QAAA,KAAK,SACgE;AAQlF,MAAa,eAAe;IAClB,MAAM,CAAc;IAE5B,YAAY,QAAgB;QAC1B,IAAI,CAAC,MAAM,GAAG,IAAA,sBAAiB,EAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9E,CAAC;IAEO,KAAK,CAAC,KAAa;QACzB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAClH,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,KAAK,CAAC,0CAA0C;cACvD,iHAAiH,CAAC,CAAC;IACzH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAiB;QAC7B,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,GAAiB;QAC5B,MAAM,EAAE,GAAG,CAAC,GAAW,EAAE,KAAa,EAAE,IAAa,EAAE,MAAM,GAAG,MAAM,EAAU,EAAE,CAChF,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,IAAA,aAAK,EAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/F,MAAM,QAAQ,GAAG;YACf,EAAE,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC;YACnD,EAAE,CAAC,UAAU,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC;YAC3D,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC;SACxD,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACX,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEnG,OAAO,UAAU;cACb,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;cAClB,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC;cACtB,EAAE,CAAC,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC;cAC1B,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC;cACrC,EAAE,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC;cACpB,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC;cACtB,EAAE,CAAC,aAAa,EAAE,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC;cACxC,EAAE,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC;cACpB,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,SAAS,CAAC;cAC9B,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC;cACtB,EAAE,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC;cACpB,EAAE,CAAC,SAAS,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC;cAChC,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,SAAS,CAAC;cAC9B,EAAE,CAAC,cAAc,EAAE,GAAG,CAAC,YAAY,CAAC;cACpC,EAAE,CAAC,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC;cAC5B,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC;cAC5B,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC;cACtB,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC;cACtB,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC;cACtB,CAAC,QAAQ,CAAC,CAAC,CAAC,mBAAmB,QAAQ,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;cAChE,CAAC,QAAQ,CAAC,CAAC,CAAC,mBAAmB,QAAQ,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;cAChE,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;cAClB,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC;cACxB,EAAE,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC;cACpB,EAAE,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC;cACpB,WAAW,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC/B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAlED,0CAkEC"}
|
|
@@ -58,9 +58,37 @@ export interface Auto24XmlItem {
|
|
|
58
58
|
dealer_name: string;
|
|
59
59
|
dealer_phone: string;
|
|
60
60
|
}
|
|
61
|
-
export interface OoyyoXmlItem
|
|
62
|
-
|
|
61
|
+
export interface OoyyoXmlItem {
|
|
62
|
+
url: string;
|
|
63
|
+
title: string;
|
|
64
|
+
content: string;
|
|
65
|
+
price: string;
|
|
66
|
+
priceUnit: 'EUR';
|
|
67
|
+
make: string;
|
|
68
|
+
model: string;
|
|
69
|
+
engine_size: string;
|
|
70
|
+
trim: string;
|
|
71
|
+
body_type: string;
|
|
72
|
+
color: string;
|
|
73
|
+
year: string;
|
|
63
74
|
mileage: string;
|
|
75
|
+
fuel_type: string;
|
|
76
|
+
transmission: string;
|
|
77
|
+
euronorm: string;
|
|
78
|
+
power: string;
|
|
79
|
+
doors: string;
|
|
80
|
+
seats: string;
|
|
81
|
+
gears: string;
|
|
82
|
+
location: {
|
|
83
|
+
city: string;
|
|
84
|
+
postcode: string;
|
|
85
|
+
region: string;
|
|
86
|
+
};
|
|
87
|
+
pictures: string[];
|
|
88
|
+
vin: string;
|
|
89
|
+
is_new: string;
|
|
90
|
+
date: string;
|
|
91
|
+
time: string;
|
|
64
92
|
}
|
|
65
93
|
export interface MeinstadtJsonItem {
|
|
66
94
|
id: string;
|
|
@@ -37,6 +37,9 @@ export interface Params {
|
|
|
37
37
|
marktdeDryRun?: string;
|
|
38
38
|
marktdeImageConcurrency?: string;
|
|
39
39
|
marktdeMaxImages?: string;
|
|
40
|
+
ooyyoDryRun?: string;
|
|
41
|
+
ooyyoImageConcurrency?: string;
|
|
42
|
+
ooyyoMaxImages?: string;
|
|
40
43
|
}
|
|
41
44
|
export interface FeedItemsBigQueryTarget {
|
|
42
45
|
project: string;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { ImboImageResponseInterface } from '@bytebrand/chunkable';
|
|
2
|
+
import { DataSchemes } from '@bytebrand/car-schema';
|
|
3
|
+
import { FeedGuard, LastGood } from './feed-guard';
|
|
4
|
+
export interface OoyyoSourceCar {
|
|
5
|
+
car: DataSchemes.Car;
|
|
6
|
+
images: ImboImageResponseInterface[];
|
|
7
|
+
}
|
|
8
|
+
export interface OoyyoRunDeps {
|
|
9
|
+
forEachChunk: (fn: (cars: OoyyoSourceCar[]) => Promise<void>) => Promise<unknown>;
|
|
10
|
+
filterCar: (source: OoyyoSourceCar) => boolean;
|
|
11
|
+
resolveImages: (source: OoyyoSourceCar) => Promise<string[]>;
|
|
12
|
+
translate: (selector: string) => string;
|
|
13
|
+
region: (postcode: string, countryCode: unknown) => Promise<string>;
|
|
14
|
+
imageStats: () => Record<string, number>;
|
|
15
|
+
guard: FeedGuard;
|
|
16
|
+
}
|
|
17
|
+
export interface OoyyoRunOptions {
|
|
18
|
+
runId?: string;
|
|
19
|
+
dryRun: boolean;
|
|
20
|
+
domain: string;
|
|
21
|
+
sourceDirectory: string;
|
|
22
|
+
sourceBaseFileName: string;
|
|
23
|
+
xmlFile: string;
|
|
24
|
+
gzFile: string;
|
|
25
|
+
stateDir: string;
|
|
26
|
+
now?: () => Date;
|
|
27
|
+
}
|
|
28
|
+
export interface OoyyoRunReport {
|
|
29
|
+
runId?: string;
|
|
30
|
+
mode: 'dry-run' | 'live';
|
|
31
|
+
startedAt: string;
|
|
32
|
+
finishedAt?: string;
|
|
33
|
+
status: 'published' | 'blocked' | 'failed';
|
|
34
|
+
guard?: {
|
|
35
|
+
code: string;
|
|
36
|
+
message: string;
|
|
37
|
+
};
|
|
38
|
+
error?: string;
|
|
39
|
+
xmlFile: string;
|
|
40
|
+
gzFile: string;
|
|
41
|
+
source: {
|
|
42
|
+
files: number;
|
|
43
|
+
newestChunkAt: string | null;
|
|
44
|
+
cars: number;
|
|
45
|
+
};
|
|
46
|
+
items: number;
|
|
47
|
+
excluded: Record<string, number>;
|
|
48
|
+
missing: Record<string, number>;
|
|
49
|
+
postcodePadded: number;
|
|
50
|
+
verify?: {
|
|
51
|
+
ads: number;
|
|
52
|
+
issues: number;
|
|
53
|
+
firstIssues: Array<{
|
|
54
|
+
ad: number;
|
|
55
|
+
problem: string;
|
|
56
|
+
}>;
|
|
57
|
+
};
|
|
58
|
+
bytes?: {
|
|
59
|
+
xml: number;
|
|
60
|
+
gz: number;
|
|
61
|
+
sha256: string;
|
|
62
|
+
};
|
|
63
|
+
imagesPerItem: {
|
|
64
|
+
min: number | null;
|
|
65
|
+
max: number | null;
|
|
66
|
+
total: number;
|
|
67
|
+
};
|
|
68
|
+
images: Record<string, number>;
|
|
69
|
+
lastGood: LastGood | null;
|
|
70
|
+
lastGoodAgeHours: number | null;
|
|
71
|
+
stale: boolean;
|
|
72
|
+
}
|
|
73
|
+
export declare class OoyyoFeedRun {
|
|
74
|
+
private readonly deps;
|
|
75
|
+
private readonly options;
|
|
76
|
+
constructor(deps: OoyyoRunDeps, options: OoyyoRunOptions);
|
|
77
|
+
private now;
|
|
78
|
+
run(): Promise<OoyyoRunReport>;
|
|
79
|
+
private count;
|
|
80
|
+
private writeReport;
|
|
81
|
+
}
|