@bytebrand/export-cars 1.26.31 → 1.26.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/configs/autouncle.config.d.ts +14 -0
- package/dist/configs/autouncle.config.js +36 -3
- package/dist/configs/autouncle.config.js.map +1 -1
- package/dist/configs/meinstadt.config.d.ts +15 -0
- package/dist/configs/meinstadt.config.js +42 -3
- package/dist/configs/meinstadt.config.js.map +1 -1
- package/dist/constants/autouncle.contants.d.ts +1 -3
- package/dist/constants/autouncle.contants.js +2 -4
- package/dist/constants/autouncle.contants.js.map +1 -1
- package/dist/exporters/autouncle.exporter.d.ts +3 -2
- package/dist/exporters/autouncle.exporter.js +45 -27
- package/dist/exporters/autouncle.exporter.js.map +1 -1
- package/dist/exporters/meinstadt.exporter.d.ts +3 -2
- package/dist/exporters/meinstadt.exporter.js +40 -23
- package/dist/exporters/meinstadt.exporter.js.map +1 -1
- package/dist/formats/autouncle-json.builder.js +1 -4
- package/dist/formats/autouncle-json.builder.js.map +1 -1
- package/dist/interfaces/export-item.interface.d.ts +8 -12
- package/dist/interfaces/params.interface.d.ts +6 -0
- package/dist/services/autouncle-feed-run.d.ts +66 -0
- package/dist/services/autouncle-feed-run.js +135 -0
- package/dist/services/autouncle-feed-run.js.map +1 -0
- package/dist/services/meinstadt-feed-run.d.ts +81 -0
- package/dist/services/meinstadt-feed-run.js +166 -0
- package/dist/services/meinstadt-feed-run.js.map +1 -0
- package/dist/transformers/autouncle.transformer.d.ts +38 -22
- package/dist/transformers/autouncle.transformer.js +156 -144
- package/dist/transformers/autouncle.transformer.js.map +1 -1
- package/dist/transformers/meinstadt.transformer.d.ts +17 -13
- package/dist/transformers/meinstadt.transformer.js +88 -61
- package/dist/transformers/meinstadt.transformer.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/autouncle/AUDIT.md +97 -0
- package/docs/autouncle/PARTNER-QUESTIONS.md +53 -0
- package/docs/autouncle/RUNBOOK.md +64 -0
- package/docs/meinestadt/AUDIT.md +146 -0
- package/docs/meinestadt/PARTNER-QUESTIONS.md +54 -0
- package/docs/meinestadt/RUNBOOK.md +64 -0
- package/package.json +3 -2
- package/src/configs/autouncle.config.ts +51 -3
- package/src/configs/meinstadt.config.ts +57 -2
- package/src/constants/autouncle.contants.ts +2 -3
- package/src/exporters/autouncle.exporter.ts +59 -37
- package/src/exporters/meinstadt.exporter.ts +53 -32
- package/src/formats/autouncle-json.builder.ts +2 -4
- package/src/interfaces/export-item.interface.ts +15 -12
- package/src/interfaces/params.interface.ts +10 -0
- package/src/services/autouncle-feed-run.ts +197 -0
- package/src/services/meinstadt-feed-run.ts +240 -0
- package/src/transformers/autouncle.transformer.ts +217 -175
- package/src/transformers/meinstadt.transformer.ts +114 -68
- package/test/autouncle-feed-run.test.ts +256 -0
- package/test/autouncle-mapping.test.ts +225 -0
- package/test/fixtures/meinstadt.fixture.ts +87 -0
- package/test/meinstadt-feed-run.test.ts +288 -0
- package/test/meinstadt-mapping.test.ts +215 -0
|
@@ -1,12 +1,67 @@
|
|
|
1
|
+
import { toNumber } from 'lodash';
|
|
2
|
+
|
|
1
3
|
import { Params } from '../interfaces/params.interface';
|
|
4
|
+
import { FeedGuardOptions } from '../services/feed-guard';
|
|
2
5
|
import { Config } from './config';
|
|
3
6
|
import { ExportType, FileType, ImageType } from '../enums';
|
|
4
7
|
|
|
8
|
+
const text = (value: unknown): string => (typeof value === 'string' ? value.trim() : '');
|
|
9
|
+
|
|
10
|
+
const positiveNumber = (value: unknown, fallback: number): number => {
|
|
11
|
+
const parsed = toNumber(text(value));
|
|
12
|
+
return text(value) && Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
// A CPC tariff as the Cronicle text param holds it: a positive decimal with a dot ("0.99").
|
|
16
|
+
// Anything else (empty, 0, "0,08", "abc") counts as not configured.
|
|
17
|
+
export const parseMeinstadtCpc = (value: unknown): string | null => {
|
|
18
|
+
const raw = text(value);
|
|
19
|
+
return /^\d+(\.\d+)?$/.test(raw) && Number(raw) > 0 ? raw : null;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
// Same guard defaults as the OpenAI feed (prod-sized); DEV needs smaller minimums via the params.
|
|
23
|
+
export const MEINSTADT_DEFAULTS = {
|
|
24
|
+
imageConcurrency: 16,
|
|
25
|
+
minSourceItems: 1000,
|
|
26
|
+
minPublishedItems: 500,
|
|
27
|
+
maxDropPercent: 20,
|
|
28
|
+
maxSourceAgeHours: 26,
|
|
29
|
+
maxStaleHours: 48,
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* meinestadt.de vehicle feed (ADV-325/326): JSON array + ZIP of it, clean ADV-336 images.
|
|
34
|
+
* Dry-run is the default: the files go to resources/meinstadt-dryrun, which no upload step reads.
|
|
35
|
+
* Only meinstadtDryRun=false writes resources/meinstadt-export (the path the Cronicle FTP step
|
|
36
|
+
* uploads from), and only with a configured cpc. Each mode has its own guard state.
|
|
37
|
+
*/
|
|
5
38
|
export class MeinstadtConfig extends Config {
|
|
39
|
+
dryRun: boolean;
|
|
40
|
+
imageConcurrency: number;
|
|
41
|
+
// 0 = no limit (no partner limit known)
|
|
42
|
+
maxImages: number;
|
|
43
|
+
stateDir: string;
|
|
44
|
+
feedGuard: FeedGuardOptions;
|
|
45
|
+
|
|
6
46
|
constructor(params: Params) {
|
|
7
47
|
super(params);
|
|
8
|
-
this.
|
|
9
|
-
this.
|
|
48
|
+
this.dryRun = text(params.meinstadtDryRun).toLowerCase() !== 'false';
|
|
49
|
+
const mode = this.dryRun ? 'dryrun' : 'export';
|
|
50
|
+
this.pathesConfig.localConvertedDirectory = `resources/${ExportType.MEINSTADT}-${mode}`;
|
|
51
|
+
this.stateDir = `resources/${ExportType.MEINSTADT}-state/${mode}`;
|
|
52
|
+
this.imageType = ImageType.clean;
|
|
53
|
+
this.cpc = parseMeinstadtCpc(params.cpc);
|
|
54
|
+
this.imageConcurrency = positiveNumber(params.meinstadtImageConcurrency, MEINSTADT_DEFAULTS.imageConcurrency);
|
|
55
|
+
this.maxImages = Math.floor(positiveNumber(params.meinstadtMaxImages, 0));
|
|
56
|
+
this.feedGuard = {
|
|
57
|
+
stateDir: this.stateDir,
|
|
58
|
+
minSourceItems: positiveNumber(params.feedGuardMinSourceItems, MEINSTADT_DEFAULTS.minSourceItems),
|
|
59
|
+
minPublishedItems: positiveNumber(params.feedGuardMinPublishedItems, MEINSTADT_DEFAULTS.minPublishedItems),
|
|
60
|
+
maxDropPercent: positiveNumber(params.feedGuardMaxDropPercent, MEINSTADT_DEFAULTS.maxDropPercent),
|
|
61
|
+
maxSourceAgeHours: positiveNumber(params.feedGuardMaxSourceAgeHours, MEINSTADT_DEFAULTS.maxSourceAgeHours),
|
|
62
|
+
maxStaleHours: positiveNumber(params.feedGuardMaxStaleHours, MEINSTADT_DEFAULTS.maxStaleHours),
|
|
63
|
+
acceptDrop: text(params.feedGuardAcceptDrop).toLowerCase() === 'true',
|
|
64
|
+
};
|
|
10
65
|
this.outputFiles = [
|
|
11
66
|
`${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.MEINSTADT}.${FileType.JSON}`,
|
|
12
67
|
`${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.MEINSTADT}.${FileType.ZIP}`,
|
|
@@ -1,16 +1,23 @@
|
|
|
1
|
-
import { get } from 'lodash';
|
|
2
|
-
|
|
3
1
|
import { Injectable } from '@nestjs/common';
|
|
4
2
|
import { ModuleRef } from '@nestjs/core';
|
|
5
3
|
|
|
6
|
-
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
7
4
|
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
5
|
+
import { LoggerService } from '@bytebrand/cronicle-helper';
|
|
8
6
|
|
|
9
|
-
import { AutouncleTransformer } from '../transformers/autouncle.transformer';
|
|
10
|
-
import {
|
|
11
|
-
import { CarService, RulesService, LoaderService, ConfigService } from '../services';
|
|
7
|
+
import { AutouncleTransformer, AutouncleAddress, toAutouncleImages } from '../transformers/autouncle.transformer';
|
|
8
|
+
import { resolveOpenAiImageFields } from '../transformers/openai-image-fields';
|
|
9
|
+
import { CarService, RulesService, LoaderService, ConfigService, ImagesService } from '../services';
|
|
10
|
+
import { ImageUrlValidator } from '../services/image-url-validator.service';
|
|
11
|
+
import { FeedGuard } from '../services/feed-guard';
|
|
12
|
+
import { AutouncleFeedRun, AutouncleSourceCar } from '../services/autouncle-feed-run';
|
|
13
|
+
import { AutouncleConfig } from '../configs/autouncle.config';
|
|
12
14
|
import { Exporter } from '../interfaces/exporter.interface';
|
|
13
15
|
|
|
16
|
+
/**
|
|
17
|
+
* ExportType.AUTOUNCLE (ADV-322/323). Wires the Cronicle/Nest services into AutouncleFeedRun, which
|
|
18
|
+
* holds the logic. The output directory is no longer emptied: the new JSON replaces the old one only
|
|
19
|
+
* after the fail-closed guard passed.
|
|
20
|
+
*/
|
|
14
21
|
@Injectable()
|
|
15
22
|
export class AutouncleExporter implements Exporter {
|
|
16
23
|
constructor(
|
|
@@ -18,43 +25,58 @@ export class AutouncleExporter implements Exporter {
|
|
|
18
25
|
private configService: ConfigService,
|
|
19
26
|
private loaderService: LoaderService,
|
|
20
27
|
private rulesService: RulesService,
|
|
28
|
+
private imagesService: ImagesService,
|
|
21
29
|
private moduleRef: ModuleRef,
|
|
22
30
|
) {}
|
|
23
31
|
|
|
24
32
|
async export(): Promise<void> {
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
const
|
|
30
|
-
await autouncleJsonBuilder.buildStart();
|
|
31
|
-
|
|
32
|
-
const autouncleTransformer = await this.moduleRef.create(AutouncleTransformer);
|
|
33
|
-
|
|
34
|
-
let firstCar = true;
|
|
35
|
-
await this.loaderService.process(async (chunk: FsChunkInterface) => {
|
|
36
|
-
chunk.data = chunk.data.filter(e => this.carService.filterCar(e, this.rulesService.rules));
|
|
33
|
+
const config = this.configService.config as AutouncleConfig;
|
|
34
|
+
const [jsonFile] = config.outputFiles;
|
|
35
|
+
const validator = new ImageUrlValidator({ concurrency: config.imageConcurrency });
|
|
36
|
+
const transformer = await this.moduleRef.create(AutouncleTransformer);
|
|
37
|
+
const addresses = new Map<string, Promise<AutouncleAddress | null>>();
|
|
37
38
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
39
|
+
const run = new AutouncleFeedRun(
|
|
40
|
+
{
|
|
41
|
+
forEachChunk: fn => this.loaderService.process((chunk: FsChunkInterface) => fn(chunk.data as AutouncleSourceCar[])),
|
|
42
|
+
filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
|
|
43
|
+
dealerAddress: (ownerId) => {
|
|
44
|
+
if (!addresses.has(ownerId)) {
|
|
45
|
+
addresses.set(ownerId, this.carService.getOwner(ownerId)
|
|
46
|
+
.then(owner => (owner ? transformer.getAddress(owner) : null))
|
|
47
|
+
.catch((error: any) => { console.error(error?.message ?? error); return null; }));
|
|
48
|
+
}
|
|
49
|
+
return addresses.get(ownerId);
|
|
50
|
+
},
|
|
51
|
+
resolveImages: async ({ car, images }) => toAutouncleImages(
|
|
52
|
+
await resolveOpenAiImageFields(this.imagesService.getCleanImages(car._id, images), validator),
|
|
53
|
+
config.maxImages,
|
|
54
|
+
),
|
|
55
|
+
imageStats: () => validator.stats,
|
|
56
|
+
translate: key => transformer.translate(key),
|
|
57
|
+
guard: new FeedGuard(config.feedGuard),
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
runId: config.runId,
|
|
61
|
+
dryRun: config.dryRun,
|
|
62
|
+
domain: config.domain,
|
|
63
|
+
sourceDirectory: config.pathesConfig.localDirectory,
|
|
64
|
+
sourceBaseFileName: config.pathesConfig.baseFileName,
|
|
65
|
+
jsonFile,
|
|
66
|
+
stateDir: config.stateDir,
|
|
67
|
+
},
|
|
68
|
+
);
|
|
46
69
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
if (firstCar) {
|
|
52
|
-
firstCar = false;
|
|
53
|
-
}
|
|
70
|
+
try {
|
|
71
|
+
const report = await run.run();
|
|
72
|
+
if (report.stale) {
|
|
73
|
+
console.log(`autouncle feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
|
|
54
74
|
}
|
|
55
|
-
})
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
75
|
+
} catch (error) {
|
|
76
|
+
// main.ts would report a rejected run as {"complete":1,"code":0}; fail the job so the chained
|
|
77
|
+
// `3.1 … part 2` copy step does not run (report + lock already handled by the run).
|
|
78
|
+
LoggerService.exitError(`autouncle feed not published: ${error?.message ?? error}`);
|
|
79
|
+
process.exit(1);
|
|
80
|
+
}
|
|
59
81
|
}
|
|
60
82
|
}
|
|
@@ -1,16 +1,23 @@
|
|
|
1
1
|
import { Injectable } from '@nestjs/common';
|
|
2
2
|
import { ModuleRef } from '@nestjs/core';
|
|
3
3
|
|
|
4
|
-
import { CronicleUtils } from '@bytebrand/cronicle-helper';
|
|
5
4
|
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
6
|
-
import {
|
|
5
|
+
import { LoggerService } from '@bytebrand/cronicle-helper';
|
|
7
6
|
|
|
8
|
-
import { CarService, RulesService, LoaderService, ConfigService } from '../services';
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
7
|
+
import { CarService, RulesService, LoaderService, ConfigService, ImagesService } from '../services';
|
|
8
|
+
import { ImageUrlValidator } from '../services/image-url-validator.service';
|
|
9
|
+
import { FeedGuard } from '../services/feed-guard';
|
|
10
|
+
import { MeinstadtFeedRun, MeinstadtSourceCar, zipSingleFile } from '../services/meinstadt-feed-run';
|
|
11
|
+
import { MeinstadtTransformer, toMeinstadtImages } from '../transformers/meinstadt.transformer';
|
|
12
|
+
import { resolveOpenAiImageFields } from '../transformers/openai-image-fields';
|
|
13
|
+
import { MeinstadtConfig } from '../configs/meinstadt.config';
|
|
12
14
|
import { Exporter } from '../interfaces/exporter.interface';
|
|
13
15
|
|
|
16
|
+
/**
|
|
17
|
+
* ExportType.MEINSTADT (ADV-325/326). Wires the Cronicle/Nest services into MeinstadtFeedRun, which
|
|
18
|
+
* holds the logic. The output directory is no longer emptied: the new JSON/ZIP replace the old ones
|
|
19
|
+
* only after the fail-closed guard passed.
|
|
20
|
+
*/
|
|
14
21
|
@Injectable()
|
|
15
22
|
export class MeinstadtExporter implements Exporter {
|
|
16
23
|
constructor(
|
|
@@ -18,38 +25,52 @@ export class MeinstadtExporter implements Exporter {
|
|
|
18
25
|
private configService: ConfigService,
|
|
19
26
|
private rulesService: RulesService,
|
|
20
27
|
private loaderService: LoaderService,
|
|
28
|
+
private imagesService: ImagesService,
|
|
21
29
|
private moduleRef: ModuleRef,
|
|
22
30
|
) {}
|
|
23
31
|
|
|
24
32
|
async export(): Promise<void> {
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
33
|
+
const config = this.configService.config as MeinstadtConfig;
|
|
34
|
+
const [jsonFile, zipFile] = config.outputFiles;
|
|
35
|
+
const validator = new ImageUrlValidator({ concurrency: config.imageConcurrency });
|
|
36
|
+
const transformer = await this.moduleRef.create(MeinstadtTransformer);
|
|
28
37
|
|
|
29
|
-
const
|
|
30
|
-
|
|
38
|
+
const run = new MeinstadtFeedRun(
|
|
39
|
+
{
|
|
40
|
+
forEachChunk: fn => this.loaderService.process((chunk: FsChunkInterface) => fn(chunk.data as MeinstadtSourceCar[])),
|
|
41
|
+
filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
|
|
42
|
+
resolveImages: async ({ car, images }) => toMeinstadtImages(
|
|
43
|
+
await resolveOpenAiImageFields(this.imagesService.getCleanImages(car._id, images), validator),
|
|
44
|
+
config.maxImages,
|
|
45
|
+
),
|
|
46
|
+
imageStats: () => validator.stats,
|
|
47
|
+
equipment: car => transformer.getDescription(car),
|
|
48
|
+
zip: zipSingleFile,
|
|
49
|
+
guard: new FeedGuard(config.feedGuard),
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
runId: config.runId,
|
|
53
|
+
dryRun: config.dryRun,
|
|
54
|
+
cpc: config.cpc,
|
|
55
|
+
domain: config.domain,
|
|
56
|
+
sourceDirectory: config.pathesConfig.localDirectory,
|
|
57
|
+
sourceBaseFileName: config.pathesConfig.baseFileName,
|
|
58
|
+
jsonFile,
|
|
59
|
+
zipFile,
|
|
60
|
+
stateDir: config.stateDir,
|
|
61
|
+
},
|
|
62
|
+
);
|
|
31
63
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
chunk.data = chunk.data.filter(e => this.carService.filterCar(e, this.rulesService.rules));
|
|
37
|
-
|
|
38
|
-
for (const data of chunk.data) {
|
|
39
|
-
// console.time(`Transform ${data.car._id}`);
|
|
40
|
-
const car = await meinstadtTransformer.transform(data);
|
|
41
|
-
// console.timeEnd(`Transform ${data.car._id}`);
|
|
42
|
-
await meinstadtJsonBuilder.addItem(car as MeinstadtJsonItem, firstCar);
|
|
43
|
-
this.carService.counter.countCar(data.images);
|
|
44
|
-
|
|
45
|
-
if (firstCar) {
|
|
46
|
-
firstCar = false;
|
|
47
|
-
}
|
|
64
|
+
try {
|
|
65
|
+
const report = await run.run();
|
|
66
|
+
if (report.stale) {
|
|
67
|
+
console.log(`meinestadt feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
|
|
48
68
|
}
|
|
49
|
-
})
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
69
|
+
} catch (error) {
|
|
70
|
+
// main.ts would report a rejected run as {"complete":1,"code":0}; fail the job so the chained
|
|
71
|
+
// upload step does not run (report + lock already handled by the run).
|
|
72
|
+
LoggerService.exitError(`meinestadt feed not published: ${error?.message ?? error}`);
|
|
73
|
+
process.exit(1);
|
|
74
|
+
}
|
|
54
75
|
}
|
|
55
76
|
}
|
|
@@ -6,11 +6,9 @@ export class AutouncleJsonBuilder extends XMLBuilderBase {
|
|
|
6
6
|
await this.write('{"ads":[');
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
+
// firstSym: the first item goes without a leading comma (it was written twice before)
|
|
9
10
|
async addItem(car: AutouncleJsonItem, firstSym?: boolean): Promise<void> {
|
|
10
|
-
|
|
11
|
-
await this.write(`${JSON.stringify(car)}`);
|
|
12
|
-
}
|
|
13
|
-
await this.write(`,${JSON.stringify(car)}`);
|
|
11
|
+
await this.write(`${firstSym ? '' : ','}${JSON.stringify(car)}`);
|
|
14
12
|
}
|
|
15
13
|
|
|
16
14
|
async buildEnd(): Promise<void> {
|
|
@@ -67,23 +67,27 @@ export interface OoyyoXmlItem extends FbXmlItem {
|
|
|
67
67
|
mileage: string;
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
+
// Keys and order as the meinestadt exporter emits them (see docs/meinestadt/AUDIT.md §3).
|
|
70
71
|
export interface MeinstadtJsonItem {
|
|
71
72
|
id: string;
|
|
72
73
|
make: string;
|
|
73
74
|
model: string;
|
|
74
|
-
|
|
75
|
+
// configured tariff; null only in dry-run without one
|
|
76
|
+
cpc: string | null;
|
|
75
77
|
listingUrl: string;
|
|
76
78
|
title: string;
|
|
77
|
-
|
|
79
|
+
// null = unknown, never "accident-free"
|
|
80
|
+
accidentFree: boolean | null;
|
|
78
81
|
mileage: number;
|
|
79
|
-
|
|
80
|
-
|
|
82
|
+
// Unix seconds, null when missing/invalid
|
|
83
|
+
initialRegistration: number | null;
|
|
84
|
+
offerCreationDate: number | null;
|
|
81
85
|
numberOfDoors: number;
|
|
82
86
|
numberOfSeats: number;
|
|
83
87
|
numberOfPreviousOwners: number;
|
|
84
88
|
price: number;
|
|
85
|
-
|
|
86
|
-
|
|
89
|
+
// PS
|
|
90
|
+
power: number;
|
|
87
91
|
co2Emissions: number;
|
|
88
92
|
fuelConsumptionCombined: number;
|
|
89
93
|
equipment: string;
|
|
@@ -92,14 +96,13 @@ export interface MeinstadtJsonItem {
|
|
|
92
96
|
emissionClass: string;
|
|
93
97
|
emissionSticker: string;
|
|
94
98
|
fuelType: string;
|
|
95
|
-
transmissionType: string;
|
|
96
|
-
sellerType: string;
|
|
97
|
-
usageState: string;
|
|
98
|
-
imagesCount: number;
|
|
99
|
-
images: string;
|
|
100
|
-
power: number;
|
|
101
99
|
'transmissionType/transmission': string;
|
|
100
|
+
// = images.length
|
|
101
|
+
imagesCount: number;
|
|
102
102
|
'customerType/sellerType': string;
|
|
103
|
+
usageState: string;
|
|
104
|
+
// validated public image URLs, hero first
|
|
105
|
+
images: string[];
|
|
103
106
|
}
|
|
104
107
|
|
|
105
108
|
export interface AutouncleJsonItem {
|
|
@@ -38,6 +38,16 @@ export interface Params {
|
|
|
38
38
|
feedGuardMaxStaleHours?: string;
|
|
39
39
|
// "true" for ONE run after a real stock drop the guard blocked
|
|
40
40
|
feedGuardAcceptDrop?: string;
|
|
41
|
+
// meinestadt feed only (ADV-325/326), Cronicle text params; see configs/meinstadt.config.ts.
|
|
42
|
+
// Live output needs meinstadtDryRun=false AND a cpc explicitly.
|
|
43
|
+
meinstadtDryRun?: string;
|
|
44
|
+
meinstadtImageConcurrency?: string;
|
|
45
|
+
meinstadtMaxImages?: string;
|
|
46
|
+
// AutoUncle feed only (ADV-322/323), Cronicle text params; see configs/autouncle.config.ts.
|
|
47
|
+
// Live output needs autouncleDryRun=false explicitly.
|
|
48
|
+
autouncleDryRun?: string;
|
|
49
|
+
autouncleImageConcurrency?: string;
|
|
50
|
+
autouncleMaxImages?: string;
|
|
41
51
|
}
|
|
42
52
|
|
|
43
53
|
// Parsed from feedItemsBq* by GoogleConfig.
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { promises as fs } from 'fs';
|
|
2
|
+
import { dirname, join } from 'path';
|
|
3
|
+
import { ImboImageResponseInterface } from '@bytebrand/chunkable';
|
|
4
|
+
import { DataSchemes } from '@bytebrand/car-schema';
|
|
5
|
+
|
|
6
|
+
import { AutouncleJsonBuilder } from '../formats/autouncle-json.builder';
|
|
7
|
+
import { AutouncleAddress, AutouncleExclusion, getAutouncleExclusion, mapAutouncleItem } from '../transformers/autouncle.transformer';
|
|
8
|
+
import { FeedGuard, FeedGuardError, LastGood, inspectSourceSnapshot } from './feed-guard';
|
|
9
|
+
|
|
10
|
+
export interface AutouncleSourceCar {
|
|
11
|
+
car: DataSchemes.Car;
|
|
12
|
+
images: ImboImageResponseInterface[];
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// Everything the run needs from the outside world — the Nest exporter wires the real services,
|
|
16
|
+
// tests pass fakes.
|
|
17
|
+
export interface AutouncleRunDeps {
|
|
18
|
+
// iterates the base export snapshot chunk by chunk (LoaderService.process)
|
|
19
|
+
forEachChunk: (fn: (cars: AutouncleSourceCar[]) => Promise<void>) => Promise<unknown>;
|
|
20
|
+
// existing shared partner filter (images count, Cronicle rules incl. the dealer allow-list)
|
|
21
|
+
filterCar: (source: AutouncleSourceCar) => boolean;
|
|
22
|
+
// dealer address of the car's owner; null = owner is not a dealer / not found
|
|
23
|
+
dealerAddress: (ownerId: string) => Promise<AutouncleAddress | null>;
|
|
24
|
+
// final validated image URLs for the car, hero first; [] = no usable image
|
|
25
|
+
resolveImages: (source: AutouncleSourceCar) => Promise<string[]>;
|
|
26
|
+
imageStats: () => Record<string, number>;
|
|
27
|
+
translate: (key: string) => string;
|
|
28
|
+
guard: FeedGuard;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface AutouncleRunOptions {
|
|
32
|
+
runId?: string;
|
|
33
|
+
dryRun: boolean;
|
|
34
|
+
domain: string;
|
|
35
|
+
sourceDirectory: string;
|
|
36
|
+
sourceBaseFileName: string;
|
|
37
|
+
jsonFile: string;
|
|
38
|
+
stateDir: string;
|
|
39
|
+
now?: () => Date;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface AutouncleRunReport {
|
|
43
|
+
runId?: string;
|
|
44
|
+
mode: 'dry-run' | 'live';
|
|
45
|
+
startedAt: string;
|
|
46
|
+
finishedAt?: string;
|
|
47
|
+
status: 'published' | 'blocked' | 'failed';
|
|
48
|
+
guard?: { code: string; message: string };
|
|
49
|
+
error?: string;
|
|
50
|
+
jsonFile: string;
|
|
51
|
+
source: { files: number; newestChunkAt: string | null; cars: number };
|
|
52
|
+
items: number;
|
|
53
|
+
dealers: number;
|
|
54
|
+
excluded: Record<string, number>;
|
|
55
|
+
imagesPerItem: { min: number | null; max: number | null; total: number };
|
|
56
|
+
images: Record<string, number>;
|
|
57
|
+
lastGood: LastGood | null;
|
|
58
|
+
lastGoodAgeHours: number | null;
|
|
59
|
+
stale: boolean;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const increment = (counter: Record<string, number>, key: string) => {
|
|
63
|
+
counter[key] = (counter[key] ?? 0) + 1;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* One AutoUncle feed run (ADV-322/323): source check -> map + validated images -> temp JSON ->
|
|
68
|
+
* count check -> atomic publish -> report. Replaces the old flow that emptied the output directory
|
|
69
|
+
* first, so an outage published an empty or partial file. Nothing outside the output directory and
|
|
70
|
+
* `stateDir` is written; a blocked run leaves the last good JSON in place.
|
|
71
|
+
*/
|
|
72
|
+
export class AutouncleFeedRun {
|
|
73
|
+
constructor(private readonly deps: AutouncleRunDeps, private readonly options: AutouncleRunOptions) { }
|
|
74
|
+
|
|
75
|
+
private now(): Date {
|
|
76
|
+
return this.options.now ? this.options.now() : new Date();
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async run(): Promise<AutouncleRunReport> {
|
|
80
|
+
const { guard } = this.deps;
|
|
81
|
+
const { jsonFile, runId, dryRun } = this.options;
|
|
82
|
+
const startedAt = this.now();
|
|
83
|
+
const lastGood = await guard.readLastGood();
|
|
84
|
+
const report: AutouncleRunReport = {
|
|
85
|
+
runId,
|
|
86
|
+
mode: dryRun ? 'dry-run' : 'live',
|
|
87
|
+
startedAt: startedAt.toISOString(),
|
|
88
|
+
status: 'failed',
|
|
89
|
+
jsonFile,
|
|
90
|
+
source: { files: 0, newestChunkAt: null, cars: 0 },
|
|
91
|
+
items: 0,
|
|
92
|
+
dealers: 0,
|
|
93
|
+
excluded: {},
|
|
94
|
+
imagesPerItem: { min: null, max: null, total: 0 },
|
|
95
|
+
images: {},
|
|
96
|
+
lastGood,
|
|
97
|
+
lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
|
|
98
|
+
stale: guard.isStale(lastGood, startedAt),
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
await guard.acquireLock(runId ?? 'local', startedAt);
|
|
102
|
+
const tmpFile = `${jsonFile}.tmp`;
|
|
103
|
+
try {
|
|
104
|
+
const snapshot = await inspectSourceSnapshot(this.options.sourceDirectory, this.options.sourceBaseFileName);
|
|
105
|
+
report.source.files = snapshot.files;
|
|
106
|
+
report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
|
|
107
|
+
guard.checkSource(snapshot, startedAt);
|
|
108
|
+
|
|
109
|
+
await fs.mkdir(dirname(jsonFile), { recursive: true });
|
|
110
|
+
// the builder appends — never onto a leftover temp file
|
|
111
|
+
await fs.rm(tmpFile, { force: true });
|
|
112
|
+
const builder = new AutouncleJsonBuilder(tmpFile);
|
|
113
|
+
await builder.buildStart();
|
|
114
|
+
|
|
115
|
+
const seen = new Set<string>();
|
|
116
|
+
const dealers = new Set<string>();
|
|
117
|
+
let first = true;
|
|
118
|
+
await this.deps.forEachChunk(async (cars) => {
|
|
119
|
+
for (const source of cars) {
|
|
120
|
+
report.source.cars += 1;
|
|
121
|
+
const id = String(source?.car?._id ?? '');
|
|
122
|
+
if (id && seen.has(id)) {
|
|
123
|
+
increment(report.excluded, AutouncleExclusion.duplicateId);
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
seen.add(id);
|
|
127
|
+
if (!this.deps.filterCar(source)) {
|
|
128
|
+
increment(report.excluded, AutouncleExclusion.filtered);
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
// cheap checks first: no image probe for a car that is excluded anyway
|
|
132
|
+
const excluded = getAutouncleExclusion(source.car);
|
|
133
|
+
if (excluded) {
|
|
134
|
+
increment(report.excluded, excluded);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
const ownerId = String(source.car.metaData.ownerId).trim();
|
|
138
|
+
const address = await this.deps.dealerAddress(ownerId);
|
|
139
|
+
if (!address) {
|
|
140
|
+
increment(report.excluded, AutouncleExclusion.noDealer);
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
const images = await this.deps.resolveImages(source);
|
|
144
|
+
const mapped = mapAutouncleItem(source.car, { domain: this.options.domain, address, images, translate: this.deps.translate });
|
|
145
|
+
if (mapped.ok === false) {
|
|
146
|
+
increment(report.excluded, mapped.reason);
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
await builder.addItem(mapped.item, first);
|
|
150
|
+
first = false;
|
|
151
|
+
dealers.add(ownerId);
|
|
152
|
+
this.count(report, mapped.item.images.length);
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
await builder.buildEnd();
|
|
156
|
+
report.dealers = dealers.size;
|
|
157
|
+
report.images = this.deps.imageStats();
|
|
158
|
+
|
|
159
|
+
const counts = { sourceItems: report.source.cars, publishedItems: report.items };
|
|
160
|
+
guard.checkCounts(counts, lastGood);
|
|
161
|
+
|
|
162
|
+
report.lastGood = await guard.publish(tmpFile, jsonFile, counts, this.now(), runId);
|
|
163
|
+
report.status = 'published';
|
|
164
|
+
report.lastGoodAgeHours = 0;
|
|
165
|
+
report.stale = false;
|
|
166
|
+
return report;
|
|
167
|
+
} catch (error) {
|
|
168
|
+
if (error instanceof FeedGuardError) {
|
|
169
|
+
report.status = 'blocked';
|
|
170
|
+
report.guard = { code: error.code, message: error.message };
|
|
171
|
+
} else {
|
|
172
|
+
report.error = error?.message ?? String(error);
|
|
173
|
+
}
|
|
174
|
+
// keep the rejected file for inspection; the published file stays untouched
|
|
175
|
+
await fs.rename(tmpFile, `${jsonFile}.rejected`).catch(() => undefined);
|
|
176
|
+
throw error;
|
|
177
|
+
} finally {
|
|
178
|
+
await this.writeReport(report);
|
|
179
|
+
await guard.releaseLock();
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
private count(report: AutouncleRunReport, images: number): void {
|
|
184
|
+
report.items += 1;
|
|
185
|
+
const stats = report.imagesPerItem;
|
|
186
|
+
stats.min = stats.min === null ? images : Math.min(stats.min, images);
|
|
187
|
+
stats.max = stats.max === null ? images : Math.max(stats.max, images);
|
|
188
|
+
stats.total += images;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private async writeReport(report: AutouncleRunReport): Promise<void> {
|
|
192
|
+
report.finishedAt = this.now().toISOString();
|
|
193
|
+
await fs.mkdir(this.options.stateDir, { recursive: true });
|
|
194
|
+
await fs.writeFile(join(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
|
|
195
|
+
console.log(`autouncle feed report: ${JSON.stringify(report)}`);
|
|
196
|
+
}
|
|
197
|
+
}
|