@bytebrand/export-cars 1.26.44 → 1.26.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/configs/marktde.config.d.ts +14 -0
- package/dist/configs/marktde.config.js +35 -2
- package/dist/configs/marktde.config.js.map +1 -1
- package/dist/constants/fb.constants.js +1 -1
- package/dist/constants/fb.constants.js.map +1 -1
- package/dist/export.schema.d.ts +2 -2
- package/dist/export.schema.js +1 -1
- package/dist/export.schema.js.map +1 -1
- package/dist/exporters/index.d.ts +1 -0
- package/dist/exporters/index.js +1 -0
- package/dist/exporters/index.js.map +1 -1
- package/dist/exporters/marktde.exporter.d.ts +13 -0
- package/dist/exporters/marktde.exporter.js +84 -0
- package/dist/exporters/marktde.exporter.js.map +1 -0
- package/dist/interfaces/params.interface.d.ts +3 -0
- package/dist/services/marktde-feed-run.d.ts +87 -0
- package/dist/services/marktde-feed-run.js +166 -0
- package/dist/services/marktde-feed-run.js.map +1 -0
- package/dist/transformers/marktde.transformer.d.ts +58 -0
- package/dist/transformers/marktde.transformer.js +158 -0
- package/dist/transformers/marktde.transformer.js.map +1 -0
- package/dist/transformers/mobile.transformer.js +1 -10
- package/dist/transformers/mobile.transformer.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/marktde/AUDIT.md +84 -0
- package/docs/marktde/PARTNER-QUESTIONS.md +49 -0
- package/docs/marktde/RUNBOOK.md +58 -0
- package/docs/marktde/TRACKING.md +69 -0
- package/package.json +2 -2
- package/src/configs/marktde.config.ts +54 -2
- package/src/constants/fb.constants.ts +34 -34
- package/src/export.schema.ts +2 -1
- package/src/exporters/index.ts +1 -0
- package/src/exporters/marktde.exporter.ts +78 -0
- package/src/interfaces/params.interface.ts +5 -0
- package/src/services/marktde-feed-run.ts +233 -0
- package/src/transformers/marktde.transformer.ts +255 -0
- package/src/transformers/mobile.transformer.ts +4 -13
- package/test/marktde-feed-run.test.ts +263 -0
- package/test/marktde-mapping.test.ts +280 -0
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { promises as fs } from 'fs';
|
|
2
|
+
import { dirname, join } from 'path';
|
|
3
|
+
import { ImboImageResponseInterface } from '@bytebrand/chunkable';
|
|
4
|
+
import { DataSchemes } from '@bytebrand/car-schema';
|
|
5
|
+
|
|
6
|
+
import { convertToCSV } from '../utils/converter.utils';
|
|
7
|
+
import { MarktdeExclusion, applyMarktdeColumns, getMarktdeExclusion } from '../transformers/marktde.transformer';
|
|
8
|
+
import { FeedGuard, FeedGuardError, LastGood, inspectSourceSnapshot } from './feed-guard';
|
|
9
|
+
|
|
10
|
+
export interface MarktdeSourceCar {
|
|
11
|
+
car: DataSchemes.Car;
|
|
12
|
+
images: ImboImageResponseInterface[];
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// Everything the run needs from the outside world — the Nest exporter wires the real services,
|
|
16
|
+
// tests pass fakes.
|
|
17
|
+
export interface MarktdeRunDeps {
|
|
18
|
+
// iterates the base export snapshot chunk by chunk (LoaderService.process)
|
|
19
|
+
forEachChunk: (fn: (cars: MarktdeSourceCar[]) => Promise<void>) => Promise<unknown>;
|
|
20
|
+
// existing shared partner filter (images count, Cronicle rules incl. the dealer filter)
|
|
21
|
+
filterCar: (source: MarktdeSourceCar) => boolean;
|
|
22
|
+
// final validated image URLs for the car, hero first; [] = no usable image
|
|
23
|
+
resolveImages: (source: MarktdeSourceCar) => Promise<string[]>;
|
|
24
|
+
// the legacy mobile.de-style row (MobileTransformer + mapping sheet), keyed by column index
|
|
25
|
+
legacyRow: (source: MarktdeSourceCar) => Promise<Record<string, unknown>>;
|
|
26
|
+
imageStats: () => Record<string, number>;
|
|
27
|
+
// packs csvFile into zipFile (entry name = basename of csvFile)
|
|
28
|
+
zip: (csvFile: string, zipFile: string) => Promise<void>;
|
|
29
|
+
guard: FeedGuard;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface MarktdeRunOptions {
|
|
33
|
+
runId?: string;
|
|
34
|
+
dryRun: boolean;
|
|
35
|
+
domain: string;
|
|
36
|
+
sourceDirectory: string;
|
|
37
|
+
sourceBaseFileName: string;
|
|
38
|
+
csvFile: string;
|
|
39
|
+
zipFile: string;
|
|
40
|
+
stateDir: string;
|
|
41
|
+
now?: () => Date;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface MarktdeRunReport {
|
|
45
|
+
runId?: string;
|
|
46
|
+
mode: 'dry-run' | 'live';
|
|
47
|
+
startedAt: string;
|
|
48
|
+
finishedAt?: string;
|
|
49
|
+
status: 'published' | 'blocked' | 'failed';
|
|
50
|
+
guard?: { code: string; message: string };
|
|
51
|
+
error?: string;
|
|
52
|
+
csvFile: string;
|
|
53
|
+
zipFile: string;
|
|
54
|
+
source: { files: number; newestChunkAt: string | null; cars: number };
|
|
55
|
+
items: number;
|
|
56
|
+
dealers: number;
|
|
57
|
+
excluded: Record<string, number>;
|
|
58
|
+
vin: Record<string, number>;
|
|
59
|
+
vat: { deductible: number; notDeductible: number; rateDefaulted: number };
|
|
60
|
+
envkv: { fuelEmpty: number; co2Empty: number };
|
|
61
|
+
description: { items: number; sanitized: number; removed: { url: number; email: number; phone: number; address: number } };
|
|
62
|
+
imagesPerItem: { min: number | null; max: number | null; total: number };
|
|
63
|
+
images: Record<string, number>;
|
|
64
|
+
lastGood: LastGood | null;
|
|
65
|
+
lastGoodAgeHours: number | null;
|
|
66
|
+
stale: boolean;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const increment = (counter: Record<string, number>, key: string) => {
|
|
70
|
+
counter[key] = (counter[key] ?? 0) + 1;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* One markt.de feed run (ADV-328): source check -> legacy row + markt.de columns + validated images ->
|
|
75
|
+
* temp CSV -> count check -> atomic publish of the CSV -> ZIP of the published CSV -> report.
|
|
76
|
+
* Replaces the old MobileExporter flow for markt.de, which emptied the output directory first (an
|
|
77
|
+
* outage published an empty or partial file) and needed the `zip` binary the DEV host lacks.
|
|
78
|
+
* Nothing outside the output directory and `stateDir` is written; a blocked run leaves the last good
|
|
79
|
+
* CSV/ZIP in place.
|
|
80
|
+
*/
|
|
81
|
+
export class MarktdeFeedRun {
|
|
82
|
+
constructor(private readonly deps: MarktdeRunDeps, private readonly options: MarktdeRunOptions) { }
|
|
83
|
+
|
|
84
|
+
private now(): Date {
|
|
85
|
+
return this.options.now ? this.options.now() : new Date();
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async run(): Promise<MarktdeRunReport> {
|
|
89
|
+
const { guard } = this.deps;
|
|
90
|
+
const { csvFile, zipFile, runId, dryRun } = this.options;
|
|
91
|
+
const startedAt = this.now();
|
|
92
|
+
const lastGood = await guard.readLastGood();
|
|
93
|
+
const report: MarktdeRunReport = {
|
|
94
|
+
runId,
|
|
95
|
+
mode: dryRun ? 'dry-run' : 'live',
|
|
96
|
+
startedAt: startedAt.toISOString(),
|
|
97
|
+
status: 'failed',
|
|
98
|
+
csvFile,
|
|
99
|
+
zipFile,
|
|
100
|
+
source: { files: 0, newestChunkAt: null, cars: 0 },
|
|
101
|
+
items: 0,
|
|
102
|
+
dealers: 0,
|
|
103
|
+
excluded: {},
|
|
104
|
+
vin: {},
|
|
105
|
+
vat: { deductible: 0, notDeductible: 0, rateDefaulted: 0 },
|
|
106
|
+
envkv: { fuelEmpty: 0, co2Empty: 0 },
|
|
107
|
+
description: { items: 0, sanitized: 0, removed: { url: 0, email: 0, phone: 0, address: 0 } },
|
|
108
|
+
imagesPerItem: { min: null, max: null, total: 0 },
|
|
109
|
+
images: {},
|
|
110
|
+
lastGood,
|
|
111
|
+
lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
|
|
112
|
+
stale: guard.isStale(lastGood, startedAt),
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
await guard.acquireLock(runId ?? 'local', startedAt);
|
|
116
|
+
const tmpFile = `${csvFile}.tmp`;
|
|
117
|
+
try {
|
|
118
|
+
const snapshot = await inspectSourceSnapshot(this.options.sourceDirectory, this.options.sourceBaseFileName);
|
|
119
|
+
report.source.files = snapshot.files;
|
|
120
|
+
report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
|
|
121
|
+
guard.checkSource(snapshot, startedAt);
|
|
122
|
+
|
|
123
|
+
await fs.mkdir(dirname(csvFile), { recursive: true });
|
|
124
|
+
await fs.rm(tmpFile, { force: true });
|
|
125
|
+
const out = await fs.open(tmpFile, 'w');
|
|
126
|
+
const seen = new Set<string>();
|
|
127
|
+
const dealers = new Set<string>();
|
|
128
|
+
try {
|
|
129
|
+
await this.deps.forEachChunk(async (cars) => {
|
|
130
|
+
for (const source of cars) {
|
|
131
|
+
report.source.cars += 1;
|
|
132
|
+
const id = String(source?.car?._id ?? '');
|
|
133
|
+
if (id && seen.has(id)) {
|
|
134
|
+
increment(report.excluded, MarktdeExclusion.duplicateId);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
seen.add(id);
|
|
138
|
+
if (!this.deps.filterCar(source)) {
|
|
139
|
+
increment(report.excluded, MarktdeExclusion.filtered);
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
// cheap checks first: no image probe for a car that is excluded anyway
|
|
143
|
+
const excluded = getMarktdeExclusion(source.car);
|
|
144
|
+
if (excluded) {
|
|
145
|
+
increment(report.excluded, excluded);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
const images = await this.deps.resolveImages(source);
|
|
149
|
+
if (!images.length) {
|
|
150
|
+
increment(report.excluded, MarktdeExclusion.noImage);
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
const row = await this.deps.legacyRow(source);
|
|
154
|
+
const facts = applyMarktdeColumns(row, source.car, { domain: this.options.domain, images });
|
|
155
|
+
await out.write(`${convertToCSV(row)}\n`, null, 'utf8');
|
|
156
|
+
dealers.add(String(source.car?.metaData?.ownerId ?? ''));
|
|
157
|
+
this.count(report, images.length, facts);
|
|
158
|
+
}
|
|
159
|
+
});
|
|
160
|
+
} finally {
|
|
161
|
+
await out.close();
|
|
162
|
+
}
|
|
163
|
+
report.dealers = dealers.size;
|
|
164
|
+
report.images = this.deps.imageStats();
|
|
165
|
+
|
|
166
|
+
const counts = { sourceItems: report.source.cars, publishedItems: report.items };
|
|
167
|
+
guard.checkCounts(counts, lastGood);
|
|
168
|
+
|
|
169
|
+
report.lastGood = await guard.publish(tmpFile, csvFile, counts, this.now(), runId);
|
|
170
|
+
const zipTmp = zipFile.replace(/\.zip$/, '.tmp.zip');
|
|
171
|
+
await fs.rm(zipTmp, { force: true });
|
|
172
|
+
await this.deps.zip(csvFile, zipTmp);
|
|
173
|
+
await fs.rename(zipTmp, zipFile);
|
|
174
|
+
report.status = 'published';
|
|
175
|
+
report.lastGoodAgeHours = 0;
|
|
176
|
+
report.stale = false;
|
|
177
|
+
return report;
|
|
178
|
+
} catch (error) {
|
|
179
|
+
if (error instanceof FeedGuardError) {
|
|
180
|
+
report.status = 'blocked';
|
|
181
|
+
report.guard = { code: error.code, message: error.message };
|
|
182
|
+
} else {
|
|
183
|
+
report.error = error?.message ?? String(error);
|
|
184
|
+
}
|
|
185
|
+
// keep the rejected file for inspection; the published files stay untouched
|
|
186
|
+
await fs.rename(tmpFile, `${csvFile}.rejected`).catch(() => undefined);
|
|
187
|
+
throw error;
|
|
188
|
+
} finally {
|
|
189
|
+
await this.writeReport(report);
|
|
190
|
+
await guard.releaseLock();
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
private count(report: MarktdeRunReport, images: number, facts: ReturnType<typeof applyMarktdeColumns>): void {
|
|
195
|
+
report.items += 1;
|
|
196
|
+
increment(report.vin, facts.vin);
|
|
197
|
+
if (facts.price.deductible) {
|
|
198
|
+
report.vat.deductible += 1;
|
|
199
|
+
} else {
|
|
200
|
+
report.vat.notDeductible += 1;
|
|
201
|
+
}
|
|
202
|
+
if (facts.price.rateDefaulted) {
|
|
203
|
+
report.vat.rateDefaulted += 1;
|
|
204
|
+
}
|
|
205
|
+
if (facts.fuelEmpty) {
|
|
206
|
+
report.envkv.fuelEmpty += 1;
|
|
207
|
+
}
|
|
208
|
+
if (facts.co2Empty) {
|
|
209
|
+
report.envkv.co2Empty += 1;
|
|
210
|
+
}
|
|
211
|
+
if (facts.descriptionRemovals) {
|
|
212
|
+
report.description.items += 1;
|
|
213
|
+
const r = facts.descriptionRemovals;
|
|
214
|
+
if (r.url + r.email + r.phone + r.address > 0) {
|
|
215
|
+
report.description.sanitized += 1;
|
|
216
|
+
}
|
|
217
|
+
for (const key of Object.keys(r) as Array<keyof typeof r>) {
|
|
218
|
+
report.description.removed[key] += r[key];
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
const stats = report.imagesPerItem;
|
|
222
|
+
stats.min = stats.min === null ? images : Math.min(stats.min, images);
|
|
223
|
+
stats.max = stats.max === null ? images : Math.max(stats.max, images);
|
|
224
|
+
stats.total += images;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
private async writeReport(report: MarktdeRunReport): Promise<void> {
|
|
228
|
+
report.finishedAt = this.now().toISOString();
|
|
229
|
+
await fs.mkdir(this.options.stateDir, { recursive: true });
|
|
230
|
+
await fs.writeFile(join(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
|
|
231
|
+
console.log(`marktde feed report: ${JSON.stringify(report)}`);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import { get, toNumber, toString } from 'lodash';
|
|
2
|
+
import { DataSchemes } from '@bytebrand/car-schema';
|
|
3
|
+
|
|
4
|
+
import { OpenAiImageSelection } from './openai-image-fields';
|
|
5
|
+
import { envkvCo2Value, envkvFuelConsumptionValue } from './google-feed-envkv';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* markt.de feed (ADV-328) on top of the legacy mobile.de-style CSV row that MobileTransformer builds
|
|
9
|
+
* from the `mobile_yescar` mapping sheet (ACF-4370). Only the columns below are replaced; every other
|
|
10
|
+
* column, the column order and the row format stay as they were. Pure functions, no I/O.
|
|
11
|
+
*
|
|
12
|
+
* Column indexes are 0-based (the sheet's importField): 1 = B, 10 = K, ... 402 = OM.
|
|
13
|
+
*/
|
|
14
|
+
export const MARKTDE_COLUMNS = {
|
|
15
|
+
vehicleId: '1', // B stable auto.de vehicle id (metaData.vehicleId)
|
|
16
|
+
price: '10', // K price (net for VAT-deductible cars, see getMarktdePriceColumns)
|
|
17
|
+
vatFlag: '11', // L 0 = VAT is added on top of K, 1 = no VAT shown separately
|
|
18
|
+
vin: '14', // O VIN (optional)
|
|
19
|
+
description: '25', // Z description (mobile.de markup)
|
|
20
|
+
images: '26', // AA image URLs, `;` separated
|
|
21
|
+
vatRate: '29', // AD VAT rate
|
|
22
|
+
fuelCombined: '98', // CU combined fuel consumption, decimal comma
|
|
23
|
+
co2: '99', // CV CO2 g/km
|
|
24
|
+
listingUrl: '402', // OM clickout link to the auto.de vehicle page
|
|
25
|
+
} as const;
|
|
26
|
+
|
|
27
|
+
// Clickout parameters (ADV-330, same scheme as AutoUncle ADV-324 / meinestadt ADV-327): the partner in
|
|
28
|
+
// `source`, so the reporting views, the CRM source and GA4 (utm_*) all see "marktde". Legacy and utm
|
|
29
|
+
// pairs carry identical values; paid / free is a contract attribute and is not encoded in the medium.
|
|
30
|
+
// Pending markt.de's OK that a structured clickout field is allowed at all.
|
|
31
|
+
export const MARKTDE_LINK_PARAMS =
|
|
32
|
+
'source=marktde&medium=listing&campaign=listing&utm_source=marktde&utm_medium=listing&utm_campaign=listing';
|
|
33
|
+
|
|
34
|
+
export const getMarktdeListingUrl = (domain: string, carId: string): string =>
|
|
35
|
+
`https://www.${domain}/search/vehicle/${carId}?${MARKTDE_LINK_PARAMS}`;
|
|
36
|
+
|
|
37
|
+
export enum MarktdeExclusion {
|
|
38
|
+
duplicateId = 'duplicate_id',
|
|
39
|
+
filtered = 'filtered',
|
|
40
|
+
noId = 'no_id',
|
|
41
|
+
noVehicleId = 'no_vehicle_id',
|
|
42
|
+
notAvailable = 'reserved_or_sold',
|
|
43
|
+
noPrice = 'no_valid_price',
|
|
44
|
+
noImage = 'no_valid_image',
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Online cars flagged like this are not sellable stock; the partner gets the file without them.
|
|
48
|
+
const NOT_AVAILABLE = ['selector_offerAvailability_reserved', 'selector_offerAvailability_sold'];
|
|
49
|
+
|
|
50
|
+
const positiveAmount = (value: unknown): number | null => {
|
|
51
|
+
if (value === null || value === undefined || value === '' || typeof value === 'boolean') {
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
const amount = toNumber(value);
|
|
55
|
+
return Number.isFinite(amount) && amount > 0 ? amount : null;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
// Cheap checks before any image probe. null = the car can be exported.
|
|
59
|
+
export const getMarktdeExclusion = (car: DataSchemes.Car): MarktdeExclusion | null => {
|
|
60
|
+
if (!toString(get(car, '_id', '')).trim()) {
|
|
61
|
+
return MarktdeExclusion.noId;
|
|
62
|
+
}
|
|
63
|
+
if (!toString(get(car, 'metaData.vehicleId', '')).trim()) {
|
|
64
|
+
return MarktdeExclusion.noVehicleId;
|
|
65
|
+
}
|
|
66
|
+
if (NOT_AVAILABLE.includes(toString(get(car, 'offer.offerAvailability', '')))) {
|
|
67
|
+
return MarktdeExclusion.notAvailable;
|
|
68
|
+
}
|
|
69
|
+
if (positiveAmount(get(car, 'price.currentSalesPrice', null)) === null) {
|
|
70
|
+
return MarktdeExclusion.noPrice;
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
// ISO 3779: 17 characters, digits and capital letters without I, O, Q.
|
|
76
|
+
const VIN_PATTERN = /^[A-HJ-NPR-Z0-9]{17}$/;
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* The car's own VIN when it is a well-formed 17-character VIN, otherwise empty. The legacy exporter
|
|
80
|
+
* sent marketing.fakeVIN or a random generated VIN for markt.de when the VIN was missing — never again.
|
|
81
|
+
*/
|
|
82
|
+
export const getMarktdeVin = (car: DataSchemes.Car): string => {
|
|
83
|
+
const vin = toString(get(car, 'metaData.vin', '')).trim().toUpperCase();
|
|
84
|
+
return VIN_PATTERN.test(vin) ? vin : '';
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
export interface MarktdePriceColumns {
|
|
88
|
+
price: string;
|
|
89
|
+
vatFlag: string;
|
|
90
|
+
vatRate: string;
|
|
91
|
+
deductible: boolean;
|
|
92
|
+
// vatRate was missing / 0 on a VAT-deductible car and 19 was used
|
|
93
|
+
rateDefaulted: boolean;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* K / L / AD with the semantics of the legacy feed (ACF-4370): price.currentSalesPrice is the gross
|
|
98
|
+
* consumer price. VAT-deductible car: K = net, L = 0, AD = rate, so gross = K * (1 + AD/100) again.
|
|
99
|
+
* Otherwise (margin scheme / private / exempt): K = the gross price as it is, L = 1, AD = 0 — no VAT
|
|
100
|
+
* is ever added on top of an already gross price. Unlike the legacy code the net price and AD always
|
|
101
|
+
* use the same rate (legacy: a null vatRate left K gross while AD said 19, i.e. +19 % on display).
|
|
102
|
+
*/
|
|
103
|
+
export const getMarktdePriceColumns = (car: DataSchemes.Car): MarktdePriceColumns | null => {
|
|
104
|
+
const raw = get(car, 'price.currentSalesPrice', null);
|
|
105
|
+
const gross = positiveAmount(raw);
|
|
106
|
+
if (gross === null) {
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
const deductible = Boolean(get(car, 'price.vatDeductible', false));
|
|
110
|
+
if (!deductible) {
|
|
111
|
+
return { price: String(raw), vatFlag: '1', vatRate: '0', deductible, rateDefaulted: false };
|
|
112
|
+
}
|
|
113
|
+
const storedRate = positiveAmount(get(car, 'price.vatRate', null));
|
|
114
|
+
const rate = storedRate ?? 19;
|
|
115
|
+
const net = (gross * 100) / (100 + rate);
|
|
116
|
+
return { price: net.toFixed(2), vatFlag: '0', vatRate: String(rate), deductible, rateDefaulted: storedRate === null };
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* The markt.de image list from the shared validated clean selection (ADV-336, same rules as the
|
|
121
|
+
* OpenAI / meinestadt / AutoUncle feeds): valid 3/4 hero first, then every other valid image. Real
|
|
122
|
+
* photos only — no marketing overlays, no placeholders. No valid hero => [] (car is skipped).
|
|
123
|
+
*/
|
|
124
|
+
export const toMarktdeImages = (selection: OpenAiImageSelection, maxImages = 0): string[] => {
|
|
125
|
+
if (!selection?.fields) {
|
|
126
|
+
return [];
|
|
127
|
+
}
|
|
128
|
+
const urls = [selection.fields.image_url, ...selection.fields.additional_image_urls];
|
|
129
|
+
return maxImages > 0 ? urls.slice(0, maxImages) : urls;
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
export interface DescriptionRemovals {
|
|
133
|
+
url: number;
|
|
134
|
+
email: number;
|
|
135
|
+
phone: number;
|
|
136
|
+
address: number;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// mobile.de description markup: `\\` is a line break, `**…**` bold. Removed tokens never eat it.
|
|
140
|
+
const URL_PATTERN = /\b(?:https?:\/\/|www\.)[^\s\\"'<>]+/gi;
|
|
141
|
+
const EMAIL_PATTERN = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
|
|
142
|
+
// Bare domains of other websites ("autohaus-muster.de"). Not the seller's own name: auto.de is the
|
|
143
|
+
// brand and the legal entity (auto.de GmbH, the commercial identification the terms require), so
|
|
144
|
+
// "auto.de", "Auto.de", "auto.de-ID" stay; only a link to it (scheme / www.) goes.
|
|
145
|
+
const DOMAIN_PATTERN = /(?<![\w.-])(?!auto\.de\b)[a-z0-9][a-z0-9-]*(?:\.[a-z0-9-]+)*\.(?:de|com|net|org|eu|info|at|ch|biz|shop|online)\b(?:\/[^\s\\"'<>]*)?/gi;
|
|
146
|
+
// Phone candidates: start with +, 00, 0 or (0, then digits and the usual separators; label optional.
|
|
147
|
+
const PHONE_PATTERN = /(?:\b(?:Tel(?:efon|efax)?|Fon|Fax|SMS|Mobil(?:funk)?|Handy|WhatsApp|Hotline)\.?\s*:?\s*)?(?:\+|\b00|\(0|\b0)[\d \t/().-]{6,}\d/gi;
|
|
148
|
+
const DATE_PATTERN = /^\d{1,2}\.\d{1,2}\.\d{2,4}$/;
|
|
149
|
+
// Street + house number ("Max-Planck-Straße 19", "Hauptstr. 5a") and postal code + town ("06796 Brehna").
|
|
150
|
+
const STREET_PATTERN = /\b[A-ZÄÖÜ][\wäöüß.-]*(?:[Ss]traße|[Ss]trasse|[Ss]tr\.|[Ww]eg|[Aa]llee|[Pp]latz|[Rr]ing|[Gg]asse|[Dd]amm|[Cc]haussee|[Uu]fer)\s*\d{1,4}(?![.,]?\d)\s?[a-z]?\b(?![.,]\d)/g;
|
|
151
|
+
const POSTCODE_TOWN_PATTERN = /\b\d{5}\s+(?!(?:Euro|EUR|Kilometer|Km|KM|km|Stück|Kilo)\b)[A-ZÄÖÜ][a-zäöüß]+(?:[ -][A-ZÄÖÜ][a-zäöüß]+)?/g;
|
|
152
|
+
// Marks a removed token until the tidy-up; never left in the output.
|
|
153
|
+
const CUT = '\u0000';
|
|
154
|
+
// Labels left behind right before a removed value ("Tel.: ", "E-Mail:").
|
|
155
|
+
const DANGLING_LABEL = /\b(?:Tel(?:efon|efax)?|Fon|Fax|SMS|Mobil(?:funk)?|Handy|WhatsApp|Hotline|E-?Mail|Mail|Web(?:seite)?|Internet|Homepage|Adresse|Anschrift)\.?[ \t]*:?[ \t]*(?=\u0000)/gi;
|
|
156
|
+
|
|
157
|
+
const digitCount = (value: string): number => (value.match(/\d/g) || []).length;
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* markt.de's terms (nutzungsbedingungen, checked 08.10.2026) forbid phone numbers, physical addresses,
|
|
161
|
+
* website links and QR codes in the listing description. The dealer's free-text header / footer of the
|
|
162
|
+
* mobile.de description often carries exactly that, so it is removed from the text; the vehicle facts
|
|
163
|
+
* stay. Heuristic by nature: the run report counts every removal for review.
|
|
164
|
+
*/
|
|
165
|
+
export const sanitizeMarktdeDescription = (text: string): { text: string; removed: DescriptionRemovals } => {
|
|
166
|
+
const removed: DescriptionRemovals = { url: 0, email: 0, phone: 0, address: 0 };
|
|
167
|
+
if (!text) {
|
|
168
|
+
return { text, removed };
|
|
169
|
+
}
|
|
170
|
+
let out = text
|
|
171
|
+
.replace(EMAIL_PATTERN, () => { removed.email += 1; return CUT; })
|
|
172
|
+
.replace(URL_PATTERN, () => { removed.url += 1; return CUT; })
|
|
173
|
+
.replace(DOMAIN_PATTERN, () => { removed.url += 1; return CUT; })
|
|
174
|
+
.replace(PHONE_PATTERN, (match) => {
|
|
175
|
+
const digits = digitCount(match);
|
|
176
|
+
if (digits < 8 || digits > 16 || DATE_PATTERN.test(match.trim())) {
|
|
177
|
+
return match;
|
|
178
|
+
}
|
|
179
|
+
removed.phone += 1;
|
|
180
|
+
return CUT;
|
|
181
|
+
})
|
|
182
|
+
.replace(STREET_PATTERN, () => { removed.address += 1; return CUT; })
|
|
183
|
+
.replace(POSTCODE_TOWN_PATTERN, () => { removed.address += 1; return CUT; });
|
|
184
|
+
if (removed.url + removed.email + removed.phone + removed.address === 0) {
|
|
185
|
+
return { text, removed };
|
|
186
|
+
}
|
|
187
|
+
// Tidy only where something was cut (CUT marks the spot): the label that introduced it, the spaces
|
|
188
|
+
// around it; a cut at the start / end of a line leaves no space. The rest of the text is untouched.
|
|
189
|
+
out = out
|
|
190
|
+
.replace(DANGLING_LABEL, '')
|
|
191
|
+
.replace(/[ \t]*(?:\u0000[ \t]*)+/g, '\u0001')
|
|
192
|
+
.replace(/(^|\\\\)\u0001/g, '$1')
|
|
193
|
+
.replace(/\u0001(?=\\\\|$)/g, '')
|
|
194
|
+
.replace(/\u0001/g, ' ');
|
|
195
|
+
return { text: out, removed };
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
export interface MarktdeRowContext {
|
|
199
|
+
domain: string;
|
|
200
|
+
// final, validated image URLs, hero first (non-empty: callers skip cars without images)
|
|
201
|
+
images: string[];
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export interface MarktdeRowFacts {
|
|
205
|
+
vin: 'sent' | 'empty' | 'invalid_dropped';
|
|
206
|
+
price: MarktdePriceColumns;
|
|
207
|
+
descriptionRemovals: DescriptionRemovals | null;
|
|
208
|
+
fuelEmpty: boolean;
|
|
209
|
+
co2Empty: boolean;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const decimalComma = (value: number | undefined): string => (value === undefined || value === null ? '' : String(value).replace('.', ','));
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Replaces the markt.de columns of a MobileTransformer row (object keyed by column index) and returns
|
|
216
|
+
* what was decided, for the run report. The row is changed in place.
|
|
217
|
+
*/
|
|
218
|
+
export const applyMarktdeColumns = (row: Record<string, unknown>, car: DataSchemes.Car, ctx: MarktdeRowContext): MarktdeRowFacts => {
|
|
219
|
+
const c = MARKTDE_COLUMNS;
|
|
220
|
+
const price = getMarktdePriceColumns(car);
|
|
221
|
+
row[c.price] = price.price;
|
|
222
|
+
row[c.vatFlag] = price.vatFlag;
|
|
223
|
+
row[c.vatRate] = price.vatRate;
|
|
224
|
+
|
|
225
|
+
const vin = getMarktdeVin(car);
|
|
226
|
+
const storedVin = toString(get(car, 'metaData.vin', '')).trim();
|
|
227
|
+
row[c.vin] = vin;
|
|
228
|
+
|
|
229
|
+
let descriptionRemovals: DescriptionRemovals | null = null;
|
|
230
|
+
const description = row[c.description];
|
|
231
|
+
if (typeof description === 'string' && description) {
|
|
232
|
+
const sanitized = sanitizeMarktdeDescription(description);
|
|
233
|
+
row[c.description] = sanitized.text;
|
|
234
|
+
descriptionRemovals = sanitized.removed;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
row[c.images] = ctx.images.join(';');
|
|
238
|
+
|
|
239
|
+
// EnVKV: the same values as the Google / Meta / AutoUncle / meinestadt feeds (WLTP first, legacy
|
|
240
|
+
// only as fallback; BEV: no fuel consumption, CO2 0). Nothing usable -> empty (legacy: "undefined").
|
|
241
|
+
const fuel = envkvFuelConsumptionValue(car);
|
|
242
|
+
const co2 = envkvCo2Value(car);
|
|
243
|
+
row[c.fuelCombined] = decimalComma(fuel);
|
|
244
|
+
row[c.co2] = co2 === undefined || co2 === null ? '' : String(co2);
|
|
245
|
+
|
|
246
|
+
row[c.listingUrl] = getMarktdeListingUrl(ctx.domain, toString(get(car, '_id', '')));
|
|
247
|
+
|
|
248
|
+
return {
|
|
249
|
+
vin: vin ? 'sent' : storedVin ? 'invalid_dropped' : 'empty',
|
|
250
|
+
price,
|
|
251
|
+
descriptionRemovals,
|
|
252
|
+
fuelEmpty: row[c.fuelCombined] === '',
|
|
253
|
+
co2Empty: row[c.co2] === '',
|
|
254
|
+
};
|
|
255
|
+
};
|
|
@@ -13,8 +13,6 @@ import { isNotEmptyArray } from '../utils';
|
|
|
13
13
|
import { ConfigService } from '../services';
|
|
14
14
|
import { ExportType } from '../enums';
|
|
15
15
|
|
|
16
|
-
const vinGenerator = require('vin-generator');
|
|
17
|
-
|
|
18
16
|
@Injectable()
|
|
19
17
|
export class MobileTransformer {
|
|
20
18
|
private currentCar: DataSchemes.Car;
|
|
@@ -346,18 +344,11 @@ export class MobileTransformer {
|
|
|
346
344
|
return String(currentSalesPrice);
|
|
347
345
|
}
|
|
348
346
|
|
|
347
|
+
// ADV-328: never a fake VIN. markt.de (the only campaign that had one) used to get
|
|
348
|
+
// marketing.fakeVIN or a random generated VIN when the car had none; marktde.transformer now
|
|
349
|
+
// sends the car's own valid VIN or nothing.
|
|
349
350
|
private getVinCode(car: DataSchemes.Car): string {
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
if (this.configService.config.campaign === 'marktde' && !vin) {
|
|
353
|
-
const fakeVin = get(car, 'marketing.fakeVIN', '');
|
|
354
|
-
if (!isEmpty(fakeVin)) {
|
|
355
|
-
return fakeVin;
|
|
356
|
-
}
|
|
357
|
-
return vinGenerator.generateVin();
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
return vin;
|
|
351
|
+
return get(car, 'metaData.vin', '');
|
|
361
352
|
}
|
|
362
353
|
|
|
363
354
|
private getAutodeUrl(car: DataSchemes.Car): string {
|