@bytebrand/export-cars 1.26.31 → 1.26.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/configs/autouncle.config.d.ts +14 -0
- package/dist/configs/autouncle.config.js +36 -3
- package/dist/configs/autouncle.config.js.map +1 -1
- package/dist/configs/meinstadt.config.d.ts +15 -0
- package/dist/configs/meinstadt.config.js +42 -3
- package/dist/configs/meinstadt.config.js.map +1 -1
- package/dist/constants/autouncle.contants.d.ts +1 -3
- package/dist/constants/autouncle.contants.js +2 -4
- package/dist/constants/autouncle.contants.js.map +1 -1
- package/dist/exporters/autouncle.exporter.d.ts +3 -2
- package/dist/exporters/autouncle.exporter.js +45 -27
- package/dist/exporters/autouncle.exporter.js.map +1 -1
- package/dist/exporters/meinstadt.exporter.d.ts +3 -2
- package/dist/exporters/meinstadt.exporter.js +40 -23
- package/dist/exporters/meinstadt.exporter.js.map +1 -1
- package/dist/formats/autouncle-json.builder.js +1 -4
- package/dist/formats/autouncle-json.builder.js.map +1 -1
- package/dist/interfaces/export-item.interface.d.ts +8 -12
- package/dist/interfaces/params.interface.d.ts +6 -0
- package/dist/services/autouncle-feed-run.d.ts +66 -0
- package/dist/services/autouncle-feed-run.js +135 -0
- package/dist/services/autouncle-feed-run.js.map +1 -0
- package/dist/services/meinstadt-feed-run.d.ts +81 -0
- package/dist/services/meinstadt-feed-run.js +166 -0
- package/dist/services/meinstadt-feed-run.js.map +1 -0
- package/dist/transformers/autouncle.transformer.d.ts +38 -22
- package/dist/transformers/autouncle.transformer.js +156 -144
- package/dist/transformers/autouncle.transformer.js.map +1 -1
- package/dist/transformers/meinstadt.transformer.d.ts +17 -13
- package/dist/transformers/meinstadt.transformer.js +88 -61
- package/dist/transformers/meinstadt.transformer.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/autouncle/AUDIT.md +97 -0
- package/docs/autouncle/PARTNER-QUESTIONS.md +53 -0
- package/docs/autouncle/RUNBOOK.md +64 -0
- package/docs/meinestadt/AUDIT.md +146 -0
- package/docs/meinestadt/PARTNER-QUESTIONS.md +54 -0
- package/docs/meinestadt/RUNBOOK.md +64 -0
- package/package.json +3 -2
- package/src/configs/autouncle.config.ts +51 -3
- package/src/configs/meinstadt.config.ts +57 -2
- package/src/constants/autouncle.contants.ts +2 -3
- package/src/exporters/autouncle.exporter.ts +59 -37
- package/src/exporters/meinstadt.exporter.ts +53 -32
- package/src/formats/autouncle-json.builder.ts +2 -4
- package/src/interfaces/export-item.interface.ts +15 -12
- package/src/interfaces/params.interface.ts +10 -0
- package/src/services/autouncle-feed-run.ts +197 -0
- package/src/services/meinstadt-feed-run.ts +240 -0
- package/src/transformers/autouncle.transformer.ts +217 -175
- package/src/transformers/meinstadt.transformer.ts +114 -68
- package/test/autouncle-feed-run.test.ts +256 -0
- package/test/autouncle-mapping.test.ts +225 -0
- package/test/fixtures/meinstadt.fixture.ts +87 -0
- package/test/meinstadt-feed-run.test.ts +288 -0
- package/test/meinstadt-mapping.test.ts +215 -0
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
// Run: npm test (ts-node, no test framework needed)
|
|
2
|
+
// Integration: real files in a temp dir (base export snapshot, output, state), real JSON writer,
|
|
3
|
+
// guard and ADV-336 clean-image selection; only HTTP (image check), UPA (equipment) and the zip
|
|
4
|
+
// binary are faked.
|
|
5
|
+
import * as assert from 'node:assert/strict';
|
|
6
|
+
import { promises as fs } from 'fs';
|
|
7
|
+
import { tmpdir } from 'os';
|
|
8
|
+
import { join } from 'path';
|
|
9
|
+
|
|
10
|
+
import { FeedGuard, FeedGuardError, FeedGuardOptions } from '../src/services/feed-guard';
|
|
11
|
+
import { MeinstadtConfigError, MeinstadtFeedRun, MeinstadtSourceCar, zipSingleFile } from '../src/services/meinstadt-feed-run';
|
|
12
|
+
import { toMeinstadtImages } from '../src/transformers/meinstadt.transformer';
|
|
13
|
+
import { selectOpenAiImageFields } from '../src/transformers/openai-image-fields';
|
|
14
|
+
import { buildImagesService } from './fixtures/images-service.harness';
|
|
15
|
+
import { NOW, car, images, translate } from './fixtures/meinstadt.fixture';
|
|
16
|
+
|
|
17
|
+
const HOUR = 60 * 60 * 1000;
|
|
18
|
+
const imagesService = buildImagesService();
|
|
19
|
+
|
|
20
|
+
interface World {
|
|
21
|
+
root: string;
|
|
22
|
+
source: string;
|
|
23
|
+
json: string;
|
|
24
|
+
zip: string;
|
|
25
|
+
state: string;
|
|
26
|
+
brokenImages: Set<string>;
|
|
27
|
+
zipCalls: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const newWorld = async (): Promise<World> => {
|
|
31
|
+
const root = await fs.mkdtemp(join(tmpdir(), 'meinstadt-'));
|
|
32
|
+
const out = join(root, 'meinstadt-dryrun');
|
|
33
|
+
return {
|
|
34
|
+
root,
|
|
35
|
+
source: join(root, 'cars-export'),
|
|
36
|
+
json: join(out, 'api_export_cars_meinstadt.json'),
|
|
37
|
+
zip: join(out, 'api_export_cars_meinstadt.zip'),
|
|
38
|
+
state: join(root, 'meinstadt-state', 'dryrun'),
|
|
39
|
+
brokenImages: new Set(),
|
|
40
|
+
zipCalls: 0,
|
|
41
|
+
};
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
const source = (id: string, overrides: Record<string, any> = {}): MeinstadtSourceCar => ({ car: car(id, overrides), images: images(id) });
|
|
45
|
+
|
|
46
|
+
const writeSource = async (world: World, cars: MeinstadtSourceCar[], chunkSize = 2, mtime = NOW) => {
|
|
47
|
+
await fs.rm(world.source, { recursive: true, force: true });
|
|
48
|
+
await fs.mkdir(world.source, { recursive: true });
|
|
49
|
+
for (let i = 0; i * chunkSize < cars.length; i++) {
|
|
50
|
+
const file = join(world.source, `cars.${i}.json`);
|
|
51
|
+
await fs.writeFile(file, JSON.stringify(cars.slice(i * chunkSize, (i + 1) * chunkSize)));
|
|
52
|
+
await fs.utimes(file, mtime, mtime);
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
const guardOptions = (world: World, extra: Partial<FeedGuardOptions> = {}): FeedGuardOptions => ({
|
|
57
|
+
stateDir: world.state,
|
|
58
|
+
minSourceItems: 1,
|
|
59
|
+
minPublishedItems: 1,
|
|
60
|
+
maxDropPercent: 20,
|
|
61
|
+
maxSourceAgeHours: 26,
|
|
62
|
+
maxStaleHours: 48,
|
|
63
|
+
acceptDrop: false,
|
|
64
|
+
...extra,
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
interface RunOptions {
|
|
68
|
+
at?: Date;
|
|
69
|
+
dryRun?: boolean;
|
|
70
|
+
cpc?: string | null;
|
|
71
|
+
guard?: Partial<FeedGuardOptions>;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const run = async (world: World, { at = NOW, dryRun = true, cpc = '0.99', guard = {} }: RunOptions = {}) => {
|
|
75
|
+
const forEachChunk = async (fn: (cars: MeinstadtSourceCar[]) => Promise<void>) => {
|
|
76
|
+
const names = (await fs.readdir(world.source)).sort((a, b) => Number(a.split('.')[1]) - Number(b.split('.')[1]));
|
|
77
|
+
for (const name of names) {
|
|
78
|
+
await fn(JSON.parse(await fs.readFile(join(world.source, name), 'utf8')));
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
const feed = new MeinstadtFeedRun(
|
|
82
|
+
{
|
|
83
|
+
forEachChunk,
|
|
84
|
+
filterCar: ({ car: c }) => (c?.metaData?.imagesCount ?? 0) > 0,
|
|
85
|
+
resolveImages: async ({ car: c, images: list }) => {
|
|
86
|
+
const clean = imagesService.getCleanImages(c._id, list);
|
|
87
|
+
return toMeinstadtImages(selectOpenAiImageFields(clean, clean.map(i => ({ url: i.url, ok: !world.brokenImages.has(i.url) }))));
|
|
88
|
+
},
|
|
89
|
+
imageStats: () => ({ ok: 1 }),
|
|
90
|
+
equipment: async c => `Ausstattung ${c._id}`,
|
|
91
|
+
// fake zip: the test only needs to see which JSON was packed
|
|
92
|
+
zip: async (json, zip) => {
|
|
93
|
+
world.zipCalls += 1;
|
|
94
|
+
await fs.writeFile(zip, `ZIP:${await fs.readFile(json, 'utf8')}`);
|
|
95
|
+
},
|
|
96
|
+
guard: new FeedGuard(guardOptions(world, guard)),
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
runId: 'test',
|
|
100
|
+
dryRun,
|
|
101
|
+
cpc,
|
|
102
|
+
domain: 'auto.de',
|
|
103
|
+
sourceDirectory: world.source,
|
|
104
|
+
sourceBaseFileName: 'cars',
|
|
105
|
+
jsonFile: world.json,
|
|
106
|
+
zipFile: world.zip,
|
|
107
|
+
stateDir: world.state,
|
|
108
|
+
translate,
|
|
109
|
+
now: () => at,
|
|
110
|
+
},
|
|
111
|
+
);
|
|
112
|
+
return feed.run();
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
const readItems = async (world: World) => JSON.parse(await fs.readFile(world.json, 'utf8'));
|
|
116
|
+
const exists = (file: string) => fs.stat(file).then(() => true, () => false);
|
|
117
|
+
|
|
118
|
+
const tests: Array<[string, () => Promise<void>]> = [
|
|
119
|
+
['first run: JSON array + ZIP of the published JSON, exclusions counted, report written', async () => {
|
|
120
|
+
const world = await newWorld();
|
|
121
|
+
const noImages = source('nopic');
|
|
122
|
+
noImages.car.metaData.imagesCount = 0;
|
|
123
|
+
const noHero = source('nohero');
|
|
124
|
+
noHero.images = noHero.images.filter(i => i.imageCodePredicted !== 'ext_f_right');
|
|
125
|
+
await writeSource(world, [
|
|
126
|
+
source('aaa1', { 'mainData.accidentDamaged': false }),
|
|
127
|
+
source('aaa2', { 'mainData.accidentDamaged': true }),
|
|
128
|
+
source('aaa3', { 'mainData.accidentDamaged': null }),
|
|
129
|
+
source('aaa1'),
|
|
130
|
+
noImages,
|
|
131
|
+
noHero,
|
|
132
|
+
]);
|
|
133
|
+
const report = await run(world);
|
|
134
|
+
const items = await readItems(world);
|
|
135
|
+
assert.ok(Array.isArray(items));
|
|
136
|
+
assert.deepEqual(items.map((i: any) => i.listingUrl.split('/vehicle/')[1].split('?')[0]), ['aaa1', 'aaa2', 'aaa3']);
|
|
137
|
+
assert.deepEqual(items.map((i: any) => i.accidentFree), [true, false, null]);
|
|
138
|
+
for (const item of items) {
|
|
139
|
+
assert.ok(Array.isArray(item.images));
|
|
140
|
+
assert.equal(item.imagesCount, item.images.length);
|
|
141
|
+
assert.equal(item.images.length, 4);
|
|
142
|
+
assert.equal(item.cpc, '0.99');
|
|
143
|
+
assert.match(item.images[0], /hero1\/large-cached\.jpeg$/);
|
|
144
|
+
}
|
|
145
|
+
assert.equal(await fs.readFile(world.zip, 'utf8'), `ZIP:${await fs.readFile(world.json, 'utf8')}`);
|
|
146
|
+
assert.equal(report.status, 'published');
|
|
147
|
+
assert.equal(report.items, 3);
|
|
148
|
+
assert.deepEqual(report.excluded, { duplicate_id: 1, filtered: 1, no_valid_image: 1 });
|
|
149
|
+
assert.deepEqual(report.accidentFree, { true: 1, false: 1, null: 1 });
|
|
150
|
+
assert.deepEqual(report.imagesPerItem, { min: 4, max: 4, total: 12 });
|
|
151
|
+
const saved = JSON.parse(await fs.readFile(join(world.state, 'last-run-report.json'), 'utf8'));
|
|
152
|
+
assert.equal(saved.status, 'published');
|
|
153
|
+
assert.equal(await exists(`${world.json}.tmp`), false);
|
|
154
|
+
assert.equal(await exists(join(world.state, 'run.lock')), false);
|
|
155
|
+
}],
|
|
156
|
+
|
|
157
|
+
['JSON file: UTF-8 umlauts, no BOM, parseable, items separated correctly', async () => {
|
|
158
|
+
const world = await newWorld();
|
|
159
|
+
await writeSource(world, [source('u1', { 'mainData.subModel': 'Größe Ü' }), source('u2')]);
|
|
160
|
+
await run(world);
|
|
161
|
+
const bytes = await fs.readFile(world.json);
|
|
162
|
+
assert.notEqual(bytes[0], 0xef);
|
|
163
|
+
assert.ok(bytes.includes(Buffer.from('Größe Ü', 'utf8')));
|
|
164
|
+
assert.equal((await readItems(world)).length, 2);
|
|
165
|
+
}],
|
|
166
|
+
|
|
167
|
+
['broken image URLs are left out and imagesCount follows; broken hero -> car not sent', async () => {
|
|
168
|
+
const world = await newWorld();
|
|
169
|
+
world.brokenImages.add('https://images.auto.de/carimage/b1/b1back1/large-cached.jpeg');
|
|
170
|
+
world.brokenImages.add('https://images.auto.de/carimage/b1/b1int01/large-cached.jpeg');
|
|
171
|
+
world.brokenImages.add('https://images.auto.de/carimage/b2/b2hero1/large-cached.jpeg');
|
|
172
|
+
await writeSource(world, [source('b1'), source('b2')]);
|
|
173
|
+
const report = await run(world);
|
|
174
|
+
const items = await readItems(world);
|
|
175
|
+
assert.equal(items.length, 1);
|
|
176
|
+
assert.equal(items[0].imagesCount, 2);
|
|
177
|
+
assert.deepEqual(items[0].images, [
|
|
178
|
+
'https://images.auto.de/carimage/b1/b1hero1/large-cached.jpeg',
|
|
179
|
+
'https://images.auto.de/carimage/b1/b1side1/large-cached.jpeg',
|
|
180
|
+
]);
|
|
181
|
+
assert.equal(report.excluded.no_valid_image, 1);
|
|
182
|
+
}],
|
|
183
|
+
|
|
184
|
+
['live without cpc: config error before anything is read or written; dry-run without cpc sends null', async () => {
|
|
185
|
+
const world = await newWorld();
|
|
186
|
+
await writeSource(world, [source('c1')]);
|
|
187
|
+
await assert.rejects(run(world, { dryRun: false, cpc: null }), (e: any) => e instanceof MeinstadtConfigError && e.code === 'cpc_missing');
|
|
188
|
+
assert.equal(await exists(world.json), false);
|
|
189
|
+
assert.equal(await exists(world.zip), false);
|
|
190
|
+
assert.equal(JSON.parse(await fs.readFile(join(world.state, 'last-run-report.json'), 'utf8')).error, 'cpc_missing');
|
|
191
|
+
assert.equal(await exists(join(world.state, 'run.lock')), false);
|
|
192
|
+
|
|
193
|
+
const report = await run(world, { dryRun: true, cpc: null });
|
|
194
|
+
assert.equal(report.status, 'published');
|
|
195
|
+
assert.equal((await readItems(world))[0].cpc, null);
|
|
196
|
+
const live = await run(world, { dryRun: false, cpc: '0.99' });
|
|
197
|
+
assert.equal(live.mode, 'live');
|
|
198
|
+
}],
|
|
199
|
+
|
|
200
|
+
['source outage: empty snapshot is blocked, last good JSON + ZIP untouched', async () => {
|
|
201
|
+
const world = await newWorld();
|
|
202
|
+
await writeSource(world, [source('s1'), source('s2')]);
|
|
203
|
+
await run(world);
|
|
204
|
+
const jsonBefore = await fs.readFile(world.json, 'utf8');
|
|
205
|
+
const zipBefore = await fs.readFile(world.zip, 'utf8');
|
|
206
|
+
await writeSource(world, []);
|
|
207
|
+
await assert.rejects(run(world, { at: new Date(NOW.getTime() + HOUR) }), (e: any) => e instanceof FeedGuardError && e.code === 'source_empty');
|
|
208
|
+
assert.equal(await fs.readFile(world.json, 'utf8'), jsonBefore);
|
|
209
|
+
assert.equal(await fs.readFile(world.zip, 'utf8'), zipBefore);
|
|
210
|
+
const saved = JSON.parse(await fs.readFile(join(world.state, 'last-run-report.json'), 'utf8'));
|
|
211
|
+
assert.equal(saved.status, 'blocked');
|
|
212
|
+
assert.equal(saved.guard.code, 'source_empty');
|
|
213
|
+
}],
|
|
214
|
+
|
|
215
|
+
['partial source: base export died after the first pages -> count drop blocked, accepted on purpose', async () => {
|
|
216
|
+
const world = await newWorld();
|
|
217
|
+
const cars = Array.from({ length: 10 }, (_, i) => source(`p${i}`));
|
|
218
|
+
await writeSource(world, cars);
|
|
219
|
+
await run(world);
|
|
220
|
+
await writeSource(world, cars.slice(0, 4));
|
|
221
|
+
const later = new Date(NOW.getTime() + HOUR);
|
|
222
|
+
await assert.rejects(run(world, { at: later }), (e: any) => e instanceof FeedGuardError && e.code === 'count_drop');
|
|
223
|
+
assert.equal((await readItems(world)).length, 10);
|
|
224
|
+
assert.equal(await exists(`${world.json}.rejected`), true);
|
|
225
|
+
const accepted = await run(world, { at: later, guard: { acceptDrop: true } });
|
|
226
|
+
assert.equal(accepted.status, 'published');
|
|
227
|
+
assert.equal((await readItems(world)).length, 4);
|
|
228
|
+
}],
|
|
229
|
+
|
|
230
|
+
['image outage: every hero broken -> published count below minimum, blocked', async () => {
|
|
231
|
+
const world = await newWorld();
|
|
232
|
+
await writeSource(world, [source('i1'), source('i2')]);
|
|
233
|
+
world.brokenImages.add('https://images.auto.de/carimage/i1/i1hero1/large-cached.jpeg');
|
|
234
|
+
world.brokenImages.add('https://images.auto.de/carimage/i2/i2hero1/large-cached.jpeg');
|
|
235
|
+
await assert.rejects(run(world), (e: any) => e instanceof FeedGuardError && e.code === 'published_too_small');
|
|
236
|
+
assert.equal(await exists(world.json), false);
|
|
237
|
+
assert.equal(world.zipCalls, 0);
|
|
238
|
+
}],
|
|
239
|
+
|
|
240
|
+
['stale snapshot is blocked; a leftover temp file is never appended to', async () => {
|
|
241
|
+
const world = await newWorld();
|
|
242
|
+
await writeSource(world, [source('t1')], 2, new Date(NOW.getTime() - 30 * HOUR));
|
|
243
|
+
await assert.rejects(run(world), (e: any) => e instanceof FeedGuardError && e.code === 'source_stale');
|
|
244
|
+
await writeSource(world, [source('t1')]);
|
|
245
|
+
await fs.mkdir(join(world.root, 'meinstadt-dryrun'), { recursive: true });
|
|
246
|
+
await fs.writeFile(`${world.json}.tmp`, '[{"junk":true}');
|
|
247
|
+
await run(world);
|
|
248
|
+
assert.equal((await readItems(world)).length, 1);
|
|
249
|
+
}],
|
|
250
|
+
|
|
251
|
+
['real ZIP: one entry api_export_cars_meinstadt.json, byte-identical to the published JSON', async () => {
|
|
252
|
+
const world = await newWorld();
|
|
253
|
+
await writeSource(world, [source('z1', { 'mainData.subModel': 'Größe' }), source('z2')]);
|
|
254
|
+
await run(world);
|
|
255
|
+
const zipFile = join(world.root, 'real.zip');
|
|
256
|
+
await zipSingleFile(world.json, zipFile);
|
|
257
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
258
|
+
const AdmZip = require('adm-zip');
|
|
259
|
+
const entries = new AdmZip(zipFile).getEntries();
|
|
260
|
+
assert.deepEqual(entries.map((e: any) => e.entryName), ['api_export_cars_meinstadt.json']);
|
|
261
|
+
assert.ok(entries[0].getData().equals(await fs.readFile(world.json)));
|
|
262
|
+
// overwrites a leftover archive instead of appending to it
|
|
263
|
+
await zipSingleFile(world.json, zipFile);
|
|
264
|
+
assert.equal(new AdmZip(zipFile).getEntries().length, 1);
|
|
265
|
+
}],
|
|
266
|
+
|
|
267
|
+
['lock: a second run while one holds the lock is refused', async () => {
|
|
268
|
+
const world = await newWorld();
|
|
269
|
+
await writeSource(world, [source('l1')]);
|
|
270
|
+
await fs.mkdir(world.state, { recursive: true });
|
|
271
|
+
await fs.writeFile(join(world.state, 'run.lock'), '{}');
|
|
272
|
+
await assert.rejects(run(world), (e: any) => e instanceof FeedGuardError && e.code === 'locked');
|
|
273
|
+
}],
|
|
274
|
+
];
|
|
275
|
+
|
|
276
|
+
(async () => {
|
|
277
|
+
let failed = 0;
|
|
278
|
+
for (const [name, fn] of tests) {
|
|
279
|
+
try {
|
|
280
|
+
await fn();
|
|
281
|
+
console.log(`ok ${name}`);
|
|
282
|
+
} catch (e) {
|
|
283
|
+
failed += 1;
|
|
284
|
+
console.error(`FAIL ${name}\n ${e.stack ?? e.message}`);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
if (failed) process.exit(1);
|
|
288
|
+
})();
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
// Run: npm test (ts-node, no test framework needed)
|
|
2
|
+
// meinestadt field mapping (ADV-325) and image list (ADV-326) — pure functions, no I/O.
|
|
3
|
+
import * as assert from 'node:assert/strict';
|
|
4
|
+
import { de } from '@bytebrand/i18n-dictionaries';
|
|
5
|
+
|
|
6
|
+
import { MeinstadtConfig, parseMeinstadtCpc } from '../src/configs/meinstadt.config';
|
|
7
|
+
import { Environment, ExportType, ImageType } from '../src/enums';
|
|
8
|
+
import {
|
|
9
|
+
mapMeinstadtItem,
|
|
10
|
+
toAccidentFree,
|
|
11
|
+
toMeinstadtImages,
|
|
12
|
+
toUnixSeconds,
|
|
13
|
+
translateSelector,
|
|
14
|
+
} from '../src/transformers/meinstadt.transformer';
|
|
15
|
+
import { selectOpenAiImageFields } from '../src/transformers/openai-image-fields';
|
|
16
|
+
import { buildImagesService } from './fixtures/images-service.harness';
|
|
17
|
+
import { car, eightImages, images, translate } from './fixtures/meinstadt.fixture';
|
|
18
|
+
|
|
19
|
+
const imagesService = buildImagesService();
|
|
20
|
+
const ctx = (extra: Record<string, any> = {}) => ({ domain: 'auto.de', cpc: '0.99', equipment: 'ABS', images: ['https://x/1.jpeg'], translate, ...extra });
|
|
21
|
+
|
|
22
|
+
const tests: Array<[string, () => void | Promise<void>]> = [
|
|
23
|
+
['accidentFree: damaged=false -> true, damaged=true -> false, unknown -> null (never accident-free)', () => {
|
|
24
|
+
assert.equal(toAccidentFree(false), true);
|
|
25
|
+
assert.equal(toAccidentFree(true), false);
|
|
26
|
+
assert.equal(toAccidentFree(null), null);
|
|
27
|
+
assert.equal(toAccidentFree(undefined), null);
|
|
28
|
+
assert.equal(toAccidentFree('false'), null);
|
|
29
|
+
assert.equal(mapMeinstadtItem(car('a1', { 'mainData.accidentDamaged': false }), ctx()).accidentFree, true);
|
|
30
|
+
assert.equal(mapMeinstadtItem(car('a2', { 'mainData.accidentDamaged': true }), ctx()).accidentFree, false);
|
|
31
|
+
assert.equal(mapMeinstadtItem(car('a3', { 'mainData.accidentDamaged': null }), ctx()).accidentFree, null);
|
|
32
|
+
const noField = car('a4');
|
|
33
|
+
delete noField.mainData.accidentDamaged;
|
|
34
|
+
assert.equal(mapMeinstadtItem(noField, ctx()).accidentFree, null);
|
|
35
|
+
}],
|
|
36
|
+
|
|
37
|
+
['cpc: only a positive decimal counts as configured, no hidden 0.08 / 0 default', () => {
|
|
38
|
+
assert.equal(parseMeinstadtCpc('0.99'), '0.99');
|
|
39
|
+
assert.equal(parseMeinstadtCpc(' 0.5 '), '0.5');
|
|
40
|
+
assert.equal(parseMeinstadtCpc('1'), '1');
|
|
41
|
+
for (const value of [undefined, null, '', ' ', '0', '0.00', '0,08', '-0.1', 'abc', 0.08]) {
|
|
42
|
+
assert.equal(parseMeinstadtCpc(value), null, `cpc ${JSON.stringify(value)}`);
|
|
43
|
+
}
|
|
44
|
+
const params = { environment: Environment.DEVELOP, exportType: ExportType.MEINSTADT };
|
|
45
|
+
assert.equal(new MeinstadtConfig(params).cpc, null);
|
|
46
|
+
assert.equal(mapMeinstadtItem(car('c1'), ctx({ cpc: null })).cpc, null);
|
|
47
|
+
assert.equal(new MeinstadtConfig({ ...params, cpc: '0.99' }).cpc, '0.99');
|
|
48
|
+
}],
|
|
49
|
+
|
|
50
|
+
['config: dry-run by default (own dir + state), live only with meinstadtDryRun=false; clean images', () => {
|
|
51
|
+
const params = { environment: Environment.DEVELOP, exportType: ExportType.MEINSTADT };
|
|
52
|
+
const dry = new MeinstadtConfig(params);
|
|
53
|
+
assert.equal(dry.dryRun, true);
|
|
54
|
+
assert.equal(dry.imageType, ImageType.clean);
|
|
55
|
+
assert.deepEqual(dry.outputFiles, [
|
|
56
|
+
'resources/meinstadt-dryrun/api_export_cars_meinstadt.json',
|
|
57
|
+
'resources/meinstadt-dryrun/api_export_cars_meinstadt.zip',
|
|
58
|
+
]);
|
|
59
|
+
assert.equal(dry.stateDir, 'resources/meinstadt-state/dryrun');
|
|
60
|
+
assert.equal(new MeinstadtConfig({ ...params, meinstadtDryRun: 'TRUE' }).dryRun, true);
|
|
61
|
+
assert.equal(new MeinstadtConfig({ ...params, meinstadtDryRun: 'no' }).dryRun, true);
|
|
62
|
+
const live = new MeinstadtConfig({ ...params, meinstadtDryRun: 'false' });
|
|
63
|
+
assert.equal(live.dryRun, false);
|
|
64
|
+
// live keeps the historical path the Cronicle upload step reads
|
|
65
|
+
assert.deepEqual(live.outputFiles, [
|
|
66
|
+
'resources/meinstadt-export/api_export_cars_meinstadt.json',
|
|
67
|
+
'resources/meinstadt-export/api_export_cars_meinstadt.zip',
|
|
68
|
+
]);
|
|
69
|
+
assert.equal(live.stateDir, 'resources/meinstadt-state/export');
|
|
70
|
+
assert.equal(dry.maxImages, 0);
|
|
71
|
+
assert.equal(new MeinstadtConfig({ ...params, meinstadtMaxImages: '1' }).maxImages, 1);
|
|
72
|
+
assert.equal(new MeinstadtConfig({ ...params, feedGuardMinSourceItems: '5' }).feedGuard.minSourceItems, 5);
|
|
73
|
+
}],
|
|
74
|
+
|
|
75
|
+
['dates: Unix seconds kept, missing / 0 / negative / garbage -> null (no 1970), Date and ISO converted', () => {
|
|
76
|
+
assert.equal(toUnixSeconds(1750809600), 1750809600);
|
|
77
|
+
assert.equal(toUnixSeconds(0), null);
|
|
78
|
+
assert.equal(toUnixSeconds(-86400), null);
|
|
79
|
+
assert.equal(toUnixSeconds(null), null);
|
|
80
|
+
assert.equal(toUnixSeconds(undefined), null);
|
|
81
|
+
assert.equal(toUnixSeconds(''), null);
|
|
82
|
+
assert.equal(toUnixSeconds('garbage'), null);
|
|
83
|
+
assert.equal(toUnixSeconds(NaN), null);
|
|
84
|
+
assert.equal(toUnixSeconds(new Date('2025-06-25T00:00:00Z')), 1750809600);
|
|
85
|
+
assert.equal(toUnixSeconds('2025-06-25T00:00:00Z'), 1750809600);
|
|
86
|
+
assert.equal(toUnixSeconds(new Date(0)), null);
|
|
87
|
+
const item = mapMeinstadtItem(car('d1', { 'mainData.firstRegistration': 0, 'metaData.creationDate': 1783408438 }), ctx());
|
|
88
|
+
assert.equal(item.initialRegistration, null);
|
|
89
|
+
// the CBD creation date is passed through, never replaced by the export time
|
|
90
|
+
assert.equal(item.offerCreationDate, 1783408438);
|
|
91
|
+
assert.equal(mapMeinstadtItem(car('d2', { 'mainData.firstRegistration': null }), ctx()).initialRegistration, null);
|
|
92
|
+
}],
|
|
93
|
+
|
|
94
|
+
['power = PS (not kW), price = currentSalesPrice (not dealer purchasePrice, not the rate)', () => {
|
|
95
|
+
const item = mapMeinstadtItem(car('p1'), ctx());
|
|
96
|
+
assert.equal(item.power, 116);
|
|
97
|
+
assert.equal(item.price, 15148);
|
|
98
|
+
assert.equal(item.mileage, 13000);
|
|
99
|
+
}],
|
|
100
|
+
|
|
101
|
+
['German labels; selector_unknown / unmapped selectors -> null, not "Unbekannt"', () => {
|
|
102
|
+
assert.equal(translateSelector('selector_unknown', translate), null);
|
|
103
|
+
assert.equal(translateSelector('selector_category_combi', translate), 'Kombi');
|
|
104
|
+
assert.equal(translateSelector('selector_does_not_exist', translate), null);
|
|
105
|
+
assert.equal(translateSelector(null, translate), null);
|
|
106
|
+
// default translator = i18n-dictionaries `de`, which maps selector_unknown to "Unbekannt"
|
|
107
|
+
assert.equal(de.selector_unknown, 'Unbekannt');
|
|
108
|
+
assert.equal(translateSelector('selector_unknown'), null);
|
|
109
|
+
assert.equal(translateSelector('selector_fuel_petrol'), de.selector_fuel_petrol);
|
|
110
|
+
const item = mapMeinstadtItem(car('l1', {
|
|
111
|
+
'mainData.category': 'selector_unknown',
|
|
112
|
+
'environmentEmissions.emissionClass': 'selector_unknown',
|
|
113
|
+
'driveSuspension.gearbox': 'selector_unknown',
|
|
114
|
+
}), ctx());
|
|
115
|
+
assert.equal(item.bodyType, null);
|
|
116
|
+
assert.equal(item.emissionClass, null);
|
|
117
|
+
assert.equal(item['transmissionType/transmission'], null);
|
|
118
|
+
assert.equal(item.usageState, 'Gebraucht');
|
|
119
|
+
assert.equal(mapMeinstadtItem(car('l2', { 'mainData.condition': 'selector_unknown', 'mainData.mileage': 5 }), ctx()).usageState, 'Neu');
|
|
120
|
+
}],
|
|
121
|
+
|
|
122
|
+
['item: historical keys and order, slash keys kept literally, no city/zip/lat/long', () => {
|
|
123
|
+
const item = mapMeinstadtItem(car('k1'), ctx({ images: ['https://x/1.jpeg', 'https://x/2.jpeg'] }));
|
|
124
|
+
assert.deepEqual(Object.keys(item), [
|
|
125
|
+
'id', 'make', 'model', 'cpc', 'listingUrl', 'title', 'accidentFree', 'mileage', 'initialRegistration',
|
|
126
|
+
'offerCreationDate', 'numberOfDoors', 'numberOfSeats', 'numberOfPreviousOwners', 'price', 'power',
|
|
127
|
+
'co2Emissions', 'fuelConsumptionCombined', 'equipment', 'color', 'bodyType', 'emissionClass',
|
|
128
|
+
'emissionSticker', 'fuelType', 'transmissionType/transmission', 'imagesCount', 'customerType/sellerType',
|
|
129
|
+
'usageState', 'images',
|
|
130
|
+
]);
|
|
131
|
+
assert.deepEqual(item, {
|
|
132
|
+
id: 'VK1',
|
|
133
|
+
make: 'Opel',
|
|
134
|
+
model: 'Astra',
|
|
135
|
+
cpc: '0.99',
|
|
136
|
+
listingUrl: 'https://www.auto.de/search/vehicle/k1?source=classifieds&medium=listing&campaign=meinestadt',
|
|
137
|
+
title: 'Opel Astra 1.2 Turbo Edition',
|
|
138
|
+
accidentFree: true,
|
|
139
|
+
mileage: 13000,
|
|
140
|
+
initialRegistration: 1750809600,
|
|
141
|
+
offerCreationDate: 1783408438,
|
|
142
|
+
numberOfDoors: 5,
|
|
143
|
+
numberOfSeats: 5,
|
|
144
|
+
numberOfPreviousOwners: 1,
|
|
145
|
+
price: 15148,
|
|
146
|
+
power: 116,
|
|
147
|
+
co2Emissions: 137,
|
|
148
|
+
fuelConsumptionCombined: 6.1,
|
|
149
|
+
equipment: 'ABS',
|
|
150
|
+
color: 'Rot',
|
|
151
|
+
bodyType: 'Kombi',
|
|
152
|
+
emissionClass: 'Euro 6',
|
|
153
|
+
emissionSticker: 'Feinstaubplakette grün (4)',
|
|
154
|
+
fuelType: 'Benzin',
|
|
155
|
+
'transmissionType/transmission': 'Manuell',
|
|
156
|
+
imagesCount: 2,
|
|
157
|
+
'customerType/sellerType': 'Händler',
|
|
158
|
+
usageState: 'Gebraucht',
|
|
159
|
+
images: ['https://x/1.jpeg', 'https://x/2.jpeg'],
|
|
160
|
+
});
|
|
161
|
+
}],
|
|
162
|
+
|
|
163
|
+
['images: 8 raw images, only 1 valid -> exactly 1 URL and imagesCount 1', () => {
|
|
164
|
+
const clean = imagesService.getCleanImages('car-8', eightImages('e'));
|
|
165
|
+
assert.equal(clean.length, 8);
|
|
166
|
+
const valid = clean[0].url;
|
|
167
|
+
const selection = selectOpenAiImageFields(clean, clean.map(i => ({ url: i.url, ok: i.url === valid })));
|
|
168
|
+
const list = toMeinstadtImages(selection);
|
|
169
|
+
assert.deepEqual(list, [valid]);
|
|
170
|
+
const item = mapMeinstadtItem(car('car-8'), ctx({ images: list }));
|
|
171
|
+
assert.deepEqual(item.images, [valid]);
|
|
172
|
+
assert.equal(item.imagesCount, 1);
|
|
173
|
+
}],
|
|
174
|
+
|
|
175
|
+
['images: URL array (not ids, not a string), hero first, all valid in clean order, count = length', () => {
|
|
176
|
+
const clean = imagesService.getCleanImages('car-g', images('g'));
|
|
177
|
+
const list = toMeinstadtImages(selectOpenAiImageFields(clean, clean.map(i => ({ url: i.url, ok: true }))));
|
|
178
|
+
assert.deepEqual(list, [
|
|
179
|
+
'https://images.auto.de/carimage/car-g/ghero1/large-cached.jpeg',
|
|
180
|
+
'https://images.auto.de/carimage/car-g/gside1/large-cached.jpeg',
|
|
181
|
+
'https://images.auto.de/carimage/car-g/gback1/large-cached.jpeg',
|
|
182
|
+
'https://images.auto.de/carimage/car-g/gint01/large-cached.jpeg',
|
|
183
|
+
]);
|
|
184
|
+
const item = mapMeinstadtItem(car('car-g'), ctx({ images: list }));
|
|
185
|
+
assert.ok(Array.isArray(item.images));
|
|
186
|
+
assert.equal(item.imagesCount, item.images.length);
|
|
187
|
+
assert.ok(item.images.every(url => url.startsWith('https://') && !url.includes('/noimage/')));
|
|
188
|
+
}],
|
|
189
|
+
|
|
190
|
+
['images: no valid 3/4 hero -> [] (car skipped); maxImages cuts the list only', () => {
|
|
191
|
+
const clean = imagesService.getCleanImages('car-h', images('h'));
|
|
192
|
+
const heroBroken = clean.map(i => ({ url: i.url, ok: !i.heroCandidate }));
|
|
193
|
+
assert.deepEqual(toMeinstadtImages(selectOpenAiImageFields(clean, heroBroken)), []);
|
|
194
|
+
assert.deepEqual(toMeinstadtImages({ fields: null, rejected: [] }), []);
|
|
195
|
+
const allOk = selectOpenAiImageFields(clean, clean.map(i => ({ url: i.url, ok: true })));
|
|
196
|
+
assert.deepEqual(toMeinstadtImages(allOk, 1), [clean[0].url]);
|
|
197
|
+
assert.equal(toMeinstadtImages(allOk, 0).length, 4);
|
|
198
|
+
// the selection itself is not modified by the cut
|
|
199
|
+
assert.equal(allOk.fields.additional_image_urls.length, 3);
|
|
200
|
+
}],
|
|
201
|
+
];
|
|
202
|
+
|
|
203
|
+
(async () => {
|
|
204
|
+
let failed = 0;
|
|
205
|
+
for (const [name, fn] of tests) {
|
|
206
|
+
try {
|
|
207
|
+
await fn();
|
|
208
|
+
console.log(`ok ${name}`);
|
|
209
|
+
} catch (e) {
|
|
210
|
+
failed += 1;
|
|
211
|
+
console.error(`FAIL ${name}\n ${e.message}`);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
if (failed) process.exit(1);
|
|
215
|
+
})();
|