@bytebrand/export-cars 1.26.46 → 1.26.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/configs/ooyyo.config.d.ts +14 -0
- package/dist/configs/ooyyo.config.js +40 -6
- package/dist/configs/ooyyo.config.js.map +1 -1
- package/dist/exporters/ooyyo.exporter.d.ts +5 -4
- package/dist/exporters/ooyyo.exporter.js +45 -25
- package/dist/exporters/ooyyo.exporter.js.map +1 -1
- package/dist/formats/ooyyo-xml.builder.d.ts +7 -2
- package/dist/formats/ooyyo-xml.builder.js +59 -36
- package/dist/formats/ooyyo-xml.builder.js.map +1 -1
- package/dist/interfaces/export-item.interface.d.ts +30 -2
- package/dist/interfaces/params.interface.d.ts +3 -0
- package/dist/services/mapping.service.d.ts +2 -0
- package/dist/services/mapping.service.js +33 -13
- package/dist/services/mapping.service.js.map +1 -1
- package/dist/services/ooyyo-feed-run.d.ts +81 -0
- package/dist/services/ooyyo-feed-run.js +183 -0
- package/dist/services/ooyyo-feed-run.js.map +1 -0
- package/dist/services/ooyyo-feed-verify.d.ts +13 -0
- package/dist/services/ooyyo-feed-verify.js +104 -0
- package/dist/services/ooyyo-feed-verify.js.map +1 -0
- package/dist/transformers/ooyyo.transformer.d.ts +51 -11
- package/dist/transformers/ooyyo.transformer.js +268 -95
- package/dist/transformers/ooyyo.transformer.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/marktde/AUDIT.md +1 -1
- package/docs/marktde/RUNBOOK.md +3 -3
- package/docs/marktde/TRACKING.md +8 -2
- package/docs/meinestadt/PARTNER-QUESTIONS.md +17 -0
- package/docs/meinestadt/TRACKING.md +107 -0
- package/docs/ooyyo/AUDIT.md +81 -0
- package/docs/ooyyo/PARTNER-QUESTIONS.md +32 -0
- package/docs/ooyyo/RUNBOOK.md +67 -0
- package/docs/ooyyo/TRACKING.md +54 -0
- package/package.json +3 -2
- package/src/configs/ooyyo.config.ts +65 -15
- package/src/constants/fb.constants.ts +34 -34
- package/src/exporters/ooyyo.exporter.ts +75 -48
- package/src/formats/ooyyo-xml.builder.ts +87 -48
- package/src/interfaces/export-item.interface.ts +43 -2
- package/src/interfaces/params.interface.ts +5 -0
- package/src/services/mapping.service.ts +54 -13
- package/src/services/ooyyo-feed-run.ts +265 -0
- package/src/services/ooyyo-feed-verify.ts +131 -0
- package/src/transformers/ooyyo.transformer.ts +378 -109
- package/test/fixtures/ooyyo/OoyyoFormat.xml +125 -0
- package/test/fixtures/ooyyo/OoyyoFormat.xsd +116 -0
- package/test/fixtures/ooyyo.fixture.ts +53 -0
- package/test/mapping-csv.test.ts +65 -0
- package/test/ooyyo-feed-run.test.ts +293 -0
- package/test/ooyyo-mapping.test.ts +276 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<xsd:schema elementFormDefault="qualified" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
|
|
3
|
+
<xsd:element name="ooyyo">
|
|
4
|
+
<xsd:complexType>
|
|
5
|
+
<xsd:choice minOccurs="0" maxOccurs="unbounded">
|
|
6
|
+
<xsd:element ref="ad"/>
|
|
7
|
+
</xsd:choice>
|
|
8
|
+
</xsd:complexType>
|
|
9
|
+
</xsd:element>
|
|
10
|
+
<xsd:element name="ad">
|
|
11
|
+
<xsd:complexType mixed="true">
|
|
12
|
+
<xsd:choice minOccurs="0" maxOccurs="unbounded">
|
|
13
|
+
<xsd:element ref="url"/>
|
|
14
|
+
<xsd:element ref="title"/>
|
|
15
|
+
<xsd:element ref="content"/>
|
|
16
|
+
<xsd:element ref="price"/>
|
|
17
|
+
<xsd:element ref="make"/>
|
|
18
|
+
<xsd:element ref="model"/>
|
|
19
|
+
<xsd:element ref="engine_size"/>
|
|
20
|
+
<xsd:element ref="trim"/>
|
|
21
|
+
<xsd:element ref="body_type"/>
|
|
22
|
+
<xsd:element ref="color"/>
|
|
23
|
+
<xsd:element ref="year"/>
|
|
24
|
+
<xsd:element ref="mileage"/>
|
|
25
|
+
<xsd:element ref="fuel_type"/>
|
|
26
|
+
<xsd:element ref="transmission"/>
|
|
27
|
+
<xsd:element ref="euronorm"/>
|
|
28
|
+
<xsd:element ref="power"/>
|
|
29
|
+
<xsd:element ref="doors"/>
|
|
30
|
+
<xsd:element ref="seats"/>
|
|
31
|
+
<xsd:element ref="gears"/>
|
|
32
|
+
<xsd:element ref="location"/>
|
|
33
|
+
<xsd:element ref="pictures"/>
|
|
34
|
+
<xsd:element ref="vin"/>
|
|
35
|
+
<xsd:element ref="is_new"/>
|
|
36
|
+
<xsd:element ref="date"/>
|
|
37
|
+
<xsd:element ref="time"/>
|
|
38
|
+
</xsd:choice>
|
|
39
|
+
</xsd:complexType>
|
|
40
|
+
</xsd:element>
|
|
41
|
+
<xsd:element name="url" type="xsd:string"/>
|
|
42
|
+
<xsd:element name="title" type="xsd:string"/>
|
|
43
|
+
<xsd:element name="content" type="xsd:string"/>
|
|
44
|
+
<xsd:element name="price">
|
|
45
|
+
<xsd:complexType>
|
|
46
|
+
<xsd:simpleContent>
|
|
47
|
+
<xsd:extension base="xsd:string">
|
|
48
|
+
<xsd:attribute name="unit" type="xsd:string" use="required"/>
|
|
49
|
+
</xsd:extension>
|
|
50
|
+
</xsd:simpleContent>
|
|
51
|
+
</xsd:complexType>
|
|
52
|
+
</xsd:element>
|
|
53
|
+
<xsd:element name="make" type="xsd:string"/>
|
|
54
|
+
<xsd:element name="model" type="xsd:string"/>
|
|
55
|
+
<xsd:element name="engine_size">
|
|
56
|
+
<xsd:complexType>
|
|
57
|
+
<xsd:simpleContent>
|
|
58
|
+
<xsd:extension base="xsd:string">
|
|
59
|
+
<xsd:attribute name="unit" type="xsd:string" use="required"/>
|
|
60
|
+
</xsd:extension>
|
|
61
|
+
</xsd:simpleContent>
|
|
62
|
+
</xsd:complexType>
|
|
63
|
+
</xsd:element>
|
|
64
|
+
<xsd:element name="trim" type="xsd:string"/>
|
|
65
|
+
<xsd:element name="body_type" type="xsd:string"/>
|
|
66
|
+
<xsd:element name="color" type="xsd:string"/>
|
|
67
|
+
<xsd:element name="year" type="xsd:string"/>
|
|
68
|
+
<xsd:element name="mileage">
|
|
69
|
+
<xsd:complexType>
|
|
70
|
+
<xsd:simpleContent>
|
|
71
|
+
<xsd:extension base="xsd:string">
|
|
72
|
+
<xsd:attribute name="unit" type="xsd:string" use="required"/>
|
|
73
|
+
</xsd:extension>
|
|
74
|
+
</xsd:simpleContent>
|
|
75
|
+
</xsd:complexType>
|
|
76
|
+
</xsd:element>
|
|
77
|
+
<xsd:element name="fuel_type" type="xsd:string"/>
|
|
78
|
+
<xsd:element name="transmission" type="xsd:string"/>
|
|
79
|
+
<xsd:element name="euronorm" type="xsd:string"/>
|
|
80
|
+
<xsd:element name="power">
|
|
81
|
+
<xsd:complexType>
|
|
82
|
+
<xsd:simpleContent>
|
|
83
|
+
<xsd:extension base="xsd:string">
|
|
84
|
+
<xsd:attribute name="unit" type="xsd:string" use="required"/>
|
|
85
|
+
</xsd:extension>
|
|
86
|
+
</xsd:simpleContent>
|
|
87
|
+
</xsd:complexType>
|
|
88
|
+
</xsd:element>
|
|
89
|
+
<xsd:element name="doors" type="xsd:string"/>
|
|
90
|
+
<xsd:element name="seats" type="xsd:string"/>
|
|
91
|
+
<xsd:element name="gears" type="xsd:string"/>
|
|
92
|
+
<xsd:element name="location">
|
|
93
|
+
<xsd:complexType mixed="true">
|
|
94
|
+
<xsd:choice minOccurs="0" maxOccurs="unbounded">
|
|
95
|
+
<xsd:element ref="city"/>
|
|
96
|
+
<xsd:element ref="postcode"/>
|
|
97
|
+
<xsd:element ref="region"/>
|
|
98
|
+
</xsd:choice>
|
|
99
|
+
</xsd:complexType>
|
|
100
|
+
</xsd:element>
|
|
101
|
+
<xsd:element name="city" type="xsd:string"/>
|
|
102
|
+
<xsd:element name="postcode" type="xsd:string"/>
|
|
103
|
+
<xsd:element name="region" type="xsd:string"/>
|
|
104
|
+
<xsd:element name="pictures">
|
|
105
|
+
<xsd:complexType mixed="true">
|
|
106
|
+
<xsd:choice minOccurs="0" maxOccurs="unbounded">
|
|
107
|
+
<xsd:element ref="picture"/>
|
|
108
|
+
</xsd:choice>
|
|
109
|
+
</xsd:complexType>
|
|
110
|
+
</xsd:element>
|
|
111
|
+
<xsd:element name="picture" type="xsd:string"/>
|
|
112
|
+
<xsd:element name="vin" type="xsd:string"/>
|
|
113
|
+
<xsd:element name="is_new" type="xsd:string"/>
|
|
114
|
+
<xsd:element name="date" type="xsd:string"/>
|
|
115
|
+
<xsd:element name="time" type="xsd:string"/>
|
|
116
|
+
</xsd:schema>
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { car as baseCar, images } from './meinstadt.fixture';
|
|
2
|
+
|
|
3
|
+
export { NOW, images } from './meinstadt.fixture';
|
|
4
|
+
|
|
5
|
+
// German labels as @bytebrand/i18n-dictionaries `de` returns them (OoyyoTransformer.translate).
|
|
6
|
+
const LABELS: Record<string, string> = {
|
|
7
|
+
selector_exteriorColor_red: 'Außenfarbe Rot',
|
|
8
|
+
selector_category_combi: 'Kombi',
|
|
9
|
+
selector_category_suv: 'SUV/Geländewagen',
|
|
10
|
+
selector_fuel_petrol: 'Benzin',
|
|
11
|
+
selector_fuel_electrical: 'Elektro',
|
|
12
|
+
selector_gearbox_manualShift: 'Manuell',
|
|
13
|
+
selector_gearbox_automatic: 'Automatik',
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export const translate = (selector: string): string => LABELS[selector] ?? '';
|
|
17
|
+
|
|
18
|
+
export const region = async (postcode: string): Promise<string> =>
|
|
19
|
+
({ '04103': 'Sachsen', '10115': 'Berlin', '80331': 'Bayern' } as Record<string, string>)[postcode] ?? '';
|
|
20
|
+
|
|
21
|
+
// A complete, sellable OOYYO car (CBD shape as the base export delivers it).
|
|
22
|
+
export const car = (id: string, overrides: Record<string, any> = {}): any => baseCar(id, {
|
|
23
|
+
'metaData.creationDate': 1783408438, // 07.07.2026 07:13:58 UTC = 09:13 Berlin
|
|
24
|
+
'metaData.vin': 'WVWZZZ1KZAW000001',
|
|
25
|
+
'mainData.constructionYear': 2024,
|
|
26
|
+
'engineData.cubicCapacity': 1199,
|
|
27
|
+
'driveSuspension.gears': 6,
|
|
28
|
+
'location.zipCode': 4103,
|
|
29
|
+
'location.city': 'Leipzig',
|
|
30
|
+
'location.countryCode': 'DE',
|
|
31
|
+
'offer.offerDescription': '**Ausstattung**\\\\* ABS\\\\* Klimaanlage\\\\Fahrzeug in gutem Zustand – Prüfung bestanden.',
|
|
32
|
+
...overrides,
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
export const source = (id: string, overrides: Record<string, any> = {}) => ({ car: car(id, overrides), images: images(id) });
|
|
36
|
+
|
|
37
|
+
// 20 representative cars: makes, used/new, petrol/BEV, postcodes with and without leading zero.
|
|
38
|
+
export const twentyCars = () => Array.from({ length: 20 }, (_, i) => {
|
|
39
|
+
const id = `car${String(i).padStart(2, '0')}`;
|
|
40
|
+
const variants: Record<string, any>[] = [
|
|
41
|
+
{},
|
|
42
|
+
{ 'mainData.make': 'Škoda', 'mainData.model': 'Octavia', 'mainData.subModel': 'Combi 2.0 TDI Größe L' },
|
|
43
|
+
{ 'mainData.condition': 'selector_condition_new', 'mainData.firstRegistration': null, 'mainData.mileage': 0, 'mainData.constructionYear': 2026 },
|
|
44
|
+
{ 'consumption.fuel': 'selector_fuel_electrical', 'engineData.cubicCapacity': null, 'mainData.make': 'Tesla', 'mainData.model': 'Model 3' },
|
|
45
|
+
{ 'location.zipCode': 10115, 'location.city': 'Berlin' },
|
|
46
|
+
{ 'location.zipCode': 80331, 'location.city': 'München', 'mainData.category': 'selector_category_suv' },
|
|
47
|
+
{ 'environmentEmissions.emissionClass': 'selector_unknown' },
|
|
48
|
+
{ 'offer.offerDescription': '' },
|
|
49
|
+
{ 'metaData.vin': 'FAKEVIN' },
|
|
50
|
+
{ 'driveSuspension.gearbox': 'selector_gearbox_automatic', 'price.currentSalesPrice': 26900 },
|
|
51
|
+
];
|
|
52
|
+
return source(id, variants[i % variants.length]);
|
|
53
|
+
});
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// Run: npm test (ts-node, no test framework needed)
|
|
2
|
+
// Mapping rules from the public CSV of the mapping sheet tab (no Sheets API key) — pure, no I/O.
|
|
3
|
+
import * as assert from 'node:assert/strict';
|
|
4
|
+
|
|
5
|
+
import { MappingService, mappingSheetCsvUrl } from '../src/services/mapping.service';
|
|
6
|
+
|
|
7
|
+
const service = new MappingService({ config: {} } as any);
|
|
8
|
+
|
|
9
|
+
// Excerpt of the tab mobile_yescar as gviz returns it (13.11.2026): quoted cells, an unnamed column,
|
|
10
|
+
// trailing empty columns, selector rows that group onto the first row of their importField.
|
|
11
|
+
const CSV = [
|
|
12
|
+
'"importField","importFieldType","autodeField","autodeFieldType","importFieldSelector","autodeFieldSelector","","custom","",""',
|
|
13
|
+
'"[\'0\']","string","persistentId","string","","","","yes","",""',
|
|
14
|
+
'"[\'2\']","categoryOrClass_selector","mainData.category","selector","cabrio/roadster","selector_category_convertible","note","","",""',
|
|
15
|
+
'"[\'2\']","categoryOrClass_selector","mainData.category","selector","Kombi, Van","selector_category_combi","","","",""',
|
|
16
|
+
'"[\'3\']","string","mainData.make","string","","","","","",""',
|
|
17
|
+
'"[\'110\']","selector","driveSuspension.gearbox","selector","1","selector_gearbox_manualShift","","","",""',
|
|
18
|
+
'"[\'110\']","selector","driveSuspension.gearbox","selector","3","selector_gearbox_automatic","","","",""',
|
|
19
|
+
'"[\'999\']","string","","string","","","","","",""',
|
|
20
|
+
'',
|
|
21
|
+
].join('\n');
|
|
22
|
+
|
|
23
|
+
const tests: Array<[string, () => void]> = [
|
|
24
|
+
['url: gviz CSV of the tab by title, title URL-encoded, no key', () => {
|
|
25
|
+
assert.equal(
|
|
26
|
+
mappingSheetCsvUrl('147qz9sTgs1wi13ua8YKa0IgfxAn1X_qH6LkGnAnzeyY', 'mobile_yescar'),
|
|
27
|
+
'https://docs.google.com/spreadsheets/d/147qz9sTgs1wi13ua8YKa0IgfxAn1X_qH6LkGnAnzeyY/gviz/tq?tqx=out:csv&sheet=mobile_yescar',
|
|
28
|
+
);
|
|
29
|
+
assert.ok(mappingSheetCsvUrl('x', 'a b&c').endsWith('sheet=a%20b%26c'));
|
|
30
|
+
}],
|
|
31
|
+
|
|
32
|
+
['rules: same post-processing as the Sheets API rows (empty keys dropped, selectors grouped, rows without autodeField out)', () => {
|
|
33
|
+
const rules = service.rulesFromCsv(CSV, 'mobile_yescar') as any[];
|
|
34
|
+
assert.deepEqual(rules.map(r => r.importField), ["['0']", "['2']", "['3']", "['110']"]);
|
|
35
|
+
assert.deepEqual(rules[0], { importField: "['0']", importFieldType: 'string', autodeField: 'persistentId', autodeFieldType: 'string', custom: 'yes' });
|
|
36
|
+
assert.deepEqual(rules[1].selectors, ['cabrio/roadster:selector_category_convertible', 'Kombi, Van:selector_category_combi']);
|
|
37
|
+
assert.equal(rules[1].importFieldSelector, undefined);
|
|
38
|
+
assert.equal(rules[1][''], undefined, 'unnamed column is not part of the rule');
|
|
39
|
+
assert.deepEqual(rules[3].selectors, ['1:selector_gearbox_manualShift', '3:selector_gearbox_automatic']);
|
|
40
|
+
assert.deepEqual(Object.keys(rules[2]).sort(), ['autodeField', 'autodeFieldType', 'importField', 'importFieldType']);
|
|
41
|
+
}],
|
|
42
|
+
|
|
43
|
+
['not the tab (sign-in page / other table) -> error, never an empty mapping', () => {
|
|
44
|
+
assert.throws(() => service.rulesFromCsv('<!DOCTYPE html><html><body>Sign in</body></html>', 'mobile_yescar'), /not readable as CSV.*shared/);
|
|
45
|
+
assert.throws(() => service.rulesFromCsv('', 'mobile_yescar'), /not readable as CSV/);
|
|
46
|
+
assert.throws(() => service.rulesFromCsv('"a","b"\n"1","2"\n', 'mobile_yescar'), /missing columns importField/);
|
|
47
|
+
}],
|
|
48
|
+
|
|
49
|
+
['a tab with the columns but no usable rule -> error', () => {
|
|
50
|
+
const header = '"importField","importFieldType","autodeField","autodeFieldType"';
|
|
51
|
+
assert.throws(() => service.rulesFromCsv(`${header}\n"[\'1\']","string","",""\n`, 'mobile'), /has no rule/);
|
|
52
|
+
}],
|
|
53
|
+
];
|
|
54
|
+
|
|
55
|
+
let failed = 0;
|
|
56
|
+
for (const [name, fn] of tests) {
|
|
57
|
+
try {
|
|
58
|
+
fn();
|
|
59
|
+
console.log(`ok ${name}`);
|
|
60
|
+
} catch (e) {
|
|
61
|
+
failed += 1;
|
|
62
|
+
console.error(`FAIL ${name}\n ${e.message}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
if (failed) process.exit(1);
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
// Run: npm test (ts-node, no test framework needed)
|
|
2
|
+
// Integration (ADV-331): real files in a temp dir (base export snapshot, output, state), real XML
|
|
3
|
+
// builder, read-back verifier, gzip, guard and ADV-336 clean-image selection; only the HTTP image
|
|
4
|
+
// check and the postcode table are faked. Covers the partner's full-replacement risk: a partly valid,
|
|
5
|
+
// empty, stale or collapsed snapshot is never published, the last good file stays.
|
|
6
|
+
import * as assert from 'node:assert/strict';
|
|
7
|
+
import { promises as fs } from 'fs';
|
|
8
|
+
import { tmpdir } from 'os';
|
|
9
|
+
import { join } from 'path';
|
|
10
|
+
import { gunzipSync } from 'zlib';
|
|
11
|
+
import { XMLParser } from 'fast-xml-parser';
|
|
12
|
+
|
|
13
|
+
import { FeedGuard, FeedGuardError, FeedGuardOptions } from '../src/services/feed-guard';
|
|
14
|
+
import { OoyyoFeedRun, OoyyoSourceCar } from '../src/services/ooyyo-feed-run';
|
|
15
|
+
import { verifyOoyyoXml } from '../src/services/ooyyo-feed-verify';
|
|
16
|
+
import { OoyyoXmlBuilder } from '../src/formats/ooyyo-xml.builder';
|
|
17
|
+
import { OoyyoExclusion, toOoyyoImages } from '../src/transformers/ooyyo.transformer';
|
|
18
|
+
import { selectOpenAiImageFields } from '../src/transformers/openai-image-fields';
|
|
19
|
+
import { buildImagesService } from './fixtures/images-service.harness';
|
|
20
|
+
import { NOW, region, source, translate, twentyCars } from './fixtures/ooyyo.fixture';
|
|
21
|
+
|
|
22
|
+
const HOUR = 60 * 60 * 1000;
|
|
23
|
+
const imagesService = buildImagesService();
|
|
24
|
+
|
|
25
|
+
interface World {
|
|
26
|
+
root: string;
|
|
27
|
+
source: string;
|
|
28
|
+
xml: string;
|
|
29
|
+
gz: string;
|
|
30
|
+
state: string;
|
|
31
|
+
brokenImages: Set<string>;
|
|
32
|
+
allImagesBroken: boolean;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const newWorld = async (): Promise<World> => {
|
|
36
|
+
const root = await fs.mkdtemp(join(tmpdir(), 'ooyyo-'));
|
|
37
|
+
return {
|
|
38
|
+
root,
|
|
39
|
+
source: join(root, 'cars-export'),
|
|
40
|
+
xml: join(root, 'ooyyo-dryrun', 'ooyyo.xml'),
|
|
41
|
+
gz: join(root, 'ooyyo-dryrun', 'ooyyo.xml.gz'),
|
|
42
|
+
state: join(root, 'ooyyo-state', 'dryrun'),
|
|
43
|
+
brokenImages: new Set(),
|
|
44
|
+
allImagesBroken: false,
|
|
45
|
+
};
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
const writeSource = async (world: World, cars: OoyyoSourceCar[], chunkSize = 5, mtime = NOW) => {
|
|
49
|
+
await fs.rm(world.source, { recursive: true, force: true });
|
|
50
|
+
await fs.mkdir(world.source, { recursive: true });
|
|
51
|
+
for (let i = 0; i * chunkSize < cars.length; i++) {
|
|
52
|
+
const file = join(world.source, `cars.${i}.json`);
|
|
53
|
+
await fs.writeFile(file, JSON.stringify(cars.slice(i * chunkSize, (i + 1) * chunkSize)));
|
|
54
|
+
await fs.utimes(file, mtime, mtime);
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const guardOptions = (world: World, extra: Partial<FeedGuardOptions> = {}): FeedGuardOptions => ({
|
|
59
|
+
stateDir: world.state,
|
|
60
|
+
minSourceItems: 1,
|
|
61
|
+
minPublishedItems: 1,
|
|
62
|
+
maxDropPercent: 20,
|
|
63
|
+
maxSourceAgeHours: 26,
|
|
64
|
+
maxStaleHours: 48,
|
|
65
|
+
acceptDrop: false,
|
|
66
|
+
...extra,
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
const run = async (world: World, { at = NOW, guard = {} }: { at?: Date; guard?: Partial<FeedGuardOptions> } = {}) => {
|
|
70
|
+
const forEachChunk = async (fn: (cars: OoyyoSourceCar[]) => Promise<void>) => {
|
|
71
|
+
const names = (await fs.readdir(world.source)).sort((a, b) => Number(a.split('.')[1]) - Number(b.split('.')[1]));
|
|
72
|
+
for (const name of names) {
|
|
73
|
+
await fn(JSON.parse(await fs.readFile(join(world.source, name), 'utf8')));
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
const feed = new OoyyoFeedRun(
|
|
77
|
+
{
|
|
78
|
+
forEachChunk,
|
|
79
|
+
filterCar: ({ car: c }) => (c?.metaData?.imagesCount ?? 0) > 0,
|
|
80
|
+
resolveImages: async ({ car: c, images: list }) => {
|
|
81
|
+
const clean = imagesService.getCleanImages(c._id, list);
|
|
82
|
+
const checks = clean.map(i => ({ url: i.url, ok: !world.allImagesBroken && !world.brokenImages.has(i.url) }));
|
|
83
|
+
return toOoyyoImages(selectOpenAiImageFields(clean, checks));
|
|
84
|
+
},
|
|
85
|
+
translate,
|
|
86
|
+
region,
|
|
87
|
+
imageStats: () => ({ ok: 1 }),
|
|
88
|
+
guard: new FeedGuard(guardOptions(world, guard)),
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
runId: 'test',
|
|
92
|
+
dryRun: true,
|
|
93
|
+
domain: 'auto.de',
|
|
94
|
+
sourceDirectory: world.source,
|
|
95
|
+
sourceBaseFileName: 'cars',
|
|
96
|
+
xmlFile: world.xml,
|
|
97
|
+
gzFile: world.gz,
|
|
98
|
+
stateDir: world.state,
|
|
99
|
+
now: () => at,
|
|
100
|
+
},
|
|
101
|
+
);
|
|
102
|
+
return feed.run();
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
const blocked = async (promise: Promise<unknown>, code: string) => {
|
|
106
|
+
await assert.rejects(promise, (error: any) => error instanceof FeedGuardError && error.code === code);
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
const parse = (xml: string) => new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@', parseTagValue: false, isArray: n => n === 'ad' || n === 'picture' })
|
|
110
|
+
.parse(xml).ooyyo;
|
|
111
|
+
const readAds = async (world: World) => parse(await fs.readFile(world.xml, 'utf8')).ad as any[];
|
|
112
|
+
const idsOf = (ads: any[]) => ads.map(ad => String(ad.url).split('/vehicle/')[1].split('?')[0]);
|
|
113
|
+
const exists = (file: string) => fs.stat(file).then(() => true, () => false);
|
|
114
|
+
const report = async (world: World) => JSON.parse(await fs.readFile(join(world.state, 'last-run-report.json'), 'utf8'));
|
|
115
|
+
|
|
116
|
+
const tests: Array<[string, () => Promise<void>]> = [
|
|
117
|
+
['20 representative cars: all published, XML verified, gzip identical, partner rules hold on every ad', async () => {
|
|
118
|
+
const world = await newWorld();
|
|
119
|
+
await writeSource(world, twentyCars());
|
|
120
|
+
const r = await run(world);
|
|
121
|
+
assert.equal(r.status, 'published');
|
|
122
|
+
assert.equal(r.items, 20);
|
|
123
|
+
assert.deepEqual(r.excluded, {});
|
|
124
|
+
assert.equal(r.verify.issues, 0);
|
|
125
|
+
const xml = await fs.readFile(world.xml, 'utf8');
|
|
126
|
+
assert.ok(xml.startsWith('<?xml version="1.0" encoding="UTF-8"?>\n<ooyyo xsi:noNamespaceSchemaLocation="OoyyoFormat.xsd"'));
|
|
127
|
+
assert.equal(gunzipSync(await fs.readFile(world.gz)).toString('utf8'), xml);
|
|
128
|
+
assert.deepEqual(verifyOoyyoXml(xml, 20).issues, []);
|
|
129
|
+
const ads = await readAds(world);
|
|
130
|
+
assert.equal(ads.length, 20);
|
|
131
|
+
for (const ad of ads) {
|
|
132
|
+
assert.equal(ad.price['@unit'], 'EUR');
|
|
133
|
+
assert.match(ad.price['#text'], /^[1-9]\d*$/);
|
|
134
|
+
assert.match(ad.year, /^20\d\d$/);
|
|
135
|
+
assert.ok(ad.pictures.picture.length >= 1);
|
|
136
|
+
assert.ok(ad.url.length <= 1500);
|
|
137
|
+
assert.ok(!JSON.stringify(ad).includes('noimage'));
|
|
138
|
+
}
|
|
139
|
+
assert.equal(ads[1].make, 'Škoda');
|
|
140
|
+
assert.equal(ads[1].trim, 'Combi 2.0 TDI Größe L');
|
|
141
|
+
assert.equal(ads[0].location.postcode, '04103');
|
|
142
|
+
assert.equal(ads[0].location.region, 'Sachsen');
|
|
143
|
+
assert.equal(ads[6].euronorm, undefined, 'unknown euronorm left out');
|
|
144
|
+
assert.equal(ads[7].content, undefined, 'empty description left out');
|
|
145
|
+
assert.equal(ads[8].vin, undefined, 'malformed VIN left out');
|
|
146
|
+
// 4103 -> 04103 on every car except the 4 Berlin / München variants
|
|
147
|
+
assert.equal(r.postcodePadded, 16);
|
|
148
|
+
assert.equal(r.lastGood.publishedItems, 20);
|
|
149
|
+
assert.ok(r.bytes.gz < r.bytes.xml);
|
|
150
|
+
}],
|
|
151
|
+
|
|
152
|
+
['cars OOYYO would reject are excluded with a reason, the rest is published', async () => {
|
|
153
|
+
const world = await newWorld();
|
|
154
|
+
const noHero = source('nohero');
|
|
155
|
+
noHero.images = noHero.images.filter(i => i.imageCodePredicted !== 'ext_f_right');
|
|
156
|
+
const noPics = source('nopics');
|
|
157
|
+
noPics.car.metaData.imagesCount = 0;
|
|
158
|
+
await writeSource(world, [
|
|
159
|
+
...twentyCars(),
|
|
160
|
+
source('car00'), // duplicate id
|
|
161
|
+
source('sold', { 'offer.offerAvailability': 'selector_offerAvailability_sold' }),
|
|
162
|
+
source('resv', { 'offer.offerAvailability': 'selector_offerAvailability_reserved' }),
|
|
163
|
+
source('noprice', { 'price.currentSalesPrice': 0 }),
|
|
164
|
+
source('sonst', { 'mainData.model': 'SONSTIGE' }),
|
|
165
|
+
source('noyear', { 'mainData.firstRegistration': null, 'mainData.constructionYear': null }),
|
|
166
|
+
source('x'.repeat(1500)), // URL > 1,500 characters
|
|
167
|
+
noHero,
|
|
168
|
+
noPics,
|
|
169
|
+
]);
|
|
170
|
+
const r = await run(world);
|
|
171
|
+
assert.equal(r.items, 20);
|
|
172
|
+
assert.deepEqual(r.excluded, {
|
|
173
|
+
[OoyyoExclusion.duplicateId]: 1,
|
|
174
|
+
[OoyyoExclusion.notAvailable]: 2,
|
|
175
|
+
[OoyyoExclusion.noPrice]: 1,
|
|
176
|
+
[OoyyoExclusion.noModel]: 1,
|
|
177
|
+
[OoyyoExclusion.noYear]: 1,
|
|
178
|
+
[OoyyoExclusion.urlTooLong]: 1,
|
|
179
|
+
[OoyyoExclusion.noImage]: 1,
|
|
180
|
+
[OoyyoExclusion.filtered]: 1,
|
|
181
|
+
});
|
|
182
|
+
assert.equal(idsOf(await readAds(world)).length, 20);
|
|
183
|
+
}],
|
|
184
|
+
|
|
185
|
+
['1 bad ad out of 20 in the written file (builder regression): NOT published, live file keeps all 20', async () => {
|
|
186
|
+
const world = await newWorld();
|
|
187
|
+
await writeSource(world, twentyCars());
|
|
188
|
+
await run(world);
|
|
189
|
+
const goodXml = await fs.readFile(world.xml, 'utf8');
|
|
190
|
+
const goodGz = await fs.readFile(world.gz);
|
|
191
|
+
const original = OoyyoXmlBuilder.toXml;
|
|
192
|
+
let n = 0;
|
|
193
|
+
OoyyoXmlBuilder.toXml = (item) => {
|
|
194
|
+
const xml = original(item);
|
|
195
|
+
n += 1;
|
|
196
|
+
return n === 7 ? xml.replace('unit="EUR"', 'unit="euro"') : xml;
|
|
197
|
+
};
|
|
198
|
+
try {
|
|
199
|
+
await blocked(run(world, { at: new Date(NOW.getTime() + HOUR) }), 'verify_failed');
|
|
200
|
+
} finally {
|
|
201
|
+
OoyyoXmlBuilder.toXml = original;
|
|
202
|
+
}
|
|
203
|
+
assert.equal(await fs.readFile(world.xml, 'utf8'), goodXml, 'published XML untouched');
|
|
204
|
+
assert.ok((await fs.readFile(world.gz)).equals(goodGz), 'published gzip untouched');
|
|
205
|
+
assert.ok(await exists(`${world.xml}.rejected`));
|
|
206
|
+
const rep = await report(world);
|
|
207
|
+
assert.equal(rep.status, 'blocked');
|
|
208
|
+
assert.equal(rep.verify.issues, 2); // unit + partner price rule, both on ad 6
|
|
209
|
+
assert.ok(rep.verify.firstIssues.every((i: any) => i.ad === 6));
|
|
210
|
+
assert.equal(rep.lastGood.publishedItems, 20);
|
|
211
|
+
assert.ok(!(await exists(world.gz.replace(/\.gz$/, '.tmp.gz'))));
|
|
212
|
+
}],
|
|
213
|
+
|
|
214
|
+
['zero readable listings (image service down): blocked, last good kept', async () => {
|
|
215
|
+
const world = await newWorld();
|
|
216
|
+
await writeSource(world, twentyCars());
|
|
217
|
+
await run(world);
|
|
218
|
+
world.allImagesBroken = true;
|
|
219
|
+
await blocked(run(world, { at: new Date(NOW.getTime() + HOUR) }), 'published_too_small');
|
|
220
|
+
assert.equal((await readAds(world)).length, 20);
|
|
221
|
+
assert.deepEqual((await report(world)).excluded, { [OoyyoExclusion.noImage]: 20 });
|
|
222
|
+
}],
|
|
223
|
+
|
|
224
|
+
['source outage / partial / stale snapshot: blocked before any output is touched', async () => {
|
|
225
|
+
const world = await newWorld();
|
|
226
|
+
await writeSource(world, twentyCars());
|
|
227
|
+
await run(world);
|
|
228
|
+
const good = await fs.readFile(world.xml, 'utf8');
|
|
229
|
+
// missing chunk (transport / base export failure mid-way)
|
|
230
|
+
await fs.rm(join(world.source, 'cars.1.json'));
|
|
231
|
+
await blocked(run(world), 'source_partial');
|
|
232
|
+
// empty snapshot
|
|
233
|
+
await fs.rm(world.source, { recursive: true, force: true });
|
|
234
|
+
await blocked(run(world), 'source_empty');
|
|
235
|
+
// stale snapshot (base export did not run for > 26 h)
|
|
236
|
+
await writeSource(world, twentyCars(), 5, new Date(NOW.getTime() - 27 * HOUR));
|
|
237
|
+
await blocked(run(world), 'source_stale');
|
|
238
|
+
assert.equal(await fs.readFile(world.xml, 'utf8'), good);
|
|
239
|
+
}],
|
|
240
|
+
|
|
241
|
+
['sold / deleted cars leave with the next run (normal sync); a collapse needs acceptDrop', async () => {
|
|
242
|
+
const world = await newWorld();
|
|
243
|
+
const cars = twentyCars();
|
|
244
|
+
await writeSource(world, cars);
|
|
245
|
+
await run(world);
|
|
246
|
+
// 2 of 20 sold / deleted: -10 %, published
|
|
247
|
+
await writeSource(world, cars.slice(2));
|
|
248
|
+
const r = await run(world, { at: new Date(NOW.getTime() + HOUR) });
|
|
249
|
+
assert.equal(r.items, 18);
|
|
250
|
+
assert.ok(!idsOf(await readAds(world)).includes('car00'));
|
|
251
|
+
// half of the stock gone at once: blocked
|
|
252
|
+
await writeSource(world, cars.slice(2, 11));
|
|
253
|
+
await blocked(run(world, { at: new Date(NOW.getTime() + 2 * HOUR) }), 'count_drop');
|
|
254
|
+
assert.equal((await readAds(world)).length, 18);
|
|
255
|
+
// a real stock reduction is confirmed once with acceptDrop
|
|
256
|
+
const accepted = await run(world, { at: new Date(NOW.getTime() + 3 * HOUR), guard: { acceptDrop: true } });
|
|
257
|
+
assert.equal(accepted.items, 9);
|
|
258
|
+
}],
|
|
259
|
+
|
|
260
|
+
['stale last good is reported; a concurrent run is locked out', async () => {
|
|
261
|
+
const world = await newWorld();
|
|
262
|
+
await writeSource(world, twentyCars());
|
|
263
|
+
await run(world);
|
|
264
|
+
world.allImagesBroken = true;
|
|
265
|
+
const later = new Date(NOW.getTime() + 49 * HOUR);
|
|
266
|
+
await writeSource(world, twentyCars(), 5, later);
|
|
267
|
+
await blocked(run(world, { at: later }), 'published_too_small');
|
|
268
|
+
const rep = await report(world);
|
|
269
|
+
assert.equal(rep.stale, true);
|
|
270
|
+
assert.ok(rep.lastGoodAgeHours > 48);
|
|
271
|
+
world.allImagesBroken = false;
|
|
272
|
+
await fs.writeFile(join(world.state, 'run.lock'), JSON.stringify({ runId: 'other' }));
|
|
273
|
+
await blocked(run(world, { at: later }), 'locked');
|
|
274
|
+
}],
|
|
275
|
+
];
|
|
276
|
+
|
|
277
|
+
(async () => {
|
|
278
|
+
let failed = 0;
|
|
279
|
+
for (const [name, fn] of tests) {
|
|
280
|
+
try {
|
|
281
|
+
await fn();
|
|
282
|
+
console.log(`ok - ${name}`);
|
|
283
|
+
} catch (error) {
|
|
284
|
+
failed += 1;
|
|
285
|
+
console.log(`not ok - ${name}\n${error?.stack ?? error}`);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
if (failed) {
|
|
289
|
+
console.log(`${failed} of ${tests.length} failed`);
|
|
290
|
+
process.exit(1);
|
|
291
|
+
}
|
|
292
|
+
console.log(`ooyyo-feed-run: ${tests.length} passed`);
|
|
293
|
+
})();
|