@bytebrand/export-cars 1.26.46 → 1.26.48

package/test/ooyyo-mapping.test.ts

@@ -0,0 +1,276 @@
+// Run: npm test (ts-node, no test framework needed)
+// OOYYO mapping, partner acceptance rules and XML structure (ADV-331), against the official
+// OoyyoFormat.xsd / OoyyoFormat.xml in test/fixtures/ooyyo.
+import * as assert from 'node:assert/strict';
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import { XMLParser } from 'fast-xml-parser';
+
+import { OoyyoXmlBuilder, cdata } from '../src/formats/ooyyo-xml.builder';
+import {
+  OOYYO_LINK_PARAMS, OoyyoExclusion, getOoyyoContent, getOoyyoDateTime, getOoyyoEuronorm, getOoyyoPostcode, getOoyyoPrice,
+  getOoyyoYear, mapOoyyoItem, validateOoyyoItem,
+} from '../src/transformers/ooyyo.transformer';
+import {
+  OOYYO_AD_ELEMENTS, OOYYO_LOCATION_ELEMENTS, OOYYO_UNIT_ELEMENTS, verifyOoyyoXml,
+} from '../src/services/ooyyo-feed-verify';
+import { NOW, car, translate } from './fixtures/ooyyo.fixture';
+
+const FIXTURES = join(__dirname, 'fixtures', 'ooyyo');
+const IMAGES = ['https://images.auto.de/carimage/x/hero/large', 'https://images.auto.de/carimage/x/side/large'];
+
+const map = (overrides: Record<string, any> = {}, ctx: Record<string, any> = {}) =>
+  mapOoyyoItem(car('abc-1', overrides), { domain: 'auto.de', images: IMAGES, translate, region: 'Sachsen', now: NOW, ...ctx });
+
+const okItem = (overrides: Record<string, any> = {}) => {
+  const result = map(overrides);
+  assert.equal(result.ok, true, JSON.stringify(result));
+  return (result as any).item;
+};
+
+const reason = (overrides: Record<string, any> = {}, ctx: Record<string, any> = {}) => {
+  const result = map(overrides, ctx);
+  assert.equal(result.ok, false);
+  return (result as any).reason;
+};
+
+// Element names / required unit attributes straight from the XSD file.
+const xsd = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@' })
+  .parse(readFileSync(join(FIXTURES, 'OoyyoFormat.xsd'), 'utf8'))['xsd:schema'];
+const xsdElements: any[] = xsd['xsd:element'];
+const refsOf = (name: string): string[] => {
+  const el = xsdElements.find(e => e['@name'] === name);
+  return [].concat(el['xsd:complexType']['xsd:choice']['xsd:element']).map((e: any) => e['@ref']);
+};
+const unitRequired = xsdElements
+  .filter(e => {
+    const attr = e?.['xsd:complexType']?.['xsd:simpleContent']?.['xsd:extension']?.['xsd:attribute'];
+    return attr && attr['@name'] === 'unit' && attr['@use'] === 'required';
+  })
+  .map(e => e['@name']);
+
+const tests: Array<[string, () => void]> = [
+  ['element / unit lists of the verifier are exactly those of OoyyoFormat.xsd', () => {
+    assert.deepEqual(refsOf('ooyyo'), ['ad']);
+    assert.deepEqual(refsOf('ad'), OOYYO_AD_ELEMENTS);
+    assert.deepEqual(refsOf('location'), OOYYO_LOCATION_ELEMENTS);
+    assert.deepEqual(refsOf('pictures'), ['picture']);
+    assert.deepEqual(unitRequired.sort(), [...OOYYO_UNIT_ELEMENTS].sort());
+  }],
+
+  ['full item: German copy, EUR integer price, units, string postcode with leading zero, Berlin date/time', () => {
+    const item = okItem();
+    assert.equal(item.url, `https://www.auto.de/search/vehicle/abc-1?${OOYYO_LINK_PARAMS}`);
+    assert.equal(item.title, 'Opel Astra 1.2 Turbo Edition');
+    assert.equal(item.content, 'Ausstattung\n* ABS\n* Klimaanlage\nFahrzeug in gutem Zustand – Prüfung bestanden.');
+    assert.equal(item.price, '15148');
+    assert.equal(item.priceUnit, 'EUR');
+    assert.equal(item.year, '2025');
+    assert.equal(item.mileage, '13000');
+    assert.equal(item.engine_size, '1199');
+    assert.equal(item.power, '85');
+    assert.equal(item.color, 'Rot');
+    assert.equal(item.body_type, 'Kombi');
+    assert.equal(item.fuel_type, 'Benzin');
+    assert.equal(item.transmission, 'Manuell');
+    assert.equal(item.euronorm, '6');
+    assert.deepEqual(item.location, { city: 'Leipzig', postcode: '04103', region: 'Sachsen' });
+    assert.equal(item.vin, 'WVWZZZ1KZAW000001');
+    assert.equal(item.is_new, '0');
+    assert.equal(item.date, '07/07/2026');
+    assert.equal(item.time, '09:13');
+    assert.deepEqual(item.pictures, IMAGES);
+  }],
+
+  ['clickout URL: partner in source + identical utm_*, no click id, no duplicate params', () => {
+    const url = new URL(okItem().url);
+    assert.equal(url.protocol, 'https:');
+    assert.equal(url.pathname, '/search/vehicle/abc-1');
+    const keys = [...url.searchParams.keys()];
+    assert.equal(new Set(keys).size, keys.length);
+    assert.deepEqual(Object.fromEntries(url.searchParams), {
+      source: 'ooyyo', medium: 'listing', campaign: 'listing', utm_source: 'ooyyo', utm_medium: 'listing', utm_campaign: 'listing',
+    });
+    assert.ok(!/click|gclid|fbclid|ooyyo_id/i.test(keys.join(',')));
+  }],
+
+  ['price: currentSalesPrice only, whole euros; leasing rate / dealer cost never used', () => {
+    assert.equal(getOoyyoPrice(26900), '26900');
+    assert.equal(getOoyyoPrice('26900'), '26900');
+    assert.equal(getOoyyoPrice(26900.4), '26900');
+    for (const bad of [null, undefined, '', 0, -1, 'abc', true]) {
+      assert.equal(getOoyyoPrice(bad), null);
+    }
+    assert.equal(okItem({ 'price.currentSalesPrice': 9990, 'rate.monthlyRate': 99 }).price, '9990');
+    assert.equal(reason({ 'price.currentSalesPrice': null, 'price.purchasePrice': 11000 }), OoyyoExclusion.noPrice);
+  }],
+
+  ['year: first registration, else construction year; never invented (legacy: now - 2 months)', () => {
+    assert.equal(getOoyyoYear(car('a'), NOW), '2025');
+    assert.equal(getOoyyoYear(car('a', { 'mainData.firstRegistration': null, 'mainData.constructionYear': 2026 }), NOW), '2026');
+    assert.equal(getOoyyoYear(car('a', { 'mainData.firstRegistration': 0, 'mainData.constructionYear': null }), NOW), '');
+    // ms instead of s => year far in the future, not used
+    assert.equal(getOoyyoYear(car('a', { 'mainData.firstRegistration': 1750809600000, 'mainData.constructionYear': null }), NOW), '');
+    assert.equal(reason({ 'mainData.condition': 'selector_condition_new', 'mainData.firstRegistration': null, 'mainData.constructionYear': null }),
+      OoyyoExclusion.noYear);
+    const newCar = okItem({ 'mainData.condition': 'selector_condition_new', 'mainData.firstRegistration': null, 'mainData.mileage': 0, 'mainData.constructionYear': 2026 });
+    assert.equal(newCar.year, '2026');
+    assert.equal(newCar.is_new, '1');
+    assert.equal(newCar.mileage, '0');
+  }],
+
+  ['make / model OOYYO cannot recognise are excluded (SONSTIGE fallback, empty)', () => {
+    assert.equal(reason({ 'mainData.model': 'SONSTIGE' }), OoyyoExclusion.noModel);
+    assert.equal(reason({ 'mainData.model': '' }), OoyyoExclusion.noModel);
+    assert.equal(reason({ 'mainData.make': '' }), OoyyoExclusion.noMake);
+    assert.equal(reason({ 'mainData.make': 'selector_unknown' }), OoyyoExclusion.noMake);
+  }],
+
+  ['photos: at least one HTTPS photo, URLs <= 1,500 chars; vehicle URL <= 1,500 chars', () => {
+    assert.equal(reason({}, { images: [] }), OoyyoExclusion.noImage);
+    assert.equal(reason({}, { images: ['http://images.auto.de/a.jpg'] }), OoyyoExclusion.noImage);
+    const p1500 = `https://images.auto.de/${'a'.repeat(1500 - 'https://images.auto.de/'.length)}`;
+    assert.equal(p1500.length, 1500);
+    assert.equal(map({}, { images: [p1500] }).ok, true);
+    assert.equal(reason({}, { images: [`${p1500}b`] }), OoyyoExclusion.urlTooLong);
+    const fixed = `https://www.auto.de/search/vehicle/?${OOYYO_LINK_PARAMS}`.length;
+    assert.equal(map({ _id: 'x'.repeat(1500 - fixed) }).ok, true);
+    assert.equal(reason({ _id: 'x'.repeat(1501 - fixed) }), OoyyoExclusion.urlTooLong);
+  }],
+
+  ['reserved / sold are not listed', () => {
+    assert.equal(reason({ 'offer.offerAvailability': 'selector_offerAvailability_sold' }), OoyyoExclusion.notAvailable);
+    assert.equal(reason({ 'offer.offerAvailability': 'selector_offerAvailability_reserved' }), OoyyoExclusion.notAvailable);
+  }],
+
+  ['postcode as string: leading zero restored for DE, invalid dropped, foreign kept', () => {
+    assert.equal(getOoyyoPostcode(4103, 'DE'), '04103');
+    assert.equal(getOoyyoPostcode('04103', 'DE'), '04103');
+    assert.equal(getOoyyoPostcode(10115, 'de'), '10115');
+    assert.equal(getOoyyoPostcode(1, 'DE'), '');
+    assert.equal(getOoyyoPostcode(999, null), '');
+    assert.equal(getOoyyoPostcode(null, 'DE'), '');
+    assert.equal(getOoyyoPostcode('1010', 'AT'), '1010');
+  }],
+
+  ['optional fields: unknown / 0 / invalid are left out, never "default" / "undefined" / fake VIN', () => {
+    assert.equal(getOoyyoEuronorm('selector_unknown'), '');
+    assert.equal(getOoyyoEuronorm('selector_emissionClass_euro6d_temp'), '6d-TEMP');
+    const item = okItem({
+      'mainData.doors': 0, 'driveSuspension.gears': null, 'engineData.cubicCapacity': null, 'metaData.vin': 'FAKEVIN',
+      'mainData.category': 'selector_unknown', 'mainData.condition': null, 'offer.offerDescription': null, 'metaData.creationDate': null,
+    });
+    for (const field of ['doors', 'gears', 'engine_size', 'vin', 'body_type', 'is_new', 'content', 'date', 'time']) {
+      assert.equal(item[field], '', field);
+    }
+    const xml = OoyyoXmlBuilder.toXml(item);
+    for (const tag of ['doors', 'gears', 'engine_size', 'vin', 'body_type', 'is_new', 'content', 'date', 'time']) {
+      assert.ok(!xml.includes(`<${tag}`), `${tag} left out`);
+    }
+    assert.ok(!/undefined|null|default/.test(xml));
+  }],
+
+  ['date / time: DD/MM/YYYY and HH:MM (24 h) in Europe/Berlin, summer and winter', () => {
+    assert.deepEqual(getOoyyoDateTime(1783408438), { date: '07/07/2026', time: '09:13' });
+    assert.deepEqual(getOoyyoDateTime(Date.parse('2026-01-15T22:30:00Z') / 1000), { date: '15/01/2026', time: '23:30' });
+    assert.deepEqual(getOoyyoDateTime(Date.parse('2026-03-31T22:05:00Z') / 1000), { date: '01/04/2026', time: '00:05' });
+    assert.deepEqual(getOoyyoDateTime(null), { date: '', time: '' });
+  }],
+
+  ['content: mobile.de markup to plain text, umlauts intact, nothing added', () => {
+    assert.equal(getOoyyoContent('**Top**\\\\Größe: L\\\\\\\\\\\\\\\\Ende'), 'Top\nGröße: L\n\nEnde');
+    assert.equal(getOoyyoContent('   '), '');
+    assert.equal(getOoyyoContent(undefined), '');
+  }],
+
+  ['XML: CDATA-safe ("]]>" split, invalid control chars removed), UTF-8 umlauts, & unescaped in CDATA', () => {
+    assert.equal(cdata('a]]>b'), '<![CDATA[a]]]]><![CDATA[>b]]>');
+    assert.equal(cdata('a\u0001b\u000Bc\td'), '<![CDATA[abc\td]]>');
+    const item = okItem({ 'mainData.make': 'Škoda', 'mainData.model': 'Octavia', 'offer.offerDescription': 'Größe ]]> Ölwechsel\u0007 & Co' });
+    const xml = `<?xml version="1.0" encoding="UTF-8"?>\n<ooyyo>\n${OoyyoXmlBuilder.toXml(item)}</ooyyo>\n`;
+    const parsed = new XMLParser({ ignoreAttributes: false, parseTagValue: false }).parse(xml);
+    assert.equal(parsed.ooyyo.ad.content, 'Größe ]]> Ölwechsel & Co');
+    assert.equal(parsed.ooyyo.ad.make, 'Škoda');
+    assert.equal(parsed.ooyyo.ad.url, item.url);
+    assert.deepEqual(verifyOoyyoXml(xml, 1).issues, []);
+  }],
+
+  ['XML structure: root as in the partner sample, price unit EUR, units on every unit element', () => {
+    const item = okItem();
+    const ad = OoyyoXmlBuilder.toXml(item);
+    assert.ok(ad.includes('<price unit="EUR"><![CDATA[15148]]></price>'));
+    assert.ok(ad.includes('<engine_size unit="cc">'));
+    assert.ok(ad.includes('<mileage unit="km">'));
+    assert.ok(ad.includes('<power unit="kw">'));
+    assert.ok(!ad.includes('unit="euro"'));
+    const sample = readFileSync(join(FIXTURES, 'OoyyoFormat.xml'), 'utf8');
+    assert.ok(sample.includes('<ooyyo xsi:noNamespaceSchemaLocation="OoyyoFormat.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">'));
+    // element order as in the sample
+    const order = [...ad.matchAll(/^ {4}<([a-z_]+)/gm)].map(m => m[1]);
+    const sampleOrder = [...sample.split('</ad>')[0].matchAll(/^ {8}<([a-z_]+)/gm)].map(m => m[1]);
+    assert.deepEqual(order, sampleOrder.filter(t => order.includes(t)));
+  }],
+
+  ['verifier rejects the legacy / partner-sample defects (unit euro, empty <power/>, noimage-only, no year)', () => {
+    const legacy = `<?xml version="1.0" encoding="UTF-8"?>
+    <ooyyo xsi:noNamespaceSchemaLocation="OoyyoFormat.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<ad>
+<url><![CDATA[https://www.autode-dev.de/search/vehicle/42c3?source=classifieds&medium=listing&campaign=ooyyo]]></url>
+<content/>
+<price unit="euro"><![CDATA[13348]]></price>
+<make><![CDATA[MG]]></make>
+<model><![CDATA[MG3]]></model>
+<engine_size/>
+<year><![CDATA[2025]]></year>
+<euronorm><![CDATA[default]]></euronorm>
+<pictures>
+<picture><![CDATA[https://images.auto.de/carimage/noimage/7pNvWELuWhZX/large-cached.jpeg]]></picture>
+</pictures>
+</ad>
+<ad>
+<url><![CDATA[https://www.autode-dev.de/search/vehicle/42c4]]></url>
+<price unit="EUR"><![CDATA[9000]]></price>
+<make><![CDATA[MG]]></make>
+<model><![CDATA[ZS]]></model>
+<pictures><picture><![CDATA[https://images.auto.de/a.jpg]]></picture></pictures>
+</ad>
+</ooyyo>`;
+    const { ads, issues } = verifyOoyyoXml(legacy, 2);
+    assert.equal(ads, 2);
+    const problems = issues.map(i => `${i.ad}:${i.problem}`);
+    assert.ok(problems.some(p => p.startsWith('0:<price> unit="euro"')), problems.join('\n'));
+    assert.ok(problems.some(p => p.startsWith('0:<engine_size> unit=""')), problems.join('\n'));
+    assert.ok(problems.some(p => p === `0:rejected by partner rules: ${OoyyoExclusion.noPrice}`), problems.join('\n'));
+    assert.ok(problems.some(p => p === `1:rejected by partner rules: ${OoyyoExclusion.noYear}`), problems.join('\n'));
+    // empty unit-less / broken / foreign root files
+    assert.equal(verifyOoyyoXml('<ooyyo><ad></ooyyo>', 0).issues[0].ad, -1);
+    assert.match(verifyOoyyoXml('<feed></feed>', 0).issues[0].problem, /root element/);
+    assert.match(verifyOoyyoXml('<ooyyo></ooyyo>', 3).issues[0].problem, /holds 0 ads, the run wrote 3/);
+  }],
+
+  ['validateOoyyoItem: the partner minimum (photo, year, price EUR, make, model, url)', () => {
+    const item = okItem();
+    assert.equal(validateOoyyoItem(item), null);
+    assert.equal(validateOoyyoItem({ ...item, priceUnit: 'euro' as any }), OoyyoExclusion.noPrice);
+    assert.equal(validateOoyyoItem({ ...item, price: '0' }), OoyyoExclusion.noPrice);
+    assert.equal(validateOoyyoItem({ ...item, year: '' }), OoyyoExclusion.noYear);
+    assert.equal(validateOoyyoItem({ ...item, pictures: [] }), OoyyoExclusion.noImage);
+    assert.equal(validateOoyyoItem({ ...item, url: 'http://www.auto.de/x' }), OoyyoExclusion.noId);
+  }],
+];
+
+let failed = 0;
+for (const [name, fn] of tests) {
+  try {
+    fn();
+    console.log(`ok - ${name}`);
+  } catch (error) {
+    failed += 1;
+    console.log(`not ok - ${name}\n${error?.stack ?? error}`);
+  }
+}
+if (failed) {
+  console.log(`${failed} of ${tests.length} failed`);
+  process.exit(1);
+}
+console.log(`ooyyo-mapping: ${tests.length} passed`);