@bytebrand/export-cars 1.26.29 → 1.26.30

package/docs/openai-ads/AUDIT.md

@@ -128,7 +128,7 @@ Null policy "EXCLUDE" = row not written + counted in the run report under the re
 | OpenAI field | Req. | Source path | Transform / format | Null / invalid policy | Test case |
 |---|---|---|---|---|---|
 | `item_id` | R | `car._id` | string as-is (UUID) | missing → EXCLUDE `no_id`; duplicate in run → keep first, count `duplicate_id` | dup id; id equals GA4 `item_id` |
-| `title` | R | `mainData.make`, `mainData.model`, `mainData.subModel` | `normalizeFeedText(make model subModel)`, German as delivered, ≤150 at word boundary, no HTML escape | make or model empty → EXCLUDE `no_title` | umlauts, `*AHK*Leder*`, `&`, 200-char subModel |
+| `title` | R | `mainData.make`, `mainData.model`, `mainData.subModel` | `normalizeFeedText(make model subModel)`, German as delivered, ≤150 at word boundary, no HTML escape; placeholder model `SONSTIGE` left out; UTF-8-read-as-Latin-1/cp1252 (`ü`) repaired (1.26.30) | neither a real model nor a variant → EXCLUDE `no_title` | umlauts, `*AHK*Leder*`, `&`, 200-char subModel |
 | `description` | R | structured fields only — NOT the dealer summary templates of the Google feed (free text, finance claims unverifiable, Q6) | `Gebrauchtwagen: <title>. Erstzulassung: 03/2021 · Kilometerstand: 45.000 km · Kraftstoff: Benzin · Getriebe: Manuell · Leistung: 110 kW (150 PS) · Karosserie · Außenfarbe`, ≤5000, real mileage (no 901/1001 floors) | facts without a German label are left out | sparse car, no `selector_*` leaks |
 | `url` | R | `car._id` | `https://www.{domain}/search/vehicle/{_id}` (same as Google/fb; no UTM — tracking via ADV-337) | always set | url host per env |
 | `brand` | R | `mainData.make` | raw manufacturer name (NOT Google's "Mercedes"/"VW" rewrite) | empty → EXCLUDE | Mercedes-Benz stays Mercedes-Benz |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bytebrand/export-cars",
-  "version": "1.26.29",
+  "version": "1.26.30",
   "description": "",
   "author": "bytebrand",
   "license": "UNLICENSED",

package/src/transformers/openai-ads.transformer.ts

@@ -14,9 +14,24 @@ import { OpenAiImageFields, encodeCommasInUrl } from './openai-image-fields';
 export const OPENAI_TITLE_MAX_LENGTH = 150;
 export const OPENAI_DESCRIPTION_MAX_LENGTH = 5000;
 
-// Placeholder makes from import mapping (SONSTIGE = "other") — not a real manufacturer, so the
-// required `brand` would be invented.
-const PLACEHOLDER_MAKES = /^(sonstige|sonstiges|andere|other|unknown|unbekannt)$/i;
+// Placeholder make/model values from import mapping (SONSTIGE = "other"). As `brand` the value would
+// be invented (car excluded); as model it is left out of title and description.
+const PLACEHOLDER_NAMES = /^(sonstige|sonstiges|andere|other|unknown|unbekannt)$/i;
+
+// UTF-8 text that was decoded as Latin-1 / Windows-1252 somewhere upstream ("Navi über" for
+// "Navi über", "ß" for "ß", "„" for "„"). Continuation bytes 0x80–0xBF show up as the Latin-1
+// character of the same code, or — for 0x80–0x9F — as the Windows-1252 character below.
+const CP1252_HIGH: Record<string, number> = {
+  '€': 0x80, '‚': 0x82, 'ƒ': 0x83, '„': 0x84, '…': 0x85, '†': 0x86,
+  '‡': 0x87, 'ˆ': 0x88, '‰': 0x89, 'Š': 0x8a, '‹': 0x8b, 'Œ': 0x8c,
+  'Ž': 0x8e, '‘': 0x91, '’': 0x92, '“': 0x93, '”': 0x94, '•': 0x95,
+  '–': 0x96, '—': 0x97, '˜': 0x98, '™': 0x99, 'š': 0x9a, '›': 0x9b,
+  'œ': 0x9c, 'ž': 0x9e, 'Ÿ': 0x9f,
+};
+const CONTINUATION = `[\\u0080-\\u00BF${Object.keys(CP1252_HIGH).join('')}]`;
+// a C2–C5 lead + one continuation (Latin-1 Supplement / Latin Extended-A), or E2 + two
+// (general punctuation, €)
+const MOJIBAKE_SEQUENCE = new RegExp(`[\\u00C2-\\u00C5]${CONTINUATION}|\\u00E2${CONTINUATION}{2}`, 'g');
 
 const OFFER_AVAILABILITY_OUT_OF_STOCK: Record<string, OpenAiAdsOutOfStockReason> = {
   selector_offerAvailability_reserved: OpenAiAdsOutOfStockReason.reserved,
@@ -55,8 +70,8 @@ export const mapOpenAiAdsVehicle = (car: DataSchemes.Car, ctx: OpenAiAdsMappingC
     return { ok: false, reason: OpenAiAdsExclusion.invalidPrice };
   }
 
-  const brand = trim(toString(get(car, 'mainData.make', '')));
-  if (!brand || PLACEHOLDER_MAKES.test(brand)) {
+  const brand = repairMojibake(trim(toString(get(car, 'mainData.make', ''))));
+  if (!brand || PLACEHOLDER_NAMES.test(brand)) {
     return { ok: false, reason: OpenAiAdsExclusion.noBrand };
   }
 
@@ -124,13 +139,30 @@ export const toOpenAiMinorUnits = (value: unknown): number | null => {
 
 // "<make> <model> <variant>" as delivered (German market names), dealer junk punctuation removed
 // with the same normalizer as the Meta feed, cut at a word boundary. No HTML escaping: CSV quoting
-// handles every character.
+// handles every character. A placeholder model (SONSTIGE) is left out ("MG 1.5 Hybrid Luxury");
+// with neither a real model nor a variant there is no title and the car is not sent.
 export const buildOpenAiTitle = (car: DataSchemes.Car): string => {
-  const parts = ['mainData.make', 'mainData.model', 'mainData.subModel'].map(path => toString(get(car, path, '')));
-  const title = normalizeFeedText(parts.join(' '));
+  const [make, model, subModel] = ['mainData.make', 'mainData.model', 'mainData.subModel']
+    .map(path => repairMojibake(trim(toString(get(car, path, '')))));
+  const realModel = PLACEHOLDER_NAMES.test(model) ? '' : model;
+  if (!normalizeFeedText(`${realModel} ${subModel}`)) {
+    return '';
+  }
+  const title = normalizeFeedText([make, realModel, subModel].join(' '));
   return truncate(title, { length: OPENAI_TITLE_MAX_LENGTH, separator: ' ', omission: '' }).trim();
 };
 
+// Repairs only sequences that decode to one character of the expected ranges (Latin-1 Supplement,
+// Latin Extended-A, general punctuation, €), so regular German text ("ä°", "ß") is never touched.
+export const repairMojibake = (text: string): string =>
+  (text || '').replace(MOJIBAKE_SEQUENCE, (sequence) => {
+    const bytes = [...sequence].map(char => CP1252_HIGH[char] ?? char.charCodeAt(0));
+    const decoded = Buffer.from(bytes).toString('utf8');
+    const code = decoded.length === 1 ? decoded.charCodeAt(0) : -1;
+    const expected = (code >= 0x00a0 && code <= 0x017f) || (code >= 0x2010 && code <= 0x20ac);
+    return expected ? decoded : sequence;
+  });
+
 // new/used from mainData.condition. selector_unknown is only "used" when the car has a first
 // registration in the past (a registered car is used for advertising purposes); otherwise
 // the car is excluded rather than defaulted — OpenAI treats an omitted condition as "new".

package/test/openai-ads-mapping.test.ts

@@ -6,6 +6,7 @@ import {
   asOutOfStock,
   formatOpenAiPrice,
   mapOpenAiAdsVehicle,
+  repairMojibake,
   toOpenAiMinorUnits,
   withOpenAiImages,
 } from '../src/transformers/openai-ads.transformer';
@@ -69,6 +70,26 @@ const cases: Array<[string, () => void]> = [
     const long = okRow(car('a', { 'mainData.subModel': 'Ausstattung '.repeat(30) })).title;
     assert.ok(long.length <= 150 && !long.endsWith(' '), long);
   }],
+  ['title: placeholder model SONSTIGE is left out of title and description; nothing left -> excluded', () => {
+    const row = okRow(car('a', { 'mainData.make': 'MG', 'mainData.model': 'SONSTIGE', 'mainData.subModel': '1.5 Hybrid Luxury' }));
+    assert.equal(row.title, 'MG 1.5 Hybrid Luxury');
+    assert.ok(!/sonstige/i.test(row.description), row.description);
+    assert.equal(okRow(car('a', { 'mainData.model': 'Sonstige', 'mainData.subModel': 'Golf' })).title, 'Volkswagen Golf');
+    assert.equal(reason(car('a', { 'mainData.model': 'SONSTIGE', 'mainData.subModel': '' })), X.noTitle);
+    assert.equal(reason(car('a', { 'mainData.model': '', 'mainData.subModel': '***' })), X.noTitle);
+    // a real model that merely contains the word stays as it is
+    assert.equal(okRow(car('a', { 'mainData.model': 'Sonstige Modelle 2', 'mainData.subModel': '' })).title, 'Volkswagen Sonstige Modelle 2');
+  }],
+  ['title: UTF-8 read as Latin-1 upstream is repaired, regular umlauts untouched', () => {
+    assert.equal(repairMojibake('AHK/Navi über SmartLink'), 'AHK/Navi über SmartLink');
+    assert.equal(repairMojibake('Größe Äußeres'), 'Größe Äußeres');
+    assert.equal(repairMojibake('Sitzheizung – Paket „1“'), 'Sitzheizung – Paket „1“');
+    assert.equal(repairMojibake('Größe Äußeres 360° Škoda Č'), 'Größe Äußeres 360° Škoda Č');
+    assert.equal(repairMojibake(''), '');
+    const row = okRow(car('a', { 'mainData.make': 'Skoda', 'mainData.model': 'Octavia', 'mainData.subModel': 'Navi über SmartLink' }));
+    assert.equal(row.title, 'Skoda Octavia Navi über SmartLink');
+    assert.ok(row.description.includes('Navi über SmartLink'));
+  }],
   ['condition: new/used; unknown only used with a past first registration', () => {
     assert.equal(okRow(car('a', { 'mainData.condition': 'selector_condition_new' })).condition, 'new');
     assert.equal(okRow(car('a')).condition, 'used');