@bytebrand/export-cars 1.26.46 → 1.26.48

package/src/services/ooyyo-feed-run.ts

@@ -0,0 +1,265 @@
+import { createReadStream, createWriteStream, promises as fs } from 'fs';
+import { dirname, join } from 'path';
+import { pipeline } from 'stream/promises';
+import { createGzip, gunzipSync } from 'zlib';
+import { createHash } from 'crypto';
+import { ImboImageResponseInterface } from '@bytebrand/chunkable';
+import { DataSchemes } from '@bytebrand/car-schema';
+
+import { OoyyoXmlItem } from '../interfaces';
+import { OoyyoXmlBuilder } from '../formats/ooyyo-xml.builder';
+import { OoyyoExclusion, getOoyyoExclusion, getOoyyoPostcode, mapOoyyoItem } from '../transformers/ooyyo.transformer';
+import { FeedGuard, FeedGuardError, LastGood, inspectSourceSnapshot } from './feed-guard';
+import { verifyOoyyoXml } from './ooyyo-feed-verify';
+
+export interface OoyyoSourceCar {
+  car: DataSchemes.Car;
+  images: ImboImageResponseInterface[];
+}
+
+// Everything the run needs from the outside world — the Nest exporter wires the real services,
+// tests pass fakes.
+export interface OoyyoRunDeps {
+  // iterates the base export snapshot chunk by chunk (LoaderService.process)
+  forEachChunk: (fn: (cars: OoyyoSourceCar[]) => Promise<void>) => Promise<unknown>;
+  // existing shared partner filter (images count, Cronicle rules)
+  filterCar: (source: OoyyoSourceCar) => boolean;
+  // final validated image URLs for the car, hero first; [] = no usable image
+  resolveImages: (source: OoyyoSourceCar) => Promise<string[]>;
+  // German label of a selector, '' when unknown
+  translate: (selector: string) => string;
+  // Bundesland of a German postcode, '' when unknown
+  region: (postcode: string, countryCode: unknown) => Promise<string>;
+  imageStats: () => Record<string, number>;
+  guard: FeedGuard;
+}
+
+export interface OoyyoRunOptions {
+  runId?: string;
+  dryRun: boolean;
+  domain: string;
+  sourceDirectory: string;
+  sourceBaseFileName: string;
+  xmlFile: string;
+  gzFile: string;
+  stateDir: string;
+  now?: () => Date;
+}
+
+export interface OoyyoRunReport {
+  runId?: string;
+  mode: 'dry-run' | 'live';
+  startedAt: string;
+  finishedAt?: string;
+  status: 'published' | 'blocked' | 'failed';
+  guard?: { code: string; message: string };
+  error?: string;
+  xmlFile: string;
+  gzFile: string;
+  source: { files: number; newestChunkAt: string | null; cars: number };
+  items: number;
+  excluded: Record<string, number>;
+  // optional fields left out because the source had no (valid) value
+  missing: Record<string, number>;
+  postcodePadded: number;
+  verify?: { ads: number; issues: number; firstIssues: Array<{ ad: number; problem: string }> };
+  bytes?: { xml: number; gz: number; sha256: string };
+  imagesPerItem: { min: number | null; max: number | null; total: number };
+  images: Record<string, number>;
+  lastGood: LastGood | null;
+  lastGoodAgeHours: number | null;
+  stale: boolean;
+}
+
+const increment = (counter: Record<string, number>, key: string) => {
+  counter[key] = (counter[key] ?? 0) + 1;
+};
+
+const OPTIONAL_FIELDS = ['content', 'engine_size', 'body_type', 'color', 'mileage', 'fuel_type', 'transmission', 'euronorm',
+  'power', 'doors', 'seats', 'gears', 'vin', 'is_new', 'date'] as const;
+
+const exists = (file: string) => fs.stat(file).then(() => true, () => false);
+
+/**
+ * One OOYYO feed run (ADV-331): source check -> map + semantic partner rules + validated images ->
+ * temp XML -> re-read + verify the whole temp file -> count check -> gzip + verify -> atomic publish
+ * of .xml.gz and .xml -> report.
+ *
+ * OOYYO pulls the file once a day and REPLACES its whole inventory with whatever it can read; a
+ * partly valid file shrinks the live listing to that part (partner, 19.10.2026). So nothing is
+ * published unless the snapshot is complete and fresh (FeedGuard.checkSource), every written ad
+ * passes the partner rules when read back (verifyOoyyoXml), and the count did not collapse against the
+ * last good run (FeedGuard.checkCounts). A blocked run leaves the last good files in place (OOYYO keeps
+ * reading them) and fails the Cronicle job. Nothing outside the output directory and `stateDir` is
+ * written; the legacy flow emptied the output directory first and tarred with the `zip` binary.
+ */
+export class OoyyoFeedRun {
+  constructor(private readonly deps: OoyyoRunDeps, private readonly options: OoyyoRunOptions) { }
+
+  private now(): Date {
+    return this.options.now ? this.options.now() : new Date();
+  }
+
+  async run(): Promise<OoyyoRunReport> {
+    const { guard } = this.deps;
+    const { xmlFile, gzFile, runId, dryRun } = this.options;
+    const startedAt = this.now();
+    const lastGood = await guard.readLastGood();
+    const report: OoyyoRunReport = {
+      runId,
+      mode: dryRun ? 'dry-run' : 'live',
+      startedAt: startedAt.toISOString(),
+      status: 'failed',
+      xmlFile,
+      gzFile,
+      source: { files: 0, newestChunkAt: null, cars: 0 },
+      items: 0,
+      excluded: {},
+      missing: {},
+      postcodePadded: 0,
+      imagesPerItem: { min: null, max: null, total: 0 },
+      images: {},
+      lastGood,
+      lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
+      stale: guard.isStale(lastGood, startedAt),
+    };
+
+    await guard.acquireLock(runId ?? 'local', startedAt);
+    const tmpXml = `${xmlFile}.tmp`;
+    const tmpGz = gzFile.replace(/\.gz$/, '.tmp.gz');
+    try {
+      const snapshot = await inspectSourceSnapshot(this.options.sourceDirectory, this.options.sourceBaseFileName);
+      report.source.files = snapshot.files;
+      report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
+      guard.checkSource(snapshot, startedAt);
+
+      await fs.mkdir(dirname(xmlFile), { recursive: true });
+      await fs.rm(tmpXml, { force: true });
+      await fs.rm(tmpGz, { force: true });
+      const builder = new OoyyoXmlBuilder(tmpXml);
+      await builder.buildStart();
+
+      const seen = new Set<string>();
+      try {
+        await this.deps.forEachChunk(async (cars) => {
+          for (const source of cars) {
+            report.source.cars += 1;
+            const id = String(source?.car?._id ?? '');
+            if (id && seen.has(id)) {
+              increment(report.excluded, OoyyoExclusion.duplicateId);
+              continue;
+            }
+            seen.add(id);
+            if (!this.deps.filterCar(source)) {
+              increment(report.excluded, OoyyoExclusion.filtered);
+              continue;
+            }
+            // cheap checks first: no image probe for a car that is excluded anyway
+            const excluded = getOoyyoExclusion(source.car, startedAt);
+            if (excluded) {
+              increment(report.excluded, excluded);
+              continue;
+            }
+            const images = await this.deps.resolveImages(source);
+            if (!images.length) {
+              increment(report.excluded, OoyyoExclusion.noImage);
+              continue;
+            }
+            const country = source.car?.location?.countryCode;
+            const postcode = getOoyyoPostcode(source.car?.location?.zipCode, country);
+            const mapped = mapOoyyoItem(source.car, {
+              domain: this.options.domain,
+              images,
+              translate: this.deps.translate,
+              region: await this.deps.region(postcode, country),
+              now: startedAt,
+            });
+            if (mapped.ok === false) {
+              increment(report.excluded, mapped.reason);
+              continue;
+            }
+            await builder.addItem(mapped.item);
+            this.count(report, mapped.item, String(source.car?.location?.zipCode ?? ''));
+          }
+        });
+      } finally {
+        await builder.buildEnd();
+      }
+      report.images = this.deps.imageStats();
+
+      // read the file back exactly as it will be published
+      const verify = verifyOoyyoXml(await fs.readFile(tmpXml, 'utf8'), report.items);
+      report.verify = { ads: verify.ads, issues: verify.issues.length, firstIssues: verify.issues.slice(0, 20) };
+      if (verify.issues.length) {
+        throw new FeedGuardError('verify_failed', `${verify.issues.length} problem(s) in the written XML, first: ${verify.issues[0].problem}`);
+      }
+
+      const counts = { sourceItems: report.source.cars, publishedItems: report.items };
+      guard.checkCounts(counts, lastGood);
+
+      const xmlBytes = await fs.readFile(tmpXml);
+      await pipeline(createReadStream(tmpXml), createGzip({ level: 9 }), createWriteStream(tmpGz));
+      const gzBytes = await fs.readFile(tmpGz);
+      if (!gunzipSync(gzBytes).equals(xmlBytes)) {
+        throw new Error('gzip round trip does not match the XML');
+      }
+      report.bytes = { xml: xmlBytes.length, gz: gzBytes.length, sha256: createHash('sha256').update(xmlBytes).digest('hex') };
+
+      // .gz first, then the .xml through the guard (which records last-good): both renames are atomic
+      // and milliseconds apart; whichever URL OOYYO reads always holds a complete, verified snapshot.
+      if (await exists(gzFile)) {
+        await fs.copyFile(gzFile, `${gzFile}.prev`);
+      }
+      await fs.rename(tmpGz, gzFile);
+      report.lastGood = await guard.publish(tmpXml, xmlFile, counts, this.now(), runId);
+      report.status = 'published';
+      report.lastGoodAgeHours = 0;
+      report.stale = false;
+      return report;
+    } catch (error) {
+      if (error instanceof FeedGuardError) {
+        report.status = 'blocked';
+        report.guard = { code: error.code, message: error.message };
+      } else {
+        report.error = error?.message ?? String(error);
+      }
+      // keep the rejected file for inspection; the published files stay untouched
+      await fs.rename(tmpXml, `${xmlFile}.rejected`).catch(() => undefined);
+      await fs.rm(tmpGz, { force: true }).catch(() => undefined);
+      throw error;
+    } finally {
+      await this.writeReport(report);
+      await guard.releaseLock();
+    }
+  }
+
+  private count(report: OoyyoRunReport, item: OoyyoXmlItem, rawZip: string): void {
+    report.items += 1;
+    for (const field of OPTIONAL_FIELDS) {
+      if (!item[field]) {
+        increment(report.missing, field);
+      }
+    }
+    if (!item.location.postcode) {
+      increment(report.missing, 'postcode');
+    }
+    if (!item.location.region) {
+      increment(report.missing, 'region');
+    }
+    if (item.location.postcode && item.location.postcode !== rawZip) {
+      report.postcodePadded += 1;
+    }
+    const images = item.pictures.length;
+    const stats = report.imagesPerItem;
+    stats.min = stats.min === null ? images : Math.min(stats.min, images);
+    stats.max = stats.max === null ? images : Math.max(stats.max, images);
+    stats.total += images;
+  }
+
+  private async writeReport(report: OoyyoRunReport): Promise<void> {
+    report.finishedAt = this.now().toISOString();
+    await fs.mkdir(this.options.stateDir, { recursive: true });
+    await fs.writeFile(join(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
+    console.log(`ooyyo feed report: ${JSON.stringify(report)}`);
+  }
+}

package/src/services/ooyyo-feed-verify.ts

@@ -0,0 +1,131 @@
+import { XMLParser, XMLValidator } from 'fast-xml-parser';
+
+import { OoyyoXmlItem } from '../interfaces';
+import { OOYYO_PRICE_UNIT, validateOoyyoItem } from '../transformers/ooyyo.transformer';
+
+/**
+ * Element names and unit attributes of the official OoyyoFormat.xsd (test/fixtures/ooyyo; the test
+ * suite re-derives both lists from the XSD file, so a schema change fails the build).
+ */
+export const OOYYO_AD_ELEMENTS = [
+  'url', 'title', 'content', 'price', 'make', 'model', 'engine_size', 'trim', 'body_type', 'color', 'year',
+  'mileage', 'fuel_type', 'transmission', 'euronorm', 'power', 'doors', 'seats', 'gears', 'location',
+  'pictures', 'vin', 'is_new', 'date', 'time',
+];
+export const OOYYO_LOCATION_ELEMENTS = ['city', 'postcode', 'region'];
+// elements whose `unit` attribute is use="required"
+export const OOYYO_UNIT_ELEMENTS = ['price', 'engine_size', 'mileage', 'power'];
+// unit values from the partner sample / partner mail (price: EUR only, 19.10.2026)
+export const OOYYO_UNITS: Record<string, string[]> = {
+  price: [OOYYO_PRICE_UNIT],
+  engine_size: ['cc', 'l'],
+  mileage: ['km', 'mi'],
+  power: ['kw', 'hp'],
+};
+
+export interface OoyyoVerifyIssue {
+  ad: number;
+  problem: string;
+}
+
+export interface OoyyoVerifyResult {
+  ads: number;
+  issues: OoyyoVerifyIssue[];
+}
+
+const TEXT = '#text';
+
+const valueOf = (node: unknown): string => {
+  if (node === null || node === undefined) {
+    return '';
+  }
+  if (typeof node === 'object') {
+    return String((node as Record<string, unknown>)[TEXT] ?? '');
+  }
+  return String(node);
+};
+
+const asList = (node: unknown): unknown[] => (Array.isArray(node) ? node : node === undefined ? [] : [node]);
+
+/**
+ * Re-reads the finished XML (before it is published) the way an importer would and checks the whole
+ * snapshot: well-formed, root <ooyyo>, only XSD elements, required unit attributes with a known value,
+ * every <ad> passes the partner's semantic rules, no duplicate vehicle URL, and exactly `expectedAds`
+ * ads. A file that fails any of this is never published — OOYYO replaces its whole inventory with the
+ * readable subset of a partly valid file (partner, 19.10.2026).
+ */
+export const verifyOoyyoXml = (xml: string, expectedAds: number): OoyyoVerifyResult => {
+  const issues: OoyyoVerifyIssue[] = [];
+  const wellFormed = XMLValidator.validate(xml);
+  if (wellFormed !== true) {
+    return { ads: 0, issues: [{ ad: -1, problem: `not well-formed: ${wellFormed.err.msg} (line ${wellFormed.err.line})` }] };
+  }
+  const parser = new XMLParser({
+    ignoreAttributes: false,
+    attributeNamePrefix: '@',
+    parseTagValue: false,
+    parseAttributeValue: false,
+    trimValues: false,
+    isArray: name => name === 'ad' || name === 'picture',
+  });
+  const doc = parser.parse(xml);
+  const rootNames = Object.keys(doc).filter(k => k !== '?xml');
+  if (rootNames.length !== 1 || rootNames[0] !== 'ooyyo') {
+    return { ads: 0, issues: [{ ad: -1, problem: `root element is ${rootNames.join(',') || 'missing'}, expected ooyyo` }] };
+  }
+  const ads = asList(doc.ooyyo?.ad) as Array<Record<string, unknown>>;
+  const urls = new Set<string>();
+
+  ads.forEach((ad, index) => {
+    for (const key of Object.keys(ad)) {
+      if (key.startsWith('@') || key === TEXT) {
+        continue;
+      }
+      if (!OOYYO_AD_ELEMENTS.includes(key)) {
+        issues.push({ ad: index, problem: `element <${key}> is not in OoyyoFormat.xsd` });
+      }
+      if (Array.isArray(ad[key])) {
+        issues.push({ ad: index, problem: `element <${key}> appears more than once` });
+      }
+    }
+    for (const tag of OOYYO_UNIT_ELEMENTS) {
+      if (ad[tag] === undefined) {
+        continue;
+      }
+      const unit = typeof ad[tag] === 'object' ? String((ad[tag] as Record<string, unknown>)['@unit'] ?? '') : '';
+      if (!OOYYO_UNITS[tag].includes(unit)) {
+        issues.push({ ad: index, problem: `<${tag}> unit="${unit}" (allowed: ${OOYYO_UNITS[tag].join(', ')})` });
+      }
+    }
+    const location = (ad.location ?? {}) as Record<string, unknown>;
+    for (const key of Object.keys(location)) {
+      if (!key.startsWith('@') && key !== TEXT && !OOYYO_LOCATION_ELEMENTS.includes(key)) {
+        issues.push({ ad: index, problem: `element <location><${key}> is not in OoyyoFormat.xsd` });
+      }
+    }
+    const pictures = asList((ad.pictures as Record<string, unknown> | undefined)?.picture).map(valueOf);
+    const priceUnit = typeof ad.price === 'object' ? (ad.price as Record<string, unknown>)['@unit'] : undefined;
+    const item: Partial<OoyyoXmlItem> = {
+      url: valueOf(ad.url),
+      price: valueOf(ad.price),
+      priceUnit: priceUnit as 'EUR',
+      make: valueOf(ad.make),
+      model: valueOf(ad.model),
+      year: valueOf(ad.year),
+      pictures,
+    };
+    const invalid = validateOoyyoItem(item);
+    if (invalid) {
+      issues.push({ ad: index, problem: `rejected by partner rules: ${invalid}` });
+    }
+    if (item.url && urls.has(item.url)) {
+      issues.push({ ad: index, problem: 'duplicate vehicle url' });
+    }
+    urls.add(item.url);
+  });
+
+  if (ads.length !== expectedAds) {
+    issues.push({ ad: -1, problem: `file holds ${ads.length} ads, the run wrote ${expectedAds}` });
+  }
+  return { ads: ads.length, issues };
+};