@bytebrand/export-cars 1.26.46 → 1.26.48

package/src/exporters/ooyyo.exporter.ts

@@ -1,48 +1,75 @@
-import { Injectable } from '@nestjs/common';
-import { CarService, LoaderService, ConfigService, RulesService } from '../services';
-import { FsChunkInterface } from '@bytebrand/chunkable';
-import { Exporter } from '../interfaces/exporter.interface';
-import { CronicleUtils } from '@bytebrand/cronicle-helper';
-import { ModuleRef } from '@nestjs/core';
-import { Zipper } from '@bytebrand/zipper';
-import { OoyyoXmlBuilder } from '../formats/ooyyo-xml.builder';
-import { OoyyoTransformer } from '../transformers/ooyyo.transformer';
-
-@Injectable()
-export class OoyyoExporter implements Exporter {
-  constructor(
-    private carService: CarService,
-    private configService: ConfigService,
-    private loaderService: LoaderService,
-    private moduleRef: ModuleRef,
-    private rulesService: RulesService,
-  ) {}
-
-  async export(): Promise<void> {
-    const [fileXml, fileGZ] = this.configService.config.outputFiles;
-    await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
-    await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
-
-    const ooyyoXmlBuilder = new OoyyoXmlBuilder(fileXml);
-    await ooyyoXmlBuilder.buildStart();
-
-    const ooyyoTransformer = await this.moduleRef.create(OoyyoTransformer);
-
-    await this.loaderService.process(async (chunk: FsChunkInterface) => {
-      const chunkData = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
-
-        for (let i = 0; i < chunkData.length; i += 1) {
-          const { car, images } = chunkData[i];
-          // console.time(`Transform ${car._id}`);
-          await ooyyoXmlBuilder.addItem(await ooyyoTransformer.transform(car, images));
-          // console.timeEnd(`Transform ${car._id}`);
-          this.carService.counter.countCar(images);
-        }
-    });
-
-    await ooyyoXmlBuilder.buildEnd();
-    console.log(this.carService.counter.getCountedCars());
-
-    fileGZ && await Zipper.tar({ dest: fileGZ, fileList: [fileXml] });
-  }
-}
+import { Injectable } from '@nestjs/common';
+import { ModuleRef } from '@nestjs/core';
+
+import { FsChunkInterface } from '@bytebrand/chunkable';
+import { LoggerService } from '@bytebrand/cronicle-helper';
+
+import { CarService, RulesService, LoaderService, ConfigService, ImagesService } from '../services';
+import { ImageUrlValidator } from '../services/image-url-validator.service';
+import { FeedGuard } from '../services/feed-guard';
+import { OoyyoFeedRun, OoyyoSourceCar } from '../services/ooyyo-feed-run';
+import { OoyyoTransformer, toOoyyoImages } from '../transformers/ooyyo.transformer';
+import { resolveOpenAiImageFields } from '../transformers/openai-image-fields';
+import { OoyyoConfig } from '../configs/ooyyo.config';
+import { Exporter } from '../interfaces/exporter.interface';
+
+/**
+ * ExportType.OOYYO (ADV-331). Wires the Cronicle/Nest services into OoyyoFeedRun, which holds the
+ * logic. The output directory is no longer emptied: the new XML / XML.gz replace the old ones only
+ * after the snapshot, the written file and the counts passed.
+ */
+@Injectable()
+export class OoyyoExporter implements Exporter {
+  constructor(
+    private carService: CarService,
+    private configService: ConfigService,
+    private loaderService: LoaderService,
+    private rulesService: RulesService,
+    private imagesService: ImagesService,
+    private moduleRef: ModuleRef,
+  ) {}
+
+  async export(): Promise<void> {
+    const config = this.configService.config as OoyyoConfig;
+    const [xmlFile, gzFile] = config.outputFiles;
+    const validator = new ImageUrlValidator({ concurrency: config.imageConcurrency });
+    const transformer = await this.moduleRef.create(OoyyoTransformer);
+
+    const run = new OoyyoFeedRun(
+      {
+        forEachChunk: fn => this.loaderService.process((chunk: FsChunkInterface) => fn(chunk.data as OoyyoSourceCar[])),
+        filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
+        resolveImages: async ({ car, images }) => toOoyyoImages(
+          await resolveOpenAiImageFields(this.imagesService.getCleanImages(car._id, images), validator),
+          config.maxImages,
+        ),
+        translate: selector => transformer.translate(selector),
+        region: (postcode, country) => transformer.region(postcode, country),
+        imageStats: () => validator.stats,
+        guard: new FeedGuard(config.feedGuard),
+      },
+      {
+        runId: config.runId,
+        dryRun: config.dryRun,
+        domain: config.domain,
+        sourceDirectory: config.pathesConfig.localDirectory,
+        sourceBaseFileName: config.pathesConfig.baseFileName,
+        xmlFile,
+        gzFile,
+        stateDir: config.stateDir,
+      },
+    );
+
+    try {
+      const report = await run.run();
+      if (report.stale) {
+        console.log(`ooyyo feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
+      }
+    } catch (error) {
+      // main.ts would report a rejected run as {"complete":1,"code":0}; fail the job so the chained
+      // `6.1 … part 2` step does not run (report + lock already handled by the run).
+      LoggerService.exitError(`ooyyo feed not published: ${error?.message ?? error}`);
+      process.exit(1);
+    }
+  }
+}

package/src/formats/ooyyo-xml.builder.ts

@@ -1,48 +1,87 @@
-import { XMLBuilderBase } from '@bytebrand/cronicle-helper';
-import { OoyyoXmlItem } from '../interfaces';
-
-export class OoyyoXmlBuilder extends XMLBuilderBase {
-  async buildStart(): Promise<void> {
-    await this.write(`<?xml version="1.0" encoding="UTF-8"?>
-    <ooyyo xsi:noNamespaceSchemaLocation="OoyyoFormat.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">`);
-  }
-
-  async addItem(car: OoyyoXmlItem): Promise<void> {
-    let item = '';
-
-    item += '\n<ad>\n';
-    item += this.getElement('url', car.url, true);
-    item += this.getElement('title', car.title, true);
-    item += this.getElement('content', car.content, true);
-    item += this.getElement('price', car.price, true, { unit: 'euro' });
-    item += this.getElement('make', car.make, true);
-    item += this.getElement('model', car.model, true);
-    item += this.getElement('trim', car.trim, true);
-    item += this.getElement('engine_size', car.engine_size, true, { unit: 'cc' });
-    item += this.getElement('body_type', car.body_type, true);
-    item += this.getElement('color', car.color, true);
-    item += this.getElement('year', car.year, true);
-    item += this.getElement('mileage', car.mileage, true, { unit: 'km' });
-    item += this.getElement('fuel_type', car.fuel_type, true);
-    item += this.getElement('transmission', car.transmission, true);
-    item += this.getElement('euronorm', car.euronorm, true);
-    item += this.getElement('power', car.power, true, { unit: 'kw' });
-    item += this.getElement('doors', car.doors, true);
-    item += this.getElement('seats', car.seats, true);
-    item += this.getElement('gears', car.gears, true);
-    item += this.getElementsTree('location', car.location, true);
-    item += this.getElementsList('pictures', 'picture', car.pictures, true);
-    item += this.getElement('vin', car.vin, true);
-    item += this.getElement('is_new', car.is_new, true);
-    item += this.getElement('date', car.date, true);
-    item += this.getElement('time', car.time, true);
-    item += '</ad>';
-
-    await this.write(item);
-  }
-
-  async buildEnd(): Promise<void> {
-    await this.write('\n</ooyyo>');
-    await this.end();
-  }
-}
+import { createWriteStream, WriteStream } from 'fs';
+import { OoyyoXmlItem } from '../interfaces';
+
+// Characters XML 1.0 does not allow at all (a single one makes the whole file unreadable).
+// eslint-disable-next-line no-control-regex
+const INVALID_XML_CHARS = /[\u0000-\u0008\u000B\u000C\u000E-\u001F￾￿]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;
+
+export const stripInvalidXmlChars = (value: string): string => value.replace(INVALID_XML_CHARS, '');
+
+// CDATA cannot contain "]]>": split it across two sections. Everything else goes in verbatim
+// (UTF-8 umlauts, "&" in URLs), exactly as the partner sample wraps every value in CDATA.
+export const cdata = (value: string): string =>
+  `<![CDATA[${stripInvalidXmlChars(value).replace(/]]>/g, ']]]]><![CDATA[>')}]]>`;
+
+/**
+ * Writes the OOYYO XML (ADV-331): UTF-8, root/namespace exactly as OoyyoFormat.xml, element order as
+ * in the sample, every value in CDATA. Empty optional elements are left out — the legacy builder
+ * wrote `<engine_size/>` / `<power/>` without the unit attribute the XSD requires, so such a file was
+ * not schema-valid. `price` always carries unit="EUR".
+ */
+export class OoyyoXmlBuilder {
+  private stream: WriteStream;
+
+  constructor(fileName: string) {
+    this.stream = createWriteStream(fileName, { flags: 'w', encoding: 'utf8' });
+  }
+
+  private write(chunk: string): Promise<void> {
+    return new Promise((resolve, reject) => this.stream.write(chunk, error => (error ? reject(error) : resolve())));
+  }
+
+  async buildStart(): Promise<void> {
+    await this.write('<?xml version="1.0" encoding="UTF-8"?>\n'
+      + '<ooyyo xsi:noNamespaceSchemaLocation="OoyyoFormat.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">\n');
+  }
+
+  async addItem(car: OoyyoXmlItem): Promise<void> {
+    await this.write(OoyyoXmlBuilder.toXml(car));
+  }
+
+  static toXml(car: OoyyoXmlItem): string {
+    const el = (tag: string, value: string, unit?: string, indent = '    '): string =>
+      (value ? `${indent}<${tag}${unit ? ` unit="${unit}"` : ''}>${cdata(value)}</${tag}>\n` : '');
+    const location = [
+      el('city', car.location?.city, undefined, '      '),
+      el('postcode', car.location?.postcode, undefined, '      '),
+      el('region', car.location?.region, undefined, '      '),
+    ].join('');
+    const pictures = (car.pictures ?? []).map(url => el('picture', url, undefined, '      ')).join('');
+
+    return '  <ad>\n'
+      + el('url', car.url)
+      + el('title', car.title)
+      + el('content', car.content)
+      + el('price', car.price, car.priceUnit)
+      + el('make', car.make)
+      + el('model', car.model)
+      + el('engine_size', car.engine_size, 'cc')
+      + el('trim', car.trim)
+      + el('body_type', car.body_type)
+      + el('color', car.color)
+      + el('year', car.year)
+      + el('mileage', car.mileage, 'km')
+      + el('fuel_type', car.fuel_type)
+      + el('transmission', car.transmission)
+      + el('euronorm', car.euronorm)
+      + el('power', car.power, 'kw')
+      + el('doors', car.doors)
+      + el('seats', car.seats)
+      + el('gears', car.gears)
+      + (location ? `    <location>\n${location}    </location>\n` : '')
+      + (pictures ? `    <pictures>\n${pictures}    </pictures>\n` : '')
+      + el('vin', car.vin)
+      + el('is_new', car.is_new)
+      + el('date', car.date)
+      + el('time', car.time)
+      + '  </ad>\n';
+  }
+
+  async buildEnd(): Promise<void> {
+    await this.write('</ooyyo>\n');
+    await new Promise<void>((resolve, reject) => {
+      this.stream.on('error', reject);
+      this.stream.end(() => resolve());
+    });
+  }
+}

package/src/interfaces/export-item.interface.ts

@@ -62,9 +62,50 @@ export interface Auto24XmlItem {
   dealer_phone: string;
 }
 
-export interface OoyyoXmlItem extends FbXmlItem {
-  euronorm: string;
+// One OOYYO <ad> (ADV-331). '' = element left out (every element is optional in OoyyoFormat.xsd;
+// the importer's mandatory fields are enforced by validateOoyyoItem).
+export interface OoyyoXmlItem {
+  url: string;
+  title: string;
+  content: string;
+  // whole euros
+  price: string;
+  // always 'EUR' (partner, 19.10.2026)
+  priceUnit: 'EUR';
+  make: string;
+  model: string;
+  // ccm, unit="cc"
+  engine_size: string;
+  trim: string;
+  body_type: string;
+  color: string;
+  year: string;
+  // unit="km"
   mileage: string;
+  fuel_type: string;
+  transmission: string;
+  euronorm: string;
+  // unit="kw"
+  power: string;
+  doors: string;
+  seats: string;
+  gears: string;
+  location: {
+    city: string;
+    // string, leading zero kept
+    postcode: string;
+    // Bundesland
+    region: string;
+  };
+  // validated public HTTPS photo URLs, hero first
+  pictures: string[];
+  vin: string;
+  // '1' new, '0' used
+  is_new: string;
+  // DD/MM/YYYY, Europe/Berlin
+  date: string;
+  // HH:MM, Europe/Berlin
+  time: string;
 }
 
 // Keys and order as the meinestadt exporter emits them (see docs/meinestadt/AUDIT.md §3).

package/src/interfaces/params.interface.ts

@@ -53,6 +53,11 @@ export interface Params {
   marktdeDryRun?: string;
   marktdeImageConcurrency?: string;
   marktdeMaxImages?: string;
+  // OOYYO feed only (ADV-331), Cronicle text params; see configs/ooyyo.config.ts.
+  // Live output needs ooyyoDryRun=false explicitly.
+  ooyyoDryRun?: string;
+  ooyyoImageConcurrency?: string;
+  ooyyoMaxImages?: string;
 }
 
 // Parsed from feedItemsBq* by GoogleConfig.

package/src/services/mapping.service.ts

@@ -1,11 +1,22 @@
 import { Injectable } from '@nestjs/common';
 import axios from 'axios';
+import { CronicleUtils } from '@bytebrand/cronicle-helper';
 import { MappingRules } from '../interfaces/mapping-rules.interface';
 import { GoogleSpreadsheet, GoogleSpreadsheetRow } from 'google-spreadsheet';
 import { ConfigService } from './config.service';
 import { DealersMapping, MetaPushPriorityMapping } from '../interfaces/dealers-mapping.interface';
 import { parseCsvToObjects } from '../utils';
 
+// Public CSV of one tab of a Google sheet, addressed by the tab title — no API key. The sheet must stay
+// readable by everyone with the link. (Checked 13.11.2026: gviz CSV = the raw export of the tabs
+// mobile_yescar and mobile, cell for cell; gviz only adds trailing empty columns.)
+export const mappingSheetCsvUrl = (spreadsheetId: string, title: string): string =>
+  `https://docs.google.com/spreadsheets/d/${spreadsheetId}/gviz/tq?tqx=out:csv&sheet=${encodeURIComponent(title)}`;
+
+// Columns every mapping tab read by getRules() has; missing ones mean the response is not the tab
+// (e.g. Google's sign-in page once the sheet is no longer shared).
+const REQUIRED_MAPPING_COLUMNS = ['importField', 'importFieldType', 'autodeField', 'autodeFieldType'];
+
 @Injectable()
 export class MappingService {
   constructor(private configService: ConfigService) {}
@@ -13,24 +24,54 @@ export class MappingService {
   private dealers: any;
   private metaPushPriorities: MetaPushPriorityMapping[];
 
+  /**
+   * Mapping rules of the export type's tab (mappingSpreadsheetTitle) of the mapping sheet. Read from the
+   * sheet's public CSV instead of the Google Sheets API: the API needed the event's `apikey`, and an
+   * expired key (DEV 9c.0, found 12.11.2026) silently broke every feed on the sheet. Fails the run
+   * when the tab cannot be read or yields no rule — never an export without a mapping.
+   */
   async getRules(): Promise<MappingRules[]> {
     if (!this.rules) {
-      const spreadsheet = new GoogleSpreadsheet(this.configService.config.mappingSpreadsheetId);
-      spreadsheet.useApiKey(this.configService.config.apikey);
-      await spreadsheet.loadInfo();
-      const rows = await spreadsheet.sheetsByTitle[this.configService.config.mappingSpreadsheetTitle].getRows();
-      const mappingRules = this.rowsToRules(rows) as MappingRules[];
+      const { mappingSpreadsheetId, mappingSpreadsheetTitle } = this.configService.config;
+      const url = mappingSheetCsvUrl(mappingSpreadsheetId, mappingSpreadsheetTitle);
+      const { data } = await CronicleUtils.retry(
+        () => axios.get<string>(url, { responseType: 'text', timeout: 30_000 }),
+        3,
+        5_000,
+        true,
+      );
+      this.rules = this.rulesFromCsv(data, mappingSpreadsheetTitle);
+      console.log(`mapping: ${this.rules.length} rules from the public CSV of tab ${mappingSpreadsheetTitle}`);
+    }
+    return this.rules;
+  }
 
-      mappingRules.map((rule, _i, selfRules) => {
-        this.removeEmptyKeys(rule);
-        if (rule.importFieldType && rule.importFieldType.includes('selector')) {
-          this.groupSelectors(rule, selfRules);
-        }
-      });
+  // CSV text of a mapping tab -> rules, with the same post-processing the Sheets API rows had.
+  rulesFromCsv(csv: string, title: string): MappingRules[] {
+    const rows = parseCsvToObjects<Record<string, string>>(typeof csv === 'string' ? csv : '');
+    const columns = rows.length ? Object.keys(rows[0]) : [];
+    const missing = REQUIRED_MAPPING_COLUMNS.filter(column => !columns.includes(column));
+    if (missing.length) {
+      throw new Error(`mapping sheet tab "${title}" not readable as CSV (missing columns ${missing.join(', ')}) — is the sheet still shared with everyone who has the link?`);
+    }
+    // unnamed columns (notes next to the table) are not part of a rule
+    const mappingRules = rows.map((row) => {
+      delete row[''];
+      return row;
+    }) as unknown as MappingRules[];
+
+    mappingRules.forEach((rule, _i, selfRules) => {
+      this.removeEmptyKeys(rule);
+      if (rule.importFieldType && rule.importFieldType.includes('selector')) {
+        this.groupSelectors(rule, selfRules);
+      }
+    });
 
-      this.rules = this.filterResultedRows(mappingRules);
+    const rules = this.filterResultedRows(mappingRules);
+    if (!rules.length) {
+      throw new Error(`mapping sheet tab "${title}" has no rule`);
     }
-    return this.rules;
+    return rules;
   }
 
   async getDealersMapping(): Promise<DealersMapping[]> {