@bytebrand/export-cars 1.26.39 → 1.26.41

package/src/services/criteo-feed-run.ts

@@ -0,0 +1,253 @@
+import { promises as fs } from 'fs';
+import { dirname, join } from 'path';
+import { get, toNumber, toString } from 'lodash';
+import { ImboImageResponseInterface } from '@bytebrand/chunkable';
+import { DataSchemes } from '@bytebrand/car-schema';
+
+import { GoogleXmlBuilder } from '../formats/google-xml.builder';
+import { CriteoXmlItem } from '../interfaces';
+import { FeedGuard, FeedGuardError, LastGood, inspectSourceSnapshot } from './feed-guard';
+
+export interface CriteoSourceCar {
+  car: DataSchemes.Car;
+  images: ImboImageResponseInterface[];
+}
+
+export enum CriteoExclusion {
+  noId = 'no_id',
+  duplicateId = 'duplicate_id',
+  filtered = 'filtered',
+  notAvailable = 'reserved_or_sold',
+  noPrice = 'no_valid_price',
+  noImage = 'no_marketing_image',
+}
+
+// Online cars flagged like this are not sellable stock (same rule as the AutoUncle feed). Criteo
+// (16.10.2026): only currently marketable stock goes into the full file; a car that leaves it is
+// removed by the next full import.
+const NOT_AVAILABLE = ['selector_offerAvailability_reserved', 'selector_offerAvailability_sold'];
+
+// Everything the run needs from the outside world — the Nest exporter wires the real services,
+// tests pass fakes.
+export interface CriteoRunDeps {
+  // iterates the base export snapshot chunk by chunk (LoaderService.process)
+  forEachChunk: (fn: (cars: CriteoSourceCar[]) => Promise<void>) => Promise<unknown>;
+  // existing shared partner filter (images count, Cronicle rules)
+  filterCar: (source: CriteoSourceCar) => boolean;
+  // CriteoTransformer.transform: null = no real marketing image
+  transform: (source: CriteoSourceCar) => Promise<CriteoXmlItem | null>;
+  guard: FeedGuard;
+}
+
+export interface CriteoRunOptions {
+  runId?: string;
+  domain: string;
+  sourceDirectory: string;
+  sourceBaseFileName: string;
+  xmlFile: string;
+  stateDir: string;
+  now?: () => Date;
+}
+
+export interface CriteoRunReport {
+  runId?: string;
+  startedAt: string;
+  finishedAt?: string;
+  durationMs?: number;
+  status: 'published' | 'blocked' | 'failed';
+  guard?: { code: string; message: string };
+  error?: string;
+  xmlFile: string;
+  source: { files: number; newestChunkAt: string | null; ageHours: number | null; cars: number };
+  items: number;
+  excluded: Record<string, number>;
+  withRate: number;
+  withoutEnergyClass: number;
+  // against the ids of the last published file; removed ids are dropped by Criteo's next full import
+  ids: { added: number; removed: number; kept: number; removedSample: string[] } | null;
+  lastGood: LastGood | null;
+  lastGoodAgeHours: number | null;
+  stale: boolean;
+}
+
+const HOUR_MS = 60 * 60 * 1000;
+const PRICE = /^\d+(\.\d+)? EUR$/;
+
+const increment = (counter: Record<string, number>, key: string) => {
+  counter[key] = (counter[key] ?? 0) + 1;
+};
+
+/**
+ * One Criteo catalog run (ADV-319): source check -> map -> temp XML -> count check -> atomic
+ * publish -> report. Nothing outside the output directory and `stateDir` is written. A blocked run
+ * leaves the last good XML in place and fails the job, so the chained upload step does not run.
+ * The XML format, enums and field order stay the legacy Criteo ones.
+ */
+export class CriteoFeedRun {
+  constructor(private readonly deps: CriteoRunDeps, private readonly options: CriteoRunOptions) { }
+
+  private now(): Date {
+    return this.options.now ? this.options.now() : new Date();
+  }
+
+  private get idsFile(): string {
+    return join(this.options.stateDir, 'published-ids.json');
+  }
+
+  async run(): Promise<CriteoRunReport> {
+    const { guard } = this.deps;
+    const { xmlFile, runId } = this.options;
+    const startedAt = this.now();
+    const lastGood = await guard.readLastGood();
+    const report: CriteoRunReport = {
+      runId,
+      startedAt: startedAt.toISOString(),
+      status: 'failed',
+      xmlFile,
+      source: { files: 0, newestChunkAt: null, ageHours: null, cars: 0 },
+      items: 0,
+      excluded: {},
+      withRate: 0,
+      withoutEnergyClass: 0,
+      ids: null,
+      lastGood,
+      lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
+      stale: guard.isStale(lastGood, startedAt),
+    };
+
+    await guard.acquireLock(runId ?? 'local', startedAt);
+    const tmpFile = `${xmlFile}.tmp`;
+    try {
+      const snapshot = await inspectSourceSnapshot(this.options.sourceDirectory, this.options.sourceBaseFileName);
+      report.source.files = snapshot.files;
+      report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
+      report.source.ageHours = snapshot.newestMtime
+        ? Number(((startedAt.getTime() - snapshot.newestMtime.getTime()) / HOUR_MS).toFixed(2))
+        : null;
+      guard.checkSource(snapshot, startedAt);
+
+      await fs.mkdir(dirname(xmlFile), { recursive: true });
+      // the builder appends — never onto a leftover temp file
+      await fs.rm(tmpFile, { force: true });
+      const builder = new GoogleXmlBuilder(tmpFile);
+      await builder.buildStart(this.options.domain);
+
+      const seen = new Set<string>();
+      const published: string[] = [];
+      await this.deps.forEachChunk(async (cars) => {
+        for (const source of cars) {
+          report.source.cars += 1;
+          const reason = this.exclusion(source, seen);
+          if (reason) {
+            increment(report.excluded, reason);
+            continue;
+          }
+          const item = await this.deps.transform(source);
+          if (!item) {
+            increment(report.excluded, CriteoExclusion.noImage);
+            continue;
+          }
+          // the transformer formats the price; a value it could not format is not sent
+          if (!PRICE.test(item.price) || !PRICE.test(item.sale_price)) {
+            increment(report.excluded, CriteoExclusion.noPrice);
+            continue;
+          }
+          await builder.addItem(item);
+          published.push(item.id);
+          report.items += 1;
+          if (item.installment) {
+            report.withRate += 1;
+          }
+          if (!item.energy_efficiency_class) {
+            report.withoutEnergyClass += 1;
+          }
+        }
+      });
+      await builder.buildEnd();
+
+      const counts = { sourceItems: report.source.cars, publishedItems: report.items };
+      guard.checkCounts(counts, lastGood);
+
+      const previousIds = await this.readPublishedIds();
+      report.lastGood = await guard.publish(tmpFile, xmlFile, counts, this.now(), runId);
+      await this.writePublishedIds(published);
+      report.ids = this.diffIds(previousIds, published);
+      report.status = 'published';
+      report.lastGoodAgeHours = 0;
+      report.stale = false;
+      return report;
+    } catch (error) {
+      if (error instanceof FeedGuardError) {
+        report.status = 'blocked';
+        report.guard = { code: error.code, message: error.message };
+      } else {
+        report.error = error?.message ?? String(error);
+      }
+      // keep the rejected file for inspection; the published file stays untouched
+      await fs.rename(tmpFile, `${xmlFile}.rejected`).catch(() => undefined);
+      throw error;
+    } finally {
+      await this.writeReport(report, startedAt);
+      await guard.releaseLock();
+    }
+  }
+
+  private exclusion(source: CriteoSourceCar, seen: Set<string>): CriteoExclusion | null {
+    const id = toString(get(source, 'car._id', ''));
+    if (!id) {
+      return CriteoExclusion.noId;
+    }
+    if (seen.has(id)) {
+      return CriteoExclusion.duplicateId;
+    }
+    seen.add(id);
+    if (!this.deps.filterCar(source)) {
+      return CriteoExclusion.filtered;
+    }
+    if (NOT_AVAILABLE.includes(toString(get(source.car, 'offer.offerAvailability', '')))) {
+      return CriteoExclusion.notAvailable;
+    }
+    const price = toNumber(get(source.car, 'price.currentSalesPrice', null));
+    if (!Number.isFinite(price) || price <= 0) {
+      return CriteoExclusion.noPrice;
+    }
+    return null;
+  }
+
+  private async readPublishedIds(): Promise<string[] | null> {
+    try {
+      return JSON.parse(await fs.readFile(this.idsFile, 'utf8')) as string[];
+    } catch (error) {
+      if (error?.code === 'ENOENT') {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  private async writePublishedIds(ids: string[]): Promise<void> {
+    await fs.mkdir(this.options.stateDir, { recursive: true });
+    await fs.writeFile(`${this.idsFile}.tmp`, JSON.stringify(ids), 'utf8');
+    await fs.rename(`${this.idsFile}.tmp`, this.idsFile);
+  }
+
+  private diffIds(previous: string[] | null, current: string[]): CriteoRunReport['ids'] {
+    if (!previous) {
+      return { added: current.length, removed: 0, kept: 0, removedSample: [] };
+    }
+    const before = new Set(previous);
+    const now = new Set(current);
+    const removed = previous.filter(id => !now.has(id));
+    const kept = current.filter(id => before.has(id)).length;
+    return { added: current.length - kept, removed: removed.length, kept, removedSample: removed.slice(0, 20) };
+  }
+
+  private async writeReport(report: CriteoRunReport, startedAt: Date): Promise<void> {
+    const finishedAt = this.now();
+    report.finishedAt = finishedAt.toISOString();
+    report.durationMs = finishedAt.getTime() - startedAt.getTime();
+    await fs.mkdir(this.options.stateDir, { recursive: true });
+    await fs.writeFile(join(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
+    console.log(`criteo feed report: ${JSON.stringify(report)}`);
+  }
+}

package/src/transformers/autouncle.transformer.ts

@@ -11,6 +11,13 @@ import { DEFAULT_REGION, MIN_MILEAGES } from '../constants/autouncle.contants';
 import { AutouncleJsonItem } from '../interfaces';
 import { OpenAiImageSelection } from './openai-image-fields';
 import { toUnixSeconds } from './meinstadt.transformer';
+import { GoogleEnvkvAttribute, envkvCo2Value, envkvFuelConsumptionValue } from './google-feed-envkv';
+
+// back_link parameters (ADV-324): the partner in `source`, so the reporting views, the CRM source and
+// GA4 (utm_*) all see "autouncle". Legacy and utm pairs carry identical values (no parameterConflict).
+// Paid / unpaid is a contract attribute and is not encoded here. Pending AutoUncle's approval.
+export const AUTOUNCLE_LINK_PARAMS =
+  'source=autouncle&medium=listing&campaign=listing&utm_source=autouncle&utm_medium=listing&utm_campaign=listing';
 
 export enum AutouncleExclusion {
   duplicateId = 'duplicate_id',
@@ -39,6 +46,8 @@ export interface AutouncleMappingContext {
   images: string[];
   // German label of a selector (TranslatorExtended.translateVal)
   translate: (key: string) => string;
+  // EnVKV attributes whose value is a CBD estimate and must not be published (publication gate, enforce)
+  withheldEnvkv?: GoogleEnvkvAttribute[];
 }
 
 export type AutouncleMapResult = { ok: true; item: AutouncleJsonItem } | { ok: false; reason: AutouncleExclusion };
@@ -179,11 +188,15 @@ export const mapAutouncleItem = (car: DataSchemes.Car, ctx: AutouncleMappingCont
   const sellerId = toString(get(car, 'metaData.ownerId', '')).trim();
   const price = getAutounclePrice(get(car, 'price.currentSalesPrice', null));
   const firstRegistration = get(car, 'mainData.firstRegistration', null);
+  // EnVKV (ADV-322): the WLTP picks of the Google feed (legacy fields only as their fallback), never an
+  // estimated value. The energy class is derived from CO2, so it goes with a withheld CO2 value.
+  const withheld = ctx.withheldEnvkv ?? [];
+  const co2Withheld = withheld.includes('co2_emissions');
 
   return {
     ok: true,
     item: {
-      back_link: `https://www.${ctx.domain}/search/vehicle/${autodeId}?source=classifieds&medium=cpc&campaign=autouncle`,
+      back_link: `https://www.${ctx.domain}/search/vehicle/${autodeId}?${AUTOUNCLE_LINK_PARAMS}`,
       make: toString(get(car, 'mainData.make', '')),
       model_name: toString(get(car, 'mainData.model', '')),
       model_variant: toString(get(car, 'mainData.subModel', '')),
@@ -210,10 +223,10 @@ export const mapAutouncleItem = (car: DataSchemes.Car, ctx: AutouncleMappingCont
       seller_id: sellerId,
       description: toString(get(car, 'offer.offerDescription', '')),
       images: ctx.images.map(url => ({ url })),
-      fuel_economy: toString(get(car, 'consumption.consumptionCombined', '')),
-      co2_emission: toString(get(car, 'environmentEmissions.co2', '')),
+      fuel_economy: withheld.includes('fuel_consumption') ? '' : toString(envkvFuelConsumptionValue(car) ?? ''),
+      co2_emission: co2Withheld ? '' : toString(envkvCo2Value(car) ?? ''),
       // ADV-187: canonical CO2-derived WLTP class, not the unvalidated legacy NEFZ class.
-      co2_effiency_class: toString(get(car, 'environmentEmissions.wltpEnergyEfficiencyClass', '')),
+      co2_effiency_class: co2Withheld ? '' : toString(get(car, 'environmentEmissions.wltpEnergyEfficiencyClass', '')),
       euro_emission_class: toString(get(car, 'environmentEmissions.emissionClass', '')),
       enviromental_sticker: toString(get(car, 'environmentEmissions.emissionSticker', '')),
     },

package/src/transformers/google-feed-envkv.ts

@@ -143,6 +143,16 @@ export function googleCo2(car: DataSchemes.Car): string | undefined {
   return picked ? `${picked.value}g/km` : undefined;
 }
 
+// The same picks as plain numbers, for feeds that send values without units (AutoUncle, ADV-322).
+// Fuel: undefined for a BEV. CO2: 0 for a BEV (no field behind it, so never gated).
+export function envkvFuelConsumptionValue(car: DataSchemes.Car): number | undefined {
+  return pickFuelConsumption(car)?.value;
+}
+
+export function envkvCo2Value(car: DataSchemes.Car): number | undefined {
+  return isBev(car) ? 0 : pickCo2(car)?.value;
+}
+
 // The EnVKV attributes of the Google item (keys of GoogleXmlItem).
 export type GoogleEnvkvAttribute =
   | 'fuel_consumption'

package/test/autouncle-feed-run.test.ts

@@ -7,7 +7,7 @@ import { tmpdir } from 'os';
 import { join } from 'path';
 
 import { FeedGuard, FeedGuardError, FeedGuardOptions } from '../src/services/feed-guard';
-import { AutouncleFeedRun, AutouncleSourceCar } from '../src/services/autouncle-feed-run';
+import { AutouncleFeedRun, AutouncleRunDeps, AutouncleSourceCar } from '../src/services/autouncle-feed-run';
 import { toAutouncleImages } from '../src/transformers/autouncle.transformer';
 import { selectOpenAiImageFields } from '../src/transformers/openai-image-fields';
 import { buildImagesService } from './fixtures/images-service.harness';
@@ -63,7 +63,9 @@ const guardOptions = (world: World, extra: Partial<FeedGuardOptions> = {}): Feed
   ...extra,
 });
 
-const run = async (world: World, { at = NOW, guard = {} }: { at?: Date; guard?: Partial<FeedGuardOptions> } = {}) => {
+type Gate = AutouncleRunDeps['gate'];
+
+const run = async (world: World, { at = NOW, guard = {}, gate }: { at?: Date; guard?: Partial<FeedGuardOptions>; gate?: Gate } = {}) => {
   const forEachChunk = async (fn: (cars: AutouncleSourceCar[]) => Promise<void>) => {
     const names = (await fs.readdir(world.source)).sort((a, b) => Number(a.split('.')[1]) - Number(b.split('.')[1]));
     for (const name of names) {
@@ -85,6 +87,7 @@ const run = async (world: World, { at = NOW, guard = {} }: { at?: Date; guard?:
       imageStats: () => ({ ok: 1 }),
       translate: key => key,
       guard: new FeedGuard(guardOptions(world, guard)),
+      gate,
     },
     {
       runId: 'test',
@@ -231,6 +234,26 @@ const tests: Array<[string, () => Promise<void>]> = [
     assert.equal(await exists(world.json), false);
   }],
 
+  ['EnVKV gate: enforce sends estimated values empty, audit only counts them; off without a gate', async () => {
+    const world = await newWorld();
+    await writeSource(world, [source('v1'), source('v2')]);
+    // v1: the CO2 value it would send is a CBD estimate
+    const estimatedAttributes = (c: any): any[] => (c._id === 'v1' ? ['co2_emissions'] : []);
+
+    const off = await run(world);
+    assert.deepEqual(off.envkv, { gate: 'off', estimated: {} });
+
+    const audit = await run(world, { at: new Date(NOW.getTime() + HOUR), gate: { mode: 'audit', estimatedAttributes } });
+    assert.deepEqual(audit.envkv, { gate: 'audit', estimated: { co2_emissions: 1 } });
+    assert.deepEqual((await readAds(world)).map((a: any) => a.co2_emission), ['137', '137']);
+
+    const enforce = await run(world, { at: new Date(NOW.getTime() + 2 * HOUR), gate: { mode: 'enforce', estimatedAttributes } });
+    assert.deepEqual(enforce.envkv, { gate: 'enforce', estimated: { co2_emissions: 1 } });
+    const ads = await readAds(world);
+    assert.deepEqual(ads.map((a: any) => [a.co2_emission, a.fuel_economy]), [['', '6.1'], ['137', '6.1']]);
+    assert.equal(ads.length, 2, 'a withheld value never drops the car');
+  }],
+
   ['second run with the same snapshot: byte-identical file (stable ids, order, links)', async () => {
     const world = await newWorld();
     await writeSource(world, [source('i1'), source('i2'), source('i3')]);

package/test/autouncle-mapping.test.ts

@@ -62,7 +62,8 @@ const tests: Array<[string, () => void | Promise<void>]> = [
     assert.equal(a.seller_id, 'dealer-7');
     assert.equal(b.seller_id, 'dealer-7');
     assert.notEqual(a.back_link, b.back_link);
-    assert.equal(a.back_link, 'https://www.auto.de/search/vehicle/car-a?source=classifieds&medium=cpc&campaign=autouncle');
+    // ADV-324: partner in source; identical legacy + utm pairs; no paid medium
+    assert.equal(a.back_link, 'https://www.auto.de/search/vehicle/car-a?source=autouncle&medium=listing&campaign=listing&utm_source=autouncle&utm_medium=listing&utm_campaign=listing');
     // same car again -> byte-identical back_link (no relisting under a new URL)
     assert.equal(item(car('car-a', { 'metaData.ownerId': 'dealer-7' })).back_link, a.back_link);
     // seller_id is never a vehicle id
@@ -129,6 +130,30 @@ const tests: Array<[string, () => void | Promise<void>]> = [
     assert.equal(item(car('u5')).power, '85');
   }],
 
+  ['EnVKV: WLTP picks of the Google feed, legacy only as fallback; BEV no fuel, CO2 0; 0 / missing -> ""', () => {
+    const wltp = item(car('e1', { 'consumption.wltpCombined': 5.8, 'environmentEmissions.wltpCo2': 132 }));
+    assert.deepEqual([wltp.fuel_economy, wltp.co2_emission], ['5.8', '132']);
+    // fixture has only the legacy fields (6.1 l, 137 g) -> sent as before
+    const legacy = item(car('e2'));
+    assert.deepEqual([legacy.fuel_economy, legacy.co2_emission], ['6.1', '137']);
+    // CBD mirrors kWh into consumptionCombined for BEVs: never sent as fuel economy
+    const bev = item(car('e3', { 'consumption.fuel': 'selector_fuel_electrical', 'consumption.consumptionCombined': 16.5, 'environmentEmissions.co2': null }));
+    assert.deepEqual([bev.fuel_economy, bev.co2_emission], ['', '0']);
+    const none = item(car('e4', { 'consumption.consumptionCombined': 0, 'environmentEmissions.co2': null }));
+    assert.deepEqual([none.fuel_economy, none.co2_emission], ['', '']);
+  }],
+
+  ['EnVKV: estimated values withheld by the publication gate; the CO2-derived class goes with CO2', () => {
+    const c = car('w1', { 'consumption.wltpCombined': 5.8, 'environmentEmissions.wltpCo2': 132, 'environmentEmissions.wltpEnergyEfficiencyClass': 'D' });
+    const sent = item(c);
+    assert.deepEqual([sent.fuel_economy, sent.co2_emission, sent.co2_effiency_class], ['5.8', '132', 'D']);
+    const noFuel = item(c, { withheldEnvkv: ['fuel_consumption'] });
+    assert.deepEqual([noFuel.fuel_economy, noFuel.co2_emission, noFuel.co2_effiency_class], ['', '132', 'D']);
+    const noCo2 = item(c, { withheldEnvkv: ['co2_emissions'] });
+    assert.deepEqual([noCo2.fuel_economy, noCo2.co2_emission, noCo2.co2_effiency_class], ['5.8', '', '']);
+    assert.deepEqual(Object.keys(noCo2), HISTORIC_KEYS);
+  }],
+
   ['seller address: the dealer\'s own address, region from the postal code table', () => {
     const a = item(car('s1'));
     assert.equal(a.seller_city, 'Sandersdorf-Brehna');
@@ -201,12 +226,26 @@ const tests: Array<[string, () => void | Promise<void>]> = [
     assert.equal(dry.maxImages, 0);
     assert.equal(dry.feedGuard.minPublishedItems, 1);
     assert.equal(new AutouncleConfig({ ...base, autouncleDryRun: 'no' } as any).dryRun, true);
-    const live = new AutouncleConfig({ ...base, autouncleDryRun: 'false', autouncleMaxImages: '12', feedGuardMinPublishedItems: '50' } as any);
+    const gateTable = { publicationGateBqProject: 'p', publicationGateBqDataset: 'd', publicationGateBqTable: 't' };
+    const live = new AutouncleConfig({ ...base, ...gateTable, publicationGate: 'enforce', autouncleDryRun: 'false', autouncleMaxImages: '12', feedGuardMinPublishedItems: '50' } as any);
     assert.equal(live.dryRun, false);
     assert.equal(live.outputFiles[0], 'resources/autouncle-export/api_export_cars_autouncle.json');
     assert.equal(live.stateDir, 'resources/autouncle-state/export');
     assert.equal(live.maxImages, 12);
     assert.equal(live.feedGuard.minPublishedItems, 50);
+    assert.deepEqual(live.publicationGate, { mode: 'enforce', project: 'p', dataset: 'd', table: 't' });
+  }],
+
+  ['config: publication gate off by default for a dry-run; a live run needs enforce; audit / enforce need the table', () => {
+    const base = { environment: Environment.DEVELOP, exportType: ExportType.AUTOUNCLE };
+    const gateTable = { publicationGateBqProject: 'p', publicationGateBqDataset: 'd', publicationGateBqTable: 't' };
+    assert.equal(new AutouncleConfig({ ...base } as any).publicationGate, undefined);
+    assert.equal(new AutouncleConfig({ ...base, publicationGate: 'off' } as any).publicationGate, undefined);
+    assert.equal(new AutouncleConfig({ ...base, ...gateTable, publicationGate: 'Audit' } as any).publicationGate.mode, 'audit');
+    assert.throws(() => new AutouncleConfig({ ...base, autouncleDryRun: 'false' } as any), /needs publicationGate=enforce/);
+    assert.throws(() => new AutouncleConfig({ ...base, ...gateTable, publicationGate: 'audit', autouncleDryRun: 'false' } as any), /needs publicationGate=enforce/);
+    assert.throws(() => new AutouncleConfig({ ...base, publicationGate: 'enforce' } as any), /needs publicationGateBqProject/);
+    assert.throws(() => new AutouncleConfig({ ...base, publicationGate: 'strict' } as any), /must be off, audit or enforce/);
   }],
 ];