@bytebrand/export-cars 1.26.25 → 1.26.27

package/src/services/feed-items-bigquery.writer.ts

@@ -0,0 +1,101 @@
+import { join } from 'path';
+import { BigQueryLogger } from '@bytebrand/bigquery-logger';
+import { FeedItemsBigQueryTarget } from '../interfaces/params.interface';
+import { FeedItemRow } from '../transformers/google-feed-item-row';
+
+const BATCH_SIZE = 500;
+// After this many failed batches the writer gives up for the run (missing table, bad credentials).
+const MAX_FAILED_BATCHES = 3;
+const CLOSE_TIMEOUT_MS = 60000;
+
+class FeedItemsBigQueryLogger extends BigQueryLogger {
+  async insertRows(rows: FeedItemRow[]): Promise<void> {
+    await this.client
+      .dataset(this.options.BigQueryDataset)
+      .table(this.options.BigQueryLogTable)
+      .insert(rows, { skipInvalidRows: true, ignoreUnknownValues: true });
+  }
+}
+
+// Audit trail of the feed items: rows are buffered and inserted in batches of 500 in the background,
+// one batch at a time. Errors are logged and swallowed — the feed export must never fail because of
+// BigQuery.
+export class FeedItemsBigQueryWriter {
+  private logger: FeedItemsBigQueryLogger;
+  private buffer: FeedItemRow[] = [];
+  private queue: Promise<void> = Promise.resolve();
+  private failedBatches = 0;
+  private insertedRows = 0;
+
+  constructor(private target: FeedItemsBigQueryTarget) {
+    try {
+      const credentials = require(join(process.cwd(), 'credentials/google-cloud-project-id.json'));
+      this.logger = new FeedItemsBigQueryLogger(credentials, {
+        BigQueryProject: target.project,
+        BigQueryDataset: target.dataset,
+        BigQueryLogTable: target.table,
+      });
+    } catch (e) {
+      // First line only: a missing credentials file comes with the whole require stack.
+      console.log(`Feed items BigQuery disabled: ${String(e?.message || e).split('\n')[0]}`);
+    }
+  }
+
+  add(row: FeedItemRow): void {
+    if (!this.enabled) return;
+    this.buffer.push(row);
+    if (this.buffer.length >= BATCH_SIZE) this.flush();
+  }
+
+  // Waits for the pending batches, but never longer than CLOSE_TIMEOUT_MS: a hanging BigQuery
+  // must not hold up the feed export (the XML is already written at this point).
+  async close(): Promise<void> {
+    this.flush();
+    let timer: NodeJS.Timeout | undefined;
+    const timeout = new Promise<'timeout'>((resolve) => {
+      timer = setTimeout(() => resolve('timeout'), CLOSE_TIMEOUT_MS);
+    });
+    const outcome = await Promise.race([this.queue.then(() => 'done' as const), timeout]);
+    if (timer) clearTimeout(timer);
+    if (outcome === 'timeout') console.error(`Feed items BigQuery: gave up waiting after ${CLOSE_TIMEOUT_MS} ms`);
+    if (this.logger) {
+      console.log(`Feed items BigQuery: ${this.insertedRows} rows to ${this.tableName}, ${this.failedBatches} failed batches`);
+    }
+  }
+
+  private get enabled(): boolean {
+    return !!this.logger && this.failedBatches < MAX_FAILED_BATCHES;
+  }
+
+  private get tableName(): string {
+    return `${this.target.project}.${this.target.dataset}.${this.target.table}`;
+  }
+
+  private flush(): void {
+    if (!this.buffer.length) return;
+    const rows = this.buffer;
+    this.buffer = [];
+    this.queue = this.queue.then(() => this.insert(rows));
+  }
+
+  private async insert(rows: FeedItemRow[]): Promise<void> {
+    if (!this.enabled) return;
+    try {
+      await this.logger.insertRows(rows);
+      this.insertedRows += rows.length;
+    } catch (e) {
+      // PartialFailureError: the valid rows went in (skipInvalidRows), `errors` lists the rejected
+      // ones; the first is enough to see the cause.
+      const rejected: any[] = e?.name === 'PartialFailureError' && Array.isArray(e.errors) ? e.errors : null;
+      const firstRowError = rejected?.[0]?.errors?.[0];
+      const detail = firstRowError ? ` (${firstRowError.reason}: ${firstRowError.message})` : '';
+      if (rejected) {
+        this.insertedRows += rows.length - rejected.length;
+        console.log(`Feed items BigQuery rejected ${rejected.length} of ${rows.length} rows${detail}`);
+      } else {
+        this.failedBatches++;
+        console.log(`Feed items BigQuery insert of ${rows.length} rows failed: ${e?.message || e}`);
+      }
+    }
+  }
+}

package/src/transformers/google-feed-envkv.ts

@@ -35,16 +35,26 @@ export function googleEngine(car: DataSchemes.Car): string {
 
 // BEV: combined WLTP range (CoC 49.2). Plug-in: the EAER (`wltpTotalRange`, CoC 49.5) when present,
 // else the range the dealer delivered. City ranges are never used; 0 counts as missing.
-export function googleElectricRange(car: DataSchemes.Car): string | undefined {
-  let range: number | undefined;
-  if (isBev(car)) {
-    range = positive(get(car, 'battery.wltpRange')) ?? positive(get(car, 'battery.batteryRangeElectric'));
-  } else if (isPlugIn(car)) {
-    range = positive(get(car, 'battery.wltpTotalRange'))
-      ?? positive(get(car, 'battery.wltpRange'))
-      ?? positive(get(car, 'battery.batteryRangeElectric'));
+const BEV_RANGE_FIELDS = ['battery.wltpRange', 'battery.batteryRangeElectric'];
+const PLUG_IN_RANGE_FIELDS = ['battery.wltpTotalRange', 'battery.wltpRange', 'battery.batteryRangeElectric'];
+
+const pickElectricRange = (car: DataSchemes.Car): { field: string; range: number } | undefined => {
+  const fields = isBev(car) ? BEV_RANGE_FIELDS : isPlugIn(car) ? PLUG_IN_RANGE_FIELDS : [];
+  for (const field of fields) {
+    const range = positive(get(car, field));
+    if (range) return { field, range };
   }
-  return range ? `${range}km` : undefined;
+  return undefined;
+};
+
+export function googleElectricRange(car: DataSchemes.Car): string | undefined {
+  const picked = pickElectricRange(car);
+  return picked ? `${picked.range}km` : undefined;
+}
+
+// The car field googleElectricRange took its value from (feed audit), null when nothing is sent.
+export function googleElectricRangeSource(car: DataSchemes.Car): string | null {
+  return pickElectricRange(car)?.field ?? null;
 }
 
 // Plug-in hybrids: Google and Pkw-EnVKV (CoC 49.4) want the WEIGHTED combined values for fuel,
@@ -62,9 +72,15 @@ const isConfirmedPlugIn = (car: DataSchemes.Car): boolean => isPlugIn(car) && !!
 // (CBD mirrors kWh into consumptionCombined for them).
 export function googleFuelConsumption(car: DataSchemes.Car): string | undefined {
   if (isBev(car)) return undefined;
-  const value = isConfirmedPlugIn(car)
-    ? positive(get(car, 'consumption.wltpWeightedCombined')) ?? positive(get(car, 'consumption.consumptionCombined'))
-    : positive(get(car, 'consumption.consumptionCombined'));
+  let value: number | undefined;
+  if (isConfirmedPlugIn(car)) {
+    value = positive(get(car, 'consumption.wltpWeightedCombined')) ?? positive(get(car, 'consumption.consumptionCombined'));
+  } else if (isPlugIn(car)) {
+    value = positive(get(car, 'consumption.consumptionCombined')); // unconfirmed plug-in: as before
+  } else {
+    // ICE / HEV: the WLTP combined value the VDP and EnVKV use (EnVKV stage D3), the legacy field as fallback
+    value = positive(get(car, 'consumption.wltpCombined')) ?? positive(get(car, 'consumption.consumptionCombined'));
+  }
   return value ? `${value}l/100km` : undefined;
 }
 
@@ -95,8 +111,10 @@ export function googleEnergyConsumption(car: DataSchemes.Car): string | undefine
 // the stored value. A missing or 0 value is omitted.
 export function googleCo2(car: DataSchemes.Car): string | undefined {
   if (isBev(car)) return '0g/km';
-  const value = isConfirmedPlugIn(car)
-    ? positive(get(car, 'environmentEmissions.wltpCo2')) ?? positive(get(car, 'environmentEmissions.co2'))
-    : positive(get(car, 'environmentEmissions.co2'));
+  // a confirmed plug-in and ICE / HEV send the WLTP value (weighted for the plug-in); an unconfirmed
+  // plug-in keeps the stored legacy value, as before
+  const value = isPlugIn(car) && !isConfirmedPlugIn(car)
+    ? positive(get(car, 'environmentEmissions.co2'))
+    : positive(get(car, 'environmentEmissions.wltpCo2')) ?? positive(get(car, 'environmentEmissions.co2'));
   return value ? `${value}g/km` : undefined;
 }

package/src/transformers/google-feed-item-row.ts

@@ -0,0 +1,71 @@
+import { get, isBoolean } from 'lodash';
+import { DataSchemes } from '@bytebrand/car-schema';
+import { GoogleXmlItem } from '../interfaces';
+import { googleElectricRangeSource } from './google-feed-envkv';
+
+// One BigQuery row per item the Google feed sent (schema/feed-items.json), so a Merchant Center
+// issue can be traced back to the exact values of a run. The attribute strings are the ones in the
+// XML; null means the attribute was omitted.
+export interface FeedItemRow {
+  partitionDay: string;
+  exportedAt: string;
+  feed: string;
+  runId: string | null;
+  itemId: string;
+  vehicleId: string | null;
+  make: string | null;
+  model: string | null;
+  condition: string | null;
+  fuel: string | null;
+  hybridPlugin: boolean | null;
+  engine: string | null;
+  electricRange: string | null;
+  fuelConsumption: string | null;
+  fuelConsumptionDischarged: string | null;
+  energyConsumption: string | null;
+  co2Emissions: string | null;
+  rangeSourceField: string | null;
+  price: number | null;
+  availability: string | null;
+}
+
+const orNull = (value: unknown): string | null =>
+  value === undefined || value === null || value === '' ? null : String(value);
+
+// "12345.5 EUR" -> 12345.5
+const sentPrice = (price: string | undefined): number | null => {
+  const n = parseFloat(price);
+  return Number.isFinite(n) ? n : null;
+};
+
+// exportedAt is the run's start, shared by all rows of a run; partitionDay is its UTC day.
+export function buildGoogleFeedItemRow(
+  car: DataSchemes.Car,
+  item: GoogleXmlItem,
+  runId: string | null,
+  exportedAt: Date,
+): FeedItemRow {
+  const hybridPlugin = get(car, 'engineData.hybridPlugin');
+  return {
+    partitionDay: exportedAt.toISOString().slice(0, 10),
+    exportedAt: exportedAt.toISOString(),
+    feed: 'google',
+    runId: orNull(runId),
+    itemId: String(item.id),
+    vehicleId: orNull(get(car, 'metaData.vehicleId')),
+    make: orNull(item.brand),
+    model: orNull(item.model),
+    condition: orNull(item.condition),
+    fuel: orNull(get(car, 'consumption.fuel')),
+    hybridPlugin: isBoolean(hybridPlugin) ? hybridPlugin : null,
+    engine: orNull(item.engine),
+    electricRange: orNull(item.electric_range),
+    fuelConsumption: orNull(item.fuel_consumption),
+    fuelConsumptionDischarged: orNull(item.fuel_consumption_discharged_battery),
+    energyConsumption: orNull(item.energy_consumption),
+    co2Emissions: orNull(item.co2_emissions),
+    rangeSourceField: item.electric_range ? googleElectricRangeSource(car) : null,
+    price: sentPrice(item.price),
+    availability: orNull(item.availability),
+  };
+}

package/test/google-feed-envkv.test.ts

@@ -87,15 +87,20 @@ const cases: Array<[string, () => void]> = [
     assert.equal(googleEnergyConsumption(phev), undefined);
     assert.equal(googleCo2(phev), '32g/km');
   }],
-  ['non-plug-in hybrids and combustion cars keep the stored values and send no electricity', () => {
+  ['non-plug-in hybrids and combustion cars send the WLTP values (legacy as fallback) and no electricity', () => {
     const hev = car('selector_fuel_hybridElectricalPetrol', {
       consumption: { consumptionCombined: 5.1, wltpCombined: 5.3, wltpWeightedCombined: 2.3, wltpWeightedPowerCombined: 21.5 },
       environmentEmissions: { co2: 116, wltpCo2: 119 },
     });
-    assert.equal(googleFuelConsumption(hev), '5.1l/100km');
+    assert.equal(googleFuelConsumption(hev), '5.3l/100km');
     assert.equal(googleFuelConsumptionDischarged(hev), undefined);
     assert.equal(googleEnergyConsumption(hev), undefined);
-    assert.equal(googleCo2(hev), '116g/km');
+    assert.equal(googleCo2(hev), '119g/km');
+    // D3: Mercedes GLC 200 sent 146.7 g (legacy) while CBD / VDP / EnVKV hold the dealer's WLTP 138 g
+    const glc = car('selector_fuel_diesel', { consumption: { consumptionCombined: 5.6, wltpCombined: 5.2 }, environmentEmissions: { co2: 146.7, wltpCo2: 138 } });
+    assert.equal(googleCo2(glc), '138g/km');
+    assert.equal(googleFuelConsumption(glc), '5.2l/100km');
+    assert.equal(googleCo2(car('selector_fuel_diesel', { environmentEmissions: { co2: 150 } })), '150g/km');
   }],
   ['a plug-in flag without an electric range keeps the stored values (ADV-124 flags every hybrid)', () => {
     const flagged = car('selector_fuel_hybridElectricalPetrol', {

package/test/google-feed-item-row.test.ts

@@ -0,0 +1,121 @@
+// Run: npm test (ts-node, no test framework needed)
+import * as assert from 'node:assert/strict';
+import { buildGoogleFeedItemRow } from '../src/transformers/google-feed-item-row';
+import { googleElectricRange, googleElectricRangeSource } from '../src/transformers/google-feed-envkv';
+
+const exportedAt = new Date('2026-10-08T23:30:00.000Z');
+
+const car = (fuel: string, extra: Record<string, any> = {}): any => ({
+  _id: 'cbd-1',
+  metaData: { vehicleId: 'veh-1' },
+  consumption: { fuel },
+  engineData: 'hybridPlugin' in extra ? { hybridPlugin: extra.hybridPlugin } : {},
+  battery: extra.battery || {},
+});
+
+// Shape of what GoogleTransformer returns after dropping the empty optional attributes.
+const item = (extra: Record<string, any> = {}): any => ({
+  id: 'cbd-1',
+  brand: 'VW',
+  model: 'Golf',
+  condition: 'Used',
+  engine: 'Gasoline',
+  price: '19990.5 EUR',
+  availability: 'in_stock',
+  ...extra,
+});
+
+const cases: Array<[string, () => void]> = [
+  ['combustion car: sent values, omitted attributes are null', () => {
+    const row = buildGoogleFeedItemRow(
+      car('selector_fuel_petrol', { hybridPlugin: false }),
+      item({ fuel_consumption: '5.1l/100km', co2_emissions: '118g/km' }),
+      '12345',
+      exportedAt,
+    );
+    assert.deepEqual(row, {
+      partitionDay: '2026-10-08',
+      exportedAt: '2026-10-08T23:30:00.000Z',
+      feed: 'google',
+      runId: '12345',
+      itemId: 'cbd-1',
+      vehicleId: 'veh-1',
+      make: 'VW',
+      model: 'Golf',
+      condition: 'Used',
+      fuel: 'selector_fuel_petrol',
+      hybridPlugin: false,
+      engine: 'Gasoline',
+      electricRange: null,
+      fuelConsumption: '5.1l/100km',
+      fuelConsumptionDischarged: null,
+      energyConsumption: null,
+      co2Emissions: '118g/km',
+      rangeSourceField: null,
+      price: 19990.5,
+      availability: 'in_stock',
+    });
+  }],
+  ['BEV: range source is the field the range came from', () => {
+    const bev = car('selector_fuel_electrical', { battery: { batteryRangeElectric: 400 } });
+    const row = buildGoogleFeedItemRow(
+      bev,
+      item({ engine: 'Electric', electric_range: googleElectricRange(bev), energy_consumption: '16.5kWh/100km', co2_emissions: '0g/km' }),
+      '12345',
+      exportedAt,
+    );
+    assert.equal(row.electricRange, '400km');
+    assert.equal(row.rangeSourceField, 'battery.batteryRangeElectric');
+    assert.equal(row.hybridPlugin, null);
+    assert.equal(row.energyConsumption, '16.5kWh/100km');
+    assert.equal(row.co2Emissions, '0g/km');
+  }],
+  ['plug-in: EAER first, then WLTP range, then delivered range', () => {
+    const phev = (battery: object) => car('selector_fuel_hybridElectricalPetrol', { hybridPlugin: true, battery });
+    assert.equal(googleElectricRangeSource(phev({ wltpTotalRange: 62, wltpRange: 59 })), 'battery.wltpTotalRange');
+    assert.equal(googleElectricRangeSource(phev({ wltpTotalRange: 0, wltpRange: 59 })), 'battery.wltpRange');
+    assert.equal(googleElectricRangeSource(phev({ batteryRangeElectric: 50 })), 'battery.batteryRangeElectric');
+    assert.equal(googleElectricRangeSource(phev({ wltpTotalRangeCity: 70 })), null);
+  }],
+  ['range source agrees with googleElectricRange', () => {
+    const cars = [
+      car('selector_fuel_electrical', { battery: { wltpRange: 451, batteryRangeElectric: 400 } }),
+      car('selector_fuel_electrical', { battery: { wltpRange: 0 } }),
+      car('selector_fuel_hybridElectricalPetrol', { hybridPlugin: false, battery: { wltpRange: 50 } }),
+      car('selector_fuel_petrol', { battery: { wltpRange: 50 } }),
+    ];
+    for (const c of cars) {
+      assert.equal(googleElectricRangeSource(c) === null, googleElectricRange(c) === undefined);
+    }
+    assert.equal(googleElectricRangeSource(cars[0]), 'battery.wltpRange');
+  }],
+  ['no range source when the item carries no electric_range', () => {
+    const bev = car('selector_fuel_electrical', { battery: { wltpRange: 451 } });
+    assert.equal(buildGoogleFeedItemRow(bev, item(), null, exportedAt).rangeSourceField, null);
+  }],
+  ['missing run id, vehicle id and unparsable price become null', () => {
+    const row = buildGoogleFeedItemRow(
+      { consumption: {} } as any,
+      item({ price: ' EUR', availability: undefined }),
+      undefined,
+      exportedAt,
+    );
+    assert.equal(row.runId, null);
+    assert.equal(row.vehicleId, null);
+    assert.equal(row.fuel, null);
+    assert.equal(row.price, null);
+    assert.equal(row.availability, null);
+  }],
+];
+
+let failed = 0;
+for (const [name, fn] of cases) {
+  try {
+    fn();
+    console.log(`ok   ${name}`);
+  } catch (e) {
+    failed++;
+    console.log(`FAIL ${name}\n     ${(e as Error).message}`);
+  }
+}
+if (failed) process.exit(1);