@bytebrand/export-cars 1.26.26 → 1.26.28

package/src/services/publication-gate.ts

@@ -0,0 +1,119 @@
+import { join } from 'path';
+import { get } from 'lodash';
+import { BigQueryLogger } from '@bytebrand/bigquery-logger';
+import { DataSchemes } from '@bytebrand/car-schema';
+import { PublicationGateConfig } from '../interfaces/params.interface';
+import { GoogleEnvkvAttribute, googleEnvkvSourceFields } from '../transformers/google-feed-envkv';
+
+// EnVKV stage C8 — publication gate for estimated EnVKV values in the Google feed.
+//
+// No car field says whether a value was delivered or estimated; car-basic-data writes every change it
+// makes to the enrichment audit log (BigQuery). At the start of the export the gate reads the latest
+// log row per car and field; a field counts as estimated when that row came from an estimate source
+// and its final value is still the value stored on the car (a later import with a delivered value
+// replaces it without a log row, so the stored value no longer matches).
+//
+//   off      no BigQuery call, the feed as before (default)
+//   audit    the feed as before, the estimated attributes are recorded in the feed items table
+//   enforce  estimated attributes are left out of the item (and recorded)
+
+// finalValueSource values of CBD's enrichment log that are estimates: the CO2 estimator (EEA register /
+// model), the make-model template sheet and the global defaults.
+export const ESTIMATE_SOURCES = ['co2-estimate', 'mmm', 'globalDefault'];
+
+// Every car field a Google EnVKV attribute can be taken from (google-feed-envkv picks).
+export const GATED_FIELDS = [
+  'consumption.wltpWeightedCombined',
+  'consumption.wltpCombined',
+  'consumption.consumptionCombined',
+  'consumption.wltpWeightedPowerCombined',
+  'consumption.wltpPowerCombined',
+  'consumption.consumptionPowerCombined',
+  'environmentEmissions.wltpCo2',
+  'environmentEmissions.co2',
+  'battery.wltpTotalRange',
+  'battery.wltpRange',
+  'battery.batteryRangeElectric',
+];
+
+// '7.10' and 7.1 are the same value; non-numeric values compare as strings.
+const sameValue = (logged: string, stored: unknown): boolean => {
+  if (stored === undefined || stored === null) return false;
+  const a = Number(logged);
+  const b = Number(stored);
+  if (Number.isFinite(a) && Number.isFinite(b)) return Math.abs(a - b) < 1e-9;
+  return logged === String(stored);
+};
+
+export class PublicationGate {
+  // estimates: `${carId}|${field}` -> final value of the latest estimate row
+  constructor(readonly mode: 'off' | 'audit' | 'enforce', private estimates: Map<string, string>) {}
+
+  static off(): PublicationGate {
+    return new PublicationGate('off', new Map());
+  }
+
+  get size(): number {
+    return this.estimates.size;
+  }
+
+  // The attributes of this car whose value comes from an estimated field, in a fixed order.
+  estimatedAttributes(car: DataSchemes.Car): GoogleEnvkvAttribute[] {
+    if (this.mode === 'off') return [];
+    const carId = String(get(car, '_id', ''));
+    const attributes: GoogleEnvkvAttribute[] = [];
+    for (const [attribute, field] of Object.entries(googleEnvkvSourceFields(car))) {
+      const logged = this.estimates.get(`${carId}|${field}`);
+      if (logged !== undefined && sameValue(logged, get(car, field))) attributes.push(attribute as GoogleEnvkvAttribute);
+    }
+    return attributes;
+  }
+
+  // enforce: drops the estimated attributes from the item. Returns the item for chaining.
+  apply<T extends Partial<Record<GoogleEnvkvAttribute, string>>>(item: T, attributes: GoogleEnvkvAttribute[]): T {
+    if (this.mode !== 'enforce') return item;
+    for (const attribute of attributes) delete item[attribute];
+    return item;
+  }
+}
+
+class EnrichmentLogReader extends BigQueryLogger {
+  async latestEstimates(table: string): Promise<Map<string, string>> {
+    const query = `
+      SELECT carId, fieldName, finalValue
+      FROM (
+        SELECT carId, fieldName, finalValue, finalValueSource,
+               ROW_NUMBER() OVER (PARTITION BY carId, fieldName ORDER BY timestamp DESC) AS rn
+        FROM \`${table}\`
+        WHERE fieldName IN UNNEST(@fields)
+      )
+      WHERE rn = 1 AND finalValueSource IN UNNEST(@sources) AND finalValue IS NOT NULL`;
+    const [rows] = await this.client.query({ query, params: { fields: GATED_FIELDS, sources: ESTIMATE_SOURCES } });
+    const estimates = new Map<string, string>();
+    for (const row of rows) estimates.set(`${row.carId}|${row.fieldName}`, String(row.finalValue));
+    return estimates;
+  }
+}
+
+// Reads the estimates once per run. audit: a failure only switches the gate off. enforce: a failure
+// fails the export, so the last published feed stays in place instead of a feed with unchecked estimates.
+export async function loadPublicationGate(config: PublicationGateConfig | undefined): Promise<PublicationGate> {
+  if (!config || config.mode === 'off') return PublicationGate.off();
+  const table = `${config.project}.${config.dataset}.${config.table}`;
+  try {
+    const credentials = require(join(process.cwd(), 'credentials/google-cloud-project-id.json'));
+    const reader = new EnrichmentLogReader(credentials, {
+      BigQueryProject: config.project,
+      BigQueryDataset: config.dataset,
+      BigQueryLogTable: config.table,
+    });
+    const estimates = await reader.latestEstimates(table);
+    console.log(`Publication gate ${config.mode}: ${estimates.size} estimated car fields from ${table}`);
+    return new PublicationGate(config.mode, estimates);
+  } catch (e) {
+    const message = String(e?.message || e).split('\n')[0];
+    if (config.mode === 'enforce') throw new Error(`Publication gate: cannot read ${table}: ${message}`);
+    console.error(`Publication gate disabled for this run: cannot read ${table}: ${message}`);
+    return PublicationGate.off();
+  }
+}

package/src/transformers/google-feed-envkv.ts

@@ -70,43 +70,99 @@ const isConfirmedPlugIn = (car: DataSchemes.Car): boolean => isPlugIn(car) && !!
 
 // Missing or 0 is omitted instead of sending "0l/100km". BEVs carry no fuel consumption
 // (CBD mirrors kWh into consumptionCombined for them).
-export function googleFuelConsumption(car: DataSchemes.Car): string | undefined {
-  if (isBev(car)) return undefined;
-  const value = isConfirmedPlugIn(car)
-    ? positive(get(car, 'consumption.wltpWeightedCombined')) ?? positive(get(car, 'consumption.consumptionCombined'))
-    : positive(get(car, 'consumption.consumptionCombined'));
-  return value ? `${value}l/100km` : undefined;
+//
+// Each attribute is picked from the first positive field of a list; the pick keeps the field so the
+// publication gate (EnVKV C8) can tell whether the sent value was estimated by CBD.
+interface Picked {
+  field: string;
+  value: number;
 }
 
+const firstPositive = (car: DataSchemes.Car, fields: string[]): Picked | undefined => {
+  for (const field of fields) {
+    const value = positive(get(car, field));
+    if (value) return { field, value };
+  }
+  return undefined;
+};
+
+const pickFuelConsumption = (car: DataSchemes.Car): Picked | undefined => {
+  if (isBev(car)) return undefined;
+  if (isConfirmedPlugIn(car)) return firstPositive(car, ['consumption.wltpWeightedCombined', 'consumption.consumptionCombined']);
+  if (isPlugIn(car)) return firstPositive(car, ['consumption.consumptionCombined']); // unconfirmed plug-in: as before
+  // ICE / HEV: the WLTP combined value the VDP and EnVKV use (EnVKV stage D3), the legacy field as fallback
+  return firstPositive(car, ['consumption.wltpCombined', 'consumption.consumptionCombined']);
+};
+
 // Confirmed plug-in: wltpCombined, no fallback (the legacy combined value is close to the weighted
 // one, sending it here would repeat the weighted figure). Unconfirmed plug-in: the stored combined
 // value, as before.
-export function googleFuelConsumptionDischarged(car: DataSchemes.Car): string | undefined {
+const pickFuelConsumptionDischarged = (car: DataSchemes.Car): Picked | undefined => {
   if (!isPlugIn(car)) return undefined;
-  const value = isConfirmedPlugIn(car)
-    ? positive(get(car, 'consumption.wltpCombined'))
-    : positive(get(car, 'consumption.consumptionCombined')) ?? positive(get(car, 'consumption.wltpCombined'));
-  return value ? `${value}l/100km` : undefined;
-}
+  return isConfirmedPlugIn(car)
+    ? firstPositive(car, ['consumption.wltpCombined'])
+    : firstPositive(car, ['consumption.consumptionCombined', 'consumption.wltpCombined']);
+};
 
 // BEV: combined consumption. Confirmed plug-in: weighted combined. Unconfirmed plug-in: the stored
 // electric value, as before. Nothing for other cars.
+const pickEnergyConsumption = (car: DataSchemes.Car): Picked | undefined => {
+  if (isConfirmedPlugIn(car)) return firstPositive(car, ['consumption.wltpWeightedPowerCombined']);
+  if (isBev(car) || isPlugIn(car)) return firstPositive(car, ['consumption.wltpPowerCombined', 'consumption.consumptionPowerCombined']);
+  return undefined;
+};
+
+// A BEV emits 0 g/km by definition (no field behind it); a confirmed plug-in and ICE / HEV send the
+// WLTP value (weighted for the plug-in); an unconfirmed plug-in keeps the stored legacy value, as before.
+const pickCo2 = (car: DataSchemes.Car): Picked | undefined => {
+  if (isBev(car)) return undefined;
+  return isPlugIn(car) && !isConfirmedPlugIn(car)
+    ? firstPositive(car, ['environmentEmissions.co2'])
+    : firstPositive(car, ['environmentEmissions.wltpCo2', 'environmentEmissions.co2']);
+};
+
+export function googleFuelConsumption(car: DataSchemes.Car): string | undefined {
+  const picked = pickFuelConsumption(car);
+  return picked ? `${picked.value}l/100km` : undefined;
+}
+
+export function googleFuelConsumptionDischarged(car: DataSchemes.Car): string | undefined {
+  const picked = pickFuelConsumptionDischarged(car);
+  return picked ? `${picked.value}l/100km` : undefined;
+}
+
 export function googleEnergyConsumption(car: DataSchemes.Car): string | undefined {
-  let value: number | undefined;
-  if (isConfirmedPlugIn(car)) {
-    value = positive(get(car, 'consumption.wltpWeightedPowerCombined'));
-  } else if (isBev(car) || isPlugIn(car)) {
-    value = positive(get(car, 'consumption.wltpPowerCombined')) ?? positive(get(car, 'consumption.consumptionPowerCombined'));
-  }
-  return value ? `${value}kWh/100km` : undefined;
+  const picked = pickEnergyConsumption(car);
+  return picked ? `${picked.value}kWh/100km` : undefined;
 }
 
-// A BEV emits 0 g/km by definition; a confirmed plug-in sends the weighted WLTP CO2; every other car
-// the stored value. A missing or 0 value is omitted.
+// A missing or 0 value is omitted.
 export function googleCo2(car: DataSchemes.Car): string | undefined {
   if (isBev(car)) return '0g/km';
-  const value = isConfirmedPlugIn(car)
-    ? positive(get(car, 'environmentEmissions.wltpCo2')) ?? positive(get(car, 'environmentEmissions.co2'))
-    : positive(get(car, 'environmentEmissions.co2'));
-  return value ? `${value}g/km` : undefined;
+  const picked = pickCo2(car);
+  return picked ? `${picked.value}g/km` : undefined;
+}
+
+// The EnVKV attributes of the Google item (keys of GoogleXmlItem).
+export type GoogleEnvkvAttribute =
+  | 'fuel_consumption'
+  | 'fuel_consumption_discharged_battery'
+  | 'energy_consumption'
+  | 'electric_range'
+  | 'co2_emissions';
+
+// For every EnVKV attribute the item sends: the car field its value came from.
+export function googleEnvkvSourceFields(car: DataSchemes.Car): Partial<Record<GoogleEnvkvAttribute, string>> {
+  const picks: Record<GoogleEnvkvAttribute, { field: string } | undefined> = {
+    fuel_consumption: pickFuelConsumption(car),
+    fuel_consumption_discharged_battery: pickFuelConsumptionDischarged(car),
+    energy_consumption: pickEnergyConsumption(car),
+    electric_range: pickElectricRange(car),
+    co2_emissions: pickCo2(car),
+  };
+  const fields: Partial<Record<GoogleEnvkvAttribute, string>> = {};
+  for (const [attribute, picked] of Object.entries(picks)) {
+    if (picked) fields[attribute as GoogleEnvkvAttribute] = picked.field;
+  }
+  return fields;
 }

package/src/transformers/google-feed-item-row.ts

@@ -1,71 +1,78 @@
-import { get, isBoolean } from 'lodash';
-import { DataSchemes } from '@bytebrand/car-schema';
-import { GoogleXmlItem } from '../interfaces';
-import { googleElectricRangeSource } from './google-feed-envkv';
-
-// One BigQuery row per item the Google feed sent (schema/feed-items.json), so a Merchant Center
-// issue can be traced back to the exact values of a run. The attribute strings are the ones in the
-// XML; null means the attribute was omitted.
-export interface FeedItemRow {
-  partitionDay: string;
-  exportedAt: string;
-  feed: string;
-  runId: string | null;
-  itemId: string;
-  vehicleId: string | null;
-  make: string | null;
-  model: string | null;
-  condition: string | null;
-  fuel: string | null;
-  hybridPlugin: boolean | null;
-  engine: string | null;
-  electricRange: string | null;
-  fuelConsumption: string | null;
-  fuelConsumptionDischarged: string | null;
-  energyConsumption: string | null;
-  co2Emissions: string | null;
-  rangeSourceField: string | null;
-  price: number | null;
-  availability: string | null;
-}
-
-const orNull = (value: unknown): string | null =>
-  value === undefined || value === null || value === '' ? null : String(value);
-
-// "12345.5 EUR" -> 12345.5
-const sentPrice = (price: string | undefined): number | null => {
-  const n = parseFloat(price);
-  return Number.isFinite(n) ? n : null;
-};
-
-// exportedAt is the run's start, shared by all rows of a run; partitionDay is its UTC day.
-export function buildGoogleFeedItemRow(
-  car: DataSchemes.Car,
-  item: GoogleXmlItem,
-  runId: string | null,
-  exportedAt: Date,
-): FeedItemRow {
-  const hybridPlugin = get(car, 'engineData.hybridPlugin');
-  return {
-    partitionDay: exportedAt.toISOString().slice(0, 10),
-    exportedAt: exportedAt.toISOString(),
-    feed: 'google',
-    runId: orNull(runId),
-    itemId: String(item.id),
-    vehicleId: orNull(get(car, 'metaData.vehicleId')),
-    make: orNull(item.brand),
-    model: orNull(item.model),
-    condition: orNull(item.condition),
-    fuel: orNull(get(car, 'consumption.fuel')),
-    hybridPlugin: isBoolean(hybridPlugin) ? hybridPlugin : null,
-    engine: orNull(item.engine),
-    electricRange: orNull(item.electric_range),
-    fuelConsumption: orNull(item.fuel_consumption),
-    fuelConsumptionDischarged: orNull(item.fuel_consumption_discharged_battery),
-    energyConsumption: orNull(item.energy_consumption),
-    co2Emissions: orNull(item.co2_emissions),
-    rangeSourceField: item.electric_range ? googleElectricRangeSource(car) : null,
-    price: sentPrice(item.price),
-    availability: orNull(item.availability),
-  };
-}
+import { get, isBoolean } from 'lodash';
+import { DataSchemes } from '@bytebrand/car-schema';
+import { GoogleXmlItem } from '../interfaces';
+import { googleElectricRangeSource } from './google-feed-envkv';
+
+// One BigQuery row per item the Google feed sent (schema/feed-items.json), so a Merchant Center
+// issue can be traced back to the exact values of a run. The attribute strings are the ones in the
+// XML; null means the attribute was omitted.
+export interface FeedItemRow {
+  partitionDay: string;
+  exportedAt: string;
+  feed: string;
+  runId: string | null;
+  itemId: string;
+  vehicleId: string | null;
+  make: string | null;
+  model: string | null;
+  condition: string | null;
+  fuel: string | null;
+  hybridPlugin: boolean | null;
+  engine: string | null;
+  electricRange: string | null;
+  fuelConsumption: string | null;
+  fuelConsumptionDischarged: string | null;
+  energyConsumption: string | null;
+  co2Emissions: string | null;
+  rangeSourceField: string | null;
+  price: number | null;
+  availability: string | null;
+  // EnVKV C8: gate mode of the run and the attributes whose value CBD estimated (comma-separated);
+  // in enforce mode these attributes were left out of the item.
+  publicationGate: string | null;
+  estimatedAttributes: string | null;
+}
+
+const orNull = (value: unknown): string | null =>
+  value === undefined || value === null || value === '' ? null : String(value);
+
+// "12345.5 EUR" -> 12345.5
+const sentPrice = (price: string | undefined): number | null => {
+  const n = parseFloat(price);
+  return Number.isFinite(n) ? n : null;
+};
+
+// exportedAt is the run's start, shared by all rows of a run; partitionDay is its UTC day.
+export function buildGoogleFeedItemRow(
+  car: DataSchemes.Car,
+  item: GoogleXmlItem,
+  runId: string | null,
+  exportedAt: Date,
+  gate: { mode: string; estimatedAttributes: string[] } | null = null,
+): FeedItemRow {
+  const hybridPlugin = get(car, 'engineData.hybridPlugin');
+  return {
+    partitionDay: exportedAt.toISOString().slice(0, 10),
+    exportedAt: exportedAt.toISOString(),
+    feed: 'google',
+    runId: orNull(runId),
+    itemId: String(item.id),
+    vehicleId: orNull(get(car, 'metaData.vehicleId')),
+    make: orNull(item.brand),
+    model: orNull(item.model),
+    condition: orNull(item.condition),
+    fuel: orNull(get(car, 'consumption.fuel')),
+    hybridPlugin: isBoolean(hybridPlugin) ? hybridPlugin : null,
+    engine: orNull(item.engine),
+    electricRange: orNull(item.electric_range),
+    fuelConsumption: orNull(item.fuel_consumption),
+    fuelConsumptionDischarged: orNull(item.fuel_consumption_discharged_battery),
+    energyConsumption: orNull(item.energy_consumption),
+    co2Emissions: orNull(item.co2_emissions),
+    rangeSourceField: item.electric_range ? googleElectricRangeSource(car) : null,
+    price: sentPrice(item.price),
+    availability: orNull(item.availability),
+    publicationGate: gate ? gate.mode : null,
+    estimatedAttributes: gate && gate.estimatedAttributes.length ? gate.estimatedAttributes.join(',') : null,
+  };
+}

package/test/google-feed-envkv.test.ts

@@ -87,15 +87,20 @@ const cases: Array<[string, () => void]> = [
     assert.equal(googleEnergyConsumption(phev), undefined);
     assert.equal(googleCo2(phev), '32g/km');
   }],
-  ['non-plug-in hybrids and combustion cars keep the stored values and send no electricity', () => {
+  ['non-plug-in hybrids and combustion cars send the WLTP values (legacy as fallback) and no electricity', () => {
     const hev = car('selector_fuel_hybridElectricalPetrol', {
       consumption: { consumptionCombined: 5.1, wltpCombined: 5.3, wltpWeightedCombined: 2.3, wltpWeightedPowerCombined: 21.5 },
       environmentEmissions: { co2: 116, wltpCo2: 119 },
     });
-    assert.equal(googleFuelConsumption(hev), '5.1l/100km');
+    assert.equal(googleFuelConsumption(hev), '5.3l/100km');
     assert.equal(googleFuelConsumptionDischarged(hev), undefined);
     assert.equal(googleEnergyConsumption(hev), undefined);
-    assert.equal(googleCo2(hev), '116g/km');
+    assert.equal(googleCo2(hev), '119g/km');
+    // D3: Mercedes GLC 200 sent 146.7 g (legacy) while CBD / VDP / EnVKV hold the dealer's WLTP 138 g
+    const glc = car('selector_fuel_diesel', { consumption: { consumptionCombined: 5.6, wltpCombined: 5.2 }, environmentEmissions: { co2: 146.7, wltpCo2: 138 } });
+    assert.equal(googleCo2(glc), '138g/km');
+    assert.equal(googleFuelConsumption(glc), '5.2l/100km');
+    assert.equal(googleCo2(car('selector_fuel_diesel', { environmentEmissions: { co2: 150 } })), '150g/km');
   }],
   ['a plug-in flag without an electric range keeps the stored values (ADV-124 flags every hybrid)', () => {
     const flagged = car('selector_fuel_hybridElectricalPetrol', {