@bytebrand/export-cars 1.26.18 → 1.26.19-hotfix.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bytebrand/export-cars",
-  "version": "1.26.18",
+  "version": "1.26.19-hotfix.2",
   "description": "",
   "author": "bytebrand",
   "license": "UNLICENSED",
@@ -10,10 +10,12 @@
     "prepublishOnly": "npm run build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
     "lint": "eslint -c .eslintrc.js --ext .ts src/",
-    "start:dev": "nest start --watch"
+    "start:dev": "nest start --watch",
+    "test": "ts-node --transpile-only test/google-feed-envkv.test.ts && ts-node --transpile-only test/google-feed-item-row.test.ts && ts-node --transpile-only test/publication-gate.test.ts"
   },
   "dependencies": {
     "@bytebrand/autode-request": "^1.19.2",
+    "@bytebrand/bigquery-logger": "^2.1.2",
     "@bytebrand/car-basic-data-scope": "^9.11.24",
     "@bytebrand/car-description": "^1.17.20",
     "@bytebrand/car-schema": "^3.14.15",

package/schema/feed-items.json

@@ -0,0 +1,134 @@
+[
+  {
+    "name": "partitionDay",
+    "type": "DATE",
+    "mode": "NULLABLE",
+    "description": "UTC day of exportedAt; the table is partitioned by it"
+  },
+  {
+    "name": "exportedAt",
+    "type": "TIMESTAMP",
+    "mode": "NULLABLE",
+    "description": "Start of the export run (same for every row of a run)"
+  },
+  {
+    "name": "feed",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "Feed the item was written to ('google')"
+  },
+  {
+    "name": "runId",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "Cronicle job id of the run"
+  },
+  {
+    "name": "itemId",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:id> = CBD _id"
+  },
+  {
+    "name": "vehicleId",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "metaData.vehicleId"
+  },
+  {
+    "name": "make",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:brand> as sent"
+  },
+  {
+    "name": "model",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:model> as sent"
+  },
+  {
+    "name": "condition",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:condition> as sent"
+  },
+  {
+    "name": "fuel",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "consumption.fuel selector"
+  },
+  {
+    "name": "hybridPlugin",
+    "type": "BOOLEAN",
+    "mode": "NULLABLE",
+    "description": "engineData.hybridPlugin, null when unset"
+  },
+  {
+    "name": "engine",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:engine> as sent"
+  },
+  {
+    "name": "electricRange",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:electric_range> as sent, null when omitted"
+  },
+  {
+    "name": "fuelConsumption",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:fuel_consumption> as sent, null when omitted"
+  },
+  {
+    "name": "fuelConsumptionDischarged",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:fuel_consumption_discharged_battery> as sent, null when omitted"
+  },
+  {
+    "name": "energyConsumption",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:energy_consumption> as sent, null when omitted"
+  },
+  {
+    "name": "co2Emissions",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:co2_emissions> as sent, null when omitted"
+  },
+  {
+    "name": "rangeSourceField",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "Car field electric_range came from: battery.wltpRange / battery.wltpTotalRange / battery.batteryRangeElectric"
+  },
+  {
+    "name": "price",
+    "type": "FLOAT",
+    "mode": "NULLABLE",
+    "description": "<g:price> amount as sent (EUR)"
+  },
+  {
+    "name": "availability",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:availability> as sent"
+  },
+  {
+    "name": "publicationGate",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "EnVKV C8 gate mode of the run (audit / enforce), null when off"
+  },
+  {
+    "name": "estimatedAttributes",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "Comma-separated EnVKV attributes whose value CBD estimated; left out of the item in enforce mode"
+  }
+]

package/src/configs/config.ts

@@ -1,5 +1,5 @@
 import { Environment, ExportType, ImageType } from '../enums';
-import { Params } from '../interfaces/params.interface';
+import { FeedItemsBigQueryTarget, Params, PublicationGateConfig } from '../interfaces/params.interface';
 
 export class Config {
   apikey: string;
@@ -39,6 +39,12 @@ export class Config {
   dealersSpreadsheetId: string;
   dealersSpreadsheetTitle: string;
   metaPushPriorityCsvUrl: string;
+  // Cronicle job id of the run (set by ExportRunner), stored with the feed item rows.
+  runId?: string;
+  // Set only when the job configures the feed items BigQuery table; unset = no BigQuery calls.
+  feedItemsBigQuery?: FeedItemsBigQueryTarget;
+  // EnVKV C8 publication gate of the Google feed; unset = off.
+  publicationGate?: PublicationGateConfig;
 
   protected environmentConfig = {
     [Environment.DEVELOP]: {

package/src/configs/facebook-shop.config.ts

@@ -1,13 +1,13 @@
 import { Params } from '../interfaces/params.interface';
 import { Config } from './config';
 import { ExportType, FileType } from '../enums';
-import { MAX_FEED_SALES_PRICE_EUR } from '../constants/feed-price.constants';
+import { resolveMaxFeedSalesPrice } from '../constants/feed-price.constants';
 
 export class FacebookShopConfig extends Config {
   constructor(params: Params) {
     super(params);
     this.pathesConfig.localConvertedDirectory = `resources/${ExportType.FACEBOOK_SHOP}-export`;
-    this.maxCurrentSalesPrice = MAX_FEED_SALES_PRICE_EUR;
+    this.maxCurrentSalesPrice = resolveMaxFeedSalesPrice(params.maxCurrentSalesPrice);
     this.imageUrl = this.environmentConfig[params.environment].baseImageUrl;
     this.outputFiles = [
       `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.FACEBOOK_SHOP}.${FileType.XML}`,

package/src/configs/fb.config.ts

@@ -1,5 +1,5 @@
 import { ExportType, FileType, ImageType } from 'src/enums';
-import { MAX_FEED_SALES_PRICE_EUR } from '../constants/feed-price.constants';
+import { resolveMaxFeedSalesPrice } from '../constants/feed-price.constants';
 import { Params } from '../interfaces/params.interface';
 import { Config } from './config';
 
@@ -7,7 +7,7 @@ export class FbConfig extends Config {
   constructor(params: Params) {
     super(params);
     this.pathesConfig.localConvertedDirectory = `resources/${ExportType.FB}-export`;
-    this.maxCurrentSalesPrice = MAX_FEED_SALES_PRICE_EUR;
+    this.maxCurrentSalesPrice = resolveMaxFeedSalesPrice(params.maxCurrentSalesPrice);
     this.imageUrl = this.environmentConfig[params.environment].imageUrl;
     this.imageType = ImageType.marketing;
     this.marketingImageConfigId = this.environmentConfig[params.environment].fbMarketingImageId;

package/src/configs/google.config.ts

@@ -1,14 +1,39 @@
 import { ExportType, FileType, ImageType } from 'src/enums';
-import { MAX_FEED_SALES_PRICE_EUR } from '../constants/feed-price.constants';
-import { Params } from '../interfaces/params.interface';
+import { resolveMaxFeedSalesPrice } from '../constants/feed-price.constants';
+import { Params, PublicationGateConfig } from '../interfaces/params.interface';
 import { Config } from './config';
 
+const trimmed = (value: unknown): string => (typeof value === 'string' ? value.trim() : '');
+
 export class GoogleConfig extends Config {
   constructor(params: Params) {
     super(params);
     this.pathesConfig.localConvertedDirectory = `resources/${ExportType.GOOGLE}-export`;
     this.imageType = ImageType.default;
-    this.maxCurrentSalesPrice = MAX_FEED_SALES_PRICE_EUR;
+    this.maxCurrentSalesPrice = resolveMaxFeedSalesPrice(params.maxCurrentSalesPrice);
+    const feedItemsBigQuery = {
+      project: trimmed(params.feedItemsBqProject),
+      dataset: trimmed(params.feedItemsBqDataset),
+      table: trimmed(params.feedItemsBqTable),
+    };
+    if (feedItemsBigQuery.project && feedItemsBigQuery.dataset && feedItemsBigQuery.table) {
+      this.feedItemsBigQuery = feedItemsBigQuery;
+    }
+    const gateMode = trimmed(params.publicationGate).toLowerCase();
+    if (gateMode === 'audit' || gateMode === 'enforce') {
+      const gate = {
+        mode: gateMode as PublicationGateConfig['mode'],
+        project: trimmed(params.publicationGateBqProject),
+        dataset: trimmed(params.publicationGateBqDataset),
+        table: trimmed(params.publicationGateBqTable),
+      };
+      if (!gate.project || !gate.dataset || !gate.table) {
+        throw new Error(`publicationGate=${gateMode} needs publicationGateBqProject, publicationGateBqDataset and publicationGateBqTable`);
+      }
+      this.publicationGate = gate;
+    } else if (gateMode && gateMode !== 'off') {
+      throw new Error(`publicationGate must be off, audit or enforce, got "${gateMode}"`);
+    }
     this.outputFiles = [
       `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.GOOGLE}.${FileType.XML}`,
     ];

package/src/constants/feed-price.constants.ts

@@ -1,4 +1,11 @@
+import { toNumber } from 'lodash';
+
 // Shared eligibility cap for the Google Vehicle Ads and Meta AIA/AMA feeds:
 // only cars whose end-customer selling price (price.currentSalesPrice) is
 // ≤ €80,000 may be exported. €80,000.00 itself is still included.
 export const MAX_FEED_SALES_PRICE_EUR = 80_000;
+
+// The Cronicle "Max currentSalesPrice" plugin param (a text field) can override
+// the default cap per event; empty/invalid/non-positive values fall back to it.
+export const resolveMaxFeedSalesPrice = (raw?: string): number =>
+  toNumber(raw) > 0 ? toNumber(raw) : MAX_FEED_SALES_PRICE_EUR;

package/src/export.command.ts

@@ -27,6 +27,7 @@ export class ExportRunner extends CommandRunner {
         LoggerService.log(`Running job: ${input.id} - ${JSON.stringify(input)} \n`);
 
         this.configService.config = new exportSchema[input.params.exportType].config(input.params);
+        this.configService.config.runId = input.id;
         const exporter = await this.moduleRef.create<Exporter>(exportSchema[input.params.exportType].exporter);
         await exporter.export().catch(reject);
 

package/src/exporters/facebook-shop.exporter.ts

@@ -1,6 +1,5 @@
 import { Injectable } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
-import { basename } from 'path';
 
 import { CronicleUtils } from '@bytebrand/cronicle-helper';
 import { FsChunkInterface } from '@bytebrand/chunkable';
@@ -25,9 +24,8 @@ export class FacebookShopExporter implements Exporter {
     await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
     await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
 
-    const selfUrl = `https://ftp.${this.configService.config.domain}/feeds/${basename(fileXml)}`;
     const facebookShopXmlBuilder = new FacebookShopXmlBuilder(fileXml);
-    await facebookShopXmlBuilder.buildStart(selfUrl);
+    await facebookShopXmlBuilder.buildStart();
 
     const facebookShopTransformer = await this.moduleRef.create(FacebookShopTransformer);
 

package/src/exporters/google.exporter.ts

@@ -6,6 +6,9 @@ import { FsChunkInterface } from '@bytebrand/chunkable';
 
 import { CarService, ConfigService, LoaderService, RulesService } from '../services';
 import { GoogleTransformer } from '../transformers/google.transformer';
+import { buildGoogleFeedItemRow } from '../transformers/google-feed-item-row';
+import { FeedItemsBigQueryWriter } from '../services/feed-items-bigquery.writer';
+import { loadPublicationGate } from '../services/publication-gate';
 import { GoogleXmlBuilder } from '../formats/google-xml.builder';
 import { Exporter } from '../interfaces/exporter.interface';
 
@@ -20,6 +23,9 @@ export class GoogleExporter implements Exporter {
   ) { }
 
   async export(): Promise<void> {
+    // EnVKV C8: estimated EnVKV values (CBD enrichment log); off unless the job sets publicationGate.
+    // Read first: in enforce mode a failure stops the run before the output directory is emptied.
+    const gate = await loadPublicationGate(this.configService.config.publicationGate);
     const [fileName] = this.configService.config.outputFiles;
     await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
     await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
@@ -29,18 +35,33 @@ export class GoogleExporter implements Exporter {
 
     const googleTransformer = await this.moduleRef.create(GoogleTransformer);
 
+    // Opt-in audit of the sent items (feedItemsBq* job params); null = no BigQuery calls at all.
+    const { feedItemsBigQuery, runId } = this.configService.config;
+    const feedItemsWriter = feedItemsBigQuery ? new FeedItemsBigQueryWriter(feedItemsBigQuery) : null;
+    const exportedAt = new Date();
+    let gatedItems = 0;
+
     await this.loaderService.process(async (chunk: FsChunkInterface) => {
       chunk.data = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
 
       for (const data of chunk.data) {
         const car = await googleTransformer.transform(data);
-        if (car)
-          await googleXmlBuilder.addItem(car);
+        if (car) {
+          const estimatedAttributes = gate.estimatedAttributes(data.car);
+          if (estimatedAttributes.length) gatedItems += 1;
+          await googleXmlBuilder.addItem(gate.apply(car, estimatedAttributes));
+          const gateInfo = gate.mode === 'off' ? null : { mode: gate.mode, estimatedAttributes };
+          feedItemsWriter?.add(buildGoogleFeedItemRow(data.car, car, runId, exportedAt, gateInfo));
+        }
         this.carService.counter.countCar(data.images);
       }
     });
 
     await googleXmlBuilder.buildEnd();
+    if (gate.mode !== 'off') {
+      console.log(`Publication gate ${gate.mode}: ${gatedItems} items with estimated EnVKV values${gate.mode === 'enforce' ? ' (attributes left out)' : ''}`);
+    }
+    await feedItemsWriter?.close();
     console.log(this.carService.counter.getCountedCars());
   }
 }

package/src/formats/facebook-shop.builder.ts

@@ -8,18 +8,17 @@ export class FacebookShopXmlBuilder extends XMLBuilderBase {
   // No xmlns:g / g: prefixes here — that namespace belongs to the old Google Merchant /
   // Facebook Shop product-catalog schema. Meta's Automotive Model Ads Vehicle Offers Feed
   // uses plain, undecorated tag names (see facebook-shop.builder field reference above).
-  async buildStart(selfUrl?: string): Promise<void> {
+  // Envelope is <listings>/<listing>, not RSS <rss><channel>/<item>: Meta Commerce Manager
+  // rejects the RSS form for a Vehicle Offers catalog as an invalid file format (ADV-369).
+  async buildStart(): Promise<void> {
     await this.write(
-      '<?xml version="1.0" encoding="utf-8"?>\n<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">' +
-        '\n<channel>\n<title>auto.de Vehicle Offers Feed</title>' +
-        '\n<description>Automotive Model Ads Vehicle Offers Feed für auto.de</description>' +
-        '\n<link>https://www.auto.de/</link>' +
-        `\n<atom:link href="${escape(selfUrl)}" rel="self" type="application/rss+xml" />\n`,
+      '<?xml version="1.0" encoding="utf-8"?>\n<listings>' +
+        '\n<title>auto.de Vehicle Offers Feed</title>\n',
     );
   }
 
   async addItem(car: FacebookShopItem): Promise<void> {
-    let item = '\n<item>\n';
+    let item = '\n<listing>\n';
 
     const { image, ...tags } = car;
 
@@ -32,7 +31,7 @@ export class FacebookShopXmlBuilder extends XMLBuilderBase {
       .map((url: string) => `<image><url>${escape(url)}</url></image>\n`)
       .join('');
 
-    item += '</item>';
+    item += '</listing>';
 
     await this.write(item);
   }
@@ -53,7 +52,7 @@ export class FacebookShopXmlBuilder extends XMLBuilderBase {
   }
 
   async buildEnd(): Promise<void> {
-    await this.write('\n</channel>\n</rss>\n');
+    await this.write('\n</listings>\n');
     await this.end();
   }
 }

package/src/interfaces/export-item.interface.ts

@@ -199,11 +199,11 @@ export interface GoogleXmlItem {
   doors?: number;
   capacity?: number;
   date_first_registered: string,
-  fuel_consumption: string,
+  fuel_consumption?: string,
   energy_consumption?: string,
   electric_range?: string,
   fuel_consumption_discharged_battery?: string,
-  co2_emissions: string,
+  co2_emissions?: string,
   identifier_exists: string,
   ads_redirect?: string;
   "certified_pre-owned"?: boolean;

package/src/interfaces/params.interface.ts

@@ -11,4 +11,33 @@ export interface Params {
   cpc?: string;
   ownerId?: string;
   metaPushPriorityCsvUrl?: string;
+  // Cronicle plugin param (text field, so it arrives as a string); overrides the
+  // default MAX_FEED_SALES_PRICE_EUR cap in the configs that apply one.
+  maxCurrentSalesPrice?: string;
+  // Google feed only, opt-in: when all three are set, every exported item is also written to
+  // `<project>.<dataset>.<table>` in BigQuery (schema/feed-items.json, see README).
+  feedItemsBqProject?: string;
+  feedItemsBqDataset?: string;
+  feedItemsBqTable?: string;
+  // Google feed only, EnVKV C8: off (default) | audit | enforce, and the car-basic-data enrichment log
+  // table the estimates are read from (see services/publication-gate.ts).
+  publicationGate?: string;
+  publicationGateBqProject?: string;
+  publicationGateBqDataset?: string;
+  publicationGateBqTable?: string;
+}
+
+// Parsed from feedItemsBq* by GoogleConfig.
+export interface FeedItemsBigQueryTarget {
+  project: string;
+  dataset: string;
+  table: string;
+}
+
+// Parsed from publicationGate* by GoogleConfig.
+export interface PublicationGateConfig {
+  mode: 'off' | 'audit' | 'enforce';
+  project: string;
+  dataset: string;
+  table: string;
 }

package/src/services/feed-items-bigquery.writer.ts

@@ -0,0 +1,101 @@
+import { join } from 'path';
+import { BigQueryLogger } from '@bytebrand/bigquery-logger';
+import { FeedItemsBigQueryTarget } from '../interfaces/params.interface';
+import { FeedItemRow } from '../transformers/google-feed-item-row';
+
+const BATCH_SIZE = 500;
+// After this many failed batches the writer gives up for the run (missing table, bad credentials).
+const MAX_FAILED_BATCHES = 3;
+const CLOSE_TIMEOUT_MS = 60000;
+
+class FeedItemsBigQueryLogger extends BigQueryLogger {
+  async insertRows(rows: FeedItemRow[]): Promise<void> {
+    await this.client
+      .dataset(this.options.BigQueryDataset)
+      .table(this.options.BigQueryLogTable)
+      .insert(rows, { skipInvalidRows: true, ignoreUnknownValues: true });
+  }
+}
+
+// Audit trail of the feed items: rows are buffered and inserted in batches of 500 in the background,
+// one batch at a time. Errors are logged and swallowed — the feed export must never fail because of
+// BigQuery.
+export class FeedItemsBigQueryWriter {
+  private logger: FeedItemsBigQueryLogger;
+  private buffer: FeedItemRow[] = [];
+  private queue: Promise<void> = Promise.resolve();
+  private failedBatches = 0;
+  private insertedRows = 0;
+
+  constructor(private target: FeedItemsBigQueryTarget) {
+    try {
+      const credentials = require(join(process.cwd(), 'credentials/google-cloud-project-id.json'));
+      this.logger = new FeedItemsBigQueryLogger(credentials, {
+        BigQueryProject: target.project,
+        BigQueryDataset: target.dataset,
+        BigQueryLogTable: target.table,
+      });
+    } catch (e) {
+      // First line only: a missing credentials file comes with the whole require stack.
+      console.log(`Feed items BigQuery disabled: ${String(e?.message || e).split('\n')[0]}`);
+    }
+  }
+
+  add(row: FeedItemRow): void {
+    if (!this.enabled) return;
+    this.buffer.push(row);
+    if (this.buffer.length >= BATCH_SIZE) this.flush();
+  }
+
+  // Waits for the pending batches, but never longer than CLOSE_TIMEOUT_MS: a hanging BigQuery
+  // must not hold up the feed export (the XML is already written at this point).
+  async close(): Promise<void> {
+    this.flush();
+    let timer: NodeJS.Timeout | undefined;
+    const timeout = new Promise<'timeout'>((resolve) => {
+      timer = setTimeout(() => resolve('timeout'), CLOSE_TIMEOUT_MS);
+    });
+    const outcome = await Promise.race([this.queue.then(() => 'done' as const), timeout]);
+    if (timer) clearTimeout(timer);
+    if (outcome === 'timeout') console.error(`Feed items BigQuery: gave up waiting after ${CLOSE_TIMEOUT_MS} ms`);
+    if (this.logger) {
+      console.log(`Feed items BigQuery: ${this.insertedRows} rows to ${this.tableName}, ${this.failedBatches} failed batches`);
+    }
+  }
+
+  private get enabled(): boolean {
+    return !!this.logger && this.failedBatches < MAX_FAILED_BATCHES;
+  }
+
+  private get tableName(): string {
+    return `${this.target.project}.${this.target.dataset}.${this.target.table}`;
+  }
+
+  private flush(): void {
+    if (!this.buffer.length) return;
+    const rows = this.buffer;
+    this.buffer = [];
+    this.queue = this.queue.then(() => this.insert(rows));
+  }
+
+  private async insert(rows: FeedItemRow[]): Promise<void> {
+    if (!this.enabled) return;
+    try {
+      await this.logger.insertRows(rows);
+      this.insertedRows += rows.length;
+    } catch (e) {
+      // PartialFailureError: the valid rows went in (skipInvalidRows), `errors` lists the rejected
+      // ones; the first is enough to see the cause.
+      const rejected: any[] = e?.name === 'PartialFailureError' && Array.isArray(e.errors) ? e.errors : null;
+      const firstRowError = rejected?.[0]?.errors?.[0];
+      const detail = firstRowError ? ` (${firstRowError.reason}: ${firstRowError.message})` : '';
+      if (rejected) {
+        this.insertedRows += rows.length - rejected.length;
+        console.log(`Feed items BigQuery rejected ${rejected.length} of ${rows.length} rows${detail}`);
+      } else {
+        this.failedBatches++;
+        console.log(`Feed items BigQuery insert of ${rows.length} rows failed: ${e?.message || e}`);
+      }
+    }
+  }
+}

package/src/services/publication-gate.ts

@@ -0,0 +1,120 @@
+import { join } from 'path';
+import { get } from 'lodash';
+import { BigQueryLogger } from '@bytebrand/bigquery-logger';
+import { DataSchemes } from '@bytebrand/car-schema';
+import { PublicationGateConfig } from '../interfaces/params.interface';
+import { GoogleEnvkvAttribute, googleEnvkvSourceFields } from '../transformers/google-feed-envkv';
+
+// EnVKV stage C8 — publication gate for estimated EnVKV values in the Google feed.
+//
+// No car field says whether a value was delivered or estimated; car-basic-data writes every change it
+// makes to the enrichment audit log (BigQuery). At the start of the export the gate reads the latest
+// log row per car and field; a field counts as estimated when that row came from an estimate source
+// and its final value is still the value stored on the car (a later import with a delivered value
+// replaces it without a log row, so the stored value no longer matches).
+//
+//   off      no BigQuery call, the feed as before (default)
+//   audit    the feed as before, the estimated attributes are recorded in the feed items table
+//   enforce  estimated attributes are left out of the item (and recorded)
+
+// finalValueSource values of CBD's enrichment log that are estimates: the CO2 estimator (EEA register /
+// model), the make-model template sheet, the global defaults, and a weighted fuel / CO2 derived from
+// its pair value.
+export const ESTIMATE_SOURCES = ['co2-estimate', 'mmm', 'globalDefault', 'derived'];
+
+// Every car field a Google EnVKV attribute can be taken from (google-feed-envkv picks).
+export const GATED_FIELDS = [
+  'consumption.wltpWeightedCombined',
+  'consumption.wltpCombined',
+  'consumption.consumptionCombined',
+  'consumption.wltpWeightedPowerCombined',
+  'consumption.wltpPowerCombined',
+  'consumption.consumptionPowerCombined',
+  'environmentEmissions.wltpCo2',
+  'environmentEmissions.co2',
+  'battery.wltpTotalRange',
+  'battery.wltpRange',
+  'battery.batteryRangeElectric',
+];
+
+// '7.10' and 7.1 are the same value; non-numeric values compare as strings.
+const sameValue = (logged: string, stored: unknown): boolean => {
+  if (stored === undefined || stored === null) return false;
+  const a = Number(logged);
+  const b = Number(stored);
+  if (Number.isFinite(a) && Number.isFinite(b)) return Math.abs(a - b) < 1e-9;
+  return logged === String(stored);
+};
+
+export class PublicationGate {
+  // estimates: `${carId}|${field}` -> final value of the latest estimate row
+  constructor(readonly mode: 'off' | 'audit' | 'enforce', private estimates: Map<string, string>) {}
+
+  static off(): PublicationGate {
+    return new PublicationGate('off', new Map());
+  }
+
+  get size(): number {
+    return this.estimates.size;
+  }
+
+  // The attributes of this car whose value comes from an estimated field, in a fixed order.
+  estimatedAttributes(car: DataSchemes.Car): GoogleEnvkvAttribute[] {
+    if (this.mode === 'off') return [];
+    const carId = String(get(car, '_id', ''));
+    const attributes: GoogleEnvkvAttribute[] = [];
+    for (const [attribute, field] of Object.entries(googleEnvkvSourceFields(car))) {
+      const logged = this.estimates.get(`${carId}|${field}`);
+      if (logged !== undefined && sameValue(logged, get(car, field))) attributes.push(attribute as GoogleEnvkvAttribute);
+    }
+    return attributes;
+  }
+
+  // enforce: drops the estimated attributes from the item. Returns the item for chaining.
+  apply<T extends Partial<Record<GoogleEnvkvAttribute, string>>>(item: T, attributes: GoogleEnvkvAttribute[]): T {
+    if (this.mode !== 'enforce') return item;
+    for (const attribute of attributes) delete item[attribute];
+    return item;
+  }
+}
+
+class EnrichmentLogReader extends BigQueryLogger {
+  async latestEstimates(table: string): Promise<Map<string, string>> {
+    const query = `
+      SELECT carId, fieldName, finalValue
+      FROM (
+        SELECT carId, fieldName, finalValue, finalValueSource,
+               ROW_NUMBER() OVER (PARTITION BY carId, fieldName ORDER BY timestamp DESC) AS rn
+        FROM \`${table}\`
+        WHERE fieldName IN UNNEST(@fields)
+      )
+      WHERE rn = 1 AND finalValueSource IN UNNEST(@sources) AND finalValue IS NOT NULL`;
+    const [rows] = await this.client.query({ query, params: { fields: GATED_FIELDS, sources: ESTIMATE_SOURCES } });
+    const estimates = new Map<string, string>();
+    for (const row of rows) estimates.set(`${row.carId}|${row.fieldName}`, String(row.finalValue));
+    return estimates;
+  }
+}
+
+// Reads the estimates once per run. audit: a failure only switches the gate off. enforce: a failure
+// fails the export, so the last published feed stays in place instead of a feed with unchecked estimates.
+export async function loadPublicationGate(config: PublicationGateConfig | undefined): Promise<PublicationGate> {
+  if (!config || config.mode === 'off') return PublicationGate.off();
+  const table = `${config.project}.${config.dataset}.${config.table}`;
+  try {
+    const credentials = require(join(process.cwd(), 'credentials/google-cloud-project-id.json'));
+    const reader = new EnrichmentLogReader(credentials, {
+      BigQueryProject: config.project,
+      BigQueryDataset: config.dataset,
+      BigQueryLogTable: config.table,
+    });
+    const estimates = await reader.latestEstimates(table);
+    console.log(`Publication gate ${config.mode}: ${estimates.size} estimated car fields from ${table}`);
+    return new PublicationGate(config.mode, estimates);
+  } catch (e) {
+    const message = String(e?.message || e).split('\n')[0];
+    if (config.mode === 'enforce') throw new Error(`Publication gate: cannot read ${table}: ${message}`);
+    console.error(`Publication gate disabled for this run: cannot read ${table}: ${message}`);
+    return PublicationGate.off();
+  }
+}