@bytebrand/export-cars 1.26.24 → 1.26.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bytebrand/export-cars",
-  "version": "1.26.24",
+  "version": "1.26.26",
   "description": "",
   "author": "bytebrand",
   "license": "UNLICENSED",
@@ -11,10 +11,11 @@
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
     "lint": "eslint -c .eslintrc.js --ext .ts src/",
     "start:dev": "nest start --watch",
-    "test": "ts-node --transpile-only test/google-feed-envkv.test.ts"
+    "test": "ts-node --transpile-only test/google-feed-envkv.test.ts && ts-node --transpile-only test/google-feed-item-row.test.ts"
   },
   "dependencies": {
     "@bytebrand/autode-request": "^1.19.2",
+    "@bytebrand/bigquery-logger": "^2.1.2",
     "@bytebrand/car-basic-data-scope": "^9.11.24",
     "@bytebrand/car-description": "^1.17.20",
     "@bytebrand/car-schema": "^3.14.15",

package/schema/feed-items.json

@@ -0,0 +1,122 @@
+[
+  {
+    "name": "partitionDay",
+    "type": "DATE",
+    "mode": "NULLABLE",
+    "description": "UTC day of exportedAt; the table is partitioned by it"
+  },
+  {
+    "name": "exportedAt",
+    "type": "TIMESTAMP",
+    "mode": "NULLABLE",
+    "description": "Start of the export run (same for every row of a run)"
+  },
+  {
+    "name": "feed",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "Feed the item was written to ('google')"
+  },
+  {
+    "name": "runId",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "Cronicle job id of the run"
+  },
+  {
+    "name": "itemId",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:id> = CBD _id"
+  },
+  {
+    "name": "vehicleId",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "metaData.vehicleId"
+  },
+  {
+    "name": "make",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:brand> as sent"
+  },
+  {
+    "name": "model",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:model> as sent"
+  },
+  {
+    "name": "condition",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:condition> as sent"
+  },
+  {
+    "name": "fuel",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "consumption.fuel selector"
+  },
+  {
+    "name": "hybridPlugin",
+    "type": "BOOLEAN",
+    "mode": "NULLABLE",
+    "description": "engineData.hybridPlugin, null when unset"
+  },
+  {
+    "name": "engine",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:engine> as sent"
+  },
+  {
+    "name": "electricRange",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:electric_range> as sent, null when omitted"
+  },
+  {
+    "name": "fuelConsumption",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:fuel_consumption> as sent, null when omitted"
+  },
+  {
+    "name": "fuelConsumptionDischarged",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:fuel_consumption_discharged_battery> as sent, null when omitted"
+  },
+  {
+    "name": "energyConsumption",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:energy_consumption> as sent, null when omitted"
+  },
+  {
+    "name": "co2Emissions",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:co2_emissions> as sent, null when omitted"
+  },
+  {
+    "name": "rangeSourceField",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "Car field electric_range came from: battery.wltpRange / battery.wltpTotalRange / battery.batteryRangeElectric"
+  },
+  {
+    "name": "price",
+    "type": "FLOAT",
+    "mode": "NULLABLE",
+    "description": "<g:price> amount as sent (EUR)"
+  },
+  {
+    "name": "availability",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "<g:availability> as sent"
+  }
+]

package/src/configs/config.ts

@@ -1,93 +1,97 @@
-import { Environment, ExportType, ImageType } from '../enums';
-import { Params } from '../interfaces/params.interface';
-
-export class Config {
-  apikey: string;
-  campaign: string;
-  userActionsUrl: string;
-  marketingImageConfigId: number;
-  upaUrl: string;
-  imboUrl: string;
-  imageUrl: string;
-  baseImageUrl: string;
-  marketingImageUrl: string;
-  metaAiaImageUrl: string;
-  imageType: ImageType;
-  downloadImage: boolean;
-  isNewCar: boolean;
-  isCriteo: boolean;
-  exportType: ExportType;
-  domain: string;
-  input: {
-    stringIN: string;
-    numberIN: string;
-    dateIN: string;
-  };
-  // When set, CarService.filterCar exports a car only if price.currentSalesPrice
-  // is a valid number in (0, maxCurrentSalesPrice] — missing/invalid price means
-  // the car is not exported. Unset = no price cap for this export type.
-  maxCurrentSalesPrice?: number;
-  pathesConfig: {
-    localDirectory: string;
-    localConvertedDirectory: string;
-    baseFileName: string;
-  };
-  outputFiles: string[];
-  cpc: string;
-  mappingSpreadsheetId: string;
-  mappingSpreadsheetTitle: string;
-  dealersSpreadsheetId: string;
-  dealersSpreadsheetTitle: string;
-  metaPushPriorityCsvUrl: string;
-
-  protected environmentConfig = {
-    [Environment.DEVELOP]: {
-      domain: 'autode-dev.de',
-      fbMarketingImageId: 8,
-      criteoMarketingImagesConfigId: 12,
-      localConvertedDirectory: 'resources/csv_dev',
-      remoteDirectory: 'import_dev',
-      imageUrl: 'https://images.autode-dev.de/cache_test',
-      baseImageUrl: 'https://images.autode-dev.de',
-    },
-    [Environment.PRODUCTION]: {
-      domain: 'auto.de',
-      fbMarketingImageId: 1,
-      criteoMarketingImagesConfigId: 3,
-      localConvertedDirectory: 'resources/csv_prod',
-      remoteDirectory: 'import',
-      imageUrl: 'https://images.auto.de/generated_images',
-      baseImageUrl: 'https://images.auto.de',
-    },
-  };
-
-  constructor(params: Params) {
-    this.apikey = params.apikey;
-    this.exportType = params.exportType;
-    this.cpc = params.cpc;
-    this.domain = this.environmentConfig[params.environment].domain;
-    this.baseImageUrl = this.environmentConfig[params.environment].baseImageUrl;
-    this.marketingImageUrl = this.environmentConfig[params.environment].imageUrl;
-    this.mappingSpreadsheetId = '147qz9sTgs1wi13ua8YKa0IgfxAn1X_qH6LkGnAnzeyY';
-    this.dealersSpreadsheetId = '1gWrcA-uRTpna1h66Mbe2Y7NA0tIhlUaQXBcS4a_UzOw';
-    this.dealersSpreadsheetTitle = 'dealers_mapping';
-    this.metaPushPriorityCsvUrl = params.metaPushPriorityCsvUrl;
-    this.userActionsUrl = `https://www.${this.domain}/service/user-actions`;
-    this.imboUrl = `https://www.${this.domain}/service/image-imbo`;
-    this.imageUrl = `https://images.${this.domain}`;
-    // Same FTP_ROOT (/generated_images) image-generator's MetaAiaImagesService saves to —
-    // independent of marketingImageUrl, which points legacy fb/criteo channels at their own path.
-    this.metaAiaImageUrl = `${this.imageUrl}/generated_images`;
-    this.upaUrl = `https://www.${this.domain}/service/user-profile-admin`;
-    this.input = {
-      stringIN: params.String,
-      numberIN: params.Number,
-      dateIN: params.Date,
-    };
-    this.pathesConfig = {
-      localDirectory: 'resources/cars-export',
-      localConvertedDirectory: 'resources/default-export',
-      baseFileName: 'cars',
-    };
-  }
-}
+import { Environment, ExportType, ImageType } from '../enums';
+import { FeedItemsBigQueryTarget, Params } from '../interfaces/params.interface';
+
+export class Config {
+  apikey: string;
+  campaign: string;
+  userActionsUrl: string;
+  marketingImageConfigId: number;
+  upaUrl: string;
+  imboUrl: string;
+  imageUrl: string;
+  baseImageUrl: string;
+  marketingImageUrl: string;
+  metaAiaImageUrl: string;
+  imageType: ImageType;
+  downloadImage: boolean;
+  isNewCar: boolean;
+  isCriteo: boolean;
+  exportType: ExportType;
+  domain: string;
+  input: {
+    stringIN: string;
+    numberIN: string;
+    dateIN: string;
+  };
+  // When set, CarService.filterCar exports a car only if price.currentSalesPrice
+  // is a valid number in (0, maxCurrentSalesPrice] — missing/invalid price means
+  // the car is not exported. Unset = no price cap for this export type.
+  maxCurrentSalesPrice?: number;
+  pathesConfig: {
+    localDirectory: string;
+    localConvertedDirectory: string;
+    baseFileName: string;
+  };
+  outputFiles: string[];
+  cpc: string;
+  mappingSpreadsheetId: string;
+  mappingSpreadsheetTitle: string;
+  dealersSpreadsheetId: string;
+  dealersSpreadsheetTitle: string;
+  metaPushPriorityCsvUrl: string;
+  // Cronicle job id of the run (set by ExportRunner), stored with the feed item rows.
+  runId?: string;
+  // Set only when the job configures the feed items BigQuery table; unset = no BigQuery calls.
+  feedItemsBigQuery?: FeedItemsBigQueryTarget;
+
+  protected environmentConfig = {
+    [Environment.DEVELOP]: {
+      domain: 'autode-dev.de',
+      fbMarketingImageId: 8,
+      criteoMarketingImagesConfigId: 12,
+      localConvertedDirectory: 'resources/csv_dev',
+      remoteDirectory: 'import_dev',
+      imageUrl: 'https://images.autode-dev.de/cache_test',
+      baseImageUrl: 'https://images.autode-dev.de',
+    },
+    [Environment.PRODUCTION]: {
+      domain: 'auto.de',
+      fbMarketingImageId: 1,
+      criteoMarketingImagesConfigId: 3,
+      localConvertedDirectory: 'resources/csv_prod',
+      remoteDirectory: 'import',
+      imageUrl: 'https://images.auto.de/generated_images',
+      baseImageUrl: 'https://images.auto.de',
+    },
+  };
+
+  constructor(params: Params) {
+    this.apikey = params.apikey;
+    this.exportType = params.exportType;
+    this.cpc = params.cpc;
+    this.domain = this.environmentConfig[params.environment].domain;
+    this.baseImageUrl = this.environmentConfig[params.environment].baseImageUrl;
+    this.marketingImageUrl = this.environmentConfig[params.environment].imageUrl;
+    this.mappingSpreadsheetId = '147qz9sTgs1wi13ua8YKa0IgfxAn1X_qH6LkGnAnzeyY';
+    this.dealersSpreadsheetId = '1gWrcA-uRTpna1h66Mbe2Y7NA0tIhlUaQXBcS4a_UzOw';
+    this.dealersSpreadsheetTitle = 'dealers_mapping';
+    this.metaPushPriorityCsvUrl = params.metaPushPriorityCsvUrl;
+    this.userActionsUrl = `https://www.${this.domain}/service/user-actions`;
+    this.imboUrl = `https://www.${this.domain}/service/image-imbo`;
+    this.imageUrl = `https://images.${this.domain}`;
+    // Same FTP_ROOT (/generated_images) image-generator's MetaAiaImagesService saves to —
+    // independent of marketingImageUrl, which points legacy fb/criteo channels at their own path.
+    this.metaAiaImageUrl = `${this.imageUrl}/generated_images`;
+    this.upaUrl = `https://www.${this.domain}/service/user-profile-admin`;
+    this.input = {
+      stringIN: params.String,
+      numberIN: params.Number,
+      dateIN: params.Date,
+    };
+    this.pathesConfig = {
+      localDirectory: 'resources/cars-export',
+      localConvertedDirectory: 'resources/default-export',
+      baseFileName: 'cars',
+    };
+  }
+}

package/src/configs/google.config.ts

@@ -1,16 +1,26 @@
-import { ExportType, FileType, ImageType } from 'src/enums';
-import { resolveMaxFeedSalesPrice } from '../constants/feed-price.constants';
-import { Params } from '../interfaces/params.interface';
-import { Config } from './config';
-
-export class GoogleConfig extends Config {
-  constructor(params: Params) {
-    super(params);
-    this.pathesConfig.localConvertedDirectory = `resources/${ExportType.GOOGLE}-export`;
-    this.imageType = ImageType.default;
-    this.maxCurrentSalesPrice = resolveMaxFeedSalesPrice(params.maxCurrentSalesPrice);
-    this.outputFiles = [
-      `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.GOOGLE}.${FileType.XML}`,
-    ];
-  }
-}
+import { ExportType, FileType, ImageType } from 'src/enums';
+import { resolveMaxFeedSalesPrice } from '../constants/feed-price.constants';
+import { Params } from '../interfaces/params.interface';
+import { Config } from './config';
+
+const trimmed = (value: unknown): string => (typeof value === 'string' ? value.trim() : '');
+
+export class GoogleConfig extends Config {
+  constructor(params: Params) {
+    super(params);
+    this.pathesConfig.localConvertedDirectory = `resources/${ExportType.GOOGLE}-export`;
+    this.imageType = ImageType.default;
+    this.maxCurrentSalesPrice = resolveMaxFeedSalesPrice(params.maxCurrentSalesPrice);
+    const feedItemsBigQuery = {
+      project: trimmed(params.feedItemsBqProject),
+      dataset: trimmed(params.feedItemsBqDataset),
+      table: trimmed(params.feedItemsBqTable),
+    };
+    if (feedItemsBigQuery.project && feedItemsBigQuery.dataset && feedItemsBigQuery.table) {
+      this.feedItemsBigQuery = feedItemsBigQuery;
+    }
+    this.outputFiles = [
+      `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.GOOGLE}.${FileType.XML}`,
+    ];
+  }
+}

package/src/export.command.ts

@@ -1,37 +1,38 @@
-import { createInterface } from 'readline';
-import { Command, CommandRunner } from 'nest-commander';
-import { ConfigService } from './services';
-import { Input } from './interfaces/input.interface';
-import { exportSchema } from './export.schema';
-import { ModuleRef } from '@nestjs/core';
-import { Exporter } from './interfaces/exporter.interface';
-import { LoggerService } from '@bytebrand/cronicle-helper';
-
-@Command({
-  name: 'export',
-  options: { isDefault: true },
-})
-export class ExportRunner extends CommandRunner {
-  constructor(
-    private configService: ConfigService,
-    private moduleRef: ModuleRef,
-  ) {
-    super();
-  }
-
-  async run(): Promise<void> {
-    const rl = createInterface({ input: process.stdin });
-    await new Promise<void>((resolve, reject) => {
-      rl.on('line', async (data: string) => {
-        const input = JSON.parse(data) as Input;
-        LoggerService.log(`Running job: ${input.id} - ${JSON.stringify(input)} \n`);
-
-        this.configService.config = new exportSchema[input.params.exportType].config(input.params);
-        const exporter = await this.moduleRef.create<Exporter>(exportSchema[input.params.exportType].exporter);
-        await exporter.export().catch(reject);
-
-        resolve(rl.close());
-      });
-    });
-  }
-}
+import { createInterface } from 'readline';
+import { Command, CommandRunner } from 'nest-commander';
+import { ConfigService } from './services';
+import { Input } from './interfaces/input.interface';
+import { exportSchema } from './export.schema';
+import { ModuleRef } from '@nestjs/core';
+import { Exporter } from './interfaces/exporter.interface';
+import { LoggerService } from '@bytebrand/cronicle-helper';
+
+@Command({
+  name: 'export',
+  options: { isDefault: true },
+})
+export class ExportRunner extends CommandRunner {
+  constructor(
+    private configService: ConfigService,
+    private moduleRef: ModuleRef,
+  ) {
+    super();
+  }
+
+  async run(): Promise<void> {
+    const rl = createInterface({ input: process.stdin });
+    await new Promise<void>((resolve, reject) => {
+      rl.on('line', async (data: string) => {
+        const input = JSON.parse(data) as Input;
+        LoggerService.log(`Running job: ${input.id} - ${JSON.stringify(input)} \n`);
+
+        this.configService.config = new exportSchema[input.params.exportType].config(input.params);
+        this.configService.config.runId = input.id;
+        const exporter = await this.moduleRef.create<Exporter>(exportSchema[input.params.exportType].exporter);
+        await exporter.export().catch(reject);
+
+        resolve(rl.close());
+      });
+    });
+  }
+}

package/src/exporters/google.exporter.ts

@@ -1,46 +1,56 @@
-import { Injectable } from '@nestjs/common';
-import { ModuleRef } from '@nestjs/core';
-
-import { CronicleUtils } from '@bytebrand/cronicle-helper';
-import { FsChunkInterface } from '@bytebrand/chunkable';
-
-import { CarService, ConfigService, LoaderService, RulesService } from '../services';
-import { GoogleTransformer } from '../transformers/google.transformer';
-import { GoogleXmlBuilder } from '../formats/google-xml.builder';
-import { Exporter } from '../interfaces/exporter.interface';
-
-@Injectable()
-export class GoogleExporter implements Exporter {
-  constructor(
-    private carService: CarService,
-    private configService: ConfigService,
-    private rulesService: RulesService,
-    private loaderService: LoaderService,
-    private moduleRef: ModuleRef,
-  ) { }
-
-  async export(): Promise<void> {
-    const [fileName] = this.configService.config.outputFiles;
-    await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
-    await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
-
-    const googleXmlBuilder = new GoogleXmlBuilder(fileName);
-    await googleXmlBuilder.buildStart(this.configService.config.domain);
-
-    const googleTransformer = await this.moduleRef.create(GoogleTransformer);
-
-    await this.loaderService.process(async (chunk: FsChunkInterface) => {
-      chunk.data = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
-
-      for (const data of chunk.data) {
-        const car = await googleTransformer.transform(data);
-        if (car)
-          await googleXmlBuilder.addItem(car);
-        this.carService.counter.countCar(data.images);
-      }
-    });
-
-    await googleXmlBuilder.buildEnd();
-    console.log(this.carService.counter.getCountedCars());
-  }
-}
+import { Injectable } from '@nestjs/common';
+import { ModuleRef } from '@nestjs/core';
+
+import { CronicleUtils } from '@bytebrand/cronicle-helper';
+import { FsChunkInterface } from '@bytebrand/chunkable';
+
+import { CarService, ConfigService, LoaderService, RulesService } from '../services';
+import { GoogleTransformer } from '../transformers/google.transformer';
+import { buildGoogleFeedItemRow } from '../transformers/google-feed-item-row';
+import { FeedItemsBigQueryWriter } from '../services/feed-items-bigquery.writer';
+import { GoogleXmlBuilder } from '../formats/google-xml.builder';
+import { Exporter } from '../interfaces/exporter.interface';
+
+@Injectable()
+export class GoogleExporter implements Exporter {
+  constructor(
+    private carService: CarService,
+    private configService: ConfigService,
+    private rulesService: RulesService,
+    private loaderService: LoaderService,
+    private moduleRef: ModuleRef,
+  ) { }
+
+  async export(): Promise<void> {
+    const [fileName] = this.configService.config.outputFiles;
+    await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
+    await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
+
+    const googleXmlBuilder = new GoogleXmlBuilder(fileName);
+    await googleXmlBuilder.buildStart(this.configService.config.domain);
+
+    const googleTransformer = await this.moduleRef.create(GoogleTransformer);
+
+    // Opt-in audit of the sent items (feedItemsBq* job params); null = no BigQuery calls at all.
+    const { feedItemsBigQuery, runId } = this.configService.config;
+    const feedItemsWriter = feedItemsBigQuery ? new FeedItemsBigQueryWriter(feedItemsBigQuery) : null;
+    const exportedAt = new Date();
+
+    await this.loaderService.process(async (chunk: FsChunkInterface) => {
+      chunk.data = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
+
+      for (const data of chunk.data) {
+        const car = await googleTransformer.transform(data);
+        if (car) {
+          await googleXmlBuilder.addItem(car);
+          feedItemsWriter?.add(buildGoogleFeedItemRow(data.car, car, runId, exportedAt));
+        }
+        this.carService.counter.countCar(data.images);
+      }
+    });
+
+    await googleXmlBuilder.buildEnd();
+    await feedItemsWriter?.close();
+    console.log(this.carService.counter.getCountedCars());
+  }
+}

package/src/interfaces/params.interface.ts

@@ -1,17 +1,29 @@
-import { Environment, ExportType } from '../enums';
-
-export interface Params {
-  environment: Environment;
-  exportType?: ExportType;
-  apikey?: string;
-  String?: string;
-  Number?: string;
-  Date?: string;
-  isNewCar?: boolean;
-  cpc?: string;
-  ownerId?: string;
-  metaPushPriorityCsvUrl?: string;
-  // Cronicle plugin param (text field, so it arrives as a string); overrides the
-  // default MAX_FEED_SALES_PRICE_EUR cap in the configs that apply one.
-  maxCurrentSalesPrice?: string;
-}
+import { Environment, ExportType } from '../enums';
+
+export interface Params {
+  environment: Environment;
+  exportType?: ExportType;
+  apikey?: string;
+  String?: string;
+  Number?: string;
+  Date?: string;
+  isNewCar?: boolean;
+  cpc?: string;
+  ownerId?: string;
+  metaPushPriorityCsvUrl?: string;
+  // Cronicle plugin param (text field, so it arrives as a string); overrides the
+  // default MAX_FEED_SALES_PRICE_EUR cap in the configs that apply one.
+  maxCurrentSalesPrice?: string;
+  // Google feed only, opt-in: when all three are set, every exported item is also written to
+  // `<project>.<dataset>.<table>` in BigQuery (schema/feed-items.json, see README).
+  feedItemsBqProject?: string;
+  feedItemsBqDataset?: string;
+  feedItemsBqTable?: string;
+}
+
+// Parsed from feedItemsBq* by GoogleConfig.
+export interface FeedItemsBigQueryTarget {
+  project: string;
+  dataset: string;
+  table: string;
+}