@bytebrand/export-cars 1.26.27 → 1.26.29

package/src/export.command.ts

@@ -1,38 +1,38 @@
-import { createInterface } from 'readline';
-import { Command, CommandRunner } from 'nest-commander';
-import { ConfigService } from './services';
-import { Input } from './interfaces/input.interface';
-import { exportSchema } from './export.schema';
-import { ModuleRef } from '@nestjs/core';
-import { Exporter } from './interfaces/exporter.interface';
-import { LoggerService } from '@bytebrand/cronicle-helper';
-
-@Command({
-  name: 'export',
-  options: { isDefault: true },
-})
-export class ExportRunner extends CommandRunner {
-  constructor(
-    private configService: ConfigService,
-    private moduleRef: ModuleRef,
-  ) {
-    super();
-  }
-
-  async run(): Promise<void> {
-    const rl = createInterface({ input: process.stdin });
-    await new Promise<void>((resolve, reject) => {
-      rl.on('line', async (data: string) => {
-        const input = JSON.parse(data) as Input;
-        LoggerService.log(`Running job: ${input.id} - ${JSON.stringify(input)} \n`);
-
-        this.configService.config = new exportSchema[input.params.exportType].config(input.params);
-        this.configService.config.runId = input.id;
-        const exporter = await this.moduleRef.create<Exporter>(exportSchema[input.params.exportType].exporter);
-        await exporter.export().catch(reject);
-
-        resolve(rl.close());
-      });
-    });
-  }
-}
+import { createInterface } from 'readline';
+import { Command, CommandRunner } from 'nest-commander';
+import { ConfigService } from './services';
+import { Input } from './interfaces/input.interface';
+import { exportSchema } from './export.schema';
+import { ModuleRef } from '@nestjs/core';
+import { Exporter } from './interfaces/exporter.interface';
+import { LoggerService } from '@bytebrand/cronicle-helper';
+
+@Command({
+  name: 'export',
+  options: { isDefault: true },
+})
+export class ExportRunner extends CommandRunner {
+  constructor(
+    private configService: ConfigService,
+    private moduleRef: ModuleRef,
+  ) {
+    super();
+  }
+
+  async run(): Promise<void> {
+    const rl = createInterface({ input: process.stdin });
+    await new Promise<void>((resolve, reject) => {
+      rl.on('line', async (data: string) => {
+        const input = JSON.parse(data) as Input;
+        LoggerService.log(`Running job: ${input.id} - ${JSON.stringify(input)} \n`);
+
+        this.configService.config = new exportSchema[input.params.exportType].config(input.params);
+        this.configService.config.runId = input.id;
+        const exporter = await this.moduleRef.create<Exporter>(exportSchema[input.params.exportType].exporter);
+        await exporter.export().catch(reject);
+
+        resolve(rl.close());
+      });
+    });
+  }
+}

package/src/export.schema.ts

@@ -23,6 +23,7 @@ import {
   AwinConfig,
   EviumConfig,
   SantanderConfig,
+  OpenAiAdsConfig,
 } from './configs';
 import {
   As24Exporter,
@@ -41,6 +42,7 @@ import {
   AwinExporter,
   SantanderExporter,
   CriteoExporter,
+  OpenAiAdsExporter,
 } from './exporters';
 
 export const exportSchema = {
@@ -132,4 +134,8 @@ export const exportSchema = {
     config: SantanderConfig,
     exporter: SantanderExporter,
   },
+  [ExportType.OPENAI_ADS]: {
+    config: OpenAiAdsConfig,
+    exporter: OpenAiAdsExporter,
+  },
 };

package/src/exporters/google.exporter.ts

@@ -1,56 +1,67 @@
-import { Injectable } from '@nestjs/common';
-import { ModuleRef } from '@nestjs/core';
-
-import { CronicleUtils } from '@bytebrand/cronicle-helper';
-import { FsChunkInterface } from '@bytebrand/chunkable';
-
-import { CarService, ConfigService, LoaderService, RulesService } from '../services';
-import { GoogleTransformer } from '../transformers/google.transformer';
-import { buildGoogleFeedItemRow } from '../transformers/google-feed-item-row';
-import { FeedItemsBigQueryWriter } from '../services/feed-items-bigquery.writer';
-import { GoogleXmlBuilder } from '../formats/google-xml.builder';
-import { Exporter } from '../interfaces/exporter.interface';
-
-@Injectable()
-export class GoogleExporter implements Exporter {
-  constructor(
-    private carService: CarService,
-    private configService: ConfigService,
-    private rulesService: RulesService,
-    private loaderService: LoaderService,
-    private moduleRef: ModuleRef,
-  ) { }
-
-  async export(): Promise<void> {
-    const [fileName] = this.configService.config.outputFiles;
-    await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
-    await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
-
-    const googleXmlBuilder = new GoogleXmlBuilder(fileName);
-    await googleXmlBuilder.buildStart(this.configService.config.domain);
-
-    const googleTransformer = await this.moduleRef.create(GoogleTransformer);
-
-    // Opt-in audit of the sent items (feedItemsBq* job params); null = no BigQuery calls at all.
-    const { feedItemsBigQuery, runId } = this.configService.config;
-    const feedItemsWriter = feedItemsBigQuery ? new FeedItemsBigQueryWriter(feedItemsBigQuery) : null;
-    const exportedAt = new Date();
-
-    await this.loaderService.process(async (chunk: FsChunkInterface) => {
-      chunk.data = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
-
-      for (const data of chunk.data) {
-        const car = await googleTransformer.transform(data);
-        if (car) {
-          await googleXmlBuilder.addItem(car);
-          feedItemsWriter?.add(buildGoogleFeedItemRow(data.car, car, runId, exportedAt));
-        }
-        this.carService.counter.countCar(data.images);
-      }
-    });
-
-    await googleXmlBuilder.buildEnd();
-    await feedItemsWriter?.close();
-    console.log(this.carService.counter.getCountedCars());
-  }
-}
+import { Injectable } from '@nestjs/common';
+import { ModuleRef } from '@nestjs/core';
+
+import { CronicleUtils } from '@bytebrand/cronicle-helper';
+import { FsChunkInterface } from '@bytebrand/chunkable';
+
+import { CarService, ConfigService, LoaderService, RulesService } from '../services';
+import { GoogleTransformer } from '../transformers/google.transformer';
+import { buildGoogleFeedItemRow } from '../transformers/google-feed-item-row';
+import { FeedItemsBigQueryWriter } from '../services/feed-items-bigquery.writer';
+import { loadPublicationGate } from '../services/publication-gate';
+import { GoogleXmlBuilder } from '../formats/google-xml.builder';
+import { Exporter } from '../interfaces/exporter.interface';
+
+@Injectable()
+export class GoogleExporter implements Exporter {
+  constructor(
+    private carService: CarService,
+    private configService: ConfigService,
+    private rulesService: RulesService,
+    private loaderService: LoaderService,
+    private moduleRef: ModuleRef,
+  ) { }
+
+  async export(): Promise<void> {
+    // EnVKV C8: estimated EnVKV values (CBD enrichment log); off unless the job sets publicationGate.
+    // Read first: in enforce mode a failure stops the run before the output directory is emptied.
+    const gate = await loadPublicationGate(this.configService.config.publicationGate);
+    const [fileName] = this.configService.config.outputFiles;
+    await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
+    await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
+
+    const googleXmlBuilder = new GoogleXmlBuilder(fileName);
+    await googleXmlBuilder.buildStart(this.configService.config.domain);
+
+    const googleTransformer = await this.moduleRef.create(GoogleTransformer);
+
+    // Opt-in audit of the sent items (feedItemsBq* job params); null = no BigQuery calls at all.
+    const { feedItemsBigQuery, runId } = this.configService.config;
+    const feedItemsWriter = feedItemsBigQuery ? new FeedItemsBigQueryWriter(feedItemsBigQuery) : null;
+    const exportedAt = new Date();
+    let gatedItems = 0;
+
+    await this.loaderService.process(async (chunk: FsChunkInterface) => {
+      chunk.data = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
+
+      for (const data of chunk.data) {
+        const car = await googleTransformer.transform(data);
+        if (car) {
+          const estimatedAttributes = gate.estimatedAttributes(data.car);
+          if (estimatedAttributes.length) gatedItems += 1;
+          await googleXmlBuilder.addItem(gate.apply(car, estimatedAttributes));
+          const gateInfo = gate.mode === 'off' ? null : { mode: gate.mode, estimatedAttributes };
+          feedItemsWriter?.add(buildGoogleFeedItemRow(data.car, car, runId, exportedAt, gateInfo));
+        }
+        this.carService.counter.countCar(data.images);
+      }
+    });
+
+    await googleXmlBuilder.buildEnd();
+    if (gate.mode !== 'off') {
+      console.log(`Publication gate ${gate.mode}: ${gatedItems} items with estimated EnVKV values${gate.mode === 'enforce' ? ' (attributes left out)' : ''}`);
+    }
+    await feedItemsWriter?.close();
+    console.log(this.carService.counter.getCountedCars());
+  }
+}

package/src/exporters/index.ts

@@ -14,3 +14,4 @@ export * from './facebook-shop.exporter';
 export * from './trovit.exporter';
 export * from './awin.exporter';
 export * from './santander.exporter';
+export * from './openai-ads.exporter';

package/src/exporters/openai-ads.exporter.ts

@@ -0,0 +1,93 @@
+import { Injectable } from '@nestjs/common';
+import { FsChunkInterface } from '@bytebrand/chunkable';
+import { LoggerService } from '@bytebrand/cronicle-helper';
+import { TranslatorExtended } from '@bytebrand/schema-tools';
+import { de, deVehicleDetailsPage, deVehicleProps } from '@bytebrand/i18n-dictionaries';
+
+import { CarService, ConfigService, ImagesService, LoaderService, RulesService } from '../services';
+import { ImageUrlValidator } from '../services/image-url-validator.service';
+import { FeedGuard } from '../services/feed-guard';
+import { DealerStatus, OpenAiAdsFeedRun, SourceCar } from '../services/openai-ads-feed-run';
+import { resolveOpenAiImageFields } from '../transformers/openai-image-fields';
+import { OpenAiAdsConfig } from '../configs/openai-ads.config';
+import { Exporter } from '../interfaces/exporter.interface';
+
+/**
+ * ExportType.OPENAI_ADS (ADV-335). Wires the Cronicle/Nest services into OpenAiAdsFeedRun, which
+ * holds the logic. Unlike the other exporters it never empties its output directory: the new file
+ * replaces the old one only after the fail-closed guard passed.
+ */
+@Injectable()
+export class OpenAiAdsExporter implements Exporter {
+  constructor(
+    private carService: CarService,
+    private configService: ConfigService,
+    private rulesService: RulesService,
+    private loaderService: LoaderService,
+    private imagesService: ImagesService,
+  ) { }
+
+  async export(): Promise<void> {
+    const config = this.configService.config as OpenAiAdsConfig;
+    const translator = new TranslatorExtended({ ...de, ...deVehicleProps, ...deVehicleDetailsPage });
+    const validator = new ImageUrlValidator({ concurrency: config.imageConcurrency });
+
+    const run = new OpenAiAdsFeedRun(
+      {
+        forEachChunk: fn => this.loaderService.process((chunk: FsChunkInterface) => fn(chunk.data as SourceCar[])),
+        filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
+        dealerStatus: ownerId => this.dealerStatus(ownerId),
+        resolveImages: ({ car, images }) =>
+          resolveOpenAiImageFields(this.imagesService.getCleanImages(car._id, images), validator),
+        imageStats: () => validator.stats,
+        guard: new FeedGuard(config.feedGuard),
+      },
+      {
+        runId: config.runId,
+        dryRun: config.dryRun,
+        sourceDirectory: config.pathesConfig.localDirectory,
+        sourceBaseFileName: config.pathesConfig.baseFileName,
+        outputFile: config.outputFiles[0],
+        stateDir: config.stateDir,
+        retentionDays: config.retentionDays,
+        mapping: {
+          domain: config.domain,
+          sellerName: config.sellerName,
+          translate: key => translator.translateVal(key),
+        },
+      },
+    );
+
+    try {
+      const report = await run.run();
+      if (report.stale) {
+        console.log(`OpenAI ads feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
+      }
+    } catch (error) {
+      // nest-commander's default serviceErrorHandler only prints a rejected run, and main.ts then
+      // reports {"complete":1,"code":0} — a blocked feed would look green in Cronicle. Fail the job
+      // here (report + lock already handled by the run).
+      LoggerService.exitError(`OpenAI ads feed not published: ${error?.message ?? error}`);
+      process.exit(1);
+    }
+  }
+
+  // One UPA lookup per dealer and run. CarService.getOwner retries 4× and does not remember
+  // failures, so without this map every car of an unknown dealer would wait for the retries again.
+  private readonly dealerStatuses = new Map<string, Promise<DealerStatus>>();
+
+  // CBD already takes blacklisted dealers' cars offline, but a deactivated dealer's cars stay
+  // online — not advertised here. A missing `active` field counts as active (only explicit false).
+  private dealerStatus(ownerId: string): Promise<DealerStatus> {
+    if (!ownerId) {
+      return Promise.resolve('unknown');
+    }
+    if (!this.dealerStatuses.has(ownerId)) {
+      this.dealerStatuses.set(ownerId, this.carService.getOwner(ownerId).then(
+        owner => (!owner ? 'unknown' : owner.active === false || owner.blacklist === true ? 'inactive' : 'active'),
+        () => 'unknown' as DealerStatus,
+      ));
+    }
+    return this.dealerStatuses.get(ownerId);
+  }
+}

package/src/formats/openai-ads-csv.builder.ts

@@ -0,0 +1,55 @@
+import { createWriteStream, WriteStream } from 'fs';
+import { once } from 'events';
+
+import { OPENAI_ADS_COLUMNS, OpenAiAdsRow } from '../interfaces/openai-ads.interface';
+
+// RFC 4180 cell: quoted only when it contains a comma, a quote, CR or LF (the OpenAI product file
+// rule); embedded quotes are doubled. Values are written as they are — callers keep free text on
+// one line, but a stray newline still stays inside its quoted cell instead of breaking the row.
+export const toCsvCell = (value: string | boolean | undefined | null): string => {
+  if (value === undefined || value === null) {
+    return '';
+  }
+  const text = typeof value === 'boolean' ? (value ? 'true' : 'false') : value;
+  return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
+};
+
+export const toCsvLine = (cells: Array<string | boolean | undefined | null>): string => `${cells.map(toCsvCell).join(',')}\n`;
+
+export const openAiAdsCsvHeader = (): string => toCsvLine([...OPENAI_ADS_COLUMNS]);
+
+export const openAiAdsCsvRow = (row: OpenAiAdsRow): string => toCsvLine(OPENAI_ADS_COLUMNS.map(column => row[column]));
+
+/**
+ * OpenAI product file (OpenAI format, not the Google-compatible profile): UTF-8 without BOM, one
+ * header row, one vehicle per row, fixed column order (OPENAI_ADS_COLUMNS). Not built on
+ * XMLBuilderBase and not shared with the Awin/Santander writers, which do not escape quotes.
+ */
+export class OpenAiAdsCsvBuilder {
+  private stream: WriteStream;
+  private rows = 0;
+
+  constructor(private readonly fileName: string) { }
+
+  async buildStart(): Promise<void> {
+    this.stream = createWriteStream(this.fileName, { encoding: 'utf8' });
+    await this.write(openAiAdsCsvHeader());
+  }
+
+  async addItem(row: OpenAiAdsRow): Promise<void> {
+    await this.write(openAiAdsCsvRow(row));
+    this.rows += 1;
+  }
+
+  async buildEnd(): Promise<number> {
+    this.stream.end();
+    await once(this.stream, 'finish');
+    return this.rows;
+  }
+
+  private async write(chunk: string): Promise<void> {
+    if (!this.stream.write(chunk)) {
+      await once(this.stream, 'drain');
+    }
+  }
+}

package/src/interfaces/openai-ads.interface.ts

@@ -0,0 +1,64 @@
+// OpenAI product file, OpenAI format (developers.openai.com/commerce/specs/file-upload/products,
+// read 2026-10-09). Only the columns below are sent: no gtin/mpn/VIN (never invented), no
+// target_countries/store_country (market comes from the account setup), no Google-compatible names.
+export const OPENAI_ADS_COLUMNS = [
+  'item_id',
+  'title',
+  'description',
+  'url',
+  'brand',
+  'seller_name',
+  'image_url',
+  'additional_image_urls',
+  'availability',
+  'price',
+  'condition',
+  'is_ads_eligible',
+] as const;
+
+export type OpenAiAdsColumn = typeof OPENAI_ADS_COLUMNS[number];
+
+export type OpenAiAdsAvailability = 'in_stock' | 'out_of_stock';
+
+export type OpenAiAdsCondition = 'new' | 'used';
+
+export interface OpenAiAdsRow {
+  // canonical auto.de vehicle id (car._id, the VDP id) — same value as GA4 item_id and the
+  // Pixel/CAPI contents[].id of ADV-337
+  item_id: string;
+  title: string;
+  description: string;
+  url: string;
+  brand: string;
+  seller_name: string;
+  image_url: string;
+  // one CSV cell, comma-separated, commas inside URLs as %2C; '' = none
+  additional_image_urls: string;
+  availability: OpenAiAdsAvailability;
+  // "17990.00 EUR": purchase price in major units — never a monthly rate
+  price: string;
+  condition: OpenAiAdsCondition;
+  is_ads_eligible: boolean;
+}
+
+// Why a car of the source snapshot is not in the file as in_stock. Counted in the run report.
+export enum OpenAiAdsExclusion {
+  noId = 'no_id',
+  duplicateId = 'duplicate_id',
+  filtered = 'filtered',
+  invalidPrice = 'invalid_price',
+  noTitle = 'no_title',
+  noBrand = 'no_brand',
+  conditionUnknown = 'condition_unknown',
+  dealerInactive = 'dealer_inactive',
+  dealerUnknown = 'dealer_unknown',
+  noImage = 'no_image',
+}
+
+// Why an item is sent as out_of_stock.
+export enum OpenAiAdsOutOfStockReason {
+  reserved = 'reserved',
+  soldFlag = 'sold_flag',
+  // sent before, not in_stock in this run (sold/offline/deleted/now excluded)
+  notInSnapshot = 'not_in_snapshot',
+}

package/src/interfaces/params.interface.ts

@@ -1,29 +1,56 @@
-import { Environment, ExportType } from '../enums';
-
-export interface Params {
-  environment: Environment;
-  exportType?: ExportType;
-  apikey?: string;
-  String?: string;
-  Number?: string;
-  Date?: string;
-  isNewCar?: boolean;
-  cpc?: string;
-  ownerId?: string;
-  metaPushPriorityCsvUrl?: string;
-  // Cronicle plugin param (text field, so it arrives as a string); overrides the
-  // default MAX_FEED_SALES_PRICE_EUR cap in the configs that apply one.
-  maxCurrentSalesPrice?: string;
-  // Google feed only, opt-in: when all three are set, every exported item is also written to
-  // `<project>.<dataset>.<table>` in BigQuery (schema/feed-items.json, see README).
-  feedItemsBqProject?: string;
-  feedItemsBqDataset?: string;
-  feedItemsBqTable?: string;
-}
-
-// Parsed from feedItemsBq* by GoogleConfig.
-export interface FeedItemsBigQueryTarget {
-  project: string;
-  dataset: string;
-  table: string;
-}
+import { Environment, ExportType } from '../enums';
+
+export interface Params {
+  environment: Environment;
+  exportType?: ExportType;
+  apikey?: string;
+  String?: string;
+  Number?: string;
+  Date?: string;
+  isNewCar?: boolean;
+  cpc?: string;
+  ownerId?: string;
+  metaPushPriorityCsvUrl?: string;
+  // Cronicle plugin param (text field, so it arrives as a string); overrides the
+  // default MAX_FEED_SALES_PRICE_EUR cap in the configs that apply one.
+  maxCurrentSalesPrice?: string;
+  // Google feed only, opt-in: when all three are set, every exported item is also written to
+  // `<project>.<dataset>.<table>` in BigQuery (schema/feed-items.json, see README).
+  feedItemsBqProject?: string;
+  feedItemsBqDataset?: string;
+  feedItemsBqTable?: string;
+  // Google feed only, EnVKV C8: off (default) | audit | enforce, and the car-basic-data enrichment log
+  // table the estimates are read from (see services/publication-gate.ts).
+  publicationGate?: string;
+  publicationGateBqProject?: string;
+  publicationGateBqDataset?: string;
+  publicationGateBqTable?: string;
+  // OpenAI Ads feed only (ADV-335). Cronicle text params, so all arrive as strings; see
+  // configs/openai-ads.config.ts for defaults. Live output needs openAiAdsDryRun=false explicitly.
+  openAiAdsDryRun?: string;
+  openAiAdsSellerName?: string;
+  openAiAdsRetentionDays?: string;
+  openAiAdsImageConcurrency?: string;
+  feedGuardMinSourceItems?: string;
+  feedGuardMinPublishedItems?: string;
+  feedGuardMaxDropPercent?: string;
+  feedGuardMaxSourceAgeHours?: string;
+  feedGuardMaxStaleHours?: string;
+  // "true" for ONE run after a real stock drop the guard blocked
+  feedGuardAcceptDrop?: string;
+}
+
+// Parsed from feedItemsBq* by GoogleConfig.
+export interface FeedItemsBigQueryTarget {
+  project: string;
+  dataset: string;
+  table: string;
+}
+
+// Parsed from publicationGate* by GoogleConfig.
+export interface PublicationGateConfig {
+  mode: 'off' | 'audit' | 'enforce';
+  project: string;
+  dataset: string;
+  table: string;
+}