@bytebrand/export-cars 1.26.27 → 1.26.29

package/src/services/images.service.ts

@@ -15,6 +15,20 @@ import { isNotEmptyArray } from '../utils';
 import { Config } from '../configs/config';
 import { ImageSize, ImageType } from '../enums';
 
+export interface CleanImage {
+  url: string;
+  imageIdentifier: string;
+  imageCode: string;
+  // 3/4 exterior shot that may serve as the main image (image_url)
+  heroCandidate: boolean;
+}
+
+// Hero candidates in priority order: the 3/4 front views (ext_f_right first, matching IMAGES_SORT_ORDER).
+export const CLEAN_HERO_IMAGE_CODES = ['ext_f_right', 'ext_f_left'];
+// Explicit JPEG variant — the extension-less `large` URL negotiates WebP from the Accept header
+// (no Vary), so a crawler could get a different format than the one we validated.
+export const CLEAN_IMAGE_VARIANT = 'large-cached.jpeg';
+
 @Injectable()
 export class ImagesService {
   private readonly IMAGES_SORT_ORDER: { [imageCode: string]: number } = {
@@ -90,6 +104,9 @@ export class ImagesService {
   async getImages(car: DataSchemes.Car, images: ImboImageResponseInterface[], imageType: ImageType, ignoreDefImages: boolean): Promise<Array<string>> {
     if (imageType === ImageType.default) {
       return this.getDefaultImages(car._id, images, ignoreDefImages);
+    } else if (imageType === ImageType.clean) {
+      // never padded with placeholders, whatever ignoreDefImages says
+      return this.getCleanImageUrls(car._id, images);
     } else {
       return await this.getMarketingImages(car._id, images, ignoreDefImages);
     }
@@ -175,6 +192,61 @@ export class ImagesService {
     });
   }
 
+  /**
+   * Clean channel profile (ADV-336): the car's own photos only, as explicit JPEG URLs, in a
+   * deterministic order — 3/4 exterior hero first, then the remaining exterior views, then the
+   * interior views (IMAGES_SORT_ORDER). Feature close-ups (`_feat_`), `conext_*` and codes the
+   * sort order doesn't know (simple_*, promo, ...) are dropped; so are repeated imageIdentifiers.
+   * Returns [] when the car has no 3/4 exterior shot — callers skip the car, never substitute a
+   * placeholder. Does not validate the URLs (see ImageUrlValidator) and never mutates `images`.
+   */
+  getCleanImages(carId: string, images: ImboImageResponseInterface[]): CleanImage[] {
+    if (!isNotEmptyArray(images)) {
+      return [];
+    }
+
+    const seen = new Set<string>();
+    const eligible = images.filter((image) => {
+      const imageCode = this.getImageCode(image);
+      if (!image?.imageIdentifier || !imageCode || seen.has(image.imageIdentifier) || !this.isCleanImageCode(imageCode)) {
+        return false;
+      }
+      seen.add(image.imageIdentifier);
+      return true;
+    });
+    const sorted = this.sortImages(eligible);
+
+    const heroIndex = CLEAN_HERO_IMAGE_CODES
+      .map(code => sorted.findIndex(image => this.getImageCode(image) === code))
+      .find(index => index >= 0);
+    if (heroIndex === undefined) {
+      return [];
+    }
+
+    const ordered = [sorted[heroIndex], ...sorted.filter((_, index) => index !== heroIndex)];
+    return ordered.map((image) => {
+      const imageCode = this.getImageCode(image);
+      return {
+        url: `${this.configService.config.baseImageUrl}/carimage/${carId}/${image.imageIdentifier}/${CLEAN_IMAGE_VARIANT}`,
+        imageIdentifier: image.imageIdentifier,
+        imageCode,
+        heroCandidate: CLEAN_HERO_IMAGE_CODES.includes(imageCode),
+      };
+    });
+  }
+
+  getCleanImageUrls(carId: string, images: ImboImageResponseInterface[]): string[] {
+    return this.getCleanImages(carId, images).map(image => image.url);
+  }
+
+  private isCleanImageCode(imageCode: string): boolean {
+    return /^(ext|int)_/.test(imageCode) && !imageCode.includes('_feat_') && imageCode in this.IMAGES_SORT_ORDER;
+  }
+
+  private getImageCode(image: ImboImageResponseInterface): string | undefined {
+    return image?.imageCodePredicted?.split('-')[0];
+  }
+
   async getMarketingImages(carId: string, images: ImboImageResponseInterface[], ignoreDefImages: boolean): Promise<string[]> {
     const imageUrl = this.configService.config.marketingImageUrl;
 

package/src/services/index.ts

@@ -7,3 +7,4 @@ export * from './images.service';
 export * from './rules.service';
 export * from './user.service';
 export * from './car.service';
+export * from './image-url-validator.service';

package/src/services/openai-ads-feed-run.ts

@@ -0,0 +1,226 @@
+import { promises as fs } from 'fs';
+import { dirname, join } from 'path';
+import { ImboImageResponseInterface } from '@bytebrand/chunkable';
+import { DataSchemes } from '@bytebrand/car-schema';
+
+import { OpenAiAdsCsvBuilder } from '../formats/openai-ads-csv.builder';
+import {
+  OpenAiAdsExclusion,
+  OpenAiAdsOutOfStockReason,
+  OpenAiAdsRow,
+} from '../interfaces/openai-ads.interface';
+import {
+  OpenAiAdsMappingContext,
+  mapOpenAiAdsVehicle,
+  withOpenAiImages,
+} from '../transformers/openai-ads.transformer';
+import { OpenAiImageSelection } from '../transformers/openai-image-fields';
+import { FeedGuard, FeedGuardError, LastGood, inspectSourceSnapshot } from './feed-guard';
+import { OpenAiAdsLedger } from './openai-ads-ledger';
+
+export interface SourceCar {
+  car: DataSchemes.Car;
+  images: ImboImageResponseInterface[];
+}
+
+export type DealerStatus = 'active' | 'inactive' | 'unknown';
+
+// Everything the run needs from the outside world — the Nest exporter wires the real services,
+// tests pass fakes.
+export interface OpenAiAdsRunDeps {
+  // iterates the base export snapshot chunk by chunk (LoaderService.process)
+  forEachChunk: (fn: (cars: SourceCar[]) => Promise<void>) => Promise<unknown>;
+  // existing shared partner filter (images count, optional price cap, Cronicle rules)
+  filterCar: (source: SourceCar) => boolean;
+  dealerStatus: (ownerId: string) => Promise<DealerStatus>;
+  // ADV-336 clean images, validated
+  resolveImages: (source: SourceCar) => Promise<OpenAiImageSelection>;
+  imageStats: () => Record<string, number>;
+  guard: FeedGuard;
+}
+
+export interface OpenAiAdsRunOptions {
+  runId?: string;
+  dryRun: boolean;
+  sourceDirectory: string;
+  sourceBaseFileName: string;
+  outputFile: string;
+  stateDir: string;
+  retentionDays: number;
+  mapping: Omit<OpenAiAdsMappingContext, 'now'>;
+  now?: () => Date;
+}
+
+export interface OpenAiAdsRunReport {
+  runId?: string;
+  mode: 'dry-run' | 'live';
+  startedAt: string;
+  finishedAt?: string;
+  status: 'published' | 'blocked' | 'failed';
+  guard?: { code: string; message: string };
+  outputFile: string;
+  source: { files: number; newestChunkAt: string | null; cars: number };
+  rows: { inStock: number; outOfStock: number; total: number };
+  outOfStock: Record<string, number>;
+  excluded: Record<string, number>;
+  reservedNeverSent: number;
+  ledger: { before: number; after: number; expired: number };
+  price: { min: number | null; max: number | null };
+  images: Record<string, number>;
+  lastGood: LastGood | null;
+  lastGoodAgeHours: number | null;
+  stale: boolean;
+}
+
+const increment = (counter: Record<string, number>, key: string) => {
+  counter[key] = (counter[key] ?? 0) + 1;
+};
+
+/**
+ * One OpenAI Ads feed run (ADV-335): source check -> map + validate -> temp CSV -> count check ->
+ * atomic publish -> ledger + report. Nothing outside `outputFile`'s directory and `stateDir` is
+ * written; transport (hosted URL / SFTP) is a separate step that only ever reads the published file.
+ */
+export class OpenAiAdsFeedRun {
+  constructor(private readonly deps: OpenAiAdsRunDeps, private readonly options: OpenAiAdsRunOptions) { }
+
+  private now(): Date {
+    return this.options.now ? this.options.now() : new Date();
+  }
+
+  async run(): Promise<OpenAiAdsRunReport> {
+    const { guard } = this.deps;
+    const { outputFile, stateDir, runId } = this.options;
+    const startedAt = this.now();
+    const lastGood = await guard.readLastGood();
+    const report: OpenAiAdsRunReport = {
+      runId,
+      mode: this.options.dryRun ? 'dry-run' : 'live',
+      startedAt: startedAt.toISOString(),
+      status: 'failed',
+      outputFile,
+      source: { files: 0, newestChunkAt: null, cars: 0 },
+      rows: { inStock: 0, outOfStock: 0, total: 0 },
+      outOfStock: {},
+      excluded: {},
+      reservedNeverSent: 0,
+      ledger: { before: 0, after: 0, expired: 0 },
+      price: { min: null, max: null },
+      images: {},
+      lastGood,
+      lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
+      stale: guard.isStale(lastGood, startedAt),
+    };
+
+    await guard.acquireLock(runId ?? 'local', startedAt);
+    const tmpFile = `${outputFile}.tmp`;
+    try {
+      const snapshot = await inspectSourceSnapshot(this.options.sourceDirectory, this.options.sourceBaseFileName);
+      report.source.files = snapshot.files;
+      report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
+      guard.checkSource(snapshot, startedAt);
+
+      const ledgerFile = join(stateDir, 'ledger.json');
+      const ledger = await OpenAiAdsLedger.load(ledgerFile);
+      report.ledger.before = ledger.size;
+
+      await fs.mkdir(dirname(outputFile), { recursive: true });
+      const builder = new OpenAiAdsCsvBuilder(tmpFile);
+      await builder.buildStart();
+
+      const seen = new Set<string>();
+      const inStockIds = new Set<string>();
+      const freshOutOfStock = new Map<string, OpenAiAdsRow>();
+      const freshReasons = new Map<string, OpenAiAdsOutOfStockReason>();
+      const mapping: OpenAiAdsMappingContext = { ...this.options.mapping, now: startedAt };
+
+      await this.deps.forEachChunk(async (cars) => {
+        for (const source of cars) {
+          report.source.cars += 1;
+          const id = String(source?.car?._id ?? '');
+          if (id && seen.has(id)) {
+            increment(report.excluded, OpenAiAdsExclusion.duplicateId);
+            continue;
+          }
+          seen.add(id);
+          if (!this.deps.filterCar(source)) {
+            increment(report.excluded, OpenAiAdsExclusion.filtered);
+            continue;
+          }
+          const mapped = mapOpenAiAdsVehicle(source.car, mapping);
+          if (mapped.ok === false) {
+            increment(report.excluded, mapped.reason);
+            continue;
+          }
+          // only cars that would be advertised are worth a dealer/image check
+          if (mapped.outOfStockReason && !ledger.has(mapped.row.item_id)) {
+            report.reservedNeverSent += 1;
+            continue;
+          }
+          const dealer = await this.deps.dealerStatus(String(source.car?.metaData?.ownerId ?? ''));
+          if (dealer !== 'active' && !mapped.outOfStockReason) {
+            increment(report.excluded, dealer === 'inactive' ? OpenAiAdsExclusion.dealerInactive : OpenAiAdsExclusion.dealerUnknown);
+            continue;
+          }
+          const images = await this.deps.resolveImages(source);
+          if (!images.fields) {
+            if (!mapped.outOfStockReason) {
+              increment(report.excluded, OpenAiAdsExclusion.noImage);
+            }
+            continue;
+          }
+          const row = withOpenAiImages(mapped.row, images.fields);
+          if (mapped.outOfStockReason) {
+            freshOutOfStock.set(row.item_id, row);
+            freshReasons.set(row.item_id, mapped.outOfStockReason);
+            continue;
+          }
+          await builder.addItem(row);
+          inStockIds.add(row.item_id);
+          ledger.markInStock(row, startedAt);
+          report.rows.inStock += 1;
+          const amount = Number(row.price.split(' ')[0]);
+          report.price.min = report.price.min === null ? amount : Math.min(report.price.min, amount);
+          report.price.max = report.price.max === null ? amount : Math.max(report.price.max, amount);
+        }
+      });
+
+      const { rows: outOfStockRows, expired } = ledger.collectOutOfStock(
+        inStockIds, freshOutOfStock, startedAt, this.options.retentionDays,
+      );
+      for (const { row, fresh } of outOfStockRows) {
+        await builder.addItem(row);
+        increment(report.outOfStock, fresh ? freshReasons.get(row.item_id) : OpenAiAdsOutOfStockReason.notInSnapshot);
+      }
+      report.rows.outOfStock = outOfStockRows.length;
+      report.rows.total = await builder.buildEnd();
+      report.ledger.expired = expired;
+      report.images = this.deps.imageStats();
+
+      const counts = { sourceItems: report.source.cars, publishedItems: report.rows.inStock };
+      guard.checkCounts(counts, lastGood);
+
+      report.lastGood = await guard.publish(tmpFile, outputFile, counts, this.now(), runId);
+      await ledger.save(ledgerFile, this.now());
+      report.ledger.after = ledger.size;
+      report.status = 'published';
+      report.lastGoodAgeHours = 0;
+      report.stale = false;
+      return report;
+    } catch (error) {
+      if (error instanceof FeedGuardError) {
+        report.status = 'blocked';
+        report.guard = { code: error.code, message: error.message };
+      }
+      // keep the rejected file for inspection; the published file stays untouched
+      await fs.rename(tmpFile, `${outputFile}.rejected`).catch(() => undefined);
+      throw error;
+    } finally {
+      report.finishedAt = this.now().toISOString();
+      await fs.mkdir(stateDir, { recursive: true });
+      await fs.writeFile(join(stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
+      console.log(`OpenAI ads feed report: ${JSON.stringify(report)}`);
+      await guard.releaseLock();
+    }
+  }
+}

package/src/services/openai-ads-ledger.ts

@@ -0,0 +1,116 @@
+import { promises as fs } from 'fs';
+import { dirname } from 'path';
+
+import { OpenAiAdsRow } from '../interfaces/openai-ads.interface';
+import { asOutOfStock } from '../transformers/openai-ads.transformer';
+
+const LEDGER_VERSION = 1;
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+interface LedgerEntry {
+  // last run in which the item was sent in_stock (ISO)
+  lastInStockAt: string;
+  // the row as last sent in_stock, without additional_image_urls (kept small)
+  row: OpenAiAdsRow;
+}
+
+interface LedgerFile {
+  version: number;
+  updatedAt: string;
+  items: Record<string, LedgerEntry>;
+}
+
+export interface OutOfStockRow {
+  row: OpenAiAdsRow;
+  // true when the row data comes from this run's snapshot (car still online but reserved/sold)
+  fresh: boolean;
+}
+
+/**
+ * Items already sent to OpenAI as in_stock (ADV-335). The base export only contains online cars,
+ * so a sold, offline or deleted car simply disappears from the source — but a missing row is not a
+ * documented removal. Every item that was sent in_stock and is not in_stock in the current run is
+ * therefore sent again as out_of_stock (same item_id) until `retentionDays` after it was last in
+ * stock; OpenAI expires items that are not refreshed for two weeks, so the default retention
+ * (15 days) covers that window. A car that comes back is sent in_stock again with the same id.
+ *
+ * The ledger is changed in memory during a run and written only by the exporter after the file
+ * was published, so a blocked run never "forgets" items.
+ */
+export class OpenAiAdsLedger {
+  private constructor(private readonly items: Record<string, LedgerEntry>) { }
+
+  static empty(): OpenAiAdsLedger {
+    return new OpenAiAdsLedger({});
+  }
+
+  static async load(file: string): Promise<OpenAiAdsLedger> {
+    let raw: string;
+    try {
+      raw = await fs.readFile(file, 'utf8');
+    } catch (error) {
+      if (error?.code === 'ENOENT') {
+        return OpenAiAdsLedger.empty();
+      }
+      throw error;
+    }
+    // A corrupt ledger must stop the run: starting empty would silently drop every out_of_stock row.
+    const parsed = JSON.parse(raw) as LedgerFile;
+    if (parsed?.version !== LEDGER_VERSION || typeof parsed.items !== 'object') {
+      throw new Error(`OpenAI ads ledger ${file} has an unknown format (version ${parsed?.version})`);
+    }
+    return new OpenAiAdsLedger(parsed.items);
+  }
+
+  get size(): number {
+    return Object.keys(this.items).length;
+  }
+
+  has(itemId: string): boolean {
+    return itemId in this.items;
+  }
+
+  markInStock(row: OpenAiAdsRow, now: Date): void {
+    this.items[row.item_id] = {
+      lastInStockAt: now.toISOString(),
+      row: { ...row, additional_image_urls: '' },
+    };
+  }
+
+  /**
+   * out_of_stock rows for every ledger item that is not in `inStockIds`. `freshRows` holds this
+   * run's data for cars that are online but reserved/sold; other items reuse the last sent row.
+   * Items past the retention window are dropped from the ledger and not sent any more.
+   */
+  collectOutOfStock(
+    inStockIds: Set<string>,
+    freshRows: Map<string, OpenAiAdsRow>,
+    now: Date,
+    retentionDays: number,
+  ): { rows: OutOfStockRow[]; expired: number } {
+    const rows: OutOfStockRow[] = [];
+    let expired = 0;
+    for (const [itemId, entry] of Object.entries(this.items)) {
+      if (inStockIds.has(itemId)) {
+        continue;
+      }
+      if (now.getTime() - Date.parse(entry.lastInStockAt) > retentionDays * DAY_MS) {
+        delete this.items[itemId];
+        expired += 1;
+        continue;
+      }
+      const fresh = freshRows.get(itemId);
+      rows.push({ row: asOutOfStock(fresh ?? entry.row), fresh: !!fresh });
+    }
+    return { rows, expired };
+  }
+
+  // Write-then-rename, so a crash never leaves a half-written ledger behind.
+  async save(file: string, now: Date): Promise<void> {
+    const content: LedgerFile = { version: LEDGER_VERSION, updatedAt: now.toISOString(), items: this.items };
+    await fs.mkdir(dirname(file), { recursive: true });
+    const tmp = `${file}.tmp`;
+    await fs.writeFile(tmp, JSON.stringify(content), 'utf8');
+    await fs.rename(tmp, file);
+  }
+}

package/src/services/publication-gate.ts

@@ -0,0 +1,119 @@
+import { join } from 'path';
+import { get } from 'lodash';
+import { BigQueryLogger } from '@bytebrand/bigquery-logger';
+import { DataSchemes } from '@bytebrand/car-schema';
+import { PublicationGateConfig } from '../interfaces/params.interface';
+import { GoogleEnvkvAttribute, googleEnvkvSourceFields } from '../transformers/google-feed-envkv';
+
+// EnVKV stage C8 — publication gate for estimated EnVKV values in the Google feed.
+//
+// No car field says whether a value was delivered or estimated; car-basic-data writes every change it
+// makes to the enrichment audit log (BigQuery). At the start of the export the gate reads the latest
+// log row per car and field; a field counts as estimated when that row came from an estimate source
+// and its final value is still the value stored on the car (a later import with a delivered value
+// replaces it without a log row, so the stored value no longer matches).
+//
+//   off      no BigQuery call, the feed as before (default)
+//   audit    the feed as before, the estimated attributes are recorded in the feed items table
+//   enforce  estimated attributes are left out of the item (and recorded)
+
+// finalValueSource values of CBD's enrichment log that are estimates: the CO2 estimator (EEA register /
+// model), the make-model template sheet and the global defaults.
+export const ESTIMATE_SOURCES = ['co2-estimate', 'mmm', 'globalDefault'];
+
+// Every car field a Google EnVKV attribute can be taken from (google-feed-envkv picks).
+export const GATED_FIELDS = [
+  'consumption.wltpWeightedCombined',
+  'consumption.wltpCombined',
+  'consumption.consumptionCombined',
+  'consumption.wltpWeightedPowerCombined',
+  'consumption.wltpPowerCombined',
+  'consumption.consumptionPowerCombined',
+  'environmentEmissions.wltpCo2',
+  'environmentEmissions.co2',
+  'battery.wltpTotalRange',
+  'battery.wltpRange',
+  'battery.batteryRangeElectric',
+];
+
+// '7.10' and 7.1 are the same value; non-numeric values compare as strings.
+const sameValue = (logged: string, stored: unknown): boolean => {
+  if (stored === undefined || stored === null) return false;
+  const a = Number(logged);
+  const b = Number(stored);
+  if (Number.isFinite(a) && Number.isFinite(b)) return Math.abs(a - b) < 1e-9;
+  return logged === String(stored);
+};
+
+export class PublicationGate {
+  // estimates: `${carId}|${field}` -> final value of the latest estimate row
+  constructor(readonly mode: 'off' | 'audit' | 'enforce', private estimates: Map<string, string>) {}
+
+  static off(): PublicationGate {
+    return new PublicationGate('off', new Map());
+  }
+
+  get size(): number {
+    return this.estimates.size;
+  }
+
+  // The attributes of this car whose value comes from an estimated field, in a fixed order.
+  estimatedAttributes(car: DataSchemes.Car): GoogleEnvkvAttribute[] {
+    if (this.mode === 'off') return [];
+    const carId = String(get(car, '_id', ''));
+    const attributes: GoogleEnvkvAttribute[] = [];
+    for (const [attribute, field] of Object.entries(googleEnvkvSourceFields(car))) {
+      const logged = this.estimates.get(`${carId}|${field}`);
+      if (logged !== undefined && sameValue(logged, get(car, field))) attributes.push(attribute as GoogleEnvkvAttribute);
+    }
+    return attributes;
+  }
+
+  // enforce: drops the estimated attributes from the item. Returns the item for chaining.
+  apply<T extends Partial<Record<GoogleEnvkvAttribute, string>>>(item: T, attributes: GoogleEnvkvAttribute[]): T {
+    if (this.mode !== 'enforce') return item;
+    for (const attribute of attributes) delete item[attribute];
+    return item;
+  }
+}
+
+class EnrichmentLogReader extends BigQueryLogger {
+  async latestEstimates(table: string): Promise<Map<string, string>> {
+    const query = `
+      SELECT carId, fieldName, finalValue
+      FROM (
+        SELECT carId, fieldName, finalValue, finalValueSource,
+               ROW_NUMBER() OVER (PARTITION BY carId, fieldName ORDER BY timestamp DESC) AS rn
+        FROM \`${table}\`
+        WHERE fieldName IN UNNEST(@fields)
+      )
+      WHERE rn = 1 AND finalValueSource IN UNNEST(@sources) AND finalValue IS NOT NULL`;
+    const [rows] = await this.client.query({ query, params: { fields: GATED_FIELDS, sources: ESTIMATE_SOURCES } });
+    const estimates = new Map<string, string>();
+    for (const row of rows) estimates.set(`${row.carId}|${row.fieldName}`, String(row.finalValue));
+    return estimates;
+  }
+}
+
+// Reads the estimates once per run. audit: a failure only switches the gate off. enforce: a failure
+// fails the export, so the last published feed stays in place instead of a feed with unchecked estimates.
+export async function loadPublicationGate(config: PublicationGateConfig | undefined): Promise<PublicationGate> {
+  if (!config || config.mode === 'off') return PublicationGate.off();
+  const table = `${config.project}.${config.dataset}.${config.table}`;
+  try {
+    const credentials = require(join(process.cwd(), 'credentials/google-cloud-project-id.json'));
+    const reader = new EnrichmentLogReader(credentials, {
+      BigQueryProject: config.project,
+      BigQueryDataset: config.dataset,
+      BigQueryLogTable: config.table,
+    });
+    const estimates = await reader.latestEstimates(table);
+    console.log(`Publication gate ${config.mode}: ${estimates.size} estimated car fields from ${table}`);
+    return new PublicationGate(config.mode, estimates);
+  } catch (e) {
+    const message = String(e?.message || e).split('\n')[0];
+    if (config.mode === 'enforce') throw new Error(`Publication gate: cannot read ${table}: ${message}`);
+    console.error(`Publication gate disabled for this run: cannot read ${table}: ${message}`);
+    return PublicationGate.off();
+  }
+}