@bytebrand/export-cars 1.26.47 → 1.26.48

package/src/services/marktde-feed-run.ts

@@ -1,233 +1,233 @@
-import { promises as fs } from 'fs';
-import { dirname, join } from 'path';
-import { ImboImageResponseInterface } from '@bytebrand/chunkable';
-import { DataSchemes } from '@bytebrand/car-schema';
-
-import { convertToCSV } from '../utils/converter.utils';
-import { MarktdeExclusion, applyMarktdeColumns, getMarktdeExclusion } from '../transformers/marktde.transformer';
-import { FeedGuard, FeedGuardError, LastGood, inspectSourceSnapshot } from './feed-guard';
-
-export interface MarktdeSourceCar {
-  car: DataSchemes.Car;
-  images: ImboImageResponseInterface[];
-}
-
-// Everything the run needs from the outside world — the Nest exporter wires the real services,
-// tests pass fakes.
-export interface MarktdeRunDeps {
-  // iterates the base export snapshot chunk by chunk (LoaderService.process)
-  forEachChunk: (fn: (cars: MarktdeSourceCar[]) => Promise<void>) => Promise<unknown>;
-  // existing shared partner filter (images count, Cronicle rules incl. the dealer filter)
-  filterCar: (source: MarktdeSourceCar) => boolean;
-  // final validated image URLs for the car, hero first; [] = no usable image
-  resolveImages: (source: MarktdeSourceCar) => Promise<string[]>;
-  // the legacy mobile.de-style row (MobileTransformer + mapping sheet), keyed by column index
-  legacyRow: (source: MarktdeSourceCar) => Promise<Record<string, unknown>>;
-  imageStats: () => Record<string, number>;
-  // packs csvFile into zipFile (entry name = basename of csvFile)
-  zip: (csvFile: string, zipFile: string) => Promise<void>;
-  guard: FeedGuard;
-}
-
-export interface MarktdeRunOptions {
-  runId?: string;
-  dryRun: boolean;
-  domain: string;
-  sourceDirectory: string;
-  sourceBaseFileName: string;
-  csvFile: string;
-  zipFile: string;
-  stateDir: string;
-  now?: () => Date;
-}
-
-export interface MarktdeRunReport {
-  runId?: string;
-  mode: 'dry-run' | 'live';
-  startedAt: string;
-  finishedAt?: string;
-  status: 'published' | 'blocked' | 'failed';
-  guard?: { code: string; message: string };
-  error?: string;
-  csvFile: string;
-  zipFile: string;
-  source: { files: number; newestChunkAt: string | null; cars: number };
-  items: number;
-  dealers: number;
-  excluded: Record<string, number>;
-  vin: Record<string, number>;
-  vat: { deductible: number; notDeductible: number; rateDefaulted: number };
-  envkv: { fuelEmpty: number; co2Empty: number };
-  description: { items: number; sanitized: number; removed: { url: number; email: number; phone: number; address: number } };
-  imagesPerItem: { min: number | null; max: number | null; total: number };
-  images: Record<string, number>;
-  lastGood: LastGood | null;
-  lastGoodAgeHours: number | null;
-  stale: boolean;
-}
-
-const increment = (counter: Record<string, number>, key: string) => {
-  counter[key] = (counter[key] ?? 0) + 1;
-};
-
-/**
- * One markt.de feed run (ADV-328): source check -> legacy row + markt.de columns + validated images ->
- * temp CSV -> count check -> atomic publish of the CSV -> ZIP of the published CSV -> report.
- * Replaces the old MobileExporter flow for markt.de, which emptied the output directory first (an
- * outage published an empty or partial file) and needed the `zip` binary the DEV host lacks.
- * Nothing outside the output directory and `stateDir` is written; a blocked run leaves the last good
- * CSV/ZIP in place.
- */
-export class MarktdeFeedRun {
-  constructor(private readonly deps: MarktdeRunDeps, private readonly options: MarktdeRunOptions) { }
-
-  private now(): Date {
-    return this.options.now ? this.options.now() : new Date();
-  }
-
-  async run(): Promise<MarktdeRunReport> {
-    const { guard } = this.deps;
-    const { csvFile, zipFile, runId, dryRun } = this.options;
-    const startedAt = this.now();
-    const lastGood = await guard.readLastGood();
-    const report: MarktdeRunReport = {
-      runId,
-      mode: dryRun ? 'dry-run' : 'live',
-      startedAt: startedAt.toISOString(),
-      status: 'failed',
-      csvFile,
-      zipFile,
-      source: { files: 0, newestChunkAt: null, cars: 0 },
-      items: 0,
-      dealers: 0,
-      excluded: {},
-      vin: {},
-      vat: { deductible: 0, notDeductible: 0, rateDefaulted: 0 },
-      envkv: { fuelEmpty: 0, co2Empty: 0 },
-      description: { items: 0, sanitized: 0, removed: { url: 0, email: 0, phone: 0, address: 0 } },
-      imagesPerItem: { min: null, max: null, total: 0 },
-      images: {},
-      lastGood,
-      lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
-      stale: guard.isStale(lastGood, startedAt),
-    };
-
-    await guard.acquireLock(runId ?? 'local', startedAt);
-    const tmpFile = `${csvFile}.tmp`;
-    try {
-      const snapshot = await inspectSourceSnapshot(this.options.sourceDirectory, this.options.sourceBaseFileName);
-      report.source.files = snapshot.files;
-      report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
-      guard.checkSource(snapshot, startedAt);
-
-      await fs.mkdir(dirname(csvFile), { recursive: true });
-      await fs.rm(tmpFile, { force: true });
-      const out = await fs.open(tmpFile, 'w');
-      const seen = new Set<string>();
-      const dealers = new Set<string>();
-      try {
-        await this.deps.forEachChunk(async (cars) => {
-          for (const source of cars) {
-            report.source.cars += 1;
-            const id = String(source?.car?._id ?? '');
-            if (id && seen.has(id)) {
-              increment(report.excluded, MarktdeExclusion.duplicateId);
-              continue;
-            }
-            seen.add(id);
-            if (!this.deps.filterCar(source)) {
-              increment(report.excluded, MarktdeExclusion.filtered);
-              continue;
-            }
-            // cheap checks first: no image probe for a car that is excluded anyway
-            const excluded = getMarktdeExclusion(source.car);
-            if (excluded) {
-              increment(report.excluded, excluded);
-              continue;
-            }
-            const images = await this.deps.resolveImages(source);
-            if (!images.length) {
-              increment(report.excluded, MarktdeExclusion.noImage);
-              continue;
-            }
-            const row = await this.deps.legacyRow(source);
-            const facts = applyMarktdeColumns(row, source.car, { domain: this.options.domain, images });
-            await out.write(`${convertToCSV(row)}\n`, null, 'utf8');
-            dealers.add(String(source.car?.metaData?.ownerId ?? ''));
-            this.count(report, images.length, facts);
-          }
-        });
-      } finally {
-        await out.close();
-      }
-      report.dealers = dealers.size;
-      report.images = this.deps.imageStats();
-
-      const counts = { sourceItems: report.source.cars, publishedItems: report.items };
-      guard.checkCounts(counts, lastGood);
-
-      report.lastGood = await guard.publish(tmpFile, csvFile, counts, this.now(), runId);
-      const zipTmp = zipFile.replace(/\.zip$/, '.tmp.zip');
-      await fs.rm(zipTmp, { force: true });
-      await this.deps.zip(csvFile, zipTmp);
-      await fs.rename(zipTmp, zipFile);
-      report.status = 'published';
-      report.lastGoodAgeHours = 0;
-      report.stale = false;
-      return report;
-    } catch (error) {
-      if (error instanceof FeedGuardError) {
-        report.status = 'blocked';
-        report.guard = { code: error.code, message: error.message };
-      } else {
-        report.error = error?.message ?? String(error);
-      }
-      // keep the rejected file for inspection; the published files stay untouched
-      await fs.rename(tmpFile, `${csvFile}.rejected`).catch(() => undefined);
-      throw error;
-    } finally {
-      await this.writeReport(report);
-      await guard.releaseLock();
-    }
-  }
-
-  private count(report: MarktdeRunReport, images: number, facts: ReturnType<typeof applyMarktdeColumns>): void {
-    report.items += 1;
-    increment(report.vin, facts.vin);
-    if (facts.price.deductible) {
-      report.vat.deductible += 1;
-    } else {
-      report.vat.notDeductible += 1;
-    }
-    if (facts.price.rateDefaulted) {
-      report.vat.rateDefaulted += 1;
-    }
-    if (facts.fuelEmpty) {
-      report.envkv.fuelEmpty += 1;
-    }
-    if (facts.co2Empty) {
-      report.envkv.co2Empty += 1;
-    }
-    if (facts.descriptionRemovals) {
-      report.description.items += 1;
-      const r = facts.descriptionRemovals;
-      if (r.url + r.email + r.phone + r.address > 0) {
-        report.description.sanitized += 1;
-      }
-      for (const key of Object.keys(r) as Array<keyof typeof r>) {
-        report.description.removed[key] += r[key];
-      }
-    }
-    const stats = report.imagesPerItem;
-    stats.min = stats.min === null ? images : Math.min(stats.min, images);
-    stats.max = stats.max === null ? images : Math.max(stats.max, images);
-    stats.total += images;
-  }
-
-  private async writeReport(report: MarktdeRunReport): Promise<void> {
-    report.finishedAt = this.now().toISOString();
-    await fs.mkdir(this.options.stateDir, { recursive: true });
-    await fs.writeFile(join(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
-    console.log(`marktde feed report: ${JSON.stringify(report)}`);
-  }
-}
+import { promises as fs } from 'fs';
+import { dirname, join } from 'path';
+import { ImboImageResponseInterface } from '@bytebrand/chunkable';
+import { DataSchemes } from '@bytebrand/car-schema';
+
+import { convertToCSV } from '../utils/converter.utils';
+import { MarktdeExclusion, applyMarktdeColumns, getMarktdeExclusion } from '../transformers/marktde.transformer';
+import { FeedGuard, FeedGuardError, LastGood, inspectSourceSnapshot } from './feed-guard';
+
+export interface MarktdeSourceCar {
+  car: DataSchemes.Car;
+  images: ImboImageResponseInterface[];
+}
+
+// Everything the run needs from the outside world — the Nest exporter wires the real services,
+// tests pass fakes.
+export interface MarktdeRunDeps {
+  // iterates the base export snapshot chunk by chunk (LoaderService.process)
+  forEachChunk: (fn: (cars: MarktdeSourceCar[]) => Promise<void>) => Promise<unknown>;
+  // existing shared partner filter (images count, Cronicle rules incl. the dealer filter)
+  filterCar: (source: MarktdeSourceCar) => boolean;
+  // final validated image URLs for the car, hero first; [] = no usable image
+  resolveImages: (source: MarktdeSourceCar) => Promise<string[]>;
+  // the legacy mobile.de-style row (MobileTransformer + mapping sheet), keyed by column index
+  legacyRow: (source: MarktdeSourceCar) => Promise<Record<string, unknown>>;
+  imageStats: () => Record<string, number>;
+  // packs csvFile into zipFile (entry name = basename of csvFile)
+  zip: (csvFile: string, zipFile: string) => Promise<void>;
+  guard: FeedGuard;
+}
+
+export interface MarktdeRunOptions {
+  runId?: string;
+  dryRun: boolean;
+  domain: string;
+  sourceDirectory: string;
+  sourceBaseFileName: string;
+  csvFile: string;
+  zipFile: string;
+  stateDir: string;
+  now?: () => Date;
+}
+
+export interface MarktdeRunReport {
+  runId?: string;
+  mode: 'dry-run' | 'live';
+  startedAt: string;
+  finishedAt?: string;
+  status: 'published' | 'blocked' | 'failed';
+  guard?: { code: string; message: string };
+  error?: string;
+  csvFile: string;
+  zipFile: string;
+  source: { files: number; newestChunkAt: string | null; cars: number };
+  items: number;
+  dealers: number;
+  excluded: Record<string, number>;
+  vin: Record<string, number>;
+  vat: { deductible: number; notDeductible: number; rateDefaulted: number };
+  envkv: { fuelEmpty: number; co2Empty: number };
+  description: { items: number; sanitized: number; removed: { url: number; email: number; phone: number; address: number } };
+  imagesPerItem: { min: number | null; max: number | null; total: number };
+  images: Record<string, number>;
+  lastGood: LastGood | null;
+  lastGoodAgeHours: number | null;
+  stale: boolean;
+}
+
+const increment = (counter: Record<string, number>, key: string) => {
+  counter[key] = (counter[key] ?? 0) + 1;
+};
+
+/**
+ * One markt.de feed run (ADV-328): source check -> legacy row + markt.de columns + validated images ->
+ * temp CSV -> count check -> atomic publish of the CSV -> ZIP of the published CSV -> report.
+ * Replaces the old MobileExporter flow for markt.de, which emptied the output directory first (an
+ * outage published an empty or partial file) and needed the `zip` binary the DEV host lacks.
+ * Nothing outside the output directory and `stateDir` is written; a blocked run leaves the last good
+ * CSV/ZIP in place.
+ */
+export class MarktdeFeedRun {
+  constructor(private readonly deps: MarktdeRunDeps, private readonly options: MarktdeRunOptions) { }
+
+  private now(): Date {
+    return this.options.now ? this.options.now() : new Date();
+  }
+
+  async run(): Promise<MarktdeRunReport> {
+    const { guard } = this.deps;
+    const { csvFile, zipFile, runId, dryRun } = this.options;
+    const startedAt = this.now();
+    const lastGood = await guard.readLastGood();
+    const report: MarktdeRunReport = {
+      runId,
+      mode: dryRun ? 'dry-run' : 'live',
+      startedAt: startedAt.toISOString(),
+      status: 'failed',
+      csvFile,
+      zipFile,
+      source: { files: 0, newestChunkAt: null, cars: 0 },
+      items: 0,
+      dealers: 0,
+      excluded: {},
+      vin: {},
+      vat: { deductible: 0, notDeductible: 0, rateDefaulted: 0 },
+      envkv: { fuelEmpty: 0, co2Empty: 0 },
+      description: { items: 0, sanitized: 0, removed: { url: 0, email: 0, phone: 0, address: 0 } },
+      imagesPerItem: { min: null, max: null, total: 0 },
+      images: {},
+      lastGood,
+      lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
+      stale: guard.isStale(lastGood, startedAt),
+    };
+
+    await guard.acquireLock(runId ?? 'local', startedAt);
+    const tmpFile = `${csvFile}.tmp`;
+    try {
+      const snapshot = await inspectSourceSnapshot(this.options.sourceDirectory, this.options.sourceBaseFileName);
+      report.source.files = snapshot.files;
+      report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
+      guard.checkSource(snapshot, startedAt);
+
+      await fs.mkdir(dirname(csvFile), { recursive: true });
+      await fs.rm(tmpFile, { force: true });
+      const out = await fs.open(tmpFile, 'w');
+      const seen = new Set<string>();
+      const dealers = new Set<string>();
+      try {
+        await this.deps.forEachChunk(async (cars) => {
+          for (const source of cars) {
+            report.source.cars += 1;
+            const id = String(source?.car?._id ?? '');
+            if (id && seen.has(id)) {
+              increment(report.excluded, MarktdeExclusion.duplicateId);
+              continue;
+            }
+            seen.add(id);
+            if (!this.deps.filterCar(source)) {
+              increment(report.excluded, MarktdeExclusion.filtered);
+              continue;
+            }
+            // cheap checks first: no image probe for a car that is excluded anyway
+            const excluded = getMarktdeExclusion(source.car);
+            if (excluded) {
+              increment(report.excluded, excluded);
+              continue;
+            }
+            const images = await this.deps.resolveImages(source);
+            if (!images.length) {
+              increment(report.excluded, MarktdeExclusion.noImage);
+              continue;
+            }
+            const row = await this.deps.legacyRow(source);
+            const facts = applyMarktdeColumns(row, source.car, { domain: this.options.domain, images });
+            await out.write(`${convertToCSV(row)}\n`, null, 'utf8');
+            dealers.add(String(source.car?.metaData?.ownerId ?? ''));
+            this.count(report, images.length, facts);
+          }
+        });
+      } finally {
+        await out.close();
+      }
+      report.dealers = dealers.size;
+      report.images = this.deps.imageStats();
+
+      const counts = { sourceItems: report.source.cars, publishedItems: report.items };
+      guard.checkCounts(counts, lastGood);
+
+      report.lastGood = await guard.publish(tmpFile, csvFile, counts, this.now(), runId);
+      const zipTmp = zipFile.replace(/\.zip$/, '.tmp.zip');
+      await fs.rm(zipTmp, { force: true });
+      await this.deps.zip(csvFile, zipTmp);
+      await fs.rename(zipTmp, zipFile);
+      report.status = 'published';
+      report.lastGoodAgeHours = 0;
+      report.stale = false;
+      return report;
+    } catch (error) {
+      if (error instanceof FeedGuardError) {
+        report.status = 'blocked';
+        report.guard = { code: error.code, message: error.message };
+      } else {
+        report.error = error?.message ?? String(error);
+      }
+      // keep the rejected file for inspection; the published files stay untouched
+      await fs.rename(tmpFile, `${csvFile}.rejected`).catch(() => undefined);
+      throw error;
+    } finally {
+      await this.writeReport(report);
+      await guard.releaseLock();
+    }
+  }
+
+  private count(report: MarktdeRunReport, images: number, facts: ReturnType<typeof applyMarktdeColumns>): void {
+    report.items += 1;
+    increment(report.vin, facts.vin);
+    if (facts.price.deductible) {
+      report.vat.deductible += 1;
+    } else {
+      report.vat.notDeductible += 1;
+    }
+    if (facts.price.rateDefaulted) {
+      report.vat.rateDefaulted += 1;
+    }
+    if (facts.fuelEmpty) {
+      report.envkv.fuelEmpty += 1;
+    }
+    if (facts.co2Empty) {
+      report.envkv.co2Empty += 1;
+    }
+    if (facts.descriptionRemovals) {
+      report.description.items += 1;
+      const r = facts.descriptionRemovals;
+      if (r.url + r.email + r.phone + r.address > 0) {
+        report.description.sanitized += 1;
+      }
+      for (const key of Object.keys(r) as Array<keyof typeof r>) {
+        report.description.removed[key] += r[key];
+      }
+    }
+    const stats = report.imagesPerItem;
+    stats.min = stats.min === null ? images : Math.min(stats.min, images);
+    stats.max = stats.max === null ? images : Math.max(stats.max, images);
+    stats.total += images;
+  }
+
+  private async writeReport(report: MarktdeRunReport): Promise<void> {
+    report.finishedAt = this.now().toISOString();
+    await fs.mkdir(this.options.stateDir, { recursive: true });
+    await fs.writeFile(join(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
+    console.log(`marktde feed report: ${JSON.stringify(report)}`);
+  }
+}