@bytebrand/export-cars 1.26.31 → 1.26.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/configs/autouncle.config.d.ts +14 -0
  2. package/dist/configs/autouncle.config.js +36 -3
  3. package/dist/configs/autouncle.config.js.map +1 -1
  4. package/dist/configs/meinstadt.config.d.ts +15 -0
  5. package/dist/configs/meinstadt.config.js +42 -3
  6. package/dist/configs/meinstadt.config.js.map +1 -1
  7. package/dist/constants/autouncle.contants.d.ts +1 -3
  8. package/dist/constants/autouncle.contants.js +2 -4
  9. package/dist/constants/autouncle.contants.js.map +1 -1
  10. package/dist/exporters/autouncle.exporter.d.ts +3 -2
  11. package/dist/exporters/autouncle.exporter.js +45 -27
  12. package/dist/exporters/autouncle.exporter.js.map +1 -1
  13. package/dist/exporters/meinstadt.exporter.d.ts +3 -2
  14. package/dist/exporters/meinstadt.exporter.js +40 -23
  15. package/dist/exporters/meinstadt.exporter.js.map +1 -1
  16. package/dist/formats/autouncle-json.builder.js +1 -4
  17. package/dist/formats/autouncle-json.builder.js.map +1 -1
  18. package/dist/interfaces/export-item.interface.d.ts +8 -12
  19. package/dist/interfaces/openai-ads.interface.d.ts +2 -1
  20. package/dist/interfaces/openai-ads.interface.js +1 -0
  21. package/dist/interfaces/openai-ads.interface.js.map +1 -1
  22. package/dist/interfaces/params.interface.d.ts +6 -0
  23. package/dist/services/autouncle-feed-run.d.ts +66 -0
  24. package/dist/services/autouncle-feed-run.js +135 -0
  25. package/dist/services/autouncle-feed-run.js.map +1 -0
  26. package/dist/services/meinstadt-feed-run.d.ts +81 -0
  27. package/dist/services/meinstadt-feed-run.js +166 -0
  28. package/dist/services/meinstadt-feed-run.js.map +1 -0
  29. package/dist/services/openai-ads-feed-run.d.ts +1 -0
  30. package/dist/services/openai-ads-feed-run.js +4 -0
  31. package/dist/services/openai-ads-feed-run.js.map +1 -1
  32. package/dist/transformers/autouncle.transformer.d.ts +38 -22
  33. package/dist/transformers/autouncle.transformer.js +156 -144
  34. package/dist/transformers/autouncle.transformer.js.map +1 -1
  35. package/dist/transformers/meinstadt.transformer.d.ts +17 -13
  36. package/dist/transformers/meinstadt.transformer.js +88 -61
  37. package/dist/transformers/meinstadt.transformer.js.map +1 -1
  38. package/dist/transformers/openai-ads-finance.d.ts +2 -0
  39. package/dist/transformers/openai-ads-finance.js +39 -0
  40. package/dist/transformers/openai-ads-finance.js.map +1 -0
  41. package/dist/transformers/openai-ads.transformer.d.ts +9 -1
  42. package/dist/transformers/openai-ads.transformer.js +25 -5
  43. package/dist/transformers/openai-ads.transformer.js.map +1 -1
  44. package/dist/tsconfig.build.tsbuildinfo +1 -1
  45. package/docs/autouncle/AUDIT.md +97 -0
  46. package/docs/autouncle/PARTNER-QUESTIONS.md +53 -0
  47. package/docs/autouncle/RUNBOOK.md +64 -0
  48. package/docs/meinestadt/AUDIT.md +146 -0
  49. package/docs/meinestadt/PARTNER-QUESTIONS.md +54 -0
  50. package/docs/meinestadt/RUNBOOK.md +64 -0
  51. package/docs/openai-ads/AUDIT.md +2 -1
  52. package/docs/openai-ads/RUNBOOK.md +26 -2
  53. package/package.json +3 -2
  54. package/src/configs/autouncle.config.ts +51 -3
  55. package/src/configs/meinstadt.config.ts +57 -2
  56. package/src/constants/autouncle.contants.ts +2 -3
  57. package/src/exporters/autouncle.exporter.ts +59 -37
  58. package/src/exporters/meinstadt.exporter.ts +53 -32
  59. package/src/formats/autouncle-json.builder.ts +2 -4
  60. package/src/interfaces/export-item.interface.ts +15 -12
  61. package/src/interfaces/openai-ads.interface.ts +2 -0
  62. package/src/interfaces/params.interface.ts +10 -0
  63. package/src/services/autouncle-feed-run.ts +197 -0
  64. package/src/services/meinstadt-feed-run.ts +240 -0
  65. package/src/services/openai-ads-feed-run.ts +6 -0
  66. package/src/transformers/autouncle.transformer.ts +217 -175
  67. package/src/transformers/meinstadt.transformer.ts +114 -68
  68. package/src/transformers/openai-ads-finance.ts +60 -0
  69. package/src/transformers/openai-ads.transformer.ts +32 -6
  70. package/test/autouncle-feed-run.test.ts +256 -0
  71. package/test/autouncle-mapping.test.ts +225 -0
  72. package/test/fixtures/meinstadt.fixture.ts +87 -0
  73. package/test/fixtures/openai-ads-finance-titles.json +325 -0
  74. package/test/meinstadt-feed-run.test.ts +288 -0
  75. package/test/meinstadt-mapping.test.ts +215 -0
  76. package/test/openai-ads-finance.test.ts +109 -0
@@ -1,12 +1,60 @@
1
- import { ExportType, FileType, ImageType } from 'src/enums';
1
+ import { toNumber } from 'lodash';
2
+
3
+ import { ExportType, FileType, ImageType } from '../enums';
2
4
  import { Params } from '../interfaces/params.interface';
5
+ import { FeedGuardOptions } from '../services/feed-guard';
3
6
  import { Config } from './config';
4
7
 
8
+ const text = (value: unknown): string => (typeof value === 'string' ? value.trim() : '');
9
+
10
+ const positiveNumber = (value: unknown, fallback: number): number => {
11
+ const parsed = toNumber(text(value));
12
+ return text(value) && Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
13
+ };
14
+
15
+ // Guard defaults. The PROD job only exports a dealer allow-list, so the published count is far below
16
+ // the base snapshot: minPublishedItems only forbids an empty file — set a real minimum per job.
17
+ export const AUTOUNCLE_DEFAULTS = {
18
+ imageConcurrency: 16,
19
+ minSourceItems: 1000,
20
+ minPublishedItems: 1,
21
+ maxDropPercent: 20,
22
+ maxSourceAgeHours: 26,
23
+ maxStaleHours: 48,
24
+ };
25
+
26
+ /**
27
+ * AutoUncle JSON feed (ADV-322/323): `{"ads":[...]}` with clean ADV-336 images.
28
+ * Dry-run is the default: the file goes to resources/autouncle-dryrun, which no Cronicle step reads.
29
+ * Only autouncleDryRun=false writes resources/autouncle-export (the path `3.1 … part 2` copies from).
30
+ * Each mode has its own guard state.
31
+ */
5
32
  export class AutouncleConfig extends Config {
33
+ dryRun: boolean;
34
+ imageConcurrency: number;
35
+ // 0 = no limit (no partner limit known)
36
+ maxImages: number;
37
+ stateDir: string;
38
+ feedGuard: FeedGuardOptions;
39
+
6
40
  constructor(params: Params) {
7
41
  super(params);
8
- this.pathesConfig.localConvertedDirectory = `resources/${ExportType.AUTOUNCLE}-export`;
9
- this.imageType = ImageType.default;
42
+ this.dryRun = text(params.autouncleDryRun).toLowerCase() !== 'false';
43
+ const mode = this.dryRun ? 'dryrun' : 'export';
44
+ this.pathesConfig.localConvertedDirectory = `resources/${ExportType.AUTOUNCLE}-${mode}`;
45
+ this.stateDir = `resources/${ExportType.AUTOUNCLE}-state/${mode}`;
46
+ this.imageType = ImageType.clean;
47
+ this.imageConcurrency = positiveNumber(params.autouncleImageConcurrency, AUTOUNCLE_DEFAULTS.imageConcurrency);
48
+ this.maxImages = Math.floor(positiveNumber(params.autouncleMaxImages, 0));
49
+ this.feedGuard = {
50
+ stateDir: this.stateDir,
51
+ minSourceItems: positiveNumber(params.feedGuardMinSourceItems, AUTOUNCLE_DEFAULTS.minSourceItems),
52
+ minPublishedItems: positiveNumber(params.feedGuardMinPublishedItems, AUTOUNCLE_DEFAULTS.minPublishedItems),
53
+ maxDropPercent: positiveNumber(params.feedGuardMaxDropPercent, AUTOUNCLE_DEFAULTS.maxDropPercent),
54
+ maxSourceAgeHours: positiveNumber(params.feedGuardMaxSourceAgeHours, AUTOUNCLE_DEFAULTS.maxSourceAgeHours),
55
+ maxStaleHours: positiveNumber(params.feedGuardMaxStaleHours, AUTOUNCLE_DEFAULTS.maxStaleHours),
56
+ acceptDrop: text(params.feedGuardAcceptDrop).toLowerCase() === 'true',
57
+ };
10
58
  this.outputFiles = [
11
59
  `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.AUTOUNCLE}.${FileType.JSON}`,
12
60
  ];
@@ -1,12 +1,67 @@
1
+ import { toNumber } from 'lodash';
2
+
1
3
  import { Params } from '../interfaces/params.interface';
4
+ import { FeedGuardOptions } from '../services/feed-guard';
2
5
  import { Config } from './config';
3
6
  import { ExportType, FileType, ImageType } from '../enums';
4
7
 
8
+ const text = (value: unknown): string => (typeof value === 'string' ? value.trim() : '');
9
+
10
+ const positiveNumber = (value: unknown, fallback: number): number => {
11
+ const parsed = toNumber(text(value));
12
+ return text(value) && Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
13
+ };
14
+
15
+ // A CPC tariff as the Cronicle text param holds it: a positive decimal with a dot ("0.99").
16
+ // Anything else (empty, 0, "0,08", "abc") counts as not configured.
17
+ export const parseMeinstadtCpc = (value: unknown): string | null => {
18
+ const raw = text(value);
19
+ return /^\d+(\.\d+)?$/.test(raw) && Number(raw) > 0 ? raw : null;
20
+ };
21
+
22
+ // Same guard defaults as the OpenAI feed (prod-sized); DEV needs smaller minimums via the params.
23
+ export const MEINSTADT_DEFAULTS = {
24
+ imageConcurrency: 16,
25
+ minSourceItems: 1000,
26
+ minPublishedItems: 500,
27
+ maxDropPercent: 20,
28
+ maxSourceAgeHours: 26,
29
+ maxStaleHours: 48,
30
+ };
31
+
32
+ /**
33
+ * meinestadt.de vehicle feed (ADV-325/326): JSON array + ZIP of it, clean ADV-336 images.
34
+ * Dry-run is the default: the files go to resources/meinstadt-dryrun, which no upload step reads.
35
+ * Only meinstadtDryRun=false writes resources/meinstadt-export (the path the Cronicle FTP step
36
+ * uploads from), and only with a configured cpc. Each mode has its own guard state.
37
+ */
5
38
  export class MeinstadtConfig extends Config {
39
+ dryRun: boolean;
40
+ imageConcurrency: number;
41
+ // 0 = no limit (no partner limit known)
42
+ maxImages: number;
43
+ stateDir: string;
44
+ feedGuard: FeedGuardOptions;
45
+
6
46
  constructor(params: Params) {
7
47
  super(params);
8
- this.pathesConfig.localConvertedDirectory = `resources/${ExportType.MEINSTADT}-export`;
9
- this.imageType = ImageType.default;
48
+ this.dryRun = text(params.meinstadtDryRun).toLowerCase() !== 'false';
49
+ const mode = this.dryRun ? 'dryrun' : 'export';
50
+ this.pathesConfig.localConvertedDirectory = `resources/${ExportType.MEINSTADT}-${mode}`;
51
+ this.stateDir = `resources/${ExportType.MEINSTADT}-state/${mode}`;
52
+ this.imageType = ImageType.clean;
53
+ this.cpc = parseMeinstadtCpc(params.cpc);
54
+ this.imageConcurrency = positiveNumber(params.meinstadtImageConcurrency, MEINSTADT_DEFAULTS.imageConcurrency);
55
+ this.maxImages = Math.floor(positiveNumber(params.meinstadtMaxImages, 0));
56
+ this.feedGuard = {
57
+ stateDir: this.stateDir,
58
+ minSourceItems: positiveNumber(params.feedGuardMinSourceItems, MEINSTADT_DEFAULTS.minSourceItems),
59
+ minPublishedItems: positiveNumber(params.feedGuardMinPublishedItems, MEINSTADT_DEFAULTS.minPublishedItems),
60
+ maxDropPercent: positiveNumber(params.feedGuardMaxDropPercent, MEINSTADT_DEFAULTS.maxDropPercent),
61
+ maxSourceAgeHours: positiveNumber(params.feedGuardMaxSourceAgeHours, MEINSTADT_DEFAULTS.maxSourceAgeHours),
62
+ maxStaleHours: positiveNumber(params.feedGuardMaxStaleHours, MEINSTADT_DEFAULTS.maxStaleHours),
63
+ acceptDrop: text(params.feedGuardAcceptDrop).toLowerCase() === 'true',
64
+ };
10
65
  this.outputFiles = [
11
66
  `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.MEINSTADT}.${FileType.JSON}`,
12
67
  `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.MEINSTADT}.${FileType.ZIP}`,
@@ -1,4 +1,3 @@
1
1
  export const MIN_MILEAGES = 0;
2
- export const DEFAULT_ZIP = '';
3
- export const DEFAULT_CITY = '';
4
- export const DEFAULT_ADDR1 = '';
2
+ // seller_region when the postal code table does not know the dealer's place
3
+ export const DEFAULT_REGION = 'Deutschland';
@@ -1,16 +1,23 @@
1
- import { get } from 'lodash';
2
-
3
1
  import { Injectable } from '@nestjs/common';
4
2
  import { ModuleRef } from '@nestjs/core';
5
3
 
6
- import { CronicleUtils } from '@bytebrand/cronicle-helper';
7
4
  import { FsChunkInterface } from '@bytebrand/chunkable';
5
+ import { LoggerService } from '@bytebrand/cronicle-helper';
8
6
 
9
- import { AutouncleTransformer } from '../transformers/autouncle.transformer';
10
- import { AutouncleJsonBuilder } from '../formats/autouncle-json.builder';
11
- import { CarService, RulesService, LoaderService, ConfigService } from '../services';
7
+ import { AutouncleTransformer, AutouncleAddress, toAutouncleImages } from '../transformers/autouncle.transformer';
8
+ import { resolveOpenAiImageFields } from '../transformers/openai-image-fields';
9
+ import { CarService, RulesService, LoaderService, ConfigService, ImagesService } from '../services';
10
+ import { ImageUrlValidator } from '../services/image-url-validator.service';
11
+ import { FeedGuard } from '../services/feed-guard';
12
+ import { AutouncleFeedRun, AutouncleSourceCar } from '../services/autouncle-feed-run';
13
+ import { AutouncleConfig } from '../configs/autouncle.config';
12
14
  import { Exporter } from '../interfaces/exporter.interface';
13
15
 
16
+ /**
17
+ * ExportType.AUTOUNCLE (ADV-322/323). Wires the Cronicle/Nest services into AutouncleFeedRun, which
18
+ * holds the logic. The output directory is no longer emptied: the new JSON replaces the old one only
19
+ * after the fail-closed guard passed.
20
+ */
14
21
  @Injectable()
15
22
  export class AutouncleExporter implements Exporter {
16
23
  constructor(
@@ -18,43 +25,58 @@ export class AutouncleExporter implements Exporter {
18
25
  private configService: ConfigService,
19
26
  private loaderService: LoaderService,
20
27
  private rulesService: RulesService,
28
+ private imagesService: ImagesService,
21
29
  private moduleRef: ModuleRef,
22
30
  ) {}
23
31
 
24
32
  async export(): Promise<void> {
25
- const [fileName] = this.configService.config.outputFiles;
26
- await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
27
- await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
28
-
29
- const autouncleJsonBuilder = new AutouncleJsonBuilder(fileName);
30
- await autouncleJsonBuilder.buildStart();
31
-
32
- const autouncleTransformer = await this.moduleRef.create(AutouncleTransformer);
33
-
34
- let firstCar = true;
35
- await this.loaderService.process(async (chunk: FsChunkInterface) => {
36
- chunk.data = chunk.data.filter(e => this.carService.filterCar(e, this.rulesService.rules));
33
+ const config = this.configService.config as AutouncleConfig;
34
+ const [jsonFile] = config.outputFiles;
35
+ const validator = new ImageUrlValidator({ concurrency: config.imageConcurrency });
36
+ const transformer = await this.moduleRef.create(AutouncleTransformer);
37
+ const addresses = new Map<string, Promise<AutouncleAddress | null>>();
37
38
 
38
- for (const data of chunk.data) {
39
- const ownerId = get(data.car, 'metaData.ownerId');
40
- // NOTES:: it must be temporary solution
41
- const owner = await this.carService.getOwner(ownerId).catch((error: any) => { console.error(error); return null; });
42
- if (!owner) { console.log(`Skip car because owner ${ownerId} is not a dealer`); continue; }
43
- // console.time(`Transform ${data.car._id}`);
44
- const car = await autouncleTransformer.transform({ owner, ...data });
45
- // console.timeEnd(`Transform ${data.car._id}`);
39
+ const run = new AutouncleFeedRun(
40
+ {
41
+ forEachChunk: fn => this.loaderService.process((chunk: FsChunkInterface) => fn(chunk.data as AutouncleSourceCar[])),
42
+ filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
43
+ dealerAddress: (ownerId) => {
44
+ if (!addresses.has(ownerId)) {
45
+ addresses.set(ownerId, this.carService.getOwner(ownerId)
46
+ .then(owner => (owner ? transformer.getAddress(owner) : null))
47
+ .catch((error: any) => { console.error(error?.message ?? error); return null; }));
48
+ }
49
+ return addresses.get(ownerId);
50
+ },
51
+ resolveImages: async ({ car, images }) => toAutouncleImages(
52
+ await resolveOpenAiImageFields(this.imagesService.getCleanImages(car._id, images), validator),
53
+ config.maxImages,
54
+ ),
55
+ imageStats: () => validator.stats,
56
+ translate: key => transformer.translate(key),
57
+ guard: new FeedGuard(config.feedGuard),
58
+ },
59
+ {
60
+ runId: config.runId,
61
+ dryRun: config.dryRun,
62
+ domain: config.domain,
63
+ sourceDirectory: config.pathesConfig.localDirectory,
64
+ sourceBaseFileName: config.pathesConfig.baseFileName,
65
+ jsonFile,
66
+ stateDir: config.stateDir,
67
+ },
68
+ );
46
69
 
47
- await autouncleJsonBuilder.addItem(car, firstCar);
48
-
49
- this.carService.counter.countCar(data.images);
50
-
51
- if (firstCar) {
52
- firstCar = false;
53
- }
70
+ try {
71
+ const report = await run.run();
72
+ if (report.stale) {
73
+ console.log(`autouncle feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
54
74
  }
55
- });
56
-
57
- await autouncleJsonBuilder.buildEnd();
58
- console.log(this.carService.counter.getCountedCars());
75
+ } catch (error) {
76
+ // main.ts would report a rejected run as {"complete":1,"code":0}; fail the job so the chained
77
+ // `3.1 … part 2` copy step does not run (report + lock already handled by the run).
78
+ LoggerService.exitError(`autouncle feed not published: ${error?.message ?? error}`);
79
+ process.exit(1);
80
+ }
59
81
  }
60
82
  }
@@ -1,16 +1,23 @@
1
1
  import { Injectable } from '@nestjs/common';
2
2
  import { ModuleRef } from '@nestjs/core';
3
3
 
4
- import { CronicleUtils } from '@bytebrand/cronicle-helper';
5
4
  import { FsChunkInterface } from '@bytebrand/chunkable';
6
- import { Zipper } from '@bytebrand/zipper';
5
+ import { LoggerService } from '@bytebrand/cronicle-helper';
7
6
 
8
- import { CarService, RulesService, LoaderService, ConfigService } from '../services';
9
- import { MeinstadtTransformer } from '../transformers/meinstadt.transformer';
10
- import { MeinstadtJsonBuilder } from '../formats/meinstadt-json.builder';
11
- import { MeinstadtJsonItem } from '../interfaces';
7
+ import { CarService, RulesService, LoaderService, ConfigService, ImagesService } from '../services';
8
+ import { ImageUrlValidator } from '../services/image-url-validator.service';
9
+ import { FeedGuard } from '../services/feed-guard';
10
+ import { MeinstadtFeedRun, MeinstadtSourceCar, zipSingleFile } from '../services/meinstadt-feed-run';
11
+ import { MeinstadtTransformer, toMeinstadtImages } from '../transformers/meinstadt.transformer';
12
+ import { resolveOpenAiImageFields } from '../transformers/openai-image-fields';
13
+ import { MeinstadtConfig } from '../configs/meinstadt.config';
12
14
  import { Exporter } from '../interfaces/exporter.interface';
13
15
 
16
+ /**
17
+ * ExportType.MEINSTADT (ADV-325/326). Wires the Cronicle/Nest services into MeinstadtFeedRun, which
18
+ * holds the logic. The output directory is no longer emptied: the new JSON/ZIP replace the old ones
19
+ * only after the fail-closed guard passed.
20
+ */
14
21
  @Injectable()
15
22
  export class MeinstadtExporter implements Exporter {
16
23
  constructor(
@@ -18,38 +25,52 @@ export class MeinstadtExporter implements Exporter {
18
25
  private configService: ConfigService,
19
26
  private rulesService: RulesService,
20
27
  private loaderService: LoaderService,
28
+ private imagesService: ImagesService,
21
29
  private moduleRef: ModuleRef,
22
30
  ) {}
23
31
 
24
32
  async export(): Promise<void> {
25
- const [fileJson, fileGZ] = this.configService.config.outputFiles;
26
- await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
27
- await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
33
+ const config = this.configService.config as MeinstadtConfig;
34
+ const [jsonFile, zipFile] = config.outputFiles;
35
+ const validator = new ImageUrlValidator({ concurrency: config.imageConcurrency });
36
+ const transformer = await this.moduleRef.create(MeinstadtTransformer);
28
37
 
29
- const meinstadtJsonBuilder = new MeinstadtJsonBuilder(fileJson);
30
- await meinstadtJsonBuilder.buildStart();
38
+ const run = new MeinstadtFeedRun(
39
+ {
40
+ forEachChunk: fn => this.loaderService.process((chunk: FsChunkInterface) => fn(chunk.data as MeinstadtSourceCar[])),
41
+ filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
42
+ resolveImages: async ({ car, images }) => toMeinstadtImages(
43
+ await resolveOpenAiImageFields(this.imagesService.getCleanImages(car._id, images), validator),
44
+ config.maxImages,
45
+ ),
46
+ imageStats: () => validator.stats,
47
+ equipment: car => transformer.getDescription(car),
48
+ zip: zipSingleFile,
49
+ guard: new FeedGuard(config.feedGuard),
50
+ },
51
+ {
52
+ runId: config.runId,
53
+ dryRun: config.dryRun,
54
+ cpc: config.cpc,
55
+ domain: config.domain,
56
+ sourceDirectory: config.pathesConfig.localDirectory,
57
+ sourceBaseFileName: config.pathesConfig.baseFileName,
58
+ jsonFile,
59
+ zipFile,
60
+ stateDir: config.stateDir,
61
+ },
62
+ );
31
63
 
32
- const meinstadtTransformer = await this.moduleRef.create(MeinstadtTransformer);
33
-
34
- let firstCar = true;
35
- await this.loaderService.process(async (chunk: FsChunkInterface) => {
36
- chunk.data = chunk.data.filter(e => this.carService.filterCar(e, this.rulesService.rules));
37
-
38
- for (const data of chunk.data) {
39
- // console.time(`Transform ${data.car._id}`);
40
- const car = await meinstadtTransformer.transform(data);
41
- // console.timeEnd(`Transform ${data.car._id}`);
42
- await meinstadtJsonBuilder.addItem(car as MeinstadtJsonItem, firstCar);
43
- this.carService.counter.countCar(data.images);
44
-
45
- if (firstCar) {
46
- firstCar = false;
47
- }
64
+ try {
65
+ const report = await run.run();
66
+ if (report.stale) {
67
+ console.log(`meinestadt feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
48
68
  }
49
- });
50
-
51
- await meinstadtJsonBuilder.buildEnd();
52
- await Zipper.zip({ dest: fileGZ, fileList: [fileJson] });
53
- console.log(this.carService.counter.getCountedCars());
69
+ } catch (error) {
70
+ // main.ts would report a rejected run as {"complete":1,"code":0}; fail the job so the chained
71
+ // upload step does not run (report + lock already handled by the run).
72
+ LoggerService.exitError(`meinestadt feed not published: ${error?.message ?? error}`);
73
+ process.exit(1);
74
+ }
54
75
  }
55
76
  }
@@ -6,11 +6,9 @@ export class AutouncleJsonBuilder extends XMLBuilderBase {
6
6
  await this.write('{"ads":[');
7
7
  }
8
8
 
9
+ // firstSym: the first item goes without a leading comma (it was written twice before)
9
10
  async addItem(car: AutouncleJsonItem, firstSym?: boolean): Promise<void> {
10
- if (firstSym) {
11
- await this.write(`${JSON.stringify(car)}`);
12
- }
13
- await this.write(`,${JSON.stringify(car)}`);
11
+ await this.write(`${firstSym ? '' : ','}${JSON.stringify(car)}`);
14
12
  }
15
13
 
16
14
  async buildEnd(): Promise<void> {
@@ -67,23 +67,27 @@ export interface OoyyoXmlItem extends FbXmlItem {
67
67
  mileage: string;
68
68
  }
69
69
 
70
+ // Keys and order as the meinestadt exporter emits them (see docs/meinestadt/AUDIT.md §3).
70
71
  export interface MeinstadtJsonItem {
71
72
  id: string;
72
73
  make: string;
73
74
  model: string;
74
- cpc: string;
75
+ // configured tariff; null only in dry-run without one
76
+ cpc: string | null;
75
77
  listingUrl: string;
76
78
  title: string;
77
- accidentFree: boolean;
79
+ // null = unknown, never "accident-free"
80
+ accidentFree: boolean | null;
78
81
  mileage: number;
79
- initialRegistration: string;
80
- offerCreationDate: string;
82
+ // Unix seconds, null when missing/invalid
83
+ initialRegistration: number | null;
84
+ offerCreationDate: number | null;
81
85
  numberOfDoors: number;
82
86
  numberOfSeats: number;
83
87
  numberOfPreviousOwners: number;
84
88
  price: number;
85
- powerPS: string;
86
- powerKS: string;
89
+ // PS
90
+ power: number;
87
91
  co2Emissions: number;
88
92
  fuelConsumptionCombined: number;
89
93
  equipment: string;
@@ -92,14 +96,13 @@ export interface MeinstadtJsonItem {
92
96
  emissionClass: string;
93
97
  emissionSticker: string;
94
98
  fuelType: string;
95
- transmissionType: string;
96
- sellerType: string;
97
- usageState: string;
98
- imagesCount: number;
99
- images: string;
100
- power: number;
101
99
  'transmissionType/transmission': string;
100
+ // = images.length
101
+ imagesCount: number;
102
102
  'customerType/sellerType': string;
103
+ usageState: string;
104
+ // validated public image URLs, hero first
105
+ images: string[];
103
106
  }
104
107
 
105
108
  export interface AutouncleJsonItem {
@@ -53,6 +53,8 @@ export enum OpenAiAdsExclusion {
53
53
  dealerInactive = 'dealer_inactive',
54
54
  dealerUnknown = 'dealer_unknown',
55
55
  noImage = 'no_image',
56
+ // finance wording left after stripping (rates, Zins, Leasing, mtl., …) — ADV-335/336 launch gate
57
+ financeClaim = 'finance_claim',
56
58
  }
57
59
 
58
60
  // Why an item is sent as out_of_stock.
@@ -38,6 +38,16 @@ export interface Params {
38
38
  feedGuardMaxStaleHours?: string;
39
39
  // "true" for ONE run after a real stock drop the guard blocked
40
40
  feedGuardAcceptDrop?: string;
41
+ // meinestadt feed only (ADV-325/326), Cronicle text params; see configs/meinstadt.config.ts.
42
+ // Live output needs meinstadtDryRun=false AND a cpc explicitly.
43
+ meinstadtDryRun?: string;
44
+ meinstadtImageConcurrency?: string;
45
+ meinstadtMaxImages?: string;
46
+ // AutoUncle feed only (ADV-322/323), Cronicle text params; see configs/autouncle.config.ts.
47
+ // Live output needs autouncleDryRun=false explicitly.
48
+ autouncleDryRun?: string;
49
+ autouncleImageConcurrency?: string;
50
+ autouncleMaxImages?: string;
41
51
  }
42
52
 
43
53
  // Parsed from feedItemsBq* by GoogleConfig.
@@ -0,0 +1,197 @@
1
+ import { promises as fs } from 'fs';
2
+ import { dirname, join } from 'path';
3
+ import { ImboImageResponseInterface } from '@bytebrand/chunkable';
4
+ import { DataSchemes } from '@bytebrand/car-schema';
5
+
6
+ import { AutouncleJsonBuilder } from '../formats/autouncle-json.builder';
7
+ import { AutouncleAddress, AutouncleExclusion, getAutouncleExclusion, mapAutouncleItem } from '../transformers/autouncle.transformer';
8
+ import { FeedGuard, FeedGuardError, LastGood, inspectSourceSnapshot } from './feed-guard';
9
+
10
+ export interface AutouncleSourceCar {
11
+ car: DataSchemes.Car;
12
+ images: ImboImageResponseInterface[];
13
+ }
14
+
15
+ // Everything the run needs from the outside world — the Nest exporter wires the real services,
16
+ // tests pass fakes.
17
+ export interface AutouncleRunDeps {
18
+ // iterates the base export snapshot chunk by chunk (LoaderService.process)
19
+ forEachChunk: (fn: (cars: AutouncleSourceCar[]) => Promise<void>) => Promise<unknown>;
20
+ // existing shared partner filter (images count, Cronicle rules incl. the dealer allow-list)
21
+ filterCar: (source: AutouncleSourceCar) => boolean;
22
+ // dealer address of the car's owner; null = owner is not a dealer / not found
23
+ dealerAddress: (ownerId: string) => Promise<AutouncleAddress | null>;
24
+ // final validated image URLs for the car, hero first; [] = no usable image
25
+ resolveImages: (source: AutouncleSourceCar) => Promise<string[]>;
26
+ imageStats: () => Record<string, number>;
27
+ translate: (key: string) => string;
28
+ guard: FeedGuard;
29
+ }
30
+
31
+ export interface AutouncleRunOptions {
32
+ runId?: string;
33
+ dryRun: boolean;
34
+ domain: string;
35
+ sourceDirectory: string;
36
+ sourceBaseFileName: string;
37
+ jsonFile: string;
38
+ stateDir: string;
39
+ now?: () => Date;
40
+ }
41
+
42
+ export interface AutouncleRunReport {
43
+ runId?: string;
44
+ mode: 'dry-run' | 'live';
45
+ startedAt: string;
46
+ finishedAt?: string;
47
+ status: 'published' | 'blocked' | 'failed';
48
+ guard?: { code: string; message: string };
49
+ error?: string;
50
+ jsonFile: string;
51
+ source: { files: number; newestChunkAt: string | null; cars: number };
52
+ items: number;
53
+ dealers: number;
54
+ excluded: Record<string, number>;
55
+ imagesPerItem: { min: number | null; max: number | null; total: number };
56
+ images: Record<string, number>;
57
+ lastGood: LastGood | null;
58
+ lastGoodAgeHours: number | null;
59
+ stale: boolean;
60
+ }
61
+
62
+ const increment = (counter: Record<string, number>, key: string) => {
63
+ counter[key] = (counter[key] ?? 0) + 1;
64
+ };
65
+
66
+ /**
67
+ * One AutoUncle feed run (ADV-322/323): source check -> map + validated images -> temp JSON ->
68
+ * count check -> atomic publish -> report. Replaces the old flow that emptied the output directory
69
+ * first, so an outage published an empty or partial file. Nothing outside the output directory and
70
+ * `stateDir` is written; a blocked run leaves the last good JSON in place.
71
+ */
72
+ export class AutouncleFeedRun {
73
+ constructor(private readonly deps: AutouncleRunDeps, private readonly options: AutouncleRunOptions) { }
74
+
75
+ private now(): Date {
76
+ return this.options.now ? this.options.now() : new Date();
77
+ }
78
+
79
+ async run(): Promise<AutouncleRunReport> {
80
+ const { guard } = this.deps;
81
+ const { jsonFile, runId, dryRun } = this.options;
82
+ const startedAt = this.now();
83
+ const lastGood = await guard.readLastGood();
84
+ const report: AutouncleRunReport = {
85
+ runId,
86
+ mode: dryRun ? 'dry-run' : 'live',
87
+ startedAt: startedAt.toISOString(),
88
+ status: 'failed',
89
+ jsonFile,
90
+ source: { files: 0, newestChunkAt: null, cars: 0 },
91
+ items: 0,
92
+ dealers: 0,
93
+ excluded: {},
94
+ imagesPerItem: { min: null, max: null, total: 0 },
95
+ images: {},
96
+ lastGood,
97
+ lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
98
+ stale: guard.isStale(lastGood, startedAt),
99
+ };
100
+
101
+ await guard.acquireLock(runId ?? 'local', startedAt);
102
+ const tmpFile = `${jsonFile}.tmp`;
103
+ try {
104
+ const snapshot = await inspectSourceSnapshot(this.options.sourceDirectory, this.options.sourceBaseFileName);
105
+ report.source.files = snapshot.files;
106
+ report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
107
+ guard.checkSource(snapshot, startedAt);
108
+
109
+ await fs.mkdir(dirname(jsonFile), { recursive: true });
110
+ // the builder appends — never onto a leftover temp file
111
+ await fs.rm(tmpFile, { force: true });
112
+ const builder = new AutouncleJsonBuilder(tmpFile);
113
+ await builder.buildStart();
114
+
115
+ const seen = new Set<string>();
116
+ const dealers = new Set<string>();
117
+ let first = true;
118
+ await this.deps.forEachChunk(async (cars) => {
119
+ for (const source of cars) {
120
+ report.source.cars += 1;
121
+ const id = String(source?.car?._id ?? '');
122
+ if (id && seen.has(id)) {
123
+ increment(report.excluded, AutouncleExclusion.duplicateId);
124
+ continue;
125
+ }
126
+ seen.add(id);
127
+ if (!this.deps.filterCar(source)) {
128
+ increment(report.excluded, AutouncleExclusion.filtered);
129
+ continue;
130
+ }
131
+ // cheap checks first: no image probe for a car that is excluded anyway
132
+ const excluded = getAutouncleExclusion(source.car);
133
+ if (excluded) {
134
+ increment(report.excluded, excluded);
135
+ continue;
136
+ }
137
+ const ownerId = String(source.car.metaData.ownerId).trim();
138
+ const address = await this.deps.dealerAddress(ownerId);
139
+ if (!address) {
140
+ increment(report.excluded, AutouncleExclusion.noDealer);
141
+ continue;
142
+ }
143
+ const images = await this.deps.resolveImages(source);
144
+ const mapped = mapAutouncleItem(source.car, { domain: this.options.domain, address, images, translate: this.deps.translate });
145
+ if (mapped.ok === false) {
146
+ increment(report.excluded, mapped.reason);
147
+ continue;
148
+ }
149
+ await builder.addItem(mapped.item, first);
150
+ first = false;
151
+ dealers.add(ownerId);
152
+ this.count(report, mapped.item.images.length);
153
+ }
154
+ });
155
+ await builder.buildEnd();
156
+ report.dealers = dealers.size;
157
+ report.images = this.deps.imageStats();
158
+
159
+ const counts = { sourceItems: report.source.cars, publishedItems: report.items };
160
+ guard.checkCounts(counts, lastGood);
161
+
162
+ report.lastGood = await guard.publish(tmpFile, jsonFile, counts, this.now(), runId);
163
+ report.status = 'published';
164
+ report.lastGoodAgeHours = 0;
165
+ report.stale = false;
166
+ return report;
167
+ } catch (error) {
168
+ if (error instanceof FeedGuardError) {
169
+ report.status = 'blocked';
170
+ report.guard = { code: error.code, message: error.message };
171
+ } else {
172
+ report.error = error?.message ?? String(error);
173
+ }
174
+ // keep the rejected file for inspection; the published file stays untouched
175
+ await fs.rename(tmpFile, `${jsonFile}.rejected`).catch(() => undefined);
176
+ throw error;
177
+ } finally {
178
+ await this.writeReport(report);
179
+ await guard.releaseLock();
180
+ }
181
+ }
182
+
183
+ private count(report: AutouncleRunReport, images: number): void {
184
+ report.items += 1;
185
+ const stats = report.imagesPerItem;
186
+ stats.min = stats.min === null ? images : Math.min(stats.min, images);
187
+ stats.max = stats.max === null ? images : Math.max(stats.max, images);
188
+ stats.total += images;
189
+ }
190
+
191
+ private async writeReport(report: AutouncleRunReport): Promise<void> {
192
+ report.finishedAt = this.now().toISOString();
193
+ await fs.mkdir(this.options.stateDir, { recursive: true });
194
+ await fs.writeFile(join(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
195
+ console.log(`autouncle feed report: ${JSON.stringify(report)}`);
196
+ }
197
+ }