@bytebrand/export-cars 1.26.28 → 1.26.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +100 -0
  2. package/dist/configs/index.d.ts +1 -0
  3. package/dist/configs/index.js +1 -0
  4. package/dist/configs/index.js.map +1 -1
  5. package/dist/configs/openai-ads.config.d.ts +22 -0
  6. package/dist/configs/openai-ads.config.js +56 -0
  7. package/dist/configs/openai-ads.config.js.map +1 -0
  8. package/dist/enums/export-type.enum.d.ts +2 -1
  9. package/dist/enums/export-type.enum.js +1 -0
  10. package/dist/enums/export-type.enum.js.map +1 -1
  11. package/dist/enums/image-type.enum.d.ts +2 -1
  12. package/dist/enums/image-type.enum.js +1 -0
  13. package/dist/enums/image-type.enum.js.map +1 -1
  14. package/dist/export.schema.d.ts +6 -2
  15. package/dist/export.schema.js +4 -0
  16. package/dist/export.schema.js.map +1 -1
  17. package/dist/exporters/index.d.ts +1 -0
  18. package/dist/exporters/index.js +1 -0
  19. package/dist/exporters/index.js.map +1 -1
  20. package/dist/exporters/openai-ads.exporter.d.ts +13 -0
  21. package/dist/exporters/openai-ads.exporter.js +91 -0
  22. package/dist/exporters/openai-ads.exporter.js.map +1 -0
  23. package/dist/formats/openai-ads-csv.builder.d.ts +15 -0
  24. package/dist/formats/openai-ads-csv.builder.js +48 -0
  25. package/dist/formats/openai-ads-csv.builder.js.map +1 -0
  26. package/dist/interfaces/openai-ads.interface.d.ts +35 -0
  27. package/dist/interfaces/openai-ads.interface.js +37 -0
  28. package/dist/interfaces/openai-ads.interface.js.map +1 -0
  29. package/dist/interfaces/params.interface.d.ts +10 -0
  30. package/dist/services/feed-guard.d.ts +44 -0
  31. package/dist/services/feed-guard.js +156 -0
  32. package/dist/services/feed-guard.js.map +1 -0
  33. package/dist/services/image-url-validator.service.d.ts +45 -0
  34. package/dist/services/image-url-validator.service.js +141 -0
  35. package/dist/services/image-url-validator.service.js.map +1 -0
  36. package/dist/services/images.service.d.ts +12 -0
  37. package/dist/services/images.service.js +46 -1
  38. package/dist/services/images.service.js.map +1 -1
  39. package/dist/services/index.d.ts +1 -0
  40. package/dist/services/index.js +1 -0
  41. package/dist/services/index.js.map +1 -1
  42. package/dist/services/openai-ads-feed-run.d.ts +74 -0
  43. package/dist/services/openai-ads-feed-run.js +151 -0
  44. package/dist/services/openai-ads-feed-run.js.map +1 -0
  45. package/dist/services/openai-ads-ledger.d.ts +19 -0
  46. package/dist/services/openai-ads-ledger.js +72 -0
  47. package/dist/services/openai-ads-ledger.js.map +1 -0
  48. package/dist/services/publication-gate.js +8 -8
  49. package/dist/transformers/openai-ads.transformer.d.ts +29 -0
  50. package/dist/transformers/openai-ads.transformer.js +166 -0
  51. package/dist/transformers/openai-ads.transformer.js.map +1 -0
  52. package/dist/transformers/openai-image-fields.d.ts +16 -0
  53. package/dist/transformers/openai-image-fields.js +34 -0
  54. package/dist/transformers/openai-image-fields.js.map +1 -0
  55. package/dist/tsconfig.build.tsbuildinfo +1 -1
  56. package/docs/openai-ads/AUDIT.md +235 -0
  57. package/docs/openai-ads/RUNBOOK.md +94 -0
  58. package/package.json +64 -64
  59. package/schema/feed-items.json +134 -134
  60. package/src/configs/config.ts +99 -99
  61. package/src/configs/index.ts +1 -0
  62. package/src/configs/openai-ads.config.ts +70 -0
  63. package/src/enums/export-type.enum.ts +2 -0
  64. package/src/enums/image-type.enum.ts +5 -0
  65. package/src/export.schema.ts +6 -0
  66. package/src/exporters/index.ts +1 -0
  67. package/src/exporters/openai-ads.exporter.ts +93 -0
  68. package/src/formats/openai-ads-csv.builder.ts +55 -0
  69. package/src/interfaces/openai-ads.interface.ts +64 -0
  70. package/src/interfaces/params.interface.ts +13 -0
  71. package/src/services/feed-guard.ts +219 -0
  72. package/src/services/image-url-validator.service.ts +181 -0
  73. package/src/services/images.service.ts +72 -0
  74. package/src/services/index.ts +1 -0
  75. package/src/services/openai-ads-feed-run.ts +226 -0
  76. package/src/services/openai-ads-ledger.ts +116 -0
  77. package/src/services/publication-gate.ts +119 -119
  78. package/src/transformers/openai-ads.transformer.ts +243 -0
  79. package/src/transformers/openai-image-fields.ts +60 -0
  80. package/test/clean-images.test.ts +86 -0
  81. package/test/fixtures/images-service.harness.ts +32 -0
  82. package/test/fixtures/images.golden.json +218 -0
  83. package/test/fixtures/imbo-images.fixture.ts +77 -0
  84. package/test/fixtures/openai-ads.fixture.ts +89 -0
  85. package/test/image-url-validator.test.ts +110 -0
  86. package/test/images-regression.test.ts +23 -0
  87. package/test/openai-ads-feed-run.test.ts +278 -0
  88. package/test/openai-ads-mapping.test.ts +157 -0
  89. package/test/openai-image-fields.test.ts +99 -0
  90. package/test/publication-gate.test.ts +86 -86
@@ -0,0 +1,93 @@
1
+ import { Injectable } from '@nestjs/common';
2
+ import { FsChunkInterface } from '@bytebrand/chunkable';
3
+ import { LoggerService } from '@bytebrand/cronicle-helper';
4
+ import { TranslatorExtended } from '@bytebrand/schema-tools';
5
+ import { de, deVehicleDetailsPage, deVehicleProps } from '@bytebrand/i18n-dictionaries';
6
+
7
+ import { CarService, ConfigService, ImagesService, LoaderService, RulesService } from '../services';
8
+ import { ImageUrlValidator } from '../services/image-url-validator.service';
9
+ import { FeedGuard } from '../services/feed-guard';
10
+ import { DealerStatus, OpenAiAdsFeedRun, SourceCar } from '../services/openai-ads-feed-run';
11
+ import { resolveOpenAiImageFields } from '../transformers/openai-image-fields';
12
+ import { OpenAiAdsConfig } from '../configs/openai-ads.config';
13
+ import { Exporter } from '../interfaces/exporter.interface';
14
+
15
+ /**
16
+ * ExportType.OPENAI_ADS (ADV-335). Wires the Cronicle/Nest services into OpenAiAdsFeedRun, which
17
+ * holds the logic. Unlike the other exporters it never empties its output directory: the new file
18
+ * replaces the old one only after the fail-closed guard passed.
19
+ */
20
+ @Injectable()
21
+ export class OpenAiAdsExporter implements Exporter {
22
+ constructor(
23
+ private carService: CarService,
24
+ private configService: ConfigService,
25
+ private rulesService: RulesService,
26
+ private loaderService: LoaderService,
27
+ private imagesService: ImagesService,
28
+ ) { }
29
+
30
+ async export(): Promise<void> {
31
+ const config = this.configService.config as OpenAiAdsConfig;
32
+ const translator = new TranslatorExtended({ ...de, ...deVehicleProps, ...deVehicleDetailsPage });
33
+ const validator = new ImageUrlValidator({ concurrency: config.imageConcurrency });
34
+
35
+ const run = new OpenAiAdsFeedRun(
36
+ {
37
+ forEachChunk: fn => this.loaderService.process((chunk: FsChunkInterface) => fn(chunk.data as SourceCar[])),
38
+ filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
39
+ dealerStatus: ownerId => this.dealerStatus(ownerId),
40
+ resolveImages: ({ car, images }) =>
41
+ resolveOpenAiImageFields(this.imagesService.getCleanImages(car._id, images), validator),
42
+ imageStats: () => validator.stats,
43
+ guard: new FeedGuard(config.feedGuard),
44
+ },
45
+ {
46
+ runId: config.runId,
47
+ dryRun: config.dryRun,
48
+ sourceDirectory: config.pathesConfig.localDirectory,
49
+ sourceBaseFileName: config.pathesConfig.baseFileName,
50
+ outputFile: config.outputFiles[0],
51
+ stateDir: config.stateDir,
52
+ retentionDays: config.retentionDays,
53
+ mapping: {
54
+ domain: config.domain,
55
+ sellerName: config.sellerName,
56
+ translate: key => translator.translateVal(key),
57
+ },
58
+ },
59
+ );
60
+
61
+ try {
62
+ const report = await run.run();
63
+ if (report.stale) {
64
+ console.log(`OpenAI ads feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
65
+ }
66
+ } catch (error) {
67
+ // nest-commander's default serviceErrorHandler only prints a rejected run, and main.ts then
68
+ // reports {"complete":1,"code":0} — a blocked feed would look green in Cronicle. Fail the job
69
+ // here (report + lock already handled by the run).
70
+ LoggerService.exitError(`OpenAI ads feed not published: ${error?.message ?? error}`);
71
+ process.exit(1);
72
+ }
73
+ }
74
+
75
+ // One UPA lookup per dealer and run. CarService.getOwner retries 4× and does not remember
76
+ // failures, so without this map every car of an unknown dealer would wait for the retries again.
77
+ private readonly dealerStatuses = new Map<string, Promise<DealerStatus>>();
78
+
79
+ // CBD already takes blacklisted dealers' cars offline, but a deactivated dealer's cars stay
80
+ // online — not advertised here. A missing `active` field counts as active (only explicit false).
81
+ private dealerStatus(ownerId: string): Promise<DealerStatus> {
82
+ if (!ownerId) {
83
+ return Promise.resolve('unknown');
84
+ }
85
+ if (!this.dealerStatuses.has(ownerId)) {
86
+ this.dealerStatuses.set(ownerId, this.carService.getOwner(ownerId).then(
87
+ owner => (!owner ? 'unknown' : owner.active === false || owner.blacklist === true ? 'inactive' : 'active'),
88
+ () => 'unknown' as DealerStatus,
89
+ ));
90
+ }
91
+ return this.dealerStatuses.get(ownerId);
92
+ }
93
+ }
@@ -0,0 +1,55 @@
1
+ import { createWriteStream, WriteStream } from 'fs';
2
+ import { once } from 'events';
3
+
4
+ import { OPENAI_ADS_COLUMNS, OpenAiAdsRow } from '../interfaces/openai-ads.interface';
5
+
6
+ // RFC 4180 cell: quoted only when it contains a comma, a quote, CR or LF (the OpenAI product file
7
+ // rule); embedded quotes are doubled. Values are written as they are — callers keep free text on
8
+ // one line, but a stray newline still stays inside its quoted cell instead of breaking the row.
9
+ export const toCsvCell = (value: string | boolean | undefined | null): string => {
10
+ if (value === undefined || value === null) {
11
+ return '';
12
+ }
13
+ const text = typeof value === 'boolean' ? (value ? 'true' : 'false') : value;
14
+ return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
15
+ };
16
+
17
+ export const toCsvLine = (cells: Array<string | boolean | undefined | null>): string => `${cells.map(toCsvCell).join(',')}\n`;
18
+
19
+ export const openAiAdsCsvHeader = (): string => toCsvLine([...OPENAI_ADS_COLUMNS]);
20
+
21
+ export const openAiAdsCsvRow = (row: OpenAiAdsRow): string => toCsvLine(OPENAI_ADS_COLUMNS.map(column => row[column]));
22
+
23
+ /**
24
+ * OpenAI product file (OpenAI format, not the Google-compatible profile): UTF-8 without BOM, one
25
+ * header row, one vehicle per row, fixed column order (OPENAI_ADS_COLUMNS). Not built on
26
+ * XMLBuilderBase and not shared with the Awin/Santander writers, which do not escape quotes.
27
+ */
28
+ export class OpenAiAdsCsvBuilder {
29
+ private stream: WriteStream;
30
+ private rows = 0;
31
+
32
+ constructor(private readonly fileName: string) { }
33
+
34
+ async buildStart(): Promise<void> {
35
+ this.stream = createWriteStream(this.fileName, { encoding: 'utf8' });
36
+ await this.write(openAiAdsCsvHeader());
37
+ }
38
+
39
+ async addItem(row: OpenAiAdsRow): Promise<void> {
40
+ await this.write(openAiAdsCsvRow(row));
41
+ this.rows += 1;
42
+ }
43
+
44
+ async buildEnd(): Promise<number> {
45
+ this.stream.end();
46
+ await once(this.stream, 'finish');
47
+ return this.rows;
48
+ }
49
+
50
+ private async write(chunk: string): Promise<void> {
51
+ if (!this.stream.write(chunk)) {
52
+ await once(this.stream, 'drain');
53
+ }
54
+ }
55
+ }
@@ -0,0 +1,64 @@
1
+ // OpenAI product file, OpenAI format (developers.openai.com/commerce/specs/file-upload/products,
2
+ // read 2026-10-09). Only the columns below are sent: no gtin/mpn/VIN (never invented), no
3
+ // target_countries/store_country (market comes from the account setup), no Google-compatible names.
4
+ export const OPENAI_ADS_COLUMNS = [
5
+ 'item_id',
6
+ 'title',
7
+ 'description',
8
+ 'url',
9
+ 'brand',
10
+ 'seller_name',
11
+ 'image_url',
12
+ 'additional_image_urls',
13
+ 'availability',
14
+ 'price',
15
+ 'condition',
16
+ 'is_ads_eligible',
17
+ ] as const;
18
+
19
+ export type OpenAiAdsColumn = typeof OPENAI_ADS_COLUMNS[number];
20
+
21
+ export type OpenAiAdsAvailability = 'in_stock' | 'out_of_stock';
22
+
23
+ export type OpenAiAdsCondition = 'new' | 'used';
24
+
25
+ export interface OpenAiAdsRow {
26
+ // canonical auto.de vehicle id (car._id, the VDP id) — same value as GA4 item_id and the
27
+ // Pixel/CAPI contents[].id of ADV-337
28
+ item_id: string;
29
+ title: string;
30
+ description: string;
31
+ url: string;
32
+ brand: string;
33
+ seller_name: string;
34
+ image_url: string;
35
+ // one CSV cell, comma-separated, commas inside URLs as %2C; '' = none
36
+ additional_image_urls: string;
37
+ availability: OpenAiAdsAvailability;
38
+ // "17990.00 EUR": purchase price in major units — never a monthly rate
39
+ price: string;
40
+ condition: OpenAiAdsCondition;
41
+ is_ads_eligible: boolean;
42
+ }
43
+
44
+ // Why a car of the source snapshot is not in the file as in_stock. Counted in the run report.
45
+ export enum OpenAiAdsExclusion {
46
+ noId = 'no_id',
47
+ duplicateId = 'duplicate_id',
48
+ filtered = 'filtered',
49
+ invalidPrice = 'invalid_price',
50
+ noTitle = 'no_title',
51
+ noBrand = 'no_brand',
52
+ conditionUnknown = 'condition_unknown',
53
+ dealerInactive = 'dealer_inactive',
54
+ dealerUnknown = 'dealer_unknown',
55
+ noImage = 'no_image',
56
+ }
57
+
58
+ // Why an item is sent as out_of_stock.
59
+ export enum OpenAiAdsOutOfStockReason {
60
+ reserved = 'reserved',
61
+ soldFlag = 'sold_flag',
62
+ // sent before, not in_stock in this run (sold/offline/deleted/now excluded)
63
+ notInSnapshot = 'not_in_snapshot',
64
+ }
@@ -25,6 +25,19 @@ export interface Params {
25
25
  publicationGateBqProject?: string;
26
26
  publicationGateBqDataset?: string;
27
27
  publicationGateBqTable?: string;
28
+ // OpenAI Ads feed only (ADV-335). Cronicle text params, so all arrive as strings; see
29
+ // configs/openai-ads.config.ts for defaults. Live output needs openAiAdsDryRun=false explicitly.
30
+ openAiAdsDryRun?: string;
31
+ openAiAdsSellerName?: string;
32
+ openAiAdsRetentionDays?: string;
33
+ openAiAdsImageConcurrency?: string;
34
+ feedGuardMinSourceItems?: string;
35
+ feedGuardMinPublishedItems?: string;
36
+ feedGuardMaxDropPercent?: string;
37
+ feedGuardMaxSourceAgeHours?: string;
38
+ feedGuardMaxStaleHours?: string;
39
+ // "true" for ONE run after a real stock drop the guard blocked
40
+ feedGuardAcceptDrop?: string;
28
41
  }
29
42
 
30
43
  // Parsed from feedItemsBq* by GoogleConfig.
@@ -0,0 +1,219 @@
1
+ import { promises as fs } from 'fs';
2
+ import { basename, dirname, join } from 'path';
3
+
4
+ /**
5
+ * Shared fail-closed guard for partner feed exporters (ADV-335). Opt-in per exporter; today only
6
+ * the OpenAI exporter uses it. Built for the 24.09 failure: base export wiped resources/cars-export,
7
+ * failed on the CBD outage, and every downstream feed was published empty.
8
+ *
9
+ * - checkSource: before any output is touched — the base export snapshot must exist, be complete
10
+ * (chunk files cars.0..n-1 without gaps) and be fresh enough.
11
+ * - checkCounts: after the new file was written to a temp path — source and published counts must
12
+ * not be below an absolute minimum or drop by more than maxDropPercent against the last good run.
13
+ * A real stock drop is accepted by re-running once with acceptDrop.
14
+ * - publish: temp file -> final path by rename (same directory, atomic), previous file kept as
15
+ * `<name>.prev`, last good counts written to the state dir. A blocked run leaves the last good
16
+ * file in place, so the partner keeps reading it; how long that is acceptable is maxStaleHours.
17
+ * - lock: one run per state dir.
18
+ */
19
+ export interface FeedGuardOptions {
20
+ // state dir of this feed (last-good.json, lock); one per export type and mode
21
+ stateDir: string;
22
+ // fail when the source snapshot has fewer cars than this
23
+ minSourceItems: number;
24
+ // fail when the published in_stock count is below this
25
+ minPublishedItems: number;
26
+ // fail when source or published count is more than this % below the last good run
27
+ maxDropPercent: number;
28
+ // fail when the newest base export chunk is older than this
29
+ maxSourceAgeHours: number;
30
+ // the last good file is reported STALE after this many hours without a successful run
31
+ maxStaleHours: number;
32
+ // one-off override for a real stock drop (Cronicle param); does not override the minimums
33
+ acceptDrop: boolean;
34
+ // a lock file older than this is treated as left over by a crashed run
35
+ lockStaleHours?: number;
36
+ }
37
+
38
+ export interface SourceSnapshot {
39
+ files: number;
40
+ missingChunks: number[];
41
+ newestMtime: Date | null;
42
+ oldestMtime: Date | null;
43
+ }
44
+
45
+ export interface FeedCounts {
46
+ sourceItems: number;
47
+ publishedItems: number;
48
+ }
49
+
50
+ export interface LastGood extends FeedCounts {
51
+ publishedAt: string;
52
+ file: string;
53
+ runId?: string;
54
+ }
55
+
56
+ export class FeedGuardError extends Error {
57
+ constructor(readonly code: string, message: string) {
58
+ super(`[feed-guard:${code}] ${message}`);
59
+ }
60
+ }
61
+
62
+ const HOUR_MS = 60 * 60 * 1000;
63
+
64
+ // Reads the base export snapshot: `<baseFileName>.<n>.json` chunk files (FsCarsLoader order).
65
+ export const inspectSourceSnapshot = async (directory: string, baseFileName: string): Promise<SourceSnapshot> => {
66
+ let names: string[];
67
+ try {
68
+ names = await fs.readdir(directory);
69
+ } catch (error) {
70
+ if (error?.code === 'ENOENT') {
71
+ return { files: 0, missingChunks: [], newestMtime: null, oldestMtime: null };
72
+ }
73
+ throw error;
74
+ }
75
+ const pattern = new RegExp(`^${baseFileName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.(\\d+)\\.json$`);
76
+ const chunks = names.map(name => ({ name, match: pattern.exec(name) })).filter(entry => entry.match);
77
+ const ids = new Set(chunks.map(entry => Number(entry.match[1])));
78
+ const maxId = ids.size ? Math.max(...ids) : -1;
79
+ const missingChunks: number[] = [];
80
+ for (let id = 0; id <= maxId; id += 1) {
81
+ if (!ids.has(id)) {
82
+ missingChunks.push(id);
83
+ }
84
+ }
85
+ let newest: number = null;
86
+ let oldest: number = null;
87
+ for (const { name } of chunks) {
88
+ const { mtimeMs } = await fs.stat(join(directory, name));
89
+ newest = newest === null ? mtimeMs : Math.max(newest, mtimeMs);
90
+ oldest = oldest === null ? mtimeMs : Math.min(oldest, mtimeMs);
91
+ }
92
+ return {
93
+ files: chunks.length,
94
+ missingChunks,
95
+ newestMtime: newest === null ? null : new Date(newest),
96
+ oldestMtime: oldest === null ? null : new Date(oldest),
97
+ };
98
+ };
99
+
100
+ const dropPercent = (current: number, previous: number): number =>
101
+ previous > 0 ? ((previous - current) / previous) * 100 : 0;
102
+
103
+ export class FeedGuard {
104
+ private lockFile: string;
105
+
106
+ constructor(private readonly options: FeedGuardOptions) { }
107
+
108
+ private get lastGoodFile(): string {
109
+ return join(this.options.stateDir, 'last-good.json');
110
+ }
111
+
112
+ async readLastGood(): Promise<LastGood | null> {
113
+ try {
114
+ return JSON.parse(await fs.readFile(this.lastGoodFile, 'utf8')) as LastGood;
115
+ } catch (error) {
116
+ if (error?.code === 'ENOENT') {
117
+ return null;
118
+ }
119
+ throw error;
120
+ }
121
+ }
122
+
123
+ async acquireLock(runId: string, now: Date): Promise<void> {
124
+ await fs.mkdir(this.options.stateDir, { recursive: true });
125
+ const lockFile = join(this.options.stateDir, 'run.lock');
126
+ const staleMs = (this.options.lockStaleHours ?? 6) * HOUR_MS;
127
+ try {
128
+ await fs.writeFile(lockFile, JSON.stringify({ runId, startedAt: now.toISOString() }), { flag: 'wx' });
129
+ } catch (error) {
130
+ if (error?.code !== 'EEXIST') {
131
+ throw error;
132
+ }
133
+ const { mtimeMs } = await fs.stat(lockFile);
134
+ if (now.getTime() - mtimeMs < staleMs) {
135
+ throw new FeedGuardError('locked', `another run holds ${lockFile} (since ${new Date(mtimeMs).toISOString()})`);
136
+ }
137
+ console.log(`feed-guard: removing stale lock ${lockFile} (${new Date(mtimeMs).toISOString()})`);
138
+ await fs.writeFile(lockFile, JSON.stringify({ runId, startedAt: now.toISOString() }));
139
+ }
140
+ this.lockFile = lockFile;
141
+ }
142
+
143
+ async releaseLock(): Promise<void> {
144
+ if (this.lockFile) {
145
+ await fs.rm(this.lockFile, { force: true });
146
+ this.lockFile = null;
147
+ }
148
+ }
149
+
150
+ checkSource(snapshot: SourceSnapshot, now: Date): void {
151
+ if (!snapshot.files) {
152
+ throw new FeedGuardError('source_empty', 'base export snapshot has no chunk files');
153
+ }
154
+ if (snapshot.missingChunks.length) {
155
+ throw new FeedGuardError('source_partial', `base export snapshot misses chunk(s) ${snapshot.missingChunks.slice(0, 10).join(', ')}`);
156
+ }
157
+ const ageHours = (now.getTime() - snapshot.newestMtime.getTime()) / HOUR_MS;
158
+ if (ageHours > this.options.maxSourceAgeHours) {
159
+ throw new FeedGuardError(
160
+ 'source_stale',
161
+ `newest base export chunk is ${ageHours.toFixed(1)} h old (max ${this.options.maxSourceAgeHours} h)`,
162
+ );
163
+ }
164
+ }
165
+
166
+ checkCounts(counts: FeedCounts, lastGood: LastGood | null): void {
167
+ if (counts.sourceItems < this.options.minSourceItems) {
168
+ throw new FeedGuardError('source_too_small', `${counts.sourceItems} source cars < minimum ${this.options.minSourceItems}`);
169
+ }
170
+ if (counts.publishedItems < this.options.minPublishedItems) {
171
+ throw new FeedGuardError(
172
+ 'published_too_small',
173
+ `${counts.publishedItems} in_stock items < minimum ${this.options.minPublishedItems}`,
174
+ );
175
+ }
176
+ if (!lastGood || this.options.acceptDrop) {
177
+ return;
178
+ }
179
+ const sourceDrop = dropPercent(counts.sourceItems, lastGood.sourceItems);
180
+ const publishedDrop = dropPercent(counts.publishedItems, lastGood.publishedItems);
181
+ if (sourceDrop > this.options.maxDropPercent || publishedDrop > this.options.maxDropPercent) {
182
+ throw new FeedGuardError(
183
+ 'count_drop',
184
+ `source ${lastGood.sourceItems} -> ${counts.sourceItems} (-${sourceDrop.toFixed(1)}%), ` +
185
+ `in_stock ${lastGood.publishedItems} -> ${counts.publishedItems} (-${publishedDrop.toFixed(1)}%), ` +
186
+ `max ${this.options.maxDropPercent}% — re-run with acceptDrop if the stock really dropped`,
187
+ );
188
+ }
189
+ }
190
+
191
+ staleHours(lastGood: LastGood | null, now: Date): number | null {
192
+ return lastGood ? (now.getTime() - Date.parse(lastGood.publishedAt)) / HOUR_MS : null;
193
+ }
194
+
195
+ isStale(lastGood: LastGood | null, now: Date): boolean {
196
+ const hours = this.staleHours(lastGood, now);
197
+ return hours !== null && hours > this.options.maxStaleHours;
198
+ }
199
+
200
+ // tmpFile must sit next to finalFile so the rename stays on one filesystem (atomic).
201
+ async publish(tmpFile: string, finalFile: string, counts: FeedCounts, now: Date, runId?: string): Promise<LastGood> {
202
+ if (dirname(tmpFile) !== dirname(finalFile)) {
203
+ throw new Error(`feed-guard: ${basename(tmpFile)} must be in the directory of ${finalFile}`);
204
+ }
205
+ try {
206
+ await fs.copyFile(finalFile, `${finalFile}.prev`);
207
+ } catch (error) {
208
+ if (error?.code !== 'ENOENT') {
209
+ throw error;
210
+ }
211
+ }
212
+ await fs.rename(tmpFile, finalFile);
213
+ const lastGood: LastGood = { ...counts, publishedAt: now.toISOString(), file: finalFile, runId };
214
+ await fs.mkdir(this.options.stateDir, { recursive: true });
215
+ await fs.writeFile(`${this.lastGoodFile}.tmp`, JSON.stringify(lastGood, null, 2), 'utf8');
216
+ await fs.rename(`${this.lastGoodFile}.tmp`, this.lastGoodFile);
217
+ return lastGood;
218
+ }
219
+ }
@@ -0,0 +1,181 @@
1
+ import axios, { AxiosResponse } from 'axios';
2
+ import Bottleneck from 'bottleneck';
3
+
4
+ export enum ImageUrlRejectReason {
5
+ invalidUrl = 'invalid_url',
6
+ notHttps = 'not_https',
7
+ credentials = 'credentials_in_url',
8
+ placeholder = 'placeholder',
9
+ httpStatus = 'http_status',
10
+ wrongMime = 'wrong_mime',
11
+ tooManyRedirects = 'too_many_redirects',
12
+ timeout = 'timeout',
13
+ networkError = 'network_error',
14
+ budgetExhausted = 'budget_exhausted',
15
+ }
16
+
17
+ export interface ImageUrlCheck {
18
+ url: string;
19
+ ok: boolean;
20
+ reason?: ImageUrlRejectReason;
21
+ status?: number;
22
+ contentType?: string;
23
+ finalUrl?: string;
24
+ }
25
+
26
+ export interface ImageUrlValidatorOptions {
27
+ // MIME types a partner is known to accept. OpenAI documents JPEG/PNG only as examples, so WebP
28
+ // (which our CDN negotiates for browsers) is rejected unless a partner explicitly allows it.
29
+ allowedMimeTypes?: string[];
30
+ requireHttps?: boolean;
31
+ timeoutMs?: number;
32
+ maxRedirects?: number;
33
+ concurrency?: number;
34
+ // Upper bound of HTTP probes per validator instance (one export run). Unlimited by default.
35
+ maxProbes?: number;
36
+ // e.g. the partner crawler's UA, to see what the crawler would get
37
+ userAgent?: string;
38
+ }
39
+
40
+ // auto.de "no image" assets (ImagesService.DEFAULT_IMAGES) — never a valid vehicle photo, and
41
+ // images.auto.de answers them with 404 + an image/jpeg body anyway.
42
+ const PLACEHOLDER_PATTERN = /\/carimage\/noimage\//i;
43
+
44
+ /**
45
+ * Budgeted, cached HTTP check of image URLs before they go into a partner feed (ADV-336).
46
+ * HEAD first, GET (body discarded) when HEAD is not supported. A URL is valid only with a final
47
+ * 200 and an allowed image MIME type — images.auto.de answers unknown images with 404 AND an
48
+ * image/jpeg placeholder body, so the Content-Type alone proves nothing. One instance = one run:
49
+ * each URL is probed at most once, results and per-reason counters are kept for the run log.
50
+ */
51
+ export class ImageUrlValidator {
52
+ private readonly options: Required<ImageUrlValidatorOptions>;
53
+ private readonly limiter: Bottleneck;
54
+ private readonly cache = new Map<string, Promise<ImageUrlCheck>>();
55
+ private readonly counters: Record<string, number> = {};
56
+ private probes = 0;
57
+
58
+ constructor(options: ImageUrlValidatorOptions = {}) {
59
+ this.options = {
60
+ allowedMimeTypes: ['image/jpeg', 'image/png'],
61
+ requireHttps: true,
62
+ timeoutMs: 5000,
63
+ maxRedirects: 3,
64
+ concurrency: 8,
65
+ maxProbes: Infinity,
66
+ userAgent: undefined,
67
+ ...options,
68
+ };
69
+ this.limiter = new Bottleneck({ maxConcurrent: this.options.concurrency });
70
+ }
71
+
72
+ validate(url: string): Promise<ImageUrlCheck> {
73
+ if (!this.cache.has(url)) {
74
+ this.cache.set(url, this.check(url).then((result) => {
75
+ const key = result.ok ? 'ok' : result.reason;
76
+ this.counters[key] = (this.counters[key] ?? 0) + 1;
77
+ return result;
78
+ }));
79
+ }
80
+ return this.cache.get(url);
81
+ }
82
+
83
+ validateAll(urls: string[]): Promise<ImageUrlCheck[]> {
84
+ return Promise.all(urls.map(url => this.validate(url)));
85
+ }
86
+
87
+ // Per-reason counts of the distinct URLs checked so far (cache hits are not counted twice).
88
+ get stats(): Record<string, number> {
89
+ return { ...this.counters };
90
+ }
91
+
92
+ private async check(url: string): Promise<ImageUrlCheck> {
93
+ const rejected = this.checkStatically(url);
94
+ if (rejected) {
95
+ return rejected;
96
+ }
97
+ if (this.probes >= this.options.maxProbes) {
98
+ return { url, ok: false, reason: ImageUrlRejectReason.budgetExhausted };
99
+ }
100
+ this.probes++;
101
+
102
+ try {
103
+ let response = await this.limiter.schedule(() => this.request('HEAD', url));
104
+ if (response.status === 405 || response.status === 501) {
105
+ response = await this.limiter.schedule(() => this.request('GET', url));
106
+ }
107
+ return this.checkResponse(url, response);
108
+ } catch (error) {
109
+ return { url, ok: false, reason: this.mapError(error) };
110
+ }
111
+ }
112
+
113
+ private checkStatically(url: string): ImageUrlCheck | null {
114
+ let parsed: URL;
115
+ try {
116
+ parsed = new URL(url);
117
+ } catch {
118
+ return { url, ok: false, reason: ImageUrlRejectReason.invalidUrl };
119
+ }
120
+ if (!['http:', 'https:'].includes(parsed.protocol)) {
121
+ return { url, ok: false, reason: ImageUrlRejectReason.invalidUrl };
122
+ }
123
+ if (this.options.requireHttps && parsed.protocol !== 'https:') {
124
+ return { url, ok: false, reason: ImageUrlRejectReason.notHttps };
125
+ }
126
+ if (parsed.username || parsed.password) {
127
+ return { url, ok: false, reason: ImageUrlRejectReason.credentials };
128
+ }
129
+ if (PLACEHOLDER_PATTERN.test(parsed.pathname)) {
130
+ return { url, ok: false, reason: ImageUrlRejectReason.placeholder };
131
+ }
132
+ return null;
133
+ }
134
+
135
+ private request(method: 'HEAD' | 'GET', url: string): Promise<AxiosResponse> {
136
+ return axios.request({
137
+ method,
138
+ url,
139
+ timeout: this.options.timeoutMs,
140
+ maxRedirects: this.options.maxRedirects,
141
+ // body is never needed — the stream is destroyed right after the headers arrive
142
+ responseType: 'stream',
143
+ headers: this.options.userAgent ? { 'User-Agent': this.options.userAgent } : undefined,
144
+ validateStatus: () => true,
145
+ }).then((response) => {
146
+ response.data?.destroy?.();
147
+ return response;
148
+ });
149
+ }
150
+
151
+ private checkResponse(url: string, response: AxiosResponse): ImageUrlCheck {
152
+ const status = response.status;
153
+ const contentType = String(response.headers?.['content-type'] ?? '').split(';')[0].trim().toLowerCase();
154
+ const finalUrl: string = response.request?.res?.responseUrl ?? url;
155
+ const result = { url, status, contentType, finalUrl };
156
+
157
+ if (status !== 200) {
158
+ return { ...result, ok: false, reason: ImageUrlRejectReason.httpStatus };
159
+ }
160
+ if (this.options.requireHttps && !finalUrl.startsWith('https:')) {
161
+ return { ...result, ok: false, reason: ImageUrlRejectReason.notHttps };
162
+ }
163
+ if (PLACEHOLDER_PATTERN.test(finalUrl)) {
164
+ return { ...result, ok: false, reason: ImageUrlRejectReason.placeholder };
165
+ }
166
+ if (!this.options.allowedMimeTypes.includes(contentType)) {
167
+ return { ...result, ok: false, reason: ImageUrlRejectReason.wrongMime };
168
+ }
169
+ return { ...result, ok: true };
170
+ }
171
+
172
+ private mapError(error: any): ImageUrlRejectReason {
173
+ if (error?.code === 'ERR_FR_TOO_MANY_REDIRECTS') {
174
+ return ImageUrlRejectReason.tooManyRedirects;
175
+ }
176
+ if (error?.code === 'ECONNABORTED' || error?.code === 'ETIMEDOUT' || /timeout/i.test(error?.message ?? '')) {
177
+ return ImageUrlRejectReason.timeout;
178
+ }
179
+ return ImageUrlRejectReason.networkError;
180
+ }
181
+ }