@bytebrand/export-cars 1.26.28 → 1.26.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -0
- package/dist/configs/index.d.ts +1 -0
- package/dist/configs/index.js +1 -0
- package/dist/configs/index.js.map +1 -1
- package/dist/configs/openai-ads.config.d.ts +22 -0
- package/dist/configs/openai-ads.config.js +56 -0
- package/dist/configs/openai-ads.config.js.map +1 -0
- package/dist/enums/export-type.enum.d.ts +2 -1
- package/dist/enums/export-type.enum.js +1 -0
- package/dist/enums/export-type.enum.js.map +1 -1
- package/dist/enums/image-type.enum.d.ts +2 -1
- package/dist/enums/image-type.enum.js +1 -0
- package/dist/enums/image-type.enum.js.map +1 -1
- package/dist/export.schema.d.ts +6 -2
- package/dist/export.schema.js +4 -0
- package/dist/export.schema.js.map +1 -1
- package/dist/exporters/index.d.ts +1 -0
- package/dist/exporters/index.js +1 -0
- package/dist/exporters/index.js.map +1 -1
- package/dist/exporters/openai-ads.exporter.d.ts +13 -0
- package/dist/exporters/openai-ads.exporter.js +91 -0
- package/dist/exporters/openai-ads.exporter.js.map +1 -0
- package/dist/formats/openai-ads-csv.builder.d.ts +15 -0
- package/dist/formats/openai-ads-csv.builder.js +48 -0
- package/dist/formats/openai-ads-csv.builder.js.map +1 -0
- package/dist/interfaces/openai-ads.interface.d.ts +35 -0
- package/dist/interfaces/openai-ads.interface.js +37 -0
- package/dist/interfaces/openai-ads.interface.js.map +1 -0
- package/dist/interfaces/params.interface.d.ts +10 -0
- package/dist/services/feed-guard.d.ts +44 -0
- package/dist/services/feed-guard.js +156 -0
- package/dist/services/feed-guard.js.map +1 -0
- package/dist/services/image-url-validator.service.d.ts +45 -0
- package/dist/services/image-url-validator.service.js +141 -0
- package/dist/services/image-url-validator.service.js.map +1 -0
- package/dist/services/images.service.d.ts +12 -0
- package/dist/services/images.service.js +46 -1
- package/dist/services/images.service.js.map +1 -1
- package/dist/services/index.d.ts +1 -0
- package/dist/services/index.js +1 -0
- package/dist/services/index.js.map +1 -1
- package/dist/services/openai-ads-feed-run.d.ts +74 -0
- package/dist/services/openai-ads-feed-run.js +151 -0
- package/dist/services/openai-ads-feed-run.js.map +1 -0
- package/dist/services/openai-ads-ledger.d.ts +19 -0
- package/dist/services/openai-ads-ledger.js +72 -0
- package/dist/services/openai-ads-ledger.js.map +1 -0
- package/dist/services/publication-gate.js +8 -8
- package/dist/transformers/openai-ads.transformer.d.ts +29 -0
- package/dist/transformers/openai-ads.transformer.js +166 -0
- package/dist/transformers/openai-ads.transformer.js.map +1 -0
- package/dist/transformers/openai-image-fields.d.ts +16 -0
- package/dist/transformers/openai-image-fields.js +34 -0
- package/dist/transformers/openai-image-fields.js.map +1 -0
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/openai-ads/AUDIT.md +235 -0
- package/docs/openai-ads/RUNBOOK.md +94 -0
- package/package.json +64 -64
- package/schema/feed-items.json +134 -134
- package/src/configs/config.ts +99 -99
- package/src/configs/index.ts +1 -0
- package/src/configs/openai-ads.config.ts +70 -0
- package/src/enums/export-type.enum.ts +2 -0
- package/src/enums/image-type.enum.ts +5 -0
- package/src/export.schema.ts +6 -0
- package/src/exporters/index.ts +1 -0
- package/src/exporters/openai-ads.exporter.ts +93 -0
- package/src/formats/openai-ads-csv.builder.ts +55 -0
- package/src/interfaces/openai-ads.interface.ts +64 -0
- package/src/interfaces/params.interface.ts +13 -0
- package/src/services/feed-guard.ts +219 -0
- package/src/services/image-url-validator.service.ts +181 -0
- package/src/services/images.service.ts +72 -0
- package/src/services/index.ts +1 -0
- package/src/services/openai-ads-feed-run.ts +226 -0
- package/src/services/openai-ads-ledger.ts +116 -0
- package/src/services/publication-gate.ts +119 -119
- package/src/transformers/openai-ads.transformer.ts +243 -0
- package/src/transformers/openai-image-fields.ts +60 -0
- package/test/clean-images.test.ts +86 -0
- package/test/fixtures/images-service.harness.ts +32 -0
- package/test/fixtures/images.golden.json +218 -0
- package/test/fixtures/imbo-images.fixture.ts +77 -0
- package/test/fixtures/openai-ads.fixture.ts +89 -0
- package/test/image-url-validator.test.ts +110 -0
- package/test/images-regression.test.ts +23 -0
- package/test/openai-ads-feed-run.test.ts +278 -0
- package/test/openai-ads-mapping.test.ts +157 -0
- package/test/openai-image-fields.test.ts +99 -0
- package/test/publication-gate.test.ts +86 -86
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { Injectable } from '@nestjs/common';
|
|
2
|
+
import { FsChunkInterface } from '@bytebrand/chunkable';
|
|
3
|
+
import { LoggerService } from '@bytebrand/cronicle-helper';
|
|
4
|
+
import { TranslatorExtended } from '@bytebrand/schema-tools';
|
|
5
|
+
import { de, deVehicleDetailsPage, deVehicleProps } from '@bytebrand/i18n-dictionaries';
|
|
6
|
+
|
|
7
|
+
import { CarService, ConfigService, ImagesService, LoaderService, RulesService } from '../services';
|
|
8
|
+
import { ImageUrlValidator } from '../services/image-url-validator.service';
|
|
9
|
+
import { FeedGuard } from '../services/feed-guard';
|
|
10
|
+
import { DealerStatus, OpenAiAdsFeedRun, SourceCar } from '../services/openai-ads-feed-run';
|
|
11
|
+
import { resolveOpenAiImageFields } from '../transformers/openai-image-fields';
|
|
12
|
+
import { OpenAiAdsConfig } from '../configs/openai-ads.config';
|
|
13
|
+
import { Exporter } from '../interfaces/exporter.interface';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* ExportType.OPENAI_ADS (ADV-335). Wires the Cronicle/Nest services into OpenAiAdsFeedRun, which
|
|
17
|
+
* holds the logic. Unlike the other exporters it never empties its output directory: the new file
|
|
18
|
+
* replaces the old one only after the fail-closed guard passed.
|
|
19
|
+
*/
|
|
20
|
+
@Injectable()
|
|
21
|
+
export class OpenAiAdsExporter implements Exporter {
|
|
22
|
+
constructor(
|
|
23
|
+
private carService: CarService,
|
|
24
|
+
private configService: ConfigService,
|
|
25
|
+
private rulesService: RulesService,
|
|
26
|
+
private loaderService: LoaderService,
|
|
27
|
+
private imagesService: ImagesService,
|
|
28
|
+
) { }
|
|
29
|
+
|
|
30
|
+
async export(): Promise<void> {
|
|
31
|
+
const config = this.configService.config as OpenAiAdsConfig;
|
|
32
|
+
const translator = new TranslatorExtended({ ...de, ...deVehicleProps, ...deVehicleDetailsPage });
|
|
33
|
+
const validator = new ImageUrlValidator({ concurrency: config.imageConcurrency });
|
|
34
|
+
|
|
35
|
+
const run = new OpenAiAdsFeedRun(
|
|
36
|
+
{
|
|
37
|
+
forEachChunk: fn => this.loaderService.process((chunk: FsChunkInterface) => fn(chunk.data as SourceCar[])),
|
|
38
|
+
filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
|
|
39
|
+
dealerStatus: ownerId => this.dealerStatus(ownerId),
|
|
40
|
+
resolveImages: ({ car, images }) =>
|
|
41
|
+
resolveOpenAiImageFields(this.imagesService.getCleanImages(car._id, images), validator),
|
|
42
|
+
imageStats: () => validator.stats,
|
|
43
|
+
guard: new FeedGuard(config.feedGuard),
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
runId: config.runId,
|
|
47
|
+
dryRun: config.dryRun,
|
|
48
|
+
sourceDirectory: config.pathesConfig.localDirectory,
|
|
49
|
+
sourceBaseFileName: config.pathesConfig.baseFileName,
|
|
50
|
+
outputFile: config.outputFiles[0],
|
|
51
|
+
stateDir: config.stateDir,
|
|
52
|
+
retentionDays: config.retentionDays,
|
|
53
|
+
mapping: {
|
|
54
|
+
domain: config.domain,
|
|
55
|
+
sellerName: config.sellerName,
|
|
56
|
+
translate: key => translator.translateVal(key),
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
try {
|
|
62
|
+
const report = await run.run();
|
|
63
|
+
if (report.stale) {
|
|
64
|
+
console.log(`OpenAI ads feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
|
|
65
|
+
}
|
|
66
|
+
} catch (error) {
|
|
67
|
+
// nest-commander's default serviceErrorHandler only prints a rejected run, and main.ts then
|
|
68
|
+
// reports {"complete":1,"code":0} — a blocked feed would look green in Cronicle. Fail the job
|
|
69
|
+
// here (report + lock already handled by the run).
|
|
70
|
+
LoggerService.exitError(`OpenAI ads feed not published: ${error?.message ?? error}`);
|
|
71
|
+
process.exit(1);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// One UPA lookup per dealer and run. CarService.getOwner retries 4× and does not remember
|
|
76
|
+
// failures, so without this map every car of an unknown dealer would wait for the retries again.
|
|
77
|
+
private readonly dealerStatuses = new Map<string, Promise<DealerStatus>>();
|
|
78
|
+
|
|
79
|
+
// CBD already takes blacklisted dealers' cars offline, but a deactivated dealer's cars stay
|
|
80
|
+
// online — not advertised here. A missing `active` field counts as active (only explicit false).
|
|
81
|
+
private dealerStatus(ownerId: string): Promise<DealerStatus> {
|
|
82
|
+
if (!ownerId) {
|
|
83
|
+
return Promise.resolve('unknown');
|
|
84
|
+
}
|
|
85
|
+
if (!this.dealerStatuses.has(ownerId)) {
|
|
86
|
+
this.dealerStatuses.set(ownerId, this.carService.getOwner(ownerId).then(
|
|
87
|
+
owner => (!owner ? 'unknown' : owner.active === false || owner.blacklist === true ? 'inactive' : 'active'),
|
|
88
|
+
() => 'unknown' as DealerStatus,
|
|
89
|
+
));
|
|
90
|
+
}
|
|
91
|
+
return this.dealerStatuses.get(ownerId);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { createWriteStream, WriteStream } from 'fs';
|
|
2
|
+
import { once } from 'events';
|
|
3
|
+
|
|
4
|
+
import { OPENAI_ADS_COLUMNS, OpenAiAdsRow } from '../interfaces/openai-ads.interface';
|
|
5
|
+
|
|
6
|
+
// RFC 4180 cell: quoted only when it contains a comma, a quote, CR or LF (the OpenAI product file
|
|
7
|
+
// rule); embedded quotes are doubled. Values are written as they are — callers keep free text on
|
|
8
|
+
// one line, but a stray newline still stays inside its quoted cell instead of breaking the row.
|
|
9
|
+
export const toCsvCell = (value: string | boolean | undefined | null): string => {
|
|
10
|
+
if (value === undefined || value === null) {
|
|
11
|
+
return '';
|
|
12
|
+
}
|
|
13
|
+
const text = typeof value === 'boolean' ? (value ? 'true' : 'false') : value;
|
|
14
|
+
return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
export const toCsvLine = (cells: Array<string | boolean | undefined | null>): string => `${cells.map(toCsvCell).join(',')}\n`;
|
|
18
|
+
|
|
19
|
+
export const openAiAdsCsvHeader = (): string => toCsvLine([...OPENAI_ADS_COLUMNS]);
|
|
20
|
+
|
|
21
|
+
export const openAiAdsCsvRow = (row: OpenAiAdsRow): string => toCsvLine(OPENAI_ADS_COLUMNS.map(column => row[column]));
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* OpenAI product file (OpenAI format, not the Google-compatible profile): UTF-8 without BOM, one
|
|
25
|
+
* header row, one vehicle per row, fixed column order (OPENAI_ADS_COLUMNS). Not built on
|
|
26
|
+
* XMLBuilderBase and not shared with the Awin/Santander writers, which do not escape quotes.
|
|
27
|
+
*/
|
|
28
|
+
export class OpenAiAdsCsvBuilder {
|
|
29
|
+
private stream: WriteStream;
|
|
30
|
+
private rows = 0;
|
|
31
|
+
|
|
32
|
+
constructor(private readonly fileName: string) { }
|
|
33
|
+
|
|
34
|
+
async buildStart(): Promise<void> {
|
|
35
|
+
this.stream = createWriteStream(this.fileName, { encoding: 'utf8' });
|
|
36
|
+
await this.write(openAiAdsCsvHeader());
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async addItem(row: OpenAiAdsRow): Promise<void> {
|
|
40
|
+
await this.write(openAiAdsCsvRow(row));
|
|
41
|
+
this.rows += 1;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async buildEnd(): Promise<number> {
|
|
45
|
+
this.stream.end();
|
|
46
|
+
await once(this.stream, 'finish');
|
|
47
|
+
return this.rows;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
private async write(chunk: string): Promise<void> {
|
|
51
|
+
if (!this.stream.write(chunk)) {
|
|
52
|
+
await once(this.stream, 'drain');
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// OpenAI product file, OpenAI format (developers.openai.com/commerce/specs/file-upload/products,
|
|
2
|
+
// read 2026-10-09). Only the columns below are sent: no gtin/mpn/VIN (never invented), no
|
|
3
|
+
// target_countries/store_country (market comes from the account setup), no Google-compatible names.
|
|
4
|
+
export const OPENAI_ADS_COLUMNS = [
|
|
5
|
+
'item_id',
|
|
6
|
+
'title',
|
|
7
|
+
'description',
|
|
8
|
+
'url',
|
|
9
|
+
'brand',
|
|
10
|
+
'seller_name',
|
|
11
|
+
'image_url',
|
|
12
|
+
'additional_image_urls',
|
|
13
|
+
'availability',
|
|
14
|
+
'price',
|
|
15
|
+
'condition',
|
|
16
|
+
'is_ads_eligible',
|
|
17
|
+
] as const;
|
|
18
|
+
|
|
19
|
+
export type OpenAiAdsColumn = typeof OPENAI_ADS_COLUMNS[number];
|
|
20
|
+
|
|
21
|
+
export type OpenAiAdsAvailability = 'in_stock' | 'out_of_stock';
|
|
22
|
+
|
|
23
|
+
export type OpenAiAdsCondition = 'new' | 'used';
|
|
24
|
+
|
|
25
|
+
export interface OpenAiAdsRow {
|
|
26
|
+
// canonical auto.de vehicle id (car._id, the VDP id) — same value as GA4 item_id and the
|
|
27
|
+
// Pixel/CAPI contents[].id of ADV-337
|
|
28
|
+
item_id: string;
|
|
29
|
+
title: string;
|
|
30
|
+
description: string;
|
|
31
|
+
url: string;
|
|
32
|
+
brand: string;
|
|
33
|
+
seller_name: string;
|
|
34
|
+
image_url: string;
|
|
35
|
+
// one CSV cell, comma-separated, commas inside URLs as %2C; '' = none
|
|
36
|
+
additional_image_urls: string;
|
|
37
|
+
availability: OpenAiAdsAvailability;
|
|
38
|
+
// "17990.00 EUR": purchase price in major units — never a monthly rate
|
|
39
|
+
price: string;
|
|
40
|
+
condition: OpenAiAdsCondition;
|
|
41
|
+
is_ads_eligible: boolean;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Why a car of the source snapshot is not in the file as in_stock. Counted in the run report.
|
|
45
|
+
export enum OpenAiAdsExclusion {
|
|
46
|
+
noId = 'no_id',
|
|
47
|
+
duplicateId = 'duplicate_id',
|
|
48
|
+
filtered = 'filtered',
|
|
49
|
+
invalidPrice = 'invalid_price',
|
|
50
|
+
noTitle = 'no_title',
|
|
51
|
+
noBrand = 'no_brand',
|
|
52
|
+
conditionUnknown = 'condition_unknown',
|
|
53
|
+
dealerInactive = 'dealer_inactive',
|
|
54
|
+
dealerUnknown = 'dealer_unknown',
|
|
55
|
+
noImage = 'no_image',
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Why an item is sent as out_of_stock.
|
|
59
|
+
export enum OpenAiAdsOutOfStockReason {
|
|
60
|
+
reserved = 'reserved',
|
|
61
|
+
soldFlag = 'sold_flag',
|
|
62
|
+
// sent before, not in_stock in this run (sold/offline/deleted/now excluded)
|
|
63
|
+
notInSnapshot = 'not_in_snapshot',
|
|
64
|
+
}
|
|
@@ -25,6 +25,19 @@ export interface Params {
|
|
|
25
25
|
publicationGateBqProject?: string;
|
|
26
26
|
publicationGateBqDataset?: string;
|
|
27
27
|
publicationGateBqTable?: string;
|
|
28
|
+
// OpenAI Ads feed only (ADV-335). Cronicle text params, so all arrive as strings; see
|
|
29
|
+
// configs/openai-ads.config.ts for defaults. Live output needs openAiAdsDryRun=false explicitly.
|
|
30
|
+
openAiAdsDryRun?: string;
|
|
31
|
+
openAiAdsSellerName?: string;
|
|
32
|
+
openAiAdsRetentionDays?: string;
|
|
33
|
+
openAiAdsImageConcurrency?: string;
|
|
34
|
+
feedGuardMinSourceItems?: string;
|
|
35
|
+
feedGuardMinPublishedItems?: string;
|
|
36
|
+
feedGuardMaxDropPercent?: string;
|
|
37
|
+
feedGuardMaxSourceAgeHours?: string;
|
|
38
|
+
feedGuardMaxStaleHours?: string;
|
|
39
|
+
// "true" for ONE run after a real stock drop the guard blocked
|
|
40
|
+
feedGuardAcceptDrop?: string;
|
|
28
41
|
}
|
|
29
42
|
|
|
30
43
|
// Parsed from feedItemsBq* by GoogleConfig.
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import { promises as fs } from 'fs';
|
|
2
|
+
import { basename, dirname, join } from 'path';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Shared fail-closed guard for partner feed exporters (ADV-335). Opt-in per exporter; today only
|
|
6
|
+
* the OpenAI exporter uses it. Built for the 24.09 failure: base export wiped resources/cars-export,
|
|
7
|
+
* failed on the CBD outage, and every downstream feed was published empty.
|
|
8
|
+
*
|
|
9
|
+
* - checkSource: before any output is touched — the base export snapshot must exist, be complete
|
|
10
|
+
* (chunk files cars.0..n-1 without gaps) and be fresh enough.
|
|
11
|
+
* - checkCounts: after the new file was written to a temp path — source and published counts must
|
|
12
|
+
* not be below an absolute minimum or drop by more than maxDropPercent against the last good run.
|
|
13
|
+
* A real stock drop is accepted by re-running once with acceptDrop.
|
|
14
|
+
* - publish: temp file -> final path by rename (same directory, atomic), previous file kept as
|
|
15
|
+
* `<name>.prev`, last good counts written to the state dir. A blocked run leaves the last good
|
|
16
|
+
* file in place, so the partner keeps reading it; how long that is acceptable is maxStaleHours.
|
|
17
|
+
* - lock: one run per state dir.
|
|
18
|
+
*/
|
|
19
|
+
export interface FeedGuardOptions {
|
|
20
|
+
// state dir of this feed (last-good.json, lock); one per export type and mode
|
|
21
|
+
stateDir: string;
|
|
22
|
+
// fail when the source snapshot has fewer cars than this
|
|
23
|
+
minSourceItems: number;
|
|
24
|
+
// fail when the published in_stock count is below this
|
|
25
|
+
minPublishedItems: number;
|
|
26
|
+
// fail when source or published count is more than this % below the last good run
|
|
27
|
+
maxDropPercent: number;
|
|
28
|
+
// fail when the newest base export chunk is older than this
|
|
29
|
+
maxSourceAgeHours: number;
|
|
30
|
+
// the last good file is reported STALE after this many hours without a successful run
|
|
31
|
+
maxStaleHours: number;
|
|
32
|
+
// one-off override for a real stock drop (Cronicle param); does not override the minimums
|
|
33
|
+
acceptDrop: boolean;
|
|
34
|
+
// a lock file older than this is treated as left over by a crashed run
|
|
35
|
+
lockStaleHours?: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface SourceSnapshot {
|
|
39
|
+
files: number;
|
|
40
|
+
missingChunks: number[];
|
|
41
|
+
newestMtime: Date | null;
|
|
42
|
+
oldestMtime: Date | null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface FeedCounts {
|
|
46
|
+
sourceItems: number;
|
|
47
|
+
publishedItems: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface LastGood extends FeedCounts {
|
|
51
|
+
publishedAt: string;
|
|
52
|
+
file: string;
|
|
53
|
+
runId?: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export class FeedGuardError extends Error {
|
|
57
|
+
constructor(readonly code: string, message: string) {
|
|
58
|
+
super(`[feed-guard:${code}] ${message}`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const HOUR_MS = 60 * 60 * 1000;
|
|
63
|
+
|
|
64
|
+
// Reads the base export snapshot: `<baseFileName>.<n>.json` chunk files (FsCarsLoader order).
|
|
65
|
+
export const inspectSourceSnapshot = async (directory: string, baseFileName: string): Promise<SourceSnapshot> => {
|
|
66
|
+
let names: string[];
|
|
67
|
+
try {
|
|
68
|
+
names = await fs.readdir(directory);
|
|
69
|
+
} catch (error) {
|
|
70
|
+
if (error?.code === 'ENOENT') {
|
|
71
|
+
return { files: 0, missingChunks: [], newestMtime: null, oldestMtime: null };
|
|
72
|
+
}
|
|
73
|
+
throw error;
|
|
74
|
+
}
|
|
75
|
+
const pattern = new RegExp(`^${baseFileName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.(\\d+)\\.json$`);
|
|
76
|
+
const chunks = names.map(name => ({ name, match: pattern.exec(name) })).filter(entry => entry.match);
|
|
77
|
+
const ids = new Set(chunks.map(entry => Number(entry.match[1])));
|
|
78
|
+
const maxId = ids.size ? Math.max(...ids) : -1;
|
|
79
|
+
const missingChunks: number[] = [];
|
|
80
|
+
for (let id = 0; id <= maxId; id += 1) {
|
|
81
|
+
if (!ids.has(id)) {
|
|
82
|
+
missingChunks.push(id);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
let newest: number = null;
|
|
86
|
+
let oldest: number = null;
|
|
87
|
+
for (const { name } of chunks) {
|
|
88
|
+
const { mtimeMs } = await fs.stat(join(directory, name));
|
|
89
|
+
newest = newest === null ? mtimeMs : Math.max(newest, mtimeMs);
|
|
90
|
+
oldest = oldest === null ? mtimeMs : Math.min(oldest, mtimeMs);
|
|
91
|
+
}
|
|
92
|
+
return {
|
|
93
|
+
files: chunks.length,
|
|
94
|
+
missingChunks,
|
|
95
|
+
newestMtime: newest === null ? null : new Date(newest),
|
|
96
|
+
oldestMtime: oldest === null ? null : new Date(oldest),
|
|
97
|
+
};
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
const dropPercent = (current: number, previous: number): number =>
|
|
101
|
+
previous > 0 ? ((previous - current) / previous) * 100 : 0;
|
|
102
|
+
|
|
103
|
+
export class FeedGuard {
|
|
104
|
+
private lockFile: string;
|
|
105
|
+
|
|
106
|
+
constructor(private readonly options: FeedGuardOptions) { }
|
|
107
|
+
|
|
108
|
+
private get lastGoodFile(): string {
|
|
109
|
+
return join(this.options.stateDir, 'last-good.json');
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async readLastGood(): Promise<LastGood | null> {
|
|
113
|
+
try {
|
|
114
|
+
return JSON.parse(await fs.readFile(this.lastGoodFile, 'utf8')) as LastGood;
|
|
115
|
+
} catch (error) {
|
|
116
|
+
if (error?.code === 'ENOENT') {
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
throw error;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async acquireLock(runId: string, now: Date): Promise<void> {
|
|
124
|
+
await fs.mkdir(this.options.stateDir, { recursive: true });
|
|
125
|
+
const lockFile = join(this.options.stateDir, 'run.lock');
|
|
126
|
+
const staleMs = (this.options.lockStaleHours ?? 6) * HOUR_MS;
|
|
127
|
+
try {
|
|
128
|
+
await fs.writeFile(lockFile, JSON.stringify({ runId, startedAt: now.toISOString() }), { flag: 'wx' });
|
|
129
|
+
} catch (error) {
|
|
130
|
+
if (error?.code !== 'EEXIST') {
|
|
131
|
+
throw error;
|
|
132
|
+
}
|
|
133
|
+
const { mtimeMs } = await fs.stat(lockFile);
|
|
134
|
+
if (now.getTime() - mtimeMs < staleMs) {
|
|
135
|
+
throw new FeedGuardError('locked', `another run holds ${lockFile} (since ${new Date(mtimeMs).toISOString()})`);
|
|
136
|
+
}
|
|
137
|
+
console.log(`feed-guard: removing stale lock ${lockFile} (${new Date(mtimeMs).toISOString()})`);
|
|
138
|
+
await fs.writeFile(lockFile, JSON.stringify({ runId, startedAt: now.toISOString() }));
|
|
139
|
+
}
|
|
140
|
+
this.lockFile = lockFile;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async releaseLock(): Promise<void> {
|
|
144
|
+
if (this.lockFile) {
|
|
145
|
+
await fs.rm(this.lockFile, { force: true });
|
|
146
|
+
this.lockFile = null;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
checkSource(snapshot: SourceSnapshot, now: Date): void {
|
|
151
|
+
if (!snapshot.files) {
|
|
152
|
+
throw new FeedGuardError('source_empty', 'base export snapshot has no chunk files');
|
|
153
|
+
}
|
|
154
|
+
if (snapshot.missingChunks.length) {
|
|
155
|
+
throw new FeedGuardError('source_partial', `base export snapshot misses chunk(s) ${snapshot.missingChunks.slice(0, 10).join(', ')}`);
|
|
156
|
+
}
|
|
157
|
+
const ageHours = (now.getTime() - snapshot.newestMtime.getTime()) / HOUR_MS;
|
|
158
|
+
if (ageHours > this.options.maxSourceAgeHours) {
|
|
159
|
+
throw new FeedGuardError(
|
|
160
|
+
'source_stale',
|
|
161
|
+
`newest base export chunk is ${ageHours.toFixed(1)} h old (max ${this.options.maxSourceAgeHours} h)`,
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
checkCounts(counts: FeedCounts, lastGood: LastGood | null): void {
|
|
167
|
+
if (counts.sourceItems < this.options.minSourceItems) {
|
|
168
|
+
throw new FeedGuardError('source_too_small', `${counts.sourceItems} source cars < minimum ${this.options.minSourceItems}`);
|
|
169
|
+
}
|
|
170
|
+
if (counts.publishedItems < this.options.minPublishedItems) {
|
|
171
|
+
throw new FeedGuardError(
|
|
172
|
+
'published_too_small',
|
|
173
|
+
`${counts.publishedItems} in_stock items < minimum ${this.options.minPublishedItems}`,
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
if (!lastGood || this.options.acceptDrop) {
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
const sourceDrop = dropPercent(counts.sourceItems, lastGood.sourceItems);
|
|
180
|
+
const publishedDrop = dropPercent(counts.publishedItems, lastGood.publishedItems);
|
|
181
|
+
if (sourceDrop > this.options.maxDropPercent || publishedDrop > this.options.maxDropPercent) {
|
|
182
|
+
throw new FeedGuardError(
|
|
183
|
+
'count_drop',
|
|
184
|
+
`source ${lastGood.sourceItems} -> ${counts.sourceItems} (-${sourceDrop.toFixed(1)}%), ` +
|
|
185
|
+
`in_stock ${lastGood.publishedItems} -> ${counts.publishedItems} (-${publishedDrop.toFixed(1)}%), ` +
|
|
186
|
+
`max ${this.options.maxDropPercent}% — re-run with acceptDrop if the stock really dropped`,
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
staleHours(lastGood: LastGood | null, now: Date): number | null {
|
|
192
|
+
return lastGood ? (now.getTime() - Date.parse(lastGood.publishedAt)) / HOUR_MS : null;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
isStale(lastGood: LastGood | null, now: Date): boolean {
|
|
196
|
+
const hours = this.staleHours(lastGood, now);
|
|
197
|
+
return hours !== null && hours > this.options.maxStaleHours;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// tmpFile must sit next to finalFile so the rename stays on one filesystem (atomic).
|
|
201
|
+
async publish(tmpFile: string, finalFile: string, counts: FeedCounts, now: Date, runId?: string): Promise<LastGood> {
|
|
202
|
+
if (dirname(tmpFile) !== dirname(finalFile)) {
|
|
203
|
+
throw new Error(`feed-guard: ${basename(tmpFile)} must be in the directory of ${finalFile}`);
|
|
204
|
+
}
|
|
205
|
+
try {
|
|
206
|
+
await fs.copyFile(finalFile, `${finalFile}.prev`);
|
|
207
|
+
} catch (error) {
|
|
208
|
+
if (error?.code !== 'ENOENT') {
|
|
209
|
+
throw error;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
await fs.rename(tmpFile, finalFile);
|
|
213
|
+
const lastGood: LastGood = { ...counts, publishedAt: now.toISOString(), file: finalFile, runId };
|
|
214
|
+
await fs.mkdir(this.options.stateDir, { recursive: true });
|
|
215
|
+
await fs.writeFile(`${this.lastGoodFile}.tmp`, JSON.stringify(lastGood, null, 2), 'utf8');
|
|
216
|
+
await fs.rename(`${this.lastGoodFile}.tmp`, this.lastGoodFile);
|
|
217
|
+
return lastGood;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import axios, { AxiosResponse } from 'axios';
|
|
2
|
+
import Bottleneck from 'bottleneck';
|
|
3
|
+
|
|
4
|
+
export enum ImageUrlRejectReason {
|
|
5
|
+
invalidUrl = 'invalid_url',
|
|
6
|
+
notHttps = 'not_https',
|
|
7
|
+
credentials = 'credentials_in_url',
|
|
8
|
+
placeholder = 'placeholder',
|
|
9
|
+
httpStatus = 'http_status',
|
|
10
|
+
wrongMime = 'wrong_mime',
|
|
11
|
+
tooManyRedirects = 'too_many_redirects',
|
|
12
|
+
timeout = 'timeout',
|
|
13
|
+
networkError = 'network_error',
|
|
14
|
+
budgetExhausted = 'budget_exhausted',
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface ImageUrlCheck {
|
|
18
|
+
url: string;
|
|
19
|
+
ok: boolean;
|
|
20
|
+
reason?: ImageUrlRejectReason;
|
|
21
|
+
status?: number;
|
|
22
|
+
contentType?: string;
|
|
23
|
+
finalUrl?: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface ImageUrlValidatorOptions {
|
|
27
|
+
// MIME types a partner is known to accept. OpenAI documents JPEG/PNG only as examples, so WebP
|
|
28
|
+
// (which our CDN negotiates for browsers) is rejected unless a partner explicitly allows it.
|
|
29
|
+
allowedMimeTypes?: string[];
|
|
30
|
+
requireHttps?: boolean;
|
|
31
|
+
timeoutMs?: number;
|
|
32
|
+
maxRedirects?: number;
|
|
33
|
+
concurrency?: number;
|
|
34
|
+
// Upper bound of HTTP probes per validator instance (one export run). Unlimited by default.
|
|
35
|
+
maxProbes?: number;
|
|
36
|
+
// e.g. the partner crawler's UA, to see what the crawler would get
|
|
37
|
+
userAgent?: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// auto.de "no image" assets (ImagesService.DEFAULT_IMAGES) — never a valid vehicle photo, and
|
|
41
|
+
// images.auto.de answers them with 404 + an image/jpeg body anyway.
|
|
42
|
+
const PLACEHOLDER_PATTERN = /\/carimage\/noimage\//i;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Budgeted, cached HTTP check of image URLs before they go into a partner feed (ADV-336).
|
|
46
|
+
* HEAD first, GET (body discarded) when HEAD is not supported. A URL is valid only with a final
|
|
47
|
+
* 200 and an allowed image MIME type — images.auto.de answers unknown images with 404 AND an
|
|
48
|
+
* image/jpeg placeholder body, so the Content-Type alone proves nothing. One instance = one run:
|
|
49
|
+
* each URL is probed at most once, results and per-reason counters are kept for the run log.
|
|
50
|
+
*/
|
|
51
|
+
export class ImageUrlValidator {
|
|
52
|
+
private readonly options: Required<ImageUrlValidatorOptions>;
|
|
53
|
+
private readonly limiter: Bottleneck;
|
|
54
|
+
private readonly cache = new Map<string, Promise<ImageUrlCheck>>();
|
|
55
|
+
private readonly counters: Record<string, number> = {};
|
|
56
|
+
private probes = 0;
|
|
57
|
+
|
|
58
|
+
constructor(options: ImageUrlValidatorOptions = {}) {
|
|
59
|
+
this.options = {
|
|
60
|
+
allowedMimeTypes: ['image/jpeg', 'image/png'],
|
|
61
|
+
requireHttps: true,
|
|
62
|
+
timeoutMs: 5000,
|
|
63
|
+
maxRedirects: 3,
|
|
64
|
+
concurrency: 8,
|
|
65
|
+
maxProbes: Infinity,
|
|
66
|
+
userAgent: undefined,
|
|
67
|
+
...options,
|
|
68
|
+
};
|
|
69
|
+
this.limiter = new Bottleneck({ maxConcurrent: this.options.concurrency });
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
validate(url: string): Promise<ImageUrlCheck> {
|
|
73
|
+
if (!this.cache.has(url)) {
|
|
74
|
+
this.cache.set(url, this.check(url).then((result) => {
|
|
75
|
+
const key = result.ok ? 'ok' : result.reason;
|
|
76
|
+
this.counters[key] = (this.counters[key] ?? 0) + 1;
|
|
77
|
+
return result;
|
|
78
|
+
}));
|
|
79
|
+
}
|
|
80
|
+
return this.cache.get(url);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
validateAll(urls: string[]): Promise<ImageUrlCheck[]> {
|
|
84
|
+
return Promise.all(urls.map(url => this.validate(url)));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Per-reason counts of the distinct URLs checked so far (cache hits are not counted twice).
|
|
88
|
+
get stats(): Record<string, number> {
|
|
89
|
+
return { ...this.counters };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private async check(url: string): Promise<ImageUrlCheck> {
|
|
93
|
+
const rejected = this.checkStatically(url);
|
|
94
|
+
if (rejected) {
|
|
95
|
+
return rejected;
|
|
96
|
+
}
|
|
97
|
+
if (this.probes >= this.options.maxProbes) {
|
|
98
|
+
return { url, ok: false, reason: ImageUrlRejectReason.budgetExhausted };
|
|
99
|
+
}
|
|
100
|
+
this.probes++;
|
|
101
|
+
|
|
102
|
+
try {
|
|
103
|
+
let response = await this.limiter.schedule(() => this.request('HEAD', url));
|
|
104
|
+
if (response.status === 405 || response.status === 501) {
|
|
105
|
+
response = await this.limiter.schedule(() => this.request('GET', url));
|
|
106
|
+
}
|
|
107
|
+
return this.checkResponse(url, response);
|
|
108
|
+
} catch (error) {
|
|
109
|
+
return { url, ok: false, reason: this.mapError(error) };
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
private checkStatically(url: string): ImageUrlCheck | null {
|
|
114
|
+
let parsed: URL;
|
|
115
|
+
try {
|
|
116
|
+
parsed = new URL(url);
|
|
117
|
+
} catch {
|
|
118
|
+
return { url, ok: false, reason: ImageUrlRejectReason.invalidUrl };
|
|
119
|
+
}
|
|
120
|
+
if (!['http:', 'https:'].includes(parsed.protocol)) {
|
|
121
|
+
return { url, ok: false, reason: ImageUrlRejectReason.invalidUrl };
|
|
122
|
+
}
|
|
123
|
+
if (this.options.requireHttps && parsed.protocol !== 'https:') {
|
|
124
|
+
return { url, ok: false, reason: ImageUrlRejectReason.notHttps };
|
|
125
|
+
}
|
|
126
|
+
if (parsed.username || parsed.password) {
|
|
127
|
+
return { url, ok: false, reason: ImageUrlRejectReason.credentials };
|
|
128
|
+
}
|
|
129
|
+
if (PLACEHOLDER_PATTERN.test(parsed.pathname)) {
|
|
130
|
+
return { url, ok: false, reason: ImageUrlRejectReason.placeholder };
|
|
131
|
+
}
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
private request(method: 'HEAD' | 'GET', url: string): Promise<AxiosResponse> {
|
|
136
|
+
return axios.request({
|
|
137
|
+
method,
|
|
138
|
+
url,
|
|
139
|
+
timeout: this.options.timeoutMs,
|
|
140
|
+
maxRedirects: this.options.maxRedirects,
|
|
141
|
+
// body is never needed — the stream is destroyed right after the headers arrive
|
|
142
|
+
responseType: 'stream',
|
|
143
|
+
headers: this.options.userAgent ? { 'User-Agent': this.options.userAgent } : undefined,
|
|
144
|
+
validateStatus: () => true,
|
|
145
|
+
}).then((response) => {
|
|
146
|
+
response.data?.destroy?.();
|
|
147
|
+
return response;
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
private checkResponse(url: string, response: AxiosResponse): ImageUrlCheck {
|
|
152
|
+
const status = response.status;
|
|
153
|
+
const contentType = String(response.headers?.['content-type'] ?? '').split(';')[0].trim().toLowerCase();
|
|
154
|
+
const finalUrl: string = response.request?.res?.responseUrl ?? url;
|
|
155
|
+
const result = { url, status, contentType, finalUrl };
|
|
156
|
+
|
|
157
|
+
if (status !== 200) {
|
|
158
|
+
return { ...result, ok: false, reason: ImageUrlRejectReason.httpStatus };
|
|
159
|
+
}
|
|
160
|
+
if (this.options.requireHttps && !finalUrl.startsWith('https:')) {
|
|
161
|
+
return { ...result, ok: false, reason: ImageUrlRejectReason.notHttps };
|
|
162
|
+
}
|
|
163
|
+
if (PLACEHOLDER_PATTERN.test(finalUrl)) {
|
|
164
|
+
return { ...result, ok: false, reason: ImageUrlRejectReason.placeholder };
|
|
165
|
+
}
|
|
166
|
+
if (!this.options.allowedMimeTypes.includes(contentType)) {
|
|
167
|
+
return { ...result, ok: false, reason: ImageUrlRejectReason.wrongMime };
|
|
168
|
+
}
|
|
169
|
+
return { ...result, ok: true };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
private mapError(error: any): ImageUrlRejectReason {
|
|
173
|
+
if (error?.code === 'ERR_FR_TOO_MANY_REDIRECTS') {
|
|
174
|
+
return ImageUrlRejectReason.tooManyRedirects;
|
|
175
|
+
}
|
|
176
|
+
if (error?.code === 'ECONNABORTED' || error?.code === 'ETIMEDOUT' || /timeout/i.test(error?.message ?? '')) {
|
|
177
|
+
return ImageUrlRejectReason.timeout;
|
|
178
|
+
}
|
|
179
|
+
return ImageUrlRejectReason.networkError;
|
|
180
|
+
}
|
|
181
|
+
}
|