@bytebrand/export-cars 1.26.40 → 1.26.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/configs/autouncle.config.js +22 -0
- package/dist/configs/autouncle.config.js.map +1 -1
- package/dist/exporters/autouncle.exporter.js +10 -0
- package/dist/exporters/autouncle.exporter.js.map +1 -1
- package/dist/services/autouncle-feed-run.d.ts +6 -0
- package/dist/services/autouncle-feed-run.js +11 -1
- package/dist/services/autouncle-feed-run.js.map +1 -1
- package/dist/transformers/autouncle.transformer.d.ts +4 -0
- package/dist/transformers/autouncle.transformer.js +10 -5
- package/dist/transformers/autouncle.transformer.js.map +1 -1
- package/dist/transformers/google-feed-envkv.d.ts +2 -0
- package/dist/transformers/google-feed-envkv.js +9 -1
- package/dist/transformers/google-feed-envkv.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/autouncle/AUDIT.md +103 -97
- package/docs/autouncle/PARTNER-QUESTIONS.md +3 -1
- package/docs/autouncle/RUNBOOK.md +2 -0
- package/docs/criteo/RUNBOOK.md +86 -86
- package/package.json +1 -1
- package/src/configs/autouncle.config.ts +26 -1
- package/src/configs/criteo.config.ts +54 -54
- package/src/exporters/autouncle.exporter.ts +12 -0
- package/src/exporters/criteo.exporter.ts +63 -63
- package/src/services/autouncle-feed-run.ts +19 -2
- package/src/services/criteo-feed-run.ts +253 -253
- package/src/transformers/autouncle.transformer.ts +20 -4
- package/src/transformers/google-feed-envkv.ts +10 -0
- package/test/autouncle-feed-run.test.ts +25 -2
- package/test/autouncle-mapping.test.ts +41 -2
- package/test/criteo-feed-run.test.ts +365 -365
- package/test/criteo-mapping.test.ts +122 -122
- package/test/fixtures/criteo-20-cases.json +581 -581
|
@@ -1,253 +1,253 @@
|
|
|
1
|
-
import { promises as fs } from 'fs';
|
|
2
|
-
import { dirname, join } from 'path';
|
|
3
|
-
import { get, toNumber, toString } from 'lodash';
|
|
4
|
-
import { ImboImageResponseInterface } from '@bytebrand/chunkable';
|
|
5
|
-
import { DataSchemes } from '@bytebrand/car-schema';
|
|
6
|
-
|
|
7
|
-
import { GoogleXmlBuilder } from '../formats/google-xml.builder';
|
|
8
|
-
import { CriteoXmlItem } from '../interfaces';
|
|
9
|
-
import { FeedGuard, FeedGuardError, LastGood, inspectSourceSnapshot } from './feed-guard';
|
|
10
|
-
|
|
11
|
-
export interface CriteoSourceCar {
|
|
12
|
-
car: DataSchemes.Car;
|
|
13
|
-
images: ImboImageResponseInterface[];
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
export enum CriteoExclusion {
|
|
17
|
-
noId = 'no_id',
|
|
18
|
-
duplicateId = 'duplicate_id',
|
|
19
|
-
filtered = 'filtered',
|
|
20
|
-
notAvailable = 'reserved_or_sold',
|
|
21
|
-
noPrice = 'no_valid_price',
|
|
22
|
-
noImage = 'no_marketing_image',
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
// Online cars flagged like this are not sellable stock (same rule as the AutoUncle feed). Criteo
|
|
26
|
-
// (16.10.2026): only currently marketable stock goes into the full file; a car that leaves it is
|
|
27
|
-
// removed by the next full import.
|
|
28
|
-
const NOT_AVAILABLE = ['selector_offerAvailability_reserved', 'selector_offerAvailability_sold'];
|
|
29
|
-
|
|
30
|
-
// Everything the run needs from the outside world — the Nest exporter wires the real services,
|
|
31
|
-
// tests pass fakes.
|
|
32
|
-
export interface CriteoRunDeps {
|
|
33
|
-
// iterates the base export snapshot chunk by chunk (LoaderService.process)
|
|
34
|
-
forEachChunk: (fn: (cars: CriteoSourceCar[]) => Promise<void>) => Promise<unknown>;
|
|
35
|
-
// existing shared partner filter (images count, Cronicle rules)
|
|
36
|
-
filterCar: (source: CriteoSourceCar) => boolean;
|
|
37
|
-
// CriteoTransformer.transform: null = no real marketing image
|
|
38
|
-
transform: (source: CriteoSourceCar) => Promise<CriteoXmlItem | null>;
|
|
39
|
-
guard: FeedGuard;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
export interface CriteoRunOptions {
|
|
43
|
-
runId?: string;
|
|
44
|
-
domain: string;
|
|
45
|
-
sourceDirectory: string;
|
|
46
|
-
sourceBaseFileName: string;
|
|
47
|
-
xmlFile: string;
|
|
48
|
-
stateDir: string;
|
|
49
|
-
now?: () => Date;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export interface CriteoRunReport {
|
|
53
|
-
runId?: string;
|
|
54
|
-
startedAt: string;
|
|
55
|
-
finishedAt?: string;
|
|
56
|
-
durationMs?: number;
|
|
57
|
-
status: 'published' | 'blocked' | 'failed';
|
|
58
|
-
guard?: { code: string; message: string };
|
|
59
|
-
error?: string;
|
|
60
|
-
xmlFile: string;
|
|
61
|
-
source: { files: number; newestChunkAt: string | null; ageHours: number | null; cars: number };
|
|
62
|
-
items: number;
|
|
63
|
-
excluded: Record<string, number>;
|
|
64
|
-
withRate: number;
|
|
65
|
-
withoutEnergyClass: number;
|
|
66
|
-
// against the ids of the last published file; removed ids are dropped by Criteo's next full import
|
|
67
|
-
ids: { added: number; removed: number; kept: number; removedSample: string[] } | null;
|
|
68
|
-
lastGood: LastGood | null;
|
|
69
|
-
lastGoodAgeHours: number | null;
|
|
70
|
-
stale: boolean;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
const HOUR_MS = 60 * 60 * 1000;
|
|
74
|
-
const PRICE = /^\d+(\.\d+)? EUR$/;
|
|
75
|
-
|
|
76
|
-
const increment = (counter: Record<string, number>, key: string) => {
|
|
77
|
-
counter[key] = (counter[key] ?? 0) + 1;
|
|
78
|
-
};
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* One Criteo catalog run (ADV-319): source check -> map -> temp XML -> count check -> atomic
|
|
82
|
-
* publish -> report. Nothing outside the output directory and `stateDir` is written. A blocked run
|
|
83
|
-
* leaves the last good XML in place and fails the job, so the chained upload step does not run.
|
|
84
|
-
* The XML format, enums and field order stay the legacy Criteo ones.
|
|
85
|
-
*/
|
|
86
|
-
export class CriteoFeedRun {
|
|
87
|
-
constructor(private readonly deps: CriteoRunDeps, private readonly options: CriteoRunOptions) { }
|
|
88
|
-
|
|
89
|
-
private now(): Date {
|
|
90
|
-
return this.options.now ? this.options.now() : new Date();
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
private get idsFile(): string {
|
|
94
|
-
return join(this.options.stateDir, 'published-ids.json');
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
async run(): Promise<CriteoRunReport> {
|
|
98
|
-
const { guard } = this.deps;
|
|
99
|
-
const { xmlFile, runId } = this.options;
|
|
100
|
-
const startedAt = this.now();
|
|
101
|
-
const lastGood = await guard.readLastGood();
|
|
102
|
-
const report: CriteoRunReport = {
|
|
103
|
-
runId,
|
|
104
|
-
startedAt: startedAt.toISOString(),
|
|
105
|
-
status: 'failed',
|
|
106
|
-
xmlFile,
|
|
107
|
-
source: { files: 0, newestChunkAt: null, ageHours: null, cars: 0 },
|
|
108
|
-
items: 0,
|
|
109
|
-
excluded: {},
|
|
110
|
-
withRate: 0,
|
|
111
|
-
withoutEnergyClass: 0,
|
|
112
|
-
ids: null,
|
|
113
|
-
lastGood,
|
|
114
|
-
lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
|
|
115
|
-
stale: guard.isStale(lastGood, startedAt),
|
|
116
|
-
};
|
|
117
|
-
|
|
118
|
-
await guard.acquireLock(runId ?? 'local', startedAt);
|
|
119
|
-
const tmpFile = `${xmlFile}.tmp`;
|
|
120
|
-
try {
|
|
121
|
-
const snapshot = await inspectSourceSnapshot(this.options.sourceDirectory, this.options.sourceBaseFileName);
|
|
122
|
-
report.source.files = snapshot.files;
|
|
123
|
-
report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
|
|
124
|
-
report.source.ageHours = snapshot.newestMtime
|
|
125
|
-
? Number(((startedAt.getTime() - snapshot.newestMtime.getTime()) / HOUR_MS).toFixed(2))
|
|
126
|
-
: null;
|
|
127
|
-
guard.checkSource(snapshot, startedAt);
|
|
128
|
-
|
|
129
|
-
await fs.mkdir(dirname(xmlFile), { recursive: true });
|
|
130
|
-
// the builder appends — never onto a leftover temp file
|
|
131
|
-
await fs.rm(tmpFile, { force: true });
|
|
132
|
-
const builder = new GoogleXmlBuilder(tmpFile);
|
|
133
|
-
await builder.buildStart(this.options.domain);
|
|
134
|
-
|
|
135
|
-
const seen = new Set<string>();
|
|
136
|
-
const published: string[] = [];
|
|
137
|
-
await this.deps.forEachChunk(async (cars) => {
|
|
138
|
-
for (const source of cars) {
|
|
139
|
-
report.source.cars += 1;
|
|
140
|
-
const reason = this.exclusion(source, seen);
|
|
141
|
-
if (reason) {
|
|
142
|
-
increment(report.excluded, reason);
|
|
143
|
-
continue;
|
|
144
|
-
}
|
|
145
|
-
const item = await this.deps.transform(source);
|
|
146
|
-
if (!item) {
|
|
147
|
-
increment(report.excluded, CriteoExclusion.noImage);
|
|
148
|
-
continue;
|
|
149
|
-
}
|
|
150
|
-
// the transformer formats the price; a value it could not format is not sent
|
|
151
|
-
if (!PRICE.test(item.price) || !PRICE.test(item.sale_price)) {
|
|
152
|
-
increment(report.excluded, CriteoExclusion.noPrice);
|
|
153
|
-
continue;
|
|
154
|
-
}
|
|
155
|
-
await builder.addItem(item);
|
|
156
|
-
published.push(item.id);
|
|
157
|
-
report.items += 1;
|
|
158
|
-
if (item.installment) {
|
|
159
|
-
report.withRate += 1;
|
|
160
|
-
}
|
|
161
|
-
if (!item.energy_efficiency_class) {
|
|
162
|
-
report.withoutEnergyClass += 1;
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
});
|
|
166
|
-
await builder.buildEnd();
|
|
167
|
-
|
|
168
|
-
const counts = { sourceItems: report.source.cars, publishedItems: report.items };
|
|
169
|
-
guard.checkCounts(counts, lastGood);
|
|
170
|
-
|
|
171
|
-
const previousIds = await this.readPublishedIds();
|
|
172
|
-
report.lastGood = await guard.publish(tmpFile, xmlFile, counts, this.now(), runId);
|
|
173
|
-
await this.writePublishedIds(published);
|
|
174
|
-
report.ids = this.diffIds(previousIds, published);
|
|
175
|
-
report.status = 'published';
|
|
176
|
-
report.lastGoodAgeHours = 0;
|
|
177
|
-
report.stale = false;
|
|
178
|
-
return report;
|
|
179
|
-
} catch (error) {
|
|
180
|
-
if (error instanceof FeedGuardError) {
|
|
181
|
-
report.status = 'blocked';
|
|
182
|
-
report.guard = { code: error.code, message: error.message };
|
|
183
|
-
} else {
|
|
184
|
-
report.error = error?.message ?? String(error);
|
|
185
|
-
}
|
|
186
|
-
// keep the rejected file for inspection; the published file stays untouched
|
|
187
|
-
await fs.rename(tmpFile, `${xmlFile}.rejected`).catch(() => undefined);
|
|
188
|
-
throw error;
|
|
189
|
-
} finally {
|
|
190
|
-
await this.writeReport(report, startedAt);
|
|
191
|
-
await guard.releaseLock();
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
private exclusion(source: CriteoSourceCar, seen: Set<string>): CriteoExclusion | null {
|
|
196
|
-
const id = toString(get(source, 'car._id', ''));
|
|
197
|
-
if (!id) {
|
|
198
|
-
return CriteoExclusion.noId;
|
|
199
|
-
}
|
|
200
|
-
if (seen.has(id)) {
|
|
201
|
-
return CriteoExclusion.duplicateId;
|
|
202
|
-
}
|
|
203
|
-
seen.add(id);
|
|
204
|
-
if (!this.deps.filterCar(source)) {
|
|
205
|
-
return CriteoExclusion.filtered;
|
|
206
|
-
}
|
|
207
|
-
if (NOT_AVAILABLE.includes(toString(get(source.car, 'offer.offerAvailability', '')))) {
|
|
208
|
-
return CriteoExclusion.notAvailable;
|
|
209
|
-
}
|
|
210
|
-
const price = toNumber(get(source.car, 'price.currentSalesPrice', null));
|
|
211
|
-
if (!Number.isFinite(price) || price <= 0) {
|
|
212
|
-
return CriteoExclusion.noPrice;
|
|
213
|
-
}
|
|
214
|
-
return null;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
private async readPublishedIds(): Promise<string[] | null> {
|
|
218
|
-
try {
|
|
219
|
-
return JSON.parse(await fs.readFile(this.idsFile, 'utf8')) as string[];
|
|
220
|
-
} catch (error) {
|
|
221
|
-
if (error?.code === 'ENOENT') {
|
|
222
|
-
return null;
|
|
223
|
-
}
|
|
224
|
-
throw error;
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
private async writePublishedIds(ids: string[]): Promise<void> {
|
|
229
|
-
await fs.mkdir(this.options.stateDir, { recursive: true });
|
|
230
|
-
await fs.writeFile(`${this.idsFile}.tmp`, JSON.stringify(ids), 'utf8');
|
|
231
|
-
await fs.rename(`${this.idsFile}.tmp`, this.idsFile);
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
private diffIds(previous: string[] | null, current: string[]): CriteoRunReport['ids'] {
|
|
235
|
-
if (!previous) {
|
|
236
|
-
return { added: current.length, removed: 0, kept: 0, removedSample: [] };
|
|
237
|
-
}
|
|
238
|
-
const before = new Set(previous);
|
|
239
|
-
const now = new Set(current);
|
|
240
|
-
const removed = previous.filter(id => !now.has(id));
|
|
241
|
-
const kept = current.filter(id => before.has(id)).length;
|
|
242
|
-
return { added: current.length - kept, removed: removed.length, kept, removedSample: removed.slice(0, 20) };
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
private async writeReport(report: CriteoRunReport, startedAt: Date): Promise<void> {
|
|
246
|
-
const finishedAt = this.now();
|
|
247
|
-
report.finishedAt = finishedAt.toISOString();
|
|
248
|
-
report.durationMs = finishedAt.getTime() - startedAt.getTime();
|
|
249
|
-
await fs.mkdir(this.options.stateDir, { recursive: true });
|
|
250
|
-
await fs.writeFile(join(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
|
|
251
|
-
console.log(`criteo feed report: ${JSON.stringify(report)}`);
|
|
252
|
-
}
|
|
253
|
-
}
|
|
1
|
+
import { promises as fs } from 'fs';
|
|
2
|
+
import { dirname, join } from 'path';
|
|
3
|
+
import { get, toNumber, toString } from 'lodash';
|
|
4
|
+
import { ImboImageResponseInterface } from '@bytebrand/chunkable';
|
|
5
|
+
import { DataSchemes } from '@bytebrand/car-schema';
|
|
6
|
+
|
|
7
|
+
import { GoogleXmlBuilder } from '../formats/google-xml.builder';
|
|
8
|
+
import { CriteoXmlItem } from '../interfaces';
|
|
9
|
+
import { FeedGuard, FeedGuardError, LastGood, inspectSourceSnapshot } from './feed-guard';
|
|
10
|
+
|
|
11
|
+
export interface CriteoSourceCar {
|
|
12
|
+
car: DataSchemes.Car;
|
|
13
|
+
images: ImboImageResponseInterface[];
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export enum CriteoExclusion {
|
|
17
|
+
noId = 'no_id',
|
|
18
|
+
duplicateId = 'duplicate_id',
|
|
19
|
+
filtered = 'filtered',
|
|
20
|
+
notAvailable = 'reserved_or_sold',
|
|
21
|
+
noPrice = 'no_valid_price',
|
|
22
|
+
noImage = 'no_marketing_image',
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Online cars flagged like this are not sellable stock (same rule as the AutoUncle feed). Criteo
|
|
26
|
+
// (16.10.2026): only currently marketable stock goes into the full file; a car that leaves it is
|
|
27
|
+
// removed by the next full import.
|
|
28
|
+
const NOT_AVAILABLE = ['selector_offerAvailability_reserved', 'selector_offerAvailability_sold'];
|
|
29
|
+
|
|
30
|
+
// Everything the run needs from the outside world — the Nest exporter wires the real services,
|
|
31
|
+
// tests pass fakes.
|
|
32
|
+
export interface CriteoRunDeps {
|
|
33
|
+
// iterates the base export snapshot chunk by chunk (LoaderService.process)
|
|
34
|
+
forEachChunk: (fn: (cars: CriteoSourceCar[]) => Promise<void>) => Promise<unknown>;
|
|
35
|
+
// existing shared partner filter (images count, Cronicle rules)
|
|
36
|
+
filterCar: (source: CriteoSourceCar) => boolean;
|
|
37
|
+
// CriteoTransformer.transform: null = no real marketing image
|
|
38
|
+
transform: (source: CriteoSourceCar) => Promise<CriteoXmlItem | null>;
|
|
39
|
+
guard: FeedGuard;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface CriteoRunOptions {
|
|
43
|
+
runId?: string;
|
|
44
|
+
domain: string;
|
|
45
|
+
sourceDirectory: string;
|
|
46
|
+
sourceBaseFileName: string;
|
|
47
|
+
xmlFile: string;
|
|
48
|
+
stateDir: string;
|
|
49
|
+
now?: () => Date;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface CriteoRunReport {
|
|
53
|
+
runId?: string;
|
|
54
|
+
startedAt: string;
|
|
55
|
+
finishedAt?: string;
|
|
56
|
+
durationMs?: number;
|
|
57
|
+
status: 'published' | 'blocked' | 'failed';
|
|
58
|
+
guard?: { code: string; message: string };
|
|
59
|
+
error?: string;
|
|
60
|
+
xmlFile: string;
|
|
61
|
+
source: { files: number; newestChunkAt: string | null; ageHours: number | null; cars: number };
|
|
62
|
+
items: number;
|
|
63
|
+
excluded: Record<string, number>;
|
|
64
|
+
withRate: number;
|
|
65
|
+
withoutEnergyClass: number;
|
|
66
|
+
// against the ids of the last published file; removed ids are dropped by Criteo's next full import
|
|
67
|
+
ids: { added: number; removed: number; kept: number; removedSample: string[] } | null;
|
|
68
|
+
lastGood: LastGood | null;
|
|
69
|
+
lastGoodAgeHours: number | null;
|
|
70
|
+
stale: boolean;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const HOUR_MS = 60 * 60 * 1000;
|
|
74
|
+
const PRICE = /^\d+(\.\d+)? EUR$/;
|
|
75
|
+
|
|
76
|
+
const increment = (counter: Record<string, number>, key: string) => {
|
|
77
|
+
counter[key] = (counter[key] ?? 0) + 1;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* One Criteo catalog run (ADV-319): source check -> map -> temp XML -> count check -> atomic
|
|
82
|
+
* publish -> report. Nothing outside the output directory and `stateDir` is written. A blocked run
|
|
83
|
+
* leaves the last good XML in place and fails the job, so the chained upload step does not run.
|
|
84
|
+
* The XML format, enums and field order stay the legacy Criteo ones.
|
|
85
|
+
*/
|
|
86
|
+
export class CriteoFeedRun {
|
|
87
|
+
constructor(private readonly deps: CriteoRunDeps, private readonly options: CriteoRunOptions) { }
|
|
88
|
+
|
|
89
|
+
private now(): Date {
|
|
90
|
+
return this.options.now ? this.options.now() : new Date();
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
private get idsFile(): string {
|
|
94
|
+
return join(this.options.stateDir, 'published-ids.json');
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async run(): Promise<CriteoRunReport> {
|
|
98
|
+
const { guard } = this.deps;
|
|
99
|
+
const { xmlFile, runId } = this.options;
|
|
100
|
+
const startedAt = this.now();
|
|
101
|
+
const lastGood = await guard.readLastGood();
|
|
102
|
+
const report: CriteoRunReport = {
|
|
103
|
+
runId,
|
|
104
|
+
startedAt: startedAt.toISOString(),
|
|
105
|
+
status: 'failed',
|
|
106
|
+
xmlFile,
|
|
107
|
+
source: { files: 0, newestChunkAt: null, ageHours: null, cars: 0 },
|
|
108
|
+
items: 0,
|
|
109
|
+
excluded: {},
|
|
110
|
+
withRate: 0,
|
|
111
|
+
withoutEnergyClass: 0,
|
|
112
|
+
ids: null,
|
|
113
|
+
lastGood,
|
|
114
|
+
lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
|
|
115
|
+
stale: guard.isStale(lastGood, startedAt),
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
await guard.acquireLock(runId ?? 'local', startedAt);
|
|
119
|
+
const tmpFile = `${xmlFile}.tmp`;
|
|
120
|
+
try {
|
|
121
|
+
const snapshot = await inspectSourceSnapshot(this.options.sourceDirectory, this.options.sourceBaseFileName);
|
|
122
|
+
report.source.files = snapshot.files;
|
|
123
|
+
report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
|
|
124
|
+
report.source.ageHours = snapshot.newestMtime
|
|
125
|
+
? Number(((startedAt.getTime() - snapshot.newestMtime.getTime()) / HOUR_MS).toFixed(2))
|
|
126
|
+
: null;
|
|
127
|
+
guard.checkSource(snapshot, startedAt);
|
|
128
|
+
|
|
129
|
+
await fs.mkdir(dirname(xmlFile), { recursive: true });
|
|
130
|
+
// the builder appends — never onto a leftover temp file
|
|
131
|
+
await fs.rm(tmpFile, { force: true });
|
|
132
|
+
const builder = new GoogleXmlBuilder(tmpFile);
|
|
133
|
+
await builder.buildStart(this.options.domain);
|
|
134
|
+
|
|
135
|
+
const seen = new Set<string>();
|
|
136
|
+
const published: string[] = [];
|
|
137
|
+
await this.deps.forEachChunk(async (cars) => {
|
|
138
|
+
for (const source of cars) {
|
|
139
|
+
report.source.cars += 1;
|
|
140
|
+
const reason = this.exclusion(source, seen);
|
|
141
|
+
if (reason) {
|
|
142
|
+
increment(report.excluded, reason);
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
const item = await this.deps.transform(source);
|
|
146
|
+
if (!item) {
|
|
147
|
+
increment(report.excluded, CriteoExclusion.noImage);
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
// the transformer formats the price; a value it could not format is not sent
|
|
151
|
+
if (!PRICE.test(item.price) || !PRICE.test(item.sale_price)) {
|
|
152
|
+
increment(report.excluded, CriteoExclusion.noPrice);
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
await builder.addItem(item);
|
|
156
|
+
published.push(item.id);
|
|
157
|
+
report.items += 1;
|
|
158
|
+
if (item.installment) {
|
|
159
|
+
report.withRate += 1;
|
|
160
|
+
}
|
|
161
|
+
if (!item.energy_efficiency_class) {
|
|
162
|
+
report.withoutEnergyClass += 1;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
await builder.buildEnd();
|
|
167
|
+
|
|
168
|
+
const counts = { sourceItems: report.source.cars, publishedItems: report.items };
|
|
169
|
+
guard.checkCounts(counts, lastGood);
|
|
170
|
+
|
|
171
|
+
const previousIds = await this.readPublishedIds();
|
|
172
|
+
report.lastGood = await guard.publish(tmpFile, xmlFile, counts, this.now(), runId);
|
|
173
|
+
await this.writePublishedIds(published);
|
|
174
|
+
report.ids = this.diffIds(previousIds, published);
|
|
175
|
+
report.status = 'published';
|
|
176
|
+
report.lastGoodAgeHours = 0;
|
|
177
|
+
report.stale = false;
|
|
178
|
+
return report;
|
|
179
|
+
} catch (error) {
|
|
180
|
+
if (error instanceof FeedGuardError) {
|
|
181
|
+
report.status = 'blocked';
|
|
182
|
+
report.guard = { code: error.code, message: error.message };
|
|
183
|
+
} else {
|
|
184
|
+
report.error = error?.message ?? String(error);
|
|
185
|
+
}
|
|
186
|
+
// keep the rejected file for inspection; the published file stays untouched
|
|
187
|
+
await fs.rename(tmpFile, `${xmlFile}.rejected`).catch(() => undefined);
|
|
188
|
+
throw error;
|
|
189
|
+
} finally {
|
|
190
|
+
await this.writeReport(report, startedAt);
|
|
191
|
+
await guard.releaseLock();
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
private exclusion(source: CriteoSourceCar, seen: Set<string>): CriteoExclusion | null {
|
|
196
|
+
const id = toString(get(source, 'car._id', ''));
|
|
197
|
+
if (!id) {
|
|
198
|
+
return CriteoExclusion.noId;
|
|
199
|
+
}
|
|
200
|
+
if (seen.has(id)) {
|
|
201
|
+
return CriteoExclusion.duplicateId;
|
|
202
|
+
}
|
|
203
|
+
seen.add(id);
|
|
204
|
+
if (!this.deps.filterCar(source)) {
|
|
205
|
+
return CriteoExclusion.filtered;
|
|
206
|
+
}
|
|
207
|
+
if (NOT_AVAILABLE.includes(toString(get(source.car, 'offer.offerAvailability', '')))) {
|
|
208
|
+
return CriteoExclusion.notAvailable;
|
|
209
|
+
}
|
|
210
|
+
const price = toNumber(get(source.car, 'price.currentSalesPrice', null));
|
|
211
|
+
if (!Number.isFinite(price) || price <= 0) {
|
|
212
|
+
return CriteoExclusion.noPrice;
|
|
213
|
+
}
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
private async readPublishedIds(): Promise<string[] | null> {
|
|
218
|
+
try {
|
|
219
|
+
return JSON.parse(await fs.readFile(this.idsFile, 'utf8')) as string[];
|
|
220
|
+
} catch (error) {
|
|
221
|
+
if (error?.code === 'ENOENT') {
|
|
222
|
+
return null;
|
|
223
|
+
}
|
|
224
|
+
throw error;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
private async writePublishedIds(ids: string[]): Promise<void> {
|
|
229
|
+
await fs.mkdir(this.options.stateDir, { recursive: true });
|
|
230
|
+
await fs.writeFile(`${this.idsFile}.tmp`, JSON.stringify(ids), 'utf8');
|
|
231
|
+
await fs.rename(`${this.idsFile}.tmp`, this.idsFile);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
private diffIds(previous: string[] | null, current: string[]): CriteoRunReport['ids'] {
|
|
235
|
+
if (!previous) {
|
|
236
|
+
return { added: current.length, removed: 0, kept: 0, removedSample: [] };
|
|
237
|
+
}
|
|
238
|
+
const before = new Set(previous);
|
|
239
|
+
const now = new Set(current);
|
|
240
|
+
const removed = previous.filter(id => !now.has(id));
|
|
241
|
+
const kept = current.filter(id => before.has(id)).length;
|
|
242
|
+
return { added: current.length - kept, removed: removed.length, kept, removedSample: removed.slice(0, 20) };
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
private async writeReport(report: CriteoRunReport, startedAt: Date): Promise<void> {
|
|
246
|
+
const finishedAt = this.now();
|
|
247
|
+
report.finishedAt = finishedAt.toISOString();
|
|
248
|
+
report.durationMs = finishedAt.getTime() - startedAt.getTime();
|
|
249
|
+
await fs.mkdir(this.options.stateDir, { recursive: true });
|
|
250
|
+
await fs.writeFile(join(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
|
|
251
|
+
console.log(`criteo feed report: ${JSON.stringify(report)}`);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
@@ -11,6 +11,16 @@ import { DEFAULT_REGION, MIN_MILEAGES } from '../constants/autouncle.contants';
|
|
|
11
11
|
import { AutouncleJsonItem } from '../interfaces';
|
|
12
12
|
import { OpenAiImageSelection } from './openai-image-fields';
|
|
13
13
|
import { toUnixSeconds } from './meinstadt.transformer';
|
|
14
|
+
import { GoogleEnvkvAttribute, envkvCo2Value, envkvFuelConsumptionValue } from './google-feed-envkv';
|
|
15
|
+
|
|
16
|
+
// back_link parameters (ADV-324): the partner in `source`, so the reporting views, the CRM source and
|
|
17
|
+
// GA4 (utm_*) all see "autouncle". Legacy and utm pairs carry identical values (no parameterConflict).
|
|
18
|
+
// Paid / unpaid is a contract attribute and is not encoded here. Pending AutoUncle's approval.
|
|
19
|
+
export const AUTOUNCLE_LINK_PARAMS =
|
|
20
|
+
'source=autouncle&medium=listing&campaign=listing&utm_source=autouncle&utm_medium=listing&utm_campaign=listing';
|
|
21
|
+
|
|
22
|
+
// The EnVKV attributes the feed sends (fuel_economy, co2_emission); the gate knows more (range, kWh, …).
|
|
23
|
+
export const AUTOUNCLE_ENVKV_ATTRIBUTES: GoogleEnvkvAttribute[] = ['fuel_consumption', 'co2_emissions'];
|
|
14
24
|
|
|
15
25
|
export enum AutouncleExclusion {
|
|
16
26
|
duplicateId = 'duplicate_id',
|
|
@@ -39,6 +49,8 @@ export interface AutouncleMappingContext {
|
|
|
39
49
|
images: string[];
|
|
40
50
|
// German label of a selector (TranslatorExtended.translateVal)
|
|
41
51
|
translate: (key: string) => string;
|
|
52
|
+
// EnVKV attributes whose value is a CBD estimate and must not be published (publication gate, enforce)
|
|
53
|
+
withheldEnvkv?: GoogleEnvkvAttribute[];
|
|
42
54
|
}
|
|
43
55
|
|
|
44
56
|
export type AutouncleMapResult = { ok: true; item: AutouncleJsonItem } | { ok: false; reason: AutouncleExclusion };
|
|
@@ -179,11 +191,15 @@ export const mapAutouncleItem = (car: DataSchemes.Car, ctx: AutouncleMappingCont
|
|
|
179
191
|
const sellerId = toString(get(car, 'metaData.ownerId', '')).trim();
|
|
180
192
|
const price = getAutounclePrice(get(car, 'price.currentSalesPrice', null));
|
|
181
193
|
const firstRegistration = get(car, 'mainData.firstRegistration', null);
|
|
194
|
+
// EnVKV (ADV-322): the WLTP picks of the Google feed (legacy fields only as their fallback), never an
|
|
195
|
+
// estimated value. The energy class is derived from CO2, so it goes with a withheld CO2 value.
|
|
196
|
+
const withheld = ctx.withheldEnvkv ?? [];
|
|
197
|
+
const co2Withheld = withheld.includes('co2_emissions');
|
|
182
198
|
|
|
183
199
|
return {
|
|
184
200
|
ok: true,
|
|
185
201
|
item: {
|
|
186
|
-
back_link: `https://www.${ctx.domain}/search/vehicle/${autodeId}
|
|
202
|
+
back_link: `https://www.${ctx.domain}/search/vehicle/${autodeId}?${AUTOUNCLE_LINK_PARAMS}`,
|
|
187
203
|
make: toString(get(car, 'mainData.make', '')),
|
|
188
204
|
model_name: toString(get(car, 'mainData.model', '')),
|
|
189
205
|
model_variant: toString(get(car, 'mainData.subModel', '')),
|
|
@@ -210,10 +226,10 @@ export const mapAutouncleItem = (car: DataSchemes.Car, ctx: AutouncleMappingCont
|
|
|
210
226
|
seller_id: sellerId,
|
|
211
227
|
description: toString(get(car, 'offer.offerDescription', '')),
|
|
212
228
|
images: ctx.images.map(url => ({ url })),
|
|
213
|
-
fuel_economy: toString(
|
|
214
|
-
co2_emission: toString(
|
|
229
|
+
fuel_economy: withheld.includes('fuel_consumption') ? '' : toString(envkvFuelConsumptionValue(car) ?? ''),
|
|
230
|
+
co2_emission: co2Withheld ? '' : toString(envkvCo2Value(car) ?? ''),
|
|
215
231
|
// ADV-187: canonical CO2-derived WLTP class, not the unvalidated legacy NEFZ class.
|
|
216
|
-
co2_effiency_class: toString(get(car, 'environmentEmissions.wltpEnergyEfficiencyClass', '')),
|
|
232
|
+
co2_effiency_class: co2Withheld ? '' : toString(get(car, 'environmentEmissions.wltpEnergyEfficiencyClass', '')),
|
|
217
233
|
euro_emission_class: toString(get(car, 'environmentEmissions.emissionClass', '')),
|
|
218
234
|
enviromental_sticker: toString(get(car, 'environmentEmissions.emissionSticker', '')),
|
|
219
235
|
},
|
|
@@ -143,6 +143,16 @@ export function googleCo2(car: DataSchemes.Car): string | undefined {
|
|
|
143
143
|
return picked ? `${picked.value}g/km` : undefined;
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
+
// The same picks as plain numbers, for feeds that send values without units (AutoUncle, ADV-322).
|
|
147
|
+
// Fuel: undefined for a BEV. CO2: 0 for a BEV (no field behind it, so never gated).
|
|
148
|
+
export function envkvFuelConsumptionValue(car: DataSchemes.Car): number | undefined {
|
|
149
|
+
return pickFuelConsumption(car)?.value;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export function envkvCo2Value(car: DataSchemes.Car): number | undefined {
|
|
153
|
+
return isBev(car) ? 0 : pickCo2(car)?.value;
|
|
154
|
+
}
|
|
155
|
+
|
|
146
156
|
// The EnVKV attributes of the Google item (keys of GoogleXmlItem).
|
|
147
157
|
export type GoogleEnvkvAttribute =
|
|
148
158
|
| 'fuel_consumption'
|
|
@@ -7,7 +7,7 @@ import { tmpdir } from 'os';
|
|
|
7
7
|
import { join } from 'path';
|
|
8
8
|
|
|
9
9
|
import { FeedGuard, FeedGuardError, FeedGuardOptions } from '../src/services/feed-guard';
|
|
10
|
-
import { AutouncleFeedRun, AutouncleSourceCar } from '../src/services/autouncle-feed-run';
|
|
10
|
+
import { AutouncleFeedRun, AutouncleRunDeps, AutouncleSourceCar } from '../src/services/autouncle-feed-run';
|
|
11
11
|
import { toAutouncleImages } from '../src/transformers/autouncle.transformer';
|
|
12
12
|
import { selectOpenAiImageFields } from '../src/transformers/openai-image-fields';
|
|
13
13
|
import { buildImagesService } from './fixtures/images-service.harness';
|
|
@@ -63,7 +63,9 @@ const guardOptions = (world: World, extra: Partial<FeedGuardOptions> = {}): Feed
|
|
|
63
63
|
...extra,
|
|
64
64
|
});
|
|
65
65
|
|
|
66
|
-
|
|
66
|
+
type Gate = AutouncleRunDeps['gate'];
|
|
67
|
+
|
|
68
|
+
const run = async (world: World, { at = NOW, guard = {}, gate }: { at?: Date; guard?: Partial<FeedGuardOptions>; gate?: Gate } = {}) => {
|
|
67
69
|
const forEachChunk = async (fn: (cars: AutouncleSourceCar[]) => Promise<void>) => {
|
|
68
70
|
const names = (await fs.readdir(world.source)).sort((a, b) => Number(a.split('.')[1]) - Number(b.split('.')[1]));
|
|
69
71
|
for (const name of names) {
|
|
@@ -85,6 +87,7 @@ const run = async (world: World, { at = NOW, guard = {} }: { at?: Date; guard?:
|
|
|
85
87
|
imageStats: () => ({ ok: 1 }),
|
|
86
88
|
translate: key => key,
|
|
87
89
|
guard: new FeedGuard(guardOptions(world, guard)),
|
|
90
|
+
gate,
|
|
88
91
|
},
|
|
89
92
|
{
|
|
90
93
|
runId: 'test',
|
|
@@ -231,6 +234,26 @@ const tests: Array<[string, () => Promise<void>]> = [
|
|
|
231
234
|
assert.equal(await exists(world.json), false);
|
|
232
235
|
}],
|
|
233
236
|
|
|
237
|
+
['EnVKV gate: enforce sends estimated values empty, audit only counts them; off without a gate', async () => {
|
|
238
|
+
const world = await newWorld();
|
|
239
|
+
await writeSource(world, [source('v1'), source('v2')]);
|
|
240
|
+
// v1: the CO2 value it would send is a CBD estimate; the range is too, but the feed has no range key
|
|
241
|
+
const estimatedAttributes = (c: any): any[] => (c._id === 'v1' ? ['co2_emissions', 'electric_range'] : []);
|
|
242
|
+
|
|
243
|
+
const off = await run(world);
|
|
244
|
+
assert.deepEqual(off.envkv, { gate: 'off', estimated: {} });
|
|
245
|
+
|
|
246
|
+
const audit = await run(world, { at: new Date(NOW.getTime() + HOUR), gate: { mode: 'audit', estimatedAttributes } });
|
|
247
|
+
assert.deepEqual(audit.envkv, { gate: 'audit', estimated: { co2_emissions: 1 } });
|
|
248
|
+
assert.deepEqual((await readAds(world)).map((a: any) => a.co2_emission), ['137', '137']);
|
|
249
|
+
|
|
250
|
+
const enforce = await run(world, { at: new Date(NOW.getTime() + 2 * HOUR), gate: { mode: 'enforce', estimatedAttributes } });
|
|
251
|
+
assert.deepEqual(enforce.envkv, { gate: 'enforce', estimated: { co2_emissions: 1 } });
|
|
252
|
+
const ads = await readAds(world);
|
|
253
|
+
assert.deepEqual(ads.map((a: any) => [a.co2_emission, a.fuel_economy]), [['', '6.1'], ['137', '6.1']]);
|
|
254
|
+
assert.equal(ads.length, 2, 'a withheld value never drops the car');
|
|
255
|
+
}],
|
|
256
|
+
|
|
234
257
|
['second run with the same snapshot: byte-identical file (stable ids, order, links)', async () => {
|
|
235
258
|
const world = await newWorld();
|
|
236
259
|
await writeSource(world, [source('i1'), source('i2'), source('i3')]);
|