@bytebrand/export-cars 1.26.27 → 1.26.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +128 -0
- package/dist/configs/config.d.ts +2 -1
- package/dist/configs/config.js +1 -0
- package/dist/configs/config.js.map +1 -1
- package/dist/configs/google.config.js +16 -0
- package/dist/configs/google.config.js.map +1 -1
- package/dist/configs/index.d.ts +1 -0
- package/dist/configs/index.js +1 -0
- package/dist/configs/index.js.map +1 -1
- package/dist/configs/openai-ads.config.d.ts +22 -0
- package/dist/configs/openai-ads.config.js +56 -0
- package/dist/configs/openai-ads.config.js.map +1 -0
- package/dist/enums/export-type.enum.d.ts +2 -1
- package/dist/enums/export-type.enum.js +1 -0
- package/dist/enums/export-type.enum.js.map +1 -1
- package/dist/enums/image-type.enum.d.ts +2 -1
- package/dist/enums/image-type.enum.js +1 -0
- package/dist/enums/image-type.enum.js.map +1 -1
- package/dist/export.schema.d.ts +6 -2
- package/dist/export.schema.js +4 -0
- package/dist/export.schema.js.map +1 -1
- package/dist/exporters/google.exporter.js +12 -2
- package/dist/exporters/google.exporter.js.map +1 -1
- package/dist/exporters/index.d.ts +1 -0
- package/dist/exporters/index.js +1 -0
- package/dist/exporters/index.js.map +1 -1
- package/dist/exporters/openai-ads.exporter.d.ts +13 -0
- package/dist/exporters/openai-ads.exporter.js +91 -0
- package/dist/exporters/openai-ads.exporter.js.map +1 -0
- package/dist/formats/openai-ads-csv.builder.d.ts +15 -0
- package/dist/formats/openai-ads-csv.builder.js +48 -0
- package/dist/formats/openai-ads-csv.builder.js.map +1 -0
- package/dist/interfaces/openai-ads.interface.d.ts +35 -0
- package/dist/interfaces/openai-ads.interface.js +37 -0
- package/dist/interfaces/openai-ads.interface.js.map +1 -0
- package/dist/interfaces/params.interface.d.ts +20 -0
- package/dist/services/feed-guard.d.ts +44 -0
- package/dist/services/feed-guard.js +156 -0
- package/dist/services/feed-guard.js.map +1 -0
- package/dist/services/image-url-validator.service.d.ts +45 -0
- package/dist/services/image-url-validator.service.js +141 -0
- package/dist/services/image-url-validator.service.js.map +1 -0
- package/dist/services/images.service.d.ts +12 -0
- package/dist/services/images.service.js +46 -1
- package/dist/services/images.service.js.map +1 -1
- package/dist/services/index.d.ts +1 -0
- package/dist/services/index.js +1 -0
- package/dist/services/index.js.map +1 -1
- package/dist/services/openai-ads-feed-run.d.ts +74 -0
- package/dist/services/openai-ads-feed-run.js +151 -0
- package/dist/services/openai-ads-feed-run.js.map +1 -0
- package/dist/services/openai-ads-ledger.d.ts +19 -0
- package/dist/services/openai-ads-ledger.js +72 -0
- package/dist/services/openai-ads-ledger.js.map +1 -0
- package/dist/services/publication-gate.d.ts +15 -0
- package/dist/services/publication-gate.js +107 -0
- package/dist/services/publication-gate.js.map +1 -0
- package/dist/transformers/google-feed-envkv.d.ts +2 -0
- package/dist/transformers/google-feed-envkv.js +62 -31
- package/dist/transformers/google-feed-envkv.js.map +1 -1
- package/dist/transformers/google-feed-item-row.d.ts +6 -1
- package/dist/transformers/google-feed-item-row.js +3 -1
- package/dist/transformers/google-feed-item-row.js.map +1 -1
- package/dist/transformers/openai-ads.transformer.d.ts +28 -0
- package/dist/transformers/openai-ads.transformer.js +144 -0
- package/dist/transformers/openai-ads.transformer.js.map +1 -0
- package/dist/transformers/openai-image-fields.d.ts +16 -0
- package/dist/transformers/openai-image-fields.js +34 -0
- package/dist/transformers/openai-image-fields.js.map +1 -0
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/openai-ads/AUDIT.md +235 -0
- package/docs/openai-ads/RUNBOOK.md +94 -0
- package/package.json +64 -64
- package/schema/feed-items.json +134 -122
- package/src/configs/config.ts +99 -97
- package/src/configs/google.config.ts +41 -26
- package/src/configs/index.ts +1 -0
- package/src/configs/openai-ads.config.ts +70 -0
- package/src/enums/export-type.enum.ts +2 -0
- package/src/enums/image-type.enum.ts +5 -0
- package/src/export.command.ts +38 -38
- package/src/export.schema.ts +6 -0
- package/src/exporters/google.exporter.ts +67 -56
- package/src/exporters/index.ts +1 -0
- package/src/exporters/openai-ads.exporter.ts +93 -0
- package/src/formats/openai-ads-csv.builder.ts +55 -0
- package/src/interfaces/openai-ads.interface.ts +64 -0
- package/src/interfaces/params.interface.ts +56 -29
- package/src/services/feed-guard.ts +219 -0
- package/src/services/feed-items-bigquery.writer.ts +101 -101
- package/src/services/image-url-validator.service.ts +181 -0
- package/src/services/images.service.ts +72 -0
- package/src/services/index.ts +1 -0
- package/src/services/openai-ads-feed-run.ts +226 -0
- package/src/services/openai-ads-ledger.ts +116 -0
- package/src/services/publication-gate.ts +119 -0
- package/src/transformers/google-feed-envkv.ts +81 -33
- package/src/transformers/google-feed-item-row.ts +78 -71
- package/src/transformers/openai-ads.transformer.ts +211 -0
- package/src/transformers/openai-image-fields.ts +60 -0
- package/test/clean-images.test.ts +86 -0
- package/test/fixtures/images-service.harness.ts +32 -0
- package/test/fixtures/images.golden.json +218 -0
- package/test/fixtures/imbo-images.fixture.ts +77 -0
- package/test/fixtures/openai-ads.fixture.ts +89 -0
- package/test/google-feed-item-row.test.ts +135 -121
- package/test/image-url-validator.test.ts +110 -0
- package/test/images-regression.test.ts +23 -0
- package/test/openai-ads-feed-run.test.ts +278 -0
- package/test/openai-ads-mapping.test.ts +136 -0
- package/test/openai-image-fields.test.ts +99 -0
- package/test/publication-gate.test.ts +86 -0
|
@@ -70,51 +70,99 @@ const isConfirmedPlugIn = (car: DataSchemes.Car): boolean => isPlugIn(car) && !!
|
|
|
70
70
|
|
|
71
71
|
// Missing or 0 is omitted instead of sending "0l/100km". BEVs carry no fuel consumption
|
|
72
72
|
// (CBD mirrors kWh into consumptionCombined for them).
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
value = positive(get(car, 'consumption.consumptionCombined')); // unconfirmed plug-in: as before
|
|
80
|
-
} else {
|
|
81
|
-
// ICE / HEV: the WLTP combined value the VDP and EnVKV use (EnVKV stage D3), the legacy field as fallback
|
|
82
|
-
value = positive(get(car, 'consumption.wltpCombined')) ?? positive(get(car, 'consumption.consumptionCombined'));
|
|
83
|
-
}
|
|
84
|
-
return value ? `${value}l/100km` : undefined;
|
|
73
|
+
//
|
|
74
|
+
// Each attribute is picked from the first positive field of a list; the pick keeps the field so the
|
|
75
|
+
// publication gate (EnVKV C8) can tell whether the sent value was estimated by CBD.
|
|
76
|
+
interface Picked {
|
|
77
|
+
field: string;
|
|
78
|
+
value: number;
|
|
85
79
|
}
|
|
86
80
|
|
|
81
|
+
const firstPositive = (car: DataSchemes.Car, fields: string[]): Picked | undefined => {
|
|
82
|
+
for (const field of fields) {
|
|
83
|
+
const value = positive(get(car, field));
|
|
84
|
+
if (value) return { field, value };
|
|
85
|
+
}
|
|
86
|
+
return undefined;
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
const pickFuelConsumption = (car: DataSchemes.Car): Picked | undefined => {
|
|
90
|
+
if (isBev(car)) return undefined;
|
|
91
|
+
if (isConfirmedPlugIn(car)) return firstPositive(car, ['consumption.wltpWeightedCombined', 'consumption.consumptionCombined']);
|
|
92
|
+
if (isPlugIn(car)) return firstPositive(car, ['consumption.consumptionCombined']); // unconfirmed plug-in: as before
|
|
93
|
+
// ICE / HEV: the WLTP combined value the VDP and EnVKV use (EnVKV stage D3), the legacy field as fallback
|
|
94
|
+
return firstPositive(car, ['consumption.wltpCombined', 'consumption.consumptionCombined']);
|
|
95
|
+
};
|
|
96
|
+
|
|
87
97
|
// Confirmed plug-in: wltpCombined, no fallback (the legacy combined value is close to the weighted
|
|
88
98
|
// one, sending it here would repeat the weighted figure). Unconfirmed plug-in: the stored combined
|
|
89
99
|
// value, as before.
|
|
90
|
-
|
|
100
|
+
const pickFuelConsumptionDischarged = (car: DataSchemes.Car): Picked | undefined => {
|
|
91
101
|
if (!isPlugIn(car)) return undefined;
|
|
92
|
-
|
|
93
|
-
?
|
|
94
|
-
:
|
|
95
|
-
|
|
96
|
-
}
|
|
102
|
+
return isConfirmedPlugIn(car)
|
|
103
|
+
? firstPositive(car, ['consumption.wltpCombined'])
|
|
104
|
+
: firstPositive(car, ['consumption.consumptionCombined', 'consumption.wltpCombined']);
|
|
105
|
+
};
|
|
97
106
|
|
|
98
107
|
// BEV: combined consumption. Confirmed plug-in: weighted combined. Unconfirmed plug-in: the stored
|
|
99
108
|
// electric value, as before. Nothing for other cars.
|
|
109
|
+
const pickEnergyConsumption = (car: DataSchemes.Car): Picked | undefined => {
|
|
110
|
+
if (isConfirmedPlugIn(car)) return firstPositive(car, ['consumption.wltpWeightedPowerCombined']);
|
|
111
|
+
if (isBev(car) || isPlugIn(car)) return firstPositive(car, ['consumption.wltpPowerCombined', 'consumption.consumptionPowerCombined']);
|
|
112
|
+
return undefined;
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
// A BEV emits 0 g/km by definition (no field behind it); a confirmed plug-in and ICE / HEV send the
|
|
116
|
+
// WLTP value (weighted for the plug-in); an unconfirmed plug-in keeps the stored legacy value, as before.
|
|
117
|
+
const pickCo2 = (car: DataSchemes.Car): Picked | undefined => {
|
|
118
|
+
if (isBev(car)) return undefined;
|
|
119
|
+
return isPlugIn(car) && !isConfirmedPlugIn(car)
|
|
120
|
+
? firstPositive(car, ['environmentEmissions.co2'])
|
|
121
|
+
: firstPositive(car, ['environmentEmissions.wltpCo2', 'environmentEmissions.co2']);
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
export function googleFuelConsumption(car: DataSchemes.Car): string | undefined {
|
|
125
|
+
const picked = pickFuelConsumption(car);
|
|
126
|
+
return picked ? `${picked.value}l/100km` : undefined;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export function googleFuelConsumptionDischarged(car: DataSchemes.Car): string | undefined {
|
|
130
|
+
const picked = pickFuelConsumptionDischarged(car);
|
|
131
|
+
return picked ? `${picked.value}l/100km` : undefined;
|
|
132
|
+
}
|
|
133
|
+
|
|
100
134
|
export function googleEnergyConsumption(car: DataSchemes.Car): string | undefined {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
value = positive(get(car, 'consumption.wltpWeightedPowerCombined'));
|
|
104
|
-
} else if (isBev(car) || isPlugIn(car)) {
|
|
105
|
-
value = positive(get(car, 'consumption.wltpPowerCombined')) ?? positive(get(car, 'consumption.consumptionPowerCombined'));
|
|
106
|
-
}
|
|
107
|
-
return value ? `${value}kWh/100km` : undefined;
|
|
135
|
+
const picked = pickEnergyConsumption(car);
|
|
136
|
+
return picked ? `${picked.value}kWh/100km` : undefined;
|
|
108
137
|
}
|
|
109
138
|
|
|
110
|
-
// A
|
|
111
|
-
// the stored value. A missing or 0 value is omitted.
|
|
139
|
+
// A missing or 0 value is omitted.
|
|
112
140
|
export function googleCo2(car: DataSchemes.Car): string | undefined {
|
|
113
141
|
if (isBev(car)) return '0g/km';
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
142
|
+
const picked = pickCo2(car);
|
|
143
|
+
return picked ? `${picked.value}g/km` : undefined;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// The EnVKV attributes of the Google item (keys of GoogleXmlItem).
|
|
147
|
+
export type GoogleEnvkvAttribute =
|
|
148
|
+
| 'fuel_consumption'
|
|
149
|
+
| 'fuel_consumption_discharged_battery'
|
|
150
|
+
| 'energy_consumption'
|
|
151
|
+
| 'electric_range'
|
|
152
|
+
| 'co2_emissions';
|
|
153
|
+
|
|
154
|
+
// For every EnVKV attribute the item sends: the car field its value came from.
|
|
155
|
+
export function googleEnvkvSourceFields(car: DataSchemes.Car): Partial<Record<GoogleEnvkvAttribute, string>> {
|
|
156
|
+
const picks: Record<GoogleEnvkvAttribute, { field: string } | undefined> = {
|
|
157
|
+
fuel_consumption: pickFuelConsumption(car),
|
|
158
|
+
fuel_consumption_discharged_battery: pickFuelConsumptionDischarged(car),
|
|
159
|
+
energy_consumption: pickEnergyConsumption(car),
|
|
160
|
+
electric_range: pickElectricRange(car),
|
|
161
|
+
co2_emissions: pickCo2(car),
|
|
162
|
+
};
|
|
163
|
+
const fields: Partial<Record<GoogleEnvkvAttribute, string>> = {};
|
|
164
|
+
for (const [attribute, picked] of Object.entries(picks)) {
|
|
165
|
+
if (picked) fields[attribute as GoogleEnvkvAttribute] = picked.field;
|
|
166
|
+
}
|
|
167
|
+
return fields;
|
|
120
168
|
}
|
|
@@ -1,71 +1,78 @@
|
|
|
1
|
-
import { get, isBoolean } from 'lodash';
|
|
2
|
-
import { DataSchemes } from '@bytebrand/car-schema';
|
|
3
|
-
import { GoogleXmlItem } from '../interfaces';
|
|
4
|
-
import { googleElectricRangeSource } from './google-feed-envkv';
|
|
5
|
-
|
|
6
|
-
// One BigQuery row per item the Google feed sent (schema/feed-items.json), so a Merchant Center
|
|
7
|
-
// issue can be traced back to the exact values of a run. The attribute strings are the ones in the
|
|
8
|
-
// XML; null means the attribute was omitted.
|
|
9
|
-
export interface FeedItemRow {
|
|
10
|
-
partitionDay: string;
|
|
11
|
-
exportedAt: string;
|
|
12
|
-
feed: string;
|
|
13
|
-
runId: string | null;
|
|
14
|
-
itemId: string;
|
|
15
|
-
vehicleId: string | null;
|
|
16
|
-
make: string | null;
|
|
17
|
-
model: string | null;
|
|
18
|
-
condition: string | null;
|
|
19
|
-
fuel: string | null;
|
|
20
|
-
hybridPlugin: boolean | null;
|
|
21
|
-
engine: string | null;
|
|
22
|
-
electricRange: string | null;
|
|
23
|
-
fuelConsumption: string | null;
|
|
24
|
-
fuelConsumptionDischarged: string | null;
|
|
25
|
-
energyConsumption: string | null;
|
|
26
|
-
co2Emissions: string | null;
|
|
27
|
-
rangeSourceField: string | null;
|
|
28
|
-
price: number | null;
|
|
29
|
-
availability: string | null;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
1
|
+
import { get, isBoolean } from 'lodash';
|
|
2
|
+
import { DataSchemes } from '@bytebrand/car-schema';
|
|
3
|
+
import { GoogleXmlItem } from '../interfaces';
|
|
4
|
+
import { googleElectricRangeSource } from './google-feed-envkv';
|
|
5
|
+
|
|
6
|
+
// One BigQuery row per item the Google feed sent (schema/feed-items.json), so a Merchant Center
|
|
7
|
+
// issue can be traced back to the exact values of a run. The attribute strings are the ones in the
|
|
8
|
+
// XML; null means the attribute was omitted.
|
|
9
|
+
export interface FeedItemRow {
|
|
10
|
+
partitionDay: string;
|
|
11
|
+
exportedAt: string;
|
|
12
|
+
feed: string;
|
|
13
|
+
runId: string | null;
|
|
14
|
+
itemId: string;
|
|
15
|
+
vehicleId: string | null;
|
|
16
|
+
make: string | null;
|
|
17
|
+
model: string | null;
|
|
18
|
+
condition: string | null;
|
|
19
|
+
fuel: string | null;
|
|
20
|
+
hybridPlugin: boolean | null;
|
|
21
|
+
engine: string | null;
|
|
22
|
+
electricRange: string | null;
|
|
23
|
+
fuelConsumption: string | null;
|
|
24
|
+
fuelConsumptionDischarged: string | null;
|
|
25
|
+
energyConsumption: string | null;
|
|
26
|
+
co2Emissions: string | null;
|
|
27
|
+
rangeSourceField: string | null;
|
|
28
|
+
price: number | null;
|
|
29
|
+
availability: string | null;
|
|
30
|
+
// EnVKV C8: gate mode of the run and the attributes whose value CBD estimated (comma-separated);
|
|
31
|
+
// in enforce mode these attributes were left out of the item.
|
|
32
|
+
publicationGate: string | null;
|
|
33
|
+
estimatedAttributes: string | null;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const orNull = (value: unknown): string | null =>
|
|
37
|
+
value === undefined || value === null || value === '' ? null : String(value);
|
|
38
|
+
|
|
39
|
+
// "12345.5 EUR" -> 12345.5
|
|
40
|
+
const sentPrice = (price: string | undefined): number | null => {
|
|
41
|
+
const n = parseFloat(price);
|
|
42
|
+
return Number.isFinite(n) ? n : null;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
// exportedAt is the run's start, shared by all rows of a run; partitionDay is its UTC day.
|
|
46
|
+
export function buildGoogleFeedItemRow(
|
|
47
|
+
car: DataSchemes.Car,
|
|
48
|
+
item: GoogleXmlItem,
|
|
49
|
+
runId: string | null,
|
|
50
|
+
exportedAt: Date,
|
|
51
|
+
gate: { mode: string; estimatedAttributes: string[] } | null = null,
|
|
52
|
+
): FeedItemRow {
|
|
53
|
+
const hybridPlugin = get(car, 'engineData.hybridPlugin');
|
|
54
|
+
return {
|
|
55
|
+
partitionDay: exportedAt.toISOString().slice(0, 10),
|
|
56
|
+
exportedAt: exportedAt.toISOString(),
|
|
57
|
+
feed: 'google',
|
|
58
|
+
runId: orNull(runId),
|
|
59
|
+
itemId: String(item.id),
|
|
60
|
+
vehicleId: orNull(get(car, 'metaData.vehicleId')),
|
|
61
|
+
make: orNull(item.brand),
|
|
62
|
+
model: orNull(item.model),
|
|
63
|
+
condition: orNull(item.condition),
|
|
64
|
+
fuel: orNull(get(car, 'consumption.fuel')),
|
|
65
|
+
hybridPlugin: isBoolean(hybridPlugin) ? hybridPlugin : null,
|
|
66
|
+
engine: orNull(item.engine),
|
|
67
|
+
electricRange: orNull(item.electric_range),
|
|
68
|
+
fuelConsumption: orNull(item.fuel_consumption),
|
|
69
|
+
fuelConsumptionDischarged: orNull(item.fuel_consumption_discharged_battery),
|
|
70
|
+
energyConsumption: orNull(item.energy_consumption),
|
|
71
|
+
co2Emissions: orNull(item.co2_emissions),
|
|
72
|
+
rangeSourceField: item.electric_range ? googleElectricRangeSource(car) : null,
|
|
73
|
+
price: sentPrice(item.price),
|
|
74
|
+
availability: orNull(item.availability),
|
|
75
|
+
publicationGate: gate ? gate.mode : null,
|
|
76
|
+
estimatedAttributes: gate && gate.estimatedAttributes.length ? gate.estimatedAttributes.join(',') : null,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import { get, toNumber, toString, trim, truncate } from 'lodash';
|
|
2
|
+
import { DataSchemes } from '@bytebrand/car-schema';
|
|
3
|
+
|
|
4
|
+
import { normalizeFeedText } from '../utils';
|
|
5
|
+
import {
|
|
6
|
+
OpenAiAdsCondition,
|
|
7
|
+
OpenAiAdsExclusion,
|
|
8
|
+
OpenAiAdsOutOfStockReason,
|
|
9
|
+
OpenAiAdsRow,
|
|
10
|
+
} from '../interfaces/openai-ads.interface';
|
|
11
|
+
import { OpenAiImageFields, encodeCommasInUrl } from './openai-image-fields';
|
|
12
|
+
|
|
13
|
+
// OpenAI product file limits (guidance): title ≤ 150, description ≤ 5,000 characters.
|
|
14
|
+
export const OPENAI_TITLE_MAX_LENGTH = 150;
|
|
15
|
+
export const OPENAI_DESCRIPTION_MAX_LENGTH = 5000;
|
|
16
|
+
|
|
17
|
+
// Placeholder makes from import mapping (SONSTIGE = "other") — not a real manufacturer, so the
|
|
18
|
+
// required `brand` would be invented.
|
|
19
|
+
const PLACEHOLDER_MAKES = /^(sonstige|sonstiges|andere|other|unknown|unbekannt)$/i;
|
|
20
|
+
|
|
21
|
+
const OFFER_AVAILABILITY_OUT_OF_STOCK: Record<string, OpenAiAdsOutOfStockReason> = {
|
|
22
|
+
selector_offerAvailability_reserved: OpenAiAdsOutOfStockReason.reserved,
|
|
23
|
+
selector_offerAvailability_sold: OpenAiAdsOutOfStockReason.soldFlag,
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export interface OpenAiAdsMappingContext {
|
|
27
|
+
domain: string;
|
|
28
|
+
sellerName: string;
|
|
29
|
+
// German label of a schema selector (TranslatorExtended.translateVal); returns the key itself
|
|
30
|
+
// when it has no label
|
|
31
|
+
translate: (key: string) => string;
|
|
32
|
+
now: Date;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Row without the media columns: mapped first so cheap exclusions happen before any image probe.
|
|
36
|
+
export type OpenAiAdsBaseRow = Omit<OpenAiAdsRow, 'image_url' | 'additional_image_urls'>;
|
|
37
|
+
|
|
38
|
+
export type OpenAiAdsMapResult =
|
|
39
|
+
| { ok: true; row: OpenAiAdsBaseRow; outOfStockReason?: OpenAiAdsOutOfStockReason }
|
|
40
|
+
| { ok: false; reason: OpenAiAdsExclusion };
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Maps one CBD car (an online car of the base export snapshot) to the OpenAI columns.
|
|
44
|
+
* The row comes out in_stock; `outOfStockReason` is set when the car is online but flagged
|
|
45
|
+
* reserved/sold — the exporter then sends it as out_of_stock (only if it was advertised before).
|
|
46
|
+
*/
|
|
47
|
+
export const mapOpenAiAdsVehicle = (car: DataSchemes.Car, ctx: OpenAiAdsMappingContext): OpenAiAdsMapResult => {
|
|
48
|
+
const itemId = toString(get(car, '_id', '')).trim();
|
|
49
|
+
if (!itemId) {
|
|
50
|
+
return { ok: false, reason: OpenAiAdsExclusion.noId };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const price = formatOpenAiPrice(get(car, 'price.currentSalesPrice', null));
|
|
54
|
+
if (!price) {
|
|
55
|
+
return { ok: false, reason: OpenAiAdsExclusion.invalidPrice };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const brand = trim(toString(get(car, 'mainData.make', '')));
|
|
59
|
+
if (!brand || PLACEHOLDER_MAKES.test(brand)) {
|
|
60
|
+
return { ok: false, reason: OpenAiAdsExclusion.noBrand };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const title = buildOpenAiTitle(car);
|
|
64
|
+
if (!title) {
|
|
65
|
+
return { ok: false, reason: OpenAiAdsExclusion.noTitle };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const condition = resolveOpenAiCondition(car, ctx.now);
|
|
69
|
+
if (!condition) {
|
|
70
|
+
return { ok: false, reason: OpenAiAdsExclusion.conditionUnknown };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const row: OpenAiAdsBaseRow = {
|
|
74
|
+
item_id: itemId,
|
|
75
|
+
title,
|
|
76
|
+
description: buildOpenAiDescription(car, title, condition, ctx),
|
|
77
|
+
url: `https://www.${ctx.domain}/search/vehicle/${itemId}`,
|
|
78
|
+
brand,
|
|
79
|
+
seller_name: ctx.sellerName,
|
|
80
|
+
availability: 'in_stock',
|
|
81
|
+
price,
|
|
82
|
+
condition,
|
|
83
|
+
is_ads_eligible: true,
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
const outOfStockReason = OFFER_AVAILABILITY_OUT_OF_STOCK[toString(get(car, 'offer.offerAvailability', ''))];
|
|
87
|
+
return outOfStockReason ? { ok: true, row, outOfStockReason } : { ok: true, row };
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
export const withOpenAiImages = (row: OpenAiAdsBaseRow, images: OpenAiImageFields): OpenAiAdsRow => ({
|
|
91
|
+
...row,
|
|
92
|
+
image_url: images.image_url,
|
|
93
|
+
// the CSV writer quotes the cell; commas inside a URL are %2C so the list stays parseable
|
|
94
|
+
additional_image_urls: images.additional_image_urls.map(encodeCommasInUrl).join(','),
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
// The same vehicle, switched off: kept in the file so OpenAI stops delivering it instead of
|
|
98
|
+
// relying on the row disappearing (not documented as a removal).
|
|
99
|
+
export const asOutOfStock = (row: OpenAiAdsRow): OpenAiAdsRow => ({
|
|
100
|
+
...row,
|
|
101
|
+
availability: 'out_of_stock',
|
|
102
|
+
is_ads_eligible: false,
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
// Purchase price (price.currentSalesPrice, EUR gross) in major units with exactly two decimals and
|
|
106
|
+
// no thousands separator: 17990 -> "17990.00 EUR". Never a monthly financing/leasing rate.
|
|
107
|
+
// null = not a valid positive price; the car is not exported.
|
|
108
|
+
export const formatOpenAiPrice = (value: unknown): string | null => {
|
|
109
|
+
if (value === null || value === undefined || value === '' || typeof value === 'boolean') {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
const amount = toNumber(value);
|
|
113
|
+
if (!Number.isFinite(amount) || amount <= 0) {
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
return `${amount.toFixed(2)} EUR`;
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
// Integer minor units (cents) for the deferred delta API: 17990.5 -> 1799050.
|
|
120
|
+
export const toOpenAiMinorUnits = (value: unknown): number | null => {
|
|
121
|
+
const formatted = formatOpenAiPrice(value);
|
|
122
|
+
return formatted ? Math.round(toNumber(formatted.split(' ')[0]) * 100) : null;
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
// "<make> <model> <variant>" as delivered (German market names), dealer junk punctuation removed
|
|
126
|
+
// with the same normalizer as the Meta feed, cut at a word boundary. No HTML escaping: CSV quoting
|
|
127
|
+
// handles every character.
|
|
128
|
+
export const buildOpenAiTitle = (car: DataSchemes.Car): string => {
|
|
129
|
+
const parts = ['mainData.make', 'mainData.model', 'mainData.subModel'].map(path => toString(get(car, path, '')));
|
|
130
|
+
const title = normalizeFeedText(parts.join(' '));
|
|
131
|
+
return truncate(title, { length: OPENAI_TITLE_MAX_LENGTH, separator: ' ', omission: '' }).trim();
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
// new/used from mainData.condition. selector_unknown is only "used" when the car has a first
|
|
135
|
+
// registration in the past (a registered car is used for advertising purposes); otherwise
|
|
136
|
+
// the car is excluded rather than defaulted — OpenAI treats an omitted condition as "new".
|
|
137
|
+
export const resolveOpenAiCondition = (car: DataSchemes.Car, now: Date): OpenAiAdsCondition | null => {
|
|
138
|
+
const condition = toString(get(car, 'mainData.condition', ''));
|
|
139
|
+
if (condition === 'selector_condition_new') {
|
|
140
|
+
return 'new';
|
|
141
|
+
}
|
|
142
|
+
if (condition === 'selector_condition_used') {
|
|
143
|
+
return 'used';
|
|
144
|
+
}
|
|
145
|
+
const firstRegistration = toNumber(get(car, 'mainData.firstRegistration', 0));
|
|
146
|
+
return firstRegistration > 0 && firstRegistration * 1000 <= now.getTime() ? 'used' : null;
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Plain German text from structured vehicle data only. Dealer summary templates (Google feed) are
|
|
151
|
+
* not used: their content is free text we cannot check for finance claims. No rates, no prices,
|
|
152
|
+
* no marketing promises.
|
|
153
|
+
* e.g. "Gebrauchtwagen: VW Golf 1.5 TSI Life. Erstzulassung: 03/2021 · Kilometerstand: 45.000 km · …"
|
|
154
|
+
*/
|
|
155
|
+
export const buildOpenAiDescription = (
|
|
156
|
+
car: DataSchemes.Car,
|
|
157
|
+
title: string,
|
|
158
|
+
condition: OpenAiAdsCondition,
|
|
159
|
+
ctx: OpenAiAdsMappingContext,
|
|
160
|
+
): string => {
|
|
161
|
+
const label = (path: string): string | null => {
|
|
162
|
+
const key = toString(get(car, path, ''));
|
|
163
|
+
if (!key || key === 'selector_unknown') {
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
const value = trim(ctx.translate(key));
|
|
167
|
+
return value && !value.startsWith('selector_') && value !== 'Unbekannt' ? value : null;
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
const facts: Array<[string, string | null]> = [
|
|
171
|
+
['Erstzulassung', formatFirstRegistration(get(car, 'mainData.firstRegistration'))],
|
|
172
|
+
['Kilometerstand', formatMileage(get(car, 'mainData.mileage'))],
|
|
173
|
+
['Kraftstoff', label('consumption.fuel')],
|
|
174
|
+
['Getriebe', label('driveSuspension.gearbox')],
|
|
175
|
+
['Leistung', formatPower(get(car, 'engineData.powerKW'), get(car, 'engineData.powerPS'))],
|
|
176
|
+
['Karosserie', label('mainData.category')],
|
|
177
|
+
['Außenfarbe', label('exterior.exteriorColor')],
|
|
178
|
+
];
|
|
179
|
+
|
|
180
|
+
const intro = `${condition === 'new' ? 'Neuwagen' : 'Gebrauchtwagen'}: ${title}.`;
|
|
181
|
+
const details = facts.filter(([, value]) => value).map(([name, value]) => `${name}: ${value}`).join(' · ');
|
|
182
|
+
const text = (details ? `${intro} ${details}` : intro).replace(/\s+/g, ' ').trim();
|
|
183
|
+
return truncate(text, { length: OPENAI_DESCRIPTION_MAX_LENGTH, separator: ' ', omission: '' }).trim();
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
const DE_NUMBER = new Intl.NumberFormat('de-DE', { maximumFractionDigits: 0 });
|
|
187
|
+
|
|
188
|
+
const formatFirstRegistration = (value: unknown): string | null => {
|
|
189
|
+
const seconds = toNumber(value);
|
|
190
|
+
if (!Number.isFinite(seconds) || seconds <= 0) {
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
const date = new Date(seconds * 1000);
|
|
194
|
+
return `${String(date.getUTCMonth() + 1).padStart(2, '0')}/${date.getUTCFullYear()}`;
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
// Real odometer value — no 901/1001 km floors as in the Meta/Google feeds.
|
|
198
|
+
const formatMileage = (value: unknown): string | null => {
|
|
199
|
+
const km = toNumber(value);
|
|
200
|
+
return Number.isFinite(km) && km > 0 ? `${DE_NUMBER.format(km)} km` : null;
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
const formatPower = (kw: unknown, ps: unknown): string | null => {
|
|
204
|
+
const powerKw = toNumber(kw);
|
|
205
|
+
if (!Number.isFinite(powerKw) || powerKw <= 0) {
|
|
206
|
+
return null;
|
|
207
|
+
}
|
|
208
|
+
const powerPs = toNumber(ps);
|
|
209
|
+
const psPart = Number.isFinite(powerPs) && powerPs > 0 ? ` (${DE_NUMBER.format(powerPs)} PS)` : '';
|
|
210
|
+
return `${DE_NUMBER.format(powerKw)} kW${psPart}`;
|
|
211
|
+
};
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { CleanImage } from '../services/images.service';
|
|
2
|
+
import { ImageUrlCheck, ImageUrlValidator } from '../services/image-url-validator.service';
|
|
3
|
+
|
|
4
|
+
// OpenAI product file (Stable schema) media fields. Not the Google-compatible image_link /
|
|
5
|
+
// additional_image_link pair — the two feed formats must not be mixed.
|
|
6
|
+
export interface OpenAiImageFields {
|
|
7
|
+
image_url: string;
|
|
8
|
+
additional_image_urls: string[];
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface OpenAiImageSelection {
|
|
12
|
+
fields: OpenAiImageFields | null;
|
|
13
|
+
// every clean image that failed validation, for the run log / reject tracking
|
|
14
|
+
rejected: ImageUrlCheck[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* image_url = first valid 3/4 exterior hero candidate; additional_image_urls = every other valid
|
|
19
|
+
* image in clean-profile order (exterior before interior), without an artificial count limit —
|
|
20
|
+
* OpenAI states none. Invalid optional URLs are dropped. No valid hero => fields null: the car must
|
|
21
|
+
* be skipped and counted, never sent with a placeholder or a non-hero view as its main image.
|
|
22
|
+
*/
|
|
23
|
+
export const selectOpenAiImageFields = (images: CleanImage[], checks: ImageUrlCheck[]): OpenAiImageSelection => {
|
|
24
|
+
const byUrl = new Map(checks.map(check => [check.url, check]));
|
|
25
|
+
const isValid = (image: CleanImage) => byUrl.get(image.url)?.ok === true;
|
|
26
|
+
const rejected = images.map(image => byUrl.get(image.url)).filter(check => check && !check.ok);
|
|
27
|
+
|
|
28
|
+
const hero = images.find(image => image.heroCandidate && isValid(image));
|
|
29
|
+
if (!hero) {
|
|
30
|
+
return { fields: null, rejected };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
return {
|
|
34
|
+
fields: {
|
|
35
|
+
image_url: hero.url,
|
|
36
|
+
additional_image_urls: images.filter(image => image !== hero && isValid(image)).map(image => image.url),
|
|
37
|
+
},
|
|
38
|
+
rejected,
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
export const resolveOpenAiImageFields = async (
|
|
43
|
+
images: CleanImage[],
|
|
44
|
+
validator: ImageUrlValidator,
|
|
45
|
+
): Promise<OpenAiImageSelection> => {
|
|
46
|
+
const checks = await validator.validateAll(images.map(image => image.url));
|
|
47
|
+
return selectOpenAiImageFields(images, checks);
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
// Percent-encode commas inside a URL (spec: "Percent-encode commas inside a URL as %2C").
|
|
51
|
+
export const encodeCommasInUrl = (url: string): string => url.replace(/,/g, '%2C');
|
|
52
|
+
|
|
53
|
+
// additional_image_urls as ONE CSV cell: comma-separated (no spaces, no semicolons), commas inside
|
|
54
|
+
// URLs encoded, quoted for the CSV writer. JSONL keeps the plain array instead.
|
|
55
|
+
export const serializeAdditionalImageUrlsForCsv = (urls: string[]): string =>
|
|
56
|
+
quoteCsvCell(urls.map(encodeCommasInUrl).join(','));
|
|
57
|
+
|
|
58
|
+
export const serializeAdditionalImageUrlsForJsonl = (urls: string[]): string[] => urls.map(encodeCommasInUrl);
|
|
59
|
+
|
|
60
|
+
export const quoteCsvCell = (value: string): string => `"${value.replace(/"/g, '""')}"`;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// Run: npm test (ts-node, no test framework needed)
|
|
2
|
+
import * as assert from 'node:assert/strict';
|
|
3
|
+
import { ImageType } from '../src/enums';
|
|
4
|
+
import { buildImagesService } from './fixtures/images-service.harness';
|
|
5
|
+
import { IMBO_FIXTURES, cloneImages } from './fixtures/imbo-images.fixture';
|
|
6
|
+
|
|
7
|
+
const service = buildImagesService();
|
|
8
|
+
const clean = (name: string) => {
|
|
9
|
+
const { carId, images } = IMBO_FIXTURES[name];
|
|
10
|
+
return service.getCleanImages(carId, cloneImages(images));
|
|
11
|
+
};
|
|
12
|
+
const ids = (name: string) => clean(name).map(i => i.imageIdentifier);
|
|
13
|
+
|
|
14
|
+
const cases: Array<[string, () => void | Promise<void>]> = [
|
|
15
|
+
['3/4 exterior hero first, then exterior, then interior; feat/conext/unknown dropped', () => {
|
|
16
|
+
assert.deepEqual(ids('full'), [
|
|
17
|
+
'extRight0001', // ext_f_right (hero)
|
|
18
|
+
'extCowl00001', 'extLeft00001', 'extSide00001', 'extBack00001', 'extOpened001',
|
|
19
|
+
'intDash00001', 'intSeats0001', 'intTrunk0001',
|
|
20
|
+
]);
|
|
21
|
+
}],
|
|
22
|
+
['explicit JPEG variant on the images host, never a placeholder', () => {
|
|
23
|
+
const urls = clean('full').map(i => i.url);
|
|
24
|
+
assert.equal(urls[0], 'https://images.auto.de/carimage/2ce1d71f-851f-4123-ae16-187ddd7923c5/extRight0001/large-cached.jpeg');
|
|
25
|
+
assert.ok(urls.every(u => u.endsWith('/large-cached.jpeg')));
|
|
26
|
+
assert.ok(urls.every(u => !u.includes('/noimage/')));
|
|
27
|
+
}],
|
|
28
|
+
['hero candidates are only the 3/4 front views', () => {
|
|
29
|
+
assert.deepEqual(clean('full').filter(i => i.heroCandidate).map(i => i.imageCode), ['ext_f_right', 'ext_f_left']);
|
|
30
|
+
}],
|
|
31
|
+
['no ext_f_right: ext_f_left becomes the hero', () => {
|
|
32
|
+
assert.deepEqual(ids('leftOnly'), ['extLeft00002', 'extBack00002', 'intDash00002']);
|
|
33
|
+
}],
|
|
34
|
+
['ext_f_left is lifted above ext_f_cowl when it is the hero', () => {
|
|
35
|
+
const images = [
|
|
36
|
+
{ imageIdentifier: 'cowl', imageCodePredicted: 'ext_f_cowl' },
|
|
37
|
+
{ imageIdentifier: 'left', imageCodePredicted: 'ext_f_left' },
|
|
38
|
+
];
|
|
39
|
+
assert.deepEqual(service.getCleanImages('car', images).map(i => i.imageIdentifier), ['left', 'cowl']);
|
|
40
|
+
}],
|
|
41
|
+
['no 3/4 exterior shot: no images at all (car is skipped, no fallback hero)', () => {
|
|
42
|
+
assert.deepEqual(clean('noHero'), []);
|
|
43
|
+
assert.deepEqual(clean('onlyUnknown'), []);
|
|
44
|
+
assert.deepEqual(clean('empty'), []);
|
|
45
|
+
assert.deepEqual(service.getCleanImages('car', undefined), []);
|
|
46
|
+
}],
|
|
47
|
+
['repeated imageIdentifier appears once; distinct shots with the same code keep Imbo order', () => {
|
|
48
|
+
assert.deepEqual(ids('duplicates'), ['extRight0004', 'extRight0005', 'intDash00004']);
|
|
49
|
+
}],
|
|
50
|
+
['images without imageCodePredicted or imageIdentifier are ignored', () => {
|
|
51
|
+
const images: any[] = [
|
|
52
|
+
{ imageIdentifier: 'noCode' },
|
|
53
|
+
{ imageCodePredicted: 'ext_f_right' },
|
|
54
|
+
{ imageIdentifier: 'right', imageCodePredicted: 'ext_f_right' },
|
|
55
|
+
];
|
|
56
|
+
assert.deepEqual(service.getCleanImages('car', images).map(i => i.imageIdentifier), ['right']);
|
|
57
|
+
}],
|
|
58
|
+
['deterministic and does not reorder the caller\'s image list', () => {
|
|
59
|
+
const images = cloneImages(IMBO_FIXTURES.full.images);
|
|
60
|
+
const before = images.map(i => i.imageIdentifier);
|
|
61
|
+
const first = service.getCleanImages('car', images);
|
|
62
|
+
assert.deepEqual(images.map(i => i.imageIdentifier), before);
|
|
63
|
+
assert.deepEqual(service.getCleanImages('car', images), first);
|
|
64
|
+
}],
|
|
65
|
+
['ImageType.clean via getImages: same URLs, no placeholders even with ignoreDefImages=false', async () => {
|
|
66
|
+
const { carId, images } = IMBO_FIXTURES.leftOnly;
|
|
67
|
+
const urls = await service.getImages({ _id: carId } as any, cloneImages(images), ImageType.clean, false);
|
|
68
|
+
assert.deepEqual(urls, service.getCleanImageUrls(carId, cloneImages(images)));
|
|
69
|
+
assert.equal(urls.length, 3);
|
|
70
|
+
assert.deepEqual(await service.getImages({ _id: 'x' } as any, [], ImageType.clean, false), []);
|
|
71
|
+
}],
|
|
72
|
+
];
|
|
73
|
+
|
|
74
|
+
(async () => {
|
|
75
|
+
let failed = 0;
|
|
76
|
+
for (const [name, fn] of cases) {
|
|
77
|
+
try {
|
|
78
|
+
await fn();
|
|
79
|
+
console.log(`ok ${name}`);
|
|
80
|
+
} catch (e) {
|
|
81
|
+
failed += 1;
|
|
82
|
+
console.error(`FAIL ${name}\n ${e.message}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (failed) process.exit(1);
|
|
86
|
+
})();
|