@bytebrand/export-cars 1.26.47 → 1.26.48

package/dist/services/ooyyo-feed-run.js

@@ -0,0 +1,183 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OoyyoFeedRun = void 0;
+const fs_1 = require("fs");
+const path_1 = require("path");
+const promises_1 = require("stream/promises");
+const zlib_1 = require("zlib");
+const crypto_1 = require("crypto");
+const ooyyo_xml_builder_1 = require("../formats/ooyyo-xml.builder");
+const ooyyo_transformer_1 = require("../transformers/ooyyo.transformer");
+const feed_guard_1 = require("./feed-guard");
+const ooyyo_feed_verify_1 = require("./ooyyo-feed-verify");
+const increment = (counter, key) => {
+    counter[key] = (counter[key] ?? 0) + 1;
+};
+const OPTIONAL_FIELDS = ['content', 'engine_size', 'body_type', 'color', 'mileage', 'fuel_type', 'transmission', 'euronorm',
+    'power', 'doors', 'seats', 'gears', 'vin', 'is_new', 'date'];
+const exists = (file) => fs_1.promises.stat(file).then(() => true, () => false);
+class OoyyoFeedRun {
+    deps;
+    options;
+    constructor(deps, options) {
+        this.deps = deps;
+        this.options = options;
+    }
+    now() {
+        return this.options.now ? this.options.now() : new Date();
+    }
+    async run() {
+        const { guard } = this.deps;
+        const { xmlFile, gzFile, runId, dryRun } = this.options;
+        const startedAt = this.now();
+        const lastGood = await guard.readLastGood();
+        const report = {
+            runId,
+            mode: dryRun ? 'dry-run' : 'live',
+            startedAt: startedAt.toISOString(),
+            status: 'failed',
+            xmlFile,
+            gzFile,
+            source: { files: 0, newestChunkAt: null, cars: 0 },
+            items: 0,
+            excluded: {},
+            missing: {},
+            postcodePadded: 0,
+            imagesPerItem: { min: null, max: null, total: 0 },
+            images: {},
+            lastGood,
+            lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
+            stale: guard.isStale(lastGood, startedAt),
+        };
+        await guard.acquireLock(runId ?? 'local', startedAt);
+        const tmpXml = `${xmlFile}.tmp`;
+        const tmpGz = gzFile.replace(/\.gz$/, '.tmp.gz');
+        try {
+            const snapshot = await (0, feed_guard_1.inspectSourceSnapshot)(this.options.sourceDirectory, this.options.sourceBaseFileName);
+            report.source.files = snapshot.files;
+            report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
+            guard.checkSource(snapshot, startedAt);
+            await fs_1.promises.mkdir((0, path_1.dirname)(xmlFile), { recursive: true });
+            await fs_1.promises.rm(tmpXml, { force: true });
+            await fs_1.promises.rm(tmpGz, { force: true });
+            const builder = new ooyyo_xml_builder_1.OoyyoXmlBuilder(tmpXml);
+            await builder.buildStart();
+            const seen = new Set();
+            try {
+                await this.deps.forEachChunk(async (cars) => {
+                    for (const source of cars) {
+                        report.source.cars += 1;
+                        const id = String(source?.car?._id ?? '');
+                        if (id && seen.has(id)) {
+                            increment(report.excluded, ooyyo_transformer_1.OoyyoExclusion.duplicateId);
+                            continue;
+                        }
+                        seen.add(id);
+                        if (!this.deps.filterCar(source)) {
+                            increment(report.excluded, ooyyo_transformer_1.OoyyoExclusion.filtered);
+                            continue;
+                        }
+                        const excluded = (0, ooyyo_transformer_1.getOoyyoExclusion)(source.car, startedAt);
+                        if (excluded) {
+                            increment(report.excluded, excluded);
+                            continue;
+                        }
+                        const images = await this.deps.resolveImages(source);
+                        if (!images.length) {
+                            increment(report.excluded, ooyyo_transformer_1.OoyyoExclusion.noImage);
+                            continue;
+                        }
+                        const country = source.car?.location?.countryCode;
+                        const postcode = (0, ooyyo_transformer_1.getOoyyoPostcode)(source.car?.location?.zipCode, country);
+                        const mapped = (0, ooyyo_transformer_1.mapOoyyoItem)(source.car, {
+                            domain: this.options.domain,
+                            images,
+                            translate: this.deps.translate,
+                            region: await this.deps.region(postcode, country),
+                            now: startedAt,
+                        });
+                        if (mapped.ok === false) {
+                            increment(report.excluded, mapped.reason);
+                            continue;
+                        }
+                        await builder.addItem(mapped.item);
+                        this.count(report, mapped.item, String(source.car?.location?.zipCode ?? ''));
+                    }
+                });
+            }
+            finally {
+                await builder.buildEnd();
+            }
+            report.images = this.deps.imageStats();
+            const verify = (0, ooyyo_feed_verify_1.verifyOoyyoXml)(await fs_1.promises.readFile(tmpXml, 'utf8'), report.items);
+            report.verify = { ads: verify.ads, issues: verify.issues.length, firstIssues: verify.issues.slice(0, 20) };
+            if (verify.issues.length) {
+                throw new feed_guard_1.FeedGuardError('verify_failed', `${verify.issues.length} problem(s) in the written XML, first: ${verify.issues[0].problem}`);
+            }
+            const counts = { sourceItems: report.source.cars, publishedItems: report.items };
+            guard.checkCounts(counts, lastGood);
+            const xmlBytes = await fs_1.promises.readFile(tmpXml);
+            await (0, promises_1.pipeline)((0, fs_1.createReadStream)(tmpXml), (0, zlib_1.createGzip)({ level: 9 }), (0, fs_1.createWriteStream)(tmpGz));
+            const gzBytes = await fs_1.promises.readFile(tmpGz);
+            if (!(0, zlib_1.gunzipSync)(gzBytes).equals(xmlBytes)) {
+                throw new Error('gzip round trip does not match the XML');
+            }
+            report.bytes = { xml: xmlBytes.length, gz: gzBytes.length, sha256: (0, crypto_1.createHash)('sha256').update(xmlBytes).digest('hex') };
+            if (await exists(gzFile)) {
+                await fs_1.promises.copyFile(gzFile, `${gzFile}.prev`);
+            }
+            await fs_1.promises.rename(tmpGz, gzFile);
+            report.lastGood = await guard.publish(tmpXml, xmlFile, counts, this.now(), runId);
+            report.status = 'published';
+            report.lastGoodAgeHours = 0;
+            report.stale = false;
+            return report;
+        }
+        catch (error) {
+            if (error instanceof feed_guard_1.FeedGuardError) {
+                report.status = 'blocked';
+                report.guard = { code: error.code, message: error.message };
+            }
+            else {
+                report.error = error?.message ?? String(error);
+            }
+            await fs_1.promises.rename(tmpXml, `${xmlFile}.rejected`).catch(() => undefined);
+            await fs_1.promises.rm(tmpGz, { force: true }).catch(() => undefined);
+            throw error;
+        }
+        finally {
+            await this.writeReport(report);
+            await guard.releaseLock();
+        }
+    }
+    count(report, item, rawZip) {
+        report.items += 1;
+        for (const field of OPTIONAL_FIELDS) {
+            if (!item[field]) {
+                increment(report.missing, field);
+            }
+        }
+        if (!item.location.postcode) {
+            increment(report.missing, 'postcode');
+        }
+        if (!item.location.region) {
+            increment(report.missing, 'region');
+        }
+        if (item.location.postcode && item.location.postcode !== rawZip) {
+            report.postcodePadded += 1;
+        }
+        const images = item.pictures.length;
+        const stats = report.imagesPerItem;
+        stats.min = stats.min === null ? images : Math.min(stats.min, images);
+        stats.max = stats.max === null ? images : Math.max(stats.max, images);
+        stats.total += images;
+    }
+    async writeReport(report) {
+        report.finishedAt = this.now().toISOString();
+        await fs_1.promises.mkdir(this.options.stateDir, { recursive: true });
+        await fs_1.promises.writeFile((0, path_1.join)(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
+        console.log(`ooyyo feed report: ${JSON.stringify(report)}`);
+    }
+}
+exports.OoyyoFeedRun = OoyyoFeedRun;
+//# sourceMappingURL=ooyyo-feed-run.js.map

package/dist/services/ooyyo-feed-run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ooyyo-feed-run.js","sourceRoot":"","sources":["../../src/services/ooyyo-feed-run.ts"],"names":[],"mappings":";;;AAAA,2BAAyE;AACzE,+BAAqC;AACrC,8CAA2C;AAC3C,+BAA8C;AAC9C,mCAAoC;AAKpC,oEAA+D;AAC/D,yEAAsH;AACtH,6CAA0F;AAC1F,2DAAqD;AA6DrD,MAAM,SAAS,GAAG,CAAC,OAA+B,EAAE,GAAW,EAAE,EAAE;IACjE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,SAAS,EAAE,aAAa,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,UAAU;IACzH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAU,CAAC;AAExE,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,aAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;AAe7E,MAAa,YAAY;IACM;IAAqC;IAAlE,YAA6B,IAAkB,EAAmB,OAAwB;QAA7D,SAAI,GAAJ,IAAI,CAAc;QAAmB,YAAO,GAAP,OAAO,CAAiB;IAAI,CAAC;IAEvF,GAAG;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;QAC5B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAmB;YAC7B,KAAK;YACL,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YACjC,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;YAClC,MAAM,EAAE,QAAQ;YAChB,OAAO;YACP,MAAM;YACN,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE;YAClD,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,EAAE;YACX,cAAc,EAAE,CAAC;YACjB,aAAa,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE;YACjD,MAAM,EAAE,EAAE;YACV,QAAQ;YACR,gBAAgB,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC;YACvD,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC;SAC1C,CAAC;QAEF,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,IAAI,OAAO,EAAE,SAAS,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,GAAG,OAAO,MAAM,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACjD,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,IAAA,kCAAqB,EAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAC5G,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,aAAa,GAAG,QAAQ,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC;YAC1E,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAEvC,MAAM,aAAE,CAAC,KAAK,CAAC,IAAA,cAAO,EAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,MAAM,aAAE,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACrC,MAAM,aAAE,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACpC,MAAM,OAAO,GAAG,IAAI,mCAAe,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YAE3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,IAAI;gBACF,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC1C,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE;wBACzB,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;wBACxB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;wBAC1C,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;4BACtB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,kCAAc,CAAC,WAAW,CAAC,CAAC;4BACvD,SAAS;yBACV;wBACD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACb,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;4BAChC,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,kCAAc,CAAC,QAAQ,CAAC,CAAC;4BACpD,SAAS;yBACV;wBAED,MAAM,QAAQ,GAAG,IAAA,qCAAiB,EAAC,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;wBAC1D,IAAI,QAAQ,EAAE;4BACZ,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;4BACrC,SAAS;yBACV;wBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;wBACrD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;4BAClB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,kCAAc,CAAC,OAAO,CAAC,CAAC;4BACnD,SAAS;yBACV;wBACD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,WAAW,CAAC;wBAClD,MAAM,QAAQ,GAAG,IAAA,oCAAgB,EAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;wBAC1E,MAAM,MAAM,GAAG,IAAA,gCAAY,EAAC,MAAM,CAAC,GAAG,EAAE;4BACtC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;4BAC3B,MAAM;4BACN,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS;4BAC9B,MAAM,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC;4BACjD,GAAG,EAAE,SAAS;yBACf,CAAC,CAAC;wBACH,IAAI,MAAM,CAAC,EAAE,KAAK,KAAK,EAAE;4BACvB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;4BAC1C,SAAS;yBACV;wBACD,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;wBACnC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;qBAC9E;gBACH,CAAC,CAAC,CAAC;aACJ;oBAAS;gBACR,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;aAC1B;YACD,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YAGvC,MAAM,MAAM,GAAG,IAAA,kCAAc,EAAC,MAAM,aAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YAC/E,MAAM,CAAC,MAAM,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YAC3G,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE;gBACxB,MAAM,IAAI,2BAAc,CAAC,eAAe,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,0CAA0C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;aACxI;YAED,MAAM,MAAM,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YACjF,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAEpC,MAAM,QAAQ,GAAG,MAAM,aAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC3C,MAAM,IAAA,mBAAQ,EAAC,IAAA,qBAAgB,EAAC,MAAM,CAAC,EAAE,IAAA,iBAAU,EAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,IAAA,sBAAiB,EAAC,KAAK,CAAC,CAAC,CAAC;YAC7F,MAAM,OAAO,GAAG,MAAM,aAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACzC,IAAI,CAAC,IAAA,iBAAU,EAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;gBACzC,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;aAC3D;YACD,MAAM,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAIzH,IAAI,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE;gBACxB,MAAM,aAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;aAC7C;YACD,MAAM,aAAE,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC/B,MAAM,CAAC,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAC;YAClF,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC5B,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACrB,OAAO,MAAM,CAAC;SACf;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,2BAAc,EAAE;gBACnC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;gBAC1B,MAAM,CAAC,KAAK,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;aAC7D;iBAAM;gBACL,MAAM,CAAC,KAAK,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;aAChD;YAED,MAAM,aAAE,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,OAAO,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACtE,MAAM,aAAE,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC3D,MAAM,KAAK,CAAC;SACb;gBAAS;YACR,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC/B,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;SAC3B;IACH,CAAC;IAEO,KAAK,CAAC,MAAsB,EAAE,IAAkB,EAAE,MAAc;QACtE,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;QAClB,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE;YACnC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBAChB,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;aAClC;SACF;QACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE;YAC3B,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;SACvC;QACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YACzB,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;SACrC;QACD,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,KAAK,MAAM,EAAE;YAC/D,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;SAC5B;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACpC,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,CAAC;QACnC,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACtE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACtE,KAAK,CAAC,KAAK,IAAI,MAAM,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAsB;QAC9C,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,aAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,aAAE,CAAC,SAAS,CAAC,IAAA,WAAI,EAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,sBAAsB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACjH,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC9D,CAAC;CACF;AAzKD,oCAyKC"}

package/dist/services/ooyyo-feed-verify.d.ts

@@ -0,0 +1,13 @@
+export declare const OOYYO_AD_ELEMENTS: string[];
+export declare const OOYYO_LOCATION_ELEMENTS: string[];
+export declare const OOYYO_UNIT_ELEMENTS: string[];
+export declare const OOYYO_UNITS: Record<string, string[]>;
+export interface OoyyoVerifyIssue {
+    ad: number;
+    problem: string;
+}
+export interface OoyyoVerifyResult {
+    ads: number;
+    issues: OoyyoVerifyIssue[];
+}
+export declare const verifyOoyyoXml: (xml: string, expectedAds: number) => OoyyoVerifyResult;

package/dist/services/ooyyo-feed-verify.js

@@ -0,0 +1,104 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.verifyOoyyoXml = exports.OOYYO_UNITS = exports.OOYYO_UNIT_ELEMENTS = exports.OOYYO_LOCATION_ELEMENTS = exports.OOYYO_AD_ELEMENTS = void 0;
+const fast_xml_parser_1 = require("fast-xml-parser");
+const ooyyo_transformer_1 = require("../transformers/ooyyo.transformer");
+exports.OOYYO_AD_ELEMENTS = [
+    'url', 'title', 'content', 'price', 'make', 'model', 'engine_size', 'trim', 'body_type', 'color', 'year',
+    'mileage', 'fuel_type', 'transmission', 'euronorm', 'power', 'doors', 'seats', 'gears', 'location',
+    'pictures', 'vin', 'is_new', 'date', 'time',
+];
+exports.OOYYO_LOCATION_ELEMENTS = ['city', 'postcode', 'region'];
+exports.OOYYO_UNIT_ELEMENTS = ['price', 'engine_size', 'mileage', 'power'];
+exports.OOYYO_UNITS = {
+    price: [ooyyo_transformer_1.OOYYO_PRICE_UNIT],
+    engine_size: ['cc', 'l'],
+    mileage: ['km', 'mi'],
+    power: ['kw', 'hp'],
+};
+const TEXT = '#text';
+const valueOf = (node) => {
+    if (node === null || node === undefined) {
+        return '';
+    }
+    if (typeof node === 'object') {
+        return String(node[TEXT] ?? '');
+    }
+    return String(node);
+};
+const asList = (node) => (Array.isArray(node) ? node : node === undefined ? [] : [node]);
+const verifyOoyyoXml = (xml, expectedAds) => {
+    const issues = [];
+    const wellFormed = fast_xml_parser_1.XMLValidator.validate(xml);
+    if (wellFormed !== true) {
+        return { ads: 0, issues: [{ ad: -1, problem: `not well-formed: ${wellFormed.err.msg} (line ${wellFormed.err.line})` }] };
+    }
+    const parser = new fast_xml_parser_1.XMLParser({
+        ignoreAttributes: false,
+        attributeNamePrefix: '@',
+        parseTagValue: false,
+        parseAttributeValue: false,
+        trimValues: false,
+        isArray: name => name === 'ad' || name === 'picture',
+    });
+    const doc = parser.parse(xml);
+    const rootNames = Object.keys(doc).filter(k => k !== '?xml');
+    if (rootNames.length !== 1 || rootNames[0] !== 'ooyyo') {
+        return { ads: 0, issues: [{ ad: -1, problem: `root element is ${rootNames.join(',') || 'missing'}, expected ooyyo` }] };
+    }
+    const ads = asList(doc.ooyyo?.ad);
+    const urls = new Set();
+    ads.forEach((ad, index) => {
+        for (const key of Object.keys(ad)) {
+            if (key.startsWith('@') || key === TEXT) {
+                continue;
+            }
+            if (!exports.OOYYO_AD_ELEMENTS.includes(key)) {
+                issues.push({ ad: index, problem: `element <${key}> is not in OoyyoFormat.xsd` });
+            }
+            if (Array.isArray(ad[key])) {
+                issues.push({ ad: index, problem: `element <${key}> appears more than once` });
+            }
+        }
+        for (const tag of exports.OOYYO_UNIT_ELEMENTS) {
+            if (ad[tag] === undefined) {
+                continue;
+            }
+            const unit = typeof ad[tag] === 'object' ? String(ad[tag]['@unit'] ?? '') : '';
+            if (!exports.OOYYO_UNITS[tag].includes(unit)) {
+                issues.push({ ad: index, problem: `<${tag}> unit="${unit}" (allowed: ${exports.OOYYO_UNITS[tag].join(', ')})` });
+            }
+        }
+        const location = (ad.location ?? {});
+        for (const key of Object.keys(location)) {
+            if (!key.startsWith('@') && key !== TEXT && !exports.OOYYO_LOCATION_ELEMENTS.includes(key)) {
+                issues.push({ ad: index, problem: `element <location><${key}> is not in OoyyoFormat.xsd` });
+            }
+        }
+        const pictures = asList(ad.pictures?.picture).map(valueOf);
+        const priceUnit = typeof ad.price === 'object' ? ad.price['@unit'] : undefined;
+        const item = {
+            url: valueOf(ad.url),
+            price: valueOf(ad.price),
+            priceUnit: priceUnit,
+            make: valueOf(ad.make),
+            model: valueOf(ad.model),
+            year: valueOf(ad.year),
+            pictures,
+        };
+        const invalid = (0, ooyyo_transformer_1.validateOoyyoItem)(item);
+        if (invalid) {
+            issues.push({ ad: index, problem: `rejected by partner rules: ${invalid}` });
+        }
+        if (item.url && urls.has(item.url)) {
+            issues.push({ ad: index, problem: 'duplicate vehicle url' });
+        }
+        urls.add(item.url);
+    });
+    if (ads.length !== expectedAds) {
+        issues.push({ ad: -1, problem: `file holds ${ads.length} ads, the run wrote ${expectedAds}` });
+    }
+    return { ads: ads.length, issues };
+};
+exports.verifyOoyyoXml = verifyOoyyoXml;
+//# sourceMappingURL=ooyyo-feed-verify.js.map

package/dist/services/ooyyo-feed-verify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ooyyo-feed-verify.js","sourceRoot":"","sources":["../../src/services/ooyyo-feed-verify.ts"],"names":[],"mappings":";;;AAAA,qDAA0D;AAG1D,yEAAwF;AAM3E,QAAA,iBAAiB,GAAG;IAC/B,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM;IACxG,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU;IAClG,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM;CAC5C,CAAC;AACW,QAAA,uBAAuB,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AAEzD,QAAA,mBAAmB,GAAG,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;AAEnE,QAAA,WAAW,GAA6B;IACnD,KAAK,EAAE,CAAC,oCAAgB,CAAC;IACzB,WAAW,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;IACxB,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;IACrB,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;CACpB,CAAC;AAYF,MAAM,IAAI,GAAG,OAAO,CAAC;AAErB,MAAM,OAAO,GAAG,CAAC,IAAa,EAAU,EAAE;IACxC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS,EAAE;QACvC,OAAO,EAAE,CAAC;KACX;IACD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;QAC5B,OAAO,MAAM,CAAE,IAAgC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;KAC9D;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,IAAa,EAAa,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAStG,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,WAAmB,EAAqB,EAAE;IACpF,MAAM,MAAM,GAAuB,EAAE,CAAC;IACtC,MAAM,UAAU,GAAG,8BAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,UAAU,KAAK,IAAI,EAAE;QACvB,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,oBAAoB,UAAU,CAAC,GAAG,CAAC,GAAG,UAAU,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,CAAC;KAC1H;IACD,MAAM,MAAM,GAAG,IAAI,2BAAS,CAAC;QAC3B,gBAAgB,EAAE,KAAK;QACvB,mBAAmB,EAAE,GAAG;QACxB,aAAa,EAAE,KAAK;QACpB,mBAAmB,EAAE,KAAK;QAC1B,UAAU,EAAE,KAAK;QACjB,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS;KACrD,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC;IAC7D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE;QACtD,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,mBAAmB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,SAAS,kBAAkB,EAAE,CAAC,EAAE,CAAC;KACzH;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAmC,CAAC;IACpE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE;QACxB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;YACjC,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,IAAI,EAAE;gBACvC,SAAS;aACV;YACD,IAAI,CAAC,yBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACpC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,GAAG,6BAA6B,EAAE,CAAC,CAAC;aACnF;YACD,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE;gBAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,GAAG,0BAA0B,EAAE,CAAC,CAAC;aAChF;SACF;QACD,KAAK,MAAM,GAAG,IAAI,2BAAmB,EAAE;YACrC,IAAI,EAAE,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE;gBACzB,SAAS;aACV;YACD,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAE,EAAE,CAAC,GAAG,CAA6B,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5G,IAAI,CAAC,mBAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;gBACpC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,GAAG,WAAW,IAAI,eAAe,mBAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;aAC1G;SACF;QACD,MAAM,QAAQ,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAA4B,CAAC;QAChE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;YACvC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,+BAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBAClF,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,sBAAsB,GAAG,6BAA6B,EAAE,CAAC,CAAC;aAC7F;SACF;QACD,MAAM,QAAQ,GAAG,MAAM,CAAE,EAAE,CAAC,QAAgD,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACpG,MAAM,SAAS,GAAG,OAAO,EAAE,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAE,EAAE,CAAC,KAAiC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5G,MAAM,IAAI,GAA0B;YAClC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC;YACpB,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC;YACxB,SAAS,EAAE,SAAkB;YAC7B,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC;YACtB,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC;YACxB,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC;YACtB,QAAQ;SACT,CAAC;QACF,MAAM,OAAO,GAAG,IAAA,qCAAiB,EAAC,IAAI,CAAC,CAAC;QACxC,IAAI,OAAO,EAAE;YACX,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,8BAA8B,OAAO,EAAE,EAAE,CAAC,CAAC;SAC9E;QACD,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YAClC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC,CAAC;SAC9D;QACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,EAAE;QAC9B,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,cAAc,GAAG,CAAC,MAAM,uBAAuB,WAAW,EAAE,EAAE,CAAC,CAAC;KAChG;IACD,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC,CAAC;AA1EW,QAAA,cAAc,kBA0EzB"}

package/dist/transformers/ooyyo.transformer.d.ts

@@ -1,16 +1,56 @@
 import { DataSchemes } from '@bytebrand/car-schema';
 import { OoyyoXmlItem } from '../interfaces';
-import { ImboImageResponseInterface } from '@bytebrand/chunkable';
-import { ConfigService, ImagesService } from '../services';
+import { OpenAiImageSelection } from './openai-image-fields';
+export declare const OOYYO_LINK_PARAMS = "source=ooyyo&medium=listing&campaign=listing&utm_source=ooyyo&utm_medium=listing&utm_campaign=listing";
+export declare const OOYYO_MAX_URL_LENGTH = 1500;
+export declare const OOYYO_PRICE_UNIT = "EUR";
+export declare const OOYYO_TIME_ZONE = "Europe/Berlin";
+export declare const getOoyyoListingUrl: (domain: string, carId: string) => string;
+export declare enum OoyyoExclusion {
+    duplicateId = "duplicate_id",
+    filtered = "filtered",
+    noId = "no_id",
+    notAvailable = "reserved_or_sold",
+    noPrice = "no_valid_price",
+    noMake = "no_make",
+    noModel = "no_recognisable_model",
+    noYear = "no_valid_year",
+    noImage = "no_valid_image",
+    urlTooLong = "url_too_long"
+}
+export declare const getOoyyoPrice: (value: unknown) => string | null;
+export declare const getOoyyoYear: (car: DataSchemes.Car, now?: Date) => string;
+export declare const getOoyyoPostcode: (zip: unknown, countryCode: unknown) => string;
+export declare const getOoyyoEuronorm: (selector: unknown) => string;
+export declare const getOoyyoDateTime: (creationDate: unknown) => {
+    date: string;
+    time: string;
+};
+export declare const getOoyyoContent: (description: unknown) => string;
+export declare const getOoyyoTitle: (car: DataSchemes.Car) => string;
+export declare const getOoyyoVin: (car: DataSchemes.Car) => string;
+export declare const toOoyyoImages: (selection: OpenAiImageSelection, maxImages?: number) => string[];
+export declare const validateOoyyoItem: (item: Partial<OoyyoXmlItem>) => OoyyoExclusion | null;
+export declare const getOoyyoExclusion: (car: DataSchemes.Car, now?: Date) => OoyyoExclusion | null;
+export interface OoyyoMappingContext {
+    domain: string;
+    images: string[];
+    translate: (selector: string) => string;
+    region: string;
+    now?: Date;
+}
+export type OoyyoMapResult = {
+    ok: true;
+    item: OoyyoXmlItem;
+} | {
+    ok: false;
+    reason: OoyyoExclusion;
+};
+export declare const mapOoyyoItem: (car: DataSchemes.Car, ctx: OoyyoMappingContext) => OoyyoMapResult;
 export declare class OoyyoTransformer {
-    private configService;
-    private imageService;
     private translator;
-    constructor(configService: ConfigService, imageService: ImagesService);
-    transform(car: DataSchemes.Car, imboImages: ImboImageResponseInterface[]): Promise<OoyyoXmlItem>;
-    private getTitle;
-    private getCondition;
-    private translateSelector;
-    private getEmissionClass;
-    private formatTimestamp;
+    private postalCodes;
+    private regions;
+    translate(selector: string): string;
+    region(postcode: string, countryCode: unknown): Promise<string>;
 }