@bytebrand/export-cars 1.26.39 → 1.26.40

package/dist/configs/criteo.config.d.ts

@@ -1,5 +1,15 @@
 import { Params } from '../interfaces/params.interface';
+import { FeedGuardOptions } from '../services/feed-guard';
 import { Config } from './config';
+export declare const CRITEO_DEFAULTS: {
+    minSourceItems: number;
+    minPublishedItems: number;
+    maxDropPercent: number;
+    maxSourceAgeHours: number;
+    maxStaleHours: number;
+};
 export declare class CriteoConfig extends Config {
+    stateDir: string;
+    feedGuard: FeedGuardOptions;
     constructor(params: Params);
 }

package/dist/configs/criteo.config.js

@@ -1,15 +1,40 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CriteoConfig = void 0;
+exports.CriteoConfig = exports.CRITEO_DEFAULTS = void 0;
+const lodash_1 = require("lodash");
 const enums_1 = require("../enums");
 const config_1 = require("./config");
+const text = (value) => (typeof value === 'string' ? value.trim() : '');
+const positiveNumber = (value, fallback) => {
+    const parsed = (0, lodash_1.toNumber)(text(value));
+    return text(value) && Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};
+exports.CRITEO_DEFAULTS = {
+    minSourceItems: 1000,
+    minPublishedItems: 500,
+    maxDropPercent: 20,
+    maxSourceAgeHours: 26,
+    maxStaleHours: 48,
+};
 class CriteoConfig extends config_1.Config {
+    stateDir;
+    feedGuard;
     constructor(params) {
         super(params);
         this.campaign = enums_1.ExportType.CRITEO;
         this.pathesConfig.localConvertedDirectory = `resources/${enums_1.ExportType.CRITEO}-export`;
+        this.stateDir = `resources/${enums_1.ExportType.CRITEO}-state`;
         this.imageType = enums_1.ImageType.marketing;
         this.marketingImageConfigId = this.environmentConfig[params.environment].criteoMarketingImagesConfigId;
+        this.feedGuard = {
+            stateDir: this.stateDir,
+            minSourceItems: positiveNumber(params.feedGuardMinSourceItems, exports.CRITEO_DEFAULTS.minSourceItems),
+            minPublishedItems: positiveNumber(params.feedGuardMinPublishedItems, exports.CRITEO_DEFAULTS.minPublishedItems),
+            maxDropPercent: positiveNumber(params.feedGuardMaxDropPercent, exports.CRITEO_DEFAULTS.maxDropPercent),
+            maxSourceAgeHours: positiveNumber(params.feedGuardMaxSourceAgeHours, exports.CRITEO_DEFAULTS.maxSourceAgeHours),
+            maxStaleHours: positiveNumber(params.feedGuardMaxStaleHours, exports.CRITEO_DEFAULTS.maxStaleHours),
+            acceptDrop: text(params.feedGuardAcceptDrop).toLowerCase() === 'true',
+        };
         this.outputFiles = [
             `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${enums_1.ExportType.CRITEO}.${enums_1.FileType.XML}`,
         ];

package/dist/configs/criteo.config.js.map

@@ -1 +1 @@
-{"version":3,"file":"criteo.config.js","sourceRoot":"","sources":["../../src/configs/criteo.config.ts"],"names":[],"mappings":";;;AAAA,oCAA4D;AAE5D,qCAAkC;AAElC,MAAa,YAAa,SAAQ,eAAM;IACtC,YAAY,MAAc;QACxB,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,QAAQ,GAAG,kBAAU,CAAC,MAAM,CAAC;QAClC,IAAI,CAAC,YAAY,CAAC,uBAAuB,GAAG,aAAa,kBAAU,CAAC,MAAM,SAAS,CAAC;QACpF,IAAI,CAAC,SAAS,GAAG,iBAAS,CAAC,SAAS,CAAC;QACrC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,6BAA6B,CAAC;QACvG,IAAI,CAAC,WAAW,GAAG;YACjB,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,oBAAoB,kBAAU,CAAC,MAAM,IAAI,gBAAQ,CAAC,GAAG,EAAE;SACpG,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;CACF;AAZD,oCAYC"}
+{"version":3,"file":"criteo.config.js","sourceRoot":"","sources":["../../src/configs/criteo.config.ts"],"names":[],"mappings":";;;AAAA,mCAAkC;AAElC,oCAA4D;AAG5D,qCAAkC;AAElC,MAAM,IAAI,GAAG,CAAC,KAAc,EAAU,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAEzF,MAAM,cAAc,GAAG,CAAC,KAAc,EAAE,QAAgB,EAAU,EAAE;IAClE,MAAM,MAAM,GAAG,IAAA,iBAAQ,EAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACrC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClF,CAAC,CAAC;AAIW,QAAA,eAAe,GAAG;IAC7B,cAAc,EAAE,IAAI;IACpB,iBAAiB,EAAE,GAAG;IACtB,cAAc,EAAE,EAAE;IAClB,iBAAiB,EAAE,EAAE;IACrB,aAAa,EAAE,EAAE;CAClB,CAAC;AAMF,MAAa,YAAa,SAAQ,eAAM;IACtC,QAAQ,CAAS;IACjB,SAAS,CAAmB;IAE5B,YAAY,MAAc;QACxB,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,QAAQ,GAAG,kBAAU,CAAC,MAAM,CAAC;QAClC,IAAI,CAAC,YAAY,CAAC,uBAAuB,GAAG,aAAa,kBAAU,CAAC,MAAM,SAAS,CAAC;QACpF,IAAI,CAAC,QAAQ,GAAG,aAAa,kBAAU,CAAC,MAAM,QAAQ,CAAC;QACvD,IAAI,CAAC,SAAS,GAAG,iBAAS,CAAC,SAAS,CAAC;QACrC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,6BAA6B,CAAC;QACvG,IAAI,CAAC,SAAS,GAAG;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,uBAAe,CAAC,cAAc,CAAC;YAC9F,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,uBAAe,CAAC,iBAAiB,CAAC;YACvG,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,uBAAe,CAAC,cAAc,CAAC;YAC9F,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,uBAAe,CAAC,iBAAiB,CAAC;YACvG,aAAa,EAAE,cAAc,CAAC,MAAM,CAAC,sBAAsB,EAAE,uBAAe,CAAC,aAAa,CAAC;YAC3F,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM;SACtE,CAAC;QACF,IAAI,CAAC,WAAW,GAAG;YACjB,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,oBAAoB,kBAAU,CAAC,MAAM,IAAI,gBAAQ,CAAC,GAAG,EAAE;SACpG,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;CACF;AAzBD,oCAyBC"}

package/dist/exporters/criteo.exporter.js

@@ -14,7 +14,8 @@ const common_1 = require("@nestjs/common");
 const core_1 = require("@nestjs/core");
 const cronicle_helper_1 = require("@bytebrand/cronicle-helper");
 const services_1 = require("../services");
-const google_xml_builder_1 = require("../formats/google-xml.builder");
+const feed_guard_1 = require("../services/feed-guard");
+const criteo_feed_run_1 = require("../services/criteo-feed-run");
 const criteo_transformer_1 = require("../transformers/criteo.transformer");
 let CriteoExporter = class CriteoExporter {
     carService;
@@ -30,29 +31,32 @@ let CriteoExporter = class CriteoExporter {
         this.moduleRef = moduleRef;
     }
     async export() {
-        const [fileName] = this.configService.config.outputFiles;
-        await cronicle_helper_1.CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
-        await cronicle_helper_1.CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
-        const googleXmlBuilder = new google_xml_builder_1.GoogleXmlBuilder(fileName);
-        await googleXmlBuilder.buildStart(this.configService.config.domain);
-        const criteoTransformer = await this.moduleRef.create(criteo_transformer_1.CriteoTransformer);
-        let withoutImage = 0;
-        await this.loaderService.process(async (chunk) => {
-            chunk.data = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
-            for (const data of chunk.data) {
-                const car = await criteoTransformer.transform(data);
-                if (car) {
-                    await googleXmlBuilder.addItem(car);
-                }
-                else {
-                    withoutImage += 1;
-                }
-                this.carService.counter.countCar(data.images);
-            }
+        const config = this.configService.config;
+        const [xmlFile] = config.outputFiles;
+        const transformer = await this.moduleRef.create(criteo_transformer_1.CriteoTransformer);
+        const run = new criteo_feed_run_1.CriteoFeedRun({
+            forEachChunk: fn => this.loaderService.process((chunk) => fn(chunk.data)),
+            filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
+            transform: source => transformer.transform(source),
+            guard: new feed_guard_1.FeedGuard(config.feedGuard),
+        }, {
+            runId: config.runId,
+            domain: config.domain,
+            sourceDirectory: config.pathesConfig.localDirectory,
+            sourceBaseFileName: config.pathesConfig.baseFileName,
+            xmlFile,
+            stateDir: config.stateDir,
         });
-        await googleXmlBuilder.buildEnd();
-        console.log(this.carService.counter.getCountedCars());
-        console.log(`criteo: ${withoutImage} cars skipped without a marketing image`);
+        try {
+            const report = await run.run();
+            if (report.stale) {
+                console.log(`criteo feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
+            }
+        }
+        catch (error) {
+            cronicle_helper_1.LoggerService.exitError(`criteo feed not published: ${error?.message ?? error}`);
+            process.exit(1);
+        }
     }
 };
 exports.CriteoExporter = CriteoExporter;

package/dist/exporters/criteo.exporter.js.map

@@ -1 +1 @@
-{"version":3,"file":"criteo.exporter.js","sourceRoot":"","sources":["../../src/exporters/criteo.exporter.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,uCAAyC;AAEzC,gEAA2D;AAG3D,0CAAqF;AACrF,sEAAiE;AAEjE,2EAAwE;AAGjE,IAAM,cAAc,GAApB,MAAM,cAAc;IAEjB;IACA;IACA;IACA;IACA;IALT,YACS,UAAsB,EACtB,aAA4B,EAC5B,YAA0B,EAC1B,aAA4B,EAC5B,SAAoB;QAJpB,eAAU,GAAV,UAAU,CAAY;QACtB,kBAAa,GAAb,aAAa,CAAe;QAC5B,iBAAY,GAAZ,YAAY,CAAc;QAC1B,kBAAa,GAAb,aAAa,CAAe;QAC5B,cAAS,GAAT,SAAS,CAAW;IACzB,CAAC;IAEL,KAAK,CAAC,MAAM;QACX,MAAM,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC;QACzD,MAAM,+BAAa,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,uBAAuB,CAAC,CAAC;QAC9F,MAAM,+BAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,uBAAuB,CAAC,CAAC;QAE7F,MAAM,gBAAgB,GAAG,IAAI,qCAAgB,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,gBAAgB,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAEpE,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,sCAAiB,CAAC,CAAC;QACzE,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,EAAE,KAAuB,EAAE,EAAE;YAClE,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;YAE/F,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE;gBAC9B,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBACpD,IAAI,GAAG,EAAE;oBACR,MAAM,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;iBACpC;qBAAM;oBACN,YAAY,IAAI,CAAC,CAAC;iBAClB;gBACD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAC9C;QACF,CAAC,CAAC,CAAC;QAEH,MAAM,gBAAgB,CAAC,QAAQ,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,WAAW,YAAY,yCAAyC,CAAC,CAAC;IAC/E,CAAC;CACD,CAAA;AAtCY,wCAAc;yBAAd,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAGS,qBAAU;QACP,wBAAa;QACd,uBAAY;QACX,wBAAa;QACjB,gBAAS;GANjB,cAAc,CAsC1B"}
+{"version":3,"file":"criteo.exporter.js","sourceRoot":"","sources":["../../src/exporters/criteo.exporter.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,uCAAyC;AAGzC,gEAA2D;AAE3D,0CAAqF;AACrF,uDAAmD;AACnD,iEAA6E;AAG7E,2EAAwE;AAQjE,IAAM,cAAc,GAApB,MAAM,cAAc;IAEjB;IACA;IACA;IACA;IACA;IALT,YACS,UAAsB,EACtB,aAA4B,EAC5B,YAA0B,EAC1B,aAA4B,EAC5B,SAAoB;QAJpB,eAAU,GAAV,UAAU,CAAY;QACtB,kBAAa,GAAb,aAAa,CAAe;QAC5B,iBAAY,GAAZ,YAAY,CAAc;QAC1B,kBAAa,GAAb,aAAa,CAAe;QAC5B,cAAS,GAAT,SAAS,CAAW;IACzB,CAAC;IAEL,KAAK,CAAC,MAAM;QACX,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,MAAsB,CAAC;QACzD,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QACrC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,sCAAiB,CAAC,CAAC;QAEnE,MAAM,GAAG,GAAG,IAAI,+BAAa,CAC5B;YACC,YAAY,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,KAAuB,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,IAAyB,CAAC,CAAC;YAChH,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;YAC/E,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC;YAClD,KAAK,EAAE,IAAI,sBAAS,CAAC,MAAM,CAAC,SAAS,CAAC;SACtC,EACD;YACC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,cAAc;YACnD,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,YAAY;YACpD,OAAO;YACP,QAAQ,EAAE,MAAM,CAAC,QAAQ;SACzB,CACD,CAAC;QAEF,IAAI;YACH,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;YAC/B,IAAI,MAAM,CAAC,KAAK,EAAE;gBACjB,OAAO,CAAC,GAAG,CAAC,qCAAqC,MAAM,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;aAC9F;SACD;QAAC,OAAO,KAAK,EAAE;YAGf,+BAAa,CAAC,SAAS,CAAC,8BAA8B,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC;YACjF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAChB;IACF,CAAC;CACD,CAAA;AA3CY,wCAAc;yBAAd,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAGS,qBAAU;QACP,wBAAa;QACd,uBAAY;QACX,wBAAa;QACjB,gBAAS;GANjB,cAAc,CA2C1B"}

package/dist/services/criteo-feed-run.d.ts

@@ -0,0 +1,76 @@
+import { ImboImageResponseInterface } from '@bytebrand/chunkable';
+import { DataSchemes } from '@bytebrand/car-schema';
+import { CriteoXmlItem } from '../interfaces';
+import { FeedGuard, LastGood } from './feed-guard';
+export interface CriteoSourceCar {
+    car: DataSchemes.Car;
+    images: ImboImageResponseInterface[];
+}
+export declare enum CriteoExclusion {
+    noId = "no_id",
+    duplicateId = "duplicate_id",
+    filtered = "filtered",
+    notAvailable = "reserved_or_sold",
+    noPrice = "no_valid_price",
+    noImage = "no_marketing_image"
+}
+export interface CriteoRunDeps {
+    forEachChunk: (fn: (cars: CriteoSourceCar[]) => Promise<void>) => Promise<unknown>;
+    filterCar: (source: CriteoSourceCar) => boolean;
+    transform: (source: CriteoSourceCar) => Promise<CriteoXmlItem | null>;
+    guard: FeedGuard;
+}
+export interface CriteoRunOptions {
+    runId?: string;
+    domain: string;
+    sourceDirectory: string;
+    sourceBaseFileName: string;
+    xmlFile: string;
+    stateDir: string;
+    now?: () => Date;
+}
+export interface CriteoRunReport {
+    runId?: string;
+    startedAt: string;
+    finishedAt?: string;
+    durationMs?: number;
+    status: 'published' | 'blocked' | 'failed';
+    guard?: {
+        code: string;
+        message: string;
+    };
+    error?: string;
+    xmlFile: string;
+    source: {
+        files: number;
+        newestChunkAt: string | null;
+        ageHours: number | null;
+        cars: number;
+    };
+    items: number;
+    excluded: Record<string, number>;
+    withRate: number;
+    withoutEnergyClass: number;
+    ids: {
+        added: number;
+        removed: number;
+        kept: number;
+        removedSample: string[];
+    } | null;
+    lastGood: LastGood | null;
+    lastGoodAgeHours: number | null;
+    stale: boolean;
+}
+export declare class CriteoFeedRun {
+    private readonly deps;
+    private readonly options;
+    constructor(deps: CriteoRunDeps, options: CriteoRunOptions);
+    private now;
+    private get idsFile();
+    run(): Promise<CriteoRunReport>;
+    private exclusion;
+    private readPublishedIds;
+    private writePublishedIds;
+    private diffIds;
+    private writeReport;
+}

package/dist/services/criteo-feed-run.js

@@ -0,0 +1,186 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CriteoFeedRun = exports.CriteoExclusion = void 0;
+const fs_1 = require("fs");
+const path_1 = require("path");
+const lodash_1 = require("lodash");
+const google_xml_builder_1 = require("../formats/google-xml.builder");
+const feed_guard_1 = require("./feed-guard");
+var CriteoExclusion;
+(function (CriteoExclusion) {
+    CriteoExclusion["noId"] = "no_id";
+    CriteoExclusion["duplicateId"] = "duplicate_id";
+    CriteoExclusion["filtered"] = "filtered";
+    CriteoExclusion["notAvailable"] = "reserved_or_sold";
+    CriteoExclusion["noPrice"] = "no_valid_price";
+    CriteoExclusion["noImage"] = "no_marketing_image";
+})(CriteoExclusion || (exports.CriteoExclusion = CriteoExclusion = {}));
+const NOT_AVAILABLE = ['selector_offerAvailability_reserved', 'selector_offerAvailability_sold'];
+const HOUR_MS = 60 * 60 * 1000;
+const PRICE = /^\d+(\.\d+)? EUR$/;
+const increment = (counter, key) => {
+    counter[key] = (counter[key] ?? 0) + 1;
+};
+class CriteoFeedRun {
+    deps;
+    options;
+    constructor(deps, options) {
+        this.deps = deps;
+        this.options = options;
+    }
+    now() {
+        return this.options.now ? this.options.now() : new Date();
+    }
+    get idsFile() {
+        return (0, path_1.join)(this.options.stateDir, 'published-ids.json');
+    }
+    async run() {
+        const { guard } = this.deps;
+        const { xmlFile, runId } = this.options;
+        const startedAt = this.now();
+        const lastGood = await guard.readLastGood();
+        const report = {
+            runId,
+            startedAt: startedAt.toISOString(),
+            status: 'failed',
+            xmlFile,
+            source: { files: 0, newestChunkAt: null, ageHours: null, cars: 0 },
+            items: 0,
+            excluded: {},
+            withRate: 0,
+            withoutEnergyClass: 0,
+            ids: null,
+            lastGood,
+            lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
+            stale: guard.isStale(lastGood, startedAt),
+        };
+        await guard.acquireLock(runId ?? 'local', startedAt);
+        const tmpFile = `${xmlFile}.tmp`;
+        try {
+            const snapshot = await (0, feed_guard_1.inspectSourceSnapshot)(this.options.sourceDirectory, this.options.sourceBaseFileName);
+            report.source.files = snapshot.files;
+            report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
+            report.source.ageHours = snapshot.newestMtime
+                ? Number(((startedAt.getTime() - snapshot.newestMtime.getTime()) / HOUR_MS).toFixed(2))
+                : null;
+            guard.checkSource(snapshot, startedAt);
+            await fs_1.promises.mkdir((0, path_1.dirname)(xmlFile), { recursive: true });
+            await fs_1.promises.rm(tmpFile, { force: true });
+            const builder = new google_xml_builder_1.GoogleXmlBuilder(tmpFile);
+            await builder.buildStart(this.options.domain);
+            const seen = new Set();
+            const published = [];
+            await this.deps.forEachChunk(async (cars) => {
+                for (const source of cars) {
+                    report.source.cars += 1;
+                    const reason = this.exclusion(source, seen);
+                    if (reason) {
+                        increment(report.excluded, reason);
+                        continue;
+                    }
+                    const item = await this.deps.transform(source);
+                    if (!item) {
+                        increment(report.excluded, CriteoExclusion.noImage);
+                        continue;
+                    }
+                    if (!PRICE.test(item.price) || !PRICE.test(item.sale_price)) {
+                        increment(report.excluded, CriteoExclusion.noPrice);
+                        continue;
+                    }
+                    await builder.addItem(item);
+                    published.push(item.id);
+                    report.items += 1;
+                    if (item.installment) {
+                        report.withRate += 1;
+                    }
+                    if (!item.energy_efficiency_class) {
+                        report.withoutEnergyClass += 1;
+                    }
+                }
+            });
+            await builder.buildEnd();
+            const counts = { sourceItems: report.source.cars, publishedItems: report.items };
+            guard.checkCounts(counts, lastGood);
+            const previousIds = await this.readPublishedIds();
+            report.lastGood = await guard.publish(tmpFile, xmlFile, counts, this.now(), runId);
+            await this.writePublishedIds(published);
+            report.ids = this.diffIds(previousIds, published);
+            report.status = 'published';
+            report.lastGoodAgeHours = 0;
+            report.stale = false;
+            return report;
+        }
+        catch (error) {
+            if (error instanceof feed_guard_1.FeedGuardError) {
+                report.status = 'blocked';
+                report.guard = { code: error.code, message: error.message };
+            }
+            else {
+                report.error = error?.message ?? String(error);
+            }
+            await fs_1.promises.rename(tmpFile, `${xmlFile}.rejected`).catch(() => undefined);
+            throw error;
+        }
+        finally {
+            await this.writeReport(report, startedAt);
+            await guard.releaseLock();
+        }
+    }
+    exclusion(source, seen) {
+        const id = (0, lodash_1.toString)((0, lodash_1.get)(source, 'car._id', ''));
+        if (!id) {
+            return CriteoExclusion.noId;
+        }
+        if (seen.has(id)) {
+            return CriteoExclusion.duplicateId;
+        }
+        seen.add(id);
+        if (!this.deps.filterCar(source)) {
+            return CriteoExclusion.filtered;
+        }
+        if (NOT_AVAILABLE.includes((0, lodash_1.toString)((0, lodash_1.get)(source.car, 'offer.offerAvailability', '')))) {
+            return CriteoExclusion.notAvailable;
+        }
+        const price = (0, lodash_1.toNumber)((0, lodash_1.get)(source.car, 'price.currentSalesPrice', null));
+        if (!Number.isFinite(price) || price <= 0) {
+            return CriteoExclusion.noPrice;
+        }
+        return null;
+    }
+    async readPublishedIds() {
+        try {
+            return JSON.parse(await fs_1.promises.readFile(this.idsFile, 'utf8'));
+        }
+        catch (error) {
+            if (error?.code === 'ENOENT') {
+                return null;
+            }
+            throw error;
+        }
+    }
+    async writePublishedIds(ids) {
+        await fs_1.promises.mkdir(this.options.stateDir, { recursive: true });
+        await fs_1.promises.writeFile(`${this.idsFile}.tmp`, JSON.stringify(ids), 'utf8');
+        await fs_1.promises.rename(`${this.idsFile}.tmp`, this.idsFile);
+    }
+    diffIds(previous, current) {
+        if (!previous) {
+            return { added: current.length, removed: 0, kept: 0, removedSample: [] };
+        }
+        const before = new Set(previous);
+        const now = new Set(current);
+        const removed = previous.filter(id => !now.has(id));
+        const kept = current.filter(id => before.has(id)).length;
+        return { added: current.length - kept, removed: removed.length, kept, removedSample: removed.slice(0, 20) };
+    }
+    async writeReport(report, startedAt) {
+        const finishedAt = this.now();
+        report.finishedAt = finishedAt.toISOString();
+        report.durationMs = finishedAt.getTime() - startedAt.getTime();
+        await fs_1.promises.mkdir(this.options.stateDir, { recursive: true });
+        await fs_1.promises.writeFile((0, path_1.join)(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
+        console.log(`criteo feed report: ${JSON.stringify(report)}`);
+    }
+}
+exports.CriteoFeedRun = CriteoFeedRun;
+//# sourceMappingURL=criteo-feed-run.js.map

package/dist/services/criteo-feed-run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"criteo-feed-run.js","sourceRoot":"","sources":["../../src/services/criteo-feed-run.ts"],"names":[],"mappings":";;;AAAA,2BAAoC;AACpC,+BAAqC;AACrC,mCAAiD;AAIjD,sEAAiE;AAEjE,6CAA0F;AAO1F,IAAY,eAOX;AAPD,WAAY,eAAe;IACzB,iCAAc,CAAA;IACd,+CAA4B,CAAA;IAC5B,wCAAqB,CAAA;IACrB,oDAAiC,CAAA;IACjC,6CAA0B,CAAA;IAC1B,iDAA8B,CAAA;AAChC,CAAC,EAPW,eAAe,+BAAf,eAAe,QAO1B;AAKD,MAAM,aAAa,GAAG,CAAC,qCAAqC,EAAE,iCAAiC,CAAC,CAAC;AA6CjG,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC/B,MAAM,KAAK,GAAG,mBAAmB,CAAC;AAElC,MAAM,SAAS,GAAG,CAAC,OAA+B,EAAE,GAAW,EAAE,EAAE;IACjE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAC;AAQF,MAAa,aAAa;IACK;IAAsC;IAAnE,YAA6B,IAAmB,EAAmB,OAAyB;QAA/D,SAAI,GAAJ,IAAI,CAAe;QAAmB,YAAO,GAAP,OAAO,CAAkB;IAAI,CAAC;IAEzF,GAAG;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IAC5D,CAAC;IAED,IAAY,OAAO;QACjB,OAAO,IAAA,WAAI,EAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;QAC5B,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAoB;YAC9B,KAAK;YACL,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;YAClC,MAAM,EAAE,QAAQ;YAChB,OAAO;YACP,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE;YAClE,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE,CAAC;YACX,kBAAkB,EAAE,CAAC;YACrB,GAAG,EAAE,IAAI;YACT,QAAQ;YACR,gBAAgB,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC;YACvD,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC;SAC1C,CAAC;QAEF,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,IAAI,OAAO,EAAE,SAAS,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,GAAG,OAAO,MAAM,CAAC;QACjC,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,IAAA,kCAAqB,EAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAC5G,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,aAAa,GAAG,QAAQ,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC;YAC1E,MAAM,CAAC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,WAAW;gBAC3C,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACvF,CAAC,CAAC,IAAI,CAAC;YACT,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAEvC,MAAM,aAAE,CAAC,KAAK,CAAC,IAAA,cAAO,EAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEtD,MAAM,aAAE,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,MAAM,OAAO,GAAG,IAAI,qCAAgB,CAAC,OAAO,CAAC,CAAC;YAC9C,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAE9C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBAC1C,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE;oBACzB,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;oBACxB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;oBAC5C,IAAI,MAAM,EAAE;wBACV,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;wBACnC,SAAS;qBACV;oBACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;oBAC/C,IAAI,CAAC,IAAI,EAAE;wBACT,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;wBACpD,SAAS;qBACV;oBAED,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;wBAC3D,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;wBACpD,SAAS;qBACV;oBACD,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBAC5B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACxB,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;oBAClB,IAAI,IAAI,CAAC,WAAW,EAAE;wBACpB,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;qBACtB;oBACD,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE;wBACjC,MAAM,CAAC,kBAAkB,IAAI,CAAC,CAAC;qBAChC;iBACF;YACH,CAAC,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;YAEzB,MAAM,MAAM,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YACjF,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAEpC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,MAAM,CAAC,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAC;YACnF,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACxC,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC5B,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACrB,OAAO,MAAM,CAAC;SACf;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,2BAAc,EAAE;gBACnC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;gBAC1B,MAAM,CAAC,KAAK,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;aAC7D;iBAAM;gBACL,MAAM,CAAC,KAAK,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;aAChD;YAED,MAAM,aAAE,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,OAAO,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACvE,MAAM,KAAK,CAAC;SACb;gBAAS;YACR,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC1C,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;SAC3B;IACH,CAAC;IAEO,SAAS,CAAC,MAAuB,EAAE,IAAiB;QAC1D,MAAM,EAAE,GAAG,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;QAChD,IAAI,CAAC,EAAE,EAAE;YACP,OAAO,eAAe,CAAC,IAAI,CAAC;SAC7B;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAChB,OAAO,eAAe,CAAC,WAAW,CAAC;SACpC;QACD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACb,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;YAChC,OAAO,eAAe,CAAC,QAAQ,CAAC;SACjC;QACD,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,MAAM,CAAC,GAAG,EAAE,yBAAyB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE;YACpF,OAAO,eAAe,CAAC,YAAY,CAAC;SACrC;QACD,MAAM,KAAK,GAAG,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,MAAM,CAAC,GAAG,EAAE,yBAAyB,EAAE,IAAI,CAAC,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE;YACzC,OAAO,eAAe,CAAC,OAAO,CAAC;SAChC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC5B,IAAI;YACF,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,aAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAa,CAAC;SACxE;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ,EAAE;gBAC5B,OAAO,IAAI,CAAC;aACb;YACD,MAAM,KAAK,CAAC;SACb;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,GAAa;QAC3C,MAAM,aAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,aAAE,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,OAAO,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;QACvE,MAAM,aAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACvD,CAAC;IAEO,OAAO,CAAC,QAAyB,EAAE,OAAiB;QAC1D,IAAI,CAAC,QAAQ,EAAE;YACb,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC;SAC1E;QACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACpD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QACzD,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAC9G,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAuB,EAAE,SAAe;QAChE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,CAAC,UAAU,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,CAAC,UAAU,GAAG,UAAU,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;QAC/D,MAAM,aAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,aAAE,CAAC,SAAS,CAAC,IAAA,WAAI,EAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,sBAAsB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACjH,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;CACF;AAvKD,sCAuKC"}