@bytebrand/export-cars 1.26.44 → 1.26.46

package/dist/configs/marktde.config.d.ts

@@ -1,5 +1,19 @@
 import { Params } from '../interfaces/params.interface';
+import { FeedGuardOptions } from '../services/feed-guard';
 import { Config } from './config';
+export declare const MARKTDE_DEFAULTS: {
+    imageConcurrency: number;
+    minSourceItems: number;
+    minPublishedItems: number;
+    maxDropPercent: number;
+    maxSourceAgeHours: number;
+    maxStaleHours: number;
+};
 export declare class MarktdeConfig extends Config {
+    dryRun: boolean;
+    imageConcurrency: number;
+    maxImages: number;
+    stateDir: string;
+    feedGuard: FeedGuardOptions;
     constructor(params: Params);
 }

package/dist/configs/marktde.config.js

@@ -1,16 +1,49 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MarktdeConfig = void 0;
+exports.MarktdeConfig = exports.MARKTDE_DEFAULTS = void 0;
+const lodash_1 = require("lodash");
 const enums_1 = require("../enums");
 const config_1 = require("./config");
+const text = (value) => (typeof value === 'string' ? value.trim() : '');
+const positiveNumber = (value, fallback) => {
+    const parsed = (0, lodash_1.toNumber)(text(value));
+    return text(value) && Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};
+exports.MARKTDE_DEFAULTS = {
+    imageConcurrency: 16,
+    minSourceItems: 1000,
+    minPublishedItems: 1,
+    maxDropPercent: 20,
+    maxSourceAgeHours: 26,
+    maxStaleHours: 48,
+};
 class MarktdeConfig extends config_1.Config {
+    dryRun;
+    imageConcurrency;
+    maxImages;
+    stateDir;
+    feedGuard;
     constructor(params) {
         super(params);
         this.campaign = enums_1.ExportType.MARKTDE;
-        this.pathesConfig.localConvertedDirectory = `resources/${enums_1.ExportType.MARKTDE}-export`;
+        this.dryRun = text(params.marktdeDryRun).toLowerCase() !== 'false';
+        const mode = this.dryRun ? 'dryrun' : 'export';
+        this.pathesConfig.localConvertedDirectory = `resources/${enums_1.ExportType.MARKTDE}-${mode}`;
+        this.stateDir = `resources/${enums_1.ExportType.MARKTDE}-state/${mode}`;
         this.imageUrl = `https://images.${this.domain}/carimage`;
         this.mappingSpreadsheetTitle = 'mobile_yescar';
         this.imageType = enums_1.ImageType.default;
+        this.imageConcurrency = positiveNumber(params.marktdeImageConcurrency, exports.MARKTDE_DEFAULTS.imageConcurrency);
+        this.maxImages = Math.floor(positiveNumber(params.marktdeMaxImages, 0));
+        this.feedGuard = {
+            stateDir: this.stateDir,
+            minSourceItems: positiveNumber(params.feedGuardMinSourceItems, exports.MARKTDE_DEFAULTS.minSourceItems),
+            minPublishedItems: positiveNumber(params.feedGuardMinPublishedItems, exports.MARKTDE_DEFAULTS.minPublishedItems),
+            maxDropPercent: positiveNumber(params.feedGuardMaxDropPercent, exports.MARKTDE_DEFAULTS.maxDropPercent),
+            maxSourceAgeHours: positiveNumber(params.feedGuardMaxSourceAgeHours, exports.MARKTDE_DEFAULTS.maxSourceAgeHours),
+            maxStaleHours: positiveNumber(params.feedGuardMaxStaleHours, exports.MARKTDE_DEFAULTS.maxStaleHours),
+            acceptDrop: text(params.feedGuardAcceptDrop).toLowerCase() === 'true',
+        };
         this.outputFiles = [
             `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${enums_1.ExportType.MARKTDE}.${enums_1.FileType.CSV}`,
             `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${enums_1.ExportType.MARKTDE}.${enums_1.FileType.ZIP}`,

package/dist/configs/marktde.config.js.map

@@ -1 +1 @@
-{"version":3,"file":"marktde.config.js","sourceRoot":"","sources":["../../src/configs/marktde.config.ts"],"names":[],"mappings":";;;AAAA,oCAA4D;AAE5D,qCAAkC;AAElC,MAAa,aAAc,SAAQ,eAAM;IACvC,YAAY,MAAc;QACxB,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,QAAQ,GAAG,kBAAU,CAAC,OAAO,CAAC;QACnC,IAAI,CAAC,YAAY,CAAC,uBAAuB,GAAG,aAAa,kBAAU,CAAC,OAAO,SAAS,CAAC;QACrF,IAAI,CAAC,QAAQ,GAAG,kBAAkB,IAAI,CAAC,MAAM,WAAW,CAAC;QACzD,IAAI,CAAC,uBAAuB,GAAG,eAAe,CAAC;QAC/C,IAAI,CAAC,SAAS,GAAG,iBAAS,CAAC,OAAO,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG;YACjB,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,oBAAoB,kBAAU,CAAC,OAAO,IAAI,gBAAQ,CAAC,GAAG,EAAE;YACpG,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,oBAAoB,kBAAU,CAAC,OAAO,IAAI,gBAAQ,CAAC,GAAG,EAAE;SACrG,CAAC;IACJ,CAAC;CACF;AAbD,sCAaC"}
+{"version":3,"file":"marktde.config.js","sourceRoot":"","sources":["../../src/configs/marktde.config.ts"],"names":[],"mappings":";;;AAAA,mCAAkC;AAElC,oCAA2D;AAG3D,qCAAkC;AAElC,MAAM,IAAI,GAAG,CAAC,KAAc,EAAU,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAEzF,MAAM,cAAc,GAAG,CAAC,KAAc,EAAE,QAAgB,EAAU,EAAE;IAClE,MAAM,MAAM,GAAG,IAAA,iBAAQ,EAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACrC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClF,CAAC,CAAC;AAIW,QAAA,gBAAgB,GAAG;IAC9B,gBAAgB,EAAE,EAAE;IACpB,cAAc,EAAE,IAAI;IACpB,iBAAiB,EAAE,CAAC;IACpB,cAAc,EAAE,EAAE;IAClB,iBAAiB,EAAE,EAAE;IACrB,aAAa,EAAE,EAAE;CAClB,CAAC;AAUF,MAAa,aAAc,SAAQ,eAAM;IACvC,MAAM,CAAU;IAChB,gBAAgB,CAAS;IAEzB,SAAS,CAAS;IAClB,QAAQ,CAAS;IACjB,SAAS,CAAmB;IAE5B,YAAY,MAAc;QACxB,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,QAAQ,GAAG,kBAAU,CAAC,OAAO,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC;QACnE,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC/C,IAAI,CAAC,YAAY,CAAC,uBAAuB,GAAG,aAAa,kBAAU,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;QACtF,IAAI,CAAC,QAAQ,GAAG,aAAa,kBAAU,CAAC,OAAO,UAAU,IAAI,EAAE,CAAC;QAChE,IAAI,CAAC,QAAQ,GAAG,kBAAkB,IAAI,CAAC,MAAM,WAAW,CAAC;QACzD,IAAI,CAAC,uBAAuB,GAAG,eAAe,CAAC;QAG/C,IAAI,CAAC,SAAS,GAAG,iBAAS,CAAC,OAAO,CAAC;QACnC,IAAI,CAAC,gBAAgB,GAAG,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,wBAAgB,CAAC,gBAAgB,CAAC,CAAC;QAC1G,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAC;QACxE,IAAI,CAAC,SAAS,GAAG;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,wBAAgB,CAAC,cAAc,CAAC;YAC/F,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,wBAAgB,CAAC,iBAAiB,CAAC;YACxG,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,wBAAgB,CAAC,cAAc,CAAC;YAC/F,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,wBAAgB,CAAC,iBAAiB,CAAC;YACxG,aAAa,EAAE,cAAc,CAAC,MAAM,CAAC,sBAAsB,EAAE,wBAAgB,CAAC,aAAa,CAAC;YAC5F,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM;SACtE,CAAC;QACF,IAAI,CAAC,WAAW,GAAG;YACjB,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,oBAAoB,kBAAU,CAAC,OAAO,IAAI,gBAAQ,CAAC,GAAG,EAAE;YACpG,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,oBAAoB,kBAAU,CAAC,OAAO,IAAI,gBAAQ,CAAC,GAAG,EAAE;SACrG,CAAC;IACJ,CAAC;CACF;AApCD,sCAoCC"}

package/dist/constants/fb.constants.js

@@ -22,7 +22,7 @@ exports.AMOUNT_QUALIFIER_APR = 'APR';
 exports.TERM_QUALIFIER_MONTHS = 'months';
 exports.OFFER_DISCLAIMER = 'Für Ihre Finanzierungswünsche stellen wir eine Bonitätsanfrage. Bonität ' +
     'vorausgesetzt, ist dies ein repräsentatives Berechnungsbeispiel gem. der Angaben, welches 2/3 aller Kunden, ' +
-    'im Sinne des § 17a Abs. 4 PAngV, erhalten. Dieses freibleibende Angebot der Santander Consumer Bank AG, ' +
+    'im Sinne des § 17a Abs. 4 PAngV, erhalten. Dieses freibleibende Angebot der Openbank Deutschland AG, ' +
     'Santander-Platz 1, 41061 Mönchengladbach wird vermittelt durch die auto.de GmbH, Max-Planck-Str. 19, 06796 ' +
     'Sandersdorf-Brehna, die als ungebundener Vermittler nicht beratend tätig ist. Irrtümer vorbehalten. Preise ' +
     'ggf. inkl. MwSt.';

package/dist/constants/fb.constants.js.map

@@ -1 +1 @@
-{"version":3,"file":"fb.constants.js","sourceRoot":"","sources":["../../src/constants/fb.constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,kBAAkB,GAAG,EAAE,CAAC;AACxB,QAAA,8BAA8B,GAAG,EAAE,CAAC;AACpC,QAAA,eAAe,GAAG,GAAG,CAAC;AACtB,QAAA,WAAW,GAAG,OAAO,CAAC;AACtB,QAAA,YAAY,GAAG,oBAAoB,CAAC;AACpC,QAAA,aAAa,GAAG,oBAAoB,CAAC;AACrC,QAAA,oBAAoB,GAAG,IAAI,CAAC;AAC5B,QAAA,2BAA2B,GAAG,UAAU,CAAC;AACzC,QAAA,iCAAiC,GAAG,CAAC,mBAAmB,EAAE,mBAAmB,CAAC,CAAC;AAI/E,QAAA,qBAAqB,GAAG,UAAU,CAAC;AAGnC,QAAA,uBAAuB,GAAG,WAAW,CAAC;AAInD,IAAY,SAIX;AAJD,WAAY,SAAS;IACnB,kCAAqB,CAAA;IACrB,8BAAiB,CAAA;IACjB,yBAAY,CAAA;AACd,CAAC,EAJW,SAAS,yBAAT,SAAS,QAIpB;AAEY,QAAA,oBAAoB,GAAG,KAAK,CAAC;AAC7B,QAAA,qBAAqB,GAAG,QAAQ,CAAC;AAEjC,QAAA,gBAAgB,GAAG,0EAA0E;IACxG,8GAA8G;IAC9G,0GAA0G;IAC1G,6GAA6G;IAC7G,6GAA6G;IAC7G,kBAAkB,CAAC"}
+{"version":3,"file":"fb.constants.js","sourceRoot":"","sources":["../../src/constants/fb.constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,kBAAkB,GAAG,EAAE,CAAC;AACxB,QAAA,8BAA8B,GAAG,EAAE,CAAC;AACpC,QAAA,eAAe,GAAG,GAAG,CAAC;AACtB,QAAA,WAAW,GAAG,OAAO,CAAC;AACtB,QAAA,YAAY,GAAG,oBAAoB,CAAC;AACpC,QAAA,aAAa,GAAG,oBAAoB,CAAC;AACrC,QAAA,oBAAoB,GAAG,IAAI,CAAC;AAC5B,QAAA,2BAA2B,GAAG,UAAU,CAAC;AACzC,QAAA,iCAAiC,GAAG,CAAC,mBAAmB,EAAE,mBAAmB,CAAC,CAAC;AAI/E,QAAA,qBAAqB,GAAG,UAAU,CAAC;AAGnC,QAAA,uBAAuB,GAAG,WAAW,CAAC;AAInD,IAAY,SAIX;AAJD,WAAY,SAAS;IACnB,kCAAqB,CAAA;IACrB,8BAAiB,CAAA;IACjB,yBAAY,CAAA;AACd,CAAC,EAJW,SAAS,yBAAT,SAAS,QAIpB;AAEY,QAAA,oBAAoB,GAAG,KAAK,CAAC;AAC7B,QAAA,qBAAqB,GAAG,QAAQ,CAAC;AAEjC,QAAA,gBAAgB,GAAG,0EAA0E;IACxG,8GAA8G;IAC9G,uGAAuG;IACvG,6GAA6G;IAC7G,6GAA6G;IAC7G,kBAAkB,CAAC"}

package/dist/export.schema.d.ts

@@ -1,5 +1,5 @@
 import { AlleskralleConfig, As24Config, Auto24Config, AutouncleConfig, DasparkingConfig, EautoConfig, FbConfig, GoogleConfig, MarktdeConfig, MeinstadtConfig, OoyyoConfig, YescarConfig, KleinanzeigeFocusConfig, AutoboerseConfig, ByOwnerConfig, FacebookShopConfig, CriteoConfig, TrovitConfig, MitulaConfig, AwinConfig, EviumConfig, SantanderConfig, OpenAiAdsConfig } from './configs';
-import { As24Exporter, Auto24Exporter, AutouncleExporter, ByOwnerExporter, FbExporter, GoogleExporter, MeinstadtExporter, MobileExporter, OoyyoExporter, ModixExporter, RubrikkExporter, FacebookShopExporter, TrovitExporter, AwinExporter, SantanderExporter, CriteoExporter, OpenAiAdsExporter } from './exporters';
+import { As24Exporter, Auto24Exporter, AutouncleExporter, ByOwnerExporter, FbExporter, GoogleExporter, MarktdeExporter, MeinstadtExporter, MobileExporter, OoyyoExporter, ModixExporter, RubrikkExporter, FacebookShopExporter, TrovitExporter, AwinExporter, SantanderExporter, CriteoExporter, OpenAiAdsExporter } from './exporters';
 export declare const exportSchema: {
     eauto: {
         config: typeof EautoConfig;
@@ -11,7 +11,7 @@ export declare const exportSchema: {
     };
     marktde: {
         config: typeof MarktdeConfig;
-        exporter: typeof MobileExporter;
+        exporter: typeof MarktdeExporter;
     };
     google: {
         config: typeof GoogleConfig;

package/dist/export.schema.js

@@ -15,7 +15,7 @@ exports.exportSchema = {
     },
     [enums_1.ExportType.MARKTDE]: {
         config: configs_1.MarktdeConfig,
-        exporter: exporters_1.MobileExporter,
+        exporter: exporters_1.MarktdeExporter,
     },
     [enums_1.ExportType.GOOGLE]: {
         config: configs_1.GoogleConfig,

package/dist/export.schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"export.schema.js","sourceRoot":"","sources":["../src/export.schema.ts"],"names":[],"mappings":";;;AAAA,mCAAqC;AAErC,uCAwBmB;AACnB,2CAkBqB;AAER,QAAA,YAAY,GAAG;IAC1B,CAAC,kBAAU,CAAC,KAAK,CAAC,EAAE;QAClB,MAAM,EAAE,qBAAW;QACnB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,OAAO,CAAC,EAAE;QACpB,MAAM,EAAE,uBAAa;QACrB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,EAAE,CAAC,EAAE;QACf,MAAM,EAAE,kBAAQ;QAChB,QAAQ,EAAE,sBAAU;KACrB;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,KAAK,CAAC,EAAE;QAClB,MAAM,EAAE,qBAAW;QACnB,QAAQ,EAAE,yBAAa;KACxB;IACD,CAAC,kBAAU,CAAC,SAAS,CAAC,EAAE;QACtB,MAAM,EAAE,yBAAe;QACvB,QAAQ,EAAE,6BAAiB;KAC5B;IACD,CAAC,kBAAU,CAAC,QAAQ,CAAC,EAAE;QACrB,MAAM,EAAE,uBAAa;QACrB,QAAQ,EAAE,2BAAe;KAC1B;IACD,CAAC,kBAAU,CAAC,SAAS,CAAC,EAAE;QACtB,MAAM,EAAE,yBAAe;QACvB,QAAQ,EAAE,6BAAiB;KAC5B;IACD,CAAC,kBAAU,CAAC,IAAI,CAAC,EAAE;QACjB,MAAM,EAAE,oBAAU;QAClB,QAAQ,EAAE,wBAAY;KACvB;IACD,CAAC,kBAAU,CAAC,WAAW,CAAC,EAAE;QACxB,MAAM,EAAE,2BAAiB;QACzB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,UAAU,CAAC,EAAE;QACvB,MAAM,EAAE,0BAAgB;QACxB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,kBAAkB,CAAC,EAAE;QAC/B,MAAM,EAAE,iCAAuB;QAC/B,QAAQ,EAAE,2BAAe;KAC1B;IACD,CAAC,kBAAU,CAAC,UAAU,CAAC,EAAE;QACvB,MAAM,EAAE,0BAAgB;QACxB,QAAQ,EAAE,yBAAa;KACxB;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,aAAa,CAAC,EAAE;QAC1B,MAAM,EAAE,4BAAkB;QAC1B,QAAQ,EAAE,gCAAoB;KAC/B;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,IAAI,CAAC,EAAE;QACjB,MAAM,EAAE,oBAAU;QAClB,QAAQ,EAAE,wBAAY;KACvB;IACD,CAAC,kBAAU,CAAC,KAAK,CAAC,EAAE;QAClB,MAAM,EAAE,qBAAW;QACnB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,SAAS,CAAC,EAAE;QACtB,MAAM,EAAE,yBAAe;QACvB,QAAQ,EAAE,6BAAiB;KAC5B;IACD,CAAC,kBAAU,CAAC,UAAU,CAAC,EAAE;QACvB,MAAM,EAAE,yBAAe;QACvB,QAAQ,EAAE,6BAAiB;KAC5B;CACF,CAAC"}
+{"version":3,"file":"export.schema.js","sourceRoot":"","sources":["../src/export.schema.ts"],"names":[],"mappings":";;;AAAA,mCAAqC;AAErC,uCAwBmB;AACnB,2CAmBqB;AAER,QAAA,YAAY,GAAG;IAC1B,CAAC,kBAAU,CAAC,KAAK,CAAC,EAAE;QAClB,MAAM,EAAE,qBAAW;QACnB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,OAAO,CAAC,EAAE;QACpB,MAAM,EAAE,uBAAa;QACrB,QAAQ,EAAE,2BAAe;KAC1B;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,EAAE,CAAC,EAAE;QACf,MAAM,EAAE,kBAAQ;QAChB,QAAQ,EAAE,sBAAU;KACrB;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,KAAK,CAAC,EAAE;QAClB,MAAM,EAAE,qBAAW;QACnB,QAAQ,EAAE,yBAAa;KACxB;IACD,CAAC,kBAAU,CAAC,SAAS,CAAC,EAAE;QACtB,MAAM,EAAE,yBAAe;QACvB,QAAQ,EAAE,6BAAiB;KAC5B;IACD,CAAC,kBAAU,CAAC,QAAQ,CAAC,EAAE;QACrB,MAAM,EAAE,uBAAa;QACrB,QAAQ,EAAE,2BAAe;KAC1B;IACD,CAAC,kBAAU,CAAC,SAAS,CAAC,EAAE;QACtB,MAAM,EAAE,yBAAe;QACvB,QAAQ,EAAE,6BAAiB;KAC5B;IACD,CAAC,kBAAU,CAAC,IAAI,CAAC,EAAE;QACjB,MAAM,EAAE,oBAAU;QAClB,QAAQ,EAAE,wBAAY;KACvB;IACD,CAAC,kBAAU,CAAC,WAAW,CAAC,EAAE;QACxB,MAAM,EAAE,2BAAiB;QACzB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,UAAU,CAAC,EAAE;QACvB,MAAM,EAAE,0BAAgB;QACxB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,kBAAkB,CAAC,EAAE;QAC/B,MAAM,EAAE,iCAAuB;QAC/B,QAAQ,EAAE,2BAAe;KAC1B;IACD,CAAC,kBAAU,CAAC,UAAU,CAAC,EAAE;QACvB,MAAM,EAAE,0BAAgB;QACxB,QAAQ,EAAE,yBAAa;KACxB;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,aAAa,CAAC,EAAE;QAC1B,MAAM,EAAE,4BAAkB;QAC1B,QAAQ,EAAE,gCAAoB;KAC/B;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,sBAAY;QACpB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,IAAI,CAAC,EAAE;QACjB,MAAM,EAAE,oBAAU;QAClB,QAAQ,EAAE,wBAAY;KACvB;IACD,CAAC,kBAAU,CAAC,KAAK,CAAC,EAAE;QAClB,MAAM,EAAE,qBAAW;QACnB,QAAQ,EAAE,0BAAc;KACzB;IACD,CAAC,kBAAU,CAAC,SAAS,CAAC,EAAE;QACtB,MAAM,EAAE,yBAAe;QACvB,QAAQ,EAAE,6BAAiB;KAC5B;IACD,CAAC,kBAAU,CAAC,UAAU,CAAC,EAAE;QACvB,MAAM,EAAE,yBAAe;QACvB,QAAQ,EAAE,6BAAiB;KAC5B;CACF,CAAC"}

package/dist/exporters/index.d.ts

@@ -5,6 +5,7 @@ export * from './by-owner.exporter';
 export * from './criteo.exporter';
 export * from './fb.exporter';
 export * from './google.exporter';
+export * from './marktde.exporter';
 export * from './meinstadt.exporter';
 export * from './mobile.exporter';
 export * from './modix.exporter';

package/dist/exporters/index.js

@@ -21,6 +21,7 @@ __exportStar(require("./by-owner.exporter"), exports);
 __exportStar(require("./criteo.exporter"), exports);
 __exportStar(require("./fb.exporter"), exports);
 __exportStar(require("./google.exporter"), exports);
+__exportStar(require("./marktde.exporter"), exports);
 __exportStar(require("./meinstadt.exporter"), exports);
 __exportStar(require("./mobile.exporter"), exports);
 __exportStar(require("./modix.exporter"), exports);

package/dist/exporters/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/exporters/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kDAAgC;AAChC,oDAAkC;AAClC,uDAAqC;AACrC,sDAAoC;AACpC,oDAAkC;AAClC,gDAA8B;AAC9B,oDAAkC;AAClC,uDAAqC;AACrC,oDAAkC;AAClC,mDAAiC;AACjC,mDAAiC;AACjC,qDAAmC;AACnC,2DAAyC;AACzC,oDAAkC;AAClC,kDAAgC;AAChC,uDAAqC;AACrC,wDAAsC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/exporters/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kDAAgC;AAChC,oDAAkC;AAClC,uDAAqC;AACrC,sDAAoC;AACpC,oDAAkC;AAClC,gDAA8B;AAC9B,oDAAkC;AAClC,qDAAmC;AACnC,uDAAqC;AACrC,oDAAkC;AAClC,mDAAiC;AACjC,mDAAiC;AACjC,qDAAmC;AACnC,2DAAyC;AACzC,oDAAkC;AAClC,kDAAgC;AAChC,uDAAqC;AACrC,wDAAsC"}

package/dist/exporters/marktde.exporter.d.ts

@@ -0,0 +1,13 @@
+import { ModuleRef } from '@nestjs/core';
+import { CarService, RulesService, LoaderService, ConfigService, ImagesService } from '../services';
+import { Exporter } from '../interfaces/exporter.interface';
+export declare class MarktdeExporter implements Exporter {
+    private carService;
+    private configService;
+    private rulesService;
+    private loaderService;
+    private imagesService;
+    private moduleRef;
+    constructor(carService: CarService, configService: ConfigService, rulesService: RulesService, loaderService: LoaderService, imagesService: ImagesService, moduleRef: ModuleRef);
+    export(): Promise<void>;
+}

package/dist/exporters/marktde.exporter.js

@@ -0,0 +1,84 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MarktdeExporter = void 0;
+const common_1 = require("@nestjs/common");
+const core_1 = require("@nestjs/core");
+const cronicle_helper_1 = require("@bytebrand/cronicle-helper");
+const services_1 = require("../services");
+const image_url_validator_service_1 = require("../services/image-url-validator.service");
+const feed_guard_1 = require("../services/feed-guard");
+const marktde_feed_run_1 = require("../services/marktde-feed-run");
+const meinstadt_feed_run_1 = require("../services/meinstadt-feed-run");
+const mobile_transformer_1 = require("../transformers/mobile.transformer");
+const marktde_transformer_1 = require("../transformers/marktde.transformer");
+const openai_image_fields_1 = require("../transformers/openai-image-fields");
+let MarktdeExporter = class MarktdeExporter {
+    carService;
+    configService;
+    rulesService;
+    loaderService;
+    imagesService;
+    moduleRef;
+    constructor(carService, configService, rulesService, loaderService, imagesService, moduleRef) {
+        this.carService = carService;
+        this.configService = configService;
+        this.rulesService = rulesService;
+        this.loaderService = loaderService;
+        this.imagesService = imagesService;
+        this.moduleRef = moduleRef;
+    }
+    async export() {
+        const config = this.configService.config;
+        const [csvFile, zipFile] = config.outputFiles;
+        const validator = new image_url_validator_service_1.ImageUrlValidator({ concurrency: config.imageConcurrency });
+        const transformer = await this.moduleRef.create(mobile_transformer_1.MobileTransformer);
+        const run = new marktde_feed_run_1.MarktdeFeedRun({
+            forEachChunk: fn => this.loaderService.process((chunk) => fn(chunk.data)),
+            filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
+            resolveImages: async ({ car, images }) => (0, marktde_transformer_1.toMarktdeImages)(await (0, openai_image_fields_1.resolveOpenAiImageFields)(this.imagesService.getCleanImages(car._id, images), validator), config.maxImages),
+            legacyRow: async ({ car, images }) => (await transformer.transform(car, images)),
+            imageStats: () => validator.stats,
+            zip: meinstadt_feed_run_1.zipSingleFile,
+            guard: new feed_guard_1.FeedGuard(config.feedGuard),
+        }, {
+            runId: config.runId,
+            dryRun: config.dryRun,
+            domain: config.domain,
+            sourceDirectory: config.pathesConfig.localDirectory,
+            sourceBaseFileName: config.pathesConfig.baseFileName,
+            csvFile,
+            zipFile,
+            stateDir: config.stateDir,
+        });
+        try {
+            const report = await run.run();
+            if (report.stale) {
+                console.log(`marktde feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
+            }
+        }
+        catch (error) {
+            cronicle_helper_1.LoggerService.exitError(`marktde feed not published: ${error?.message ?? error}`);
+            process.exit(1);
+        }
+    }
+};
+exports.MarktdeExporter = MarktdeExporter;
+exports.MarktdeExporter = MarktdeExporter = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [services_1.CarService,
+        services_1.ConfigService,
+        services_1.RulesService,
+        services_1.LoaderService,
+        services_1.ImagesService,
+        core_1.ModuleRef])
+], MarktdeExporter);
+//# sourceMappingURL=marktde.exporter.js.map

package/dist/exporters/marktde.exporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"marktde.exporter.js","sourceRoot":"","sources":["../../src/exporters/marktde.exporter.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,uCAAyC;AAGzC,gEAA2D;AAE3D,0CAAoG;AACpG,yFAA4E;AAC5E,uDAAmD;AACnD,mEAAgF;AAChF,uEAA+D;AAC/D,2EAAuE;AACvE,6EAAsE;AACtE,6EAA+E;AAWxE,IAAM,eAAe,GAArB,MAAM,eAAe;IAEhB;IACA;IACA;IACA;IACA;IACA;IANV,YACU,UAAsB,EACtB,aAA4B,EAC5B,YAA0B,EAC1B,aAA4B,EAC5B,aAA4B,EAC5B,SAAoB;QALpB,eAAU,GAAV,UAAU,CAAY;QACtB,kBAAa,GAAb,aAAa,CAAe;QAC5B,iBAAY,GAAZ,YAAY,CAAc;QAC1B,kBAAa,GAAb,aAAa,CAAe;QAC5B,kBAAa,GAAb,aAAa,CAAe;QAC5B,cAAS,GAAT,SAAS,CAAW;IAC3B,CAAC;IAEJ,KAAK,CAAC,MAAM;QACV,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,MAAuB,CAAC;QAC1D,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QAC9C,MAAM,SAAS,GAAG,IAAI,+CAAiB,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,sCAAiB,CAAC,CAAC;QAEnE,MAAM,GAAG,GAAG,IAAI,iCAAc,CAC5B;YACE,YAAY,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,KAAuB,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,IAA0B,CAAC,CAAC;YACjH,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;YAC/E,aAAa,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,IAAA,qCAAe,EACvD,MAAM,IAAA,8CAAwB,EAAC,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,SAAS,CAAC,EAC7F,MAAM,CAAC,SAAS,CACjB;YACD,SAAS,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,WAAW,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAA4B;YAC3G,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK;YACjC,GAAG,EAAE,kCAAa;YAClB,KAAK,EAAE,IAAI,sBAAS,CAAC,MAAM,CAAC,SAAS,CAAC;SACvC,EACD;YACE,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,cAAc;YACnD,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,YAAY;YACpD,OAAO;YACP,OAAO;YACP,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CACF,CAAC;QAEF,IAAI;YACF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;YAC/B,IAAI,MAAM,CAAC,KAAK,EAAE;gBAChB,OAAO,CAAC,GAAG,CAAC,sCAAsC,MAAM,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;aAChG;SACF;QAAC,OAAO,KAAK,EAAE;YAGd,+BAAa,CAAC,SAAS,CAAC,+BAA+B,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC;YAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;CACF,CAAA;AArDY,0CAAe;0BAAf,eAAe;IAD3B,IAAA,mBAAU,GAAE;qCAGW,qBAAU;QACP,wBAAa;QACd,uBAAY;QACX,wBAAa;QACb,wBAAa;QACjB,gBAAS;GAPnB,eAAe,CAqD3B"}

package/dist/interfaces/params.interface.d.ts

@@ -34,6 +34,9 @@ export interface Params {
     autouncleDryRun?: string;
     autouncleImageConcurrency?: string;
     autouncleMaxImages?: string;
+    marktdeDryRun?: string;
+    marktdeImageConcurrency?: string;
+    marktdeMaxImages?: string;
 }
 export interface FeedItemsBigQueryTarget {
     project: string;

package/dist/services/marktde-feed-run.d.ts

@@ -0,0 +1,87 @@
+import { ImboImageResponseInterface } from '@bytebrand/chunkable';
+import { DataSchemes } from '@bytebrand/car-schema';
+import { FeedGuard, LastGood } from './feed-guard';
+export interface MarktdeSourceCar {
+    car: DataSchemes.Car;
+    images: ImboImageResponseInterface[];
+}
+export interface MarktdeRunDeps {
+    forEachChunk: (fn: (cars: MarktdeSourceCar[]) => Promise<void>) => Promise<unknown>;
+    filterCar: (source: MarktdeSourceCar) => boolean;
+    resolveImages: (source: MarktdeSourceCar) => Promise<string[]>;
+    legacyRow: (source: MarktdeSourceCar) => Promise<Record<string, unknown>>;
+    imageStats: () => Record<string, number>;
+    zip: (csvFile: string, zipFile: string) => Promise<void>;
+    guard: FeedGuard;
+}
+export interface MarktdeRunOptions {
+    runId?: string;
+    dryRun: boolean;
+    domain: string;
+    sourceDirectory: string;
+    sourceBaseFileName: string;
+    csvFile: string;
+    zipFile: string;
+    stateDir: string;
+    now?: () => Date;
+}
+export interface MarktdeRunReport {
+    runId?: string;
+    mode: 'dry-run' | 'live';
+    startedAt: string;
+    finishedAt?: string;
+    status: 'published' | 'blocked' | 'failed';
+    guard?: {
+        code: string;
+        message: string;
+    };
+    error?: string;
+    csvFile: string;
+    zipFile: string;
+    source: {
+        files: number;
+        newestChunkAt: string | null;
+        cars: number;
+    };
+    items: number;
+    dealers: number;
+    excluded: Record<string, number>;
+    vin: Record<string, number>;
+    vat: {
+        deductible: number;
+        notDeductible: number;
+        rateDefaulted: number;
+    };
+    envkv: {
+        fuelEmpty: number;
+        co2Empty: number;
+    };
+    description: {
+        items: number;
+        sanitized: number;
+        removed: {
+            url: number;
+            email: number;
+            phone: number;
+            address: number;
+        };
+    };
+    imagesPerItem: {
+        min: number | null;
+        max: number | null;
+        total: number;
+    };
+    images: Record<string, number>;
+    lastGood: LastGood | null;
+    lastGoodAgeHours: number | null;
+    stale: boolean;
+}
+export declare class MarktdeFeedRun {
+    private readonly deps;
+    private readonly options;
+    constructor(deps: MarktdeRunDeps, options: MarktdeRunOptions);
+    private now;
+    run(): Promise<MarktdeRunReport>;
+    private count;
+    private writeReport;
+}

package/dist/services/marktde-feed-run.js

@@ -0,0 +1,166 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MarktdeFeedRun = void 0;
+const fs_1 = require("fs");
+const path_1 = require("path");
+const converter_utils_1 = require("../utils/converter.utils");
+const marktde_transformer_1 = require("../transformers/marktde.transformer");
+const feed_guard_1 = require("./feed-guard");
+const increment = (counter, key) => {
+    counter[key] = (counter[key] ?? 0) + 1;
+};
+class MarktdeFeedRun {
+    deps;
+    options;
+    constructor(deps, options) {
+        this.deps = deps;
+        this.options = options;
+    }
+    now() {
+        return this.options.now ? this.options.now() : new Date();
+    }
+    async run() {
+        const { guard } = this.deps;
+        const { csvFile, zipFile, runId, dryRun } = this.options;
+        const startedAt = this.now();
+        const lastGood = await guard.readLastGood();
+        const report = {
+            runId,
+            mode: dryRun ? 'dry-run' : 'live',
+            startedAt: startedAt.toISOString(),
+            status: 'failed',
+            csvFile,
+            zipFile,
+            source: { files: 0, newestChunkAt: null, cars: 0 },
+            items: 0,
+            dealers: 0,
+            excluded: {},
+            vin: {},
+            vat: { deductible: 0, notDeductible: 0, rateDefaulted: 0 },
+            envkv: { fuelEmpty: 0, co2Empty: 0 },
+            description: { items: 0, sanitized: 0, removed: { url: 0, email: 0, phone: 0, address: 0 } },
+            imagesPerItem: { min: null, max: null, total: 0 },
+            images: {},
+            lastGood,
+            lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
+            stale: guard.isStale(lastGood, startedAt),
+        };
+        await guard.acquireLock(runId ?? 'local', startedAt);
+        const tmpFile = `${csvFile}.tmp`;
+        try {
+            const snapshot = await (0, feed_guard_1.inspectSourceSnapshot)(this.options.sourceDirectory, this.options.sourceBaseFileName);
+            report.source.files = snapshot.files;
+            report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
+            guard.checkSource(snapshot, startedAt);
+            await fs_1.promises.mkdir((0, path_1.dirname)(csvFile), { recursive: true });
+            await fs_1.promises.rm(tmpFile, { force: true });
+            const out = await fs_1.promises.open(tmpFile, 'w');
+            const seen = new Set();
+            const dealers = new Set();
+            try {
+                await this.deps.forEachChunk(async (cars) => {
+                    for (const source of cars) {
+                        report.source.cars += 1;
+                        const id = String(source?.car?._id ?? '');
+                        if (id && seen.has(id)) {
+                            increment(report.excluded, marktde_transformer_1.MarktdeExclusion.duplicateId);
+                            continue;
+                        }
+                        seen.add(id);
+                        if (!this.deps.filterCar(source)) {
+                            increment(report.excluded, marktde_transformer_1.MarktdeExclusion.filtered);
+                            continue;
+                        }
+                        const excluded = (0, marktde_transformer_1.getMarktdeExclusion)(source.car);
+                        if (excluded) {
+                            increment(report.excluded, excluded);
+                            continue;
+                        }
+                        const images = await this.deps.resolveImages(source);
+                        if (!images.length) {
+                            increment(report.excluded, marktde_transformer_1.MarktdeExclusion.noImage);
+                            continue;
+                        }
+                        const row = await this.deps.legacyRow(source);
+                        const facts = (0, marktde_transformer_1.applyMarktdeColumns)(row, source.car, { domain: this.options.domain, images });
+                        await out.write(`${(0, converter_utils_1.convertToCSV)(row)}\n`, null, 'utf8');
+                        dealers.add(String(source.car?.metaData?.ownerId ?? ''));
+                        this.count(report, images.length, facts);
+                    }
+                });
+            }
+            finally {
+                await out.close();
+            }
+            report.dealers = dealers.size;
+            report.images = this.deps.imageStats();
+            const counts = { sourceItems: report.source.cars, publishedItems: report.items };
+            guard.checkCounts(counts, lastGood);
+            report.lastGood = await guard.publish(tmpFile, csvFile, counts, this.now(), runId);
+            const zipTmp = zipFile.replace(/\.zip$/, '.tmp.zip');
+            await fs_1.promises.rm(zipTmp, { force: true });
+            await this.deps.zip(csvFile, zipTmp);
+            await fs_1.promises.rename(zipTmp, zipFile);
+            report.status = 'published';
+            report.lastGoodAgeHours = 0;
+            report.stale = false;
+            return report;
+        }
+        catch (error) {
+            if (error instanceof feed_guard_1.FeedGuardError) {
+                report.status = 'blocked';
+                report.guard = { code: error.code, message: error.message };
+            }
+            else {
+                report.error = error?.message ?? String(error);
+            }
+            await fs_1.promises.rename(tmpFile, `${csvFile}.rejected`).catch(() => undefined);
+            throw error;
+        }
+        finally {
+            await this.writeReport(report);
+            await guard.releaseLock();
+        }
+    }
+    count(report, images, facts) {
+        report.items += 1;
+        increment(report.vin, facts.vin);
+        if (facts.price.deductible) {
+            report.vat.deductible += 1;
+        }
+        else {
+            report.vat.notDeductible += 1;
+        }
+        if (facts.price.rateDefaulted) {
+            report.vat.rateDefaulted += 1;
+        }
+        if (facts.fuelEmpty) {
+            report.envkv.fuelEmpty += 1;
+        }
+        if (facts.co2Empty) {
+            report.envkv.co2Empty += 1;
+        }
+        if (facts.descriptionRemovals) {
+            report.description.items += 1;
+            const r = facts.descriptionRemovals;
+            if (r.url + r.email + r.phone + r.address > 0) {
+                report.description.sanitized += 1;
+            }
+            for (const key of Object.keys(r)) {
+                report.description.removed[key] += r[key];
+            }
+        }
+        const stats = report.imagesPerItem;
+        stats.min = stats.min === null ? images : Math.min(stats.min, images);
+        stats.max = stats.max === null ? images : Math.max(stats.max, images);
+        stats.total += images;
+    }
+    async writeReport(report) {
+        report.finishedAt = this.now().toISOString();
+        await fs_1.promises.mkdir(this.options.stateDir, { recursive: true });
+        await fs_1.promises.writeFile((0, path_1.join)(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
+        console.log(`marktde feed report: ${JSON.stringify(report)}`);
+    }
+}
+exports.MarktdeFeedRun = MarktdeFeedRun;
+//# sourceMappingURL=marktde-feed-run.js.map

package/dist/services/marktde-feed-run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"marktde-feed-run.js","sourceRoot":"","sources":["../../src/services/marktde-feed-run.ts"],"names":[],"mappings":";;;AAAA,2BAAoC;AACpC,+BAAqC;AAIrC,8DAAwD;AACxD,6EAAiH;AACjH,6CAA0F;AA6D1F,MAAM,SAAS,GAAG,CAAC,OAA+B,EAAE,GAAW,EAAE,EAAE;IACjE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAC;AAUF,MAAa,cAAc;IACI;IAAuC;IAApE,YAA6B,IAAoB,EAAmB,OAA0B;QAAjE,SAAI,GAAJ,IAAI,CAAgB;QAAmB,YAAO,GAAP,OAAO,CAAmB;IAAI,CAAC;IAE3F,GAAG;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;QAC5B,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACzD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAqB;YAC/B,KAAK;YACL,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YACjC,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;YAClC,MAAM,EAAE,QAAQ;YAChB,OAAO;YACP,OAAO;YACP,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE;YAClD,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,CAAC;YACV,QAAQ,EAAE,EAAE;YACZ,GAAG,EAAE,EAAE;YACP,GAAG,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAC1D,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE;YACpC,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE;YAC5F,aAAa,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE;YACjD,MAAM,EAAE,EAAE;YACV,QAAQ;YACR,gBAAgB,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC;YACvD,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC;SAC1C,CAAC;QAEF,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,IAAI,OAAO,EAAE,SAAS,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,GAAG,OAAO,MAAM,CAAC;QACjC,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,IAAA,kCAAqB,EAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAC5G,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,aAAa,GAAG,QAAQ,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC;YAC1E,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAEvC,MAAM,aAAE,CAAC,KAAK,CAAC,IAAA,cAAO,EAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,MAAM,aAAE,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,MAAM,GAAG,GAAG,MAAM,aAAE,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACxC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;YAClC,IAAI;gBACF,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC1C,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE;wBACzB,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;wBACxB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;wBAC1C,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;4BACtB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,sCAAgB,CAAC,WAAW,CAAC,CAAC;4BACzD,SAAS;yBACV;wBACD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACb,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;4BAChC,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,sCAAgB,CAAC,QAAQ,CAAC,CAAC;4BACtD,SAAS;yBACV;wBAED,MAAM,QAAQ,GAAG,IAAA,yCAAmB,EAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBACjD,IAAI,QAAQ,EAAE;4BACZ,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;4BACrC,SAAS;yBACV;wBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;wBACrD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;4BAClB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,sCAAgB,CAAC,OAAO,CAAC,CAAC;4BACrD,SAAS;yBACV;wBACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;wBAC9C,MAAM,KAAK,GAAG,IAAA,yCAAmB,EAAC,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;wBAC5F,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,IAAA,8BAAY,EAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;wBACxD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;wBACzD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;qBAC1C;gBACH,CAAC,CAAC,CAAC;aACJ;oBAAS;gBACR,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC;aACnB;YACD,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC9B,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YAEvC,MAAM,MAAM,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YACjF,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAEpC,MAAM,CAAC,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAC;YACnF,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACrD,MAAM,aAAE,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACrC,MAAM,aAAE,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACjC,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC5B,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACrB,OAAO,MAAM,CAAC;SACf;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,2BAAc,EAAE;gBACnC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;gBAC1B,MAAM,CAAC,KAAK,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;aAC7D;iBAAM;gBACL,MAAM,CAAC,KAAK,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;aAChD;YAED,MAAM,aAAE,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,OAAO,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACvE,MAAM,KAAK,CAAC;SACb;gBAAS;YACR,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC/B,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;SAC3B;IACH,CAAC;IAEO,KAAK,CAAC,MAAwB,EAAE,MAAc,EAAE,KAA6C;QACnG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;QAClB,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE;YAC1B,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;SAC5B;aAAM;YACL,MAAM,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,CAAC;SAC/B;QACD,IAAI,KAAK,CAAC,KAAK,CAAC,aAAa,EAAE;YAC7B,MAAM,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,CAAC;SAC/B;QACD,IAAI,KAAK,CAAC,SAAS,EAAE;YACnB,MAAM,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;SAC7B;QACD,IAAI,KAAK,CAAC,QAAQ,EAAE;YAClB,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;SAC5B;QACD,IAAI,KAAK,CAAC,mBAAmB,EAAE;YAC7B,MAAM,CAAC,WAAW,CAAC,KAAK,IAAI,CAAC,CAAC;YAC9B,MAAM,CAAC,GAAG,KAAK,CAAC,mBAAmB,CAAC;YACpC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,OAAO,GAAG,CAAC,EAAE;gBAC7C,MAAM,CAAC,WAAW,CAAC,SAAS,IAAI,CAAC,CAAC;aACnC;YACD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAA0B,EAAE;gBACzD,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;aAC3C;SACF;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,CAAC;QACnC,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACtE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACtE,KAAK,CAAC,KAAK,IAAI,MAAM,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAwB;QAChD,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,aAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,aAAE,CAAC,SAAS,CAAC,IAAA,WAAI,EAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,sBAAsB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACjH,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAChE,CAAC;CACF;AAxJD,wCAwJC"}

package/dist/transformers/marktde.transformer.d.ts

@@ -0,0 +1,58 @@
+import { DataSchemes } from '@bytebrand/car-schema';
+import { OpenAiImageSelection } from './openai-image-fields';
+export declare const MARKTDE_COLUMNS: {
+    readonly vehicleId: "1";
+    readonly price: "10";
+    readonly vatFlag: "11";
+    readonly vin: "14";
+    readonly description: "25";
+    readonly images: "26";
+    readonly vatRate: "29";
+    readonly fuelCombined: "98";
+    readonly co2: "99";
+    readonly listingUrl: "402";
+};
+export declare const MARKTDE_LINK_PARAMS = "source=marktde&medium=listing&campaign=listing&utm_source=marktde&utm_medium=listing&utm_campaign=listing";
+export declare const getMarktdeListingUrl: (domain: string, carId: string) => string;
+export declare enum MarktdeExclusion {
+    duplicateId = "duplicate_id",
+    filtered = "filtered",
+    noId = "no_id",
+    noVehicleId = "no_vehicle_id",
+    notAvailable = "reserved_or_sold",
+    noPrice = "no_valid_price",
+    noImage = "no_valid_image"
+}
+export declare const getMarktdeExclusion: (car: DataSchemes.Car) => MarktdeExclusion | null;
+export declare const getMarktdeVin: (car: DataSchemes.Car) => string;
+export interface MarktdePriceColumns {
+    price: string;
+    vatFlag: string;
+    vatRate: string;
+    deductible: boolean;
+    rateDefaulted: boolean;
+}
+export declare const getMarktdePriceColumns: (car: DataSchemes.Car) => MarktdePriceColumns | null;
+export declare const toMarktdeImages: (selection: OpenAiImageSelection, maxImages?: number) => string[];
+export interface DescriptionRemovals {
+    url: number;
+    email: number;
+    phone: number;
+    address: number;
+}
+export declare const sanitizeMarktdeDescription: (text: string) => {
+    text: string;
+    removed: DescriptionRemovals;
+};
+export interface MarktdeRowContext {
+    domain: string;
+    images: string[];
+}
+export interface MarktdeRowFacts {
+    vin: 'sent' | 'empty' | 'invalid_dropped';
+    price: MarktdePriceColumns;
+    descriptionRemovals: DescriptionRemovals | null;
+    fuelEmpty: boolean;
+    co2Empty: boolean;
+}
+export declare const applyMarktdeColumns: (row: Record<string, unknown>, car: DataSchemes.Car, ctx: MarktdeRowContext) => MarktdeRowFacts;