@bytebrand/export-cars 1.26.39 → 1.26.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/configs/autouncle.config.js +22 -0
- package/dist/configs/autouncle.config.js.map +1 -1
- package/dist/configs/criteo.config.d.ts +10 -0
- package/dist/configs/criteo.config.js +26 -1
- package/dist/configs/criteo.config.js.map +1 -1
- package/dist/exporters/autouncle.exporter.js +10 -0
- package/dist/exporters/autouncle.exporter.js.map +1 -1
- package/dist/exporters/criteo.exporter.js +27 -23
- package/dist/exporters/criteo.exporter.js.map +1 -1
- package/dist/services/autouncle-feed-run.d.ts +6 -0
- package/dist/services/autouncle-feed-run.js +10 -1
- package/dist/services/autouncle-feed-run.js.map +1 -1
- package/dist/services/criteo-feed-run.d.ts +76 -0
- package/dist/services/criteo-feed-run.js +186 -0
- package/dist/services/criteo-feed-run.js.map +1 -0
- package/dist/transformers/autouncle.transformer.d.ts +3 -0
- package/dist/transformers/autouncle.transformer.js +9 -5
- package/dist/transformers/autouncle.transformer.js.map +1 -1
- package/dist/transformers/google-feed-envkv.d.ts +2 -0
- package/dist/transformers/google-feed-envkv.js +9 -1
- package/dist/transformers/google-feed-envkv.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/docs/autouncle/AUDIT.md +7 -3
- package/docs/autouncle/PARTNER-QUESTIONS.md +3 -1
- package/docs/autouncle/RUNBOOK.md +2 -0
- package/docs/criteo/RUNBOOK.md +86 -0
- package/package.json +2 -2
- package/src/configs/autouncle.config.ts +26 -1
- package/src/configs/criteo.config.ts +37 -0
- package/src/exporters/autouncle.exporter.ts +12 -0
- package/src/exporters/criteo.exporter.ts +40 -28
- package/src/services/autouncle-feed-run.ts +15 -1
- package/src/services/criteo-feed-run.ts +253 -0
- package/src/transformers/autouncle.transformer.ts +17 -4
- package/src/transformers/google-feed-envkv.ts +10 -0
- package/test/autouncle-feed-run.test.ts +25 -2
- package/test/autouncle-mapping.test.ts +41 -2
- package/test/criteo-feed-run.test.ts +365 -0
- package/test/criteo-mapping.test.ts +122 -122
- package/test/fixtures/criteo-20-cases.json +582 -0
|
@@ -41,10 +41,32 @@ class AutouncleConfig extends config_1.Config {
|
|
|
41
41
|
maxStaleHours: positiveNumber(params.feedGuardMaxStaleHours, exports.AUTOUNCLE_DEFAULTS.maxStaleHours),
|
|
42
42
|
acceptDrop: text(params.feedGuardAcceptDrop).toLowerCase() === 'true',
|
|
43
43
|
};
|
|
44
|
+
this.publicationGate = parsePublicationGate(params);
|
|
45
|
+
if (!this.dryRun && this.publicationGate?.mode !== 'enforce') {
|
|
46
|
+
throw new Error('a live AutoUncle run (autouncleDryRun=false) needs publicationGate=enforce');
|
|
47
|
+
}
|
|
44
48
|
this.outputFiles = [
|
|
45
49
|
`${this.pathesConfig.localConvertedDirectory}/api_export_cars_${enums_1.ExportType.AUTOUNCLE}.${enums_1.FileType.JSON}`,
|
|
46
50
|
];
|
|
47
51
|
}
|
|
48
52
|
}
|
|
49
53
|
exports.AutouncleConfig = AutouncleConfig;
|
|
54
|
+
function parsePublicationGate(params) {
|
|
55
|
+
const mode = text(params.publicationGate).toLowerCase();
|
|
56
|
+
if (!mode || mode === 'off')
|
|
57
|
+
return undefined;
|
|
58
|
+
if (mode !== 'audit' && mode !== 'enforce') {
|
|
59
|
+
throw new Error(`publicationGate must be off, audit or enforce, got "${mode}"`);
|
|
60
|
+
}
|
|
61
|
+
const gate = {
|
|
62
|
+
mode: mode,
|
|
63
|
+
project: text(params.publicationGateBqProject),
|
|
64
|
+
dataset: text(params.publicationGateBqDataset),
|
|
65
|
+
table: text(params.publicationGateBqTable),
|
|
66
|
+
};
|
|
67
|
+
if (!gate.project || !gate.dataset || !gate.table) {
|
|
68
|
+
throw new Error(`publicationGate=${mode} needs publicationGateBqProject, publicationGateBqDataset and publicationGateBqTable`);
|
|
69
|
+
}
|
|
70
|
+
return gate;
|
|
71
|
+
}
|
|
50
72
|
//# sourceMappingURL=autouncle.config.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"autouncle.config.js","sourceRoot":"","sources":["../../src/configs/autouncle.config.ts"],"names":[],"mappings":";;;AAAA,mCAAkC;AAElC,oCAA2D;AAG3D,qCAAkC;AAElC,MAAM,IAAI,GAAG,CAAC,KAAc,EAAU,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAEzF,MAAM,cAAc,GAAG,CAAC,KAAc,EAAE,QAAgB,EAAU,EAAE;IAClE,MAAM,MAAM,GAAG,IAAA,iBAAQ,EAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACrC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClF,CAAC,CAAC;AAIW,QAAA,kBAAkB,GAAG;IAChC,gBAAgB,EAAE,EAAE;IACpB,cAAc,EAAE,IAAI;IACpB,iBAAiB,EAAE,CAAC;IACpB,cAAc,EAAE,EAAE;IAClB,iBAAiB,EAAE,EAAE;IACrB,aAAa,EAAE,EAAE;CAClB,CAAC;AAQF,MAAa,eAAgB,SAAQ,eAAM;IACzC,MAAM,CAAU;IAChB,gBAAgB,CAAS;IAEzB,SAAS,CAAS;IAClB,QAAQ,CAAS;IACjB,SAAS,CAAmB;IAE5B,YAAY,MAAc;QACxB,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC;QACrE,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC/C,IAAI,CAAC,YAAY,CAAC,uBAAuB,GAAG,aAAa,kBAAU,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;QACxF,IAAI,CAAC,QAAQ,GAAG,aAAa,kBAAU,CAAC,SAAS,UAAU,IAAI,EAAE,CAAC;QAClE,IAAI,CAAC,SAAS,GAAG,iBAAS,CAAC,KAAK,CAAC;QACjC,IAAI,CAAC,gBAAgB,GAAG,cAAc,CAAC,MAAM,CAAC,yBAAyB,EAAE,0BAAkB,CAAC,gBAAgB,CAAC,CAAC;QAC9G,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,SAAS,GAAG;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,0BAAkB,CAAC,cAAc,CAAC;YACjG,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,0BAAkB,CAAC,iBAAiB,CAAC;YAC1G,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,0BAAkB,CAAC,cAAc,CAAC;YACjG,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,0BAAkB,CAAC,iBAAiB,CAAC;YAC1G,aAAa,EAAE,cAAc,CAAC,MAAM,CAAC,sBAAsB,EAAE,0BAAkB,CAAC,aAAa,CAAC;YAC9F,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM;SACtE,CAAC;QACF,IAAI,CAAC,WAAW,GAAG;YACjB,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,oBAAoB,kBAAU,CAAC,SAAS,IAAI,gBAAQ,CAAC,IAAI,EAAE;SACxG,CAAC;IACJ,CAAC;CACF;
|
|
1
|
+
{"version":3,"file":"autouncle.config.js","sourceRoot":"","sources":["../../src/configs/autouncle.config.ts"],"names":[],"mappings":";;;AAAA,mCAAkC;AAElC,oCAA2D;AAG3D,qCAAkC;AAElC,MAAM,IAAI,GAAG,CAAC,KAAc,EAAU,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAEzF,MAAM,cAAc,GAAG,CAAC,KAAc,EAAE,QAAgB,EAAU,EAAE;IAClE,MAAM,MAAM,GAAG,IAAA,iBAAQ,EAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACrC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClF,CAAC,CAAC;AAIW,QAAA,kBAAkB,GAAG;IAChC,gBAAgB,EAAE,EAAE;IACpB,cAAc,EAAE,IAAI;IACpB,iBAAiB,EAAE,CAAC;IACpB,cAAc,EAAE,EAAE;IAClB,iBAAiB,EAAE,EAAE;IACrB,aAAa,EAAE,EAAE;CAClB,CAAC;AAQF,MAAa,eAAgB,SAAQ,eAAM;IACzC,MAAM,CAAU;IAChB,gBAAgB,CAAS;IAEzB,SAAS,CAAS;IAClB,QAAQ,CAAS;IACjB,SAAS,CAAmB;IAE5B,YAAY,MAAc;QACxB,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC;QACrE,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC/C,IAAI,CAAC,YAAY,CAAC,uBAAuB,GAAG,aAAa,kBAAU,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;QACxF,IAAI,CAAC,QAAQ,GAAG,aAAa,kBAAU,CAAC,SAAS,UAAU,IAAI,EAAE,CAAC;QAClE,IAAI,CAAC,SAAS,GAAG,iBAAS,CAAC,KAAK,CAAC;QACjC,IAAI,CAAC,gBAAgB,GAAG,cAAc,CAAC,MAAM,CAAC,yBAAyB,EAAE,0BAAkB,CAAC,gBAAgB,CAAC,CAAC;QAC9G,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,SAAS,GAAG;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,0BAAkB,CAAC,cAAc,CAAC;YACjG,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,0BAAkB,CAAC,iBAAiB,CAAC;YAC1G,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,0BAAkB,CAAC,cAAc,CAAC;YACjG,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,0BAAkB,CAAC,iBAAiB,CAAC;YAC1G,aAAa,EAAE,cAAc,CAAC,MAAM,CAAC,sBAAsB,EAAE,0BAAkB,CAAC,aAAa,CAAC;YAC9F,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM;SACtE,CAAC;QACF,IAAI,CAAC,eAAe,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,KAAK,SAAS,EAAE;YAC5D,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;SAC/F;QACD,IAAI,CAAC,WAAW,GAAG;YACjB,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,oBAAoB,kBAAU,CAAC,SAAS,IAAI,gBAAQ,CAAC,IAAI,EAAE;SACxG,CAAC;IACJ,CAAC;CACF;AAnCD,0CAmCC;AAID,SAAS,oBAAoB,CAAC,MAAc;IAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;IACxD,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,SAAS,CAAC;IAC9C,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,SAAS,EAAE;QAC1C,MAAM,IAAI,KAAK,CAAC,uDAAuD,IAAI,GAAG,CAAC,CAAC;KACjF;IACD,MAAM,IAAI,GAAG;QACX,IAAI,EAAE,IAAqC;QAC3C,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,wBAAwB,CAAC;QAC9C,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,wBAAwB,CAAC;QAC9C,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC;KAC3C,CAAC;IACF,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;QACjD,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,sFAAsF,CAAC,CAAC;KAChI;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
import { Params } from '../interfaces/params.interface';
|
|
2
|
+
import { FeedGuardOptions } from '../services/feed-guard';
|
|
2
3
|
import { Config } from './config';
|
|
4
|
+
export declare const CRITEO_DEFAULTS: {
|
|
5
|
+
minSourceItems: number;
|
|
6
|
+
minPublishedItems: number;
|
|
7
|
+
maxDropPercent: number;
|
|
8
|
+
maxSourceAgeHours: number;
|
|
9
|
+
maxStaleHours: number;
|
|
10
|
+
};
|
|
3
11
|
export declare class CriteoConfig extends Config {
|
|
12
|
+
stateDir: string;
|
|
13
|
+
feedGuard: FeedGuardOptions;
|
|
4
14
|
constructor(params: Params);
|
|
5
15
|
}
|
|
@@ -1,15 +1,40 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.CriteoConfig = void 0;
|
|
3
|
+
exports.CriteoConfig = exports.CRITEO_DEFAULTS = void 0;
|
|
4
|
+
const lodash_1 = require("lodash");
|
|
4
5
|
const enums_1 = require("../enums");
|
|
5
6
|
const config_1 = require("./config");
|
|
7
|
+
const text = (value) => (typeof value === 'string' ? value.trim() : '');
|
|
8
|
+
const positiveNumber = (value, fallback) => {
|
|
9
|
+
const parsed = (0, lodash_1.toNumber)(text(value));
|
|
10
|
+
return text(value) && Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
11
|
+
};
|
|
12
|
+
exports.CRITEO_DEFAULTS = {
|
|
13
|
+
minSourceItems: 1000,
|
|
14
|
+
minPublishedItems: 500,
|
|
15
|
+
maxDropPercent: 20,
|
|
16
|
+
maxSourceAgeHours: 26,
|
|
17
|
+
maxStaleHours: 48,
|
|
18
|
+
};
|
|
6
19
|
class CriteoConfig extends config_1.Config {
|
|
20
|
+
stateDir;
|
|
21
|
+
feedGuard;
|
|
7
22
|
constructor(params) {
|
|
8
23
|
super(params);
|
|
9
24
|
this.campaign = enums_1.ExportType.CRITEO;
|
|
10
25
|
this.pathesConfig.localConvertedDirectory = `resources/${enums_1.ExportType.CRITEO}-export`;
|
|
26
|
+
this.stateDir = `resources/${enums_1.ExportType.CRITEO}-state`;
|
|
11
27
|
this.imageType = enums_1.ImageType.marketing;
|
|
12
28
|
this.marketingImageConfigId = this.environmentConfig[params.environment].criteoMarketingImagesConfigId;
|
|
29
|
+
this.feedGuard = {
|
|
30
|
+
stateDir: this.stateDir,
|
|
31
|
+
minSourceItems: positiveNumber(params.feedGuardMinSourceItems, exports.CRITEO_DEFAULTS.minSourceItems),
|
|
32
|
+
minPublishedItems: positiveNumber(params.feedGuardMinPublishedItems, exports.CRITEO_DEFAULTS.minPublishedItems),
|
|
33
|
+
maxDropPercent: positiveNumber(params.feedGuardMaxDropPercent, exports.CRITEO_DEFAULTS.maxDropPercent),
|
|
34
|
+
maxSourceAgeHours: positiveNumber(params.feedGuardMaxSourceAgeHours, exports.CRITEO_DEFAULTS.maxSourceAgeHours),
|
|
35
|
+
maxStaleHours: positiveNumber(params.feedGuardMaxStaleHours, exports.CRITEO_DEFAULTS.maxStaleHours),
|
|
36
|
+
acceptDrop: text(params.feedGuardAcceptDrop).toLowerCase() === 'true',
|
|
37
|
+
};
|
|
13
38
|
this.outputFiles = [
|
|
14
39
|
`${this.pathesConfig.localConvertedDirectory}/api_export_cars_${enums_1.ExportType.CRITEO}.${enums_1.FileType.XML}`,
|
|
15
40
|
];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"criteo.config.js","sourceRoot":"","sources":["../../src/configs/criteo.config.ts"],"names":[],"mappings":";;;AAAA,oCAA4D;
|
|
1
|
+
{"version":3,"file":"criteo.config.js","sourceRoot":"","sources":["../../src/configs/criteo.config.ts"],"names":[],"mappings":";;;AAAA,mCAAkC;AAElC,oCAA4D;AAG5D,qCAAkC;AAElC,MAAM,IAAI,GAAG,CAAC,KAAc,EAAU,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAEzF,MAAM,cAAc,GAAG,CAAC,KAAc,EAAE,QAAgB,EAAU,EAAE;IAClE,MAAM,MAAM,GAAG,IAAA,iBAAQ,EAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACrC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClF,CAAC,CAAC;AAIW,QAAA,eAAe,GAAG;IAC7B,cAAc,EAAE,IAAI;IACpB,iBAAiB,EAAE,GAAG;IACtB,cAAc,EAAE,EAAE;IAClB,iBAAiB,EAAE,EAAE;IACrB,aAAa,EAAE,EAAE;CAClB,CAAC;AAMF,MAAa,YAAa,SAAQ,eAAM;IACtC,QAAQ,CAAS;IACjB,SAAS,CAAmB;IAE5B,YAAY,MAAc;QACxB,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,QAAQ,GAAG,kBAAU,CAAC,MAAM,CAAC;QAClC,IAAI,CAAC,YAAY,CAAC,uBAAuB,GAAG,aAAa,kBAAU,CAAC,MAAM,SAAS,CAAC;QACpF,IAAI,CAAC,QAAQ,GAAG,aAAa,kBAAU,CAAC,MAAM,QAAQ,CAAC;QACvD,IAAI,CAAC,SAAS,GAAG,iBAAS,CAAC,SAAS,CAAC;QACrC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,6BAA6B,CAAC;QACvG,IAAI,CAAC,SAAS,GAAG;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,uBAAe,CAAC,cAAc,CAAC;YAC9F,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,uBAAe,CAAC,iBAAiB,CAAC;YACvG,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,uBAAuB,EAAE,uBAAe,CAAC,cAAc,CAAC;YAC9F,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,0BAA0B,EAAE,uBAAe,CAAC,iBAAiB,CAAC;YACvG,aAAa,EAAE,cAAc,CAAC,MAAM,CAAC,sBAAsB,EAAE,uBAAe,CAAC,aAAa,CAAC;YAC3F,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM;SACtE,CAAC;QACF,IAAI,CAAC,WAAW,GAAG;YACjB,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,oBAAoB,kBAAU,CAAC,MAAM,IAAI,gBAAQ,CAAC,GAAG,EAAE;SACpG,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;CACF;AAzBD,oCAyBC"}
|
|
@@ -18,6 +18,7 @@ const openai_image_fields_1 = require("../transformers/openai-image-fields");
|
|
|
18
18
|
const services_1 = require("../services");
|
|
19
19
|
const image_url_validator_service_1 = require("../services/image-url-validator.service");
|
|
20
20
|
const feed_guard_1 = require("../services/feed-guard");
|
|
21
|
+
const publication_gate_1 = require("../services/publication-gate");
|
|
21
22
|
const autouncle_feed_run_1 = require("../services/autouncle-feed-run");
|
|
22
23
|
let AutouncleExporter = class AutouncleExporter {
|
|
23
24
|
carService;
|
|
@@ -40,6 +41,14 @@ let AutouncleExporter = class AutouncleExporter {
|
|
|
40
41
|
const validator = new image_url_validator_service_1.ImageUrlValidator({ concurrency: config.imageConcurrency });
|
|
41
42
|
const transformer = await this.moduleRef.create(autouncle_transformer_1.AutouncleTransformer);
|
|
42
43
|
const addresses = new Map();
|
|
44
|
+
let gate;
|
|
45
|
+
try {
|
|
46
|
+
gate = await (0, publication_gate_1.loadPublicationGate)(config.publicationGate);
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
cronicle_helper_1.LoggerService.exitError(`autouncle feed not published: ${error?.message ?? error}`);
|
|
50
|
+
process.exit(1);
|
|
51
|
+
}
|
|
43
52
|
const run = new autouncle_feed_run_1.AutouncleFeedRun({
|
|
44
53
|
forEachChunk: fn => this.loaderService.process((chunk) => fn(chunk.data)),
|
|
45
54
|
filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
|
|
@@ -55,6 +64,7 @@ let AutouncleExporter = class AutouncleExporter {
|
|
|
55
64
|
imageStats: () => validator.stats,
|
|
56
65
|
translate: key => transformer.translate(key),
|
|
57
66
|
guard: new feed_guard_1.FeedGuard(config.feedGuard),
|
|
67
|
+
gate,
|
|
58
68
|
}, {
|
|
59
69
|
runId: config.runId,
|
|
60
70
|
dryRun: config.dryRun,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"autouncle.exporter.js","sourceRoot":"","sources":["../../src/exporters/autouncle.exporter.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,uCAAyC;AAGzC,gEAA2D;AAE3D,iFAAkH;AAClH,6EAA+E;AAC/E,0CAAoG;AACpG,yFAA4E;AAC5E,uDAAmD;AACnD,uEAAsF;AAU/E,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAElB;IACA;IACA;IACA;IACA;IACA;IANV,YACU,UAAsB,EACtB,aAA4B,EAC5B,aAA4B,EAC5B,YAA0B,EAC1B,aAA4B,EAC5B,SAAoB;QALpB,eAAU,GAAV,UAAU,CAAY;QACtB,kBAAa,GAAb,aAAa,CAAe;QAC5B,kBAAa,GAAb,aAAa,CAAe;QAC5B,iBAAY,GAAZ,YAAY,CAAc;QAC1B,kBAAa,GAAb,aAAa,CAAe;QAC5B,cAAS,GAAT,SAAS,CAAW;IAC3B,CAAC;IAEJ,KAAK,CAAC,MAAM;QACV,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,MAAyB,CAAC;QAC5D,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,MAAM,SAAS,GAAG,IAAI,+CAAiB,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,4CAAoB,CAAC,CAAC;QACtE,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4C,CAAC;
|
|
1
|
+
{"version":3,"file":"autouncle.exporter.js","sourceRoot":"","sources":["../../src/exporters/autouncle.exporter.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,uCAAyC;AAGzC,gEAA2D;AAE3D,iFAAkH;AAClH,6EAA+E;AAC/E,0CAAoG;AACpG,yFAA4E;AAC5E,uDAAmD;AACnD,mEAAoF;AACpF,uEAAsF;AAU/E,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAElB;IACA;IACA;IACA;IACA;IACA;IANV,YACU,UAAsB,EACtB,aAA4B,EAC5B,aAA4B,EAC5B,YAA0B,EAC1B,aAA4B,EAC5B,SAAoB;QALpB,eAAU,GAAV,UAAU,CAAY;QACtB,kBAAa,GAAb,aAAa,CAAe;QAC5B,kBAAa,GAAb,aAAa,CAAe;QAC5B,iBAAY,GAAZ,YAAY,CAAc;QAC1B,kBAAa,GAAb,aAAa,CAAe;QAC5B,cAAS,GAAT,SAAS,CAAW;IAC3B,CAAC;IAEJ,KAAK,CAAC,MAAM;QACV,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,MAAyB,CAAC;QAC5D,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,MAAM,SAAS,GAAG,IAAI,+CAAiB,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,4CAAoB,CAAC,CAAC;QACtE,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4C,CAAC;QAItE,IAAI,IAAqB,CAAC;QAC1B,IAAI;YACF,IAAI,GAAG,MAAM,IAAA,sCAAmB,EAAC,MAAM,CAAC,eAAe,CAAC,CAAC;SAC1D;QAAC,OAAO,KAAK,EAAE;YACd,+BAAa,CAAC,SAAS,CAAC,iCAAiC,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC;YACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,MAAM,GAAG,GAAG,IAAI,qCAAgB,CAC9B;YACE,YAAY,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,KAAuB,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,IAA4B,CAAC,CAAC;YACnH,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;YAC/E,aAAa,EAAE,CAAC,OAAO,EAAE,EAAE;gBACzB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;oBAC3B,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC;yBACrD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;yBAC7D,KAAK,CAAC,CAAC,KAAU,EAAE,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBACrF;gBACD,OAAO,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;YACD,aAAa,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,IAAA,yCAAiB,EACzD,MAAM,IAAA,8CAAwB,EAAC,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,SAAS,CAAC,EAC7F,MAAM,CAAC,SAAS,CACjB;YACD,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK;YACjC,SAAS,EAAE,GAAG,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC;YAC5C,KAAK,EAAE,IAAI,sBAAS,CAAC,MAAM,CAAC,SAAS,CAAC;YACtC,IAAI;SACL,EACD;YACE,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,cAAc;YACnD,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,YAAY;YACpD,QAAQ;YACR,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CACF,CAAC;QAEF,IAAI;YACF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;YAC/B,IAAI,MAAM,CAAC,KAAK,EAAE;gBAChB,OAAO,CAAC,GAAG,CAAC,wCAAwC,MAAM,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;aAClG;SACF;QAAC,OAAO,KAAK,EAAE;YAGd,+BAAa,CAAC,SAAS,CAAC,iCAAiC,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC;YACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;CACF,CAAA;AAvEY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAGW,qBAAU;QACP,wBAAa;QACb,wBAAa;QACd,uBAAY;QACX,wBAAa;QACjB,gBAAS;GAPnB,iBAAiB,CAuE7B"}
|
|
@@ -14,7 +14,8 @@ const common_1 = require("@nestjs/common");
|
|
|
14
14
|
const core_1 = require("@nestjs/core");
|
|
15
15
|
const cronicle_helper_1 = require("@bytebrand/cronicle-helper");
|
|
16
16
|
const services_1 = require("../services");
|
|
17
|
-
const
|
|
17
|
+
const feed_guard_1 = require("../services/feed-guard");
|
|
18
|
+
const criteo_feed_run_1 = require("../services/criteo-feed-run");
|
|
18
19
|
const criteo_transformer_1 = require("../transformers/criteo.transformer");
|
|
19
20
|
let CriteoExporter = class CriteoExporter {
|
|
20
21
|
carService;
|
|
@@ -30,29 +31,32 @@ let CriteoExporter = class CriteoExporter {
|
|
|
30
31
|
this.moduleRef = moduleRef;
|
|
31
32
|
}
|
|
32
33
|
async export() {
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
await
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
withoutImage += 1;
|
|
49
|
-
}
|
|
50
|
-
this.carService.counter.countCar(data.images);
|
|
51
|
-
}
|
|
34
|
+
const config = this.configService.config;
|
|
35
|
+
const [xmlFile] = config.outputFiles;
|
|
36
|
+
const transformer = await this.moduleRef.create(criteo_transformer_1.CriteoTransformer);
|
|
37
|
+
const run = new criteo_feed_run_1.CriteoFeedRun({
|
|
38
|
+
forEachChunk: fn => this.loaderService.process((chunk) => fn(chunk.data)),
|
|
39
|
+
filterCar: source => this.carService.filterCar(source, this.rulesService.rules),
|
|
40
|
+
transform: source => transformer.transform(source),
|
|
41
|
+
guard: new feed_guard_1.FeedGuard(config.feedGuard),
|
|
42
|
+
}, {
|
|
43
|
+
runId: config.runId,
|
|
44
|
+
domain: config.domain,
|
|
45
|
+
sourceDirectory: config.pathesConfig.localDirectory,
|
|
46
|
+
sourceBaseFileName: config.pathesConfig.baseFileName,
|
|
47
|
+
xmlFile,
|
|
48
|
+
stateDir: config.stateDir,
|
|
52
49
|
});
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
50
|
+
try {
|
|
51
|
+
const report = await run.run();
|
|
52
|
+
if (report.stale) {
|
|
53
|
+
console.log(`criteo feed STALE: last good file ${report.lastGoodAgeHours?.toFixed(1)} h old`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
cronicle_helper_1.LoggerService.exitError(`criteo feed not published: ${error?.message ?? error}`);
|
|
58
|
+
process.exit(1);
|
|
59
|
+
}
|
|
56
60
|
}
|
|
57
61
|
};
|
|
58
62
|
exports.CriteoExporter = CriteoExporter;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"criteo.exporter.js","sourceRoot":"","sources":["../../src/exporters/criteo.exporter.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,uCAAyC;
|
|
1
|
+
{"version":3,"file":"criteo.exporter.js","sourceRoot":"","sources":["../../src/exporters/criteo.exporter.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA4C;AAC5C,uCAAyC;AAGzC,gEAA2D;AAE3D,0CAAqF;AACrF,uDAAmD;AACnD,iEAA6E;AAG7E,2EAAwE;AAQjE,IAAM,cAAc,GAApB,MAAM,cAAc;IAEjB;IACA;IACA;IACA;IACA;IALT,YACS,UAAsB,EACtB,aAA4B,EAC5B,YAA0B,EAC1B,aAA4B,EAC5B,SAAoB;QAJpB,eAAU,GAAV,UAAU,CAAY;QACtB,kBAAa,GAAb,aAAa,CAAe;QAC5B,iBAAY,GAAZ,YAAY,CAAc;QAC1B,kBAAa,GAAb,aAAa,CAAe;QAC5B,cAAS,GAAT,SAAS,CAAW;IACzB,CAAC;IAEL,KAAK,CAAC,MAAM;QACX,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,MAAsB,CAAC;QACzD,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QACrC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,sCAAiB,CAAC,CAAC;QAEnE,MAAM,GAAG,GAAG,IAAI,+BAAa,CAC5B;YACC,YAAY,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,KAAuB,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,IAAyB,CAAC,CAAC;YAChH,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;YAC/E,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC;YAClD,KAAK,EAAE,IAAI,sBAAS,CAAC,MAAM,CAAC,SAAS,CAAC;SACtC,EACD;YACC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,cAAc;YACnD,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,YAAY;YACpD,OAAO;YACP,QAAQ,EAAE,MAAM,CAAC,QAAQ;SACzB,CACD,CAAC;QAEF,IAAI;YACH,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;YAC/B,IAAI,MAAM,CAAC,KAAK,EAAE;gBACjB,OAAO,CAAC,GAAG,CAAC,qCAAqC,MAAM,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;aAC9F;SACD;QAAC,OAAO,KAAK,EAAE;YAGf,+BAAa,CAAC,SAAS,CAAC,8BAA8B,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC;YACjF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAChB;IACF,CAAC;CACD,CAAA;AA3CY,wCAAc;yBAAd,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAGS,qBAAU;QACP,wBAAa;QACd,uBAAY;QACX,wBAAa;QACjB,gBAAS;GANjB,cAAc,CA2C1B"}
|
|
@@ -2,6 +2,7 @@ import { ImboImageResponseInterface } from '@bytebrand/chunkable';
|
|
|
2
2
|
import { DataSchemes } from '@bytebrand/car-schema';
|
|
3
3
|
import { AutouncleAddress } from '../transformers/autouncle.transformer';
|
|
4
4
|
import { FeedGuard, LastGood } from './feed-guard';
|
|
5
|
+
import { PublicationGate } from './publication-gate';
|
|
5
6
|
export interface AutouncleSourceCar {
|
|
6
7
|
car: DataSchemes.Car;
|
|
7
8
|
images: ImboImageResponseInterface[];
|
|
@@ -14,6 +15,7 @@ export interface AutouncleRunDeps {
|
|
|
14
15
|
imageStats: () => Record<string, number>;
|
|
15
16
|
translate: (key: string) => string;
|
|
16
17
|
guard: FeedGuard;
|
|
18
|
+
gate?: Pick<PublicationGate, 'mode' | 'estimatedAttributes'>;
|
|
17
19
|
}
|
|
18
20
|
export interface AutouncleRunOptions {
|
|
19
21
|
runId?: string;
|
|
@@ -51,6 +53,10 @@ export interface AutouncleRunReport {
|
|
|
51
53
|
total: number;
|
|
52
54
|
};
|
|
53
55
|
images: Record<string, number>;
|
|
56
|
+
envkv: {
|
|
57
|
+
gate: 'off' | 'audit' | 'enforce';
|
|
58
|
+
estimated: Record<string, number>;
|
|
59
|
+
};
|
|
54
60
|
lastGood: LastGood | null;
|
|
55
61
|
lastGoodAgeHours: number | null;
|
|
56
62
|
stale: boolean;
|
|
@@ -36,6 +36,7 @@ class AutouncleFeedRun {
|
|
|
36
36
|
excluded: {},
|
|
37
37
|
imagesPerItem: { min: null, max: null, total: 0 },
|
|
38
38
|
images: {},
|
|
39
|
+
envkv: { gate: this.deps.gate?.mode ?? 'off', estimated: {} },
|
|
39
40
|
lastGood,
|
|
40
41
|
lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
|
|
41
42
|
stale: guard.isStale(lastGood, startedAt),
|
|
@@ -79,11 +80,19 @@ class AutouncleFeedRun {
|
|
|
79
80
|
continue;
|
|
80
81
|
}
|
|
81
82
|
const images = await this.deps.resolveImages(source);
|
|
82
|
-
const
|
|
83
|
+
const estimated = this.deps.gate ? this.deps.gate.estimatedAttributes(source.car) : [];
|
|
84
|
+
const mapped = (0, autouncle_transformer_1.mapAutouncleItem)(source.car, {
|
|
85
|
+
domain: this.options.domain,
|
|
86
|
+
address,
|
|
87
|
+
images,
|
|
88
|
+
translate: this.deps.translate,
|
|
89
|
+
withheldEnvkv: this.deps.gate?.mode === 'enforce' ? estimated : [],
|
|
90
|
+
});
|
|
83
91
|
if (mapped.ok === false) {
|
|
84
92
|
increment(report.excluded, mapped.reason);
|
|
85
93
|
continue;
|
|
86
94
|
}
|
|
95
|
+
estimated.forEach(attribute => increment(report.envkv.estimated, attribute));
|
|
87
96
|
await builder.addItem(mapped.item, first);
|
|
88
97
|
first = false;
|
|
89
98
|
dealers.add(ownerId);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"autouncle-feed-run.js","sourceRoot":"","sources":["../../src/services/autouncle-feed-run.ts"],"names":[],"mappings":";;;AAAA,2BAAoC;AACpC,+BAAqC;AAIrC,8EAAyE;AACzE,iFAAsI;AACtI,6CAA0F;
|
|
1
|
+
{"version":3,"file":"autouncle-feed-run.js","sourceRoot":"","sources":["../../src/services/autouncle-feed-run.ts"],"names":[],"mappings":";;;AAAA,2BAAoC;AACpC,+BAAqC;AAIrC,8EAAyE;AACzE,iFAAsI;AACtI,6CAA0F;AA2D1F,MAAM,SAAS,GAAG,CAAC,OAA+B,EAAE,GAAW,EAAE,EAAE;IACjE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAC;AAQF,MAAa,gBAAgB;IACE;IAAyC;IAAtE,YAA6B,IAAsB,EAAmB,OAA4B;QAArE,SAAI,GAAJ,IAAI,CAAkB;QAAmB,YAAO,GAAP,OAAO,CAAqB;IAAI,CAAC;IAE/F,GAAG;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;QAC5B,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAuB;YACjC,KAAK;YACL,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YACjC,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;YAClC,MAAM,EAAE,QAAQ;YAChB,QAAQ;YACR,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE;YAClD,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,CAAC;YACV,QAAQ,EAAE,EAAE;YACZ,aAAa,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE;YACjD,MAAM,EAAE,EAAE;YACV,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,IAAI,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE;YAC7D,QAAQ;YACR,gBAAgB,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC;YACvD,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC;SAC1C,CAAC;QAEF,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,IAAI,OAAO,EAAE,SAAS,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,GAAG,QAAQ,MAAM,CAAC;QAClC,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,IAAA,kCAAqB,EAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAC5G,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,aAAa,GAAG,QAAQ,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC;YAC1E,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAEvC,MAAM,aAAE,CAAC,KAAK,CAAC,IAAA,cAAO,EAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEvD,MAAM,aAAE,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,MAAM,OAAO,GAAG,IAAI,6CAAoB,CAAC,OAAO,CAAC,CAAC;YAClD,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YAE3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;YAClC,IAAI,KAAK,GAAG,IAAI,CAAC;YACjB,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBAC1C,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE;oBACzB,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;oBACxB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;oBAC1C,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;wBACtB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,0CAAkB,CAAC,WAAW,CAAC,CAAC;wBAC3D,SAAS;qBACV;oBACD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACb,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;wBAChC,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,0CAAkB,CAAC,QAAQ,CAAC,CAAC;wBACxD,SAAS;qBACV;oBAED,MAAM,QAAQ,GAAG,IAAA,6CAAqB,EAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBACnD,IAAI,QAAQ,EAAE;wBACZ,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;wBACrC,SAAS;qBACV;oBACD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;oBACvD,IAAI,CAAC,OAAO,EAAE;wBACZ,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,0CAAkB,CAAC,QAAQ,CAAC,CAAC;wBACxD,SAAS;qBACV;oBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;oBACrD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACvF,MAAM,MAAM,GAAG,IAAA,wCAAgB,EAAC,MAAM,CAAC,GAAG,EAAE;wBAC1C,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;wBAC3B,OAAO;wBACP,MAAM;wBACN,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS;wBAC9B,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;qBACnE,CAAC,CAAC;oBACH,IAAI,MAAM,CAAC,EAAE,KAAK,KAAK,EAAE;wBACvB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;wBAC1C,SAAS;qBACV;oBACD,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;oBAC7E,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;oBAC1C,KAAK,GAAG,KAAK,CAAC;oBACd,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACrB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iBAC/C;YACH,CAAC,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;YACzB,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC9B,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YAEvC,MAAM,MAAM,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YACjF,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAEpC,MAAM,CAAC,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAC;YACpF,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC5B,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACrB,OAAO,MAAM,CAAC;SACf;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,2BAAc,EAAE;gBACnC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;gBAC1B,MAAM,CAAC,KAAK,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;aAC7D;iBAAM;gBACL,MAAM,CAAC,KAAK,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;aAChD;YAED,MAAM,aAAE,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,QAAQ,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACxE,MAAM,KAAK,CAAC;SACb;gBAAS;YACR,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC/B,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;SAC3B;IACH,CAAC;IAEO,KAAK,CAAC,MAA0B,EAAE,MAAc;QACtD,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;QAClB,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,CAAC;QACnC,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACtE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACtE,KAAK,CAAC,KAAK,IAAI,MAAM,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAA0B;QAClD,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,aAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,aAAE,CAAC,SAAS,CAAC,IAAA,WAAI,EAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,sBAAsB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACjH,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC;CACF;AAtID,4CAsIC"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { ImboImageResponseInterface } from '@bytebrand/chunkable';
|
|
2
|
+
import { DataSchemes } from '@bytebrand/car-schema';
|
|
3
|
+
import { CriteoXmlItem } from '../interfaces';
|
|
4
|
+
import { FeedGuard, LastGood } from './feed-guard';
|
|
5
|
+
export interface CriteoSourceCar {
|
|
6
|
+
car: DataSchemes.Car;
|
|
7
|
+
images: ImboImageResponseInterface[];
|
|
8
|
+
}
|
|
9
|
+
export declare enum CriteoExclusion {
|
|
10
|
+
noId = "no_id",
|
|
11
|
+
duplicateId = "duplicate_id",
|
|
12
|
+
filtered = "filtered",
|
|
13
|
+
notAvailable = "reserved_or_sold",
|
|
14
|
+
noPrice = "no_valid_price",
|
|
15
|
+
noImage = "no_marketing_image"
|
|
16
|
+
}
|
|
17
|
+
export interface CriteoRunDeps {
|
|
18
|
+
forEachChunk: (fn: (cars: CriteoSourceCar[]) => Promise<void>) => Promise<unknown>;
|
|
19
|
+
filterCar: (source: CriteoSourceCar) => boolean;
|
|
20
|
+
transform: (source: CriteoSourceCar) => Promise<CriteoXmlItem | null>;
|
|
21
|
+
guard: FeedGuard;
|
|
22
|
+
}
|
|
23
|
+
export interface CriteoRunOptions {
|
|
24
|
+
runId?: string;
|
|
25
|
+
domain: string;
|
|
26
|
+
sourceDirectory: string;
|
|
27
|
+
sourceBaseFileName: string;
|
|
28
|
+
xmlFile: string;
|
|
29
|
+
stateDir: string;
|
|
30
|
+
now?: () => Date;
|
|
31
|
+
}
|
|
32
|
+
export interface CriteoRunReport {
|
|
33
|
+
runId?: string;
|
|
34
|
+
startedAt: string;
|
|
35
|
+
finishedAt?: string;
|
|
36
|
+
durationMs?: number;
|
|
37
|
+
status: 'published' | 'blocked' | 'failed';
|
|
38
|
+
guard?: {
|
|
39
|
+
code: string;
|
|
40
|
+
message: string;
|
|
41
|
+
};
|
|
42
|
+
error?: string;
|
|
43
|
+
xmlFile: string;
|
|
44
|
+
source: {
|
|
45
|
+
files: number;
|
|
46
|
+
newestChunkAt: string | null;
|
|
47
|
+
ageHours: number | null;
|
|
48
|
+
cars: number;
|
|
49
|
+
};
|
|
50
|
+
items: number;
|
|
51
|
+
excluded: Record<string, number>;
|
|
52
|
+
withRate: number;
|
|
53
|
+
withoutEnergyClass: number;
|
|
54
|
+
ids: {
|
|
55
|
+
added: number;
|
|
56
|
+
removed: number;
|
|
57
|
+
kept: number;
|
|
58
|
+
removedSample: string[];
|
|
59
|
+
} | null;
|
|
60
|
+
lastGood: LastGood | null;
|
|
61
|
+
lastGoodAgeHours: number | null;
|
|
62
|
+
stale: boolean;
|
|
63
|
+
}
|
|
64
|
+
export declare class CriteoFeedRun {
|
|
65
|
+
private readonly deps;
|
|
66
|
+
private readonly options;
|
|
67
|
+
constructor(deps: CriteoRunDeps, options: CriteoRunOptions);
|
|
68
|
+
private now;
|
|
69
|
+
private get idsFile();
|
|
70
|
+
run(): Promise<CriteoRunReport>;
|
|
71
|
+
private exclusion;
|
|
72
|
+
private readPublishedIds;
|
|
73
|
+
private writePublishedIds;
|
|
74
|
+
private diffIds;
|
|
75
|
+
private writeReport;
|
|
76
|
+
}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CriteoFeedRun = exports.CriteoExclusion = void 0;
|
|
4
|
+
const fs_1 = require("fs");
|
|
5
|
+
const path_1 = require("path");
|
|
6
|
+
const lodash_1 = require("lodash");
|
|
7
|
+
const google_xml_builder_1 = require("../formats/google-xml.builder");
|
|
8
|
+
const feed_guard_1 = require("./feed-guard");
|
|
9
|
+
var CriteoExclusion;
|
|
10
|
+
(function (CriteoExclusion) {
|
|
11
|
+
CriteoExclusion["noId"] = "no_id";
|
|
12
|
+
CriteoExclusion["duplicateId"] = "duplicate_id";
|
|
13
|
+
CriteoExclusion["filtered"] = "filtered";
|
|
14
|
+
CriteoExclusion["notAvailable"] = "reserved_or_sold";
|
|
15
|
+
CriteoExclusion["noPrice"] = "no_valid_price";
|
|
16
|
+
CriteoExclusion["noImage"] = "no_marketing_image";
|
|
17
|
+
})(CriteoExclusion || (exports.CriteoExclusion = CriteoExclusion = {}));
|
|
18
|
+
const NOT_AVAILABLE = ['selector_offerAvailability_reserved', 'selector_offerAvailability_sold'];
|
|
19
|
+
const HOUR_MS = 60 * 60 * 1000;
|
|
20
|
+
const PRICE = /^\d+(\.\d+)? EUR$/;
|
|
21
|
+
const increment = (counter, key) => {
|
|
22
|
+
counter[key] = (counter[key] ?? 0) + 1;
|
|
23
|
+
};
|
|
24
|
+
class CriteoFeedRun {
|
|
25
|
+
deps;
|
|
26
|
+
options;
|
|
27
|
+
constructor(deps, options) {
|
|
28
|
+
this.deps = deps;
|
|
29
|
+
this.options = options;
|
|
30
|
+
}
|
|
31
|
+
now() {
|
|
32
|
+
return this.options.now ? this.options.now() : new Date();
|
|
33
|
+
}
|
|
34
|
+
get idsFile() {
|
|
35
|
+
return (0, path_1.join)(this.options.stateDir, 'published-ids.json');
|
|
36
|
+
}
|
|
37
|
+
async run() {
|
|
38
|
+
const { guard } = this.deps;
|
|
39
|
+
const { xmlFile, runId } = this.options;
|
|
40
|
+
const startedAt = this.now();
|
|
41
|
+
const lastGood = await guard.readLastGood();
|
|
42
|
+
const report = {
|
|
43
|
+
runId,
|
|
44
|
+
startedAt: startedAt.toISOString(),
|
|
45
|
+
status: 'failed',
|
|
46
|
+
xmlFile,
|
|
47
|
+
source: { files: 0, newestChunkAt: null, ageHours: null, cars: 0 },
|
|
48
|
+
items: 0,
|
|
49
|
+
excluded: {},
|
|
50
|
+
withRate: 0,
|
|
51
|
+
withoutEnergyClass: 0,
|
|
52
|
+
ids: null,
|
|
53
|
+
lastGood,
|
|
54
|
+
lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
|
|
55
|
+
stale: guard.isStale(lastGood, startedAt),
|
|
56
|
+
};
|
|
57
|
+
await guard.acquireLock(runId ?? 'local', startedAt);
|
|
58
|
+
const tmpFile = `${xmlFile}.tmp`;
|
|
59
|
+
try {
|
|
60
|
+
const snapshot = await (0, feed_guard_1.inspectSourceSnapshot)(this.options.sourceDirectory, this.options.sourceBaseFileName);
|
|
61
|
+
report.source.files = snapshot.files;
|
|
62
|
+
report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
|
|
63
|
+
report.source.ageHours = snapshot.newestMtime
|
|
64
|
+
? Number(((startedAt.getTime() - snapshot.newestMtime.getTime()) / HOUR_MS).toFixed(2))
|
|
65
|
+
: null;
|
|
66
|
+
guard.checkSource(snapshot, startedAt);
|
|
67
|
+
await fs_1.promises.mkdir((0, path_1.dirname)(xmlFile), { recursive: true });
|
|
68
|
+
await fs_1.promises.rm(tmpFile, { force: true });
|
|
69
|
+
const builder = new google_xml_builder_1.GoogleXmlBuilder(tmpFile);
|
|
70
|
+
await builder.buildStart(this.options.domain);
|
|
71
|
+
const seen = new Set();
|
|
72
|
+
const published = [];
|
|
73
|
+
await this.deps.forEachChunk(async (cars) => {
|
|
74
|
+
for (const source of cars) {
|
|
75
|
+
report.source.cars += 1;
|
|
76
|
+
const reason = this.exclusion(source, seen);
|
|
77
|
+
if (reason) {
|
|
78
|
+
increment(report.excluded, reason);
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
const item = await this.deps.transform(source);
|
|
82
|
+
if (!item) {
|
|
83
|
+
increment(report.excluded, CriteoExclusion.noImage);
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
if (!PRICE.test(item.price) || !PRICE.test(item.sale_price)) {
|
|
87
|
+
increment(report.excluded, CriteoExclusion.noPrice);
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
await builder.addItem(item);
|
|
91
|
+
published.push(item.id);
|
|
92
|
+
report.items += 1;
|
|
93
|
+
if (item.installment) {
|
|
94
|
+
report.withRate += 1;
|
|
95
|
+
}
|
|
96
|
+
if (!item.energy_efficiency_class) {
|
|
97
|
+
report.withoutEnergyClass += 1;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
await builder.buildEnd();
|
|
102
|
+
const counts = { sourceItems: report.source.cars, publishedItems: report.items };
|
|
103
|
+
guard.checkCounts(counts, lastGood);
|
|
104
|
+
const previousIds = await this.readPublishedIds();
|
|
105
|
+
report.lastGood = await guard.publish(tmpFile, xmlFile, counts, this.now(), runId);
|
|
106
|
+
await this.writePublishedIds(published);
|
|
107
|
+
report.ids = this.diffIds(previousIds, published);
|
|
108
|
+
report.status = 'published';
|
|
109
|
+
report.lastGoodAgeHours = 0;
|
|
110
|
+
report.stale = false;
|
|
111
|
+
return report;
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
if (error instanceof feed_guard_1.FeedGuardError) {
|
|
115
|
+
report.status = 'blocked';
|
|
116
|
+
report.guard = { code: error.code, message: error.message };
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
report.error = error?.message ?? String(error);
|
|
120
|
+
}
|
|
121
|
+
await fs_1.promises.rename(tmpFile, `${xmlFile}.rejected`).catch(() => undefined);
|
|
122
|
+
throw error;
|
|
123
|
+
}
|
|
124
|
+
finally {
|
|
125
|
+
await this.writeReport(report, startedAt);
|
|
126
|
+
await guard.releaseLock();
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
exclusion(source, seen) {
|
|
130
|
+
const id = (0, lodash_1.toString)((0, lodash_1.get)(source, 'car._id', ''));
|
|
131
|
+
if (!id) {
|
|
132
|
+
return CriteoExclusion.noId;
|
|
133
|
+
}
|
|
134
|
+
if (seen.has(id)) {
|
|
135
|
+
return CriteoExclusion.duplicateId;
|
|
136
|
+
}
|
|
137
|
+
seen.add(id);
|
|
138
|
+
if (!this.deps.filterCar(source)) {
|
|
139
|
+
return CriteoExclusion.filtered;
|
|
140
|
+
}
|
|
141
|
+
if (NOT_AVAILABLE.includes((0, lodash_1.toString)((0, lodash_1.get)(source.car, 'offer.offerAvailability', '')))) {
|
|
142
|
+
return CriteoExclusion.notAvailable;
|
|
143
|
+
}
|
|
144
|
+
const price = (0, lodash_1.toNumber)((0, lodash_1.get)(source.car, 'price.currentSalesPrice', null));
|
|
145
|
+
if (!Number.isFinite(price) || price <= 0) {
|
|
146
|
+
return CriteoExclusion.noPrice;
|
|
147
|
+
}
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
async readPublishedIds() {
|
|
151
|
+
try {
|
|
152
|
+
return JSON.parse(await fs_1.promises.readFile(this.idsFile, 'utf8'));
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
if (error?.code === 'ENOENT') {
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
throw error;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
async writePublishedIds(ids) {
|
|
162
|
+
await fs_1.promises.mkdir(this.options.stateDir, { recursive: true });
|
|
163
|
+
await fs_1.promises.writeFile(`${this.idsFile}.tmp`, JSON.stringify(ids), 'utf8');
|
|
164
|
+
await fs_1.promises.rename(`${this.idsFile}.tmp`, this.idsFile);
|
|
165
|
+
}
|
|
166
|
+
diffIds(previous, current) {
|
|
167
|
+
if (!previous) {
|
|
168
|
+
return { added: current.length, removed: 0, kept: 0, removedSample: [] };
|
|
169
|
+
}
|
|
170
|
+
const before = new Set(previous);
|
|
171
|
+
const now = new Set(current);
|
|
172
|
+
const removed = previous.filter(id => !now.has(id));
|
|
173
|
+
const kept = current.filter(id => before.has(id)).length;
|
|
174
|
+
return { added: current.length - kept, removed: removed.length, kept, removedSample: removed.slice(0, 20) };
|
|
175
|
+
}
|
|
176
|
+
async writeReport(report, startedAt) {
|
|
177
|
+
const finishedAt = this.now();
|
|
178
|
+
report.finishedAt = finishedAt.toISOString();
|
|
179
|
+
report.durationMs = finishedAt.getTime() - startedAt.getTime();
|
|
180
|
+
await fs_1.promises.mkdir(this.options.stateDir, { recursive: true });
|
|
181
|
+
await fs_1.promises.writeFile((0, path_1.join)(this.options.stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
|
|
182
|
+
console.log(`criteo feed report: ${JSON.stringify(report)}`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
exports.CriteoFeedRun = CriteoFeedRun;
|
|
186
|
+
//# sourceMappingURL=criteo-feed-run.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"criteo-feed-run.js","sourceRoot":"","sources":["../../src/services/criteo-feed-run.ts"],"names":[],"mappings":";;;AAAA,2BAAoC;AACpC,+BAAqC;AACrC,mCAAiD;AAIjD,sEAAiE;AAEjE,6CAA0F;AAO1F,IAAY,eAOX;AAPD,WAAY,eAAe;IACzB,iCAAc,CAAA;IACd,+CAA4B,CAAA;IAC5B,wCAAqB,CAAA;IACrB,oDAAiC,CAAA;IACjC,6CAA0B,CAAA;IAC1B,iDAA8B,CAAA;AAChC,CAAC,EAPW,eAAe,+BAAf,eAAe,QAO1B;AAKD,MAAM,aAAa,GAAG,CAAC,qCAAqC,EAAE,iCAAiC,CAAC,CAAC;AA6CjG,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC/B,MAAM,KAAK,GAAG,mBAAmB,CAAC;AAElC,MAAM,SAAS,GAAG,CAAC,OAA+B,EAAE,GAAW,EAAE,EAAE;IACjE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAC;AAQF,MAAa,aAAa;IACK;IAAsC;IAAnE,YAA6B,IAAmB,EAAmB,OAAyB;QAA/D,SAAI,GAAJ,IAAI,CAAe;QAAmB,YAAO,GAAP,OAAO,CAAkB;IAAI,CAAC;IAEzF,GAAG;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IAC5D,CAAC;IAED,IAAY,OAAO;QACjB,OAAO,IAAA,WAAI,EAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;QAC5B,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAoB;YAC9B,KAAK;YACL,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;YAClC,MAAM,EAAE,QAAQ;YAChB,OAAO;YACP,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE;YAClE,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE,CAAC;YACX,kBAAkB,EAAE,CAAC;YACrB,GAAG,EAAE,IAAI;YACT,QAAQ;YACR,gBAAgB,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC;YACvD,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC;SAC1C,CAAC;QAEF,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,IAAI,OAAO,EAAE,SAAS,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,GAAG,OAAO,MAAM,CAAC;QACjC,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,IAAA,kCAAqB,EAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAC5G,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,aAAa,GAAG,QAAQ,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC;YAC1E,MAAM,CAAC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,WAAW;gBAC3C,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACvF,CAAC,CAAC,IAAI,CAAC;YACT,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAEvC,MAAM,aAAE,CAAC,KAAK,CAAC,IAAA,cAAO,EAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEtD,MAAM,aAAE,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,MAAM,OAAO,GAAG,IAAI,qCAAgB,CAAC,OAAO,CAAC,CAAC;YAC9C,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAE9C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBAC1C,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE;oBACzB,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;oBACxB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;oBAC5C,IAAI,MAAM,EAAE;wBACV,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;wBACnC,SAAS;qBACV;oBACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;oBAC/C,IAAI,CAAC,IAAI,EAAE;wBACT,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;wBACpD,SAAS;qBACV;oBAED,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;wBAC3D,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;wBACpD,SAAS;qBACV;oBACD,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBAC5B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACxB,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;oBAClB,IAAI,IAAI,CAAC,WAAW,EAAE;wBACpB,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;qBACtB;oBACD,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE;wBACjC,MAAM,CAAC,kBAAkB,IAAI,CAAC,CAAC;qBAChC;iBACF;YACH,CAAC,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;YAEzB,MAAM,MAAM,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YACjF,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAEpC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,MAAM,CAAC,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAC;YACnF,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACxC,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC5B,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACrB,OAAO,MAAM,CAAC;SACf;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,2BAAc,EAAE;gBACnC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;gBAC1B,MAAM,CAAC,KAAK,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;aAC7D;iBAAM;gBACL,MAAM,CAAC,KAAK,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;aAChD;YAED,MAAM,aAAE,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,OAAO,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACvE,MAAM,KAAK,CAAC;SACb;gBAAS;YACR,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC1C,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;SAC3B;IACH,CAAC;IAEO,SAAS,CAAC,MAAuB,EAAE,IAAiB;QAC1D,MAAM,EAAE,GAAG,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;QAChD,IAAI,CAAC,EAAE,EAAE;YACP,OAAO,eAAe,CAAC,IAAI,CAAC;SAC7B;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAChB,OAAO,eAAe,CAAC,WAAW,CAAC;SACpC;QACD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACb,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;YAChC,OAAO,eAAe,CAAC,QAAQ,CAAC;SACjC;QACD,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,MAAM,CAAC,GAAG,EAAE,yBAAyB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE;YACpF,OAAO,eAAe,CAAC,YAAY,CAAC;SACrC;QACD,MAAM,KAAK,GAAG,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,MAAM,CAAC,GAAG,EAAE,yBAAyB,EAAE,IAAI,CAAC,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE;YACzC,OAAO,eAAe,CAAC,OAAO,CAAC;SAChC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC5B,IAAI;YACF,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,aAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAa,CAAC;SACxE;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ,EAAE;gBAC5B,OAAO,IAAI,CAAC;aACb;YACD,MAAM,KAAK,CAAC;SACb;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,GAAa;QAC3C,MAAM,aAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,aAAE,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,OAAO,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;QACvE,MAAM,aAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACvD,CAAC;IAEO,OAAO,CAAC,QAAyB,EAAE,OAAiB;QAC1D,IAAI,CAAC,QAAQ,EAAE;YACb,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC;SAC1E;QACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACpD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QACzD,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAC9G,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAuB,EAAE,SAAe;QAChE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,CAAC,UAAU,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,CAAC,UAAU,GAAG,UAAU,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;QAC/D,MAAM,aAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,aAAE,CAAC,SAAS,CAAC,IAAA,WAAI,EAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,sBAAsB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACjH,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;CACF;AAvKD,sCAuKC"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { DataSchemes } from '@bytebrand/car-schema';
|
|
2
2
|
import { AutouncleJsonItem } from '../interfaces';
|
|
3
3
|
import { OpenAiImageSelection } from './openai-image-fields';
|
|
4
|
+
import { GoogleEnvkvAttribute } from './google-feed-envkv';
|
|
5
|
+
export declare const AUTOUNCLE_LINK_PARAMS = "source=autouncle&medium=listing&campaign=listing&utm_source=autouncle&utm_medium=listing&utm_campaign=listing";
|
|
4
6
|
export declare enum AutouncleExclusion {
|
|
5
7
|
duplicateId = "duplicate_id",
|
|
6
8
|
filtered = "filtered",
|
|
@@ -21,6 +23,7 @@ export interface AutouncleMappingContext {
|
|
|
21
23
|
address: AutouncleAddress;
|
|
22
24
|
images: string[];
|
|
23
25
|
translate: (key: string) => string;
|
|
26
|
+
withheldEnvkv?: GoogleEnvkvAttribute[];
|
|
24
27
|
}
|
|
25
28
|
export type AutouncleMapResult = {
|
|
26
29
|
ok: true;
|