@bytebrand/export-cars 1.26.27 → 1.26.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/README.md +128 -0
  2. package/dist/configs/config.d.ts +2 -1
  3. package/dist/configs/config.js +1 -0
  4. package/dist/configs/config.js.map +1 -1
  5. package/dist/configs/google.config.js +16 -0
  6. package/dist/configs/google.config.js.map +1 -1
  7. package/dist/configs/index.d.ts +1 -0
  8. package/dist/configs/index.js +1 -0
  9. package/dist/configs/index.js.map +1 -1
  10. package/dist/configs/openai-ads.config.d.ts +22 -0
  11. package/dist/configs/openai-ads.config.js +56 -0
  12. package/dist/configs/openai-ads.config.js.map +1 -0
  13. package/dist/enums/export-type.enum.d.ts +2 -1
  14. package/dist/enums/export-type.enum.js +1 -0
  15. package/dist/enums/export-type.enum.js.map +1 -1
  16. package/dist/enums/image-type.enum.d.ts +2 -1
  17. package/dist/enums/image-type.enum.js +1 -0
  18. package/dist/enums/image-type.enum.js.map +1 -1
  19. package/dist/export.schema.d.ts +6 -2
  20. package/dist/export.schema.js +4 -0
  21. package/dist/export.schema.js.map +1 -1
  22. package/dist/exporters/google.exporter.js +12 -2
  23. package/dist/exporters/google.exporter.js.map +1 -1
  24. package/dist/exporters/index.d.ts +1 -0
  25. package/dist/exporters/index.js +1 -0
  26. package/dist/exporters/index.js.map +1 -1
  27. package/dist/exporters/openai-ads.exporter.d.ts +13 -0
  28. package/dist/exporters/openai-ads.exporter.js +91 -0
  29. package/dist/exporters/openai-ads.exporter.js.map +1 -0
  30. package/dist/formats/openai-ads-csv.builder.d.ts +15 -0
  31. package/dist/formats/openai-ads-csv.builder.js +48 -0
  32. package/dist/formats/openai-ads-csv.builder.js.map +1 -0
  33. package/dist/interfaces/openai-ads.interface.d.ts +35 -0
  34. package/dist/interfaces/openai-ads.interface.js +37 -0
  35. package/dist/interfaces/openai-ads.interface.js.map +1 -0
  36. package/dist/interfaces/params.interface.d.ts +20 -0
  37. package/dist/services/feed-guard.d.ts +44 -0
  38. package/dist/services/feed-guard.js +156 -0
  39. package/dist/services/feed-guard.js.map +1 -0
  40. package/dist/services/image-url-validator.service.d.ts +45 -0
  41. package/dist/services/image-url-validator.service.js +141 -0
  42. package/dist/services/image-url-validator.service.js.map +1 -0
  43. package/dist/services/images.service.d.ts +12 -0
  44. package/dist/services/images.service.js +46 -1
  45. package/dist/services/images.service.js.map +1 -1
  46. package/dist/services/index.d.ts +1 -0
  47. package/dist/services/index.js +1 -0
  48. package/dist/services/index.js.map +1 -1
  49. package/dist/services/openai-ads-feed-run.d.ts +74 -0
  50. package/dist/services/openai-ads-feed-run.js +151 -0
  51. package/dist/services/openai-ads-feed-run.js.map +1 -0
  52. package/dist/services/openai-ads-ledger.d.ts +19 -0
  53. package/dist/services/openai-ads-ledger.js +72 -0
  54. package/dist/services/openai-ads-ledger.js.map +1 -0
  55. package/dist/services/publication-gate.d.ts +15 -0
  56. package/dist/services/publication-gate.js +107 -0
  57. package/dist/services/publication-gate.js.map +1 -0
  58. package/dist/transformers/google-feed-envkv.d.ts +2 -0
  59. package/dist/transformers/google-feed-envkv.js +62 -31
  60. package/dist/transformers/google-feed-envkv.js.map +1 -1
  61. package/dist/transformers/google-feed-item-row.d.ts +6 -1
  62. package/dist/transformers/google-feed-item-row.js +3 -1
  63. package/dist/transformers/google-feed-item-row.js.map +1 -1
  64. package/dist/transformers/openai-ads.transformer.d.ts +28 -0
  65. package/dist/transformers/openai-ads.transformer.js +144 -0
  66. package/dist/transformers/openai-ads.transformer.js.map +1 -0
  67. package/dist/transformers/openai-image-fields.d.ts +16 -0
  68. package/dist/transformers/openai-image-fields.js +34 -0
  69. package/dist/transformers/openai-image-fields.js.map +1 -0
  70. package/dist/tsconfig.build.tsbuildinfo +1 -1
  71. package/docs/openai-ads/AUDIT.md +235 -0
  72. package/docs/openai-ads/RUNBOOK.md +94 -0
  73. package/package.json +64 -64
  74. package/schema/feed-items.json +134 -122
  75. package/src/configs/config.ts +99 -97
  76. package/src/configs/google.config.ts +41 -26
  77. package/src/configs/index.ts +1 -0
  78. package/src/configs/openai-ads.config.ts +70 -0
  79. package/src/enums/export-type.enum.ts +2 -0
  80. package/src/enums/image-type.enum.ts +5 -0
  81. package/src/export.command.ts +38 -38
  82. package/src/export.schema.ts +6 -0
  83. package/src/exporters/google.exporter.ts +67 -56
  84. package/src/exporters/index.ts +1 -0
  85. package/src/exporters/openai-ads.exporter.ts +93 -0
  86. package/src/formats/openai-ads-csv.builder.ts +55 -0
  87. package/src/interfaces/openai-ads.interface.ts +64 -0
  88. package/src/interfaces/params.interface.ts +56 -29
  89. package/src/services/feed-guard.ts +219 -0
  90. package/src/services/feed-items-bigquery.writer.ts +101 -101
  91. package/src/services/image-url-validator.service.ts +181 -0
  92. package/src/services/images.service.ts +72 -0
  93. package/src/services/index.ts +1 -0
  94. package/src/services/openai-ads-feed-run.ts +226 -0
  95. package/src/services/openai-ads-ledger.ts +116 -0
  96. package/src/services/publication-gate.ts +119 -0
  97. package/src/transformers/google-feed-envkv.ts +81 -33
  98. package/src/transformers/google-feed-item-row.ts +78 -71
  99. package/src/transformers/openai-ads.transformer.ts +211 -0
  100. package/src/transformers/openai-image-fields.ts +60 -0
  101. package/test/clean-images.test.ts +86 -0
  102. package/test/fixtures/images-service.harness.ts +32 -0
  103. package/test/fixtures/images.golden.json +218 -0
  104. package/test/fixtures/imbo-images.fixture.ts +77 -0
  105. package/test/fixtures/openai-ads.fixture.ts +89 -0
  106. package/test/google-feed-item-row.test.ts +135 -121
  107. package/test/image-url-validator.test.ts +110 -0
  108. package/test/images-regression.test.ts +23 -0
  109. package/test/openai-ads-feed-run.test.ts +278 -0
  110. package/test/openai-ads-mapping.test.ts +136 -0
  111. package/test/openai-image-fields.test.ts +99 -0
  112. package/test/publication-gate.test.ts +86 -0
@@ -0,0 +1,48 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.OpenAiAdsCsvBuilder = exports.openAiAdsCsvRow = exports.openAiAdsCsvHeader = exports.toCsvLine = exports.toCsvCell = void 0;
4
+ const fs_1 = require("fs");
5
+ const events_1 = require("events");
6
+ const openai_ads_interface_1 = require("../interfaces/openai-ads.interface");
7
+ const toCsvCell = (value) => {
8
+ if (value === undefined || value === null) {
9
+ return '';
10
+ }
11
+ const text = typeof value === 'boolean' ? (value ? 'true' : 'false') : value;
12
+ return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
13
+ };
14
+ exports.toCsvCell = toCsvCell;
15
+ const toCsvLine = (cells) => `${cells.map(exports.toCsvCell).join(',')}\n`;
16
+ exports.toCsvLine = toCsvLine;
17
+ const openAiAdsCsvHeader = () => (0, exports.toCsvLine)([...openai_ads_interface_1.OPENAI_ADS_COLUMNS]);
18
+ exports.openAiAdsCsvHeader = openAiAdsCsvHeader;
19
+ const openAiAdsCsvRow = (row) => (0, exports.toCsvLine)(openai_ads_interface_1.OPENAI_ADS_COLUMNS.map(column => row[column]));
20
+ exports.openAiAdsCsvRow = openAiAdsCsvRow;
21
+ class OpenAiAdsCsvBuilder {
22
+ fileName;
23
+ stream;
24
+ rows = 0;
25
+ constructor(fileName) {
26
+ this.fileName = fileName;
27
+ }
28
+ async buildStart() {
29
+ this.stream = (0, fs_1.createWriteStream)(this.fileName, { encoding: 'utf8' });
30
+ await this.write((0, exports.openAiAdsCsvHeader)());
31
+ }
32
+ async addItem(row) {
33
+ await this.write((0, exports.openAiAdsCsvRow)(row));
34
+ this.rows += 1;
35
+ }
36
+ async buildEnd() {
37
+ this.stream.end();
38
+ await (0, events_1.once)(this.stream, 'finish');
39
+ return this.rows;
40
+ }
41
+ async write(chunk) {
42
+ if (!this.stream.write(chunk)) {
43
+ await (0, events_1.once)(this.stream, 'drain');
44
+ }
45
+ }
46
+ }
47
+ exports.OpenAiAdsCsvBuilder = OpenAiAdsCsvBuilder;
48
+ //# sourceMappingURL=openai-ads-csv.builder.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"openai-ads-csv.builder.js","sourceRoot":"","sources":["../../src/formats/openai-ads-csv.builder.ts"],"names":[],"mappings":";;;AAAA,2BAAoD;AACpD,mCAA8B;AAE9B,6EAAsF;AAK/E,MAAM,SAAS,GAAG,CAAC,KAA0C,EAAU,EAAE;IAC9E,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;QACzC,OAAO,EAAE,CAAC;KACX;IACD,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7E,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AACxE,CAAC,CAAC;AANW,QAAA,SAAS,aAMpB;AAEK,MAAM,SAAS,GAAG,CAAC,KAAiD,EAAU,EAAE,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,iBAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;AAAjH,QAAA,SAAS,aAAwG;AAEvH,MAAM,kBAAkB,GAAG,GAAW,EAAE,CAAC,IAAA,iBAAS,EAAC,CAAC,GAAG,yCAAkB,CAAC,CAAC,CAAC;AAAtE,QAAA,kBAAkB,sBAAoD;AAE5E,MAAM,eAAe,GAAG,CAAC,GAAiB,EAAU,EAAE,CAAC,IAAA,iBAAS,EAAC,yCAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAA1G,QAAA,eAAe,mBAA2F;AAOvH,MAAa,mBAAmB;IAID;IAHrB,MAAM,CAAc;IACpB,IAAI,GAAG,CAAC,CAAC;IAEjB,YAA6B,QAAgB;QAAhB,aAAQ,GAAR,QAAQ,CAAQ;IAAI,CAAC;IAElD,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,MAAM,GAAG,IAAA,sBAAiB,EAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QACrE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAA,0BAAkB,GAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAiB;QAC7B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAA,uBAAe,EAAC,GAAG,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAClB,MAAM,IAAA,aAAI,EAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,KAAa;QAC/B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YAC7B,MAAM,IAAA,aAAI,EAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SAClC;IACH,CAAC;CACF;AA3BD,kDA2BC"}
@@ -0,0 +1,35 @@
1
+ export declare const OPENAI_ADS_COLUMNS: readonly ["item_id", "title", "description", "url", "brand", "seller_name", "image_url", "additional_image_urls", "availability", "price", "condition", "is_ads_eligible"];
2
+ export type OpenAiAdsColumn = typeof OPENAI_ADS_COLUMNS[number];
3
+ export type OpenAiAdsAvailability = 'in_stock' | 'out_of_stock';
4
+ export type OpenAiAdsCondition = 'new' | 'used';
5
+ export interface OpenAiAdsRow {
6
+ item_id: string;
7
+ title: string;
8
+ description: string;
9
+ url: string;
10
+ brand: string;
11
+ seller_name: string;
12
+ image_url: string;
13
+ additional_image_urls: string;
14
+ availability: OpenAiAdsAvailability;
15
+ price: string;
16
+ condition: OpenAiAdsCondition;
17
+ is_ads_eligible: boolean;
18
+ }
19
+ export declare enum OpenAiAdsExclusion {
20
+ noId = "no_id",
21
+ duplicateId = "duplicate_id",
22
+ filtered = "filtered",
23
+ invalidPrice = "invalid_price",
24
+ noTitle = "no_title",
25
+ noBrand = "no_brand",
26
+ conditionUnknown = "condition_unknown",
27
+ dealerInactive = "dealer_inactive",
28
+ dealerUnknown = "dealer_unknown",
29
+ noImage = "no_image"
30
+ }
31
+ export declare enum OpenAiAdsOutOfStockReason {
32
+ reserved = "reserved",
33
+ soldFlag = "sold_flag",
34
+ notInSnapshot = "not_in_snapshot"
35
+ }
@@ -0,0 +1,37 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.OpenAiAdsOutOfStockReason = exports.OpenAiAdsExclusion = exports.OPENAI_ADS_COLUMNS = void 0;
4
+ exports.OPENAI_ADS_COLUMNS = [
5
+ 'item_id',
6
+ 'title',
7
+ 'description',
8
+ 'url',
9
+ 'brand',
10
+ 'seller_name',
11
+ 'image_url',
12
+ 'additional_image_urls',
13
+ 'availability',
14
+ 'price',
15
+ 'condition',
16
+ 'is_ads_eligible',
17
+ ];
18
+ var OpenAiAdsExclusion;
19
+ (function (OpenAiAdsExclusion) {
20
+ OpenAiAdsExclusion["noId"] = "no_id";
21
+ OpenAiAdsExclusion["duplicateId"] = "duplicate_id";
22
+ OpenAiAdsExclusion["filtered"] = "filtered";
23
+ OpenAiAdsExclusion["invalidPrice"] = "invalid_price";
24
+ OpenAiAdsExclusion["noTitle"] = "no_title";
25
+ OpenAiAdsExclusion["noBrand"] = "no_brand";
26
+ OpenAiAdsExclusion["conditionUnknown"] = "condition_unknown";
27
+ OpenAiAdsExclusion["dealerInactive"] = "dealer_inactive";
28
+ OpenAiAdsExclusion["dealerUnknown"] = "dealer_unknown";
29
+ OpenAiAdsExclusion["noImage"] = "no_image";
30
+ })(OpenAiAdsExclusion || (exports.OpenAiAdsExclusion = OpenAiAdsExclusion = {}));
31
+ var OpenAiAdsOutOfStockReason;
32
+ (function (OpenAiAdsOutOfStockReason) {
33
+ OpenAiAdsOutOfStockReason["reserved"] = "reserved";
34
+ OpenAiAdsOutOfStockReason["soldFlag"] = "sold_flag";
35
+ OpenAiAdsOutOfStockReason["notInSnapshot"] = "not_in_snapshot";
36
+ })(OpenAiAdsOutOfStockReason || (exports.OpenAiAdsOutOfStockReason = OpenAiAdsOutOfStockReason = {}));
37
+ //# sourceMappingURL=openai-ads.interface.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"openai-ads.interface.js","sourceRoot":"","sources":["../../src/interfaces/openai-ads.interface.ts"],"names":[],"mappings":";;;AAGa,QAAA,kBAAkB,GAAG;IAChC,SAAS;IACT,OAAO;IACP,aAAa;IACb,KAAK;IACL,OAAO;IACP,aAAa;IACb,WAAW;IACX,uBAAuB;IACvB,cAAc;IACd,OAAO;IACP,WAAW;IACX,iBAAiB;CACT,CAAC;AA4BX,IAAY,kBAWX;AAXD,WAAY,kBAAkB;IAC5B,oCAAc,CAAA;IACd,kDAA4B,CAAA;IAC5B,2CAAqB,CAAA;IACrB,oDAA8B,CAAA;IAC9B,0CAAoB,CAAA;IACpB,0CAAoB,CAAA;IACpB,4DAAsC,CAAA;IACtC,wDAAkC,CAAA;IAClC,sDAAgC,CAAA;IAChC,0CAAoB,CAAA;AACtB,CAAC,EAXW,kBAAkB,kCAAlB,kBAAkB,QAW7B;AAGD,IAAY,yBAKX;AALD,WAAY,yBAAyB;IACnC,kDAAqB,CAAA;IACrB,mDAAsB,CAAA;IAEtB,8DAAiC,CAAA;AACnC,CAAC,EALW,yBAAyB,yCAAzB,yBAAyB,QAKpC"}
@@ -14,9 +14,29 @@ export interface Params {
14
14
  feedItemsBqProject?: string;
15
15
  feedItemsBqDataset?: string;
16
16
  feedItemsBqTable?: string;
17
+ publicationGate?: string;
18
+ publicationGateBqProject?: string;
19
+ publicationGateBqDataset?: string;
20
+ publicationGateBqTable?: string;
21
+ openAiAdsDryRun?: string;
22
+ openAiAdsSellerName?: string;
23
+ openAiAdsRetentionDays?: string;
24
+ openAiAdsImageConcurrency?: string;
25
+ feedGuardMinSourceItems?: string;
26
+ feedGuardMinPublishedItems?: string;
27
+ feedGuardMaxDropPercent?: string;
28
+ feedGuardMaxSourceAgeHours?: string;
29
+ feedGuardMaxStaleHours?: string;
30
+ feedGuardAcceptDrop?: string;
17
31
  }
18
32
  export interface FeedItemsBigQueryTarget {
19
33
  project: string;
20
34
  dataset: string;
21
35
  table: string;
22
36
  }
37
+ export interface PublicationGateConfig {
38
+ mode: 'off' | 'audit' | 'enforce';
39
+ project: string;
40
+ dataset: string;
41
+ table: string;
42
+ }
@@ -0,0 +1,44 @@
1
+ export interface FeedGuardOptions {
2
+ stateDir: string;
3
+ minSourceItems: number;
4
+ minPublishedItems: number;
5
+ maxDropPercent: number;
6
+ maxSourceAgeHours: number;
7
+ maxStaleHours: number;
8
+ acceptDrop: boolean;
9
+ lockStaleHours?: number;
10
+ }
11
+ export interface SourceSnapshot {
12
+ files: number;
13
+ missingChunks: number[];
14
+ newestMtime: Date | null;
15
+ oldestMtime: Date | null;
16
+ }
17
+ export interface FeedCounts {
18
+ sourceItems: number;
19
+ publishedItems: number;
20
+ }
21
+ export interface LastGood extends FeedCounts {
22
+ publishedAt: string;
23
+ file: string;
24
+ runId?: string;
25
+ }
26
+ export declare class FeedGuardError extends Error {
27
+ readonly code: string;
28
+ constructor(code: string, message: string);
29
+ }
30
+ export declare const inspectSourceSnapshot: (directory: string, baseFileName: string) => Promise<SourceSnapshot>;
31
+ export declare class FeedGuard {
32
+ private readonly options;
33
+ private lockFile;
34
+ constructor(options: FeedGuardOptions);
35
+ private get lastGoodFile();
36
+ readLastGood(): Promise<LastGood | null>;
37
+ acquireLock(runId: string, now: Date): Promise<void>;
38
+ releaseLock(): Promise<void>;
39
+ checkSource(snapshot: SourceSnapshot, now: Date): void;
40
+ checkCounts(counts: FeedCounts, lastGood: LastGood | null): void;
41
+ staleHours(lastGood: LastGood | null, now: Date): number | null;
42
+ isStale(lastGood: LastGood | null, now: Date): boolean;
43
+ publish(tmpFile: string, finalFile: string, counts: FeedCounts, now: Date, runId?: string): Promise<LastGood>;
44
+ }
@@ -0,0 +1,156 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.FeedGuard = exports.inspectSourceSnapshot = exports.FeedGuardError = void 0;
4
+ const fs_1 = require("fs");
5
+ const path_1 = require("path");
6
+ class FeedGuardError extends Error {
7
+ code;
8
+ constructor(code, message) {
9
+ super(`[feed-guard:${code}] ${message}`);
10
+ this.code = code;
11
+ }
12
+ }
13
+ exports.FeedGuardError = FeedGuardError;
14
+ const HOUR_MS = 60 * 60 * 1000;
15
+ const inspectSourceSnapshot = async (directory, baseFileName) => {
16
+ let names;
17
+ try {
18
+ names = await fs_1.promises.readdir(directory);
19
+ }
20
+ catch (error) {
21
+ if (error?.code === 'ENOENT') {
22
+ return { files: 0, missingChunks: [], newestMtime: null, oldestMtime: null };
23
+ }
24
+ throw error;
25
+ }
26
+ const pattern = new RegExp(`^${baseFileName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.(\\d+)\\.json$`);
27
+ const chunks = names.map(name => ({ name, match: pattern.exec(name) })).filter(entry => entry.match);
28
+ const ids = new Set(chunks.map(entry => Number(entry.match[1])));
29
+ const maxId = ids.size ? Math.max(...ids) : -1;
30
+ const missingChunks = [];
31
+ for (let id = 0; id <= maxId; id += 1) {
32
+ if (!ids.has(id)) {
33
+ missingChunks.push(id);
34
+ }
35
+ }
36
+ let newest = null;
37
+ let oldest = null;
38
+ for (const { name } of chunks) {
39
+ const { mtimeMs } = await fs_1.promises.stat((0, path_1.join)(directory, name));
40
+ newest = newest === null ? mtimeMs : Math.max(newest, mtimeMs);
41
+ oldest = oldest === null ? mtimeMs : Math.min(oldest, mtimeMs);
42
+ }
43
+ return {
44
+ files: chunks.length,
45
+ missingChunks,
46
+ newestMtime: newest === null ? null : new Date(newest),
47
+ oldestMtime: oldest === null ? null : new Date(oldest),
48
+ };
49
+ };
50
+ exports.inspectSourceSnapshot = inspectSourceSnapshot;
51
+ const dropPercent = (current, previous) => previous > 0 ? ((previous - current) / previous) * 100 : 0;
52
+ class FeedGuard {
53
+ options;
54
+ lockFile;
55
+ constructor(options) {
56
+ this.options = options;
57
+ }
58
+ get lastGoodFile() {
59
+ return (0, path_1.join)(this.options.stateDir, 'last-good.json');
60
+ }
61
+ async readLastGood() {
62
+ try {
63
+ return JSON.parse(await fs_1.promises.readFile(this.lastGoodFile, 'utf8'));
64
+ }
65
+ catch (error) {
66
+ if (error?.code === 'ENOENT') {
67
+ return null;
68
+ }
69
+ throw error;
70
+ }
71
+ }
72
+ async acquireLock(runId, now) {
73
+ await fs_1.promises.mkdir(this.options.stateDir, { recursive: true });
74
+ const lockFile = (0, path_1.join)(this.options.stateDir, 'run.lock');
75
+ const staleMs = (this.options.lockStaleHours ?? 6) * HOUR_MS;
76
+ try {
77
+ await fs_1.promises.writeFile(lockFile, JSON.stringify({ runId, startedAt: now.toISOString() }), { flag: 'wx' });
78
+ }
79
+ catch (error) {
80
+ if (error?.code !== 'EEXIST') {
81
+ throw error;
82
+ }
83
+ const { mtimeMs } = await fs_1.promises.stat(lockFile);
84
+ if (now.getTime() - mtimeMs < staleMs) {
85
+ throw new FeedGuardError('locked', `another run holds ${lockFile} (since ${new Date(mtimeMs).toISOString()})`);
86
+ }
87
+ console.log(`feed-guard: removing stale lock ${lockFile} (${new Date(mtimeMs).toISOString()})`);
88
+ await fs_1.promises.writeFile(lockFile, JSON.stringify({ runId, startedAt: now.toISOString() }));
89
+ }
90
+ this.lockFile = lockFile;
91
+ }
92
+ async releaseLock() {
93
+ if (this.lockFile) {
94
+ await fs_1.promises.rm(this.lockFile, { force: true });
95
+ this.lockFile = null;
96
+ }
97
+ }
98
+ checkSource(snapshot, now) {
99
+ if (!snapshot.files) {
100
+ throw new FeedGuardError('source_empty', 'base export snapshot has no chunk files');
101
+ }
102
+ if (snapshot.missingChunks.length) {
103
+ throw new FeedGuardError('source_partial', `base export snapshot misses chunk(s) ${snapshot.missingChunks.slice(0, 10).join(', ')}`);
104
+ }
105
+ const ageHours = (now.getTime() - snapshot.newestMtime.getTime()) / HOUR_MS;
106
+ if (ageHours > this.options.maxSourceAgeHours) {
107
+ throw new FeedGuardError('source_stale', `newest base export chunk is ${ageHours.toFixed(1)} h old (max ${this.options.maxSourceAgeHours} h)`);
108
+ }
109
+ }
110
+ checkCounts(counts, lastGood) {
111
+ if (counts.sourceItems < this.options.minSourceItems) {
112
+ throw new FeedGuardError('source_too_small', `${counts.sourceItems} source cars < minimum ${this.options.minSourceItems}`);
113
+ }
114
+ if (counts.publishedItems < this.options.minPublishedItems) {
115
+ throw new FeedGuardError('published_too_small', `${counts.publishedItems} in_stock items < minimum ${this.options.minPublishedItems}`);
116
+ }
117
+ if (!lastGood || this.options.acceptDrop) {
118
+ return;
119
+ }
120
+ const sourceDrop = dropPercent(counts.sourceItems, lastGood.sourceItems);
121
+ const publishedDrop = dropPercent(counts.publishedItems, lastGood.publishedItems);
122
+ if (sourceDrop > this.options.maxDropPercent || publishedDrop > this.options.maxDropPercent) {
123
+ throw new FeedGuardError('count_drop', `source ${lastGood.sourceItems} -> ${counts.sourceItems} (-${sourceDrop.toFixed(1)}%), ` +
124
+ `in_stock ${lastGood.publishedItems} -> ${counts.publishedItems} (-${publishedDrop.toFixed(1)}%), ` +
125
+ `max ${this.options.maxDropPercent}% — re-run with acceptDrop if the stock really dropped`);
126
+ }
127
+ }
128
+ staleHours(lastGood, now) {
129
+ return lastGood ? (now.getTime() - Date.parse(lastGood.publishedAt)) / HOUR_MS : null;
130
+ }
131
+ isStale(lastGood, now) {
132
+ const hours = this.staleHours(lastGood, now);
133
+ return hours !== null && hours > this.options.maxStaleHours;
134
+ }
135
+ async publish(tmpFile, finalFile, counts, now, runId) {
136
+ if ((0, path_1.dirname)(tmpFile) !== (0, path_1.dirname)(finalFile)) {
137
+ throw new Error(`feed-guard: ${(0, path_1.basename)(tmpFile)} must be in the directory of ${finalFile}`);
138
+ }
139
+ try {
140
+ await fs_1.promises.copyFile(finalFile, `${finalFile}.prev`);
141
+ }
142
+ catch (error) {
143
+ if (error?.code !== 'ENOENT') {
144
+ throw error;
145
+ }
146
+ }
147
+ await fs_1.promises.rename(tmpFile, finalFile);
148
+ const lastGood = { ...counts, publishedAt: now.toISOString(), file: finalFile, runId };
149
+ await fs_1.promises.mkdir(this.options.stateDir, { recursive: true });
150
+ await fs_1.promises.writeFile(`${this.lastGoodFile}.tmp`, JSON.stringify(lastGood, null, 2), 'utf8');
151
+ await fs_1.promises.rename(`${this.lastGoodFile}.tmp`, this.lastGoodFile);
152
+ return lastGood;
153
+ }
154
+ }
155
+ exports.FeedGuard = FeedGuard;
156
+ //# sourceMappingURL=feed-guard.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"feed-guard.js","sourceRoot":"","sources":["../../src/services/feed-guard.ts"],"names":[],"mappings":";;;AAAA,2BAAoC;AACpC,+BAA+C;AAsD/C,MAAa,cAAe,SAAQ,KAAK;IAClB;IAArB,YAAqB,IAAY,EAAE,OAAe;QAChD,KAAK,CAAC,eAAe,IAAI,KAAK,OAAO,EAAE,CAAC,CAAC;QADtB,SAAI,GAAJ,IAAI,CAAQ;IAEjC,CAAC;CACF;AAJD,wCAIC;AAED,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAGxB,MAAM,qBAAqB,GAAG,KAAK,EAAE,SAAiB,EAAE,YAAoB,EAA2B,EAAE;IAC9G,IAAI,KAAe,CAAC;IACpB,IAAI;QACF,KAAK,GAAG,MAAM,aAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;KACrC;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ,EAAE;YAC5B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;SAC9E;QACD,MAAM,KAAK,CAAC;KACb;IACD,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,IAAI,YAAY,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACvG,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrG,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,KAAK,EAAE,EAAE,IAAI,CAAC,EAAE;QACrC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAChB,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACxB;KACF;IACD,IAAI,MAAM,GAAW,IAAI,CAAC;IAC1B,IAAI,MAAM,GAAW,IAAI,CAAC;IAC1B,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,MAAM,EAAE;QAC7B,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,aAAE,CAAC,IAAI,CAAC,IAAA,WAAI,EAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;QACzD,MAAM,GAAG,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,GAAG,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KAChE;IACD,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,aAAa;QACb,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC;QACtD,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAjCW,QAAA,qBAAqB,yBAiChC;AAEF,MAAM,WAAW,GAAG,CAAC,OAAe,EAAE,QAAgB,EAAU,EAAE,CAChE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AAE7D,MAAa,SAAS;IAGS;IAFrB,QAAQ,CAAS;IAEzB,YAA6B,OAAyB;QAAzB,YAAO,GAAP,OAAO,CAAkB;IAAI,CAAC;IAE3D,IAAY,YAAY;QACtB,OAAO,IAAA,WAAI,EAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI;YACF,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,aAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAa,CAAC;SAC7E;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ,EAAE;gBAC5B,OAAO,IAAI,CAAC;aACb;YACD,MAAM,KAAK,CAAC;SACb;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,GAAS;QACxC,MAAM,aAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC;QAC7D,IAAI;YACF,MAAM,aAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;SACvG;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ,EAAE;gBAC5B,MAAM,KAAK,CAAC;aACb;YACD,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,aAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC5C,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,GAAG,OAAO,EAAE;gBACrC,MAAM,IAAI,cAAc,CAAC,QAAQ,EAAE,qBAAqB,QAAQ,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;aAChH;YACD,OAAO,CAAC,GAAG,CAAC,mCAAmC,QAAQ,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;YAChG,MAAM,aAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;SACvF;QACD,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,IAAI,CAAC,QAAQ,EAAE;YACjB,MAAM,aAAE,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;SACtB;IACH,CAAC;IAED,WAAW,CAAC,QAAwB,EAAE,GAAS;QAC7C,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE;YACnB,MAAM,IAAI,cAAc,CAAC,cAAc,EAAE,yCAAyC,CAAC,CAAC;SACrF;QACD,IAAI,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE;YACjC,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE,wCAAwC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACtI;QACD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC;QAC5E,IAAI,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE;YAC7C,MAAM,IAAI,cAAc,CACtB,cAAc,EACd,+BAA+B,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,OAAO,CAAC,iBAAiB,KAAK,CACrG,CAAC;SACH;IACH,CAAC;IAED,WAAW,CAAC,MAAkB,EAAE,QAAyB;QACvD,IAAI,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE;YACpD,MAAM,IAAI,cAAc,CAAC,kBAAkB,EAAE,GAAG,MAAM,CAAC,WAAW,0BAA0B,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;SAC5H;QACD,IAAI,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE;YAC1D,MAAM,IAAI,cAAc,CACtB,qBAAqB,EACrB,GAAG,MAAM,CAAC,cAAc,6BAA6B,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CACtF,CAAC;SACH;QACD,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE;YACxC,OAAO;SACR;QACD,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QACzE,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;QAClF,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE;YAC3F,MAAM,IAAI,cAAc,CACtB,YAAY,EACZ,UAAU,QAAQ,CAAC,WAAW,OAAO,MAAM,CAAC,WAAW,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;gBACxF,YAAY,QAAQ,CAAC,cAAc,OAAO,MAAM,CAAC,cAAc,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;gBACnG,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,wDAAwD,CAC3F,CAAC;SACH;IACH,CAAC;IAED,UAAU,CAAC,QAAyB,EAAE,GAAS;QAC7C,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACxF,CAAC;IAED,OAAO,CAAC,QAAyB,EAAE,GAAS;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC7C,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;IAC9D,CAAC;IAGD,KAAK,CAAC,OAAO,CAAC,OAAe,EAAE,SAAiB,EAAE,MAAkB,EAAE,GAAS,EAAE,KAAc;QAC7F,IAAI,IAAA,cAAO,EAAC,OAAO,CAAC,KAAK,IAAA,cAAO,EAAC,SAAS,CAAC,EAAE;YAC3C,MAAM,IAAI,KAAK,CAAC,eAAe,IAAA,eAAQ,EAAC,OAAO,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAC;SAC9F;QACD,IAAI;YACF,MAAM,aAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,SAAS,OAAO,CAAC,CAAC;SACnD;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ,EAAE;gBAC5B,MAAM,KAAK,CAAC;aACb;SACF;QACD,MAAM,aAAE,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAa,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QACjG,MAAM,aAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,aAAE,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,YAAY,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1F,MAAM,aAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,YAAY,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAC/D,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF;AApHD,8BAoHC"}
@@ -0,0 +1,45 @@
1
+ export declare enum ImageUrlRejectReason {
2
+ invalidUrl = "invalid_url",
3
+ notHttps = "not_https",
4
+ credentials = "credentials_in_url",
5
+ placeholder = "placeholder",
6
+ httpStatus = "http_status",
7
+ wrongMime = "wrong_mime",
8
+ tooManyRedirects = "too_many_redirects",
9
+ timeout = "timeout",
10
+ networkError = "network_error",
11
+ budgetExhausted = "budget_exhausted"
12
+ }
13
+ export interface ImageUrlCheck {
14
+ url: string;
15
+ ok: boolean;
16
+ reason?: ImageUrlRejectReason;
17
+ status?: number;
18
+ contentType?: string;
19
+ finalUrl?: string;
20
+ }
21
+ export interface ImageUrlValidatorOptions {
22
+ allowedMimeTypes?: string[];
23
+ requireHttps?: boolean;
24
+ timeoutMs?: number;
25
+ maxRedirects?: number;
26
+ concurrency?: number;
27
+ maxProbes?: number;
28
+ userAgent?: string;
29
+ }
30
+ export declare class ImageUrlValidator {
31
+ private readonly options;
32
+ private readonly limiter;
33
+ private readonly cache;
34
+ private readonly counters;
35
+ private probes;
36
+ constructor(options?: ImageUrlValidatorOptions);
37
+ validate(url: string): Promise<ImageUrlCheck>;
38
+ validateAll(urls: string[]): Promise<ImageUrlCheck[]>;
39
+ get stats(): Record<string, number>;
40
+ private check;
41
+ private checkStatically;
42
+ private request;
43
+ private checkResponse;
44
+ private mapError;
45
+ }
@@ -0,0 +1,141 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ImageUrlValidator = exports.ImageUrlRejectReason = void 0;
4
+ const axios_1 = require("axios");
5
+ const bottleneck_1 = require("bottleneck");
6
+ var ImageUrlRejectReason;
7
+ (function (ImageUrlRejectReason) {
8
+ ImageUrlRejectReason["invalidUrl"] = "invalid_url";
9
+ ImageUrlRejectReason["notHttps"] = "not_https";
10
+ ImageUrlRejectReason["credentials"] = "credentials_in_url";
11
+ ImageUrlRejectReason["placeholder"] = "placeholder";
12
+ ImageUrlRejectReason["httpStatus"] = "http_status";
13
+ ImageUrlRejectReason["wrongMime"] = "wrong_mime";
14
+ ImageUrlRejectReason["tooManyRedirects"] = "too_many_redirects";
15
+ ImageUrlRejectReason["timeout"] = "timeout";
16
+ ImageUrlRejectReason["networkError"] = "network_error";
17
+ ImageUrlRejectReason["budgetExhausted"] = "budget_exhausted";
18
+ })(ImageUrlRejectReason || (exports.ImageUrlRejectReason = ImageUrlRejectReason = {}));
19
+ const PLACEHOLDER_PATTERN = /\/carimage\/noimage\//i;
20
+ class ImageUrlValidator {
21
+ options;
22
+ limiter;
23
+ cache = new Map();
24
+ counters = {};
25
+ probes = 0;
26
+ constructor(options = {}) {
27
+ this.options = {
28
+ allowedMimeTypes: ['image/jpeg', 'image/png'],
29
+ requireHttps: true,
30
+ timeoutMs: 5000,
31
+ maxRedirects: 3,
32
+ concurrency: 8,
33
+ maxProbes: Infinity,
34
+ userAgent: undefined,
35
+ ...options,
36
+ };
37
+ this.limiter = new bottleneck_1.default({ maxConcurrent: this.options.concurrency });
38
+ }
39
+ validate(url) {
40
+ if (!this.cache.has(url)) {
41
+ this.cache.set(url, this.check(url).then((result) => {
42
+ const key = result.ok ? 'ok' : result.reason;
43
+ this.counters[key] = (this.counters[key] ?? 0) + 1;
44
+ return result;
45
+ }));
46
+ }
47
+ return this.cache.get(url);
48
+ }
49
+ validateAll(urls) {
50
+ return Promise.all(urls.map(url => this.validate(url)));
51
+ }
52
+ get stats() {
53
+ return { ...this.counters };
54
+ }
55
+ async check(url) {
56
+ const rejected = this.checkStatically(url);
57
+ if (rejected) {
58
+ return rejected;
59
+ }
60
+ if (this.probes >= this.options.maxProbes) {
61
+ return { url, ok: false, reason: ImageUrlRejectReason.budgetExhausted };
62
+ }
63
+ this.probes++;
64
+ try {
65
+ let response = await this.limiter.schedule(() => this.request('HEAD', url));
66
+ if (response.status === 405 || response.status === 501) {
67
+ response = await this.limiter.schedule(() => this.request('GET', url));
68
+ }
69
+ return this.checkResponse(url, response);
70
+ }
71
+ catch (error) {
72
+ return { url, ok: false, reason: this.mapError(error) };
73
+ }
74
+ }
75
+ checkStatically(url) {
76
+ let parsed;
77
+ try {
78
+ parsed = new URL(url);
79
+ }
80
+ catch {
81
+ return { url, ok: false, reason: ImageUrlRejectReason.invalidUrl };
82
+ }
83
+ if (!['http:', 'https:'].includes(parsed.protocol)) {
84
+ return { url, ok: false, reason: ImageUrlRejectReason.invalidUrl };
85
+ }
86
+ if (this.options.requireHttps && parsed.protocol !== 'https:') {
87
+ return { url, ok: false, reason: ImageUrlRejectReason.notHttps };
88
+ }
89
+ if (parsed.username || parsed.password) {
90
+ return { url, ok: false, reason: ImageUrlRejectReason.credentials };
91
+ }
92
+ if (PLACEHOLDER_PATTERN.test(parsed.pathname)) {
93
+ return { url, ok: false, reason: ImageUrlRejectReason.placeholder };
94
+ }
95
+ return null;
96
+ }
97
+ request(method, url) {
98
+ return axios_1.default.request({
99
+ method,
100
+ url,
101
+ timeout: this.options.timeoutMs,
102
+ maxRedirects: this.options.maxRedirects,
103
+ responseType: 'stream',
104
+ headers: this.options.userAgent ? { 'User-Agent': this.options.userAgent } : undefined,
105
+ validateStatus: () => true,
106
+ }).then((response) => {
107
+ response.data?.destroy?.();
108
+ return response;
109
+ });
110
+ }
111
+ checkResponse(url, response) {
112
+ const status = response.status;
113
+ const contentType = String(response.headers?.['content-type'] ?? '').split(';')[0].trim().toLowerCase();
114
+ const finalUrl = response.request?.res?.responseUrl ?? url;
115
+ const result = { url, status, contentType, finalUrl };
116
+ if (status !== 200) {
117
+ return { ...result, ok: false, reason: ImageUrlRejectReason.httpStatus };
118
+ }
119
+ if (this.options.requireHttps && !finalUrl.startsWith('https:')) {
120
+ return { ...result, ok: false, reason: ImageUrlRejectReason.notHttps };
121
+ }
122
+ if (PLACEHOLDER_PATTERN.test(finalUrl)) {
123
+ return { ...result, ok: false, reason: ImageUrlRejectReason.placeholder };
124
+ }
125
+ if (!this.options.allowedMimeTypes.includes(contentType)) {
126
+ return { ...result, ok: false, reason: ImageUrlRejectReason.wrongMime };
127
+ }
128
+ return { ...result, ok: true };
129
+ }
130
+ mapError(error) {
131
+ if (error?.code === 'ERR_FR_TOO_MANY_REDIRECTS') {
132
+ return ImageUrlRejectReason.tooManyRedirects;
133
+ }
134
+ if (error?.code === 'ECONNABORTED' || error?.code === 'ETIMEDOUT' || /timeout/i.test(error?.message ?? '')) {
135
+ return ImageUrlRejectReason.timeout;
136
+ }
137
+ return ImageUrlRejectReason.networkError;
138
+ }
139
+ }
140
+ exports.ImageUrlValidator = ImageUrlValidator;
141
+ //# sourceMappingURL=image-url-validator.service.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"image-url-validator.service.js","sourceRoot":"","sources":["../../src/services/image-url-validator.service.ts"],"names":[],"mappings":";;;AAAA,iCAA6C;AAC7C,2CAAoC;AAEpC,IAAY,oBAWX;AAXD,WAAY,oBAAoB;IAC9B,kDAA0B,CAAA;IAC1B,8CAAsB,CAAA;IACtB,0DAAkC,CAAA;IAClC,mDAA2B,CAAA;IAC3B,kDAA0B,CAAA;IAC1B,gDAAwB,CAAA;IACxB,+DAAuC,CAAA;IACvC,2CAAmB,CAAA;IACnB,sDAA8B,CAAA;IAC9B,4DAAoC,CAAA;AACtC,CAAC,EAXW,oBAAoB,oCAApB,oBAAoB,QAW/B;AA2BD,MAAM,mBAAmB,GAAG,wBAAwB,CAAC;AASrD,MAAa,iBAAiB;IACX,OAAO,CAAqC;IAC5C,OAAO,CAAa;IACpB,KAAK,GAAG,IAAI,GAAG,EAAkC,CAAC;IAClD,QAAQ,GAA2B,EAAE,CAAC;IAC/C,MAAM,GAAG,CAAC,CAAC;IAEnB,YAAY,UAAoC,EAAE;QAChD,IAAI,CAAC,OAAO,GAAG;YACb,gBAAgB,EAAE,CAAC,YAAY,EAAE,WAAW,CAAC;YAC7C,YAAY,EAAE,IAAI;YAClB,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,CAAC;YACf,WAAW,EAAE,CAAC;YACd,SAAS,EAAE,QAAQ;YACnB,SAAS,EAAE,SAAS;YACpB,GAAG,OAAO;SACX,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,oBAAU,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,QAAQ,CAAC,GAAW;QAClB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACxB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;gBAClD,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAC7C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnD,OAAO,MAAM,CAAC;YAChB,CAAC,CAAC,CAAC,CAAC;SACL;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,WAAW,CAAC,IAAc;QACxB,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;IAGD,IAAI,KAAK;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,GAAW;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,QAAQ,EAAE;YACZ,OAAO,QAAQ,CAAC;SACjB;QACD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzC,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,CAAC,eAAe,EAAE,CAAC;SACzE;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;QAEd,IAAI;YACF,IAAI,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;YAC5E,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;gBACtD,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;aACxE;YACD,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;SAC1C;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;SACzD;IACH,CAAC;IAEO,eAAe,CAAC,GAAW;QACjC,IAAI,MAAW,CAAC;QAChB,IAAI;YACF,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;SACvB;QAAC,MAAM;YACN,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,CAAC,UAAU,EAAE,CAAC;SACpE;QACD,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;YAClD,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,CAAC,UAAU,EAAE,CAAC;SACpE;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAC7D,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE,CAAC;SAClE;QACD,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE;YACtC,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,CAAC,WAAW,EAAE,CAAC;SACrE;QACD,IAAI,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;YAC7C,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,CAAC,WAAW,EAAE,CAAC;SACrE;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,OAAO,CAAC,MAAsB,EAAE,GAAW;QACjD,OAAO,eAAK,CAAC,OAAO,CAAC;YACnB,MAAM;YACN,GAAG;YACH,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS;YAC/B,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY;YAEvC,YAAY,EAAE,QAAQ;YACtB,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS;YACtF,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI;SAC3B,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;YACnB,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC;YAC3B,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,aAAa,CAAC,GAAW,EAAE,QAAuB;QACxD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC/B,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACxG,MAAM,QAAQ,GAAW,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE,WAAW,IAAI,GAAG,CAAC;QACnE,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;QAEtD,IAAI,MAAM,KAAK,GAAG,EAAE;YAClB,OAAO,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,CAAC,UAAU,EAAE,CAAC;SAC1E;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;YAC/D,OAAO,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE,CAAC;SACxE;QACD,IAAI,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;YACtC,OAAO,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,CAAC,WAAW,EAAE,CAAC;SAC3E;QACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;YACxD,OAAO,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,CAAC,SAAS,EAAE,CAAC;SACzE;QACD,OAAO,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;IACjC,CAAC;IAEO,QAAQ,CAAC,KAAU;QACzB,IAAI,KAAK,EAAE,IAAI,KAAK,2BAA2B,EAAE;YAC/C,OAAO,oBAAoB,CAAC,gBAAgB,CAAC;SAC9C;QACD,IAAI,KAAK,EAAE,IAAI,KAAK,cAAc,IAAI,KAAK,EAAE,IAAI,KAAK,WAAW,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,EAAE,CAAC,EAAE;YAC1G,OAAO,oBAAoB,CAAC,OAAO,CAAC;SACrC;QACD,OAAO,oBAAoB,CAAC,YAAY,CAAC;IAC3C,CAAC;CACF;AAlID,8CAkIC"}
@@ -5,6 +5,14 @@ import { ConfigService } from './config.service';
5
5
  import { UserService } from './user.service';
6
6
  import { Config } from '../configs/config';
7
7
  import { ImageSize, ImageType } from '../enums';
8
+ export interface CleanImage {
9
+ url: string;
10
+ imageIdentifier: string;
11
+ imageCode: string;
12
+ heroCandidate: boolean;
13
+ }
14
+ export declare const CLEAN_HERO_IMAGE_CODES: string[];
15
+ export declare const CLEAN_IMAGE_VARIANT = "large-cached.jpeg";
8
16
  export declare class ImagesService {
9
17
  private readonly configService;
10
18
  private readonly userService;
@@ -21,6 +29,10 @@ export declare class ImagesService {
21
29
  private categorizeImages;
22
30
  getDefaultImages(carId: string, images: ImboImageResponseInterface[], ignoreDefImages: boolean): Array<string>;
23
31
  generateImageLinks(images: ImboImageResponseInterface[], carId: string, size?: ImageSize): string[];
32
+ getCleanImages(carId: string, images: ImboImageResponseInterface[]): CleanImage[];
33
+ getCleanImageUrls(carId: string, images: ImboImageResponseInterface[]): string[];
34
+ private isCleanImageCode;
35
+ private getImageCode;
24
36
  getMarketingImages(carId: string, images: ImboImageResponseInterface[], ignoreDefImages: boolean): Promise<string[]>;
25
37
  getMetaAiaImage(carId: string, images: ImboImageResponseInterface[], mode: string): Promise<string | null>;
26
38
  private imageUrlExists;