@bytebrand/export-cars 1.26.28 → 1.26.29

package/dist/services/openai-ads-feed-run.js

@@ -0,0 +1,151 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAiAdsFeedRun = void 0;
+const fs_1 = require("fs");
+const path_1 = require("path");
+const openai_ads_csv_builder_1 = require("../formats/openai-ads-csv.builder");
+const openai_ads_interface_1 = require("../interfaces/openai-ads.interface");
+const openai_ads_transformer_1 = require("../transformers/openai-ads.transformer");
+const feed_guard_1 = require("./feed-guard");
+const openai_ads_ledger_1 = require("./openai-ads-ledger");
+const increment = (counter, key) => {
+    counter[key] = (counter[key] ?? 0) + 1;
+};
+class OpenAiAdsFeedRun {
+    deps;
+    options;
+    constructor(deps, options) {
+        this.deps = deps;
+        this.options = options;
+    }
+    now() {
+        return this.options.now ? this.options.now() : new Date();
+    }
+    async run() {
+        const { guard } = this.deps;
+        const { outputFile, stateDir, runId } = this.options;
+        const startedAt = this.now();
+        const lastGood = await guard.readLastGood();
+        const report = {
+            runId,
+            mode: this.options.dryRun ? 'dry-run' : 'live',
+            startedAt: startedAt.toISOString(),
+            status: 'failed',
+            outputFile,
+            source: { files: 0, newestChunkAt: null, cars: 0 },
+            rows: { inStock: 0, outOfStock: 0, total: 0 },
+            outOfStock: {},
+            excluded: {},
+            reservedNeverSent: 0,
+            ledger: { before: 0, after: 0, expired: 0 },
+            price: { min: null, max: null },
+            images: {},
+            lastGood,
+            lastGoodAgeHours: guard.staleHours(lastGood, startedAt),
+            stale: guard.isStale(lastGood, startedAt),
+        };
+        await guard.acquireLock(runId ?? 'local', startedAt);
+        const tmpFile = `${outputFile}.tmp`;
+        try {
+            const snapshot = await (0, feed_guard_1.inspectSourceSnapshot)(this.options.sourceDirectory, this.options.sourceBaseFileName);
+            report.source.files = snapshot.files;
+            report.source.newestChunkAt = snapshot.newestMtime?.toISOString() ?? null;
+            guard.checkSource(snapshot, startedAt);
+            const ledgerFile = (0, path_1.join)(stateDir, 'ledger.json');
+            const ledger = await openai_ads_ledger_1.OpenAiAdsLedger.load(ledgerFile);
+            report.ledger.before = ledger.size;
+            await fs_1.promises.mkdir((0, path_1.dirname)(outputFile), { recursive: true });
+            const builder = new openai_ads_csv_builder_1.OpenAiAdsCsvBuilder(tmpFile);
+            await builder.buildStart();
+            const seen = new Set();
+            const inStockIds = new Set();
+            const freshOutOfStock = new Map();
+            const freshReasons = new Map();
+            const mapping = { ...this.options.mapping, now: startedAt };
+            await this.deps.forEachChunk(async (cars) => {
+                for (const source of cars) {
+                    report.source.cars += 1;
+                    const id = String(source?.car?._id ?? '');
+                    if (id && seen.has(id)) {
+                        increment(report.excluded, openai_ads_interface_1.OpenAiAdsExclusion.duplicateId);
+                        continue;
+                    }
+                    seen.add(id);
+                    if (!this.deps.filterCar(source)) {
+                        increment(report.excluded, openai_ads_interface_1.OpenAiAdsExclusion.filtered);
+                        continue;
+                    }
+                    const mapped = (0, openai_ads_transformer_1.mapOpenAiAdsVehicle)(source.car, mapping);
+                    if (mapped.ok === false) {
+                        increment(report.excluded, mapped.reason);
+                        continue;
+                    }
+                    if (mapped.outOfStockReason && !ledger.has(mapped.row.item_id)) {
+                        report.reservedNeverSent += 1;
+                        continue;
+                    }
+                    const dealer = await this.deps.dealerStatus(String(source.car?.metaData?.ownerId ?? ''));
+                    if (dealer !== 'active' && !mapped.outOfStockReason) {
+                        increment(report.excluded, dealer === 'inactive' ? openai_ads_interface_1.OpenAiAdsExclusion.dealerInactive : openai_ads_interface_1.OpenAiAdsExclusion.dealerUnknown);
+                        continue;
+                    }
+                    const images = await this.deps.resolveImages(source);
+                    if (!images.fields) {
+                        if (!mapped.outOfStockReason) {
+                            increment(report.excluded, openai_ads_interface_1.OpenAiAdsExclusion.noImage);
+                        }
+                        continue;
+                    }
+                    const row = (0, openai_ads_transformer_1.withOpenAiImages)(mapped.row, images.fields);
+                    if (mapped.outOfStockReason) {
+                        freshOutOfStock.set(row.item_id, row);
+                        freshReasons.set(row.item_id, mapped.outOfStockReason);
+                        continue;
+                    }
+                    await builder.addItem(row);
+                    inStockIds.add(row.item_id);
+                    ledger.markInStock(row, startedAt);
+                    report.rows.inStock += 1;
+                    const amount = Number(row.price.split(' ')[0]);
+                    report.price.min = report.price.min === null ? amount : Math.min(report.price.min, amount);
+                    report.price.max = report.price.max === null ? amount : Math.max(report.price.max, amount);
+                }
+            });
+            const { rows: outOfStockRows, expired } = ledger.collectOutOfStock(inStockIds, freshOutOfStock, startedAt, this.options.retentionDays);
+            for (const { row, fresh } of outOfStockRows) {
+                await builder.addItem(row);
+                increment(report.outOfStock, fresh ? freshReasons.get(row.item_id) : openai_ads_interface_1.OpenAiAdsOutOfStockReason.notInSnapshot);
+            }
+            report.rows.outOfStock = outOfStockRows.length;
+            report.rows.total = await builder.buildEnd();
+            report.ledger.expired = expired;
+            report.images = this.deps.imageStats();
+            const counts = { sourceItems: report.source.cars, publishedItems: report.rows.inStock };
+            guard.checkCounts(counts, lastGood);
+            report.lastGood = await guard.publish(tmpFile, outputFile, counts, this.now(), runId);
+            await ledger.save(ledgerFile, this.now());
+            report.ledger.after = ledger.size;
+            report.status = 'published';
+            report.lastGoodAgeHours = 0;
+            report.stale = false;
+            return report;
+        }
+        catch (error) {
+            if (error instanceof feed_guard_1.FeedGuardError) {
+                report.status = 'blocked';
+                report.guard = { code: error.code, message: error.message };
+            }
+            await fs_1.promises.rename(tmpFile, `${outputFile}.rejected`).catch(() => undefined);
+            throw error;
+        }
+        finally {
+            report.finishedAt = this.now().toISOString();
+            await fs_1.promises.mkdir(stateDir, { recursive: true });
+            await fs_1.promises.writeFile((0, path_1.join)(stateDir, 'last-run-report.json'), JSON.stringify(report, null, 2), 'utf8');
+            console.log(`OpenAI ads feed report: ${JSON.stringify(report)}`);
+            await guard.releaseLock();
+        }
+    }
+}
+exports.OpenAiAdsFeedRun = OpenAiAdsFeedRun;
+//# sourceMappingURL=openai-ads-feed-run.js.map

package/dist/services/openai-ads-feed-run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-ads-feed-run.js","sourceRoot":"","sources":["../../src/services/openai-ads-feed-run.ts"],"names":[],"mappings":";;;AAAA,2BAAoC;AACpC,+BAAqC;AAIrC,8EAAwE;AACxE,6EAI4C;AAC5C,mFAIgD;AAEhD,6CAA0F;AAC1F,2DAAsD;AAwDtD,MAAM,SAAS,GAAG,CAAC,OAA+B,EAAE,GAAW,EAAE,EAAE;IACjE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAC;AAOF,MAAa,gBAAgB;IACE;IAAyC;IAAtE,YAA6B,IAAsB,EAAmB,OAA4B;QAArE,SAAI,GAAJ,IAAI,CAAkB;QAAmB,YAAO,GAAP,OAAO,CAAqB;IAAI,CAAC;IAE/F,GAAG;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;QAC5B,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAuB;YACjC,KAAK;YACL,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YAC9C,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;YAClC,MAAM,EAAE,QAAQ;YAChB,UAAU;YACV,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE;YAClD,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;YAC7C,UAAU,EAAE,EAAE;YACd,QAAQ,EAAE,EAAE;YACZ,iBAAiB,EAAE,CAAC;YACpB,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;YAC3C,KAAK,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE;YAC/B,MAAM,EAAE,EAAE;YACV,QAAQ;YACR,gBAAgB,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC;YACvD,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC;SAC1C,CAAC;QAEF,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,IAAI,OAAO,EAAE,SAAS,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,GAAG,UAAU,MAAM,CAAC;QACpC,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,IAAA,kCAAqB,EAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAC5G,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,aAAa,GAAG,QAAQ,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC;YAC1E,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAEvC,MAAM,UAAU,GAAG,IAAA,WAAI,EAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,MAAM,mCAAe,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC;YAEnC,MAAM,aAAE,CAAC,KAAK,CAAC,IAAA,cAAO,EAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,MAAM,OAAO,GAAG,IAAI,4CAAmB,CAAC,OAAO,CAAC,CAAC;YACjD,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YAE3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;YACrC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAwB,CAAC;YACxD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAqC,CAAC;YAClE,MAAM,OAAO,GAA4B,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;YAErF,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBAC1C,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE;oBACzB,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;oBACxB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;oBAC1C,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;wBACtB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,yCAAkB,CAAC,WAAW,CAAC,CAAC;wBAC3D,SAAS;qBACV;oBACD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACb,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;wBAChC,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,yCAAkB,CAAC,QAAQ,CAAC,CAAC;wBACxD,SAAS;qBACV;oBACD,MAAM,MAAM,GAAG,IAAA,4CAAmB,EAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;oBACxD,IAAI,MAAM,CAAC,EAAE,KAAK,KAAK,EAAE;wBACvB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;wBAC1C,SAAS;qBACV;oBAED,IAAI,MAAM,CAAC,gBAAgB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;wBAC9D,MAAM,CAAC,iBAAiB,IAAI,CAAC,CAAC;wBAC9B,SAAS;qBACV;oBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;oBACzF,IAAI,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE;wBACnD,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,yCAAkB,CAAC,cAAc,CAAC,CAAC,CAAC,yCAAkB,CAAC,aAAa,CAAC,CAAC;wBACzH,SAAS;qBACV;oBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;oBACrD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;wBAClB,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE;4BAC5B,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,yCAAkB,CAAC,OAAO,CAAC,CAAC;yBACxD;wBACD,SAAS;qBACV;oBACD,MAAM,GAAG,GAAG,IAAA,yCAAgB,EAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;oBACxD,IAAI,MAAM,CAAC,gBAAgB,EAAE;wBAC3B,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;wBACtC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;wBACvD,SAAS;qBACV;oBACD,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;oBAC3B,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBAC5B,MAAM,CAAC,WAAW,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;oBACnC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;oBACzB,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/C,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;oBAC3F,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;iBAC5F;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC,iBAAiB,CAChE,UAAU,EAAE,eAAe,EAAE,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,aAAa,CACnE,CAAC;YACF,KAAK,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,cAAc,EAAE;gBAC3C,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC3B,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,gDAAyB,CAAC,aAAa,CAAC,CAAC;aAC/G;YACD,MAAM,CAAC,IAAI,CAAC,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;YAChC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YAEvC,MAAM,MAAM,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACxF,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAEpC,MAAM,CAAC,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAC;YACtF,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC;YAClC,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC5B,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACrB,OAAO,MAAM,CAAC;SACf;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,2BAAc,EAAE;gBACnC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;gBAC1B,MAAM,CAAC,KAAK,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;aAC7D;YAED,MAAM,aAAE,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,UAAU,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC1E,MAAM,KAAK,CAAC;SACb;gBAAS;YACR,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;YAC7C,MAAM,aAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,MAAM,aAAE,CAAC,SAAS,CAAC,IAAA,WAAI,EAAC,QAAQ,EAAE,sBAAsB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACpG,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACjE,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;SAC3B;IACH,CAAC;CACF;AA9ID,4CA8IC"}

package/dist/services/openai-ads-ledger.d.ts

@@ -0,0 +1,19 @@
+import { OpenAiAdsRow } from '../interfaces/openai-ads.interface';
+export interface OutOfStockRow {
+    row: OpenAiAdsRow;
+    fresh: boolean;
+}
+export declare class OpenAiAdsLedger {
+    private readonly items;
+    private constructor();
+    static empty(): OpenAiAdsLedger;
+    static load(file: string): Promise<OpenAiAdsLedger>;
+    get size(): number;
+    has(itemId: string): boolean;
+    markInStock(row: OpenAiAdsRow, now: Date): void;
+    collectOutOfStock(inStockIds: Set<string>, freshRows: Map<string, OpenAiAdsRow>, now: Date, retentionDays: number): {
+        rows: OutOfStockRow[];
+        expired: number;
+    };
+    save(file: string, now: Date): Promise<void>;
+}

package/dist/services/openai-ads-ledger.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAiAdsLedger = void 0;
+const fs_1 = require("fs");
+const path_1 = require("path");
+const openai_ads_transformer_1 = require("../transformers/openai-ads.transformer");
+const LEDGER_VERSION = 1;
+const DAY_MS = 24 * 60 * 60 * 1000;
+class OpenAiAdsLedger {
+    items;
+    constructor(items) {
+        this.items = items;
+    }
+    static empty() {
+        return new OpenAiAdsLedger({});
+    }
+    static async load(file) {
+        let raw;
+        try {
+            raw = await fs_1.promises.readFile(file, 'utf8');
+        }
+        catch (error) {
+            if (error?.code === 'ENOENT') {
+                return OpenAiAdsLedger.empty();
+            }
+            throw error;
+        }
+        const parsed = JSON.parse(raw);
+        if (parsed?.version !== LEDGER_VERSION || typeof parsed.items !== 'object') {
+            throw new Error(`OpenAI ads ledger ${file} has an unknown format (version ${parsed?.version})`);
+        }
+        return new OpenAiAdsLedger(parsed.items);
+    }
+    get size() {
+        return Object.keys(this.items).length;
+    }
+    has(itemId) {
+        return itemId in this.items;
+    }
+    markInStock(row, now) {
+        this.items[row.item_id] = {
+            lastInStockAt: now.toISOString(),
+            row: { ...row, additional_image_urls: '' },
+        };
+    }
+    collectOutOfStock(inStockIds, freshRows, now, retentionDays) {
+        const rows = [];
+        let expired = 0;
+        for (const [itemId, entry] of Object.entries(this.items)) {
+            if (inStockIds.has(itemId)) {
+                continue;
+            }
+            if (now.getTime() - Date.parse(entry.lastInStockAt) > retentionDays * DAY_MS) {
+                delete this.items[itemId];
+                expired += 1;
+                continue;
+            }
+            const fresh = freshRows.get(itemId);
+            rows.push({ row: (0, openai_ads_transformer_1.asOutOfStock)(fresh ?? entry.row), fresh: !!fresh });
+        }
+        return { rows, expired };
+    }
+    async save(file, now) {
+        const content = { version: LEDGER_VERSION, updatedAt: now.toISOString(), items: this.items };
+        await fs_1.promises.mkdir((0, path_1.dirname)(file), { recursive: true });
+        const tmp = `${file}.tmp`;
+        await fs_1.promises.writeFile(tmp, JSON.stringify(content), 'utf8');
+        await fs_1.promises.rename(tmp, file);
+    }
+}
+exports.OpenAiAdsLedger = OpenAiAdsLedger;
+//# sourceMappingURL=openai-ads-ledger.js.map

package/dist/services/openai-ads-ledger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-ads-ledger.js","sourceRoot":"","sources":["../../src/services/openai-ads-ledger.ts"],"names":[],"mappings":";;;AAAA,2BAAoC;AACpC,+BAA+B;AAG/B,mFAAsE;AAEtE,MAAM,cAAc,GAAG,CAAC,CAAC;AACzB,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAgCnC,MAAa,eAAe;IACW;IAArC,YAAqC,KAAkC;QAAlC,UAAK,GAAL,KAAK,CAA6B;IAAI,CAAC;IAE5E,MAAM,CAAC,KAAK;QACV,OAAO,IAAI,eAAe,CAAC,EAAE,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAY;QAC5B,IAAI,GAAW,CAAC;QAChB,IAAI;YACF,GAAG,GAAG,MAAM,aAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;SACvC;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ,EAAE;gBAC5B,OAAO,eAAe,CAAC,KAAK,EAAE,CAAC;aAChC;YACD,MAAM,KAAK,CAAC;SACb;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAe,CAAC;QAC7C,IAAI,MAAM,EAAE,OAAO,KAAK,cAAc,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE;YAC1E,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,mCAAmC,MAAM,EAAE,OAAO,GAAG,CAAC,CAAC;SACjG;QACD,OAAO,IAAI,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,IAAI;QACN,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IACxC,CAAC;IAED,GAAG,CAAC,MAAc;QAChB,OAAO,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC;IAC9B,CAAC;IAED,WAAW,CAAC,GAAiB,EAAE,GAAS;QACtC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG;YACxB,aAAa,EAAE,GAAG,CAAC,WAAW,EAAE;YAChC,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,qBAAqB,EAAE,EAAE,EAAE;SAC3C,CAAC;IACJ,CAAC;IAOD,iBAAiB,CACf,UAAuB,EACvB,SAAoC,EACpC,GAAS,EACT,aAAqB;QAErB,MAAM,IAAI,GAAoB,EAAE,CAAC;QACjC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;YACxD,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBAC1B,SAAS;aACV;YACD,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,aAAa,GAAG,MAAM,EAAE;gBAC5E,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC1B,OAAO,IAAI,CAAC,CAAC;gBACb,SAAS;aACV;YACD,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAA,qCAAY,EAAC,KAAK,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;SACtE;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;IAGD,KAAK,CAAC,IAAI,CAAC,IAAY,EAAE,GAAS;QAChC,MAAM,OAAO,GAAe,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QACzG,MAAM,aAAE,CAAC,KAAK,CAAC,IAAA,cAAO,EAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,GAAG,IAAI,MAAM,CAAC;QAC1B,MAAM,aAAE,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,CAAC;QACzD,MAAM,aAAE,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;CACF;AA5ED,0CA4EC"}

package/dist/services/publication-gate.js

@@ -64,14 +64,14 @@ class PublicationGate {
 exports.PublicationGate = PublicationGate;
 class EnrichmentLogReader extends bigquery_logger_1.BigQueryLogger {
     async latestEstimates(table) {
-        const query = `
-      SELECT carId, fieldName, finalValue
-      FROM (
-        SELECT carId, fieldName, finalValue, finalValueSource,
-               ROW_NUMBER() OVER (PARTITION BY carId, fieldName ORDER BY timestamp DESC) AS rn
-        FROM \`${table}\`
-        WHERE fieldName IN UNNEST(@fields)
-      )
+        const query = `
+      SELECT carId, fieldName, finalValue
+      FROM (
+        SELECT carId, fieldName, finalValue, finalValueSource,
+               ROW_NUMBER() OVER (PARTITION BY carId, fieldName ORDER BY timestamp DESC) AS rn
+        FROM \`${table}\`
+        WHERE fieldName IN UNNEST(@fields)
+      )
       WHERE rn = 1 AND finalValueSource IN UNNEST(@sources) AND finalValue IS NOT NULL`;
         const [rows] = await this.client.query({ query, params: { fields: exports.GATED_FIELDS, sources: exports.ESTIMATE_SOURCES } });
         const estimates = new Map();

package/dist/transformers/openai-ads.transformer.d.ts

@@ -0,0 +1,28 @@
+import { DataSchemes } from '@bytebrand/car-schema';
+import { OpenAiAdsCondition, OpenAiAdsExclusion, OpenAiAdsOutOfStockReason, OpenAiAdsRow } from '../interfaces/openai-ads.interface';
+import { OpenAiImageFields } from './openai-image-fields';
+export declare const OPENAI_TITLE_MAX_LENGTH = 150;
+export declare const OPENAI_DESCRIPTION_MAX_LENGTH = 5000;
+export interface OpenAiAdsMappingContext {
+    domain: string;
+    sellerName: string;
+    translate: (key: string) => string;
+    now: Date;
+}
+export type OpenAiAdsBaseRow = Omit<OpenAiAdsRow, 'image_url' | 'additional_image_urls'>;
+export type OpenAiAdsMapResult = {
+    ok: true;
+    row: OpenAiAdsBaseRow;
+    outOfStockReason?: OpenAiAdsOutOfStockReason;
+} | {
+    ok: false;
+    reason: OpenAiAdsExclusion;
+};
+export declare const mapOpenAiAdsVehicle: (car: DataSchemes.Car, ctx: OpenAiAdsMappingContext) => OpenAiAdsMapResult;
+export declare const withOpenAiImages: (row: OpenAiAdsBaseRow, images: OpenAiImageFields) => OpenAiAdsRow;
+export declare const asOutOfStock: (row: OpenAiAdsRow) => OpenAiAdsRow;
+export declare const formatOpenAiPrice: (value: unknown) => string | null;
+export declare const toOpenAiMinorUnits: (value: unknown) => number | null;
+export declare const buildOpenAiTitle: (car: DataSchemes.Car) => string;
+export declare const resolveOpenAiCondition: (car: DataSchemes.Car, now: Date) => OpenAiAdsCondition | null;
+export declare const buildOpenAiDescription: (car: DataSchemes.Car, title: string, condition: OpenAiAdsCondition, ctx: OpenAiAdsMappingContext) => string;

package/dist/transformers/openai-ads.transformer.js

@@ -0,0 +1,144 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildOpenAiDescription = exports.resolveOpenAiCondition = exports.buildOpenAiTitle = exports.toOpenAiMinorUnits = exports.formatOpenAiPrice = exports.asOutOfStock = exports.withOpenAiImages = exports.mapOpenAiAdsVehicle = exports.OPENAI_DESCRIPTION_MAX_LENGTH = exports.OPENAI_TITLE_MAX_LENGTH = void 0;
+const lodash_1 = require("lodash");
+const utils_1 = require("../utils");
+const openai_ads_interface_1 = require("../interfaces/openai-ads.interface");
+const openai_image_fields_1 = require("./openai-image-fields");
+exports.OPENAI_TITLE_MAX_LENGTH = 150;
+exports.OPENAI_DESCRIPTION_MAX_LENGTH = 5000;
+const PLACEHOLDER_MAKES = /^(sonstige|sonstiges|andere|other|unknown|unbekannt)$/i;
+const OFFER_AVAILABILITY_OUT_OF_STOCK = {
+    selector_offerAvailability_reserved: openai_ads_interface_1.OpenAiAdsOutOfStockReason.reserved,
+    selector_offerAvailability_sold: openai_ads_interface_1.OpenAiAdsOutOfStockReason.soldFlag,
+};
+const mapOpenAiAdsVehicle = (car, ctx) => {
+    const itemId = (0, lodash_1.toString)((0, lodash_1.get)(car, '_id', '')).trim();
+    if (!itemId) {
+        return { ok: false, reason: openai_ads_interface_1.OpenAiAdsExclusion.noId };
+    }
+    const price = (0, exports.formatOpenAiPrice)((0, lodash_1.get)(car, 'price.currentSalesPrice', null));
+    if (!price) {
+        return { ok: false, reason: openai_ads_interface_1.OpenAiAdsExclusion.invalidPrice };
+    }
+    const brand = (0, lodash_1.trim)((0, lodash_1.toString)((0, lodash_1.get)(car, 'mainData.make', '')));
+    if (!brand || PLACEHOLDER_MAKES.test(brand)) {
+        return { ok: false, reason: openai_ads_interface_1.OpenAiAdsExclusion.noBrand };
+    }
+    const title = (0, exports.buildOpenAiTitle)(car);
+    if (!title) {
+        return { ok: false, reason: openai_ads_interface_1.OpenAiAdsExclusion.noTitle };
+    }
+    const condition = (0, exports.resolveOpenAiCondition)(car, ctx.now);
+    if (!condition) {
+        return { ok: false, reason: openai_ads_interface_1.OpenAiAdsExclusion.conditionUnknown };
+    }
+    const row = {
+        item_id: itemId,
+        title,
+        description: (0, exports.buildOpenAiDescription)(car, title, condition, ctx),
+        url: `https://www.${ctx.domain}/search/vehicle/${itemId}`,
+        brand,
+        seller_name: ctx.sellerName,
+        availability: 'in_stock',
+        price,
+        condition,
+        is_ads_eligible: true,
+    };
+    const outOfStockReason = OFFER_AVAILABILITY_OUT_OF_STOCK[(0, lodash_1.toString)((0, lodash_1.get)(car, 'offer.offerAvailability', ''))];
+    return outOfStockReason ? { ok: true, row, outOfStockReason } : { ok: true, row };
+};
+exports.mapOpenAiAdsVehicle = mapOpenAiAdsVehicle;
+const withOpenAiImages = (row, images) => ({
+    ...row,
+    image_url: images.image_url,
+    additional_image_urls: images.additional_image_urls.map(openai_image_fields_1.encodeCommasInUrl).join(','),
+});
+exports.withOpenAiImages = withOpenAiImages;
+const asOutOfStock = (row) => ({
+    ...row,
+    availability: 'out_of_stock',
+    is_ads_eligible: false,
+});
+exports.asOutOfStock = asOutOfStock;
+const formatOpenAiPrice = (value) => {
+    if (value === null || value === undefined || value === '' || typeof value === 'boolean') {
+        return null;
+    }
+    const amount = (0, lodash_1.toNumber)(value);
+    if (!Number.isFinite(amount) || amount <= 0) {
+        return null;
+    }
+    return `${amount.toFixed(2)} EUR`;
+};
+exports.formatOpenAiPrice = formatOpenAiPrice;
+const toOpenAiMinorUnits = (value) => {
+    const formatted = (0, exports.formatOpenAiPrice)(value);
+    return formatted ? Math.round((0, lodash_1.toNumber)(formatted.split(' ')[0]) * 100) : null;
+};
+exports.toOpenAiMinorUnits = toOpenAiMinorUnits;
+const buildOpenAiTitle = (car) => {
+    const parts = ['mainData.make', 'mainData.model', 'mainData.subModel'].map(path => (0, lodash_1.toString)((0, lodash_1.get)(car, path, '')));
+    const title = (0, utils_1.normalizeFeedText)(parts.join(' '));
+    return (0, lodash_1.truncate)(title, { length: exports.OPENAI_TITLE_MAX_LENGTH, separator: ' ', omission: '' }).trim();
+};
+exports.buildOpenAiTitle = buildOpenAiTitle;
+const resolveOpenAiCondition = (car, now) => {
+    const condition = (0, lodash_1.toString)((0, lodash_1.get)(car, 'mainData.condition', ''));
+    if (condition === 'selector_condition_new') {
+        return 'new';
+    }
+    if (condition === 'selector_condition_used') {
+        return 'used';
+    }
+    const firstRegistration = (0, lodash_1.toNumber)((0, lodash_1.get)(car, 'mainData.firstRegistration', 0));
+    return firstRegistration > 0 && firstRegistration * 1000 <= now.getTime() ? 'used' : null;
+};
+exports.resolveOpenAiCondition = resolveOpenAiCondition;
+const buildOpenAiDescription = (car, title, condition, ctx) => {
+    const label = (path) => {
+        const key = (0, lodash_1.toString)((0, lodash_1.get)(car, path, ''));
+        if (!key || key === 'selector_unknown') {
+            return null;
+        }
+        const value = (0, lodash_1.trim)(ctx.translate(key));
+        return value && !value.startsWith('selector_') && value !== 'Unbekannt' ? value : null;
+    };
+    const facts = [
+        ['Erstzulassung', formatFirstRegistration((0, lodash_1.get)(car, 'mainData.firstRegistration'))],
+        ['Kilometerstand', formatMileage((0, lodash_1.get)(car, 'mainData.mileage'))],
+        ['Kraftstoff', label('consumption.fuel')],
+        ['Getriebe', label('driveSuspension.gearbox')],
+        ['Leistung', formatPower((0, lodash_1.get)(car, 'engineData.powerKW'), (0, lodash_1.get)(car, 'engineData.powerPS'))],
+        ['Karosserie', label('mainData.category')],
+        ['Außenfarbe', label('exterior.exteriorColor')],
+    ];
+    const intro = `${condition === 'new' ? 'Neuwagen' : 'Gebrauchtwagen'}: ${title}.`;
+    const details = facts.filter(([, value]) => value).map(([name, value]) => `${name}: ${value}`).join(' · ');
+    const text = (details ? `${intro} ${details}` : intro).replace(/\s+/g, ' ').trim();
+    return (0, lodash_1.truncate)(text, { length: exports.OPENAI_DESCRIPTION_MAX_LENGTH, separator: ' ', omission: '' }).trim();
+};
+exports.buildOpenAiDescription = buildOpenAiDescription;
+const DE_NUMBER = new Intl.NumberFormat('de-DE', { maximumFractionDigits: 0 });
+const formatFirstRegistration = (value) => {
+    const seconds = (0, lodash_1.toNumber)(value);
+    if (!Number.isFinite(seconds) || seconds <= 0) {
+        return null;
+    }
+    const date = new Date(seconds * 1000);
+    return `${String(date.getUTCMonth() + 1).padStart(2, '0')}/${date.getUTCFullYear()}`;
+};
+const formatMileage = (value) => {
+    const km = (0, lodash_1.toNumber)(value);
+    return Number.isFinite(km) && km > 0 ? `${DE_NUMBER.format(km)} km` : null;
+};
+const formatPower = (kw, ps) => {
+    const powerKw = (0, lodash_1.toNumber)(kw);
+    if (!Number.isFinite(powerKw) || powerKw <= 0) {
+        return null;
+    }
+    const powerPs = (0, lodash_1.toNumber)(ps);
+    const psPart = Number.isFinite(powerPs) && powerPs > 0 ? ` (${DE_NUMBER.format(powerPs)} PS)` : '';
+    return `${DE_NUMBER.format(powerKw)} kW${psPart}`;
+};
+//# sourceMappingURL=openai-ads.transformer.js.map

package/dist/transformers/openai-ads.transformer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-ads.transformer.js","sourceRoot":"","sources":["../../src/transformers/openai-ads.transformer.ts"],"names":[],"mappings":";;;AAAA,mCAAiE;AAGjE,oCAA6C;AAC7C,6EAK4C;AAC5C,+DAA6E;AAGhE,QAAA,uBAAuB,GAAG,GAAG,CAAC;AAC9B,QAAA,6BAA6B,GAAG,IAAI,CAAC;AAIlD,MAAM,iBAAiB,GAAG,wDAAwD,CAAC;AAEnF,MAAM,+BAA+B,GAA8C;IACjF,mCAAmC,EAAE,gDAAyB,CAAC,QAAQ;IACvE,+BAA+B,EAAE,gDAAyB,CAAC,QAAQ;CACpE,CAAC;AAuBK,MAAM,mBAAmB,GAAG,CAAC,GAAoB,EAAE,GAA4B,EAAsB,EAAE;IAC5G,MAAM,MAAM,GAAG,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpD,IAAI,CAAC,MAAM,EAAE;QACX,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,yCAAkB,CAAC,IAAI,EAAE,CAAC;KACvD;IAED,MAAM,KAAK,GAAG,IAAA,yBAAiB,EAAC,IAAA,YAAG,EAAC,GAAG,EAAE,yBAAyB,EAAE,IAAI,CAAC,CAAC,CAAC;IAC3E,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,yCAAkB,CAAC,YAAY,EAAE,CAAC;KAC/D;IAED,MAAM,KAAK,GAAG,IAAA,aAAI,EAAC,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,GAAG,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5D,IAAI,CAAC,KAAK,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QAC3C,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,yCAAkB,CAAC,OAAO,EAAE,CAAC;KAC1D;IAED,MAAM,KAAK,GAAG,IAAA,wBAAgB,EAAC,GAAG,CAAC,CAAC;IACpC,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,yCAAkB,CAAC,OAAO,EAAE,CAAC;KAC1D;IAED,MAAM,SAAS,GAAG,IAAA,8BAAsB,EAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACvD,IAAI,CAAC,SAAS,EAAE;QACd,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,yCAAkB,CAAC,gBAAgB,EAAE,CAAC;KACnE;IAED,MAAM,GAAG,GAAqB;QAC5B,OAAO,EAAE,MAAM;QACf,KAAK;QACL,WAAW,EAAE,IAAA,8BAAsB,EAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC;QAC/D,GAAG,EAAE,eAAe,GAAG,CAAC,MAAM,mBAAmB,MAAM,EAAE;QACzD,KAAK;QACL,WAAW,EAAE,GAAG,CAAC,UAAU;QAC3B,YAAY,EAAE,UAAU;QACxB,KAAK;QACL,SAAS;QACT,eAAe,EAAE,IAAI;KACtB,CAAC;IAEF,MAAM,gBAAgB,GAAG,+BAA+B,CAAC,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,GAAG,EAAE,yBAAyB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5G,OAAO,gBAAgB,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;AACpF,CAAC,CAAC;AAzCW,QAAA,mBAAmB,uBAyC9B;AAEK,MAAM,gBAAgB,GAAG,CAAC,GAAqB,EAAE,MAAyB,EAAgB,EAAE,CAAC,CAAC;IACnG,GAAG,GAAG;IACN,SAAS,EAAE,MAAM,CAAC,SAAS;IAE3B,qBAAqB,EAAE,MAAM,CAAC,qBAAqB,CAAC,GAAG,CAAC,uCAAiB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;CACrF,CAAC,CAAC;AALU,QAAA,gBAAgB,oBAK1B;AAII,MAAM,YAAY,GAAG,CAAC,GAAiB,EAAgB,EAAE,CAAC,CAAC;IAChE,GAAG,GAAG;IACN,YAAY,EAAE,cAAc;IAC5B,eAAe,EAAE,KAAK;CACvB,CAAC,CAAC;AAJU,QAAA,YAAY,gBAItB;AAKI,MAAM,iBAAiB,GAAG,CAAC,KAAc,EAAiB,EAAE;IACjE,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;QACvF,OAAO,IAAI,CAAC;KACb;IACD,MAAM,MAAM,GAAG,IAAA,iBAAQ,EAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE;QAC3C,OAAO,IAAI,CAAC;KACb;IACD,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;AACpC,CAAC,CAAC;AATW,QAAA,iBAAiB,qBAS5B;AAGK,MAAM,kBAAkB,GAAG,CAAC,KAAc,EAAiB,EAAE;IAClE,MAAM,SAAS,GAAG,IAAA,yBAAiB,EAAC,KAAK,CAAC,CAAC;IAC3C,OAAO,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAA,iBAAQ,EAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAChF,CAAC,CAAC;AAHW,QAAA,kBAAkB,sBAG7B;AAKK,MAAM,gBAAgB,GAAG,CAAC,GAAoB,EAAU,EAAE;IAC/D,MAAM,KAAK,GAAG,CAAC,eAAe,EAAE,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACjH,MAAM,KAAK,GAAG,IAAA,yBAAiB,EAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACjD,OAAO,IAAA,iBAAQ,EAAC,KAAK,EAAE,EAAE,MAAM,EAAE,+BAAuB,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AACnG,CAAC,CAAC;AAJW,QAAA,gBAAgB,oBAI3B;AAKK,MAAM,sBAAsB,GAAG,CAAC,GAAoB,EAAE,GAAS,EAA6B,EAAE;IACnG,MAAM,SAAS,GAAG,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,GAAG,EAAE,oBAAoB,EAAE,EAAE,CAAC,CAAC,CAAC;IAC/D,IAAI,SAAS,KAAK,wBAAwB,EAAE;QAC1C,OAAO,KAAK,CAAC;KACd;IACD,IAAI,SAAS,KAAK,yBAAyB,EAAE;QAC3C,OAAO,MAAM,CAAC;KACf;IACD,MAAM,iBAAiB,GAAG,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,GAAG,EAAE,4BAA4B,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9E,OAAO,iBAAiB,GAAG,CAAC,IAAI,iBAAiB,GAAG,IAAI,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5F,CAAC,CAAC;AAVW,QAAA,sBAAsB,0BAUjC;AAQK,MAAM,sBAAsB,GAAG,CACpC,GAAoB,EACpB,KAAa,EACb,SAA6B,EAC7B,GAA4B,EACpB,EAAE;IACV,MAAM,KAAK,GAAG,CAAC,IAAY,EAAiB,EAAE;QAC5C,MAAM,GAAG,GAAG,IAAA,iBAAQ,EAAC,IAAA,YAAG,EAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,kBAAkB,EAAE;YACtC,OAAO,IAAI,CAAC;SACb;QACD,MAAM,KAAK,GAAG,IAAA,aAAI,EAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,OAAO,KAAK,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IACzF,CAAC,CAAC;IAEF,MAAM,KAAK,GAAmC;QAC5C,CAAC,eAAe,EAAE,uBAAuB,CAAC,IAAA,YAAG,EAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC,CAAC;QAClF,CAAC,gBAAgB,EAAE,aAAa,CAAC,IAAA,YAAG,EAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC,CAAC;QAC/D,CAAC,YAAY,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACzC,CAAC,UAAU,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC9C,CAAC,UAAU,EAAE,WAAW,CAAC,IAAA,YAAG,EAAC,GAAG,EAAE,oBAAoB,CAAC,EAAE,IAAA,YAAG,EAAC,GAAG,EAAE,oBAAoB,CAAC,CAAC,CAAC;QACzF,CAAC,YAAY,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC1C,CAAC,YAAY,EAAE,KAAK,CAAC,wBAAwB,CAAC,CAAC;KAChD,CAAC;IAEF,MAAM,KAAK,GAAG,GAAG,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,gBAAgB,KAAK,KAAK,GAAG,CAAC;IAClF,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3G,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACnF,OAAO,IAAA,iBAAQ,EAAC,IAAI,EAAE,EAAE,MAAM,EAAE,qCAA6B,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AACxG,CAAC,CAAC;AA7BW,QAAA,sBAAsB,0BA6BjC;AAEF,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;AAE/E,MAAM,uBAAuB,GAAG,CAAC,KAAc,EAAiB,EAAE;IAChE,MAAM,OAAO,GAAG,IAAA,iBAAQ,EAAC,KAAK,CAAC,CAAC;IAChC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EAAE;QAC7C,OAAO,IAAI,CAAC;KACb;IACD,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACtC,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;AACvF,CAAC,CAAC;AAGF,MAAM,aAAa,GAAG,CAAC,KAAc,EAAiB,EAAE;IACtD,MAAM,EAAE,GAAG,IAAA,iBAAQ,EAAC,KAAK,CAAC,CAAC;IAC3B,OAAO,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7E,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,EAAW,EAAE,EAAW,EAAiB,EAAE;IAC9D,MAAM,OAAO,GAAG,IAAA,iBAAQ,EAAC,EAAE,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EAAE;QAC7C,OAAO,IAAI,CAAC;KACb;IACD,MAAM,OAAO,GAAG,IAAA,iBAAQ,EAAC,EAAE,CAAC,CAAC;IAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IACnG,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,MAAM,EAAE,CAAC;AACpD,CAAC,CAAC"}

package/dist/transformers/openai-image-fields.d.ts

@@ -0,0 +1,16 @@
+import { CleanImage } from '../services/images.service';
+import { ImageUrlCheck, ImageUrlValidator } from '../services/image-url-validator.service';
+export interface OpenAiImageFields {
+    image_url: string;
+    additional_image_urls: string[];
+}
+export interface OpenAiImageSelection {
+    fields: OpenAiImageFields | null;
+    rejected: ImageUrlCheck[];
+}
+export declare const selectOpenAiImageFields: (images: CleanImage[], checks: ImageUrlCheck[]) => OpenAiImageSelection;
+export declare const resolveOpenAiImageFields: (images: CleanImage[], validator: ImageUrlValidator) => Promise<OpenAiImageSelection>;
+export declare const encodeCommasInUrl: (url: string) => string;
+export declare const serializeAdditionalImageUrlsForCsv: (urls: string[]) => string;
+export declare const serializeAdditionalImageUrlsForJsonl: (urls: string[]) => string[];
+export declare const quoteCsvCell: (value: string) => string;

package/dist/transformers/openai-image-fields.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.quoteCsvCell = exports.serializeAdditionalImageUrlsForJsonl = exports.serializeAdditionalImageUrlsForCsv = exports.encodeCommasInUrl = exports.resolveOpenAiImageFields = exports.selectOpenAiImageFields = void 0;
+const selectOpenAiImageFields = (images, checks) => {
+    const byUrl = new Map(checks.map(check => [check.url, check]));
+    const isValid = (image) => byUrl.get(image.url)?.ok === true;
+    const rejected = images.map(image => byUrl.get(image.url)).filter(check => check && !check.ok);
+    const hero = images.find(image => image.heroCandidate && isValid(image));
+    if (!hero) {
+        return { fields: null, rejected };
+    }
+    return {
+        fields: {
+            image_url: hero.url,
+            additional_image_urls: images.filter(image => image !== hero && isValid(image)).map(image => image.url),
+        },
+        rejected,
+    };
+};
+exports.selectOpenAiImageFields = selectOpenAiImageFields;
+const resolveOpenAiImageFields = async (images, validator) => {
+    const checks = await validator.validateAll(images.map(image => image.url));
+    return (0, exports.selectOpenAiImageFields)(images, checks);
+};
+exports.resolveOpenAiImageFields = resolveOpenAiImageFields;
+const encodeCommasInUrl = (url) => url.replace(/,/g, '%2C');
+exports.encodeCommasInUrl = encodeCommasInUrl;
+const serializeAdditionalImageUrlsForCsv = (urls) => (0, exports.quoteCsvCell)(urls.map(exports.encodeCommasInUrl).join(','));
+exports.serializeAdditionalImageUrlsForCsv = serializeAdditionalImageUrlsForCsv;
+const serializeAdditionalImageUrlsForJsonl = (urls) => urls.map(exports.encodeCommasInUrl);
+exports.serializeAdditionalImageUrlsForJsonl = serializeAdditionalImageUrlsForJsonl;
+const quoteCsvCell = (value) => `"${value.replace(/"/g, '""')}"`;
+exports.quoteCsvCell = quoteCsvCell;
+//# sourceMappingURL=openai-image-fields.js.map

package/dist/transformers/openai-image-fields.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-image-fields.js","sourceRoot":"","sources":["../../src/transformers/openai-image-fields.ts"],"names":[],"mappings":";;;AAsBO,MAAM,uBAAuB,GAAG,CAAC,MAAoB,EAAE,MAAuB,EAAwB,EAAE;IAC7G,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,IAAI,CAAC;IACzE,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAE/F,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACzE,IAAI,CAAC,IAAI,EAAE;QACT,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;KACnC;IAED,OAAO;QACL,MAAM,EAAE;YACN,SAAS,EAAE,IAAI,CAAC,GAAG;YACnB,qBAAqB,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC;SACxG;QACD,QAAQ;KACT,CAAC;AACJ,CAAC,CAAC;AAjBW,QAAA,uBAAuB,2BAiBlC;AAEK,MAAM,wBAAwB,GAAG,KAAK,EAC3C,MAAoB,EACpB,SAA4B,EACG,EAAE;IACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3E,OAAO,IAAA,+BAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACjD,CAAC,CAAC;AANW,QAAA,wBAAwB,4BAMnC;AAGK,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAAtE,QAAA,iBAAiB,qBAAqD;AAI5E,MAAM,kCAAkC,GAAG,CAAC,IAAc,EAAU,EAAE,CAC3E,IAAA,oBAAY,EAAC,IAAI,CAAC,GAAG,CAAC,yBAAiB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AADzC,QAAA,kCAAkC,sCACO;AAE/C,MAAM,oCAAoC,GAAG,CAAC,IAAc,EAAY,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,yBAAiB,CAAC,CAAC;AAAjG,QAAA,oCAAoC,wCAA6D;AAEvG,MAAM,YAAY,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;AAA3E,QAAA,YAAY,gBAA+D"}