@bytebrand/export-cars 1.20.0 → 1.21.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bytebrand/export-cars",
-  "version": "1.20.0",
+  "version": "1.21.0",
   "description": "",
   "author": "bytebrand",
   "license": "UNLICENSED",
@@ -28,6 +28,7 @@
     "@nestjs/cache-manager": "^2.1.0",
     "@nestjs/common": "^10.2.7",
     "@nestjs/core": "^10.2.7",
+    "axios": "^1.5.1",
     "bottleneck": "^2.19.5",
     "cache-manager": "^5.2.4",
     "csv-write-stream": "^2.0.0",
@@ -61,5 +62,5 @@
   "engines": {
     "node": ">=14.0.0"
   },
-  "gitHead": "e755a559caae3bd7088cfed459bb8f80e8adc012"
+  "gitHead": "3ef72636ca23cee78df87f16e66089e1a532b053"
 }

package/src/configs/config.ts

@@ -12,6 +12,7 @@ export class Config {
   baseImageUrl: string;
   marketingImageUrl: string;
   imageType: ImageType;
+  downloadImage: boolean;
   isNewCar: boolean;
   isCriteo: boolean;
   exportType: ExportType;

package/src/configs/evium.config.ts

@@ -10,6 +10,7 @@ export class EviumConfig extends Config {
     this.imageUrl = `https://images.${this.domain}/carimage`;
     this.mappingSpreadsheetTitle = 'mobile_yescar';
     this.imageType = ImageType.default;
+    this.downloadImage = true;
     this.outputFiles = [
       `${this.pathesConfig.localConvertedDirectory}/api_export_cars__${ExportType.EVIUM}.${FileType.CSV}`,
       `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.EVIUM}.${FileType.ZIP}`,

package/src/exporters/mobile.exporter.ts

@@ -1,5 +1,5 @@
 import { writeFile } from 'fs/promises';
-import { basename, normalize, parse } from 'path';
+import { normalize, parse } from 'path';
 
 import { Injectable } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
@@ -8,10 +8,10 @@ import { CronicleUtils } from '@bytebrand/cronicle-helper';
 import { FsChunkInterface } from '@bytebrand/chunkable';
 import { Zipper } from '@bytebrand/zipper';
 
-import { CarService, RulesService, LoaderService, ConfigService } from '../services';
+import { CarService, RulesService, LoaderService, ConfigService, ImagesService } from '../services';
 import { MobileTransformer } from '../transformers/mobile.transformer';
-import { convertToCSV } from '../utils/converter.utils';
 import { Exporter } from '../interfaces/exporter.interface';
+import { convertToCSV } from '../utils/converter.utils';
 import { FileType } from '../enums';
 
 @Injectable()
@@ -21,6 +21,7 @@ export class MobileExporter implements Exporter {
     private configService: ConfigService,
     private rulesService: RulesService,
     private loaderService: LoaderService,
+    private imageService: ImagesService,
     private moduleRef: ModuleRef,
   ) { }
 
@@ -30,6 +31,7 @@ export class MobileExporter implements Exporter {
     await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
 
     const mobileTransformer = await this.moduleRef.create(MobileTransformer);
+    const imagesPath = [];
 
     await this.loaderService.process(async (chunk: FsChunkInterface) => {
       chunk.data = chunk.data.filter(e => this.carService.filterCar(e, this.rulesService.rules));
@@ -38,9 +40,12 @@ export class MobileExporter implements Exporter {
         for (let i = 0; i < chunk.data.length; i += 1) {
           const data = chunk.data[i];
           const { car, images } = data;
-          // console.time(`Transform ${data.car._id}`);
           const transformedData = await mobileTransformer.transform(car, images);
-          // console.timeEnd(`Transform ${data.car._id}`);
+
+          if (this.configService.config.downloadImage) {
+            const urls = await this.imageService.getImages(car, images, this.configService.config.imageType, true);
+            imagesPath.push(...await this.imageService.downloadInParallel(car.metaData.vehicleId, urls, this.configService.config, 10));
+          }
 
           await writeFile(fileName, Buffer.from(`${convertToCSV(transformedData)}\n`), { flag: 'a' });
 
@@ -50,9 +55,9 @@ export class MobileExporter implements Exporter {
     });
 
     const fileZIP = normalize(`${parse(fileName).dir}/${parse(fileName).name}.${FileType.ZIP}`);
-    console.log(fileZIP);
+
     if (await CronicleUtils.exists(fileName)) {
-      await Zipper.zip({ fileList: [fileName], dest: fileZIP });
+      await Zipper.zip({ fileList: [fileName, ...imagesPath], dest: fileZIP });
     }
     console.log(this.carService.counter.getCountedCars());
   }

package/src/services/images.service.ts

@@ -1,4 +1,6 @@
-import { get } from 'lodash';
+import { createWriteStream } from 'fs';
+import { extname } from 'path';
+import axios from 'axios';
 
 import { Injectable } from '@nestjs/common';
 
@@ -7,9 +9,10 @@ import { ImboImageResponseInterface } from '@bytebrand/chunkable';
 import { DataSchemes } from '@bytebrand/car-schema';
 
 import { FB_MAX_IMAGE_COUNT_FOR_DEAFULT } from '../constants/fb.constants';
-import { isNotEmptyArray } from '../utils';
 import { ConfigService } from './config.service';
 import { UserService } from './user.service';
+import { isNotEmptyArray } from '../utils';
+import { Config } from '../configs/config';
 import { ImageType } from '../enums';
 
 @Injectable()
@@ -84,30 +87,21 @@ export class ImagesService {
 
   constructor(private readonly configService: ConfigService, private readonly userService: UserService) { }
 
-  async getImages(
-    car: DataSchemes.Car,
-    images: ImboImageResponseInterface[],
-    imageType: ImageType,
-    ignoreDefImages: boolean,
-  ): Promise<Array<string>> {
+  async getImages(car: DataSchemes.Car, images: ImboImageResponseInterface[], imageType: ImageType, ignoreDefImages: boolean): Promise<Array<string>> {
     if (imageType === ImageType.default) {
-      return this.getDefaultImages(car, images, ignoreDefImages);
+      return this.getDefaultImages(car._id, images, ignoreDefImages);
     } else {
-      return await this.getMarketingImages(car, images, ignoreDefImages);
+      return await this.getMarketingImages(car._id, images, ignoreDefImages);
     }
   }
 
-  getDefaultImages(
-    car: DataSchemes.Car,
-    images: ImboImageResponseInterface[],
-    ignoreDefImages: boolean,
-  ): Array<string> {
+  getDefaultImages(carId: string, images: ImboImageResponseInterface[], ignoreDefImages: boolean): Array<string> {
     if (!isNotEmptyArray(images)) {
       return this.DEFAULT_IMAGES;
     }
 
     const sortedImages = this.sortImages(images);
-    const transformedImages = this.generateImageLinks(sortedImages, get(car, '_id'));
+    const transformedImages = this.generateImageLinks(sortedImages, carId);
 
     if (!ignoreDefImages && transformedImages.length < 17) {
       transformedImages.push(...this.DEFAULT_IMAGES);
@@ -122,11 +116,7 @@ export class ImagesService {
     });
   }
 
-  async getMarketingImages(
-    car: DataSchemes.Car,
-    images: ImboImageResponseInterface[],
-    ignoreDefImages: boolean,
-  ): Promise<string[]> {
+  async getMarketingImages(carId: string, images: ImboImageResponseInterface[], ignoreDefImages: boolean): Promise<string[]> {
     const imageUrl = this.configService.config.marketingImageUrl;
 
     const transformedImages = this.transformImagesArrayIntoObject(this.sortImages(images));
@@ -140,7 +130,7 @@ export class ImagesService {
         const imageCode = marketingImage.marketingImageCodes[i];
         const imageId = transformedImages[imageCode];
         if (imageId) {
-          imageUrls.push(this.getImagePathWithExtension(imageUrl, get(car, '_id', ''), imageId, config));
+          imageUrls.push(this.getImagePathWithExtension(imageUrl, carId, imageId, config));
           break;
         }
       }
@@ -167,24 +157,48 @@ export class ImagesService {
     return `${imageUrl}/${[carId[0], carId[1], carId[2], carId].join('/')}`;
   }
 
-  protected getImagePathWithExtension(
-    imageUrl: string,
-    carId: string,
-    imageId: string,
-    config: MarketingImageConfigsInterface,
-  ): string {
+  protected getImagePathWithExtension(imageUrl: string, carId: string, imageId: string, config: MarketingImageConfigsInterface): string {
     const { channelCode, imageExtension } = config;
     return `${this.getChannelPath(imageUrl, carId)}/${channelCode.replace(/ /g, '-')}-${imageId}.${imageExtension}`;
   }
 
   sortImages(images: ImboImageResponseInterface[]): ImboImageResponseInterface[] {
     return images.sort((a, b) => {
-      const imageCodePredictedA = a.imageCodePredicted.includes('-') ? a.imageCodePredicted.split('-')[1] : a.imageCodePredicted;
-      const imageCodePredictedB = b.imageCodePredicted.includes('-') ? b.imageCodePredicted.split('-')[1] : b.imageCodePredicted;
+      const imageCodePredictedA = a.imageCodePredicted.includes('-') ? a.imageCodePredicted.split('-')[0] : a.imageCodePredicted;
+      const imageCodePredictedB = b.imageCodePredicted.includes('-') ? b.imageCodePredicted.split('-')[0] : b.imageCodePredicted;
       const orderA = this.IMAGES_SORT_ORDER[imageCodePredictedA] || 100;
       const orderB = this.IMAGES_SORT_ORDER[imageCodePredictedB] || 100;
-  
+
       return orderA - orderB;
     });
   }
+
+  async downloadInParallel(vehicleId: string, urls: string[], config: Config, concurrency = 1): Promise<string[]> {
+    for (let i = 0; i < urls.length; i += concurrency) {
+      const downloadPromises = urls.map(
+        (url: string, j: number) => this.download(
+          url,
+          `${config.pathesConfig.localConvertedDirectory}/${vehicleId}_${(i + 1).toLocaleString('en-US', { minimumIntegerDigits: 2 })}.jpeg`
+        )
+      );
+      return Promise.all(downloadPromises);
+    }
+    return [];
+  }
+
+  private async download(url: string, imagePath: string) {
+    return new Promise<string>(async resolve => {
+      try {
+        const response = await axios.get(url, { responseType: 'stream' });
+        response.data.pipe(
+          createWriteStream(imagePath)
+            .on('finish', () => { console.log(`Downloaded: ${url}`); resolve(imagePath); })
+            .on('error', (err) => { console.error(`Failed to download: ${url}`); resolve(null); })
+        );
+      } catch (error) {
+        console.error(`Failed to download: ${url}`);
+        resolve(null);
+      }
+    });
+  };
 }

package/src/transformers/mobile.transformer.ts

@@ -12,6 +12,7 @@ import { MappingRules } from '../interfaces/mapping-rules.interface';
 import { isNotEmptyArray } from '../utils';
 import { ConfigService } from '../services';
 import { ExportType } from '../enums';
+import axios from "axios";
 
 const vinGenerator = require('vin-generator');
 
@@ -24,7 +25,7 @@ export class MobileTransformer {
     private userService: UserService,
     private configService: ConfigService,
     private imagesService: ImagesService,
-  ) {}
+  ) { }
 
   async transform(carData: DataSchemes.Car, images: ImboImageResponseInterface[]): Promise<unknown> {
     this.currentCar = carData;
@@ -364,6 +365,9 @@ export class MobileTransformer {
       false,
     );
     let imagesAsALine;
+    // console.log(' --> transformedImages', transformedImages)
+    if (this.configService.config.campaign == ExportType.EVIUM) return null;
+
     if (isNotEmptyArray(transformedImages)) {
       imagesAsALine = transformedImages.map((el: string) => el).join(';');
     }