@bytebrand/export-cars 1.0.8 → 1.0.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bytebrand/export-cars",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "",
   "author": "bytebrand",
   "license": "UNLICENSED",
@@ -64,5 +64,5 @@
   "engines": {
     "node": ">=14.0.0"
   },
-  "gitHead": "483c1f6632515345e3bb0adcce0dfc77bebfcec9"
+  "gitHead": "76f936636e6e646ed55feb87f618d492d1cad69a"
 }

package/src/config/as24.config.ts

@@ -6,5 +6,6 @@ export class As24Config extends Config {
     super(params);
     this.pathesConfig.localConvertedDirectory = 'resources/as24-export';
     this.dealersList = params.dealers.split(',');
+    this.mappingSpreadsheetTitle = 'mobile';
   }
 }

package/src/config/autoboerse.config.ts

@@ -8,5 +8,6 @@ export class AutoboerseConfig extends Config {
     this.pathesConfig.localConvertedDirectory = 'resources/autoboerse-export';
     this.imageUrl = `https://images.${this.url}/carimage`;
     this.downloadImages = true;
+    this.mappingSpreadsheetTitle = 'mobile_yescar';
   }
 }

package/src/config/config.ts

@@ -32,6 +32,9 @@ export class Config {
   cpc: string;
   ownerId: string;
   downloadImages: boolean;
+  googleSheetsApiKey: string;
+  mappingSpreadsheetId: string;
+  mappingSpreadsheetTitle: string;
 
   protected environmentConfig = {
     [Environment.LOCAL]: {
@@ -68,6 +71,8 @@ export class Config {
     this.fileName = params.fileName || `${params.environment}_${params.exportType}`;
     this.criteoId = this.environmentConfig[params.environment].criteoId;
     this.url = this.environmentConfig[params.environment].url;
+    this.googleSheetsApiKey = params.googleSheetsApiKey;
+    this.mappingSpreadsheetId = '147qz9sTgs1wi13ua8YKa0IgfxAn1X_qH6LkGnAnzeyY';
     this.userActionsUrl = `https://www.${this.url}/service/user-actions`;
     this.imboUrl = `https://www.${this.url}/service/image-imbo`;
     this.imageUrl = `https://images.${this.url}`;

package/src/config/eauto.config.ts

@@ -8,5 +8,6 @@ export class EautoConfig extends Config {
     this.pathesConfig.localConvertedDirectory = 'resources/eauto-export';
     this.fileName = '416';
     this.imageUrl = `https://images.${this.url}/carimage`;
+    this.mappingSpreadsheetTitle = 'mobile_yescar';
   }
 }

package/src/config/marktde.config.ts

@@ -7,5 +7,6 @@ export class MarktdeConfig extends Config {
     this.campaign = 'marktde';
     this.pathesConfig.localConvertedDirectory = 'resources/marktde-export';
     this.imageUrl = `https://images.${this.url}/carimage`;
+    this.mappingSpreadsheetTitle = 'mobile_yescar';
   }
 }

package/src/config/yescar.config.ts

@@ -7,5 +7,6 @@ export class YescarConfig extends Config {
     this.campaign = 'YesAuto';
     this.pathesConfig.localConvertedDirectory = 'resources/yescar-export';
     this.imageUrl = `https://images.${this.url}/carimage`;
+    this.mappingSpreadsheetTitle = 'mobile_yescar';
   }
 }

package/src/export.command.ts

@@ -1,18 +1,16 @@
 import { Command, CommandRunner, InquirerService } from 'nest-commander';
 import { ConfigService } from './services/config.service';
 import { Input } from './interfaces/input.interface';
-import { Logger } from '@nestjs/common';
 import { exportSchema } from './export.schema';
 import { ModuleRef } from '@nestjs/core';
 import { Exporter } from './interfaces/exporter.interface';
+import { LoggerService } from '@bytebrand/cronicle-helper';
 
 @Command({
   name: 'export',
   options: { isDefault: true },
 })
 export class ExportRunner extends CommandRunner {
-  private readonly logger = new Logger();
-
   constructor(
     private readonly inquirer: InquirerService,
     private configService: ConfigService,
@@ -24,7 +22,7 @@ export class ExportRunner extends CommandRunner {
   async run(): Promise<void> {
     const input = (await this.inquirer.ask<{ input: Input }>('input-questions', undefined)).input;
 
-    this.logger.log(`Running job: ${input.id} - ${JSON.stringify(input)}\n`);
+    LoggerService.log(`Running job: ${input.id} - ${JSON.stringify(input)}\n`);
 
     this.configService.config = new exportSchema[input.params.exportType].config(input.params);
     const exporter = await this.moduleRef.create<Exporter>(exportSchema[input.params.exportType].exporter);

package/src/export.schema.ts

@@ -1,3 +1,4 @@
+import { As24Config } from './config/as24.config';
 import { Auto24Config } from './config/auto24.config';
 import { AutoboerseConfig } from './config/autoboerse.config';
 import { AutouncleConfig } from './config/autouncle.config';
@@ -10,6 +11,7 @@ import { MeinstadtConfig } from './config/meinstadt.config';
 import { OoyyoConfig } from './config/ooyyo.config';
 import { YescarConfig } from './config/yescar.config';
 import { ExportType } from './enums/export-type.enum';
+import { As24Exporter } from './exporters/as24.exporter';
 import { Auto24Exporter } from './exporters/auto24.exporter';
 import { AutouncleExporter } from './exporters/autouncle.exporter';
 import { ByOwnerExporter } from './exporters/by-owner.exporter';
@@ -64,4 +66,8 @@ export const exportSchema = {
     config: AutouncleConfig,
     exporter: AutouncleExporter,
   },
+  [ExportType.AS24]: {
+    config: As24Config,
+    exporter: As24Exporter,
+  },
 };

package/src/exporters/as24.exporter.ts

@@ -7,9 +7,11 @@ import { Injectable } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
 import { As24Transformer } from '../transformers/as24.transformer';
 import { UserService } from '../services/user.service';
-import { get } from 'lodash';
+import { get, union } from 'lodash';
 import { convertToCSV } from '../utils/converter.utils';
 import { writeFile } from 'fs/promises';
+import { normalize } from 'path';
+import { Zipper } from '@bytebrand/zipper';
 
 @Injectable()
 export class As24Exporter implements Exporter {
@@ -31,6 +33,8 @@ export class As24Exporter implements Exporter {
       descriptionSchemas[dealerId] = await this.userService.getDefaultSummaryByDealerId(dealerId);
     }
 
+    const processedDealers = [];
+
     await this.loaderService.process(async (chunk: FsChunkInterface) => {
       if (chunk && chunk.data) {
         for (let i = 0; i < chunk.data.length; i += 1) {
@@ -43,28 +47,28 @@ export class As24Exporter implements Exporter {
           ) {
             const transformedData = await as24Transformer.transform(data.car, descriptionSchemas);
             const transformedDataCSV = `${convertToCSV(transformedData)}\n`;
+
             await writeFile(
               `${this.configService.config.pathesConfig.localConvertedDirectory}/${ownerId}.csv`,
               Buffer.from(transformedDataCSV),
               { flag: 'a' },
             );
+
+            union([ownerId], processedDealers);
           }
         }
         chunk.data = null;
       }
     });
 
-    // TODO: filter dealers and zip files
-
-    // const fileZIP = normalize(
-    //   `${this.configService.config.pathesConfig.localConvertedDirectory}/${this.configService.config.fileName}.zip`,
-    // );
-    // const fileCSV = normalize(
-    //   `${this.configService.config.pathesConfig.localConvertedDirectory}/${this.configService.config.fileName}.csv`,
-    // );
-
-    // if (await CronicleUtils.exists(fileCSV)) {
-    //   await Zipper.zip({ fileList: [fileCSV], dest: fileZIP });
-    // }
+    for (const dealer of processedDealers) {
+      const now = new Date();
+      const d = `${now.getFullYear()}${now.getMonth() + 1}${now.getDate()}`;
+      const fileCSV = normalize(`${this.configService.config.pathesConfig.localConvertedDirectory}/${dealer}.csv`);
+      const fileZIP = normalize(`${this.configService.config.pathesConfig.localConvertedDirectory}/${d}_${dealer}.zip`);
+      if (await CronicleUtils.exists(fileCSV)) {
+        await Zipper.zip({ fileList: [fileCSV], dest: fileZIP });
+      }
+    }
   }
 }

package/src/exporters/mobile.exporter.ts

@@ -16,6 +16,7 @@ import { lastValueFrom } from 'rxjs';
 import { createWriteStream } from 'fs';
 import { get } from 'lodash';
 import { RulesService } from '../services/rules.service';
+import { pipeline } from 'stream/promises';
 
 @Injectable()
 export class MobileExporter implements Exporter {
@@ -93,7 +94,7 @@ export class MobileExporter implements Exporter {
               responseType: 'stream',
             }),
           )
-            .then(response => {
+            .then(({ data }) => {
               const writer = createWriteStream(
                 join(
                   this.configService.config.pathesConfig.localConvertedDirectory,
@@ -101,9 +102,7 @@ export class MobileExporter implements Exporter {
                   `${vehicleId}_${index + 1}${extname(imageUrl) || '.jpg'}`,
                 ),
               );
-              response.data.pipe(writer);
-              writer.on('finish', resolve);
-              writer.on('error', reject);
+              pipeline(data, writer).then(resolve).catch(reject);
             })
             .catch(reject);
         });

package/src/interfaces/params.interface.ts

@@ -16,4 +16,5 @@ export interface Params {
   cpc?: string;
   ownerId?: string;
   fileName?: string;
+  googleSheetsApiKey?: string;
 }

package/src/main.ts

@@ -1,28 +1,19 @@
+import { LoggerService } from '@bytebrand/cronicle-helper';
 import { CommandFactory } from 'nest-commander';
 import { AppModule } from './app.module';
-import { WinstonModule } from 'nest-winston';
-import { transports as winstonTransports, format as winstonFormat } from 'winston';
 
 async function bootstrap() {
+  // setInterval(() => {
+  //   Object.entries(process.memoryUsage()).forEach(item =>
+  //     console.log(`${item[0]}: ${(item[1] / 1024 / 1024).toFixed(4)} MB`),
+  //   );
+  //   console.log('\n');
+  // }, 10000);
   try {
-    await CommandFactory.run(
-      AppModule,
-      WinstonModule.createLogger({
-        transports: [
-          new winstonTransports.Console({
-            format: winstonFormat.combine(
-              winstonFormat.splat(),
-              winstonFormat.simple(),
-              winstonFormat.colorize({ all: true }),
-            ),
-          }),
-        ],
-      }),
-    );
-    console.log({ complete: 1, code: 0 });
+    await CommandFactory.run(AppModule, console);
+    LoggerService.complete();
   } catch (e) {
-    console.error({ description: e.message || e, complete: 1, code: 1 });
+    LoggerService.exitError(e.message || e);
   }
 }
-
 bootstrap();

package/src/services/loader.service.ts

@@ -1,12 +1,11 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import { AbstractChunkable, FsChunkInterface, Loaders } from '@bytebrand/chunkable';
 import { ConfigService } from './config.service';
 import { toNumber } from 'lodash';
+import { LoggerService } from '@bytebrand/cronicle-helper';
 
 @Injectable()
 export class LoaderService {
-  private readonly logger = new Logger(LoaderService.name);
-
   constructor(private configService: ConfigService) {}
 
   private loader: AbstractChunkable;
@@ -20,7 +19,7 @@ export class LoaderService {
         await fn(chunk);
 
         const progress = (toNumber(chunk.id) + 1) / chunk.total;
-        this.logger.log(`progress - ${progress < 1 ? progress : 1}`);
+        LoggerService.progress(progress < 1 ? progress : 1);
       });
 
       this.loader.onError(reject);

package/src/services/mapping.service.ts

@@ -1,47 +1,45 @@
 import { Injectable } from '@nestjs/common';
-import { CronicleUtils } from '@bytebrand/cronicle-helper';
 import { MappingRules } from '../interfaces/mapping-rules.interface';
-import { HttpService } from '@nestjs/axios';
-import { lastValueFrom } from 'rxjs';
+import { GoogleSpreadsheet, GoogleSpreadsheetRow } from 'google-spreadsheet';
+import { ConfigService } from './config.service';
 
 @Injectable()
 export class MappingService {
-  constructor(private httpService: HttpService) {}
+  constructor(private configService: ConfigService) {}
   private rules: MappingRules[];
 
   async getRules(): Promise<MappingRules[]> {
     if (!this.rules) {
-      const { data } = await lastValueFrom(
-        this.httpService.get(
-          'https://docs.google.com/spreadsheets/d/e/2PACX-1vRh14TrYZDtk2efsnVsbfKypPWGdtGSSjgJRVlY2qrqHybaQZWP4HptLb3RQhiKfBsxQUdtltCRnwHU/pub?gid=1580940856&single=true&output=csv',
-        ),
-      );
-      const rows: MappingRules[] = await CronicleUtils.parseCSV(data);
+      const spreadsheet = new GoogleSpreadsheet(this.configService.config.mappingSpreadsheetId);
+      spreadsheet.useApiKey(this.configService.config.googleSheetsApiKey);
+      await spreadsheet.loadInfo();
+      const rows = await spreadsheet.sheetsByTitle[this.configService.config.mappingSpreadsheetTitle].getRows();
+      const mappingRules = this.rowsToRules(rows);
 
-      rows.map((row, _i, selfRows) => {
-        this.removeEmptyKeys(row);
-        if (row.importFieldType && row.importFieldType.includes('selector')) {
-          this.groupSelectors(row, selfRows);
+      mappingRules.map((rule, _i, selfRules) => {
+        this.removeEmptyKeys(rule);
+        if (rule.importFieldType && rule.importFieldType.includes('selector')) {
+          this.groupSelectors(rule, selfRules);
         }
       });
 
-      this.rules = this.filterResultedRows(rows);
+      this.rules = this.filterResultedRows(mappingRules);
     }
     return this.rules;
   }
 
-  // private rowsToRules(rows: GoogleSpreadsheetRow[]): MappingRulesInterface[] {
-  //   return rows.map(row =>
-  //     Object.assign(
-  //       {},
-  //       ...Object.entries(Object.getOwnPropertyDescriptors(row))
-  //         .filter(([_key, descriptor]) => typeof descriptor.get === 'function')
-  //         .map(([key, descriptor]) => ({
-  //           [key]: descriptor.get() || '',
-  //         })),
-  //     ),
-  //   );
-  // }
+  private rowsToRules(rows: GoogleSpreadsheetRow[]): MappingRules[] {
+    return rows.map(row =>
+      Object.assign(
+        {},
+        ...Object.entries(Object.getOwnPropertyDescriptors(row))
+          .filter(([_key, descriptor]) => typeof descriptor.get === 'function')
+          .map(([key, descriptor]) => ({
+            [key]: descriptor.get() || '',
+          })),
+      ),
+    );
+  }
 
   private filterResultedRows(rows: MappingRules[]): MappingRules[] {
     return rows.filter((value, idx, self) => {

package/src/services/rules.service.ts

@@ -1,4 +1,5 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { LoggerService } from '@bytebrand/cronicle-helper';
+import { Injectable } from '@nestjs/common';
 import { concat, toSafeInteger, toString } from 'lodash';
 import { DateRule } from '../rules/date.rule';
 import { NumberRule } from '../rules/number.rule';
@@ -7,7 +8,6 @@ import { ConfigService } from './config.service';
 
 @Injectable()
 export class RulesService {
-  private readonly logger = new Logger(RulesService.name);
   constructor(private configService: ConfigService) {}
 
   private _rules: Array<StringRule | NumberRule | DateRule>;
@@ -42,7 +42,7 @@ export class RulesService {
       });
       return rulesArr;
     } catch (error) {
-      this.logger.error(error);
+      LoggerService.error(error);
       return rulesArr;
     }
   }

package/src/transformers/as24.transformer.ts

@@ -3,11 +3,11 @@ import { DataSchemes } from '@bytebrand/car-schema';
 import { CarDescriptionFactory, DescriptionTypeEnum } from '@bytebrand/car-description';
 import { MappingService } from '../services/mapping.service';
 import { MappingRules } from '../interfaces/mapping-rules.interface';
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
+import { LoggerService } from '@bytebrand/cronicle-helper';
 
 @Injectable()
 export class As24Transformer {
-  private readonly logger = new Logger(As24Transformer.name);
   private currentCar: DataSchemes.Car;
   private descriptionSchemas: unknown;
 
@@ -97,7 +97,7 @@ export class As24Transformer {
         return this.selectorToCategoryOrClass_selector(fieldValue, rule);
       }
       default:
-        this.logger.log(`missing transform: ${transformProcedureName}`);
+        LoggerService.log(`missing transform: ${transformProcedureName}`);
         break;
     }
   }

package/src/transformers/mobile.transformer.ts

@@ -1,4 +1,4 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import { DataSchemes } from '@bytebrand/car-schema';
 import { get, isEmpty, isNil, set, toString, upperFirst } from 'lodash';
 import { MappingService } from '../services/mapping.service';
@@ -7,12 +7,12 @@ import { UserService } from '../services/user.service';
 import { CarDescriptionFactory, DescriptionTypeEnum } from '@bytebrand/car-description';
 import { ConfigService } from '../services/config.service';
 import { isNotEmptyArray } from '../utils/validator.utils';
+import { LoggerService } from '@bytebrand/cronicle-helper';
 
 const vinGenerator = require('vin-generator');
 
 @Injectable()
 export class MobileTransformer {
-  private readonly logger = new Logger(MobileTransformer.name);
   private currentCar: DataSchemes.Car;
 
   constructor(
@@ -107,7 +107,7 @@ export class MobileTransformer {
         return this.selectorToCategoryOrClass_selector(fieldValue);
       }
       default:
-        this.logger.log(`missing transform: ${transformProcedureName}`);
+        LoggerService.log(`missing transform: ${transformProcedureName}`);
         break;
     }
   }
@@ -341,7 +341,7 @@ export class MobileTransformer {
     try {
       res = this.getImagesField(car._id, get(car, 'metaData.mainImageId', null), images);
     } catch (e) {
-      this.logger.error(e);
+      LoggerService.error(e);
       throw e;
     }
     let imagesAsALine;
@@ -438,7 +438,7 @@ export class MobileTransformer {
     if (found) {
       return `Car.${found}`;
     } else {
-      this.logger.log(`Missing mapping for: ${fieldValue}`);
+      LoggerService.log(`Missing mapping for: ${fieldValue}`);
       return 'Car.OtherCar';
     }
   }