@bytebrand/export-cars 1.0.9 → 1.0.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bytebrand/export-cars",
-  "version": "1.0.9",
+  "version": "1.0.10",
   "description": "",
   "author": "bytebrand",
   "license": "UNLICENSED",
@@ -64,5 +64,5 @@
   "engines": {
     "node": ">=14.0.0"
   },
-  "gitHead": "8f6de1762a4b742afd2e9020cf9a27d4fe0d57e3"
+  "gitHead": "76f936636e6e646ed55feb87f618d492d1cad69a"
 }

package/src/config/as24.config.ts

@@ -6,5 +6,6 @@ export class As24Config extends Config {
     super(params);
     this.pathesConfig.localConvertedDirectory = 'resources/as24-export';
     this.dealersList = params.dealers.split(',');
+    this.mappingSpreadsheetTitle = 'mobile';
   }
 }

package/src/config/autoboerse.config.ts

@@ -8,5 +8,6 @@ export class AutoboerseConfig extends Config {
     this.pathesConfig.localConvertedDirectory = 'resources/autoboerse-export';
     this.imageUrl = `https://images.${this.url}/carimage`;
     this.downloadImages = true;
+    this.mappingSpreadsheetTitle = 'mobile_yescar';
   }
 }

package/src/config/config.ts

@@ -32,6 +32,9 @@ export class Config {
   cpc: string;
   ownerId: string;
   downloadImages: boolean;
+  googleSheetsApiKey: string;
+  mappingSpreadsheetId: string;
+  mappingSpreadsheetTitle: string;
 
   protected environmentConfig = {
     [Environment.LOCAL]: {
@@ -68,6 +71,8 @@ export class Config {
     this.fileName = params.fileName || `${params.environment}_${params.exportType}`;
     this.criteoId = this.environmentConfig[params.environment].criteoId;
     this.url = this.environmentConfig[params.environment].url;
+    this.googleSheetsApiKey = params.googleSheetsApiKey;
+    this.mappingSpreadsheetId = '147qz9sTgs1wi13ua8YKa0IgfxAn1X_qH6LkGnAnzeyY';
     this.userActionsUrl = `https://www.${this.url}/service/user-actions`;
     this.imboUrl = `https://www.${this.url}/service/image-imbo`;
     this.imageUrl = `https://images.${this.url}`;

package/src/config/eauto.config.ts

@@ -8,5 +8,6 @@ export class EautoConfig extends Config {
     this.pathesConfig.localConvertedDirectory = 'resources/eauto-export';
     this.fileName = '416';
     this.imageUrl = `https://images.${this.url}/carimage`;
+    this.mappingSpreadsheetTitle = 'mobile_yescar';
   }
 }

package/src/config/marktde.config.ts

@@ -7,5 +7,6 @@ export class MarktdeConfig extends Config {
     this.campaign = 'marktde';
     this.pathesConfig.localConvertedDirectory = 'resources/marktde-export';
     this.imageUrl = `https://images.${this.url}/carimage`;
+    this.mappingSpreadsheetTitle = 'mobile_yescar';
   }
 }

package/src/config/yescar.config.ts

@@ -7,5 +7,6 @@ export class YescarConfig extends Config {
     this.campaign = 'YesAuto';
     this.pathesConfig.localConvertedDirectory = 'resources/yescar-export';
     this.imageUrl = `https://images.${this.url}/carimage`;
+    this.mappingSpreadsheetTitle = 'mobile_yescar';
   }
 }

package/src/export.schema.ts

@@ -1,3 +1,4 @@
+import { As24Config } from './config/as24.config';
 import { Auto24Config } from './config/auto24.config';
 import { AutoboerseConfig } from './config/autoboerse.config';
 import { AutouncleConfig } from './config/autouncle.config';
@@ -10,6 +11,7 @@ import { MeinstadtConfig } from './config/meinstadt.config';
 import { OoyyoConfig } from './config/ooyyo.config';
 import { YescarConfig } from './config/yescar.config';
 import { ExportType } from './enums/export-type.enum';
+import { As24Exporter } from './exporters/as24.exporter';
 import { Auto24Exporter } from './exporters/auto24.exporter';
 import { AutouncleExporter } from './exporters/autouncle.exporter';
 import { ByOwnerExporter } from './exporters/by-owner.exporter';
@@ -64,4 +66,8 @@ export const exportSchema = {
     config: AutouncleConfig,
     exporter: AutouncleExporter,
   },
+  [ExportType.AS24]: {
+    config: As24Config,
+    exporter: As24Exporter,
+  },
 };

package/src/exporters/as24.exporter.ts

@@ -7,9 +7,11 @@ import { Injectable } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
 import { As24Transformer } from '../transformers/as24.transformer';
 import { UserService } from '../services/user.service';
-import { get } from 'lodash';
+import { get, union } from 'lodash';
 import { convertToCSV } from '../utils/converter.utils';
 import { writeFile } from 'fs/promises';
+import { normalize } from 'path';
+import { Zipper } from '@bytebrand/zipper';
 
 @Injectable()
 export class As24Exporter implements Exporter {
@@ -31,6 +33,8 @@ export class As24Exporter implements Exporter {
       descriptionSchemas[dealerId] = await this.userService.getDefaultSummaryByDealerId(dealerId);
     }
 
+    const processedDealers = [];
+
     await this.loaderService.process(async (chunk: FsChunkInterface) => {
       if (chunk && chunk.data) {
         for (let i = 0; i < chunk.data.length; i += 1) {
@@ -43,28 +47,28 @@ export class As24Exporter implements Exporter {
           ) {
             const transformedData = await as24Transformer.transform(data.car, descriptionSchemas);
             const transformedDataCSV = `${convertToCSV(transformedData)}\n`;
+
             await writeFile(
               `${this.configService.config.pathesConfig.localConvertedDirectory}/${ownerId}.csv`,
               Buffer.from(transformedDataCSV),
               { flag: 'a' },
             );
+
+            union([ownerId], processedDealers);
           }
         }
         chunk.data = null;
       }
     });
 
-    // TODO: filter dealers and zip files
-
-    // const fileZIP = normalize(
-    //   `${this.configService.config.pathesConfig.localConvertedDirectory}/${this.configService.config.fileName}.zip`,
-    // );
-    // const fileCSV = normalize(
-    //   `${this.configService.config.pathesConfig.localConvertedDirectory}/${this.configService.config.fileName}.csv`,
-    // );
-
-    // if (await CronicleUtils.exists(fileCSV)) {
-    //   await Zipper.zip({ fileList: [fileCSV], dest: fileZIP });
-    // }
+    for (const dealer of processedDealers) {
+      const now = new Date();
+      const d = `${now.getFullYear()}${now.getMonth() + 1}${now.getDate()}`;
+      const fileCSV = normalize(`${this.configService.config.pathesConfig.localConvertedDirectory}/${dealer}.csv`);
+      const fileZIP = normalize(`${this.configService.config.pathesConfig.localConvertedDirectory}/${d}_${dealer}.zip`);
+      if (await CronicleUtils.exists(fileCSV)) {
+        await Zipper.zip({ fileList: [fileCSV], dest: fileZIP });
+      }
+    }
   }
 }

package/src/exporters/mobile.exporter.ts

@@ -16,6 +16,7 @@ import { lastValueFrom } from 'rxjs';
 import { createWriteStream } from 'fs';
 import { get } from 'lodash';
 import { RulesService } from '../services/rules.service';
+import { pipeline } from 'stream/promises';
 
 @Injectable()
 export class MobileExporter implements Exporter {
@@ -93,7 +94,7 @@ export class MobileExporter implements Exporter {
               responseType: 'stream',
             }),
           )
-            .then(response => {
+            .then(({ data }) => {
               const writer = createWriteStream(
                 join(
                   this.configService.config.pathesConfig.localConvertedDirectory,
@@ -101,9 +102,7 @@ export class MobileExporter implements Exporter {
                   `${vehicleId}_${index + 1}${extname(imageUrl) || '.jpg'}`,
                 ),
               );
-              response.data.pipe(writer);
-              writer.on('finish', resolve);
-              writer.on('error', reject);
+              pipeline(data, writer).then(resolve).catch(reject);
             })
             .catch(reject);
         });

package/src/interfaces/params.interface.ts

@@ -16,4 +16,5 @@ export interface Params {
   cpc?: string;
   ownerId?: string;
   fileName?: string;
+  googleSheetsApiKey?: string;
 }

package/src/main.ts

@@ -3,6 +3,12 @@ import { CommandFactory } from 'nest-commander';
 import { AppModule } from './app.module';
 
 async function bootstrap() {
+  // setInterval(() => {
+  //   Object.entries(process.memoryUsage()).forEach(item =>
+  //     console.log(`${item[0]}: ${(item[1] / 1024 / 1024).toFixed(4)} MB`),
+  //   );
+  //   console.log('\n');
+  // }, 10000);
   try {
     await CommandFactory.run(AppModule, console);
     LoggerService.complete();
@@ -10,5 +16,4 @@ async function bootstrap() {
     LoggerService.exitError(e.message || e);
   }
 }
-
 bootstrap();

package/src/services/mapping.service.ts

@@ -1,47 +1,45 @@
 import { Injectable } from '@nestjs/common';
-import { CronicleUtils } from '@bytebrand/cronicle-helper';
 import { MappingRules } from '../interfaces/mapping-rules.interface';
-import { HttpService } from '@nestjs/axios';
-import { lastValueFrom } from 'rxjs';
+import { GoogleSpreadsheet, GoogleSpreadsheetRow } from 'google-spreadsheet';
+import { ConfigService } from './config.service';
 
 @Injectable()
 export class MappingService {
-  constructor(private httpService: HttpService) {}
+  constructor(private configService: ConfigService) {}
   private rules: MappingRules[];
 
   async getRules(): Promise<MappingRules[]> {
     if (!this.rules) {
-      const { data } = await lastValueFrom(
-        this.httpService.get(
-          'https://docs.google.com/spreadsheets/d/e/2PACX-1vRh14TrYZDtk2efsnVsbfKypPWGdtGSSjgJRVlY2qrqHybaQZWP4HptLb3RQhiKfBsxQUdtltCRnwHU/pub?gid=1580940856&single=true&output=csv',
-        ),
-      );
-      const rows: MappingRules[] = await CronicleUtils.parseCSV(data);
+      const spreadsheet = new GoogleSpreadsheet(this.configService.config.mappingSpreadsheetId);
+      spreadsheet.useApiKey(this.configService.config.googleSheetsApiKey);
+      await spreadsheet.loadInfo();
+      const rows = await spreadsheet.sheetsByTitle[this.configService.config.mappingSpreadsheetTitle].getRows();
+      const mappingRules = this.rowsToRules(rows);
 
-      rows.map((row, _i, selfRows) => {
-        this.removeEmptyKeys(row);
-        if (row.importFieldType && row.importFieldType.includes('selector')) {
-          this.groupSelectors(row, selfRows);
+      mappingRules.map((rule, _i, selfRules) => {
+        this.removeEmptyKeys(rule);
+        if (rule.importFieldType && rule.importFieldType.includes('selector')) {
+          this.groupSelectors(rule, selfRules);
         }
       });
 
-      this.rules = this.filterResultedRows(rows);
+      this.rules = this.filterResultedRows(mappingRules);
     }
     return this.rules;
   }
 
-  // private rowsToRules(rows: GoogleSpreadsheetRow[]): MappingRulesInterface[] {
-  //   return rows.map(row =>
-  //     Object.assign(
-  //       {},
-  //       ...Object.entries(Object.getOwnPropertyDescriptors(row))
-  //         .filter(([_key, descriptor]) => typeof descriptor.get === 'function')
-  //         .map(([key, descriptor]) => ({
-  //           [key]: descriptor.get() || '',
-  //         })),
-  //     ),
-  //   );
-  // }
+  private rowsToRules(rows: GoogleSpreadsheetRow[]): MappingRules[] {
+    return rows.map(row =>
+      Object.assign(
+        {},
+        ...Object.entries(Object.getOwnPropertyDescriptors(row))
+          .filter(([_key, descriptor]) => typeof descriptor.get === 'function')
+          .map(([key, descriptor]) => ({
+            [key]: descriptor.get() || '',
+          })),
+      ),
+    );
+  }
 
   private filterResultedRows(rows: MappingRules[]): MappingRules[] {
     return rows.filter((value, idx, self) => {