@bytebrand/export-cars 0.0.3 → 1.0.8

package/src/formats/fb-xml.builder.ts

@@ -0,0 +1,71 @@
+import { XMLBuilderBase } from '@bytebrand/cronicle-helper';
+import { isObject } from 'lodash';
+import { Description, DescriptionValue } from '../interfaces/description.interface';
+import { FbXmlItem } from '../interfaces/export-item.interface';
+
+export class FbXmlBuilder extends XMLBuilderBase {
+  async buildStart(url?: string): Promise<void> {
+    await this.write(
+      `<?xml version="1.0" encoding="UTF-8"?><listings><title>Auto de feed</title><link rel="self" href="${url}"/>`,
+    );
+  }
+
+  async addItem(car: FbXmlItem): Promise<void> {
+    const listing: string = Object.entries(car)
+      .map(([key, data]): string => {
+        if (key === 'description') {
+          return this.getDescriptionElement(key, data);
+        }
+        if (Array.isArray(data)) {
+          return this.getElementsArray(key, data);
+        }
+        if (isObject(data)) {
+          if (key === 'address') {
+            return this.getElementWithComponents(key, data, { format: 'simple' });
+          }
+          return this.getElementsTree(key, data, true);
+        }
+        return this.getElement(key, data, true);
+      })
+      .join('');
+
+    await this.write(`<listing>${listing}</listing>\n`);
+  }
+
+  private getDescriptionElement(rootTag: string, descriptionBlocks: Description[]): string {
+    const description: string = descriptionBlocks
+      .map(({ title, values }: Description) => {
+        return `<![CDATA[${title}\n${this.buildDescriptionValues(values)}.]]>`;
+      })
+      .join('\n');
+    return `<${rootTag}>${description}</${rootTag}>`;
+  }
+
+  private buildDescriptionValues(values: DescriptionValue[]): string {
+    return values
+      .map(({ title, value }: DescriptionValue) => {
+        return `${title}: ${value.join(', ')}`;
+      })
+      .join(';\n');
+  }
+
+  private getElementsArray(rootTag: string, data: object[]): string {
+    return data.reduce((elements, item) => `${elements}${this.getElementsTree(rootTag, item, true)}`, '');
+  }
+
+  private getElementWithComponents(rootTag: string, data: object, attr: object): string {
+    const components: string = Object.entries(data)
+      .map(([key, data]) => `<component name="${key}">${data}</component>`)
+      .join('');
+
+    const attrs = Object.entries(attr)
+      .map(([key, value]) => `${key}="${value}"`)
+      .join(' ');
+
+    return `<${rootTag} ${attrs}>${components}</${rootTag}>`;
+  }
+  async buildEnd(): Promise<void> {
+    await this.write('</listings>');
+    await this.end();
+  }
+}

package/src/formats/google-xml.builder.ts

@@ -0,0 +1,35 @@
+import { XMLBuilderBase } from '@bytebrand/cronicle-helper';
+import { GoogleXmlItem, FbXmlItem } from '../interfaces/export-item.interface';
+
+export class GoogleXmlBuilder extends XMLBuilderBase {
+  async buildStart(url?: string): Promise<void> {
+    await this.write(
+      `<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:g="http://base.google.com/ns/1.0"><channel><title>Auto de feed</title><link>${url}<link/>`,
+    );
+  }
+
+  async addItem(item: FbXmlItem | GoogleXmlItem): Promise<void> {
+    const listing: string = Object.entries(item)
+      .map(([key, data]): string => {
+        if (key === 'installment') {
+          return this.getInstallmentElement(key, data);
+        }
+        return this.getElement(`g:${key}`, data);
+      })
+      .join('');
+
+    await this.write(`<item>${listing}</item>\n`);
+  }
+
+  async buildEnd(): Promise<void> {
+    await this.write('</channel></rss>');
+    await this.end();
+  }
+
+  private getInstallmentElement(rootTag: string, data: { paybackPeriod: string; monthlyInstallment: string }): string {
+    return `<g:${rootTag}>
+      <g:months>${data.paybackPeriod}</g:months>
+      <g:amount>${data.monthlyInstallment} EUR</g:amount>
+    </g:${rootTag}>`;
+  }
+}

package/src/formats/meinstadt-json.builder.ts

@@ -0,0 +1,19 @@
+import { XMLBuilderBase } from '@bytebrand/cronicle-helper';
+import { MeinstadtJsonItem } from '../interfaces/export-item.interface';
+
+export class MeinstadtJsonBuilder extends XMLBuilderBase {
+  async buildStart(): Promise<void> {
+    await this.write('[');
+  }
+
+  async addItem(car: MeinstadtJsonItem, firstSym?: boolean): Promise<void> {
+    if (firstSym) {
+      await this.write(`${JSON.stringify(car)}`);
+    } else await this.write(`,${JSON.stringify(car)}`);
+  }
+
+  async buildEnd(): Promise<void> {
+    await this.write(']');
+    await this.end();
+  }
+}

package/src/formats/ooyyo-xml.builder.ts

@@ -0,0 +1,48 @@
+import { XMLBuilderBase } from '@bytebrand/cronicle-helper';
+import { OoyyoXmlItem } from '../interfaces/export-item.interface';
+
+export class OoyyoXmlBuilder extends XMLBuilderBase {
+  async buildStart(): Promise<void> {
+    await this.write(`<?xml version="1.0" encoding="UTF-8"?>
+    <ooyyo xsi:noNamespaceSchemaLocation="OoyyoFormat.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">`);
+  }
+
+  async addItem(car: OoyyoXmlItem): Promise<void> {
+    let item = '';
+
+    item += '\n<ad>\n';
+    item += this.getElement('url', car.url, true);
+    item += this.getElement('title', car.title, true);
+    item += this.getElement('content', car.content, true);
+    item += this.getElement('price', car.price, true, { unit: 'euro' });
+    item += this.getElement('make', car.make, true);
+    item += this.getElement('model', car.model, true);
+    item += this.getElement('trim', car.trim, true);
+    item += this.getElement('engine_size', car.engine_size, true, { unit: 'cc' });
+    item += this.getElement('body_type', car.body_type, true);
+    item += this.getElement('color', car.color, true);
+    item += this.getElement('year', car.year, true);
+    item += this.getElement('mileage', car.mileage, true, { unit: 'km' });
+    item += this.getElement('fuel_type', car.fuel_type, true);
+    item += this.getElement('transmission', car.transmission, true);
+    item += this.getElement('euronorm', car.euronorm, true);
+    item += this.getElement('power', car.power, true, { unit: 'kw' });
+    item += this.getElement('doors', car.doors, true);
+    item += this.getElement('seats', car.seats, true);
+    item += this.getElement('gears', car.gears, true);
+    item += this.getElementsTree('location', car.location, true);
+    item += this.getElementsList('pictures', 'picture', car.pictures, true);
+    item += this.getElement('vin', car.vin, true);
+    item += this.getElement('is_new', car.is_new, true);
+    item += this.getElement('date', car.date, true);
+    item += this.getElement('time', car.time, true);
+    item += '</ad>';
+
+    await this.write(item);
+  }
+
+  async buildEnd(): Promise<void> {
+    await this.write('\n</ooyyo>');
+    await this.end();
+  }
+}

package/src/input.questions.ts

@@ -0,0 +1,12 @@
+import { Question, QuestionSet } from 'nest-commander';
+
+@QuestionSet({ name: 'input-questions' })
+export class InputQuestions {
+  @Question({
+    message: 'Input:',
+    name: 'input',
+  })
+  parseInput(input: string) {
+    return JSON.parse(input);
+  }
+}

package/src/interfaces/description.interface.ts

@@ -0,0 +1,24 @@
+export interface Description {
+  title: string;
+  values: DescriptionValue[];
+}
+
+export interface DescriptionValue {
+  title: string;
+  value: string[];
+}
+
+export interface DescriptionInfo {
+  title: string;
+  values: DescriptionValueInfo[];
+}
+
+export interface DescriptionValueInfo {
+  title: string;
+  value: DescriptionSubValue | DescriptionSubValue[];
+}
+
+export interface DescriptionSubValue {
+  key: string;
+  data?: { [key: string]: string | number };
+}

package/src/interfaces/export-item.interface.ts

@@ -0,0 +1,147 @@
+export interface FbXmlItem {
+  url: string;
+  title: string;
+  content: string;
+  price: string;
+  make: string;
+  model: string;
+  engine_size: string;
+  trim: string;
+  body_type: string;
+  color: string;
+  year: string;
+  fuel_type: string;
+  transmission: string;
+  power: string;
+  doors: string;
+  seats: string;
+  gears: string;
+  pictures: string[];
+  vin: string;
+  is_new: string;
+  date: string;
+  time: string;
+  location: {
+    city: string;
+    postcode: string;
+    region: string;
+  };
+}
+
+export interface Auto24XmlItem {
+  autodeid: string;
+  userlink: string;
+  apilink: string;
+  brand: string;
+  modell: string;
+  cartype: string;
+  variant: string;
+  bodytype: string;
+  fueltype: string;
+  consumptionK: string;
+  co2: string;
+  efficiencyclass: string;
+  image: string;
+  registration: string;
+  images: string[];
+  price: string;
+  power: string;
+  zip: string;
+  city: string;
+  mileage: string;
+  companyname: string;
+  extras: string;
+  sellertype: string;
+}
+
+export interface OoyyoXmlItem extends FbXmlItem {
+  euronorm: string;
+  mileage: string;
+}
+
+export interface MeinstadtJsonItem {
+  id: string;
+  make: string;
+  model: string;
+  cpc: string;
+  listingUrl: string;
+  title: string;
+  accidentFree: boolean;
+  mileage: number;
+  initialRegistration: string;
+  offerCreationDate: string;
+  numberOfDoors: number;
+  numberOfSeats: number;
+  numberOfPreviousOwners: number;
+  price: number;
+  powerPS: string;
+  powerKS: string;
+  co2Emissions: number;
+  fuelConsumptionCombined: number;
+  equipment: string;
+  color: string;
+  bodyType: string;
+  emissionClass: string;
+  emissionSticker: string;
+  fuelType: string;
+  transmissionType: string;
+  sellerType: string;
+  usageState: string;
+  imagesCount: number;
+  images: string;
+  power: number;
+  'transmissionType/transmission': string;
+  'customerType/sellerType': string;
+}
+
+export interface AutouncleJsonItem {
+  back_link: string;
+  make: string;
+  model_name: string;
+  model_variant: string;
+  fuel: string;
+  body: string;
+  doors: string;
+  price: string;
+  year: string;
+  mileage: string;
+  leasing: string;
+  color: string;
+  engine_size: string;
+  power: string;
+  gear_type: string;
+  has_4wd: string;
+  seller_type: string;
+  seller_region: string;
+  seller_city: string;
+  seller_street_address: string;
+  seller_postcode: string;
+  seller_name: string;
+  seller_id: string;
+  description: string;
+  images: { url: string }[];
+  fuel_economy: string;
+  co2_emission: string;
+  co2_effiency_class: string;
+  euro_emission_class: string;
+  enviromental_sticker: string;
+  status: string;
+}
+
+export interface GoogleXmlItem {
+  id: string;
+  title: string;
+  description: string;
+  google_product_category: string;
+  product_type: string;
+  link: string;
+  image_link: string;
+  condition: string;
+  availability: string;
+  brand: any;
+  price: string;
+  sales_price: string;
+  energy_efficiency_class: string;
+  color: string;
+  installment: { paybackPeriod: any; monthlyInstallment: any };
+}

package/src/interfaces/exporter.interface.ts

@@ -0,0 +1,3 @@
+export interface Exporter {
+  export(): Promise<void>;
+}

package/src/interfaces/input.interface.ts

@@ -0,0 +1,6 @@
+import { Params } from './params.interface';
+
+export interface Input {
+  id: string;
+  params: Params;
+}

package/src/interfaces/mapping-rules.interface.ts

@@ -0,0 +1,11 @@
+export interface MappingRules {
+  importField: string;
+  importFieldType: string;
+  autodeField: string;
+  autodeFieldType: string;
+  importFieldSelector?: string;
+  autodeFieldSelector?: string;
+  selectors?: string[];
+
+  [key: string]: string | string[];
+}

package/src/interfaces/params.interface.ts

@@ -0,0 +1,19 @@
+import { Environment } from '../enums/environment.enum';
+import { ExportType } from '../enums/export-type.enum';
+
+export interface Params {
+  environment: Environment;
+  carDetailsUrl?: string;
+  exportType?: ExportType;
+  apikey?: string;
+  String?: string;
+  Number?: string;
+  Date?: string;
+  isNewCar?: boolean;
+  isCriteo?: boolean;
+  fileXML?: string;
+  dealers?: string;
+  cpc?: string;
+  ownerId?: string;
+  fileName?: string;
+}

package/src/interfaces/rule-parser.interface.ts

@@ -0,0 +1,3 @@
+export interface RuleParser<T> {
+  (e: string): T;
+}

package/src/main.ts

@@ -0,0 +1,28 @@
+import { CommandFactory } from 'nest-commander';
+import { AppModule } from './app.module';
+import { WinstonModule } from 'nest-winston';
+import { transports as winstonTransports, format as winstonFormat } from 'winston';
+
+async function bootstrap() {
+  try {
+    await CommandFactory.run(
+      AppModule,
+      WinstonModule.createLogger({
+        transports: [
+          new winstonTransports.Console({
+            format: winstonFormat.combine(
+              winstonFormat.splat(),
+              winstonFormat.simple(),
+              winstonFormat.colorize({ all: true }),
+            ),
+          }),
+        ],
+      }),
+    );
+    console.log({ complete: 1, code: 0 });
+  } catch (e) {
+    console.error({ description: e.message || e, complete: 1, code: 1 });
+  }
+}
+
+bootstrap();

package/src/rules/abstract.rule.ts

@@ -0,0 +1,37 @@
+import { DataSchemes } from '@bytebrand/car-schema';
+import { get, includes } from 'lodash';
+import { RuleParser } from '../interfaces/rule-parser.interface';
+
+export abstract class AbstractRule<T> {
+  private readonly _key: string;
+  private readonly _params: T[];
+
+  protected constructor(key: string, params: string, parser: RuleParser<T>) {
+    this._params = this.parseParams(params, parser);
+    this._key = key.trim();
+  }
+
+  getKey(): string {
+    return this._key;
+  }
+
+  getParams(): T[] {
+    return this._params;
+  }
+
+  applyRule(car: DataSchemes.Car): boolean {
+    return this.applyOR(get(car, this._key, false) as T, this._params);
+  }
+
+  protected parseParams(params: string, parser: RuleParser<T>): T[] {
+    return params.split(',').map((e: string) => parser(e.trim()));
+  }
+
+  protected applyOR(value: T, params: T[]): boolean {
+    return includes(params, value);
+  }
+
+  protected applyGTE(value: T, parameter: T): boolean {
+    return value >= parameter;
+  }
+}

package/src/rules/date.rule.ts

@@ -0,0 +1,18 @@
+import { DataSchemes } from '@bytebrand/car-schema';
+import { get } from 'lodash';
+import * as moment from 'moment';
+import { AbstractRule } from './abstract.rule';
+import { RuleParser } from '../interfaces/rule-parser.interface';
+
+export class DateRule extends AbstractRule<number> {
+  constructor(key: string, params: string, parser: RuleParser<number>) {
+    super(key, params, parser);
+  }
+
+  applyRule(car: DataSchemes.Car): boolean {
+    const value = get(car, this.getKey(), false);
+    if (!value) return false;
+    const parameter = this.getParams()[0];
+    return this.applyGTE(moment(value).unix(), parameter);
+  }
+}

package/src/rules/number.rule.ts

@@ -0,0 +1,15 @@
+import { DataSchemes } from '@bytebrand/car-schema';
+import { get } from 'lodash';
+import { RuleParser } from '../interfaces/rule-parser.interface';
+import { AbstractRule } from './abstract.rule';
+
+export class NumberRule extends AbstractRule<number> {
+  constructor(key: string, params: string, parser: RuleParser<number>) {
+    super(key, params, parser);
+  }
+
+  applyRule(car: DataSchemes.Car): boolean {
+    const parameter = this.getParams()[0];
+    return this.applyGTE(get(car, this.getKey(), 0), parameter);
+  }
+}

package/src/rules/string.rule.ts

@@ -0,0 +1,8 @@
+import { AbstractRule } from './abstract.rule';
+import { RuleParser } from '../interfaces/rule-parser.interface';
+
+export class StringRule extends AbstractRule<string> {
+  constructor(key: string, params: string, parser: RuleParser<string>) {
+    super(key, params, parser);
+  }
+}

package/src/services/car.service.ts

@@ -0,0 +1,57 @@
+import { CACHE_MANAGER, HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common';
+import { getMonthsDiff } from '../utils/date-time.utils';
+import { DataSchemes } from '@bytebrand/car-schema';
+import { Cache } from 'cache-manager';
+import Bottleneck from 'bottleneck';
+import { RequestService } from './request.service';
+import { DateRule } from '../rules/date.rule';
+import { NumberRule } from '../rules/number.rule';
+import { StringRule } from '../rules/string.rule';
+import { CarsCounter } from '@bytebrand/cronicle-helper';
+import { DealerProfileInterface } from '@bytebrand/autode-request';
+
+@Injectable()
+export class CarService {
+  constructor(
+    @Inject(CACHE_MANAGER) private cacheManager: Cache,
+    private bottleneck: Bottleneck,
+    private requestService: RequestService,
+  ) {}
+
+  private _counter: CarsCounter;
+
+  get counter() {
+    if (!this._counter) {
+      this._counter = new CarsCounter();
+    }
+    return this._counter;
+  }
+
+  filterCar({ car }: { car: DataSchemes.Car }, rules: Array<StringRule | NumberRule | DateRule>): boolean {
+    return rules.every(e => e.applyRule(car));
+  }
+
+  isCarNew({ car }: { car: DataSchemes.Car }): boolean {
+    if (car.mainData.mileage) return car.mainData.mileage <= 100;
+    else return getMonthsDiff(new Date((car.mainData.firstRegistration as number) * 1000), new Date()) <= 12;
+  }
+
+  async getOwner(ownerId: string): Promise<DealerProfileInterface> {
+    let owner = (await this.cacheManager.get(ownerId)) as DealerProfileInterface;
+    if (!owner) {
+      try {
+        owner = await this.bottleneck.schedule(() => this.requestService.upaRequest.getDealerProfile(ownerId));
+        console.log('TEST');
+      } catch (e) {
+        console.log(e);
+        if (e.statusCode !== HttpStatus.NOT_FOUND) {
+          throw e;
+        }
+        owner = null;
+      } finally {
+        await this.cacheManager.set(ownerId, owner);
+      }
+    }
+    return owner;
+  }
+}

package/src/services/config.service.ts

@@ -0,0 +1,15 @@
+import { Injectable } from '@nestjs/common';
+import { Config } from '../config/config';
+
+@Injectable()
+export class ConfigService {
+  private _config: Config;
+
+  set config(config: Config) {
+    this._config = config;
+  }
+
+  get config() {
+    return this._config;
+  }
+}

package/src/services/loader.service.ts

@@ -0,0 +1,30 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { AbstractChunkable, FsChunkInterface, Loaders } from '@bytebrand/chunkable';
+import { ConfigService } from './config.service';
+import { toNumber } from 'lodash';
+
+@Injectable()
+export class LoaderService {
+  private readonly logger = new Logger(LoaderService.name);
+
+  constructor(private configService: ConfigService) {}
+
+  private loader: AbstractChunkable;
+
+  process(fn: (chunk: FsChunkInterface) => Promise<void>): Promise<boolean> {
+    if (!this.loader) {
+      this.loader = new Loaders.FsCarsLoader(this.configService.config);
+    }
+    return new Promise((resolve, reject) => {
+      this.loader.onChunk(async (chunk: FsChunkInterface) => {
+        await fn(chunk);
+
+        const progress = (toNumber(chunk.id) + 1) / chunk.total;
+        this.logger.log(`progress - ${progress < 1 ? progress : 1}`);
+      });
+
+      this.loader.onError(reject);
+      return this.loader.start().then(resolve);
+    });
+  }
+}