@wikicasa-dev/node-common 1.0.0 → 1.1.0

package/dist/Crawler.d.ts

@@ -0,0 +1,29 @@
+import { ThenableWebDriver } from "selenium-webdriver";
+interface ICrawler {
+    drivers: Array<ThenableWebDriver>;
+    options: ICrawlerOptions;
+    first: boolean;
+    crawlFn(driver: ThenableWebDriver, url: string): Promise<any>;
+    execute(): Promise<Array<any>>;
+}
+interface ICrawlerOptions {
+    concurrent?: number;
+    throttling?: boolean | number;
+    crawlHub(driver: ThenableWebDriver): Promise<Array<string>>;
+    crawlListing(driver: ThenableWebDriver): Promise<any>;
+    nextPage(driver: ThenableWebDriver): Promise<any>;
+    crawlDetail?(driver: ThenableWebDriver): Promise<any>;
+    pageCallback?: (results: Array<Record<string, any>>, index: number) => void;
+    callback?: (results: Array<Record<string, any>>, index: number, first: boolean) => void;
+}
+export default class Crawler implements ICrawler {
+    drivers: Array<ThenableWebDriver>;
+    first: boolean;
+    options: ICrawlerOptions;
+    constructor(options: ICrawlerOptions);
+    execute(): Promise<any>;
+    createDrivers(): void;
+    destroyDrivers(): void;
+    crawlFn(driver: ThenableWebDriver, url: string): Promise<any>;
+}
+export {};

package/dist/Crawler.js

@@ -0,0 +1,85 @@
+import { pool } from "./common.js";
+import Webdriver from "selenium-webdriver";
+import Console from "./Console.js";
+import { Options } from "selenium-webdriver/chrome.js";
+export default class Crawler {
+    drivers = [];
+    first = true;
+    options = {
+        concurrent: 1,
+        throttling: false,
+        crawlListing: null,
+        nextPage: null,
+        crawlHub: null,
+        crawlDetail: null
+    };
+    constructor(options) {
+        this.options = { ...this.options, ...options };
+    }
+    execute() {
+        this.createDrivers();
+        try {
+            return new Promise(resolve => {
+                this.options.crawlHub(this.drivers[0]).then(urls => {
+                    const tasks = urls.map((url, index) => () => {
+                        const driver = this.drivers.pop();
+                        Console.log(`[${index + 1}/${Math.floor(urls.length)}] Valuating url ${urls[index]}`);
+                        return this.crawlFn(driver, url)
+                            .then(results => this.options.callback?.call(null, results, index, this.first))
+                            .finally(() => {
+                            this.first = false;
+                            this.drivers.push(driver);
+                        });
+                    });
+                    pool(tasks, this.options.concurrent).then(results => {
+                        this.destroyDrivers();
+                        resolve(results);
+                        Console.success("FINISHED");
+                    });
+                });
+            });
+        }
+        catch (e) {
+            console.error(e);
+            this.destroyDrivers();
+        }
+    }
+    createDrivers() {
+        for (let i = 0; i < this.options.concurrent; i++) {
+            this.drivers.push(new Webdriver.Builder()
+                .withCapabilities(Webdriver.Capabilities.chrome())
+                .setChromeOptions(new Options().headless())
+                .forBrowser("chrome")
+                .build());
+        }
+    }
+    destroyDrivers() {
+        this.drivers.forEach(driver => driver.close());
+    }
+    crawlFn(driver, url) {
+        const _self = this;
+        return new Promise(resolve => {
+            driver.get(url).then(() => {
+                _self.options.crawlListing(driver).then(listings => {
+                    if (_self.options.crawlDetail) {
+                        pool(listings.map((listing) => () => new Promise(resolve1 => {
+                            driver.get(listing.url).then(() => _self.options.crawlDetail(driver).then(detail => resolve1({ ...listing, ...detail })));
+                        })), 1)
+                            .then(details => {
+                            _self.options.pageCallback?.call(null, details);
+                            driver.get(url).then(() => _self.options.nextPage(driver)
+                                .then(url => _self.crawlFn(driver, url).then(acc => resolve([...details, ...acc])))
+                                .catch(() => resolve(details)));
+                        });
+                    }
+                    else {
+                        _self.options.pageCallback?.call(null, listings);
+                        _self.options.nextPage(driver)
+                            .then(url => _self.crawlFn(driver, url).then(acc => resolve([...listings, ...acc])))
+                            .catch(() => resolve(listings));
+                    }
+                });
+            });
+        });
+    }
+}

package/dist/Pool.d.ts

@@ -0,0 +1,13 @@
+export declare class Pool {
+    private name;
+    private readonly concurrency;
+    private queue;
+    private activeCount;
+    constructor(name: string, concurrency: number);
+    static run: (calls: Array<() => Promise<any>>, concurrency: number) => Promise<any[]>;
+    enqueue(...calls: Array<() => Promise<any>>): Promise<any[]>;
+    private schedule;
+    print(): string;
+    updatePrint(): void;
+    private _print;
+}

package/dist/Pool.js

@@ -0,0 +1,70 @@
+import Console, { COLORS } from "./Console.js";
+const BATCH_SIZE = 1000;
+export class Pool {
+    name;
+    concurrency;
+    queue = [];
+    activeCount;
+    constructor(name = "default", concurrency) {
+        this.name = name;
+        this.concurrency = concurrency;
+        this.queue = [];
+        this.activeCount = 0;
+        this.updatePrint();
+    }
+    static run = async (calls, concurrency) => {
+        const pool = new Pool("", concurrency);
+        const batchPromises = Array.from({ length: Math.ceil(calls.length / BATCH_SIZE) }, async (_, index) => {
+            const start = index * BATCH_SIZE;
+            const end = Math.min((index + 1) * BATCH_SIZE, calls.length);
+            const batch = calls.slice(start, end);
+            return await pool.enqueue(...batch);
+        });
+        const batchResults = await Promise.all(batchPromises);
+        return batchResults.flat();
+    };
+    enqueue(...calls) {
+        return Promise.all(calls.map(call => new Promise((resolve, reject) => {
+            this.queue.push({
+                call,
+                resolve,
+                reject,
+            });
+            this.schedule();
+        })));
+    }
+    schedule() {
+        while (this.activeCount < this.concurrency && this.queue.length > 0) {
+            const { call, resolve, reject } = this.queue.shift();
+            this.activeCount++;
+            call()
+                .then((result) => {
+                resolve(result);
+            })
+                .catch((error) => {
+                reject(error);
+            })
+                .finally(() => {
+                this.activeCount--;
+                this.updatePrint();
+                this.schedule();
+            });
+        }
+    }
+    print() {
+        return this._print;
+    }
+    updatePrint() {
+        const textStart = `${Console.color(this.name.padEnd(16), COLORS.YELLOW)} [POOL][`;
+        const textEnd = `][${this.activeCount + this.queue?.length}/${this.concurrency}]`;
+        let progress = "";
+        const max = process.stdout.columns - Math.max(textStart.length + textEnd.length, process.stdout.columns * 0.5);
+        const percent = this.activeCount / this.concurrency * max;
+        for (let j = 0; j < max; j++) {
+            const char = j < percent ? "=" : " ";
+            progress += Console.color(char, j < (max * 0.8) ? COLORS.GREEN : j < (max * 0.9) ? COLORS.YELLOW : COLORS.RED);
+        }
+        this._print = textStart + progress + textEnd;
+    }
+    _print;
+}

package/dist/common.d.ts

@@ -0,0 +1,14 @@
+export declare class CustomError extends Error {
+    critical: boolean;
+    constructor(message: string, critical?: boolean);
+}
+export declare function sequentially(promises: Array<() => Promise<unknown>>, step?: number, exit?: boolean): Promise<unknown>;
+export declare function pool(promises: Array<() => Promise<unknown>>, concurrent: number, stopIfError?: boolean): Promise<any>;
+export declare function promisesFirst<T>(fns: Array<() => Promise<T>>): Promise<T>;
+export declare function readCache(path: string): JSON;
+export declare function writeCache(path: string, data: any): void;
+export declare function readJson(path: string, strict?: boolean): Record<string, any>;
+export declare function readCSV(path: string, delimiter?: string, strict?: boolean): Record<string, any>;
+export declare function groupBy<T>(list: any, keyGetter: any): Map<string, T>;
+export declare function sleep(ms: any, feedback?: boolean): Promise<void>;
+export declare function unique<T>(array: Array<T>): Array<T> | undefined;

package/dist/common.js

@@ -0,0 +1,150 @@
+import fs from "fs";
+import { PromisePool } from "@supercharge/promise-pool";
+import Console from "./Console.js";
+import { brotliCompressSync, brotliDecompressSync } from "zlib";
+import { parse } from "csv-parse/sync";
+export class CustomError extends Error {
+    critical;
+    constructor(message, critical = false) {
+        super(message);
+        this.critical = critical;
+    }
+}
+export function sequentially(promises, step = 1, exit = false) {
+    return new Promise(resolve => {
+        let results = [];
+        const groupedPromises = promises.reduce((acc, fn, index) => {
+            const i = Math.floor(index / step);
+            if (acc[i])
+                acc[i].push(fn);
+            else
+                acc[i] = [fn];
+            return acc;
+        }, []);
+        groupedPromises.reduce((accumulatorPromise, promise) => {
+            return accumulatorPromise.then(() => {
+                return Promise.all(promise.map(f => f())).then(res => {
+                    results = [...results, ...res];
+                });
+            });
+        }, Promise.resolve([])).then(() => {
+            resolve(results);
+        });
+    });
+}
+export function pool(promises, concurrent, stopIfError = true) {
+    return new Promise((resolve, reject) => {
+        PromisePool
+            .for(promises)
+            .withConcurrency(concurrent)
+            .handleError(async (error, user, pool) => {
+            if (stopIfError || (error instanceof CustomError && error.critical)) {
+                reject(error);
+                pool.stop();
+            }
+        })
+            .useCorrespondingResults()
+            .process((fn) => fn())
+            .then((poolResult) => resolve(poolResult.results))
+            .catch((poolResult) => reject(poolResult.errors));
+    });
+}
+export async function promisesFirst(fns) {
+    for await (const f of fns) {
+        try {
+            const a = await f();
+            if (a) {
+                return a;
+            }
+        }
+        catch (e) {
+            //console.error(e);
+            //
+        }
+    }
+    throw "No promises resolve";
+}
+export function readCache(path) {
+    const brotliPath = `${path}.br`;
+    if (!fs.existsSync(brotliPath))
+        return null;
+    let data;
+    try {
+        data = JSON.parse(brotliDecompressSync(fs.readFileSync(brotliPath)).toString());
+    }
+    catch (e) {
+        data = null;
+    }
+    return data;
+}
+export function writeCache(path, data) {
+    const brotliPath = `${path}.br`;
+    try {
+        fs.writeFileSync(brotliPath, brotliCompressSync(JSON.stringify(data)), "binary");
+    }
+    catch (e) {
+        console.error("Can't pack data to file");
+    }
+}
+export function readJson(path, strict = false) {
+    if (!fs.existsSync(path) && strict)
+        throw new Error(`File ${path} not found`);
+    if (!fs.existsSync(path))
+        return null;
+    let json;
+    try {
+        json = JSON.parse(fs.readFileSync(path, "utf8"));
+    }
+    catch (e) {
+        json = null;
+    }
+    return json;
+}
+export function readCSV(path, delimiter = ",", strict = false) {
+    if (!fs.existsSync(path) && strict)
+        throw new Error(`File ${path} not found`);
+    if (!fs.existsSync(path))
+        return null;
+    let csv;
+    try {
+        csv = parse(fs.readFileSync(path, "utf8"), {
+            columns: true,
+            skip_empty_lines: true,
+            delimiter: delimiter
+        });
+    }
+    catch (e) {
+        csv = null;
+    }
+    return csv;
+}
+export function groupBy(list, keyGetter) {
+    const map = new Map();
+    list.forEach((item) => {
+        const key = keyGetter(item);
+        const collection = map.get(key);
+        if (!collection) {
+            map.set(key, [item]);
+        }
+        else {
+            collection.push(item);
+        }
+    });
+    return map;
+}
+export function sleep(ms, feedback = false) {
+    return new Promise((resolve) => {
+        const t = performance.now();
+        setTimeout(resolve, ms);
+        if (feedback)
+            setInterval(function () {
+                const left = (ms - (performance.now() - t)) / 1000;
+                Console.log(`Time left: ${Math.round(left)}s`, true);
+            }, 1000);
+    });
+}
+export function unique(array) {
+    if (!array || !array.length)
+        return;
+    return Array.from(new Set(array));
+}

package/dist/src/CallPool.js

@@ -157,28 +157,28 @@ export class CallPool {
             this.limiter(() => {
                 call()
                     .then((result) => {
-                    resolve(result);
-                })
+                        resolve(result);
+                    })
                     .catch((error) => {
-                    reject(error);
-                })
+                        reject(error);
+                    })
                     .finally(() => {
-                    this.activeCount--;
-                    this.clock--;
-                    if (this.clock < 0) {
-                        this.clock = this.CLOCK_NUMBER;
-                        if (this.mean > this.targetMean && this.mean > 10)
-                            this.concurrency = Math.max(this.concurrency - Math.floor(this.concurrency / 10), this.minConcurrency);
-                        if (this.mean < this.targetMean || this.mean < 2)
-                            this.concurrency = Math.min(this.concurrency + Math.ceil(this.concurrency / 20), this.maxConcurrency);
-                        this.targetMean = this.mean;
-                    }
-                    const t = (performance.now() - time) / 1000;
-                    this.mean = this.mean > 0 ? this.mean + (t - this.mean) / this.WEIGHT : t;
-                    this.updatePrint();
-                    //console.debug(this.name, t, this.mean, this.targetMean, `${this.activeCount}/${this.queue.length}`, this.concurrency, `${(60 / this.mean) * this.concurrency} avg/m`);
-                    this.schedule();
-                });
+                        this.activeCount--;
+                        this.clock--;
+                        if (this.clock < 0) {
+                            this.clock = this.CLOCK_NUMBER;
+                            if (this.mean > this.targetMean && this.mean > 10)
+                                this.concurrency = Math.max(this.concurrency - Math.floor(this.concurrency / 10), this.minConcurrency);
+                            if (this.mean < this.targetMean || this.mean < 2)
+                                this.concurrency = Math.min(this.concurrency + Math.ceil(this.concurrency / 20), this.maxConcurrency);
+                            this.targetMean = this.mean;
+                        }
+                        const t = (performance.now() - time) / 1000;
+                        this.mean = this.mean > 0 ? this.mean + (t - this.mean) / this.WEIGHT : t;
+                        this.updatePrint();
+                        //console.debug(this.name, t, this.mean, this.targetMean, `${this.activeCount}/${this.queue.length}`, this.concurrency, `${(60 / this.mean) * this.concurrency} avg/m`);
+                        this.schedule();
+                    });
             });
         }
     }

package/dist/src/Crawler.js

@@ -27,9 +27,9 @@ export default class Crawler {
                         return this.crawlFn(driver, url)
                             .then(results => this.options.callback?.call(null, results, index, this.first))
                             .finally(() => {
-                            this.first = false;
-                            this.drivers.push(driver);
-                        });
+                                this.first = false;
+                                this.drivers.push(driver);
+                            });
                     });
                     pool(tasks, this.options.concurrent).then(results => {
                         this.destroyDrivers();
@@ -66,11 +66,11 @@ export default class Crawler {
                             driver.get(listing.url).then(() => _self.options.crawlDetail(driver).then(detail => resolve1({ ...listing, ...detail })));
                         })), 1)
                             .then(details => {
-                            _self.options.pageCallback?.call(null, details);
-                            driver.get(url).then(() => _self.options.nextPage(driver)
-                                .then(url => _self.crawlFn(driver, url).then(acc => resolve([...details, ...acc])))
-                                .catch(() => resolve(details)));
-                        });
+                                _self.options.pageCallback?.call(null, details);
+                                driver.get(url).then(() => _self.options.nextPage(driver)
+                                    .then(url => _self.crawlFn(driver, url).then(acc => resolve([...details, ...acc])))
+                                    .catch(() => resolve(details)));
+                            });
                     }
                     else {
                         _self.options.pageCallback?.call(null, listings);

package/dist/src/Pool.js

@@ -39,16 +39,16 @@ export class Pool {
             this.activeCount++;
             call()
                 .then((result) => {
-                resolve(result);
-            })
+                    resolve(result);
+                })
                 .catch((error) => {
-                reject(error);
-            })
+                    reject(error);
+                })
                 .finally(() => {
-                this.activeCount--;
-                this.updatePrint();
-                this.schedule();
-            });
+                    this.activeCount--;
+                    this.updatePrint();
+                    this.schedule();
+                });
         }
     }
     print() {

package/dist/src/common.js

@@ -38,11 +38,11 @@ export function pool(promises, concurrent, stopIfError = true) {
             .for(promises)
             .withConcurrency(concurrent)
             .handleError(async (error, user, pool) => {
-            if (stopIfError || (error instanceof CustomError && error.critical)) {
-                reject(error);
-                pool.stop();
-            }
-        })
+                if (stopIfError || (error instanceof CustomError && error.critical)) {
+                    reject(error);
+                    pool.stop();
+                }
+            })
             .useCorrespondingResults()
             .process((fn) => fn())
             .then((poolResult) => resolve(poolResult.results))

package/dist/utils.d.ts

@@ -0,0 +1,30 @@
+import hash from "object-hash";
+export declare function arrayRotate<T>(arr: Array<T>, n: any): Array<T>;
+export declare function cleanASCII(str: string): string;
+export declare function cleanUpSpecialChars(str: string): string;
+export declare function normalizeString(str: string): string;
+export declare function trimAll(str: string): string;
+export declare function cleanUpSpecialChars2(str: string): string;
+export declare function capitalizeWords(str: string): string;
+export declare function capitalizeFirstLetter(str: string): string | undefined;
+export declare function readJson(path: string): any;
+export declare function orderBy<T extends object>(arr: Array<T>, prop: any): Array<T>;
+export declare function pan(number: number, digits: number): string;
+export declare function uniqWith<T>(array: Array<T>, compareFn: any): Array<T>;
+export declare function pop<T>(array: Array<T>, n: number): Array<T>;
+export declare function splitChunks<T>(array: Array<T>, chunkSize: number): Array<T>;
+export declare function newpopList<T>(list: Array<T>, fn?: (a: T) => boolean): Array<T>;
+export declare function popList<T extends hash.NotUndefined>(list1: Array<T>, list2: Array<T>, fn?: (a: T) => string): Array<T>;
+export declare function diffList<T extends hash.NotUndefined>(list1: Array<T>, list2: Array<T>, fn?: any): Array<T>;
+export declare function truncator(num: any, digits: any): number;
+export declare function parseNumber(num: string, thousand?: string): number;
+export declare function getRandom<T>(arr: Array<T>): T;
+export declare function shuffleArray<T>(array: T[]): T[];
+export declare function extractNumbers(str: string): string | undefined;
+export declare function parseFloatFromString(str: string): number | undefined;
+/**
+ * Convert MySQL POLYGON string into coordinates array
+ * @param {string} polygonString - The POLYGON string from MySQL
+ * @returns {Array<Array<number>>} - Array of coordinates in [longitude, latitude] format
+ */
+export declare function convertPolygonStringToArray(polygonString: string): Array<Array<number>>;