@wikicasa-dev/node-common 1.1.0 → 1.2.1

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export * from "./src/Cache";
+export { CacheES } from "./src/CacheES";
+export * from "./src/CallPool";
+export * from "./src/common";
+export * from "./src/Console";
+export * from "./src/Crawler";
+export * from "./src/Pool";
+export * from "./src/utils";

package/dist/index.js

@@ -0,0 +1,9 @@
+// src/index.ts
+export * from "./src/Cache";
+export { CacheES } from "./src/CacheES";
+export * from "./src/CallPool";
+export * from "./src/common";
+export * from "./src/Console";
+export * from "./src/Crawler";
+export * from "./src/Pool";
+export * from "./src/utils";

package/dist/src/Cache.js

@@ -31,8 +31,13 @@ export class Cache {
                 case MODE.STRING_BROTLI:
                 default: {
                     const value = await this.client.get(key);
-                    const dataRaw = brotliDecompressSync(Buffer.from(value, "binary")).toString();
-                    data = JSON.parse(dataRaw);
+                    if (value !== null) {
+                        const dataRaw = brotliDecompressSync(Buffer.from(value, "binary")).toString();
+                        data = JSON.parse(dataRaw);
+                    }
+                    else {
+                        data = null;
+                    }
                     break;
                 }
             }
@@ -86,6 +91,7 @@ export class Cache {
         catch (e) {
             console.error("Can't check data exists");
         }
+        return;
     }
     async select(db) {
         if (!this.init)
@@ -106,4 +112,3 @@ export class Cache {
         }
     }
 }
-//# sourceMappingURL=Cache.js.map

package/dist/src/CacheES.js

@@ -99,13 +99,14 @@ export class CacheES {
     async exists(id) {
         try {
             return !!(await this.client.exists({
-                index,
+                index: "redis_data",
                 id
             }));
         }
         catch (e) {
             console.error(`Can't check data exists: ${e}`);
         }
+        return;
     }
     async quit() {
         try {
@@ -116,4 +117,3 @@ export class CacheES {
         }
     }
 }
-//# sourceMappingURL=CacheES.js.map

package/dist/src/CallPool.js

@@ -31,7 +31,7 @@ export class CallPool {
     concurrency;
     minConcurrency = 1;
     maxConcurrency;
-    _print;
+    _print = "";
     limiter;
     proxy;
     CLOCK_NUMBER = 1;
@@ -41,6 +41,7 @@ export class CallPool {
     clock = this.CLOCK_NUMBER;
     queue = [];
     activeCount;
+    // @ts-ignore
     agent;
     constructor(name = "default", proxy = PROXY.STATIC, minConcurrency = 1, maxConcurrency, limitCall = 1000, limitInterval = 1000) {
         this.name = name;
@@ -136,7 +137,7 @@ export class CallPool {
             { name: "Edge", version: 116 }
         ];
         return RandomUserAgent.getRandom(function (ua) {
-            return browsers.some(b => b.name === ua.browserName && b.version <= ua.browserVersion);
+            return browsers.some(b => b.name === ua.browserName && b.version <= parseInt(ua.browserVersion));
         });
     }
     enqueue(call) {
@@ -151,34 +152,37 @@ export class CallPool {
     }
     schedule() {
         while (this.activeCount < this.concurrency && this.queue.length > 0) {
-            const { call, resolve, reject } = this.queue.shift();
+            const item = this.queue.shift();
+            if (!item)
+                return;
+            const { call, resolve, reject } = item;
             this.activeCount++;
             const time = performance.now();
             this.limiter(() => {
                 call()
                     .then((result) => {
-                        resolve(result);
-                    })
+                    resolve(result);
+                })
                     .catch((error) => {
-                        reject(error);
-                    })
+                    reject(error);
+                })
                     .finally(() => {
-                        this.activeCount--;
-                        this.clock--;
-                        if (this.clock < 0) {
-                            this.clock = this.CLOCK_NUMBER;
-                            if (this.mean > this.targetMean && this.mean > 10)
-                                this.concurrency = Math.max(this.concurrency - Math.floor(this.concurrency / 10), this.minConcurrency);
-                            if (this.mean < this.targetMean || this.mean < 2)
-                                this.concurrency = Math.min(this.concurrency + Math.ceil(this.concurrency / 20), this.maxConcurrency);
-                            this.targetMean = this.mean;
-                        }
-                        const t = (performance.now() - time) / 1000;
-                        this.mean = this.mean > 0 ? this.mean + (t - this.mean) / this.WEIGHT : t;
-                        this.updatePrint();
-                        //console.debug(this.name, t, this.mean, this.targetMean, `${this.activeCount}/${this.queue.length}`, this.concurrency, `${(60 / this.mean) * this.concurrency} avg/m`);
-                        this.schedule();
-                    });
+                    this.activeCount--;
+                    this.clock--;
+                    if (this.clock < 0) {
+                        this.clock = this.CLOCK_NUMBER;
+                        if (this.mean > this.targetMean && this.mean > 10)
+                            this.concurrency = Math.max(this.concurrency - Math.floor(this.concurrency / 10), this.minConcurrency);
+                        if (this.mean < this.targetMean || this.mean < 2)
+                            this.concurrency = Math.min(this.concurrency + Math.ceil(this.concurrency / 20), this.maxConcurrency);
+                        this.targetMean = this.mean;
+                    }
+                    const t = (performance.now() - time) / 1000;
+                    this.mean = this.mean > 0 ? this.mean + (t - this.mean) / this.WEIGHT : t;
+                    this.updatePrint();
+                    //console.debug(this.name, t, this.mean, this.targetMean, `${this.activeCount}/${this.queue.length}`, this.concurrency, `${(60 / this.mean) * this.concurrency} avg/m`);
+                    this.schedule();
+                });
             });
         }
     }
@@ -220,4 +224,3 @@ export class CallPool {
         }
     }
 }
-//# sourceMappingURL=CallPool.js.map

package/dist/src/Console.js

@@ -111,4 +111,3 @@ export default class Console {
         this.first = false;
     }
 }
-//# sourceMappingURL=Console.js.map

package/dist/src/Crawler.js

@@ -27,9 +27,9 @@ export default class Crawler {
                         return this.crawlFn(driver, url)
                             .then(results => this.options.callback?.call(null, results, index, this.first))
                             .finally(() => {
-                                this.first = false;
-                                this.drivers.push(driver);
-                            });
+                            this.first = false;
+                            this.drivers.push(driver);
+                        });
                     });
                     pool(tasks, this.options.concurrent).then(results => {
                         this.destroyDrivers();
@@ -66,11 +66,11 @@ export default class Crawler {
                             driver.get(listing.url).then(() => _self.options.crawlDetail(driver).then(detail => resolve1({ ...listing, ...detail })));
                         })), 1)
                             .then(details => {
-                                _self.options.pageCallback?.call(null, details);
-                                driver.get(url).then(() => _self.options.nextPage(driver)
-                                    .then(url => _self.crawlFn(driver, url).then(acc => resolve([...details, ...acc])))
-                                    .catch(() => resolve(details)));
-                            });
+                            _self.options.pageCallback?.call(null, details);
+                            driver.get(url).then(() => _self.options.nextPage(driver)
+                                .then(url => _self.crawlFn(driver, url).then(acc => resolve([...details, ...acc])))
+                                .catch(() => resolve(details)));
+                        });
                     }
                     else {
                         _self.options.pageCallback?.call(null, listings);
@@ -83,4 +83,3 @@ export default class Crawler {
         });
     }
 }
-//# sourceMappingURL=Crawler.js.map

package/dist/src/Pool.js

@@ -39,16 +39,16 @@ export class Pool {
             this.activeCount++;
             call()
                 .then((result) => {
-                    resolve(result);
-                })
+                resolve(result);
+            })
                 .catch((error) => {
-                    reject(error);
-                })
+                reject(error);
+            })
                 .finally(() => {
-                    this.activeCount--;
-                    this.updatePrint();
-                    this.schedule();
-                });
+                this.activeCount--;
+                this.updatePrint();
+                this.schedule();
+            });
         }
     }
     print() {
@@ -68,4 +68,3 @@ export class Pool {
     }
     _print;
 }
-//# sourceMappingURL=Pool.js.map

package/dist/src/common.js

@@ -38,11 +38,11 @@ export function pool(promises, concurrent, stopIfError = true) {
             .for(promises)
             .withConcurrency(concurrent)
             .handleError(async (error, user, pool) => {
-                if (stopIfError || (error instanceof CustomError && error.critical)) {
-                    reject(error);
-                    pool.stop();
-                }
-            })
+            if (stopIfError || (error instanceof CustomError && error.critical)) {
+                reject(error);
+                pool.stop();
+            }
+        })
             .useCorrespondingResults()
             .process((fn) => fn())
             .then((poolResult) => resolve(poolResult.results))
@@ -148,4 +148,3 @@ export function unique(array) {
         return;
     return Array.from(new Set(array));
 }
-//# sourceMappingURL=common.js.map

package/dist/{utils.d.ts → src/utils.d.ts}

@@ -7,7 +7,6 @@ export declare function trimAll(str: string): string;
 export declare function cleanUpSpecialChars2(str: string): string;
 export declare function capitalizeWords(str: string): string;
 export declare function capitalizeFirstLetter(str: string): string | undefined;
-export declare function readJson(path: string): any;
 export declare function orderBy<T extends object>(arr: Array<T>, prop: any): Array<T>;
 export declare function pan(number: number, digits: number): string;
 export declare function uniqWith<T>(array: Array<T>, compareFn: any): Array<T>;

package/dist/src/utils.js

@@ -1,4 +1,3 @@
-import fs from "fs";
 import hash from "object-hash";
 export function arrayRotate(arr, n) {
     Array.from(Array(n)).forEach((x, i) => {
@@ -40,18 +39,6 @@ export function capitalizeFirstLetter(str) {
         return;
     return str?.charAt(0).toUpperCase() + str?.slice(1);
 }
-export function readJson(path) {
-    if (!fs.existsSync(path))
-        return null;
-    let json;
-    try {
-        json = JSON.parse(fs.readFileSync(path, "utf8"));
-    }
-    catch (e) {
-        json = null;
-    }
-    return json;
-}
 export function orderBy(arr, prop) {
     arr.sort((a, b) => {
         if (a[prop] > b[prop]) {
@@ -207,4 +194,3 @@ export function convertPolygonStringToArray(polygonString) {
         return [lon, lat]; // Return [longitude, latitude]
     });
 }
-//# sourceMappingURL=utils.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wikicasa-dev/node-common",
-  "version": "1.1.0",
+  "version": "1.2.1",
   "description": "Wikicasa node common",
   "type": "module",
   "files": [

package/dist/Cache.js

@@ -1,114 +0,0 @@
-import redis from "redis";
-import { brotliCompressSync, brotliDecompressSync } from "zlib";
-import { Buffer } from "node:buffer";
-export var MODE;
-(function (MODE) {
-    MODE[MODE["STRING"] = 0] = "STRING";
-    MODE[MODE["JSON"] = 1] = "JSON";
-    MODE[MODE["HASH"] = 2] = "HASH";
-    MODE[MODE["STRING_BROTLI"] = 3] = "STRING_BROTLI";
-})(MODE || (MODE = {}));
-export class Cache {
-    init;
-    client;
-    constructor(options) {
-        this.client = redis.createClient(options);
-    }
-    async start() {
-        this.init = this.client.connect();
-    }
-    async get(key, mode = MODE.STRING_BROTLI) {
-        if (!this.init)
-            await this.start();
-        //if (!await this.client.exists(key))
-        //    return null;
-        let data;
-        try {
-            switch (mode) {
-                case MODE.STRING:
-                    data = await this.client.get(key);
-                    break;
-                case MODE.STRING_BROTLI:
-                default: {
-                    const value = await this.client.get(key);
-                    if (value !== null) {
-                        const dataRaw = brotliDecompressSync(Buffer.from(value, "binary")).toString();
-                        data = JSON.parse(dataRaw);
-                    }
-                    else {
-                        data = null;
-                    }
-                    break;
-                }
-            }
-        }
-        catch (e) {
-            data = null;
-        }
-        return data;
-    }
-    async set(key, value, mode = MODE.STRING_BROTLI) {
-        if (!this.init)
-            await this.start();
-        try {
-            switch (mode) {
-                case MODE.STRING: {
-                    await this.client.set(key, value.toString("binary"));
-                    break;
-                }
-                case MODE.JSON:
-                    // @ts-ignore
-                    await this.client.jSet(key, ".", value);
-                    break;
-                case MODE.STRING_BROTLI:
-                default: {
-                    const data = brotliCompressSync(JSON.stringify(value));
-                    await this.client.set(key, data.toString("binary"));
-                    break;
-                }
-            }
-        }
-        catch (e) {
-            console.error("Can't cache data");
-        }
-    }
-    async delete(key) {
-        if (!this.init)
-            await this.start();
-        try {
-            await this.client.del(key);
-        }
-        catch (e) {
-            console.error("Can't delete data");
-        }
-    }
-    async exists(key) {
-        if (!this.init)
-            await this.start();
-        try {
-            return !!(await this.client.exists(key));
-        }
-        catch (e) {
-            console.error("Can't check data exists");
-        }
-        return;
-    }
-    async select(db) {
-        if (!this.init)
-            await this.start();
-        try {
-            await this.client.select(db);
-        }
-        catch (e) {
-            console.error("Can't select db");
-        }
-    }
-    async quit() {
-        try {
-            await this.client.quit();
-        }
-        catch (e) {
-            console.error("Can't quit");
-        }
-    }
-}

package/dist/CacheES.js

@@ -1,119 +0,0 @@
-import { Client } from "@elastic/elasticsearch";
-import { brotliCompressSync, brotliDecompressSync } from "zlib";
-import { Buffer } from "node:buffer";
-const index = "redis_data";
-export var MODE;
-(function (MODE) {
-    MODE[MODE["STRING"] = 0] = "STRING";
-    MODE[MODE["JSON"] = 1] = "JSON";
-    MODE[MODE["HASH"] = 2] = "HASH";
-    MODE[MODE["STRING_BROTLI"] = 3] = "STRING_BROTLI";
-})(MODE || (MODE = {}));
-export class CacheES {
-    client;
-    constructor(options) {
-        this.client = new Client({
-            ...(options || {}),
-            tls: {
-                rejectUnauthorized: false
-            }
-        });
-    }
-    async get(id, mode = MODE.STRING_BROTLI) {
-        let data;
-        try {
-            switch (mode) {
-                case MODE.STRING: {
-                    const value = (await this.client.get({
-                        index,
-                        id
-                    }))._source;
-                    data = value.data;
-                    break;
-                }
-                case MODE.STRING_BROTLI:
-                default: {
-                    const value = (await this.client.get({
-                        index,
-                        id
-                    }))._source;
-                    const dataRaw = brotliDecompressSync(Buffer.from(value.data, "binary")).toString();
-                    data = JSON.parse(dataRaw);
-                    break;
-                }
-            }
-        }
-        catch (e) {
-            data = null;
-        }
-        return data;
-    }
-    async set(id, value, mode = MODE.STRING_BROTLI) {
-        try {
-            switch (mode) {
-                case MODE.STRING: {
-                    await this.client.index({
-                        index: "redis_data", // puoi cambiare il nome dell'indice
-                        id,
-                        document: {
-                            id,
-                            data: value.toString("binary")
-                        }
-                    });
-                    break;
-                }
-                case MODE.JSON:
-                    // @ts-ignore
-                    await this.client.jSet(id, ".", value);
-                    break;
-                case MODE.STRING_BROTLI:
-                default: {
-                    const data = brotliCompressSync(JSON.stringify(value));
-                    await this.client.index({
-                        index: "redis_data", // puoi cambiare il nome dell'indice
-                        id,
-                        document: {
-                            id,
-                            data: data.toString("binary")
-                        }
-                    });
-                    break;
-                }
-            }
-        }
-        catch (e) {
-            console.error(`Can't cache data: ${e}`);
-        }
-    }
-    async delete(id) {
-        try {
-            await this.client.delete({
-                index,
-                id
-            });
-        }
-        catch (e) {
-            console.error(`Can't delete data: ${e}`);
-        }
-    }
-    async exists(id) {
-        try {
-            return !!(await this.client.exists({
-                index: "redis_data",
-                id
-            }));
-        }
-        catch (e) {
-            console.error(`Can't check data exists: ${e}`);
-        }
-        return;
-    }
-    async quit() {
-        try {
-            await this.client.close();
-        }
-        catch (e) {
-            console.error(`Can't quit connection: ${e}`);
-        }
-    }
-}