@wikicasa-dev/node-common 1.0.0 → 1.1.0

package/dist/Cache.d.ts

@@ -0,0 +1,20 @@
+import { RedisClientOptions, RedisClientType, RedisDefaultModules, RedisFunctions, RedisModules, RedisScripts } from "redis";
+import { Buffer } from "node:buffer";
+export declare enum MODE {
+    STRING = 0,
+    JSON = 1,
+    HASH = 2,
+    STRING_BROTLI = 3
+}
+export declare class Cache {
+    init: Promise<any>;
+    client: RedisClientType<RedisDefaultModules & RedisModules, RedisFunctions, RedisScripts>;
+    constructor(options?: RedisClientOptions);
+    start(): Promise<void>;
+    get(key: string, mode?: MODE): Promise<unknown>;
+    set(key: string, value: Buffer | unknown, mode?: MODE): Promise<void>;
+    delete(key: string): Promise<void>;
+    exists(key: string): Promise<boolean | undefined>;
+    select(db: number): Promise<void>;
+    quit(): Promise<void>;
+}

package/dist/Cache.js

@@ -0,0 +1,114 @@
+import redis from "redis";
+import { brotliCompressSync, brotliDecompressSync } from "zlib";
+import { Buffer } from "node:buffer";
+export var MODE;
+(function (MODE) {
+    MODE[MODE["STRING"] = 0] = "STRING";
+    MODE[MODE["JSON"] = 1] = "JSON";
+    MODE[MODE["HASH"] = 2] = "HASH";
+    MODE[MODE["STRING_BROTLI"] = 3] = "STRING_BROTLI";
+})(MODE || (MODE = {}));
+export class Cache {
+    init;
+    client;
+    constructor(options) {
+        this.client = redis.createClient(options);
+    }
+    async start() {
+        this.init = this.client.connect();
+    }
+    async get(key, mode = MODE.STRING_BROTLI) {
+        if (!this.init)
+            await this.start();
+        //if (!await this.client.exists(key))
+        //    return null;
+        let data;
+        try {
+            switch (mode) {
+                case MODE.STRING:
+                    data = await this.client.get(key);
+                    break;
+                case MODE.STRING_BROTLI:
+                default: {
+                    const value = await this.client.get(key);
+                    if (value !== null) {
+                        const dataRaw = brotliDecompressSync(Buffer.from(value, "binary")).toString();
+                        data = JSON.parse(dataRaw);
+                    }
+                    else {
+                        data = null;
+                    }
+                    break;
+                }
+            }
+        }
+        catch (e) {
+            data = null;
+        }
+        return data;
+    }
+    async set(key, value, mode = MODE.STRING_BROTLI) {
+        if (!this.init)
+            await this.start();
+        try {
+            switch (mode) {
+                case MODE.STRING: {
+                    await this.client.set(key, value.toString("binary"));
+                    break;
+                }
+                case MODE.JSON:
+                    // @ts-ignore
+                    await this.client.jSet(key, ".", value);
+                    break;
+                case MODE.STRING_BROTLI:
+                default: {
+                    const data = brotliCompressSync(JSON.stringify(value));
+                    await this.client.set(key, data.toString("binary"));
+                    break;
+                }
+            }
+        }
+        catch (e) {
+            console.error("Can't cache data");
+        }
+    }
+    async delete(key) {
+        if (!this.init)
+            await this.start();
+        try {
+            await this.client.del(key);
+        }
+        catch (e) {
+            console.error("Can't delete data");
+        }
+    }
+    async exists(key) {
+        if (!this.init)
+            await this.start();
+        try {
+            return !!(await this.client.exists(key));
+        }
+        catch (e) {
+            console.error("Can't check data exists");
+        }
+        return;
+    }
+    async select(db) {
+        if (!this.init)
+            await this.start();
+        try {
+            await this.client.select(db);
+        }
+        catch (e) {
+            console.error("Can't select db");
+        }
+    }
+    async quit() {
+        try {
+            await this.client.quit();
+        }
+        catch (e) {
+            console.error("Can't quit");
+        }
+    }
+}

package/dist/CacheES.d.ts

@@ -0,0 +1,17 @@
+import { Client, ClientOptions } from "@elastic/elasticsearch";
+import { Buffer } from "node:buffer";
+export declare enum MODE {
+    STRING = 0,
+    JSON = 1,
+    HASH = 2,
+    STRING_BROTLI = 3
+}
+export declare class CacheES {
+    client: Client;
+    constructor(options?: ClientOptions);
+    get(id: string, mode?: MODE): Promise<unknown>;
+    set(id: string, value: Buffer | unknown, mode?: MODE): Promise<void>;
+    delete(id: string): Promise<void>;
+    exists(id: string): Promise<boolean | undefined>;
+    quit(): Promise<void>;
+}

package/dist/CacheES.js

@@ -0,0 +1,119 @@
+import { Client } from "@elastic/elasticsearch";
+import { brotliCompressSync, brotliDecompressSync } from "zlib";
+import { Buffer } from "node:buffer";
+const index = "redis_data";
+export var MODE;
+(function (MODE) {
+    MODE[MODE["STRING"] = 0] = "STRING";
+    MODE[MODE["JSON"] = 1] = "JSON";
+    MODE[MODE["HASH"] = 2] = "HASH";
+    MODE[MODE["STRING_BROTLI"] = 3] = "STRING_BROTLI";
+})(MODE || (MODE = {}));
+export class CacheES {
+    client;
+    constructor(options) {
+        this.client = new Client({
+            ...(options || {}),
+            tls: {
+                rejectUnauthorized: false
+            }
+        });
+    }
+    async get(id, mode = MODE.STRING_BROTLI) {
+        let data;
+        try {
+            switch (mode) {
+                case MODE.STRING: {
+                    const value = (await this.client.get({
+                        index,
+                        id
+                    }))._source;
+                    data = value.data;
+                    break;
+                }
+                case MODE.STRING_BROTLI:
+                default: {
+                    const value = (await this.client.get({
+                        index,
+                        id
+                    }))._source;
+                    const dataRaw = brotliDecompressSync(Buffer.from(value.data, "binary")).toString();
+                    data = JSON.parse(dataRaw);
+                    break;
+                }
+            }
+        }
+        catch (e) {
+            data = null;
+        }
+        return data;
+    }
+    async set(id, value, mode = MODE.STRING_BROTLI) {
+        try {
+            switch (mode) {
+                case MODE.STRING: {
+                    await this.client.index({
+                        index: "redis_data", // puoi cambiare il nome dell'indice
+                        id,
+                        document: {
+                            id,
+                            data: value.toString("binary")
+                        }
+                    });
+                    break;
+                }
+                case MODE.JSON:
+                    // @ts-ignore
+                    await this.client.jSet(id, ".", value);
+                    break;
+                case MODE.STRING_BROTLI:
+                default: {
+                    const data = brotliCompressSync(JSON.stringify(value));
+                    await this.client.index({
+                        index: "redis_data", // puoi cambiare il nome dell'indice
+                        id,
+                        document: {
+                            id,
+                            data: data.toString("binary")
+                        }
+                    });
+                    break;
+                }
+            }
+        }
+        catch (e) {
+            console.error(`Can't cache data: ${e}`);
+        }
+    }
+    async delete(id) {
+        try {
+            await this.client.delete({
+                index,
+                id
+            });
+        }
+        catch (e) {
+            console.error(`Can't delete data: ${e}`);
+        }
+    }
+    async exists(id) {
+        try {
+            return !!(await this.client.exists({
+                index: "redis_data",
+                id
+            }));
+        }
+        catch (e) {
+            console.error(`Can't check data exists: ${e}`);
+        }
+        return;
+    }
+    async quit() {
+        try {
+            await this.client.close();
+        }
+        catch (e) {
+            console.error(`Can't quit connection: ${e}`);
+        }
+    }
+}

package/dist/CallPool.d.ts

@@ -0,0 +1,32 @@
+import { AxiosRequestConfig, AxiosResponse } from "axios";
+export declare enum PROXY {
+    NONE = 0,
+    STATIC = 1,
+    DYNAMIC = 2
+}
+export declare class CallPool {
+    private name;
+    private concurrency;
+    private readonly minConcurrency;
+    private readonly maxConcurrency;
+    private _print;
+    private readonly limiter;
+    private readonly proxy;
+    CLOCK_NUMBER: number;
+    WEIGHT: number;
+    mean: number;
+    targetMean: number;
+    clock: number;
+    private queue;
+    private activeCount;
+    private agent;
+    constructor(name: string, proxy: PROXY, minConcurrency: number, maxConcurrency: number, limitCall?: number, limitInterval?: number);
+    call(url: string, config?: AxiosRequestConfig): Promise<AxiosResponse>;
+    private getRandomUserAgent;
+    enqueue(call: () => Promise<any>): Promise<any>;
+    private schedule;
+    print(): string;
+    updatePrint(): void;
+    private finish;
+    private fetchWithRetry;
+}

package/dist/CallPool.js

@@ -0,0 +1,226 @@
+import throttledQueue from "throttled-queue";
+import Console, { COLORS } from "./Console.js";
+import { EventEmitter } from "events";
+import axios from "axios";
+import { sleep } from "./common.js";
+import CacheableLookup from "cacheable-lookup";
+import Agent from "agentkeepalive";
+import * as RandomUserAgent from "random-useragent";
+import axiosRetry from "axios-retry";
+import _ from "lodash";
+import { HttpsProxyAgent } from "https-proxy-agent";
+const cacheable = new CacheableLookup();
+// Increase the limit
+EventEmitter.defaultMaxListeners = 20;
+//const a = new CallPool("test", 1, 10, 50, 1000, PROXY.DYNAMIC);
+axiosRetry(axios, {
+    retries: 1,
+    retryDelay: (retryCount, error) => {
+        Console.log(`retry attempt: ${retryCount}. Error ${error}`);
+        return retryCount * 1000;
+    },
+});
+export var PROXY;
+(function (PROXY) {
+    PROXY[PROXY["NONE"] = 0] = "NONE";
+    PROXY[PROXY["STATIC"] = 1] = "STATIC";
+    PROXY[PROXY["DYNAMIC"] = 2] = "DYNAMIC";
+})(PROXY || (PROXY = {}));
+export class CallPool {
+    name;
+    concurrency;
+    minConcurrency = 1;
+    maxConcurrency;
+    _print = "";
+    limiter;
+    proxy;
+    CLOCK_NUMBER = 1;
+    WEIGHT = 100;
+    mean = 0;
+    targetMean = 0;
+    clock = this.CLOCK_NUMBER;
+    queue = [];
+    activeCount;
+    // @ts-ignore
+    agent;
+    constructor(name = "default", proxy = PROXY.STATIC, minConcurrency = 1, maxConcurrency, limitCall = 1000, limitInterval = 1000) {
+        this.name = name;
+        this.concurrency = (minConcurrency + maxConcurrency) / 2;
+        this.minConcurrency = minConcurrency;
+        this.maxConcurrency = maxConcurrency;
+        this.queue = [];
+        this.activeCount = 0;
+        this.limiter = throttledQueue(limitCall, limitInterval, true);
+        this.proxy = proxy;
+        this.updatePrint();
+        this.agent = new Agent({
+            maxSockets: 100,
+            maxFreeSockets: 10,
+            timeout: 30000,
+            lookup: cacheable.lookupAsync,
+        });
+        /*axios("https://api.ipify.org?format=json", {
+            method: "GET",
+            proxy: null,
+        });
+
+        fetch("https://api.ipify.org?format=json", {
+            method: "GET",
+            agent: null,
+            headers: {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
+            }
+        })*/
+    }
+    call(url, config) {
+        return new Promise((resolve, reject) => {
+            this.queue.push({
+                call: async () => {
+                    /*const response = await this.fetchWithRetry(url, {
+                        ...config,
+                        agent: this.agent,
+                        headers: {
+                            ...config?.headers,
+                            "User-Agent": this.proxy !== PROXY.NONE ? RandomUserAgent.getRandom() : undefined
+                        }
+                    });
+
+                    // Andrea M: handle 400-500 errors
+                    if (!response.ok)
+                        throw response;*/
+                    /*httpAgent: new Agent({
+                            keepAlive: false,
+                            maxSockets: 100,
+                            maxFreeSockets: 10,
+                            timeout: 60000,
+                            lookup: cacheable.lookupAsync,
+                        }),
+                        httpsAgent: new Agent({
+                            keepAlive: false,
+                            maxSockets: 100,
+                            maxFreeSockets: 10,
+                            timeout: 60000,
+                            lookup: cacheable.lookupAsync,
+                        }),*/
+                    return axios(url, _.merge({
+                        method: "GET",
+                        proxy: this.proxy === PROXY.DYNAMIC ? {
+                            protocol: "http",
+                            host: "api.zyte.com",
+                            port: 8011,
+                            auth: {
+                                username: "cf3dd14c1b9c4daf9cf67007520d0c48",
+                                password: ""
+                            },
+                        } : undefined,
+                        httpsAgent: this.proxy === PROXY.STATIC ? new HttpsProxyAgent("http://wikicasa:VTt7KYb5fsqDhK6xXinS@162.19.104.35:3128") : undefined,
+                        httpAgent: this.proxy === PROXY.STATIC ? new HttpsProxyAgent("http://wikicasa:VTt7KYb5fsqDhK6xXinS@162.19.104.35:3128") : undefined,
+                        maxContentLength: Infinity,
+                        maxBodyLength: Infinity,
+                        timeout: 600000,
+                        headers: {
+                            ...(this.proxy === PROXY.DYNAMIC && { "Zyte-Geolocation": "IT" }),
+                            "User-Agent": this.proxy !== PROXY.NONE ? this.getRandomUserAgent() : undefined,
+                        }
+                    }, config));
+                },
+                resolve,
+                reject
+            });
+            this.schedule();
+        });
+    }
+    getRandomUserAgent() {
+        const browsers = [
+            { name: "Chrome", version: 109 },
+            { name: "Firefox", version: 115 },
+            { name: "Edge", version: 116 }
+        ];
+        return RandomUserAgent.getRandom(function (ua) {
+            return browsers.some(b => b.name === ua.browserName && b.version <= parseInt(ua.browserVersion));
+        });
+    }
+    enqueue(call) {
+        return new Promise((resolve, reject) => {
+            this.queue.push({
+                call,
+                resolve,
+                reject,
+            });
+            this.schedule();
+        });
+    }
+    schedule() {
+        while (this.activeCount < this.concurrency && this.queue.length > 0) {
+            const item = this.queue.shift();
+            if (!item)
+                return;
+            const { call, resolve, reject } = item;
+            this.activeCount++;
+            const time = performance.now();
+            this.limiter(() => {
+                call()
+                    .then((result) => {
+                    resolve(result);
+                })
+                    .catch((error) => {
+                    reject(error);
+                })
+                    .finally(() => {
+                    this.activeCount--;
+                    this.clock--;
+                    if (this.clock < 0) {
+                        this.clock = this.CLOCK_NUMBER;
+                        if (this.mean > this.targetMean && this.mean > 10)
+                            this.concurrency = Math.max(this.concurrency - Math.floor(this.concurrency / 10), this.minConcurrency);
+                        if (this.mean < this.targetMean || this.mean < 2)
+                            this.concurrency = Math.min(this.concurrency + Math.ceil(this.concurrency / 20), this.maxConcurrency);
+                        this.targetMean = this.mean;
+                    }
+                    const t = (performance.now() - time) / 1000;
+                    this.mean = this.mean > 0 ? this.mean + (t - this.mean) / this.WEIGHT : t;
+                    this.updatePrint();
+                    //console.debug(this.name, t, this.mean, this.targetMean, `${this.activeCount}/${this.queue.length}`, this.concurrency, `${(60 / this.mean) * this.concurrency} avg/m`);
+                    this.schedule();
+                });
+            });
+        }
+    }
+    print() {
+        return this._print;
+    }
+    updatePrint() {
+        const textStart = `${Console.color(this.name.padEnd(16), COLORS.YELLOW)} [POOL][`;
+        // eslint-disable-next-line max-len
+        const textEnd = `][${this.activeCount + this.queue?.length}/${this.concurrency}][${this.mean.toFixed(2)}s, ${((60 / this.mean) * this.activeCount).toFixed(2)} avg/m]`;
+        let progress = "";
+        const max = process.stdout.columns - Math.max(textStart.length + textEnd.length, process.stdout.columns * 0.5);
+        const percent = this.activeCount / this.concurrency * max;
+        for (let j = 0; j < max; j++) {
+            const char = j < percent ? "=" : " ";
+            progress += Console.color(char, j < (max * 0.8) ? COLORS.GREEN : j < (max * 0.9) ? COLORS.YELLOW : COLORS.RED);
+        }
+        this._print = textStart + progress + textEnd;
+    }
+    async finish() {
+        while (this.activeCount) {
+            await this.finish();
+        }
+        return;
+    }
+    async fetchWithRetry(url, options, retries = 3, sleepTime = 1000) {
+        try {
+            const response = await fetch(url, options);
+            if (!response.ok)
+                throw new Error(`HTTP error! status: ${response.status}`);
+            return response;
+        }
+        catch (error) {
+            if (retries === 0)
+                throw new Error(`Failed to fetch ${url} after several retries`);
+            Console.log(`Retrying fetch for ${url}. Attempts remaining: ${retries - 1}`);
+            await sleep(sleepTime);
+            return this.fetchWithRetry(url, options, retries - 1);
+        }
+    }
+}

package/dist/Console.d.ts

@@ -0,0 +1,36 @@
+export declare enum COLORS {
+    BLACK = "\u001B[30m",
+    RED = "\u001B[31m",
+    GREEN = "\u001B[32m",
+    YELLOW = "\u001B[33m",
+    BLUE = "\u001B[34m",
+    MAGENTA = "\u001B[35m",
+    CYAN = "\u001B[36m",
+    WHITE = "\u001B[0m"
+}
+/**
+ * @author Andrea Moraglia
+ */
+export default class Console {
+    private static prefix;
+    private static postfix;
+    private static loader;
+    private static first;
+    private static summaryLines;
+    static setPrefix(prefix: string, color?: COLORS): void;
+    static setPostfix(postfix: string, color?: COLORS): void;
+    static appendSummaryLine(fn: () => string, color?: COLORS): void;
+    static clearSummaryLine(): void;
+    private static processSummaryLines;
+    static clearPostfix(): void;
+    static clearPrefix(): void;
+    static color(str: string, color: COLORS): string;
+    static log(str?: string, spinner?: boolean, file?: string): void;
+    static error(str?: string, spinner?: boolean, file?: string): void;
+    static warn(str?: string, spinner?: boolean, file?: string): void;
+    static success(str?: string, spinner?: boolean, file?: string): void;
+    private static getPrefix;
+    private static getPostfix;
+    private static clearLines;
+    static write(str: string, color: COLORS, spinner: boolean, file?: string): void;
+}

package/dist/Console.js

@@ -0,0 +1,113 @@
+import readline from "readline";
+import fs from "fs";
+const log_stdout = process.stdout;
+export var COLORS;
+(function (COLORS) {
+    COLORS["BLACK"] = "\u001B[30m";
+    COLORS["RED"] = "\u001B[31m";
+    COLORS["GREEN"] = "\u001B[32m";
+    COLORS["YELLOW"] = "\u001B[33m";
+    COLORS["BLUE"] = "\u001B[34m";
+    COLORS["MAGENTA"] = "\u001B[35m";
+    COLORS["CYAN"] = "\u001B[36m";
+    COLORS["WHITE"] = "\u001B[0m";
+})(COLORS || (COLORS = {}));
+/**
+ * @author Andrea Moraglia
+ */
+export default class Console {
+    static prefix = null;
+    static postfix = null;
+    static loader;
+    static first = true;
+    static summaryLines = [];
+    static setPrefix(prefix, color = COLORS.WHITE) {
+        this.prefix = this.color(prefix, color);
+    }
+    static setPostfix(postfix, color = COLORS.WHITE) {
+        this.postfix = this.color(postfix, color);
+    }
+    static appendSummaryLine(fn, color = COLORS.WHITE) {
+        if (process.env.NODE_ENV === "production")
+            return;
+        this.summaryLines.push({ fn, color });
+    }
+    static clearSummaryLine() {
+        this.summaryLines = [];
+    }
+    static processSummaryLines() {
+        return this.summaryLines.map(l => this.color(l.fn(), l.color)).join("\n");
+    }
+    static clearPostfix() {
+        this.postfix = null;
+    }
+    static clearPrefix() {
+        this.prefix = null;
+    }
+    static color(str, color) {
+        return `${color}${str}${COLORS.WHITE}`;
+    }
+    static log(str = "", spinner = false, file) {
+        this.write(str, COLORS.WHITE, spinner, file);
+    }
+    static error(str = "", spinner = false, file) {
+        this.write(str, COLORS.RED, spinner, file);
+    }
+    static warn(str = "", spinner = false, file) {
+        this.write(str, COLORS.YELLOW, spinner, file);
+    }
+    static success(str = "", spinner = false, file) {
+        this.write(str, COLORS.GREEN, spinner, file);
+    }
+    static getPrefix() {
+        return this.prefix ? this.prefix + " " : "";
+    }
+    static getPostfix() {
+        return this.postfix ? this.postfix + " " : "";
+    }
+    static clearLines(number) {
+        for (let i = 0; i < number; i++) {
+            const y = i === 0 ? null : -1;
+            readline.moveCursor(log_stdout, 0, y);
+            readline.clearLine(log_stdout, 0);
+        }
+        readline.cursorTo(log_stdout, 0);
+    }
+    static write(str = "", color, spinner, file = "") {
+        if (this.loader) {
+            clearInterval(this.loader);
+            this.loader = null;
+        }
+        if (spinner) {
+            const P = ["\\", "|", "/", "-"];
+            let x = 0;
+            readline.clearLine(log_stdout, 0);
+            log_stdout.write(`\r${this.getPrefix()}${P[x++]} ${this.color(str, color)}${this.getPostfix()}`);
+            if (this.summaryLines.length) {
+                readline.clearScreenDown(log_stdout);
+                log_stdout.write(`\n${this.processSummaryLines()}`);
+                readline.moveCursor(log_stdout, 0, -this.summaryLines.length);
+            }
+            x &= 3;
+            this.loader = setInterval(() => {
+                log_stdout.write(`\r${this.getPrefix()}${P[x++]} ${this.color(str, color)}${this.getPostfix()}`);
+                x &= 3;
+            }, 250);
+        }
+        else {
+            //this.clearLines(this.summaryLines.length + (this.loader ? 1 : 0));
+            //readline.moveCursor(log_stdout, 0, this.summaryLines.length);
+            //readline.cursorTo(log_stdout, 0);
+            readline.clearLine(log_stdout, 0);
+            log_stdout.write(`\r${this.getPrefix()}${this.color(str, color)}${this.getPostfix()}\n`);
+            if (file)
+                fs.appendFileSync(file, `\r${this.getPrefix()}${this.color(str, color)}${this.getPostfix()}\n`);
+            if (this.summaryLines.length && !this.first) {
+                readline.clearScreenDown(log_stdout);
+                log_stdout.write(`\r${this.processSummaryLines()}`);
+                readline.moveCursor(log_stdout, 0, -(this.summaryLines.length - 1));
+            }
+        }
+        this.first = false;
+    }
+}