@vpalmisano/webrtcperf 4.0.0

package/build/src/stats.js

@@ -0,0 +1,1383 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Stats = exports.FastStats = void 0;
+const axios_1 = __importDefault(require("axios"));
+const chalk_1 = __importDefault(require("chalk"));
+const events = __importStar(require("events"));
+const fast_stats_1 = require("fast-stats");
+Object.defineProperty(exports, "FastStats", { enumerable: true, get: function () { return fast_stats_1.Stats; } });
+const fs = __importStar(require("fs"));
+const http = __importStar(require("http"));
+const https = __importStar(require("https"));
+const json5_1 = __importDefault(require("json5"));
+const path = __importStar(require("path"));
+const promClient = __importStar(require("prom-client"));
+const sprintf_js_1 = require("sprintf-js");
+const zlib = __importStar(require("zlib"));
+const rtcstats_1 = require("./rtcstats");
+const utils_1 = require("./utils");
+const log = (0, utils_1.logger)('webrtcperf:stats');
+function calculateFailAmountPercentile(stat, percentile = 95) {
+    return Math.round(stat.percentile(percentile));
+}
+/**
+ * StatsWriter
+ */
+class StatsWriter {
+    fname;
+    columns;
+    _header_written = false;
+    constructor(fname = 'stats.log', columns) {
+        this.fname = fname;
+        this.columns = columns;
+    }
+    /**
+     * push
+     * @param dataColumns
+     */
+    async push(dataColumns) {
+        if (!this._header_written) {
+            const data = ['datetime', ...this.columns].join(',') + '\n';
+            await fs.promises.mkdir(path.dirname(this.fname), { recursive: true });
+            await fs.promises.writeFile(this.fname, data);
+            this._header_written = true;
+        }
+        //
+        const data = [Date.now(), ...dataColumns].join(',') + '\n';
+        return fs.promises.appendFile(this.fname, data);
+    }
+}
+/**
+ * formatStatsColumns
+ * @param column
+ */
+function formatStatsColumns(column) {
+    return [
+        `${column}_length`,
+        `${column}_sum`,
+        `${column}_mean`,
+        `${column}_stdev`,
+        `${column}_5p`,
+        `${column}_95p`,
+        `${column}_min`,
+        `${column}_max`,
+    ];
+}
+/**
+ * Formats the stats for console or for file output.
+ * @param s The stats object.
+ * @param forWriter If true, format the stats to be written on file.
+ */
+function formatStats(s, forWriter = false) {
+    if (forWriter) {
+        return [
+            (0, utils_1.toPrecision)(s.length || 0, 0),
+            (0, utils_1.toPrecision)(s.sum || 0),
+            (0, utils_1.toPrecision)(s.amean() || 0),
+            (0, utils_1.toPrecision)(s.stddev() || 0),
+            (0, utils_1.toPrecision)(s.percentile(5) || 0),
+            (0, utils_1.toPrecision)(s.percentile(95) || 0),
+            (0, utils_1.toPrecision)(s.min || 0),
+            (0, utils_1.toPrecision)(s.max || 0),
+        ];
+    }
+    return {
+        length: s.length || 0,
+        sum: s.sum || 0,
+        mean: s.amean() || 0,
+        stddev: s.stddev() || 0,
+        p5: s.percentile(5) || 0,
+        p95: s.percentile(95) || 0,
+        min: s.min || 0,
+        max: s.max || 0,
+    };
+}
+/**
+ * Formats the console stats title.
+ * @param name
+ */
+function sprintfStatsTitle(name) {
+    return (0, sprintf_js_1.sprintf)((0, chalk_1.default) `-- {bold %(name)s} %(fill)s\n`, {
+        name,
+        fill: '-'.repeat(100 - name.length - 4),
+    });
+}
+/**
+ * Formats the console stats header.
+ */
+function sprintfStatsHeader() {
+    return (sprintfStatsTitle(new Date().toUTCString()) +
+        (0, sprintf_js_1.sprintf)((0, chalk_1.default) `{bold %(name)\' 30s} {bold %(length)\' 8s} {bold %(sum)\' 8s} {bold %(mean)\' 8s} {bold %(stddev)\' 8s} {bold %(p5)\' 8s} {bold %(p95)\' 8s} {bold %(min)\' 8s} {bold %(max)\' 8s}\n`, {
+            name: 'name',
+            length: 'count',
+            sum: 'sum',
+            mean: 'mean',
+            stddev: 'stddev',
+            p5: '5p',
+            p95: '95p',
+            min: 'min',
+            max: 'max',
+        }));
+}
+/**
+ * Format the stats for console output.
+ */
+function sprintfStats(name, stats, format = '.2f', unit = '', scale = 1, hideSum = false) {
+    if (!stats?.all.length) {
+        return '';
+    }
+    if (!scale) {
+        scale = 1;
+    }
+    const statsData = formatStats(stats.all);
+    return (0, sprintf_js_1.sprintf)((0, chalk_1.default) `{red {bold %(name)\' 30s}}` +
+        (0, chalk_1.default) ` {bold %(length)\' 8d}` +
+        (hideSum ? '         ' : (0, chalk_1.default) ` {bold %(sum)\' 8${format}}`) +
+        (0, chalk_1.default) ` {bold %(mean)\' 8${format}}` +
+        (0, chalk_1.default) ` {bold %(stddev)\' 8${format}}` +
+        (0, chalk_1.default) ` {bold %(p5)\' 8${format}}` +
+        (0, chalk_1.default) ` {bold %(p95)\' 8${format}}` +
+        (0, chalk_1.default) ` {bold %(min)\' 8${format}}` +
+        (0, chalk_1.default) ` {bold %(max)\' 8${format}}%(unit)s\n`, {
+        name,
+        length: statsData.length,
+        sum: statsData.sum * scale,
+        mean: statsData.mean * scale,
+        stddev: statsData.stddev * scale,
+        p5: statsData.p5 * scale,
+        p95: statsData.p95 * scale,
+        min: statsData.min * scale,
+        max: statsData.max * scale,
+        unit: unit ? (0, chalk_1.default) ` {red {bold ${unit}}}` : '',
+    });
+}
+const promPrefix = 'wst_';
+const promCreateGauge = (register, name, suffix = '', labelNames = [], collect) => {
+    return new promClient.Gauge({
+        name: `${promPrefix}${name}${suffix && '_' + suffix}`,
+        help: `${name} ${suffix}`,
+        labelNames,
+        registers: [register],
+        collect,
+    });
+};
+const calculateFailAmount = (checkValue, ruleValue) => {
+    if (ruleValue) {
+        return 100 * Math.min(1, Math.abs(checkValue - ruleValue) / ruleValue);
+    }
+    else {
+        return 100 * Math.min(1, Math.abs(checkValue));
+    }
+};
+/**
+ * The Stats collector class.
+ */
+class Stats extends events.EventEmitter {
+    statsPath;
+    detailedStatsPath;
+    prometheusPushgateway;
+    prometheusPushgatewayJobName;
+    prometheusPushgatewayAuth;
+    prometheusPushgatewayGzip;
+    showStats;
+    showPageLog;
+    statsInterval;
+    rtcStatsTimeout;
+    customMetrics = {};
+    startTimestamp;
+    enableDetailedStats;
+    startTimestampString;
+    sessions = new Map();
+    nextSessionId;
+    statsWriter;
+    detailedStatsWriter;
+    scheduler;
+    alertRules = null;
+    alertRulesFilename;
+    alertRulesFailPercentile;
+    pushStatsUrl;
+    pushStatsId;
+    serverSecret;
+    alertRulesReport = new Map();
+    gateway = null;
+    /* metricConfigGauge: promClient.Gauge<string> | null = null */
+    elapsedTimeMetric = null;
+    metrics = {};
+    alertTagsMetrics;
+    customMetricsLabels;
+    collectedStats;
+    collectedStatsConfig = {
+        url: '',
+        pages: 0,
+        startTime: 0,
+    };
+    externalCollectedStats = new Map();
+    pushStatsInstance = null;
+    running = false;
+    /**
+     * Stats aggregator class.
+     */
+    constructor({ statsPath, detailedStatsPath, prometheusPushgateway, prometheusPushgatewayJobName, prometheusPushgatewayAuth, prometheusPushgatewayGzip, showStats, showPageLog, statsInterval, rtcStatsTimeout, customMetrics, alertRules, alertRulesFilename, alertRulesFailPercentile, pushStatsUrl, pushStatsId, serverSecret, startSessionId, startTimestamp, enableDetailedStats, customMetricsLabels, }) {
+        super();
+        this.statsPath = statsPath;
+        this.detailedStatsPath = detailedStatsPath;
+        this.prometheusPushgateway = prometheusPushgateway;
+        this.prometheusPushgatewayJobName = prometheusPushgatewayJobName || 'default';
+        this.prometheusPushgatewayAuth = prometheusPushgatewayAuth || undefined;
+        this.prometheusPushgatewayGzip = prometheusPushgatewayGzip;
+        this.showStats = showStats !== undefined ? showStats : true;
+        this.showPageLog = !!showPageLog;
+        this.statsInterval = statsInterval || 10;
+        this.rtcStatsTimeout = Math.max(rtcStatsTimeout, this.statsInterval);
+        if (customMetrics.trim()) {
+            this.customMetrics = json5_1.default.parse(customMetrics);
+            log.debug(`using customMetrics: ${JSON.stringify(this.customMetrics, undefined, 2)}`);
+        }
+        this.collectedStats = this.initCollectedStats();
+        this.sessions = new Map();
+        this.nextSessionId = startSessionId;
+        this.startTimestamp = startTimestamp || Date.now();
+        this.startTimestampString = new Date(this.startTimestamp).toISOString();
+        this.enableDetailedStats = enableDetailedStats;
+        this.customMetricsLabels = customMetricsLabels
+            ? customMetricsLabels.split(',').reduce((p, label) => {
+                label = label.trim();
+                if (label) {
+                    p[label] = undefined;
+                }
+                return p;
+            }, {})
+            : {};
+        this.statsWriter = null;
+        this.detailedStatsWriter = null;
+        if (alertRules.trim()) {
+            this.alertRules = json5_1.default.parse(alertRules);
+            log.debug(`using alertRules: ${JSON.stringify(this.alertRules, undefined, 2)}`);
+        }
+        this.alertRulesFilename = alertRulesFilename;
+        this.alertRulesFailPercentile = alertRulesFailPercentile;
+        this.pushStatsUrl = pushStatsUrl;
+        this.pushStatsId = pushStatsId;
+        this.serverSecret = serverSecret;
+        if (this.pushStatsUrl) {
+            const httpAgent = new http.Agent({ keepAlive: false });
+            const httpsAgent = new https.Agent({
+                keepAlive: false,
+                rejectUnauthorized: false,
+            });
+            this.pushStatsInstance = axios_1.default.create({
+                httpAgent,
+                httpsAgent,
+                baseURL: this.pushStatsUrl,
+                auth: {
+                    username: 'admin',
+                    password: this.serverSecret,
+                },
+                maxBodyLength: 20000000,
+                transformRequest: [
+                    ...axios_1.default.defaults.transformRequest,
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    (data, headers) => {
+                        if (headers && typeof data === 'string' && data.length > 16 * 1024) {
+                            headers['Content-Encoding'] = 'gzip';
+                            return zlib.gzipSync(data);
+                        }
+                        else {
+                            return data;
+                        }
+                    },
+                ],
+            });
+        }
+    }
+    initCollectedStats() {
+        return this.statsNames.reduce((prev, name) => {
+            prev[name] = {
+                all: new fast_stats_1.Stats(),
+                byHost: {},
+                byCodec: {},
+                byParticipantAndTrack: {},
+            };
+            return prev;
+        }, {});
+    }
+    get statsNames() {
+        return Object.keys(rtcstats_1.PageStatsNames).concat(Object.keys(rtcstats_1.RtcStatsMetricNames)).concat(Object.keys(this.customMetrics));
+    }
+    /**
+     * consumeSessionId
+     * @param tabs the number of tabs to allocate in the same session.
+     */
+    consumeSessionId(tabs = 1) {
+        const id = this.nextSessionId;
+        this.nextSessionId += tabs;
+        return id;
+    }
+    /**
+     * Adds the session to the list of monitored sessions.
+     */
+    addSession(session) {
+        log.debug(`addSession ${session.id}`);
+        if (this.sessions.has(session.id)) {
+            throw new Error(`session id ${session.id} already present`);
+        }
+        session.once('stop', id => {
+            log.debug(`Session ${id} stopped`);
+            this.sessions.delete(id);
+        });
+        this.sessions.set(session.id, session);
+    }
+    /**
+     * Removes the session from list of monitored sessions.
+     * @param id the Session id
+     */
+    removeSession(id) {
+        log.debug(`removeSession ${id}`);
+        this.sessions.delete(id);
+    }
+    /**
+     * It updates the custom label value.
+     * @param label the custom metric label
+     * @param value the custom metric label value
+     */
+    setCustomMetricLabel(label, value) {
+        if (!(label in this.customMetricsLabels)) {
+            throw new Error(`Unknown custom metric label: ${label}`);
+        }
+        this.customMetricsLabels[label] = value;
+    }
+    /**
+     * start
+     */
+    async start() {
+        if (this.running) {
+            log.warn('already running');
+            return;
+        }
+        log.debug('start');
+        this.running = true;
+        if (this.statsPath) {
+            log.debug(`Logging stats into ${this.statsPath}`);
+            const headers = this.statsNames.reduce((v, name) => v.concat(formatStatsColumns(name)), []);
+            this.statsWriter = new StatsWriter(this.statsPath, headers);
+        }
+        if (this.detailedStatsPath) {
+            log.debug(`Logging stats into ${this.statsPath}`);
+            this.detailedStatsWriter = new StatsWriter(this.detailedStatsPath, [
+                'participantName',
+                'trackId',
+                ...this.statsNames,
+            ]);
+        }
+        if (this.prometheusPushgateway) {
+            const register = new promClient.Registry();
+            const agent = this.prometheusPushgateway.startsWith('https://')
+                ? new https.Agent({
+                    keepAlive: true,
+                    keepAliveMsecs: 60000,
+                    maxSockets: 5,
+                })
+                : new http.Agent({
+                    keepAlive: true,
+                    keepAliveMsecs: 60000,
+                    maxSockets: 5,
+                });
+            this.gateway = new promClient.Pushgateway(this.prometheusPushgateway, {
+                timeout: 5000,
+                auth: this.prometheusPushgatewayAuth,
+                rejectUnauthorized: false,
+                agent,
+                headers: this.prometheusPushgatewayGzip
+                    ? {
+                        'Content-Encoding': 'gzip',
+                    }
+                    : undefined,
+            }, register);
+            // promClient.collectDefaultMetrics({ prefix: promPrefix, register })
+            this.elapsedTimeMetric = promCreateGauge(register, 'elapsedTime', '', ['datetime', ...Object.keys(this.customMetricsLabels)], () => this.elapsedTimeMetric?.set({
+                datetime: this.startTimestampString,
+                ...this.customMetricsLabels,
+            }, (Date.now() - this.startTimestamp) / 1000));
+            // Export rtc stats.
+            this.statsNames.forEach(name => {
+                this.metrics[name] = {
+                    length: promCreateGauge(register, name, 'length', [
+                        'host',
+                        'codec',
+                        'datetime',
+                        ...Object.keys(this.customMetricsLabels),
+                    ]),
+                    sum: promCreateGauge(register, name, 'sum', [
+                        'host',
+                        'codec',
+                        'datetime',
+                        ...Object.keys(this.customMetricsLabels),
+                    ]),
+                    mean: promCreateGauge(register, name, 'mean', [
+                        'host',
+                        'codec',
+                        'datetime',
+                        ...Object.keys(this.customMetricsLabels),
+                    ]),
+                    stddev: promCreateGauge(register, name, 'stddev', [
+                        'host',
+                        'codec',
+                        'datetime',
+                        ...Object.keys(this.customMetricsLabels),
+                    ]),
+                    p5: promCreateGauge(register, name, 'p5', [
+                        'host',
+                        'codec',
+                        'datetime',
+                        ...Object.keys(this.customMetricsLabels),
+                    ]),
+                    p95: promCreateGauge(register, name, 'p95', [
+                        'host',
+                        'codec',
+                        'datetime',
+                        ...Object.keys(this.customMetricsLabels),
+                    ]),
+                    min: promCreateGauge(register, name, 'min', [
+                        'host',
+                        'codec',
+                        'datetime',
+                        ...Object.keys(this.customMetricsLabels),
+                    ]),
+                    max: promCreateGauge(register, name, 'max', [
+                        'host',
+                        'codec',
+                        'datetime',
+                        ...Object.keys(this.customMetricsLabels),
+                    ]),
+                    alertRules: {},
+                };
+                if (this.enableDetailedStats !== false) {
+                    this.metrics[name].value = promCreateGauge(register, name, '', [
+                        'participantName',
+                        'trackId',
+                        'datetime',
+                        ...Object.keys(this.customMetricsLabels),
+                    ]);
+                }
+                if (this.alertRules && this.alertRules[name]) {
+                    const rule = this.alertRules[name];
+                    for (const ruleKey of Object.keys(rule)) {
+                        const ruleName = `alert_${name}_${ruleKey}`;
+                        this.metrics[name].alertRules[ruleName] = {
+                            report: promCreateGauge(register, ruleName, 'report', [
+                                'rule',
+                                'datetime',
+                                ...Object.keys(this.customMetricsLabels),
+                            ]),
+                            rule: promCreateGauge(register, ruleName, '', [
+                                'rule',
+                                'datetime',
+                                ...Object.keys(this.customMetricsLabels),
+                            ]),
+                            mean: promCreateGauge(register, ruleName, 'mean', [
+                                'rule',
+                                'datetime',
+                                ...Object.keys(this.customMetricsLabels),
+                            ]),
+                        };
+                    }
+                }
+            });
+            if (this.alertRules) {
+                this.alertTagsMetrics = promCreateGauge(register, `alert_report`, '', [
+                    'datetime',
+                    'tag',
+                    ...Object.keys(this.customMetricsLabels),
+                ]);
+            }
+            await this.deletePushgatewayStats();
+        }
+        this.scheduler = new utils_1.Scheduler('stats', this.statsInterval, this.collectStats.bind(this));
+        this.scheduler.start();
+    }
+    async deletePushgatewayStats() {
+        if (!this.gateway) {
+            return;
+        }
+        try {
+            const { resp, body } = await this.gateway.delete({
+                jobName: this.prometheusPushgatewayJobName,
+            });
+            if (body.length) {
+                log.warn(`Pushgateway delete error ${resp.statusCode}: ${body}`);
+            }
+        }
+        catch (err) {
+            log.error(`Pushgateway delete error: ${err.stack}`);
+        }
+    }
+    /**
+     * collectStats
+     */
+    async collectStats(now) {
+        if (!this.running) {
+            return;
+        }
+        // log.debug(`statsInterval ${this.sessions.size} sessions`);
+        if (!this.sessions.size && !this.externalCollectedStats.size) {
+            return;
+        }
+        // Prepare config.
+        this.collectedStatsConfig.pages = 0;
+        this.collectedStatsConfig.startTime = this.startTimestamp;
+        // Reset collectedStats object.
+        Object.values(this.collectedStats).forEach(stats => {
+            stats.all.reset();
+            Object.values(stats.byHost).forEach(s => s.reset());
+            Object.values(stats.byCodec).forEach(s => s.reset());
+            stats.byParticipantAndTrack = {};
+        });
+        for (const [sessionId, session] of this.sessions.entries()) {
+            this.collectedStatsConfig.url = `${(0, utils_1.hideAuth)(session.url)}?${session.urlQuery}`;
+            this.collectedStatsConfig.pages += session.pages.size || 0;
+            const sessionStats = await session.updateStats();
+            for (const [name, obj] of Object.entries(sessionStats)) {
+                if (obj === undefined) {
+                    return;
+                }
+                //log.log(name, obj)
+                try {
+                    const collectedStats = this.collectedStats[name];
+                    if (typeof obj === 'number' && isFinite(obj)) {
+                        collectedStats.all.push(obj);
+                    }
+                    else {
+                        for (const [key, value] of Object.entries(obj)) {
+                            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                            if (typeof value === 'number' && isFinite(value)) {
+                                collectedStats.all.push(value);
+                                // Push host label.
+                                const { trackId, hostName, participantName } = (0, rtcstats_1.parseRtStatKey)(key);
+                                let stats = collectedStats.byHost[hostName];
+                                if (!stats) {
+                                    stats = collectedStats.byHost[hostName] = new fast_stats_1.Stats();
+                                }
+                                stats.push(value);
+                                // Push participant and track values.
+                                if ((0, utils_1.enabledForSession)(sessionId, this.enableDetailedStats) && participantName) {
+                                    collectedStats.byParticipantAndTrack[`${participantName}:${trackId || ''}`] = value;
+                                }
+                            }
+                            else if (typeof value === 'string') {
+                                // Codec stats.
+                                collectedStats.all.push(1);
+                                let stats = collectedStats.byCodec[value];
+                                if (!stats) {
+                                    stats = collectedStats.byCodec[value] = new fast_stats_1.Stats();
+                                }
+                                stats.push(1);
+                            }
+                        }
+                    }
+                }
+                catch (err) {
+                    log.error(`session getStats name: ${name} error: ${err.stack}`, err);
+                }
+            }
+        }
+        // Add external collected stats.
+        for (const [id, data] of this.externalCollectedStats.entries()) {
+            const { addedTime, externalStats, config } = data;
+            if (now - addedTime > this.rtcStatsTimeout * 1000) {
+                log.debug(`remove externalCollectedStats from ${id}`);
+                this.externalCollectedStats.delete(id);
+                continue;
+            }
+            log.debug(`add external stats from ${id}`);
+            // Add external config settings.
+            if (config.url) {
+                this.collectedStatsConfig.url = config.url;
+            }
+            if (config.pages) {
+                this.collectedStatsConfig.pages += config.pages;
+            }
+            // Add metrics.
+            this.statsNames.forEach(name => {
+                const stats = externalStats[name];
+                if (!stats) {
+                    return;
+                }
+                const collectedStats = this.collectedStats[name];
+                collectedStats.all.push(stats.all);
+                Object.entries(stats.byHost).forEach(([host, values]) => {
+                    if (!collectedStats.byHost[host]) {
+                        collectedStats.byHost[host] = new fast_stats_1.Stats();
+                    }
+                    collectedStats.byHost[host].push(values);
+                });
+                Object.entries(stats.byCodec).forEach(([codec, values]) => {
+                    if (!collectedStats.byCodec[codec]) {
+                        collectedStats.byCodec[codec] = new fast_stats_1.Stats();
+                    }
+                    collectedStats.byCodec[codec].push(values);
+                });
+                Object.entries(stats.byParticipantAndTrack).forEach(([label, value]) => {
+                    collectedStats.byParticipantAndTrack[label] = value;
+                });
+            });
+        }
+        this.emit('stats', this.collectedStats);
+        // Push to an external instance.
+        if (this.pushStatsInstance) {
+            const pushStats = {};
+            for (const [name, stats] of Object.entries(this.collectedStats)) {
+                pushStats[name] = {
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    all: stats.all.data,
+                    byHost: {},
+                    byCodec: {},
+                    byParticipantAndTrack: {},
+                };
+                Object.entries(stats.byHost).forEach(([host, stat]) => {
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    pushStats[name].byHost[host] = stat.data;
+                });
+                Object.entries(stats.byCodec).forEach(([codec, stat]) => {
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    pushStats[name].byCodec[codec] = stat.data;
+                });
+                Object.entries(stats.byParticipantAndTrack).forEach(([label, value]) => {
+                    pushStats[name].byParticipantAndTrack[label] = value;
+                });
+            }
+            try {
+                const res = await this.pushStatsInstance.put('/collected-stats', {
+                    id: this.pushStatsId,
+                    stats: pushStats,
+                    config: this.collectedStatsConfig,
+                });
+                log.debug(`pushStats message=${res.data.message}`);
+            }
+            catch (err) {
+                log.error(`pushStats error: ${err.stack}`);
+            }
+        }
+        // Check alerts.
+        this.checkAlertRules();
+        // Show to console.
+        this.consoleShowStats();
+        await Promise.allSettled([
+            this.writeStats(),
+            this.writeDetailedStats(),
+            this.sendToPushGateway(),
+            this.writeAlertRulesReport(),
+        ]);
+    }
+    async writeStats() {
+        if (!this.statsWriter)
+            return;
+        const values = this.statsNames.reduce((v, name) => v.concat(formatStats(this.collectedStats[name].all, true)), []);
+        await this.statsWriter.push(values);
+    }
+    async writeDetailedStats() {
+        if (!this.detailedStatsWriter)
+            return;
+        const participantTrackStats = new Map();
+        Object.entries(this.collectedStats).forEach(([name, stats]) => {
+            Object.entries(stats.byParticipantAndTrack).forEach(([label, value]) => {
+                let stats = participantTrackStats.get(label);
+                if (!stats) {
+                    stats = {};
+                    participantTrackStats.set(label, stats);
+                }
+                stats[name] = (0, utils_1.toPrecision)(value, 6);
+            });
+        });
+        for (const [label, trackStats] of participantTrackStats.entries()) {
+            const [participantName, trackId] = label.split(':', 2);
+            const values = [participantName, trackId];
+            for (const name of this.statsNames) {
+                values.push(trackStats[name] ?? '');
+            }
+            await this.detailedStatsWriter.push(values);
+        }
+    }
+    /**
+     * addCollectedStats
+     * @param id
+     * @param externalStats
+     * @param config
+     */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    addExternalCollectedStats(id, externalStats, config) {
+        log.debug(`addExternalCollectedStats from ${id}`);
+        const addedTime = Date.now();
+        this.externalCollectedStats.set(id, { addedTime, externalStats, config });
+    }
+    /**
+     * It display stats on the console.
+     */
+    consoleShowStats() {
+        if (!this.showStats) {
+            return;
+        }
+        const stats = this.collectedStats;
+        let out = sprintfStatsHeader() +
+            sprintfStats('System CPU', stats.usedCpu, '.2f', '%', undefined, true) +
+            sprintfStats('System GPU', stats.usedGpu, '.2f', '%', undefined, true) +
+            sprintfStats('System Memory', stats.usedMemory, '.2f', '%', undefined, true) +
+            sprintfStats('CPU/page', stats.cpu, '.2f', '%') +
+            sprintfStats('Memory/page', stats.memory, '.2f', 'MB') +
+            sprintfStats('Pages', stats.pages, 'd', '') +
+            sprintfStats('Errors', stats.errors, 'd', '') +
+            sprintfStats('Warnings', stats.warnings, 'd', '') +
+            sprintfStats('Peer Connections', stats.peerConnections, 'd', '') +
+            sprintfStats('audioSubscribeDelay', stats.audioSubscribeDelay, 'd', 'ms', undefined, true) +
+            sprintfStats('videoSubscribeDelay', stats.videoSubscribeDelay, 'd', 'ms', undefined, true) +
+            // inbound audio
+            sprintfStatsTitle('Inbound audio') +
+            sprintfStats('received', stats.audioBytesReceived, '.2f', 'MB', 1e-6) +
+            sprintfStats('rate', stats.audioRecvBitrates, '.2f', 'Kbps', 1e-3) +
+            sprintfStats('lost', stats.audioRecvPacketsLost, '.2f', '%', undefined, true) +
+            sprintfStats('jitter', stats.audioRecvJitter, '.2f', 's', undefined, true) +
+            sprintfStats('avgJitterBufferDelay', stats.audioRecvAvgJitterBufferDelay, '.2f', 'ms', 1e3, true) +
+            // inbound video
+            sprintfStatsTitle('Inbound video') +
+            sprintfStats('received', stats.videoRecvBytes, '.2f', 'MB', 1e-6) +
+            sprintfStats('decoded', stats.videoFramesDecoded, 'd', 'frames') +
+            sprintfStats('rate', stats.videoRecvBitrates, '.2f', 'Kbps', 1e-3) +
+            sprintfStats('lost', stats.videoRecvPacketsLost, '.2f', '%', undefined, true) +
+            sprintfStats('jitter', stats.videoRecvJitter, '.2f', 's', undefined, true) +
+            sprintfStats('avgJitterBufferDelay', stats.videoRecvAvgJitterBufferDelay, '.2f', 'ms', 1e3, true) +
+            sprintfStats('width', stats.videoRecvWidth, 'd', 'px', undefined, true) +
+            sprintfStats('height', stats.videoRecvHeight, 'd', 'px', undefined, true) +
+            sprintfStats('fps', stats.videoRecvFps, 'd', 'fps', undefined, true) +
+            sprintfStats('firCountSent', stats.firCountSent, 'd', '', undefined, true) +
+            sprintfStats('pliCountSent', stats.pliCountSent, 'd', '', undefined, true) +
+            // outbound audio
+            sprintfStatsTitle('Outbound audio') +
+            sprintfStats('sent', stats.audioBytesSent, '.2f', 'MB', 1e-6) +
+            sprintfStats('retransmitted', stats.audioRetransmittedBytesSent, '.2f', 'MB', 1e-6) +
+            sprintfStats('rate', stats.audioSentBitrates, '.2f', 'Kbps', 1e-3) +
+            sprintfStats('lost', stats.audioSentPacketsLost, '.2f', '%', undefined, true) +
+            sprintfStats('roundTripTime', stats.audioSentRoundTripTime, '.3f', 's', undefined, true) +
+            // outbound video
+            sprintfStatsTitle('Outbound video') +
+            sprintfStats('sent', stats.videoSentBytes, '.2f', 'MB', 1e-6) +
+            sprintfStats('retransmitted', stats.videoSentRetransmittedBytes, '.2f', 'MB', 1e-6) +
+            sprintfStats('rate', stats.videoSentBitrates, '.2f', 'Kbps', 1e-3) +
+            sprintfStats('lost', stats.videoSentPacketsLost, '.2f', '%', undefined, true) +
+            sprintfStats('roundTripTime', stats.videoSentRoundTripTime, '.3f', 's', undefined, true) +
+            sprintfStats('qualityLimitResolutionChanges', stats.videoQualityLimitationResolutionChanges, 'd', '') +
+            sprintfStats('qualityLimitationCpu', stats.videoQualityLimitationCpu, 'd', '%') +
+            sprintfStats('qualityLimitationBandwidth', stats.videoQualityLimitationBandwidth, 'd', '%') +
+            sprintfStats('sentActiveSpatialLayers', stats.videoSentActiveSpatialLayers, 'd', 'layers', undefined, true) +
+            sprintfStats('sentMaxBitrate', stats.videoSentMaxBitrate, '.2f', 'Kbps', 1e-3) +
+            sprintfStats('width', stats.videoSentWidth, 'd', 'px', undefined, true) +
+            sprintfStats('height', stats.videoSentHeight, 'd', 'px', undefined, true) +
+            sprintfStats('fps', stats.videoSentFps, 'd', 'fps', undefined, true) +
+            sprintfStats('firCountReceived', stats.videoFirCountReceived, 'd', '', undefined, true) +
+            sprintfStats('pliCountReceived', stats.videoPliCountReceived, 'd', '', undefined, true);
+        if (this.alertRules) {
+            const report = this.formatAlertRulesReport();
+            if (report.length) {
+                out += sprintfStatsTitle('Alert rules report');
+                out += report;
+            }
+        }
+        if (!this.showPageLog) {
+            console.clear();
+        }
+        console.log(out);
+    }
+    /**
+     * sendToPushGateway
+     */
+    async sendToPushGateway() {
+        if (!this.gateway || !this.running) {
+            return;
+        }
+        const elapsedSeconds = (Date.now() - this.startTimestamp) / 1000;
+        const datetime = this.startTimestampString;
+        Object.entries(this.metrics).forEach(([name, metric]) => {
+            if (!this.collectedStats[name]) {
+                return;
+            }
+            const setStats = (stats, host, codec) => {
+                const labels = { host, codec, datetime, ...this.customMetricsLabels };
+                const { length, sum, mean, stddev, p5, p95, min, max } = formatStats(stats);
+                metric.length.set(labels, length);
+                metric.sum.set(labels, sum);
+                metric.mean.set(labels, mean);
+                metric.stddev.set(labels, stddev);
+                metric.p5.set(labels, p5);
+                metric.p95.set(labels, p95);
+                metric.min.set(labels, min);
+                metric.max.set(labels, max);
+            };
+            setStats(this.collectedStats[name].all, 'all', 'all');
+            Object.entries(this.collectedStats[name].byHost).forEach(([host, stats]) => {
+                setStats(stats, host, 'all');
+            });
+            Object.entries(this.collectedStats[name].byCodec).forEach(([codec, stats]) => {
+                setStats(stats, 'all', codec);
+            });
+            if (metric.value) {
+                Object.entries(this.collectedStats[name].byParticipantAndTrack).forEach(([label, value]) => {
+                    const [participantName, trackId] = label.split(':', 2);
+                    metric.value?.set({
+                        participantName,
+                        trackId,
+                        datetime,
+                        ...this.customMetricsLabels,
+                    }, value);
+                });
+            }
+            // Set alerts metrics.
+            if (this.alertRules && this.alertRules[name]) {
+                const rule = this.alertRules[name];
+                // eslint-disable-next-line prefer-const
+                for (let [ruleKey, ruleValues] of Object.entries(rule)) {
+                    if (ruleKey === 'tags') {
+                        continue;
+                    }
+                    if (!Array.isArray(ruleValues)) {
+                        ruleValues = [ruleValues];
+                    }
+                    else {
+                        ruleValues = ruleValues;
+                    }
+                    for (const ruleValue of ruleValues) {
+                        // Send rule values as metrics.
+                        if (ruleValue.$after !== undefined && elapsedSeconds < ruleValue.$after) {
+                            continue;
+                        }
+                        const ruleName = `alert_${name}_${ruleKey}`;
+                        const ruleObj = this.metrics[name].alertRules[ruleName];
+                        const remove = ruleValue.$before !== undefined && elapsedSeconds > ruleValue.$before;
+                        // Send rule report as metric.
+                        const ruleDesc = this.getAlertRuleDesc(ruleKey, ruleValue);
+                        const report = this.alertRulesReport.get(name);
+                        if (report) {
+                            const ruleReport = report.get(ruleDesc);
+                            if (ruleReport) {
+                                const labels = {
+                                    rule: ruleDesc,
+                                    datetime,
+                                    ...this.customMetricsLabels,
+                                };
+                                if (!remove) {
+                                    ruleObj.report.set(labels, ruleReport.failAmountPercentile);
+                                    ruleObj.mean.set(labels, ruleReport.valueStats.amean());
+                                }
+                                else {
+                                    ruleObj.report.remove(labels);
+                                    ruleObj.mean.remove(labels);
+                                }
+                            }
+                        }
+                        // Send rules values as metrics.
+                        if (ruleValue.$eq !== undefined) {
+                            const labels = {
+                                rule: `${name} ${ruleKey} =`,
+                                datetime,
+                                ...this.customMetricsLabels,
+                            };
+                            if (!remove) {
+                                ruleObj.rule.set(labels, ruleValue.$eq);
+                            }
+                            else {
+                                ruleObj.rule.remove(labels);
+                            }
+                        }
+                        if (ruleValue.$lt !== undefined) {
+                            const labels = {
+                                rule: `${name} ${ruleKey} <`,
+                                datetime,
+                                ...this.customMetricsLabels,
+                            };
+                            if (!remove) {
+                                ruleObj.rule.set(labels, ruleValue.$lt);
+                            }
+                            else {
+                                ruleObj.rule.remove(labels);
+                            }
+                        }
+                        if (ruleValue.$lte !== undefined) {
+                            const labels = {
+                                rule: `${name} ${ruleKey} <=`,
+                                datetime,
+                                ...this.customMetricsLabels,
+                            };
+                            if (!remove) {
+                                ruleObj.rule.set(labels, ruleValue.$lte);
+                            }
+                            else {
+                                ruleObj.rule.remove(labels);
+                            }
+                        }
+                        if (ruleValue.$gt !== undefined) {
+                            const labels = {
+                                rule: `${name} ${ruleKey} >`,
+                                datetime,
+                                ...this.customMetricsLabels,
+                            };
+                            if (!remove) {
+                                ruleObj.rule.set(labels, ruleValue.$gt);
+                            }
+                            else {
+                                ruleObj.rule.remove(labels);
+                            }
+                        }
+                        if (ruleValue.$gte !== undefined) {
+                            const labels = {
+                                rule: `${name} ${ruleKey} >=`,
+                                datetime,
+                                ...this.customMetricsLabels,
+                            };
+                            if (!remove) {
+                                ruleObj.rule.set(labels, ruleValue.$gte);
+                            }
+                            else {
+                                ruleObj.rule.remove(labels);
+                            }
+                        }
+                    }
+                }
+            }
+        });
+        const alertRulesReportTags = this.getAlertRulesTags();
+        if (alertRulesReportTags && this.alertTagsMetrics) {
+            for (const [tag, stat] of alertRulesReportTags.entries()) {
+                this.alertTagsMetrics.set({ datetime, tag, ...this.customMetricsLabels }, calculateFailAmountPercentile(stat, this.alertRulesFailPercentile));
+            }
+        }
+        try {
+            const { resp, body } = await this.gateway.push({
+                jobName: this.prometheusPushgatewayJobName,
+            });
+            if (body.length) {
+                log.warn(`Pushgateway error ${resp.statusCode}: ${body}`);
+            }
+        }
+        catch (err) {
+            log.error(`Pushgateway push error: ${err.stack}`);
+        }
+    }
+    /**
+     * alertRuleDesc
+     */
+    getAlertRuleDesc(ruleKey, ruleValue) {
+        const ruleDescs = [];
+        if (ruleValue.$eq !== undefined) {
+            ruleDescs.push(`= ${ruleValue.$eq}`);
+        }
+        if (ruleValue.$gt !== undefined) {
+            ruleDescs.push(`> ${ruleValue.$gt}`);
+        }
+        if (ruleValue.$gte !== undefined) {
+            ruleDescs.push(`>= ${ruleValue.$gte}`);
+        }
+        if (ruleValue.$lt !== undefined) {
+            ruleDescs.push(`< ${ruleValue.$lt}`);
+        }
+        if (ruleValue.$lte !== undefined) {
+            ruleDescs.push(`<= ${ruleValue.$lte}`);
+        }
+        let ruleDesc = `${ruleKey} ${ruleDescs.join(' and ')}`;
+        if (ruleValue.$after !== undefined) {
+            ruleDesc += ` after ${ruleValue.$after}s`;
+        }
+        if (ruleValue.$before !== undefined) {
+            ruleDesc += ` before ${ruleValue.$before}s`;
+        }
+        return ruleDesc;
+    }
+    /**
+     * checkAlertRules
+     */
+    checkAlertRules() {
+        if (!this.alertRules || !this.running) {
+            return;
+        }
+        const now = Date.now();
+        const elapsedSeconds = (now - this.startTimestamp) / 1000;
+        for (const [key, rule] of Object.entries(this.alertRules)) {
+            if (!this.collectedStats[key]) {
+                continue;
+            }
+            let failPercentile = this.alertRulesFailPercentile;
+            const value = formatStats(this.collectedStats[key].all);
+            // eslint-disable-next-line prefer-const
+            for (let [ruleKey, ruleValues] of Object.entries(rule)) {
+                if (['tags', 'failPercentile'].includes(ruleKey)) {
+                    if (ruleKey === 'failPercentile') {
+                        failPercentile = ruleValues;
+                    }
+                    continue;
+                }
+                if (!Array.isArray(ruleValues)) {
+                    ruleValues = [ruleValues];
+                }
+                else {
+                    ruleValues = ruleValues;
+                }
+                let ruleElapsedSeconds = elapsedSeconds;
+                for (const ruleValue of ruleValues) {
+                    if ((ruleValue.$after !== undefined && elapsedSeconds < ruleValue.$after) ||
+                        (ruleValue.$before !== undefined && elapsedSeconds > ruleValue.$before)) {
+                        continue;
+                    }
+                    if (ruleValue.$after !== undefined) {
+                        ruleElapsedSeconds -= ruleValue.$after;
+                    }
+                    const checkValue = value[ruleKey];
+                    if (!isFinite(checkValue)) {
+                        continue;
+                    }
+                    const ruleDesc = this.getAlertRuleDesc(ruleKey, ruleValue);
+                    let failed = false;
+                    let failAmount = 0;
+                    if ((ruleValue.$skip_lt !== undefined && checkValue < ruleValue.$skip_lt) ||
+                        (ruleValue.$skip_lte !== undefined && checkValue <= ruleValue.$skip_lte) ||
+                        (ruleValue.$skip_gt !== undefined && checkValue > ruleValue.$skip_gt) ||
+                        (ruleValue.$skip_gte !== undefined && checkValue >= ruleValue.$skip_gte)) {
+                        continue;
+                    }
+                    if (ruleValue.$eq !== undefined) {
+                        if (checkValue !== ruleValue.$eq) {
+                            failed = true;
+                            failAmount = calculateFailAmount(checkValue, ruleValue.$eq);
+                        }
+                    }
+                    else {
+                        if (ruleValue.$lt !== undefined) {
+                            if (checkValue >= ruleValue.$lt) {
+                                failed = true;
+                                failAmount = calculateFailAmount(checkValue, ruleValue.$lt);
+                            }
+                        }
+                        else if (ruleValue.$lte !== undefined) {
+                            if (checkValue > ruleValue.$lte) {
+                                failed = true;
+                                failAmount = calculateFailAmount(checkValue, ruleValue.$lte);
+                            }
+                        }
+                        if (!failed) {
+                            if (ruleValue.$gt !== undefined) {
+                                if (checkValue <= ruleValue.$gt) {
+                                    failed = true;
+                                    failAmount = calculateFailAmount(checkValue, ruleValue.$gt);
+                                }
+                            }
+                            else if (ruleValue.$gte !== undefined) {
+                                if (checkValue < ruleValue.$gte) {
+                                    failed = true;
+                                    failAmount = calculateFailAmount(checkValue, ruleValue.$gte);
+                                }
+                            }
+                        }
+                    }
+                    // Report if failed or not.
+                    this.updateRulesReport(key, checkValue, ruleDesc, failed, failAmount, now, ruleElapsedSeconds, failPercentile);
+                }
+            }
+        }
+    }
+    /**
+     * addFailedRule
+     */
+    updateRulesReport(key, checkValue, ruleDesc, failed, failAmount, now, elapsedSeconds, failPercentile) {
+        if (failed) {
+            log.debug(`updateRulesReport ${key}.${ruleDesc} failed: ${failed} checkValue: ${checkValue} failAmount: ${failAmount} elapsedSeconds: ${elapsedSeconds}`);
+        }
+        let report = this.alertRulesReport.get(key);
+        if (!report) {
+            report = new Map();
+            this.alertRulesReport.set(key, report);
+        }
+        let reportValue = report.get(ruleDesc);
+        if (!reportValue) {
+            reportValue = {
+                totalFails: 0,
+                totalFailsTime: 0,
+                totalFailsTimePerc: 0,
+                lastFailed: 0,
+                valueStats: new fast_stats_1.Stats(),
+                failAmountStats: new fast_stats_1.Stats(),
+                failAmountPercentile: 0,
+            };
+            report.set(ruleDesc, reportValue);
+        }
+        if (failed) {
+            reportValue.totalFails += 1;
+            if (reportValue.lastFailed) {
+                reportValue.totalFailsTime += (now - reportValue.lastFailed) / 1000;
+            }
+            reportValue.lastFailed = now;
+        }
+        else {
+            reportValue.lastFailed = 0;
+        }
+        reportValue.totalFailsTimePerc = Math.round((100 * reportValue.totalFailsTime) / elapsedSeconds);
+        reportValue.valueStats.push(checkValue);
+        reportValue.failAmountStats.push(failAmount);
+        reportValue.failAmountPercentile = calculateFailAmountPercentile(reportValue.failAmountStats, failPercentile);
+    }
+    getAlertRulesTags() {
+        if (!this.alertRules) {
+            return;
+        }
+        const alertRulesReportTags = new Map();
+        for (const [key, report] of this.alertRulesReport.entries()) {
+            const tags = this.alertRules[key].tags || [];
+            for (const tag of tags) {
+                if (!alertRulesReportTags.has(tag)) {
+                    alertRulesReportTags.set(tag, new fast_stats_1.Stats());
+                }
+            }
+            for (const reportValue of report.values()) {
+                const { failAmountPercentile } = reportValue;
+                for (const tag of tags) {
+                    const stat = alertRulesReportTags.get(tag);
+                    if (!stat) {
+                        continue;
+                    }
+                    stat.push(failAmountPercentile);
+                }
+            }
+        }
+        return alertRulesReportTags;
+    }
+    /**
+     * formatAlertRulesReport
+     * @param ext
+     */
+    formatAlertRulesReport(ext = null) {
+        if (!this.alertRulesReport || !this.alertRules) {
+            return '';
+        }
+        // Update tags values.
+        const alertRulesReportTags = this.getAlertRulesTags();
+        // JSON output.
+        if (ext === 'json') {
+            const out = {
+                tags: {},
+                reports: {},
+            };
+            for (const [key, report] of this.alertRulesReport.entries()) {
+                for (const [reportDesc, reportValue] of report.entries()) {
+                    const { totalFails, totalFailsTime, valueStats, totalFailsTimePerc, failAmountStats, failAmountPercentile } = reportValue;
+                    if (totalFails) {
+                        out.reports[`${key} ${reportDesc}`] = {
+                            totalFails,
+                            totalFailsTime: Math.round(totalFailsTime),
+                            valueAverage: valueStats.amean(),
+                            totalFailsTimePerc,
+                            failAmount: failAmountPercentile,
+                            count: failAmountStats.length,
+                            // failAmountStats: (failAmountStats as any).data as number[],
+                        };
+                    }
+                }
+            }
+            for (const [tag, stat] of alertRulesReportTags.entries()) {
+                out.tags[tag] = calculateFailAmountPercentile(stat, this.alertRulesFailPercentile);
+            }
+            return JSON.stringify(out, null, 2);
+        }
+        // Textual output.
+        let out = '';
+        // Calculate max column size.
+        let colSize = 20;
+        for (const [key, report] of this.alertRulesReport.entries()) {
+            for (const [reportDesc, reportValue] of report.entries()) {
+                const { totalFails, totalFailsTimePerc } = reportValue;
+                if (totalFails && totalFailsTimePerc > 0) {
+                    const check = `${key} ${reportDesc}`;
+                    colSize = Math.max(colSize, check.length);
+                }
+            }
+        }
+        if (ext) {
+            out += (0, sprintf_js_1.sprintf)(`| %(check)-${colSize}s | %(total)-10s | %(totalFailsTime)-15s | %(totalFailsTimePerc)-15s | %(failAmount)-15s |\n`, {
+                check: 'Condition',
+                total: 'Fails',
+                totalFailsTime: 'Fail time (s)',
+                totalFailsTimePerc: 'Fail time (%)',
+                failAmount: 'Fail amount %',
+            });
+        }
+        else {
+            out += (0, sprintf_js_1.sprintf)((0, chalk_1.default) `{bold %(check)-${colSize}s} {bold %(total)-10s} {bold %(totalFailsTime)-15s} {bold %(totalFailsTimePerc)-15s} {bold %(failAmount)-15s}\n`, {
+                check: 'Condition',
+                total: 'Fails',
+                totalFailsTime: 'Fail time (s)',
+                totalFailsTimePerc: 'Fail time (%)',
+                failAmount: 'Fail amount %',
+            });
+        }
+        for (const [key, report] of this.alertRulesReport.entries()) {
+            for (const [reportDesc, reportValue] of report.entries()) {
+                const { totalFails, totalFailsTime, failAmountPercentile, totalFailsTimePerc } = reportValue;
+                if (totalFails && totalFailsTimePerc > 0) {
+                    if (ext) {
+                        out += (0, sprintf_js_1.sprintf)(`| %(check)-${colSize}s | %(totalFails)-10s | %(totalFailsTime)-15s | %(totalFailsTimePerc)-15s | %(failAmountPercentile)-15s |\n`, {
+                            check: `${key} ${reportDesc}`,
+                            totalFails,
+                            totalFailsTime: Math.round(totalFailsTime),
+                            totalFailsTimePerc,
+                            failAmountPercentile,
+                        });
+                    }
+                    else {
+                        out += (0, sprintf_js_1.sprintf)((0, chalk_1.default) `{red {bold %(check)-${colSize}s}} {bold %(totalFails)-10s} {bold %(totalFailsTime)-15s} {bold %(totalFailsTimePerc)-15s} {bold %(failAmountPercentile)-15s}\n`, {
+                            check: `${key} ${reportDesc}`,
+                            totalFails,
+                            totalFailsTime: Math.round(totalFailsTime),
+                            totalFailsTimePerc,
+                            failAmountPercentile,
+                        });
+                    }
+                }
+            }
+        }
+        // Tags report.
+        if (ext) {
+            out += (0, sprintf_js_1.sprintf)(`%(fill)s\n`, { fill: '-'.repeat(colSize + 15 + 7) });
+            out += (0, sprintf_js_1.sprintf)(`| %(name)-${colSize}s | %(failPerc)-15s |\n`, {
+                name: 'Tag',
+                failPerc: 'Fail %',
+            });
+        }
+        else {
+            out += (0, sprintf_js_1.sprintf)(`%(fill)s\n`, { fill: '-'.repeat(colSize + 15) });
+            out += (0, sprintf_js_1.sprintf)((0, chalk_1.default) `{bold %(name)-${colSize}s} {bold %(failPerc)-15s}\n`, {
+                name: 'Tag',
+                failPerc: 'Fail %',
+            });
+        }
+        for (const [tag, stat] of alertRulesReportTags.entries()) {
+            const failPerc = calculateFailAmountPercentile(stat, this.alertRulesFailPercentile);
+            if (ext) {
+                out += (0, sprintf_js_1.sprintf)(`| %(tag)-${colSize}s | %(failPerc)-15s |\n`, {
+                    tag,
+                    failPerc,
+                });
+            }
+            else {
+                const color = failPerc < 5 ? 'green' : failPerc < 25 ? 'yellowBright' : failPerc < 50 ? 'yellow' : 'red';
+                out += (0, sprintf_js_1.sprintf)((0, chalk_1.default) `{${color} {bold %(tag)-${colSize}s %(failPerc)-15s}}\n`, {
+                    tag,
+                    failPerc,
+                });
+            }
+        }
+        return out;
+    }
+    /**
+     * writeAlertRulesReport
+     */
+    async writeAlertRulesReport() {
+        if (!this.alertRules || !this.alertRulesFilename || !this.running) {
+            return;
+        }
+        log.debug(`writeAlertRulesReport writing in ${this.alertRulesFilename}`);
+        try {
+            const ext = this.alertRulesFilename.split('.').slice(-1)[0];
+            const report = this.formatAlertRulesReport(ext);
+            if (!report.length) {
+                return;
+            }
+            let out;
+            if (ext === 'log') {
+                const lines = report.split('\n').filter(line => line.length);
+                const name = `Alert rules report (${new Date().toISOString()})`;
+                out = (0, sprintf_js_1.sprintf)(`-- %(name)s %(fill)s\n`, {
+                    name,
+                    fill: '-'.repeat(Math.max(4, lines[0].length - name.length - 4)),
+                });
+                out += report;
+                out += (0, sprintf_js_1.sprintf)(`%(fill)s\n`, {
+                    fill: '-'.repeat(lines[lines.length - 1].length),
+                });
+            }
+            else {
+                out = report;
+            }
+            await fs.promises.mkdir(path.dirname(this.alertRulesFilename), {
+                recursive: true,
+            });
+            await fs.promises.writeFile(this.alertRulesFilename, out);
+        }
+        catch (err) {
+            log.error(`writeAlertRulesReport error: ${err.stack}`);
+        }
+    }
+    /**
+     * Stop the stats collector and the added Sessions.
+     */
+    async stop() {
+        if (!this.running) {
+            return;
+        }
+        this.running = false;
+        log.debug('stop');
+        if (this.scheduler) {
+            this.scheduler.stop();
+            this.scheduler = undefined;
+        }
+        for (const session of this.sessions.values()) {
+            try {
+                session.removeAllListeners();
+                await session.stop();
+            }
+            catch (err) {
+                log.error(`session stop error: ${err.stack}`);
+            }
+        }
+        this.sessions.clear();
+        this.statsWriter = null;
+        // delete metrics
+        if (this.gateway) {
+            await this.deletePushgatewayStats();
+            this.gateway = null;
+            this.metrics = {};
+        }
+        this.collectedStats = this.initCollectedStats();
+        this.externalCollectedStats.clear();
+    }
+}
+exports.Stats = Stats;
+//# sourceMappingURL=stats.js.map