@arcblock/pm2-prom-module 2.6.1

package/core/pm2.js

@@ -0,0 +1,311 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startPm2Connect = void 0;
+const pm2_1 = __importDefault(require("pm2"));
+const app_1 = require("./app");
+const utils_1 = require("../utils");
+const cpu_1 = require("../utils/cpu");
+const metrics_1 = require("../metrics");
+const app_2 = require("../metrics/app");
+const logger_1 = require("../utils/logger");
+const docker_1 = require("../utils/docker");
+const WORKER_CHECK_INTERVAL = 1000;
+const SHOW_STAT_INTERVAL = 10000;
+const APPS = {};
+const isMonitoringApp = (app) => {
+    const pm2_env = app.pm2_env;
+    if (pm2_env.axm_options.isModule ||
+        !app.name ||
+        app.pm_id === undefined // pm_id might be zero
+    ) {
+        return false;
+    }
+    return true;
+};
+const updateAppPidsData = (workingApp, pidData) => {
+    workingApp.updatePid({
+        id: pidData.id,
+        memory: pidData.memory,
+        cpu: pidData.cpu || 0,
+        pmId: pidData.pmId,
+        restartCount: pidData.restartCount,
+        createdAt: pidData.createdAt,
+        metrics: pidData.metrics,
+        status: pidData.status,
+    });
+};
+const detectActiveApps = () => {
+    const logger = (0, logger_1.getLogger)();
+    pm2_1.default.list((err, apps) => {
+        if (err)
+            return console.error(err.stack || err);
+        const pidsMonit = {};
+        const mapAppPids = {};
+        const activePM2Ids = new Set();
+        apps.forEach((appInstance) => {
+            var _a;
+            const pm2_env = appInstance.pm2_env;
+            const appName = appInstance.name;
+            if (!isMonitoringApp(appInstance) || !appName || appInstance.pm_id === undefined) {
+                return;
+            }
+            // Fill all apps pids
+            if (!mapAppPids[appName]) {
+                mapAppPids[appName] = {
+                    pids: [],
+                    restartsSum: 0,
+                };
+            }
+            mapAppPids[appName].restartsSum =
+                mapAppPids[appName].restartsSum + Number(pm2_env.restart_time || 0);
+            // Get the last app instance status
+            mapAppPids[appName].status = (_a = appInstance.pm2_env) === null || _a === void 0 ? void 0 : _a.status;
+            if (appInstance.pid && appInstance.pm_id !== undefined) {
+                mapAppPids[appName].pids.push(appInstance.pid);
+                // Fill active pm2 apps id to collect internal statistic
+                if (pm2_env.status === 'online') {
+                    activePM2Ids.add(appInstance.pm_id);
+                }
+                // Fill monitoring data
+                pidsMonit[appInstance.pid] = {
+                    cpu: 0,
+                    memory: 0,
+                    pmId: appInstance.pm_id,
+                    id: appInstance.pid,
+                    restartCount: pm2_env.restart_time || 0,
+                    createdAt: pm2_env.created_at || 0,
+                    metrics: pm2_env.axm_monitor,
+                    status: pm2_env.status,
+                };
+            }
+        });
+        Object.keys(APPS).forEach((appName) => {
+            const processingApp = mapAppPids[appName];
+            // Filters apps which do not have active pids
+            if (!processingApp) {
+                logger.debug(`Delete ${appName} because it not longer exists`);
+                const workingApp = APPS[appName];
+                const instances = workingApp.getActivePm2Ids();
+                // Clear app metrics
+                (0, app_2.deleteAppMetrics)(appName);
+                // Clear all metrics in prom-client because an app is not exists anymore
+                (0, metrics_1.deletePromAppMetrics)(appName, instances);
+                delete APPS[appName];
+            }
+            else {
+                const workingApp = APPS[appName];
+                if (workingApp) {
+                    const activePids = processingApp.pids;
+                    const removedPids = workingApp.removeNotActivePids(activePids);
+                    if (removedPids.length) {
+                        const removedIntances = removedPids.map((entry) => entry.pmId);
+                        logger.debug(`App ${appName} clear metrics. Removed PIDs ${removedIntances.toString()}`);
+                        (0, metrics_1.deletePromAppInstancesMetrics)(appName, removedIntances);
+                        if (!activePids.length) {
+                            // Delete app metrics because it does not have active PIDs anymore
+                            logger.debug(`App ${appName} does not have active PIDs. Clear app metrics`);
+                            (0, app_2.deleteAppMetrics)(appName);
+                        }
+                    }
+                    const pidsRestartsSum = workingApp
+                        .getRestartCount()
+                        .reduce((accum, item) => accum + item.value, 0);
+                    if (processingApp.restartsSum > pidsRestartsSum) {
+                        // Reset metrics when active restart app bigger then active app
+                        // This logic exist to prevent autoscaling problems if we use only !==
+                        logger.debug(`App ${appName} has been restarted. Clear app metrics`);
+                        (0, app_2.deleteAppMetrics)(appName);
+                    }
+                }
+            }
+        });
+        // Create instances for new apps
+        for (const [appName, entry] of Object.entries(mapAppPids)) {
+            if (!APPS[appName]) {
+                APPS[appName] = new app_1.App(appName);
+            }
+            const workingApp = APPS[appName];
+            if (workingApp) {
+                // Update status
+                workingApp.updateStatus(entry.status);
+            }
+        }
+        // Collect statistic from apps. Do it after all APPS created
+        if (activePM2Ids.size > 0) {
+            // logger.debug(`Collect app metrics from PIDs ${Array.from(activePM2Ids)}`);
+            sendCollectStaticticBusEvent(Array.from(activePM2Ids));
+        }
+        // Update metric with available apps
+        metrics_1.metricAvailableApps === null || metrics_1.metricAvailableApps === void 0 ? void 0 : metrics_1.metricAvailableApps.set(Object.keys(APPS).length);
+        // Get all pids to monit
+        const pids = Object.keys(pidsMonit);
+        // Get real pids data.
+        // !ATTENTION! Can not use PM2 app.monit because of incorrect values of CPU usage
+        (0, cpu_1.getPidsUsage)(pids)
+            .then((stats) => __awaiter(void 0, void 0, void 0, function* () {
+            // Fill data for all pids
+            if (stats && Object.keys(stats).length) {
+                for (const [pid, stat] of Object.entries(stats)) {
+                    const pidId = Number(pid);
+                    if (pidId && pidsMonit[pidId]) {
+                        pidsMonit[pidId].cpu = Math.round(stat.cpu * 10) / 10;
+                        pidsMonit[pidId].memory = stat.memory;
+                    }
+                }
+            }
+            // Get docker stats
+            // @ts-expect-error
+            const dockerApps = apps.filter((app) => { var _a; return (_a = app.pm2_env) === null || _a === void 0 ? void 0 : _a.BLOCKLET_DOCKER_NAME; });
+            yield (0, docker_1.getDockerStats)(
+            // @ts-expect-error
+            dockerApps.map((x) => { var _a; return (_a = x.pm2_env) === null || _a === void 0 ? void 0 : _a.BLOCKLET_DOCKER_NAME; })).then((stats) => {
+                stats.map((stat, i) => {
+                    const entry = mapAppPids[dockerApps[i].name];
+                    if (entry) {
+                        entry.pids.forEach((pid) => {
+                            pidsMonit[pid].cpu = stat.cpuUsage;
+                            pidsMonit[pid].memory = stat.memoryUsage;
+                        });
+                    }
+                });
+            });
+            for (const [appName, entry] of Object.entries(mapAppPids)) {
+                const workingApp = APPS[appName];
+                if (workingApp) {
+                    // Update pids data
+                    entry.pids.forEach((pidId) => {
+                        const monit = pidsMonit[pidId];
+                        if (monit) {
+                            updateAppPidsData(workingApp, monit);
+                        }
+                    });
+                    // Collect metrics
+                    processWorkingApp(workingApp);
+                }
+            }
+        }))
+            .catch((err) => {
+            console.error(err.stack || err);
+        });
+        // get docker stats
+        // @ts-ignore
+    });
+};
+const startPm2Connect = (conf) => {
+    const logger = (0, logger_1.getLogger)();
+    pm2_1.default.connect((err) => {
+        var _a;
+        if (err)
+            return console.error(err.stack || err);
+        const additionalMetrics = app_1.PM2_METRICS.map((entry) => {
+            return {
+                key: (0, utils_1.toUndescore)(entry.name),
+                description: `${entry.name}. Unit "${entry.unit}"`,
+            };
+        });
+        if (additionalMetrics.length) {
+            (0, metrics_1.initDynamicGaugeMetricClients)(additionalMetrics);
+        }
+        detectActiveApps();
+        // Collect statistic from running apps
+        pm2_1.default.launchBus((err, bus) => {
+            if (err)
+                return console.error(err.stack || err);
+            logger.debug('Start bus listener');
+            bus.on('process:msg', (packet) => {
+                if (packet.process &&
+                    packet.raw &&
+                    packet.raw.topic === 'pm2-prom-module:metrics' &&
+                    packet.raw.data) {
+                    const { name, pm_id } = packet.process;
+                    /*logger.debug(
+                        `Got message from app=${name} and pid=${pm_id}. Message=${JSON.stringify(
+                            packet.raw.data
+                        )}`
+                    );*/
+                    if (name && APPS[name] && packet.raw.data.metrics) {
+                        (0, app_2.processAppMetrics)(conf, {
+                            pmId: pm_id,
+                            appName: name,
+                            appResponse: packet.raw.data,
+                        });
+                    }
+                }
+            });
+        });
+        // Start timer to update available apps
+        setInterval(() => {
+            detectActiveApps();
+        }, (_a = conf.app_check_interval) !== null && _a !== void 0 ? _a : WORKER_CHECK_INTERVAL);
+        if (conf.debug) {
+            setInterval(() => {
+                if (Object.keys(APPS).length) {
+                    for (const [, app] of Object.entries(APPS)) {
+                        const cpuValues = app.getCpuThreshold().map((entry) => entry.value);
+                        const memory = Math.round(app.getTotalUsedMemory() / 1024 / 1024);
+                        const CPU = cpuValues.length ? cpuValues.toString() : '0';
+                        (0, logger_1.getLogger)().debug(`App "${app.getName()}" has ${app.getActiveWorkersCount()} worker(s). CPU: ${CPU}, Memory: ${memory}MB`);
+                    }
+                }
+                else {
+                    (0, logger_1.getLogger)().debug(`No apps available`);
+                }
+            }, SHOW_STAT_INTERVAL);
+        }
+    });
+};
+exports.startPm2Connect = startPm2Connect;
+function processWorkingApp(workingApp) {
+    const labels = { app: workingApp.getName() };
+    metrics_1.metricAppInstances === null || metrics_1.metricAppInstances === void 0 ? void 0 : metrics_1.metricAppInstances.set(labels, workingApp.getActiveWorkersCount());
+    metrics_1.metricAppAverageMemory === null || metrics_1.metricAppAverageMemory === void 0 ? void 0 : metrics_1.metricAppAverageMemory.set(labels, workingApp.getAverageUsedMemory());
+    metrics_1.metricAppTotalMemory === null || metrics_1.metricAppTotalMemory === void 0 ? void 0 : metrics_1.metricAppTotalMemory.set(labels, workingApp.getTotalUsedMemory());
+    metrics_1.metricAppAverageCpu === null || metrics_1.metricAppAverageCpu === void 0 ? void 0 : metrics_1.metricAppAverageCpu.set(labels, workingApp.getAverageCpu());
+    metrics_1.metricAppUptime === null || metrics_1.metricAppUptime === void 0 ? void 0 : metrics_1.metricAppUptime.set(labels, workingApp.getUptime());
+    metrics_1.metricAppStatus === null || metrics_1.metricAppStatus === void 0 ? void 0 : metrics_1.metricAppStatus.set(labels, workingApp.getStatus());
+    workingApp.getCurrentPidsCpu().forEach((entry) => {
+        metrics_1.metricAppPidsCpuLast === null || metrics_1.metricAppPidsCpuLast === void 0 ? void 0 : metrics_1.metricAppPidsCpuLast.set(Object.assign(Object.assign({}, labels), { instance: entry.pmId }), entry.value);
+    });
+    workingApp.getCpuThreshold().forEach((entry) => {
+        metrics_1.metricAppPidsCpuThreshold === null || metrics_1.metricAppPidsCpuThreshold === void 0 ? void 0 : metrics_1.metricAppPidsCpuThreshold.set(Object.assign(Object.assign({}, labels), { instance: entry.pmId }), entry.value);
+    });
+    workingApp.getCurrentPidsMemory().forEach((entry) => {
+        metrics_1.metricAppPidsMemory === null || metrics_1.metricAppPidsMemory === void 0 ? void 0 : metrics_1.metricAppPidsMemory.set(Object.assign(Object.assign({}, labels), { instance: entry.pmId }), entry.value);
+    });
+    workingApp.getRestartCount().forEach((entry) => {
+        metrics_1.metricAppRestartCount === null || metrics_1.metricAppRestartCount === void 0 ? void 0 : metrics_1.metricAppRestartCount.set(Object.assign(Object.assign({}, labels), { instance: entry.pmId }), entry.value);
+    });
+    workingApp.getPidPm2Metrics().forEach((entry) => {
+        Object.keys(entry.metrics).forEach((metricKey) => {
+            if (metrics_1.dynamicGaugeMetricClients[metricKey]) {
+                metrics_1.dynamicGaugeMetricClients[metricKey].set(Object.assign(Object.assign({}, labels), { instance: entry.pmId }), entry.metrics[metricKey]);
+            }
+        });
+    });
+}
+function sendCollectStaticticBusEvent(pm2Ids) {
+    // Request available metrics from all running apps
+    pm2Ids.forEach((pm2id) => {
+        pm2_1.default.sendDataToProcessId(pm2id, {
+            topic: 'pm2-prom-module:collect',
+            data: {},
+            // Required fields by pm2 but we do not use them
+            id: pm2id,
+        }, (err) => {
+            if (err)
+                return console.error(`pm2-prom-module: sendDataToProcessId ${err.stack || err}`);
+        });
+    });
+}

package/index.js

@@ -0,0 +1,106 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+// @ts-ignore
+const pmx_1 = __importDefault(require("pmx"));
+const http_1 = require("http");
+const net_1 = __importDefault(require("net"));
+const fs_1 = __importDefault(require("fs"));
+const pm2_1 = require("./core/pm2");
+const logger_1 = require("./utils/logger");
+const metrics_1 = require("./metrics");
+const DEFAULT_PREFIX = 'pm2';
+const startPromServer = (prefix, moduleConfig) => {
+    (0, metrics_1.initMetrics)(prefix);
+    const serviceName = moduleConfig.service_name;
+    const port = Number(moduleConfig.port);
+    const hostname = moduleConfig.hostname;
+    const unixSocketPath = moduleConfig.unix_socket_path;
+    const promServer = (0, http_1.createServer)((_req, res) => __awaiter(void 0, void 0, void 0, function* () {
+        const mergedRegistry = (0, metrics_1.combineAllRegistries)(Boolean(moduleConfig.aggregate_app_metrics));
+        mergedRegistry.setDefaultLabels({ serviceName });
+        res.setHeader('Content-Type', mergedRegistry.contentType);
+        res.end(yield mergedRegistry.metrics());
+        return;
+    }));
+    const listenCallback = () => {
+        const listenValue = promServer.address();
+        let listenString = '';
+        if (typeof listenValue === 'string') {
+            listenString = listenValue;
+        }
+        else {
+            listenString = `${listenValue === null || listenValue === void 0 ? void 0 : listenValue.address}:${listenValue === null || listenValue === void 0 ? void 0 : listenValue.port}`;
+        }
+        console.log(`Metrics server is available on ${listenString}`);
+    };
+    if (unixSocketPath) {
+        promServer.on('error', function (promServerError) {
+            if (promServerError.code == 'EADDRINUSE') {
+                console.log(`Listen error: "${promServerError.message}". Try to remove socket...`);
+                const clientSocket = new net_1.default.Socket();
+                clientSocket.on('error', function (clientSocketError) {
+                    if (clientSocketError.code == 'ECONNREFUSED') {
+                        console.log(`Remove old socket ${unixSocketPath}`);
+                        fs_1.default.unlinkSync(unixSocketPath);
+                        promServer.listen(unixSocketPath);
+                    }
+                });
+                clientSocket.connect({ path: unixSocketPath }, function () {
+                    console.log('Server running, giving up...');
+                    process.exit();
+                });
+            }
+        });
+        promServer.listen(unixSocketPath, listenCallback);
+    }
+    else {
+        promServer.listen(port, hostname, listenCallback);
+    }
+};
+pmx_1.default.initModule({
+    widget: {
+        el: {
+            probes: true,
+            actions: true,
+        },
+        block: {
+            actions: false,
+            issues: true,
+            meta: true,
+        },
+    },
+}, function (err, conf) {
+    var _a, _b;
+    if (err)
+        return console.error(err.stack || err);
+    const moduleConfig = conf.module_conf;
+    (0, logger_1.initLogger)({ isDebug: moduleConfig.debug });
+    (0, pm2_1.startPm2Connect)(moduleConfig);
+    startPromServer((_a = moduleConfig.prefix) !== null && _a !== void 0 ? _a : DEFAULT_PREFIX, moduleConfig);
+    pmx_1.default.configureModule({
+        human_info: [
+            ['Status', 'Module enabled'],
+            ['Debug', moduleConfig.debug ? 'Enabled' : 'Disabled'],
+            [
+                'Aggregate apps metrics',
+                moduleConfig.aggregate_app_metrics ? 'Enabled' : 'Disabled',
+            ],
+            ['Port', moduleConfig.port],
+            ['Service name', moduleConfig.service_name ? moduleConfig.service_name : `N/A`],
+            ['Check interval', `${moduleConfig.app_check_interval} ms`],
+            ['Prefix', (_b = moduleConfig.prefix) !== null && _b !== void 0 ? _b : DEFAULT_PREFIX],
+        ],
+    });
+});

package/metrics/app.js

@@ -0,0 +1,193 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAppRegistry = exports.processAppMetrics = exports.deleteAppMetrics = void 0;
+const prom_client_1 = __importDefault(require("prom-client"));
+const logger_1 = require("../utils/logger");
+const histogram_1 = require("./prom/histogram");
+const summary_1 = require("./prom/summary");
+const dynamicAppMetrics = {};
+const DEFAULT_LABELS = ['app', 'instance'];
+const parseLabels = (values) => {
+    const labels = new Set();
+    values.forEach((entry) => {
+        Object.keys(entry.labels).forEach((label) => {
+            labels.add(String(label));
+        });
+    });
+    return Array.from(labels);
+};
+const createMetricByType = (metric, labels) => {
+    switch (metric.type) {
+        case "counter" /* MetricType.Counter */: {
+            const metricEntry = new prom_client_1.default.Counter({
+                name: metric.name,
+                help: metric.help,
+                aggregator: metric.aggregator,
+                labelNames: [...DEFAULT_LABELS, ...labels],
+                registers: [],
+            });
+            return metricEntry;
+        }
+        case "gauge" /* MetricType.Gauge */: {
+            const metricEntry = new prom_client_1.default.Gauge({
+                name: metric.name,
+                help: metric.help,
+                aggregator: metric.aggregator,
+                labelNames: [...DEFAULT_LABELS, ...labels],
+                registers: [],
+            });
+            return metricEntry;
+        }
+        case "histogram" /* MetricType.Histogram */: {
+            const filteredMetrics = labels.filter((entry) => entry !== 'le');
+            const metricEntry = new histogram_1.IHistogram({
+                name: metric.name,
+                help: metric.help,
+                aggregator: metric.aggregator,
+                labelNames: [...DEFAULT_LABELS, ...filteredMetrics],
+                registers: [],
+            });
+            return metricEntry;
+        }
+        case "summary" /* MetricType.Summary */: {
+            const filteredMetrics = labels.filter((entry) => entry !== 'quantile');
+            const metricEntry = new summary_1.ISummary({
+                name: metric.name,
+                help: metric.help,
+                aggregator: metric.aggregator,
+                labelNames: [...DEFAULT_LABELS, ...filteredMetrics],
+                registers: [],
+            });
+            return metricEntry;
+        }
+        default:
+            return null;
+    }
+};
+const createRegistryMetrics = (registry) => {
+    const logger = (0, logger_1.getLogger)();
+    const metrics = {};
+    for (const [appName, appEntry] of Object.entries(dynamicAppMetrics)) {
+        for (const [metricName, pidEntry] of Object.entries(appEntry)) {
+            for (const [pm2id, metric] of Object.entries(pidEntry)) {
+                if (!metrics[metricName]) {
+                    const parsedLabels = parseLabels(metric.values);
+                    const newMetricStore = createMetricByType(metric, parsedLabels);
+                    if (newMetricStore) {
+                        metrics[metricName] = newMetricStore;
+                    }
+                }
+                const createdMetric = metrics[metricName];
+                if (!createdMetric) {
+                    logger.error(`Unsupported metric type ${metric.type} for ${metricName}`);
+                }
+                else {
+                    // Register metric
+                    registry.registerMetric(createdMetric);
+                    const defaultLabels = {
+                        app: appName,
+                        instance: pm2id,
+                    };
+                    // Fill data
+                    switch (metric.type) {
+                        case "counter" /* MetricType.Counter */: {
+                            metric.values.forEach((entry) => {
+                                try {
+                                    createdMetric.inc(Object.assign(Object.assign({}, entry.labels), defaultLabels), entry.value);
+                                }
+                                catch (error) {
+                                    logger.error(error);
+                                }
+                            });
+                            break;
+                        }
+                        case "gauge" /* MetricType.Gauge */: {
+                            metric.values.forEach((entry) => {
+                                try {
+                                    createdMetric.inc(Object.assign(Object.assign({}, entry.labels), defaultLabels), entry.value);
+                                }
+                                catch (error) {
+                                    logger.error(error);
+                                }
+                            });
+                            break;
+                        }
+                        case "histogram" /* MetricType.Histogram */: {
+                            createdMetric.setValues(defaultLabels, metric.values);
+                            break;
+                        }
+                        case "summary" /* MetricType.Summary */: {
+                            createdMetric.setValues(defaultLabels, metric.values);
+                            break;
+                        }
+                        default:
+                            break;
+                    }
+                }
+            }
+        }
+    }
+};
+const getAggregatedMetrics = () => {
+    const metrics = [];
+    for (const [appName, appEntry] of Object.entries(dynamicAppMetrics)) {
+        for (const [_metricName, pidEntry] of Object.entries(appEntry)) {
+            const pidMetrics = [];
+            for (const [_pm2id, metric] of Object.entries(pidEntry)) {
+                const metricWithApp = Object.assign({}, metric);
+                metricWithApp.values = metricWithApp.values.map((entry) => {
+                    entry.labels['app'] = appName;
+                    return entry;
+                });
+                pidMetrics.push(metricWithApp);
+            }
+            metrics.push(pidMetrics);
+        }
+    }
+    return prom_client_1.default.AggregatorRegistry.aggregate(metrics);
+};
+const deleteAppMetrics = (appName) => {
+    const logger = (0, logger_1.getLogger)();
+    if (dynamicAppMetrics[appName]) {
+        logger.debug(`Remove AppMetrics for app ${appName}`);
+        delete dynamicAppMetrics[appName];
+    }
+};
+exports.deleteAppMetrics = deleteAppMetrics;
+const processAppMetrics = (_config, data) => {
+    if (!Array.isArray(data.appResponse.metrics)) {
+        return;
+    }
+    data.appResponse.metrics.forEach((entry) => {
+        if (Array.isArray(entry.values) && entry.values.length) {
+            const metricName = entry.name;
+            if (!dynamicAppMetrics[data.appName]) {
+                dynamicAppMetrics[data.appName] = {};
+            }
+            const appKey = dynamicAppMetrics[data.appName][metricName];
+            if (!appKey) {
+                dynamicAppMetrics[data.appName][metricName] = {};
+            }
+            const pm2id = String(data.pmId);
+            dynamicAppMetrics[data.appName][metricName][pm2id] = entry;
+        }
+    });
+};
+exports.processAppMetrics = processAppMetrics;
+const getAppRegistry = (needAggregate) => {
+    if (Object.keys(dynamicAppMetrics).length) {
+        if (needAggregate) {
+            return getAggregatedMetrics();
+        }
+        else {
+            const registry = new prom_client_1.default.Registry();
+            createRegistryMetrics(registry);
+            return registry;
+        }
+    }
+    return undefined;
+};
+exports.getAppRegistry = getAppRegistry;