badmfck-api-server 4.1.25 → 4.1.26

package/dist/apiServer/http/Http.d.ts

@@ -13,6 +13,19 @@ export type THttpErr<E = any> = {
     details?: any;
 };
 export type THttpResult<T, E = any> = THttpOk<T> | THttpErr<E>;
+export type THttpOutgoingLogItem = {
+    id: number;
+    created: number;
+    time: number;
+    method: HttpMethod;
+    url: string;
+    status?: number;
+    ok: boolean;
+    attempts: number;
+    request: any;
+    response: any;
+    error?: string;
+};
 export type TRetryPolicy = {
     enabled?: boolean;
     maxAttempts?: number;
@@ -71,6 +84,12 @@ export declare class Http {
     private static _activeConnectionsByHost;
     private static _countedSockets;
     private static _socketHostKey;
+    private static _outgoingLog;
+    private static _nextOutgoingLogID;
+    private static _outgoingLogLimit;
+    private static _truncate;
+    private static _pushOutgoingLog;
+    static getOutgoingLogSnapshot(): THttpOutgoingLogItem[];
     private static ensureInit;
     static configure(partial: Partial<THttpConfig>): void;
     static createAgent(opts?: ConstructorParameters<typeof Agent>[0]): Agent;

package/dist/apiServer/http/Http.js

@@ -88,6 +88,40 @@ class Http {
     static _activeConnectionsByHost = new Map();
     static _countedSockets = new WeakSet();
     static _socketHostKey = new WeakMap();
+    static _outgoingLog = [];
+    static _nextOutgoingLogID = 0;
+    static _outgoingLogLimit = 500;
+    static _truncate(v) {
+        if (v === undefined || v === null)
+            return v;
+        if (Buffer.isBuffer(v))
+            return `[Buffer ${v.length} bytes]`;
+        let json = "";
+        try {
+            json = JSON.stringify(v, (_, val) => {
+                if (typeof val === "string" && val.length > 255)
+                    return val.slice(0, 255) + `...(${val.length})`;
+                if (Buffer.isBuffer(val))
+                    return `[Buffer ${val.length} bytes]`;
+                return val;
+            });
+        }
+        catch {
+            return "[unserializable]";
+        }
+        return json.length > 500 ? json.slice(0, 500) + `...(total ${json.length})` : json;
+    }
+    static _pushOutgoingLog(item) {
+        const entry = { id: this._nextOutgoingLogID++, ...item };
+        this._outgoingLog.push(entry);
+        if (this._outgoingLog.length > this._outgoingLogLimit)
+            this._outgoingLog.shift();
+        if (this._nextOutgoingLogID > Number.MAX_SAFE_INTEGER - 1000)
+            this._nextOutgoingLogID = 0;
+    }
+    static getOutgoingLogSnapshot() {
+        return this._outgoingLog.slice();
+    }
     static ensureInit() {
         if (this._inited)
             return;
@@ -230,6 +264,7 @@ class Http {
     static async request(opts) {
         MonitorService_1.S_STAT_REGISTRATE_HTTP_REQUEST.invoke({ url: opts.url });
         this.ensureInit();
+        const overallStarted = Date.now();
         const cfg = this.config;
         const retryCfg = {
             enabled: opts.retry?.enabled ?? cfg.retry?.enabled ?? true,
@@ -263,7 +298,24 @@ class Http {
         }
         const responseType = opts.responseType ?? "json";
         const throwOnJsonParseError = opts.throwOnJsonParseError ?? false;
+        let attemptsMade = 0;
+        const finish = (result, networkErr) => {
+            Http._pushOutgoingLog({
+                created: overallStarted,
+                time: Date.now() - overallStarted,
+                method: opts.method,
+                url,
+                status: result.status,
+                ok: result.ok,
+                attempts: attemptsMade,
+                request: Http._truncate(opts.body),
+                response: Http._truncate(result.ok ? result.data : result.error),
+                error: networkErr ? String(networkErr?.message ?? networkErr) : undefined,
+            });
+            return result;
+        };
         for (let attempt = 1; attempt <= retryCfg.maxAttempts; attempt++) {
+            attemptsMade = attempt;
             const started = Date.now();
             cfg.onRequest?.({ method: opts.method, url, attempt });
             const controller = new AbortController();
@@ -307,7 +359,7 @@ class Http {
                     }
                 }
                 if (status >= 200 && status < 300) {
-                    return { ok: true, status, data: parsed, headers: hdrs };
+                    return finish({ ok: true, status, data: parsed, headers: hdrs });
                 }
                 const retryable = retryCfg.enabled &&
                     isRetryableStatus(status, retryCfg.retryOnStatuses) &&
@@ -317,7 +369,7 @@ class Http {
                     await sleep(backoff);
                     continue;
                 }
-                return { ok: false, status, error: parsed, details: { headers: hdrs } };
+                return finish({ ok: false, status, error: parsed, details: { headers: hdrs } });
             }
             catch (err) {
                 const ms = Date.now() - started;
@@ -331,13 +383,13 @@ class Http {
                     await sleep(backoff);
                     continue;
                 }
-                return { ok: false, error: err, details: { network: true } };
+                return finish({ ok: false, error: err, details: { network: true } }, err);
             }
             finally {
                 clearTimeout(t);
             }
         }
-        return { ok: false, error: "UNKNOWN" };
+        return finish({ ok: false, error: "UNKNOWN" });
     }
     static get(url, opts) {
         return this.request({ ...(opts || {}), method: "GET", url });

package/dist/apiServer/monitor/Monitor.d.ts

@@ -20,6 +20,9 @@ export declare class Monitor extends BaseEndpoint {
     changeLogLevel(req: HTTPRequestVO): Promise<TransferPacketVO>;
     logs(req: HTTPRequestVO): Promise<TransferPacketVO>;
     netlog(req: HTTPRequestVO): Promise<TransferPacketVO>;
+    outgoingNetlog(req: HTTPRequestVO): Promise<TransferPacketVO>;
+    sysinfo(req: HTTPRequestVO): Promise<TransferPacketVO>;
+    procinfo(req: HTTPRequestVO): Promise<TransferPacketVO>;
     metrics(req: HTTPRequestVO): Promise<TransferPacketVO>;
     connections(req: HTTPRequestVO): Promise<TransferPacketVO>;
     db(req: HTTPRequestVO): Promise<TransferPacketVO>;

package/dist/apiServer/monitor/Monitor.js

@@ -15,6 +15,8 @@ const path_1 = __importDefault(require("path"));
 const Http_1 = require("../http/Http");
 const MonitorService_1 = require("../MonitorService");
 const DBService_1 = require("../DBService");
+const SystemInfo_1 = require("./SystemInfo");
+const ProcessMonitor_1 = require("./ProcessMonitor");
 exports.S_MONITOR_REGISTRATE_ACTION = new badmfck_signal_1.Signal();
 const logMap = {
     ALL: 10,
@@ -39,6 +41,9 @@ class Monitor extends BaseEndpoint_1.BaseEndpoint {
             { ignoreInterceptor: true, endpoint: "html", handler: this.html },
             { ignoreInterceptor: true, endpoint: "log", handler: this.logs },
             { ignoreInterceptor: true, endpoint: "netlog", handler: this.netlog },
+            { ignoreInterceptor: true, endpoint: "outgoingnetlog", handler: this.outgoingNetlog },
+            { ignoreInterceptor: true, endpoint: "sysinfo", handler: this.sysinfo },
+            { ignoreInterceptor: true, endpoint: "procinfo", handler: this.procinfo },
             { ignoreInterceptor: true, endpoint: "metrics", handler: this.metrics, validationModel: {
                     range: "",
                     $__range_values: ["week", "day", "hour"]
@@ -120,6 +125,18 @@ class Monitor extends BaseEndpoint_1.BaseEndpoint {
         const log = await APIService_1.REQ_HTTP_LOG.request(req.data);
         return { data: log };
     }
+    async outgoingNetlog(req) {
+        await this.checkAuthentication(req);
+        return { data: Http_1.Http.getOutgoingLogSnapshot() };
+    }
+    async sysinfo(req) {
+        await this.checkAuthentication(req);
+        return { data: await (0, SystemInfo_1.collectSystemInfo)() };
+    }
+    async procinfo(req) {
+        await this.checkAuthentication(req);
+        return { data: await ProcessMonitor_1.ProcessMonitor.getSnapshot() };
+    }
     async metrics(req) {
         await this.checkAuthentication(req);
         const result = await MonitorService_1.REQ_STAT.request(req.data);

package/dist/apiServer/monitor/ProcessMonitor.d.ts

@@ -0,0 +1,117 @@
+/// <reference types="node" />
+export declare class ProcessMonitor {
+    private static _inited;
+    private static _eld;
+    private static _gcObserver;
+    private static _gc;
+    private static _prevCpu;
+    private static _prevCpuAt;
+    private static _cpuPct;
+    private static _eluPrev;
+    private static _elu;
+    private static _processStartedAt;
+    static init(): void;
+    private static _sampleCpu;
+    private static _sampleELU;
+    private static _mapGcKind;
+    static getSnapshot(): Promise<{
+        identity: {
+            pid: number;
+            ppid: number;
+            title: string;
+            argv: string[];
+            execPath: string;
+            cwd: string;
+            nodeVersion: string;
+            v8Version: string;
+            platform: NodeJS.Platform;
+            arch: NodeJS.Architecture;
+            hostname: string;
+            user: {
+                uid: number;
+                gid: number;
+            } | null;
+            startedAtUnix: number;
+            uptimeSec: number;
+            availableParallelism: number;
+        };
+        memory: {
+            rss: number;
+            heapTotal: number;
+            heapUsed: number;
+            external: number;
+            arrayBuffers: any;
+            heap: {
+                sizeLimit: number;
+                usedHeap: number;
+                totalAvailable: number;
+                totalPhysical: number;
+                mallocedMemory: number;
+                peakMallocedMemory: number;
+                numberOfNativeContexts: number;
+                numberOfDetachedContexts: number;
+            };
+            heapSpaces: {
+                name: string;
+                size: number;
+                used: number;
+                available: number;
+                physical: number;
+            }[];
+            procStatus: Record<string, number> | null;
+        };
+        cpu: {
+            userPct: number;
+            systemPct: number;
+            userMicros: number;
+            systemMicros: number;
+            resource: any;
+        };
+        eventLoop: {
+            minMs: number;
+            maxMs: number;
+            meanMs: number;
+            stddevMs: number;
+            p50Ms: number;
+            p90Ms: number;
+            p99Ms: number;
+            utilizationPct: number;
+            activeHandles: number;
+            activeRequests: number;
+            spotLagMs: number;
+        } | null;
+        gc: {
+            totalCount: number;
+            totalTimeMs: number;
+            maxPauseMs: number;
+            byKind: Record<string, {
+                count: number;
+                timeMs: number;
+            }>;
+            lastPauses: {
+                kind: string;
+                ms: number;
+                timestamp: number;
+            }[];
+        };
+        fileDescriptors: {
+            open: number | null;
+            limitSoft: number | null;
+            limitHard: number | null;
+        };
+        io: Record<string, number> | null;
+        limits: Record<string, {
+            soft: string;
+            hard: string;
+            unit: string;
+        }> | null;
+    }>;
+    private static _readProcStatus;
+    private static _readProcIo;
+    private static _readProcLimits;
+    private static _countFds;
+    private static _parseLimitValue;
+    private static _toMsFinite;
+    private static _safePercentileMs;
+    private static _safeCount;
+}

package/dist/apiServer/monitor/ProcessMonitor.js

@@ -0,0 +1,308 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProcessMonitor = void 0;
+const os_1 = __importDefault(require("os"));
+const v8_1 = __importDefault(require("v8"));
+const perf_hooks_1 = require("perf_hooks");
+const promises_1 = __importDefault(require("fs/promises"));
+class ProcessMonitor {
+    static _inited = false;
+    static _eld = null;
+    static _gcObserver = null;
+    static _gc = {
+        totalCount: 0,
+        totalTimeMs: 0,
+        maxPauseMs: 0,
+        byKind: {},
+        lastPauses: [],
+    };
+    static _prevCpu = process.cpuUsage();
+    static _prevCpuAt = Date.now();
+    static _cpuPct = { userPct: 0, systemPct: 0 };
+    static _eluPrev = perf_hooks_1.performance.eventLoopUtilization();
+    static _elu = { idle: 0, active: 0, utilizationPct: 0 };
+    static _processStartedAt = Date.now() - process.uptime() * 1000;
+    static init() {
+        if (this._inited)
+            return;
+        this._inited = true;
+        this._eld = (0, perf_hooks_1.monitorEventLoopDelay)({ resolution: 20 });
+        this._eld.enable();
+        this._gcObserver = new perf_hooks_1.PerformanceObserver((list) => {
+            for (const entry of list.getEntries()) {
+                const kind = this._mapGcKind(entry.detail?.kind ?? entry.kind);
+                this._gc.totalCount++;
+                this._gc.totalTimeMs += entry.duration;
+                if (entry.duration > this._gc.maxPauseMs)
+                    this._gc.maxPauseMs = entry.duration;
+                const bucket = this._gc.byKind[kind] ?? { count: 0, timeMs: 0 };
+                bucket.count++;
+                bucket.timeMs += entry.duration;
+                this._gc.byKind[kind] = bucket;
+                this._gc.lastPauses.push({ kind, ms: entry.duration, timestamp: Date.now() });
+                if (this._gc.lastPauses.length > 20)
+                    this._gc.lastPauses.shift();
+            }
+        });
+        try {
+            this._gcObserver.observe({ entryTypes: ["gc"], buffered: false });
+        }
+        catch {
+            try {
+                this._gcObserver.observe({ type: "gc" });
+            }
+            catch { }
+        }
+        const cpuTimer = setInterval(() => this._sampleCpu(), 5000);
+        if (cpuTimer.unref)
+            cpuTimer.unref();
+        const eluTimer = setInterval(() => this._sampleELU(), 5000);
+        if (eluTimer.unref)
+            eluTimer.unref();
+    }
+    static _sampleCpu() {
+        const now = process.cpuUsage();
+        const nowAt = Date.now();
+        const dtMs = nowAt - this._prevCpuAt;
+        if (dtMs <= 0)
+            return;
+        const userPct = (now.user - this._prevCpu.user) / (dtMs * 1000) * 100;
+        const systemPct = (now.system - this._prevCpu.system) / (dtMs * 1000) * 100;
+        this._cpuPct = {
+            userPct: Math.round(userPct * 10) / 10,
+            systemPct: Math.round(systemPct * 10) / 10,
+        };
+        this._prevCpu = now;
+        this._prevCpuAt = nowAt;
+    }
+    static _sampleELU() {
+        const cur = perf_hooks_1.performance.eventLoopUtilization();
+        const delta = perf_hooks_1.performance.eventLoopUtilization(cur, this._eluPrev);
+        this._elu = {
+            idle: delta.idle,
+            active: delta.active,
+            utilizationPct: Math.round(delta.utilization * 10000) / 100,
+        };
+        this._eluPrev = cur;
+    }
+    static _mapGcKind(kind) {
+        if (typeof kind === "string")
+            return kind;
+        switch (kind) {
+            case 1: return "minor";
+            case 2: return "major";
+            case 4: return "incremental";
+            case 8: return "weakcb";
+            case 15: return "all";
+            default: return "unknown";
+        }
+    }
+    static async getSnapshot() {
+        const tStart = process.hrtime.bigint();
+        await new Promise(r => setImmediate(() => r()));
+        const spotLagMs = Number(process.hrtime.bigint() - tStart) / 1e6;
+        const mem = process.memoryUsage();
+        const heap = v8_1.default.getHeapStatistics();
+        const heapSpaces = v8_1.default.getHeapSpaceStatistics().map(s => ({
+            name: s.space_name,
+            size: s.space_size,
+            used: s.space_used_size,
+            available: s.space_available_size,
+            physical: s.physical_space_size,
+        }));
+        const procStatus = await this._readProcStatus();
+        const io = await this._readProcIo();
+        const limitsParsed = await this._readProcLimits();
+        const cpuTotal = process.cpuUsage();
+        let resource = null;
+        try {
+            const r = process.resourceUsage();
+            resource = {
+                maxRSS: r.maxRSS,
+                voluntaryContextSwitches: r.voluntaryContextSwitches,
+                involuntaryContextSwitches: r.involuntaryContextSwitches,
+                fsRead: r.fsRead,
+                fsWrite: r.fsWrite,
+                signalsCount: r.signalsCount,
+                ipcSent: r.ipcSent,
+                ipcReceived: r.ipcReceived,
+                userCPUTimeMs: Math.round(r.userCPUTime / 1000),
+                systemCPUTimeMs: Math.round(r.systemCPUTime / 1000),
+            };
+        }
+        catch { }
+        const fdOpen = await this._countFds();
+        const fdSoft = limitsParsed?.["Max open files"]?.soft;
+        const fdHard = limitsParsed?.["Max open files"]?.hard;
+        const h = this._eld;
+        const eventLoop = h ? {
+            minMs: this._toMsFinite(h.min),
+            maxMs: this._toMsFinite(h.max),
+            meanMs: this._toMsFinite(h.mean),
+            stddevMs: this._toMsFinite(h.stddev),
+            p50Ms: this._safePercentileMs(h, 50),
+            p90Ms: this._safePercentileMs(h, 90),
+            p99Ms: this._safePercentileMs(h, 99),
+            utilizationPct: this._elu.utilizationPct,
+            activeHandles: this._safeCount(() => process._getActiveHandles?.()?.length),
+            activeRequests: this._safeCount(() => process._getActiveRequests?.()?.length),
+            spotLagMs: Math.round(spotLagMs * 100) / 100,
+        } : null;
+        return {
+            identity: {
+                pid: process.pid,
+                ppid: process.ppid,
+                title: process.title,
+                argv: process.argv,
+                execPath: process.execPath,
+                cwd: process.cwd(),
+                nodeVersion: process.version,
+                v8Version: process.versions.v8,
+                platform: process.platform,
+                arch: process.arch,
+                hostname: os_1.default.hostname(),
+                user: typeof process.getuid === "function"
+                    ? { uid: process.getuid(), gid: process.getgid?.() ?? -1 }
+                    : null,
+                startedAtUnix: this._processStartedAt,
+                uptimeSec: Math.round(process.uptime()),
+                availableParallelism: typeof os_1.default.availableParallelism === "function"
+                    ? os_1.default.availableParallelism()
+                    : os_1.default.cpus().length,
+            },
+            memory: {
+                rss: mem.rss,
+                heapTotal: mem.heapTotal,
+                heapUsed: mem.heapUsed,
+                external: mem.external,
+                arrayBuffers: mem.arrayBuffers ?? 0,
+                heap: {
+                    sizeLimit: heap.heap_size_limit,
+                    usedHeap: heap.used_heap_size,
+                    totalAvailable: heap.total_available_size,
+                    totalPhysical: heap.total_physical_size,
+                    mallocedMemory: heap.malloced_memory,
+                    peakMallocedMemory: heap.peak_malloced_memory,
+                    numberOfNativeContexts: heap.number_of_native_contexts,
+                    numberOfDetachedContexts: heap.number_of_detached_contexts,
+                },
+                heapSpaces,
+                procStatus,
+            },
+            cpu: {
+                userPct: this._cpuPct.userPct,
+                systemPct: this._cpuPct.systemPct,
+                userMicros: cpuTotal.user,
+                systemMicros: cpuTotal.system,
+                resource,
+            },
+            eventLoop,
+            gc: {
+                totalCount: this._gc.totalCount,
+                totalTimeMs: Math.round(this._gc.totalTimeMs * 100) / 100,
+                maxPauseMs: Math.round(this._gc.maxPauseMs * 100) / 100,
+                byKind: this._gc.byKind,
+                lastPauses: this._gc.lastPauses.slice(-20),
+            },
+            fileDescriptors: {
+                open: fdOpen,
+                limitSoft: fdSoft ? this._parseLimitValue(fdSoft) : null,
+                limitHard: fdHard ? this._parseLimitValue(fdHard) : null,
+            },
+            io,
+            limits: limitsParsed,
+        };
+    }
+    static async _readProcStatus() {
+        try {
+            const txt = await promises_1.default.readFile(`/proc/${process.pid}/status`, "utf-8");
+            const keys = ["VmPeak", "VmSize", "VmHWM", "VmRSS", "VmData", "VmStk", "VmExe", "VmSwap", "Threads"];
+            const result = {};
+            for (const line of txt.split("\n")) {
+                for (const k of keys) {
+                    if (line.startsWith(k + ":")) {
+                        const m = line.match(new RegExp(`${k}:\\s+(\\d+)`));
+                        if (m) {
+                            const v = parseInt(m[1], 10);
+                            result[k] = k === "Threads" ? v : v * 1024;
+                        }
+                    }
+                }
+            }
+            return result;
+        }
+        catch {
+            return null;
+        }
+    }
+    static async _readProcIo() {
+        try {
+            const txt = await promises_1.default.readFile(`/proc/${process.pid}/io`, "utf-8");
+            const result = {};
+            for (const line of txt.split("\n")) {
+                const m = line.match(/^(\w+):\s+(\d+)$/);
+                if (m)
+                    result[m[1]] = parseInt(m[2], 10);
+            }
+            return result;
+        }
+        catch {
+            return null;
+        }
+    }
+    static async _readProcLimits() {
+        try {
+            const txt = await promises_1.default.readFile(`/proc/${process.pid}/limits`, "utf-8");
+            const result = {};
+            const lines = txt.split("\n").slice(1);
+            for (const line of lines) {
+                const m = line.match(/^(.+?)\s{2,}(\S+)\s+(\S+)\s+(\S+)\s*$/);
+                if (m)
+                    result[m[1].trim()] = { soft: m[2], hard: m[3], unit: m[4] };
+            }
+            return result;
+        }
+        catch {
+            return null;
+        }
+    }
+    static async _countFds() {
+        try {
+            const entries = await promises_1.default.readdir(`/proc/${process.pid}/fd`);
+            return entries.length;
+        }
+        catch {
+            return null;
+        }
+    }
+    static _parseLimitValue(v) {
+        if (v === "unlimited")
+            return -1;
+        const n = parseInt(v, 10);
+        return Number.isFinite(n) ? n : null;
+    }
+    static _toMsFinite(ns) {
+        return Number.isFinite(ns) ? Math.round((ns / 1e6) * 100) / 100 : 0;
+    }
+    static _safePercentileMs(h, p) {
+        try {
+            return this._toMsFinite(h.percentile(p));
+        }
+        catch {
+            return 0;
+        }
+    }
+    static _safeCount(fn) {
+        try {
+            return fn() ?? 0;
+        }
+        catch {
+            return 0;
+        }
+    }
+}
+exports.ProcessMonitor = ProcessMonitor;
+ProcessMonitor.init();

package/dist/apiServer/monitor/SystemInfo.d.ts

@@ -0,0 +1,71 @@
+/// <reference types="node" />
+export declare function collectSystemInfo(): Promise<{
+    cpu: {
+        usage: {
+            userPct: number;
+            systemPct: number;
+            idlePct: number;
+            iowaitPct: number;
+        } | null;
+        count: number;
+        models: Record<string, number>;
+        loadAvg: number[];
+        platform: NodeJS.Platform;
+        arch: NodeJS.Architecture;
+        uptimeSec: number;
+    };
+    memory: {
+        total: number;
+        free: number;
+        used: number;
+        usedPct: number;
+    };
+    disk: {
+        filesystem: string;
+        size: number;
+        used: number;
+        available: number;
+        usePct: string;
+        mount: string;
+    }[] | null;
+    connections: {
+        ports: number[];
+        total: number;
+        topRemote: {
+            ip: string;
+            count: number;
+        }[];
+    } | null;
+    nginx: {
+        installed: boolean;
+        version?: undefined;
+        status?: undefined;
+        listening?: undefined;
+        workerCount?: undefined;
+    } | {
+        installed: boolean;
+        version: string;
+        status: string;
+        listening: string[];
+        workerCount: number;
+    } | {
+        installed: boolean;
+        error: string;
+    };
+    pm2: {
+        installed: boolean;
+        error?: undefined;
+        processes?: undefined;
+    } | {
+        installed: boolean;
+        error: string;
+        processes?: undefined;
+    } | {
+        installed: boolean;
+        processes: any;
+        error?: undefined;
+    } | {
+        installed: boolean;
+        error: string;
+    };
+}>;