@daemonitor/plugins 1.1.28 → 1.1.31

package/dist/plugins/index.js

@@ -10,6 +10,7 @@ import { createNetScanPlugin } from "./monitoring/NetScanPlugin.js";
 import { createSnmpPlugin } from "./monitoring/SnmpPlugin.js";
 import { createThreeCXPlugin } from "./monitoring/ThreeCXPlugin.js";
 import { createTplinkPlugin } from "./monitoring/TplinkPlugin.js";
+import { createPatroniPlugin } from "./monitoring/PatroniPlugin.js";
 // In future, implement other plugin factory functions using the same pattern
 // import { createOSPlugin } from "./monitoring/OSPlugin.js"
 // import { createMongoDBPlugin } from "./monitoring/MongoDBPlugin.js"
@@ -31,6 +32,7 @@ export default {
     Snmp: createSnmpPlugin,
     ThreeCX: createThreeCXPlugin,
     Tplink: createTplinkPlugin,
+    Patroni: createPatroniPlugin,
     // CloudflarePlugin: createCloudflarePlugin,
     // EwelinkPlugin: createEwelinkPlugin,
     // MongoDBPlugin: createMongoDBPlugin,

package/dist/plugins/monitoring/DockerPlugin.js

@@ -2,6 +2,7 @@ import { createMonitoringPlugin } from "../../lib/MonitoringPlugin.js";
 import { exec } from "child_process";
 import { promisify } from "util";
 import { hostname } from "os";
+import { readFile } from "fs/promises";
 const execAsync = promisify(exec);
 const EXEC_OPTS = { maxBuffer: 32 * 1024 * 1024 }; // containers.length can be large
 /**
@@ -38,9 +39,78 @@ export function cleanHealthOutput(raw) {
     return stripped.length > 300 ? "…" + stripped.slice(-300) : stripped;
 }
 const HEALTH_RE = /\((healthy|unhealthy|health: starting|starting)\)/i;
-// Previous cumulative NetIO/BlockIO totals per container id, to derive a
-// per-second rate across polls. Module-level (one collect per host per cycle).
 const ioPrev = {};
+/** Sum of rbytes/wbytes across every device in a cgroup v2 io.stat. */
+export function parseIoStat(text) {
+    let a = 0, b = 0, seen = false;
+    for (const line of text.split("\n")) {
+        const r = /\brbytes=(\d+)/.exec(line);
+        const w = /\bwbytes=(\d+)/.exec(line);
+        if (r || w)
+            seen = true;
+        if (r)
+            a += Number(r[1]);
+        if (w)
+            b += Number(w[1]);
+    }
+    return seen ? { a, b } : (text.trim() === "" ? { a: 0, b: 0 } : null);
+}
+/** rx/tx bytes summed over a /proc/<pid>/net/dev, loopback excluded. */
+export function parseNetDev(text) {
+    let a = 0, b = 0, seen = false;
+    for (const line of text.split("\n")) {
+        const m = /^\s*([^:\s]+):\s*(.*)$/.exec(line);
+        if (!m || m[1] === "lo")
+            continue;
+        const f = m[2].trim().split(/\s+/).map(Number);
+        // receive: bytes packets errs drop fifo frame compressed multicast | transmit: bytes …
+        if (f.length < 9 || !Number.isFinite(f[0]) || !Number.isFinite(f[8]))
+            continue;
+        a += f[0];
+        b += f[8];
+        seen = true;
+    }
+    return seen ? { a, b } : null;
+}
+/** The cgroup v2 path in a /proc/<pid>/cgroup ("0::/system.slice/docker-….scope"), or null on v1. */
+export function cgroupV2Path(text) {
+    const m = /^0::(\/.*)$/m.exec(text);
+    return m ? m[1].trim() : null;
+}
+/**
+ * Byte rate between two cumulative readings. Undefined when there is no usable
+ * previous reading: none yet, a different source, or the counter went backwards
+ * (container restarted, counters reset).
+ */
+export function ioRate(cur, prev, dt) {
+    if (prev == null || dt <= 0 || cur < prev)
+        return undefined;
+    return Math.round((cur - prev) / dt);
+}
+/**
+ * Exact cumulative counters for one container, read from the kernel rather than
+ * `docker stats`, which prints three significant figures. At "17.8GB" one step
+ * is 100MB, so the delta between two 30s polls was almost always zero.
+ *
+ * Needs the host pid (from inspect). Works for docker and podman, systemd or
+ * cgroupfs driver, because the cgroup path comes from the process itself.
+ * Either half is null when it can't be read (cgroup v1, a hidepid /proc); the
+ * caller falls back to the stats strings for that half.
+ */
+async function readExactIo(pid, hostNet) {
+    if (!(pid > 0))
+        return { net: null, blk: null };
+    const read = (p) => readFile(p, "utf8").catch(() => null);
+    const [cg, dev] = await Promise.all([read(`/proc/${pid}/cgroup`), hostNet ? Promise.resolve(null) : read(`/proc/${pid}/net/dev`)]);
+    const path = cg ? cgroupV2Path(cg) : null;
+    const io = path ? await read(`/sys/fs/cgroup${path}/io.stat`) : null;
+    return {
+        // Host networking shares the host's interfaces; reading them would charge
+        // all host traffic to this container. docker stats reports 0 for it, and so do we.
+        net: hostNet ? { a: 0, b: 0 } : (dev ? parseNetDev(dev) : null),
+        blk: io != null ? parseIoStat(io) : null,
+    };
+}
 // docker stats renders "12MB / 7.02MB" (rx / tx, or read / write). Split + convert.
 function pairToBytes(value) {
     const [x, y] = (value || "").split("/").map((p) => p.trim());
@@ -65,7 +135,7 @@ function memToBytes(value) {
  *   2. `docker stats`      — live CPU / memory for running containers (one batch)
  *   3. `docker inspect`    — restart count, health, exit code (one batch)
  */
-async function collectContainers(bin) {
+export async function collectContainers(bin) {
     const psFmt = '{{.ID}}|{{.Names}}|{{.Image}}|{{.State}}|{{.Status}}|{{.Label "com.docker.compose.project"}}|{{.Label "com.docker.compose.service"}}';
     const { stdout: psOut } = await execAsync(`${bin} ps -a --format '${psFmt}'`, EXEC_OPTS);
     const containers = [];
@@ -94,12 +164,12 @@ async function collectContainers(bin) {
     if (!containers.length)
         return containers;
     // 2. live stats for running containers (one batched call, best-effort).
-    // NetIO/BlockIO are CUMULATIVE since container start, so we derive a per-second
-    // rate from the delta vs the previous poll (first poll → no rate).
+    // NetIO/BlockIO are CUMULATIVE since container start; they are kept here only
+    // as the fallback for counters the kernel won't give us (see readExactIo), and
+    // the rate is derived after inspect, once the pid is known.
+    const statsIo = new Map();
     try {
         const { stdout: statsOut } = await execAsync(`${bin} stats --no-stream --format '{{.ID}}|{{.CPUPerc}}|{{.MemUsage}}|{{.MemPerc}}|{{.NetIO}}|{{.BlockIO}}'`, EXEC_OPTS);
-        const now = Date.now();
-        const seen = new Set();
         for (const line of statsOut.split("\n")) {
             if (!line.trim())
                 continue;
@@ -112,28 +182,8 @@ async function collectContainers(bin) {
             c.mem = memToBytes(used);
             c.memLimit = memToBytes(limit);
             c.memPercent = parseFloat((memPerc || "").replace("%", "")) || 0;
-            const net = pairToBytes(netIO); // { a: rx, b: tx }
-            const blk = pairToBytes(blkIO); // { a: read, b: write }
-            seen.add(id);
-            const prev = ioPrev[id];
-            if (prev && now > prev.ts) {
-                const dt = (now - prev.ts) / 1000;
-                // Guard counter resets (container restart): negative delta → skip that field.
-                if (net.a >= prev.rx)
-                    c.netIn = Math.round((net.a - prev.rx) / dt);
-                if (net.b >= prev.tx)
-                    c.netOut = Math.round((net.b - prev.tx) / dt);
-                if (blk.a >= prev.rd)
-                    c.blkRead = Math.round((blk.a - prev.rd) / dt);
-                if (blk.b >= prev.wr)
-                    c.blkWrite = Math.round((blk.b - prev.wr) / dt);
-            }
-            ioPrev[id] = { rx: net.a, tx: net.b, rd: blk.a, wr: blk.b, ts: now };
+            statsIo.set(id, { net: pairToBytes(netIO), blk: pairToBytes(blkIO) });
         }
-        // Drop prev samples for containers no longer present.
-        for (const k of Object.keys(ioPrev))
-            if (!seen.has(k))
-                delete ioPrev[k];
     }
     catch (err) {
         console.error("docker: stats collection failed:", err.message);
@@ -143,24 +193,52 @@ async function collectContainers(bin) {
     // template errors with `map has no entry for key "Health"` for containers
     // without a healthcheck, which would fail the whole batch. Health is parsed
     // from the ps `Status` string ("(healthy)" / "(unhealthy)") instead.
+    // Pid + network mode ride the same call, for the exact IO counters.
+    const procInfo = new Map();
     try {
         const ids = containers.map((c) => c.id).join(" ");
-        const inspectFmt = '{{.Id}}|{{.RestartCount}}|{{.State.ExitCode}}';
+        const inspectFmt = '{{.Id}}|{{.RestartCount}}|{{.State.ExitCode}}|{{.State.Pid}}|{{.HostConfig.NetworkMode}}';
         const { stdout: inspOut } = await execAsync(`${bin} inspect --format '${inspectFmt}' ${ids}`, EXEC_OPTS);
         for (const line of inspOut.split("\n")) {
             if (!line.trim())
                 continue;
-            const [fullId, restarts, exitCode] = line.split("|");
+            const [fullId, restarts, exitCode, pid, netMode] = line.split("|");
             const c = byId.get(fullId.slice(0, 12));
             if (!c)
                 continue;
             c.restarts = parseInt(restarts, 10) || 0;
             c.exitCode = parseInt(exitCode, 10) || 0;
+            procInfo.set(c.id, { pid: parseInt(pid, 10) || 0, hostNet: (netMode || "").trim() === "host" });
         }
     }
     catch (err) {
         console.error("docker: inspect collection failed:", err.message);
     }
+    // 3b. Per-second IO rates: exact kernel counters where readable, the rounded
+    // stats strings otherwise. Only containers docker stats saw (i.e. running).
+    const now = Date.now();
+    await Promise.all([...statsIo].map(async ([id, fromStats]) => {
+        const c = byId.get(id);
+        if (!c)
+            return;
+        const info = procInfo.get(id);
+        const exact = info ? await readExactIo(info.pid, info.hostNet) : { net: null, blk: null };
+        const net = exact.net ? { ...exact.net, src: "exact" } : { ...fromStats.net, src: "stats" };
+        const blk = exact.blk ? { ...exact.blk, src: "exact" } : { ...fromStats.blk, src: "stats" };
+        const prev = ioPrev[id];
+        const dt = prev ? (now - prev.ts) / 1000 : 0;
+        const pn = prev?.net?.src === net.src ? prev.net : undefined;
+        const pb = prev?.blk?.src === blk.src ? prev.blk : undefined;
+        c.netIn = ioRate(net.a, pn?.a, dt);
+        c.netOut = ioRate(net.b, pn?.b, dt);
+        c.blkRead = ioRate(blk.a, pb?.a, dt);
+        c.blkWrite = ioRate(blk.b, pb?.b, dt);
+        ioPrev[id] = { net, blk, ts: now };
+    }));
+    // Drop prev samples for containers no longer present.
+    for (const k of Object.keys(ioPrev))
+        if (!statsIo.has(k))
+            delete ioPrev[k];
     // 4. For UNHEALTHY containers only, read the healthcheck's own last output.
     //
     // "1/13 containers unhealthy" says a problem exists and nothing about what it

package/dist/plugins/monitoring/PatroniPlugin.js

@@ -0,0 +1,169 @@
+import { hostname } from "node:os";
+import { createMonitoringPlugin } from "../../lib/MonitoringPlugin.js";
+const DEFAULTS = {
+    url: "http://localhost:8008",
+    lagWarnBytes: 32 * 1024 * 1024, // 32MB: a busy node catching up
+    lagCritBytes: 256 * 1024 * 1024, // 256MB: not keeping up at all
+    requestTimeout: 5000,
+};
+/**
+ * Roles Patroni reports for a member that is holding the primary. `master` is
+ * the pre-3.0 spelling and still shows up on older clusters; `standby_leader` is
+ * the leader of a standby cluster, which is a primary as far as placement is
+ * concerned even though it is replicating from elsewhere.
+ */
+const LEADER_ROLES = new Set(["leader", "master", "primary", "standby_leader"]);
+export function isLeaderRole(role) {
+    return LEADER_ROLES.has(String(role || "").toLowerCase());
+}
+async function getJson(url, cfg) {
+    const headers = { Accept: "application/json" };
+    if (cfg.username) {
+        const raw = `${cfg.username}:${cfg.password ?? ""}`;
+        headers.Authorization = `Basic ${Buffer.from(raw).toString("base64")}`;
+    }
+    const res = await fetch(url, {
+        headers,
+        signal: AbortSignal.timeout(cfg.requestTimeout ?? DEFAULTS.requestTimeout),
+    });
+    // Patroni answers /patroni with 503 on a node that is up but not healthy (a
+    // replica that has lost its leader, a node in maintenance). The BODY is still
+    // the node's state, and that state is the whole reason to ask, so read it
+    // rather than treating the status code as a failure.
+    const text = await res.text();
+    let body = null;
+    try {
+        body = text ? JSON.parse(text) : null;
+    }
+    catch {
+        throw new Error(`non-JSON response from ${url} (HTTP ${res.status})`);
+    }
+    if (!body && !res.ok)
+        throw new Error(`HTTP ${res.status} from ${url}`);
+    return body;
+}
+/**
+ * Bytes this member is behind the leader.
+ *
+ * /cluster reports `lag` per member and it is already in bytes, but it comes
+ * back as the string "unknown" on a member the DCS cannot currently see. That
+ * must stay distinct from 0: "unknown" means no information, and coercing it to
+ * zero reports a node nobody can reach as perfectly caught up.
+ */
+function lagOf(member) {
+    const raw = member?.lag;
+    if (raw == null)
+        return null;
+    const n = Number(raw);
+    return Number.isFinite(n) ? n : null;
+}
+function normaliseMembers(cluster) {
+    const list = Array.isArray(cluster?.members) ? cluster.members : [];
+    return list.map((m) => ({
+        name: String(m?.name ?? "unknown"),
+        role: String(m?.role ?? "unknown").toLowerCase(),
+        state: String(m?.state ?? "unknown").toLowerCase(),
+        host: m?.host ?? undefined,
+        port: typeof m?.port === "number" ? m.port : undefined,
+        apiUrl: m?.api_url ?? undefined,
+        timeline: typeof m?.timeline === "number" ? m.timeline : undefined,
+        lagBytes: lagOf(m),
+    }));
+}
+export async function collectPatroni(cfg) {
+    const base = String(cfg.url || DEFAULTS.url).replace(/\/+$/, "");
+    // Ask for both, and do not let one failure hide the other. A node whose own
+    // /patroni is 503-ing can still return a perfectly good /cluster, and that is
+    // the reading that says whether the cluster as a whole has a leader.
+    const [selfRes, clusterRes] = await Promise.allSettled([
+        getJson(`${base}/patroni`, cfg),
+        getJson(`${base}/cluster`, cfg),
+    ]);
+    if (selfRes.status === "rejected" && clusterRes.status === "rejected") {
+        throw new Error(String(selfRes.reason?.message || selfRes.reason));
+    }
+    const self = selfRes.status === "fulfilled" ? selfRes.value : null;
+    const cluster = clusterRes.status === "fulfilled" ? clusterRes.value : null;
+    const members = normaliseMembers(cluster);
+    const scope = self?.patroni?.scope || cluster?.scope || undefined;
+    // Patroni's member name defaults to the hostname but is configurable, and the
+    // server matches leader-by-name against it, so report what Patroni calls this
+    // node rather than what the OS does.
+    const member = self?.patroni?.name || self?.name || hostname();
+    const selfRole = String(self?.role ?? "unknown").toLowerCase();
+    const leaderMember = members.find((m) => isLeaderRole(m.role));
+    const selfMember = members.find((m) => m.name === member);
+    return {
+        kind: "patroni",
+        name: scope || "patroni",
+        scope,
+        member,
+        reachable: true,
+        // Self-reported role. The server trusts this over any name matching: a node
+        // saying "I am the leader" identifies the leader's SYSTEM exactly, with no
+        // guessing about which host a Patroni member name belongs to.
+        role: selfRole,
+        isLeader: isLeaderRole(selfRole),
+        state: String(self?.state ?? "unknown").toLowerCase(),
+        timeline: self?.timeline ?? selfMember?.timeline ?? null,
+        serverVersion: self?.server_version ?? null,
+        patroniVersion: self?.patroni?.version ?? null,
+        // A cluster with no leader is either mid-election (seconds) or stuck
+        // (forever). The server applies a grace window rather than paging on the gap.
+        leader: leaderMember?.name ?? null,
+        hasLeader: !!leaderMember,
+        // Patroni sets this while the DCS lock is not held by anyone.
+        clusterUnlocked: self?.cluster_unlocked === true,
+        // Maintenance mode: failover is disabled, so a dead leader will NOT be
+        // replaced. Worth surfacing because everything else looks normal.
+        paused: self?.pause === true,
+        pendingRestart: self?.pending_restart === true,
+        lagBytes: selfMember?.lagBytes ?? null,
+        lagWarnBytes: cfg.lagWarnBytes ?? DEFAULTS.lagWarnBytes,
+        lagCritBytes: cfg.lagCritBytes ?? DEFAULTS.lagCritBytes,
+        members,
+        memberCount: members.length,
+        runningCount: members.filter((m) => m.state === "running" || m.state === "streaming").length,
+        // Set when /patroni answered but /cluster did not, or vice versa: the report
+        // is partial and the server should not read a missing leader as no leader.
+        partial: !self || !cluster,
+        timestamp: Date.now(),
+    };
+}
+export function createPatroniPlugin() {
+    let refreshTimer = null;
+    const refreshFn = async (plugin) => {
+        const cfg = (plugin.config || {});
+        const uid = cfg.uniqueId || "patroni";
+        try {
+            await plugin.send(await collectPatroni(cfg), uid);
+        }
+        catch (e) {
+            // Report the failure rather than going quiet. Silence from this plugin is
+            // indistinguishable from the agent being dead, and "Patroni's API is not
+            // answering on this node" is itself a finding worth an alert.
+            await plugin.send({
+                kind: "patroni",
+                name: "patroni",
+                member: hostname(),
+                reachable: false,
+                error: String(e?.message || e).slice(0, 200),
+                timestamp: Date.now(),
+            }, uid);
+        }
+    };
+    const monitorFn = async (plugin) => {
+        await refreshFn(plugin);
+        // 30s by default rather than the usual 60s: this is the input to leader
+        // placement, so the window in which daemonitor believes the old leader is
+        // still the leader is bounded by this interval.
+        refreshTimer = setInterval(() => refreshFn(plugin), plugin.config?.refreshInterval || 30000);
+    };
+    const teardownFn = async () => {
+        if (refreshTimer)
+            clearInterval(refreshTimer);
+        refreshTimer = null;
+    };
+    return createMonitoringPlugin("patroni", "patroni", "Patroni HA Postgres monitor (leader identity, member roles, replication lag)", async () => { }, monitorFn, refreshFn, teardownFn);
+}
+export default createPatroniPlugin;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@daemonitor/plugins",
-  "version": "1.1.28",
+  "version": "1.1.31",
   "description": "daemonitor plugins",
   "types": "types/index.d.ts",
   "main": "dist/index.js",