@glassmkr/crucible 0.6.4 → 0.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +13 -0
- package/.github/workflows/docker.yml +53 -0
- package/Dockerfile +59 -0
- package/README.md +46 -2
- package/dist/collect/network.js +90 -3
- package/dist/collect/network.js.map +1 -1
- package/dist/lib/types.d.ts +13 -0
- package/docker-compose.yml +26 -0
- package/package.json +1 -1
- package/src/collect/network.ts +86 -3
- package/src/lib/types.ts +13 -0
package/.dockerignore
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
name: Build and publish Docker image
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
tags: ['v*']
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
packages: write
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
docker:
|
|
14
|
+
name: Build and push to ghcr.io and docker.io
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
|
|
19
|
+
- uses: docker/setup-buildx-action@v3
|
|
20
|
+
|
|
21
|
+
- uses: docker/login-action@v3
|
|
22
|
+
with:
|
|
23
|
+
registry: ghcr.io
|
|
24
|
+
username: ${{ github.actor }}
|
|
25
|
+
password: ${{ secrets.GITHUB_TOKEN }}
|
|
26
|
+
|
|
27
|
+
- uses: docker/login-action@v3
|
|
28
|
+
with:
|
|
29
|
+
registry: docker.io
|
|
30
|
+
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
31
|
+
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
32
|
+
|
|
33
|
+
- uses: docker/metadata-action@v5
|
|
34
|
+
id: meta
|
|
35
|
+
with:
|
|
36
|
+
images: |
|
|
37
|
+
ghcr.io/glassmkr/crucible
|
|
38
|
+
docker.io/glassmkr/crucible
|
|
39
|
+
tags: |
|
|
40
|
+
type=semver,pattern={{version}}
|
|
41
|
+
type=semver,pattern={{major}}.{{minor}}
|
|
42
|
+
type=raw,value=latest,enable={{is_default_branch}}
|
|
43
|
+
type=sha,prefix=sha-,format=short
|
|
44
|
+
|
|
45
|
+
- uses: docker/build-push-action@v6
|
|
46
|
+
with:
|
|
47
|
+
context: .
|
|
48
|
+
platforms: linux/amd64
|
|
49
|
+
push: true
|
|
50
|
+
tags: ${{ steps.meta.outputs.tags }}
|
|
51
|
+
labels: ${{ steps.meta.outputs.labels }}
|
|
52
|
+
cache-from: type=gha
|
|
53
|
+
cache-to: type=gha,mode=max
|
package/Dockerfile
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# syntax=docker/dockerfile:1.7
|
|
2
|
+
# Multi-stage build for Glassmkr Crucible monitoring agent.
|
|
3
|
+
|
|
4
|
+
# ---------- Stage 1: build TypeScript to dist/ ----------
|
|
5
|
+
FROM node:24-slim AS builder
|
|
6
|
+
WORKDIR /build
|
|
7
|
+
COPY package.json package-lock.json* ./
|
|
8
|
+
RUN npm install --include=dev --no-audit --no-fund
|
|
9
|
+
COPY tsconfig.json ./
|
|
10
|
+
COPY src ./src
|
|
11
|
+
RUN npm run build
|
|
12
|
+
|
|
13
|
+
# ---------- Stage 2: production runtime ----------
|
|
14
|
+
FROM node:24-slim AS runtime
|
|
15
|
+
|
|
16
|
+
# Hardware monitoring tools. Crucible shells out to these; they must be on PATH.
|
|
17
|
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
18
|
+
smartmontools \
|
|
19
|
+
ipmitool \
|
|
20
|
+
dmidecode \
|
|
21
|
+
lm-sensors \
|
|
22
|
+
ethtool \
|
|
23
|
+
util-linux \
|
|
24
|
+
procps \
|
|
25
|
+
net-tools \
|
|
26
|
+
iproute2 \
|
|
27
|
+
ca-certificates \
|
|
28
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
29
|
+
|
|
30
|
+
WORKDIR /app
|
|
31
|
+
|
|
32
|
+
# Production node_modules only.
|
|
33
|
+
COPY package.json package-lock.json* ./
|
|
34
|
+
RUN npm ci --omit=dev --no-audit --no-fund && npm cache clean --force
|
|
35
|
+
|
|
36
|
+
# Built code.
|
|
37
|
+
COPY --from=builder /build/dist ./dist
|
|
38
|
+
|
|
39
|
+
# Create a non-root user for future use. IPMI and SMART typically require root,
|
|
40
|
+
# so the container is expected to run with --privileged or cap_add DAC_READ_SEARCH etc.
|
|
41
|
+
# Keeping the user available lets operators drop privileges when hardware access is not needed.
|
|
42
|
+
RUN useradd --system --no-create-home --shell /usr/sbin/nologin glassmkr
|
|
43
|
+
|
|
44
|
+
# Crucible reads /etc/glassmkr/collector.yaml by default.
|
|
45
|
+
# Mount the host config directory at this path.
|
|
46
|
+
RUN mkdir -p /etc/glassmkr
|
|
47
|
+
|
|
48
|
+
# Container health: verify the Node process is actually running and hasn't crashed.
|
|
49
|
+
HEALTHCHECK --interval=60s --timeout=10s --start-period=30s --retries=3 \
|
|
50
|
+
CMD pgrep -f "node /app/dist/index.js" > /dev/null || exit 1
|
|
51
|
+
|
|
52
|
+
LABEL org.opencontainers.image.source="https://github.com/glassmkr/crucible" \
|
|
53
|
+
org.opencontainers.image.description="Glassmkr Crucible - bare metal server monitoring agent" \
|
|
54
|
+
org.opencontainers.image.licenses="MIT" \
|
|
55
|
+
org.opencontainers.image.title="Crucible" \
|
|
56
|
+
org.opencontainers.image.vendor="Glassmkr"
|
|
57
|
+
|
|
58
|
+
# Crucible does not listen on any port; data flows outbound to Forge.
|
|
59
|
+
ENTRYPOINT ["node", "/app/dist/index.js"]
|
package/README.md
CHANGED
|
@@ -3,10 +3,25 @@
|
|
|
3
3
|
[](LICENSE)
|
|
4
4
|
[](https://www.npmjs.com/package/@glassmkr/crucible)
|
|
5
5
|
|
|
6
|
-
Lightweight bare metal server monitoring agent. Collects hardware and OS health every 5 minutes and pushes snapshots to a [Forge](https://forge.glassmkr.com) dashboard, which evaluates
|
|
6
|
+
Lightweight bare metal server monitoring agent. Collects hardware and OS health every 5 minutes and pushes snapshots to a [Forge](https://forge.glassmkr.com) dashboard, which evaluates 38 alert rules and sends notifications.
|
|
7
7
|
|
|
8
8
|
Open source. MIT licensed. Built by [Glassmkr](https://glassmkr.com). See also [Bench](https://github.com/glassmkr/bench), the MCP server collection.
|
|
9
9
|
|
|
10
|
+
**Resource usage:** ~90MB RSS memory (varies by hardware: servers with more IPMI sensors use more), <0.1% CPU at 5-minute collection interval. Collects IPMI, SMART, ZFS, network bonds, security posture, conntrack, systemd, NTP, and file descriptors.
|
|
11
|
+
|
|
12
|
+
**Security:** See [glassmkr.com/security](https://glassmkr.com/security) for the full list of what Crucible does and does not collect.
|
|
13
|
+
|
|
14
|
+
## Screenshots
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
*Alerts grouped by server, with AI-generated fix commands for each rule.*
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
*Per-disk SMART status, storage capacity, and network interface bonding.*
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
*Security posture, server overview, and active alerts.*
|
|
24
|
+
|
|
10
25
|
## Install
|
|
11
26
|
|
|
12
27
|
```bash
|
|
@@ -19,6 +34,35 @@ Or use the bootstrap script:
|
|
|
19
34
|
curl -sf https://forge.glassmkr.com/install | bash
|
|
20
35
|
```
|
|
21
36
|
|
|
37
|
+
## Docker
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
# Create config directory
|
|
41
|
+
sudo mkdir -p /etc/glassmkr
|
|
42
|
+
|
|
43
|
+
# Create config (replace with your Forge credentials)
|
|
44
|
+
sudo tee /etc/glassmkr/collector.yaml << 'EOF'
|
|
45
|
+
server_name: "web-01"
|
|
46
|
+
collection:
|
|
47
|
+
interval_seconds: 300
|
|
48
|
+
ipmi: true
|
|
49
|
+
smart: true
|
|
50
|
+
forge:
|
|
51
|
+
enabled: true
|
|
52
|
+
url: "https://forge.glassmkr.com"
|
|
53
|
+
api_key: "col_YOUR_KEY_HERE"
|
|
54
|
+
EOF
|
|
55
|
+
|
|
56
|
+
# Run with docker compose
|
|
57
|
+
curl -O https://raw.githubusercontent.com/glassmkr/crucible/main/docker-compose.yml
|
|
58
|
+
docker compose up -d
|
|
59
|
+
|
|
60
|
+
# Check logs
|
|
61
|
+
docker compose logs -f crucible
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Images are published to [ghcr.io/glassmkr/crucible](https://github.com/glassmkr/crucible/pkgs/container/crucible) on every tag release. The container needs `--privileged` and `network_mode: host` for IPMI, SMART, and accurate host network monitoring. Details in the [compose file](./docker-compose.yml).
|
|
65
|
+
|
|
22
66
|
## Quick Start
|
|
23
67
|
|
|
24
68
|
1. Create an API key in the Forge dashboard (Servers, then Add server).
|
|
@@ -104,7 +148,7 @@ sudo systemctl status glassmkr-crucible
|
|
|
104
148
|
| NTP | Sync state and source |
|
|
105
149
|
| File descriptors | System-wide allocation |
|
|
106
150
|
|
|
107
|
-
Forge evaluates
|
|
151
|
+
Forge evaluates 38 alert rules server-side across OS, Storage, Network, Hardware, ZFS, Security, and Service Health, with priorities P1 Urgent through P4 Low. Full list: [forge.glassmkr.com/docs/alerts](https://forge.glassmkr.com/docs/alerts).
|
|
108
152
|
|
|
109
153
|
## Requirements
|
|
110
154
|
|
package/dist/collect/network.js
CHANGED
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
import { readProcFile, sleep } from "../lib/parse.js";
|
|
2
|
-
import { readFileSync } from "fs";
|
|
2
|
+
import { readFileSync, readdirSync } from "fs";
|
|
3
3
|
const previousCounters = new Map();
|
|
4
|
+
function readStatCounter(iface, name) {
|
|
5
|
+
try {
|
|
6
|
+
const raw = readFileSync(`/sys/class/net/${iface}/statistics/${name}`, "utf-8").trim();
|
|
7
|
+
const val = parseInt(raw, 10);
|
|
8
|
+
return Number.isFinite(val) ? val : undefined;
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return undefined;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
4
14
|
function parseNetDev() {
|
|
5
15
|
const raw = readProcFile("/proc/net/dev") || "";
|
|
6
16
|
const result = {};
|
|
@@ -30,6 +40,36 @@ function getSpeed(iface) {
|
|
|
30
40
|
return 0;
|
|
31
41
|
}
|
|
32
42
|
}
|
|
43
|
+
function getOperstate(iface) {
|
|
44
|
+
try {
|
|
45
|
+
return readFileSync(`/sys/class/net/${iface}/operstate`, "utf-8").trim();
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
return "unknown";
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
function getBondMaster(iface) {
|
|
52
|
+
try {
|
|
53
|
+
const bonds = readdirSync("/proc/net/bonding/");
|
|
54
|
+
for (const bond of bonds) {
|
|
55
|
+
const content = readFileSync(`/proc/net/bonding/${bond}`, "utf-8");
|
|
56
|
+
if (content.includes(`Slave Interface: ${iface}`))
|
|
57
|
+
return bond;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
// No bonds or /proc/net/bonding doesn't exist
|
|
62
|
+
}
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
function isBondMaster(iface) {
|
|
66
|
+
try {
|
|
67
|
+
return readdirSync("/proc/net/bonding/").includes(iface);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
33
73
|
// Compute delta, handling counter wraps (current < previous means reset, use current as delta)
|
|
34
74
|
function delta(current, previous) {
|
|
35
75
|
if (current >= previous)
|
|
@@ -48,16 +88,39 @@ export async function collectNetwork() {
|
|
|
48
88
|
continue;
|
|
49
89
|
currentIfaces.add(name);
|
|
50
90
|
const prev = previousCounters.get(name);
|
|
91
|
+
// /sys/class/net/*/statistics/ exposes finer-grained RX/TX subtype
|
|
92
|
+
// counters than /proc/net/dev. Read cumulative values here; delta is
|
|
93
|
+
// derived below against the previous cycle's snapshot.
|
|
94
|
+
const rxCrcCum = readStatCounter(name, "rx_crc_errors");
|
|
95
|
+
const rxFrameCum = readStatCounter(name, "rx_frame_errors");
|
|
96
|
+
const rxLenCum = readStatCounter(name, "rx_length_errors");
|
|
97
|
+
const txCarrierCum = readStatCounter(name, "tx_carrier_errors");
|
|
51
98
|
// Compute error/drop deltas (0 on first cycle after start or new interface)
|
|
52
99
|
let rxErrorsDelta = 0;
|
|
53
100
|
let txErrorsDelta = 0;
|
|
54
101
|
let rxDropsDelta = 0;
|
|
55
102
|
let txDropsDelta = 0;
|
|
103
|
+
let rxPacketsDelta = 0;
|
|
104
|
+
let txPacketsDelta = 0;
|
|
105
|
+
let rxCrcDelta;
|
|
106
|
+
let rxFrameDelta;
|
|
107
|
+
let rxLenDelta;
|
|
108
|
+
let txCarrierDelta;
|
|
56
109
|
if (prev) {
|
|
57
110
|
rxErrorsDelta = delta(s2.rx_errors, prev.rx_errors);
|
|
58
111
|
txErrorsDelta = delta(s2.tx_errors, prev.tx_errors);
|
|
59
112
|
rxDropsDelta = delta(s2.rx_drops, prev.rx_drops);
|
|
60
113
|
txDropsDelta = delta(s2.tx_drops, prev.tx_drops);
|
|
114
|
+
rxPacketsDelta = delta(s2.rx_packets, prev.rx_packets);
|
|
115
|
+
txPacketsDelta = delta(s2.tx_packets, prev.tx_packets);
|
|
116
|
+
if (rxCrcCum != null && prev.rx_crc_errors != null)
|
|
117
|
+
rxCrcDelta = delta(rxCrcCum, prev.rx_crc_errors);
|
|
118
|
+
if (rxFrameCum != null && prev.rx_frame_errors != null)
|
|
119
|
+
rxFrameDelta = delta(rxFrameCum, prev.rx_frame_errors);
|
|
120
|
+
if (rxLenCum != null && prev.rx_length_errors != null)
|
|
121
|
+
rxLenDelta = delta(rxLenCum, prev.rx_length_errors);
|
|
122
|
+
if (txCarrierCum != null && prev.tx_carrier_errors != null)
|
|
123
|
+
txCarrierDelta = delta(txCarrierCum, prev.tx_carrier_errors);
|
|
61
124
|
}
|
|
62
125
|
// Store current cumulative values for next cycle
|
|
63
126
|
previousCounters.set(name, {
|
|
@@ -65,8 +128,14 @@ export async function collectNetwork() {
|
|
|
65
128
|
tx_errors: s2.tx_errors,
|
|
66
129
|
rx_drops: s2.rx_drops,
|
|
67
130
|
tx_drops: s2.tx_drops,
|
|
131
|
+
rx_packets: s2.rx_packets,
|
|
132
|
+
tx_packets: s2.tx_packets,
|
|
133
|
+
rx_crc_errors: rxCrcCum,
|
|
134
|
+
rx_frame_errors: rxFrameCum,
|
|
135
|
+
rx_length_errors: rxLenCum,
|
|
136
|
+
tx_carrier_errors: txCarrierCum,
|
|
68
137
|
});
|
|
69
|
-
|
|
138
|
+
const entry = {
|
|
70
139
|
interface: name,
|
|
71
140
|
speed_mbps: getSpeed(name),
|
|
72
141
|
rx_bytes_sec: s2.rx_bytes - s1.rx_bytes, // already a 1-second delta
|
|
@@ -75,7 +144,25 @@ export async function collectNetwork() {
|
|
|
75
144
|
tx_errors: txErrorsDelta,
|
|
76
145
|
rx_drops: rxDropsDelta,
|
|
77
146
|
tx_drops: txDropsDelta,
|
|
78
|
-
|
|
147
|
+
rx_packets: rxPacketsDelta,
|
|
148
|
+
tx_packets: txPacketsDelta,
|
|
149
|
+
operstate: getOperstate(name),
|
|
150
|
+
};
|
|
151
|
+
if (rxCrcDelta !== undefined)
|
|
152
|
+
entry.rx_crc_errors = rxCrcDelta;
|
|
153
|
+
if (rxFrameDelta !== undefined)
|
|
154
|
+
entry.rx_frame_errors = rxFrameDelta;
|
|
155
|
+
if (rxLenDelta !== undefined)
|
|
156
|
+
entry.rx_length_errors = rxLenDelta;
|
|
157
|
+
if (txCarrierDelta !== undefined)
|
|
158
|
+
entry.tx_carrier_errors = txCarrierDelta;
|
|
159
|
+
const master = getBondMaster(name);
|
|
160
|
+
if (master)
|
|
161
|
+
entry.bond_master = master;
|
|
162
|
+
// Identify bond masters (have at least one slave pointing at them).
|
|
163
|
+
if (isBondMaster(name))
|
|
164
|
+
entry.is_bond_master = true;
|
|
165
|
+
results.push(entry);
|
|
79
166
|
}
|
|
80
167
|
// Remove stale interfaces that disappeared
|
|
81
168
|
for (const name of previousCounters.keys()) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/collect/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"network.js","sourceRoot":"","sources":["../../src/collect/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AAsB/C,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA4B,CAAC;AAE7D,SAAS,eAAe,CAAC,KAAa,EAAE,IAAY;IAClD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,kBAAkB,KAAK,eAAe,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QACvF,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC9B,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,WAAW;IAClB,MAAM,GAAG,GAAG,YAAY,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IAChD,MAAM,MAAM,GAA+B,EAAE,CAAC;IAC9C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC9C,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,0BAA0B;QAC1B,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,SAAS;QAC1I,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACvD,MAAM,CAAC,IAAI,CAAC,GAAG;YACb,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;YACrG,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC;SACxG,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,YAAY,CAAC,kBAAkB,KAAK,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5E,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChC,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,kBAAkB,KAAK,YAAY,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,oBAAoB,CAAC,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,qBAAqB,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;YACnE,IAAI,OAAO,CAAC,QAAQ,CAAC,oBAAoB,KAAK,EAAE,CAAC;gBAAE,OAAO,IAAI,CAAC;QACjE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,8CAA8C;IAChD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,IAAI,CAAC;QACH,OAAO,WAAW,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,+FAA+F;AAC/F,SAAS,KAAK,CAAC,OAAe,EAAE,QAAgB;IAC9C,IAAI,OAAO,IAAI,QAAQ;QAAE,OAAO,OAAO,GAAG,QAAQ,CAAC;IACnD,OAAO,OAAO,CAAC,CAAC,2BAA2B;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAC7B,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IAClB,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAE7B,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IACxC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,CAAC,EAAE;YAAE,SAAS;QAClB,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAExB,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAExC,mEAAmE;QACnE,qEAAqE;QACrE,uDAAuD;QACvD,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;QACxD,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;QAC3D,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAEhE,4EAA4E;QAC5E,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,UAA8B,CAAC;QACnC,IAAI,YAAgC,CAAC;QACrC,IAAI,UAA8B,CAAC;QACnC,IAAI,cAAkC,CAAC;QAEvC,IAAI,IAAI,EAAE,CAAC;YACT,aAAa,GAAG,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YACpD,aAAa,GAAG,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YACpD,YAAY,GAAG,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjD,YAAY,GAAG,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjD,cAAc,GAAG,KAAK,CAAC,EAAE,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACvD,cAAc,GAAG,KAAK,CAAC,EAAE,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACvD,IAAI,QAAQ,IAAI,IAAI,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI;gBAAE,UAAU,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YACrG,IAAI,UAAU,IAAI,IAAI,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI;gBAAE,YAAY,GAAG,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YAC/G,IAAI,QAAQ,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI;gBAAE,UAAU,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC3G,IAAI,YAAY,IAAI,IAAI,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI;gBAAE,cAAc,GAAG,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC3H,CAAC;QAED,iDAAiD;QACjD,gBAAgB,CAAC,GAAG,CAAC,IAAI,EAAE;YACzB,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,UAAU,EAAE,EAAE,CAAC,UAAU;YACzB,UAAU,EAAE,EAAE,CAAC,UAAU;YACzB,aAAa,EAAE,QAAQ;YACvB,eAAe,EAAE,UAAU;YAC3B,gBAAgB,EAAE,QAAQ;YAC1B,iBAAiB,EAAE,YAAY;SAChC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAgB;YACzB,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC;YAC1B,YAAY,EAAE,EAAE,CAAC,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,2BAA2B;YACpE,YAAY,EAAE,EAAE,CAAC,QAAQ,GAAG,EAAE,CAAC,QAAQ;YACvC,SAAS,EAAE,aAAa;YACxB,SAAS,EAAE,aAAa;YACxB,QAAQ,EAAE,YAAY;YACtB,QAAQ,EAAE,YAAY;YACtB,UAAU,EAAE,cAAc;YAC1B,UAAU,EAAE,cAAc;YAC1B,SAAS,EAAE,YAAY,CAAC,IAAI,CAAC;SAC9B,CAAC;QACF,IAAI,UAAU,KAAK,SAAS;YAAE,KAAK,CAAC,aAAa,GAAG,UAAU,CAAC;QAC/D,IAAI,YAAY,KAAK,SAAS;YAAE,KAAK,CAAC,eAAe,GAAG,YAAY,CAAC;QACrE,IAAI,UAAU,KAAK,SAAS;YAAE,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC;QAClE,IAAI,cAAc,KAAK,SAAS;YAAE,KAAK,CAAC,iBAAiB,GAAG,cAAc,CAAC;QAC3E,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,MAAM;YAAE,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC;QACvC,oEAAoE;QACpE,IAAI,YAAY,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAED,2CAA2C;IAC3C,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,IAAI,EAAE,EAAE,CAAC;QAC3C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/lib/types.d.ts
CHANGED
|
@@ -158,10 +158,23 @@ export interface NetworkInfo {
|
|
|
158
158
|
speed_mbps: number;
|
|
159
159
|
rx_bytes_sec: number;
|
|
160
160
|
tx_bytes_sec: number;
|
|
161
|
+
/** Delta over the collection interval (rx_errors + any subtype counter). */
|
|
161
162
|
rx_errors: number;
|
|
162
163
|
tx_errors: number;
|
|
163
164
|
rx_drops: number;
|
|
164
165
|
tx_drops: number;
|
|
166
|
+
/** Delta over the collection interval. Null if counter not available on this NIC. */
|
|
167
|
+
rx_packets?: number;
|
|
168
|
+
tx_packets?: number;
|
|
169
|
+
/** Fine-grained RX hardware-error subtypes (deltas). Null if unavailable. */
|
|
170
|
+
rx_crc_errors?: number;
|
|
171
|
+
rx_frame_errors?: number;
|
|
172
|
+
rx_length_errors?: number;
|
|
173
|
+
/** TX physical-layer fault counter (delta). Null if unavailable. */
|
|
174
|
+
tx_carrier_errors?: number;
|
|
175
|
+
operstate?: string;
|
|
176
|
+
bond_master?: string;
|
|
177
|
+
is_bond_master?: boolean;
|
|
165
178
|
}
|
|
166
179
|
export interface RaidInfo {
|
|
167
180
|
device: string;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Glassmkr Crucible - docker compose deployment
|
|
2
|
+
#
|
|
3
|
+
# Before starting, create /etc/glassmkr/collector.yaml on the host with your
|
|
4
|
+
# Forge collector key. See https://forge.glassmkr.com/docs/getting-started.
|
|
5
|
+
#
|
|
6
|
+
# Why privileged + host network:
|
|
7
|
+
# - privileged: true gives access to /dev/ipmi0 (IPMI sensors) and raw disk devices (SMART)
|
|
8
|
+
# - network_mode: host lets the agent read the real host network interfaces and bond state
|
|
9
|
+
# - /proc and /sys are mounted so the agent monitors the host, not the container
|
|
10
|
+
|
|
11
|
+
services:
|
|
12
|
+
crucible:
|
|
13
|
+
image: ghcr.io/glassmkr/crucible:latest
|
|
14
|
+
container_name: glassmkr-crucible
|
|
15
|
+
restart: unless-stopped
|
|
16
|
+
privileged: true
|
|
17
|
+
network_mode: host
|
|
18
|
+
volumes:
|
|
19
|
+
- /etc/glassmkr:/etc/glassmkr:ro
|
|
20
|
+
- /proc:/host/proc:ro
|
|
21
|
+
- /sys:/host/sys:ro
|
|
22
|
+
- /dev:/dev:ro
|
|
23
|
+
- /run/dbus:/run/dbus:ro
|
|
24
|
+
environment:
|
|
25
|
+
- HOST_PROC=/host/proc
|
|
26
|
+
- HOST_SYS=/host/sys
|
package/package.json
CHANGED
package/src/collect/network.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { readProcFile, sleep } from "../lib/parse.js";
|
|
2
|
-
import { readFileSync } from "fs";
|
|
2
|
+
import { readFileSync, readdirSync } from "fs";
|
|
3
3
|
import type { NetworkInfo } from "../lib/types.js";
|
|
4
4
|
|
|
5
5
|
interface IfaceStats {
|
|
@@ -13,10 +13,26 @@ interface PreviousCounters {
|
|
|
13
13
|
tx_errors: number;
|
|
14
14
|
rx_drops: number;
|
|
15
15
|
tx_drops: number;
|
|
16
|
+
rx_packets: number;
|
|
17
|
+
tx_packets: number;
|
|
18
|
+
rx_crc_errors?: number;
|
|
19
|
+
rx_frame_errors?: number;
|
|
20
|
+
rx_length_errors?: number;
|
|
21
|
+
tx_carrier_errors?: number;
|
|
16
22
|
}
|
|
17
23
|
|
|
18
24
|
const previousCounters = new Map<string, PreviousCounters>();
|
|
19
25
|
|
|
26
|
+
function readStatCounter(iface: string, name: string): number | undefined {
|
|
27
|
+
try {
|
|
28
|
+
const raw = readFileSync(`/sys/class/net/${iface}/statistics/${name}`, "utf-8").trim();
|
|
29
|
+
const val = parseInt(raw, 10);
|
|
30
|
+
return Number.isFinite(val) ? val : undefined;
|
|
31
|
+
} catch {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
20
36
|
function parseNetDev(): Record<string, IfaceStats> {
|
|
21
37
|
const raw = readProcFile("/proc/net/dev") || "";
|
|
22
38
|
const result: Record<string, IfaceStats> = {};
|
|
@@ -45,6 +61,35 @@ function getSpeed(iface: string): number {
|
|
|
45
61
|
}
|
|
46
62
|
}
|
|
47
63
|
|
|
64
|
+
function getOperstate(iface: string): string {
|
|
65
|
+
try {
|
|
66
|
+
return readFileSync(`/sys/class/net/${iface}/operstate`, "utf-8").trim();
|
|
67
|
+
} catch {
|
|
68
|
+
return "unknown";
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function getBondMaster(iface: string): string | undefined {
|
|
73
|
+
try {
|
|
74
|
+
const bonds = readdirSync("/proc/net/bonding/");
|
|
75
|
+
for (const bond of bonds) {
|
|
76
|
+
const content = readFileSync(`/proc/net/bonding/${bond}`, "utf-8");
|
|
77
|
+
if (content.includes(`Slave Interface: ${iface}`)) return bond;
|
|
78
|
+
}
|
|
79
|
+
} catch {
|
|
80
|
+
// No bonds or /proc/net/bonding doesn't exist
|
|
81
|
+
}
|
|
82
|
+
return undefined;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function isBondMaster(iface: string): boolean {
|
|
86
|
+
try {
|
|
87
|
+
return readdirSync("/proc/net/bonding/").includes(iface);
|
|
88
|
+
} catch {
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
48
93
|
// Compute delta, handling counter wraps (current < previous means reset, use current as delta)
|
|
49
94
|
function delta(current: number, previous: number): number {
|
|
50
95
|
if (current >= previous) return current - previous;
|
|
@@ -66,17 +111,37 @@ export async function collectNetwork(): Promise<NetworkInfo[]> {
|
|
|
66
111
|
|
|
67
112
|
const prev = previousCounters.get(name);
|
|
68
113
|
|
|
114
|
+
// /sys/class/net/*/statistics/ exposes finer-grained RX/TX subtype
|
|
115
|
+
// counters than /proc/net/dev. Read cumulative values here; delta is
|
|
116
|
+
// derived below against the previous cycle's snapshot.
|
|
117
|
+
const rxCrcCum = readStatCounter(name, "rx_crc_errors");
|
|
118
|
+
const rxFrameCum = readStatCounter(name, "rx_frame_errors");
|
|
119
|
+
const rxLenCum = readStatCounter(name, "rx_length_errors");
|
|
120
|
+
const txCarrierCum = readStatCounter(name, "tx_carrier_errors");
|
|
121
|
+
|
|
69
122
|
// Compute error/drop deltas (0 on first cycle after start or new interface)
|
|
70
123
|
let rxErrorsDelta = 0;
|
|
71
124
|
let txErrorsDelta = 0;
|
|
72
125
|
let rxDropsDelta = 0;
|
|
73
126
|
let txDropsDelta = 0;
|
|
127
|
+
let rxPacketsDelta = 0;
|
|
128
|
+
let txPacketsDelta = 0;
|
|
129
|
+
let rxCrcDelta: number | undefined;
|
|
130
|
+
let rxFrameDelta: number | undefined;
|
|
131
|
+
let rxLenDelta: number | undefined;
|
|
132
|
+
let txCarrierDelta: number | undefined;
|
|
74
133
|
|
|
75
134
|
if (prev) {
|
|
76
135
|
rxErrorsDelta = delta(s2.rx_errors, prev.rx_errors);
|
|
77
136
|
txErrorsDelta = delta(s2.tx_errors, prev.tx_errors);
|
|
78
137
|
rxDropsDelta = delta(s2.rx_drops, prev.rx_drops);
|
|
79
138
|
txDropsDelta = delta(s2.tx_drops, prev.tx_drops);
|
|
139
|
+
rxPacketsDelta = delta(s2.rx_packets, prev.rx_packets);
|
|
140
|
+
txPacketsDelta = delta(s2.tx_packets, prev.tx_packets);
|
|
141
|
+
if (rxCrcCum != null && prev.rx_crc_errors != null) rxCrcDelta = delta(rxCrcCum, prev.rx_crc_errors);
|
|
142
|
+
if (rxFrameCum != null && prev.rx_frame_errors != null) rxFrameDelta = delta(rxFrameCum, prev.rx_frame_errors);
|
|
143
|
+
if (rxLenCum != null && prev.rx_length_errors != null) rxLenDelta = delta(rxLenCum, prev.rx_length_errors);
|
|
144
|
+
if (txCarrierCum != null && prev.tx_carrier_errors != null) txCarrierDelta = delta(txCarrierCum, prev.tx_carrier_errors);
|
|
80
145
|
}
|
|
81
146
|
|
|
82
147
|
// Store current cumulative values for next cycle
|
|
@@ -85,9 +150,15 @@ export async function collectNetwork(): Promise<NetworkInfo[]> {
|
|
|
85
150
|
tx_errors: s2.tx_errors,
|
|
86
151
|
rx_drops: s2.rx_drops,
|
|
87
152
|
tx_drops: s2.tx_drops,
|
|
153
|
+
rx_packets: s2.rx_packets,
|
|
154
|
+
tx_packets: s2.tx_packets,
|
|
155
|
+
rx_crc_errors: rxCrcCum,
|
|
156
|
+
rx_frame_errors: rxFrameCum,
|
|
157
|
+
rx_length_errors: rxLenCum,
|
|
158
|
+
tx_carrier_errors: txCarrierCum,
|
|
88
159
|
});
|
|
89
160
|
|
|
90
|
-
|
|
161
|
+
const entry: NetworkInfo = {
|
|
91
162
|
interface: name,
|
|
92
163
|
speed_mbps: getSpeed(name),
|
|
93
164
|
rx_bytes_sec: s2.rx_bytes - s1.rx_bytes, // already a 1-second delta
|
|
@@ -96,7 +167,19 @@ export async function collectNetwork(): Promise<NetworkInfo[]> {
|
|
|
96
167
|
tx_errors: txErrorsDelta,
|
|
97
168
|
rx_drops: rxDropsDelta,
|
|
98
169
|
tx_drops: txDropsDelta,
|
|
99
|
-
|
|
170
|
+
rx_packets: rxPacketsDelta,
|
|
171
|
+
tx_packets: txPacketsDelta,
|
|
172
|
+
operstate: getOperstate(name),
|
|
173
|
+
};
|
|
174
|
+
if (rxCrcDelta !== undefined) entry.rx_crc_errors = rxCrcDelta;
|
|
175
|
+
if (rxFrameDelta !== undefined) entry.rx_frame_errors = rxFrameDelta;
|
|
176
|
+
if (rxLenDelta !== undefined) entry.rx_length_errors = rxLenDelta;
|
|
177
|
+
if (txCarrierDelta !== undefined) entry.tx_carrier_errors = txCarrierDelta;
|
|
178
|
+
const master = getBondMaster(name);
|
|
179
|
+
if (master) entry.bond_master = master;
|
|
180
|
+
// Identify bond masters (have at least one slave pointing at them).
|
|
181
|
+
if (isBondMaster(name)) entry.is_bond_master = true;
|
|
182
|
+
results.push(entry);
|
|
100
183
|
}
|
|
101
184
|
|
|
102
185
|
// Remove stale interfaces that disappeared
|
package/src/lib/types.ts
CHANGED
|
@@ -139,10 +139,23 @@ export interface NetworkInfo {
|
|
|
139
139
|
speed_mbps: number;
|
|
140
140
|
rx_bytes_sec: number;
|
|
141
141
|
tx_bytes_sec: number;
|
|
142
|
+
/** Delta over the collection interval (rx_errors + any subtype counter). */
|
|
142
143
|
rx_errors: number;
|
|
143
144
|
tx_errors: number;
|
|
144
145
|
rx_drops: number;
|
|
145
146
|
tx_drops: number;
|
|
147
|
+
/** Delta over the collection interval. Null if counter not available on this NIC. */
|
|
148
|
+
rx_packets?: number;
|
|
149
|
+
tx_packets?: number;
|
|
150
|
+
/** Fine-grained RX hardware-error subtypes (deltas). Null if unavailable. */
|
|
151
|
+
rx_crc_errors?: number;
|
|
152
|
+
rx_frame_errors?: number;
|
|
153
|
+
rx_length_errors?: number;
|
|
154
|
+
/** TX physical-layer fault counter (delta). Null if unavailable. */
|
|
155
|
+
tx_carrier_errors?: number;
|
|
156
|
+
operstate?: string; // "up", "down", "unknown", etc. from /sys/class/net/{iface}/operstate
|
|
157
|
+
bond_master?: string; // if this interface is a bond slave, the bond name
|
|
158
|
+
is_bond_master?: boolean; // true when this entry represents the bond aggregate
|
|
146
159
|
}
|
|
147
160
|
|
|
148
161
|
export interface RaidInfo {
|