dsh-ssh-hub 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +271 -2
- package/cordis.patch.yml +16 -0
- package/icon.svg +3 -0
- package/lib/client.js +13975 -0
- package/lib/index.js +6908 -0
- package/lib/types/activity/history.d.ts +50 -0
- package/lib/types/alerting/notify.d.ts +89 -0
- package/lib/types/alerting/prompt-summary.d.ts +28 -0
- package/lib/types/alerting/rules.d.ts +135 -0
- package/lib/types/alerting/tools.d.ts +30 -0
- package/lib/types/audit/audit-log.d.ts +84 -0
- package/lib/types/audit/recorder.d.ts +50 -0
- package/lib/types/client/api.d.ts +222 -0
- package/lib/types/client/index.d.ts +45 -0
- package/lib/types/client/locales.d.ts +319 -0
- package/lib/types/client/native-panel.d.ts +70 -0
- package/lib/types/client/panel/ClusterTab.d.ts +7 -0
- package/lib/types/client/panel/DeployTab.d.ts +10 -0
- package/lib/types/client/panel/FleetTab.d.ts +10 -0
- package/lib/types/client/panel/HostFormDialog.d.ts +12 -0
- package/lib/types/client/panel/HostsTab.d.ts +25 -0
- package/lib/types/client/panel/LogsTab.d.ts +10 -0
- package/lib/types/client/panel/SshPanel.d.ts +14 -0
- package/lib/types/client/panel/TerminalTab.d.ts +27 -0
- package/lib/types/client/panel/TransferTab.d.ts +7 -0
- package/lib/types/client/panel/TunnelEditDialog.d.ts +15 -0
- package/lib/types/client/panel/TunnelsTab.d.ts +27 -0
- package/lib/types/client/panel/controller.d.ts +253 -0
- package/lib/types/client/panel/helpers.d.ts +45 -0
- package/lib/types/client/panel/tabs.d.ts +62 -0
- package/lib/types/client/panel/xterm.css.d.ts +2 -0
- package/lib/types/client/settings-binding.d.ts +71 -0
- package/lib/types/deploy/orchestrator.d.ts +158 -0
- package/lib/types/deploy/tools.d.ts +14 -0
- package/lib/types/dsh-home.d.ts +16 -0
- package/lib/types/engine/cluster.d.ts +19 -0
- package/lib/types/engine/connection-pool.d.ts +183 -0
- package/lib/types/engine/proxy-command.d.ts +36 -0
- package/lib/types/engine/pty.d.ts +32 -0
- package/lib/types/engine/sftp.d.ts +20 -0
- package/lib/types/engine/terminal-sessions.d.ts +67 -0
- package/lib/types/engine/tunnel-discovery.d.ts +89 -0
- package/lib/types/engine/tunnel-remarks.d.ts +49 -0
- package/lib/types/engine/tunnel.d.ts +76 -0
- package/lib/types/engine.d.ts +103 -0
- package/lib/types/http.d.ts +59 -0
- package/lib/types/index.d.ts +128 -0
- package/lib/types/inspect/checks.d.ts +118 -0
- package/lib/types/inspect/inspection.d.ts +111 -0
- package/lib/types/inspect/tools.d.ts +50 -0
- package/lib/types/logs/search.d.ts +131 -0
- package/lib/types/logs/tools.d.ts +6 -0
- package/lib/types/loopback.d.ts +64 -0
- package/lib/types/monitor/loop.d.ts +133 -0
- package/lib/types/monitor/metrics.d.ts +151 -0
- package/lib/types/mount-once.d.ts +55 -0
- package/lib/types/protocol.d.ts +524 -0
- package/lib/types/routes.d.ts +82 -0
- package/lib/types/security/known-hosts.d.ts +153 -0
- package/lib/types/security/master-secret.d.ts +157 -0
- package/lib/types/security/redact.d.ts +44 -0
- package/lib/types/security/secrets.d.ts +70 -0
- package/lib/types/ssh-config.d.ts +42 -0
- package/lib/types/store.d.ts +183 -0
- package/lib/types/tool-surface.d.ts +72 -0
- package/lib/types/tools.d.ts +56 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +126 -6
- package/src/activity/history.ts +117 -0
- package/src/alerting/notify.ts +212 -0
- package/src/alerting/prompt-summary.ts +71 -0
- package/src/alerting/rules.ts +281 -0
- package/src/alerting/tools.ts +263 -0
- package/src/audit/audit-log.ts +199 -0
- package/src/audit/recorder.ts +73 -0
- package/src/client/api.ts +782 -0
- package/src/client/css-modules.d.ts +5 -0
- package/src/client/index.ts +150 -0
- package/src/client/locales.ts +654 -0
- package/src/client/native-panel.tsx +141 -0
- package/src/client/panel/ClusterTab.tsx +141 -0
- package/src/client/panel/DeployTab.tsx +314 -0
- package/src/client/panel/FleetTab.tsx +494 -0
- package/src/client/panel/HostFormDialog.tsx +289 -0
- package/src/client/panel/HostsTab.tsx +395 -0
- package/src/client/panel/LogsTab.tsx +208 -0
- package/src/client/panel/SshPanel.tsx +105 -0
- package/src/client/panel/TerminalTab.tsx +550 -0
- package/src/client/panel/TransferTab.tsx +238 -0
- package/src/client/panel/TunnelEditDialog.tsx +174 -0
- package/src/client/panel/TunnelsTab.tsx +328 -0
- package/src/client/panel/controller.ts +507 -0
- package/src/client/panel/helpers.ts +84 -0
- package/src/client/panel/panel.module.css +1734 -0
- package/src/client/panel/tabs.ts +52 -0
- package/src/client/panel/xterm.css.ts +2 -0
- package/src/client/settings-binding.ts +184 -0
- package/src/deploy/orchestrator.ts +326 -0
- package/src/deploy/tools.ts +225 -0
- package/src/dsh-home.ts +42 -0
- package/src/engine/cluster.ts +60 -0
- package/src/engine/connection-pool.ts +721 -0
- package/src/engine/proxy-command.ts +248 -0
- package/src/engine/pty.ts +78 -0
- package/src/engine/sftp.ts +229 -0
- package/src/engine/terminal-sessions.ts +280 -0
- package/src/engine/tunnel-discovery.ts +353 -0
- package/src/engine/tunnel-remarks.ts +120 -0
- package/src/engine/tunnel.ts +300 -0
- package/src/engine.ts +195 -0
- package/src/http.ts +127 -0
- package/src/index.ts +690 -0
- package/src/inspect/checks.ts +251 -0
- package/src/inspect/inspection.ts +321 -0
- package/src/inspect/tools.ts +366 -0
- package/src/logs/search.ts +251 -0
- package/src/logs/tools.ts +160 -0
- package/src/loopback.ts +119 -0
- package/src/monitor/loop.ts +265 -0
- package/src/monitor/metrics.ts +333 -0
- package/src/mount-once.ts +142 -0
- package/src/protocol.ts +534 -0
- package/src/routes.ts +998 -0
- package/src/security/known-hosts.ts +342 -0
- package/src/security/master-secret.ts +334 -0
- package/src/security/redact.ts +119 -0
- package/src/security/secrets.ts +152 -0
- package/src/ssh-config.ts +182 -0
- package/src/store.ts +757 -0
- package/src/tool-surface.ts +96 -0
- package/src/tools.ts +550 -0
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inspection checks: the vocabulary of what "inspect this fleet" means.
|
|
3
|
+
*
|
|
4
|
+
* The base plugin gives the agent a way to run a command on many hosts at
|
|
5
|
+
* once, but every run is improvised — nothing is defined, nothing is
|
|
6
|
+
* retained, nothing can be compared against last time. That is fine for
|
|
7
|
+
* "what is on this box" and useless for "is the fleet healthy".
|
|
8
|
+
*
|
|
9
|
+
* A check here is a named, typed probe with a command, a parser and a
|
|
10
|
+
* verdict rule. Checks are defined in code (they are part of the product,
|
|
11
|
+
* not user data), so their commands are known-good POSIX shell — the fleet
|
|
12
|
+
* is Linux-only by design, and the probes read only.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import type { ExecResult } from '../protocol.ts'
|
|
16
|
+
|
|
17
|
+
/** How a check's value is judged. */
|
|
18
|
+
export type CheckKind =
|
|
19
|
+
/** A percentage 0-100; the threshold is a watermark. */
|
|
20
|
+
| 'percent'
|
|
21
|
+
/** A number compared against a threshold. */
|
|
22
|
+
| 'gauge'
|
|
23
|
+
/** A boolean: does the thing exist / is it running. */
|
|
24
|
+
| 'boolean'
|
|
25
|
+
/** A free-form text value (a version, a mount path). */
|
|
26
|
+
| 'text'
|
|
27
|
+
|
|
28
|
+
/** Which way a breach is judged. */
|
|
29
|
+
export type CheckDirection =
|
|
30
|
+
/** High values are bad (disk usage). */
|
|
31
|
+
| 'above'
|
|
32
|
+
/** Low values are bad (free space, free memory). */
|
|
33
|
+
| 'below'
|
|
34
|
+
|
|
35
|
+
/** One inspection check. */
|
|
36
|
+
export interface Check {
|
|
37
|
+
/** Stable identifier, used in stored results. */
|
|
38
|
+
id: string
|
|
39
|
+
/** What the check measures, in the operator's language. */
|
|
40
|
+
title: string
|
|
41
|
+
/** How the value is judged. */
|
|
42
|
+
kind: CheckKind
|
|
43
|
+
/** Which way a breach goes. */
|
|
44
|
+
direction: CheckDirection
|
|
45
|
+
/**
|
|
46
|
+
* The breach threshold: a percentage for 'percent', a number for
|
|
47
|
+
* 'gauge', unused for the other kinds.
|
|
48
|
+
*/
|
|
49
|
+
threshold?: number
|
|
50
|
+
/** The read-only command that produces the value. */
|
|
51
|
+
command: string
|
|
52
|
+
/** Parse the command output into a value. */
|
|
53
|
+
parse: (stdout: string) => number | boolean | string | null
|
|
54
|
+
/** The unit shown in the panel. */
|
|
55
|
+
unit?: string
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The outcome of one check on one host. */
|
|
59
|
+
export interface CheckResult {
|
|
60
|
+
/** The check id. */
|
|
61
|
+
checkId: string
|
|
62
|
+
/** The host alias. */
|
|
63
|
+
alias: string
|
|
64
|
+
/** The parsed value, or null when the probe could not produce one. */
|
|
65
|
+
value: number | boolean | string | null
|
|
66
|
+
/** Whether the value breaches the threshold. */
|
|
67
|
+
breached: boolean
|
|
68
|
+
/** The raw output, trimmed, for the operator to read. */
|
|
69
|
+
raw: string
|
|
70
|
+
/** The error when the command itself failed. */
|
|
71
|
+
error?: string
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The built-in check set.
|
|
76
|
+
*
|
|
77
|
+
* Every command is read-only and POSIX-shell compatible. Nothing installs
|
|
78
|
+
* anything on the remote host and nothing writes to it — an inspection
|
|
79
|
+
* that mutates the fleet is not an inspection.
|
|
80
|
+
*/
|
|
81
|
+
export const BUILTIN_CHECKS: readonly Check[] = [
|
|
82
|
+
{
|
|
83
|
+
id: 'disk-usage',
|
|
84
|
+
title: '磁盘使用率',
|
|
85
|
+
kind: 'percent',
|
|
86
|
+
direction: 'above',
|
|
87
|
+
threshold: 85,
|
|
88
|
+
unit: '%',
|
|
89
|
+
// df -P keeps the output POSIX (one line per filesystem, no wrapping).
|
|
90
|
+
command: 'df -P / | tail -1 | awk \'{print $5}\' | tr -d "%"',
|
|
91
|
+
parse: stdout => parseNumber(stdout),
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
id: 'load-average',
|
|
95
|
+
title: '负载(5 分钟)',
|
|
96
|
+
kind: 'gauge',
|
|
97
|
+
direction: 'above',
|
|
98
|
+
threshold: 4,
|
|
99
|
+
command: 'cat /proc/loadavg | awk \'{print $2}\'',
|
|
100
|
+
parse: stdout => parseNumber(stdout),
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
id: 'memory-used',
|
|
104
|
+
title: '内存使用率',
|
|
105
|
+
kind: 'percent',
|
|
106
|
+
direction: 'above',
|
|
107
|
+
threshold: 90,
|
|
108
|
+
unit: '%',
|
|
109
|
+
// /proc/meminfo is the only source that needs no external tool.
|
|
110
|
+
command: 'awk \'/^MemTotal:/ {t=$2} /^MemAvailable:/ {a=$2} END {if (t>0) printf "%d", (t-a)*100/t}\' /proc/meminfo',
|
|
111
|
+
parse: stdout => parseNumber(stdout),
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
id: 'disk-inodes',
|
|
115
|
+
title: 'inode 使用率',
|
|
116
|
+
kind: 'percent',
|
|
117
|
+
direction: 'above',
|
|
118
|
+
threshold: 85,
|
|
119
|
+
unit: '%',
|
|
120
|
+
command: 'df -Pi / | tail -1 | awk \'{print $5}\' | tr -d "%"',
|
|
121
|
+
parse: stdout => parseNumber(stdout),
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
id: 'uptime',
|
|
125
|
+
title: '运行时长',
|
|
126
|
+
kind: 'gauge',
|
|
127
|
+
direction: 'below',
|
|
128
|
+
// A server that rebooted within an hour is usually rebooting in a loop.
|
|
129
|
+
threshold: 3600,
|
|
130
|
+
unit: '秒',
|
|
131
|
+
command: 'cat /proc/uptime | awk \'{print int($1)}\'',
|
|
132
|
+
parse: stdout => parseNumber(stdout),
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
id: 'sshd-running',
|
|
136
|
+
title: 'sshd 运行中',
|
|
137
|
+
kind: 'boolean',
|
|
138
|
+
direction: 'above',
|
|
139
|
+
command: 'pgrep -x sshd >/dev/null 2>&1 && echo running || echo stopped',
|
|
140
|
+
parse: stdout => stdout.includes('running'),
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
id: 'failed-units',
|
|
144
|
+
title: '失败的 systemd 单元',
|
|
145
|
+
kind: 'gauge',
|
|
146
|
+
direction: 'above',
|
|
147
|
+
threshold: 0,
|
|
148
|
+
command: 'systemctl list-units --state=failed --no-legend --no-pager 2>/dev/null | wc -l',
|
|
149
|
+
parse: stdout => parseNumber(stdout),
|
|
150
|
+
},
|
|
151
|
+
{
|
|
152
|
+
id: 'kernel',
|
|
153
|
+
title: '内核版本',
|
|
154
|
+
kind: 'text',
|
|
155
|
+
direction: 'above',
|
|
156
|
+
command: 'uname -r',
|
|
157
|
+
parse: stdout => stdout.trim() === '' ? null : stdout.trim(),
|
|
158
|
+
},
|
|
159
|
+
]
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Parse a number out of command output.
|
|
163
|
+
*
|
|
164
|
+
* @param stdout - the raw output.
|
|
165
|
+
* @returns the first number found, or null.
|
|
166
|
+
*/
|
|
167
|
+
export function parseNumber(stdout: string): number | null {
|
|
168
|
+
const match = stdout.match(/-?\d+(?:\.\d+)?/)
|
|
169
|
+
if (match === null) return null
|
|
170
|
+
const value = Number.parseFloat(match[0])
|
|
171
|
+
return Number.isFinite(value) ? value : null
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Find a check by id.
|
|
176
|
+
*
|
|
177
|
+
* @param id - the check id.
|
|
178
|
+
* @returns the check, or undefined.
|
|
179
|
+
*/
|
|
180
|
+
export function findCheck(id: string): Check | undefined {
|
|
181
|
+
return BUILTIN_CHECKS.find(check => check.id === id)
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Judge one parsed value against a check's threshold.
|
|
186
|
+
*
|
|
187
|
+
* A `null` value never breaches: an unreadable probe is an unknown state,
|
|
188
|
+
* not a healthy one and not a breach either. The caller records the
|
|
189
|
+
* distinction — an unknown value must not be silently counted as healthy.
|
|
190
|
+
*
|
|
191
|
+
* @param check - the check.
|
|
192
|
+
* @param value - the parsed value.
|
|
193
|
+
* @returns whether the value breaches.
|
|
194
|
+
*/
|
|
195
|
+
export function isBreached(check: Check, value: number | boolean | string | null): boolean {
|
|
196
|
+
if (value === null) return false
|
|
197
|
+
if (check.kind === 'boolean') {
|
|
198
|
+
// For a boolean, "running" is the good state; the direction says which
|
|
199
|
+
// way is bad, so 'above' means "false is bad".
|
|
200
|
+
const asBoolean = value === true || value === 'running' || value === 'true'
|
|
201
|
+
return check.direction === 'above' ? !asBoolean : asBoolean
|
|
202
|
+
}
|
|
203
|
+
if (check.kind === 'text') return false
|
|
204
|
+
if (typeof value !== 'number' || check.threshold === undefined) return false
|
|
205
|
+
return check.direction === 'above' ? value > check.threshold : value < check.threshold
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Turn one exec result into a check result.
|
|
210
|
+
*
|
|
211
|
+
* @param check - the check that was run.
|
|
212
|
+
* @param alias - the host alias.
|
|
213
|
+
* @param result - the exec result.
|
|
214
|
+
* @returns the check result.
|
|
215
|
+
*/
|
|
216
|
+
export function toCheckResult(check: Check, alias: string, result: ExecResult): CheckResult {
|
|
217
|
+
if (result.error !== undefined || !result.success) {
|
|
218
|
+
return {
|
|
219
|
+
checkId: check.id,
|
|
220
|
+
alias,
|
|
221
|
+
value: null,
|
|
222
|
+
breached: false,
|
|
223
|
+
raw: result.stderr.trim().slice(0, 2000),
|
|
224
|
+
error: result.error ?? `command failed with exit code ${result.exitCode ?? 'null'}`,
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
const raw = result.stdout.trim()
|
|
228
|
+
const value = check.parse(raw)
|
|
229
|
+
return {
|
|
230
|
+
checkId: check.id,
|
|
231
|
+
alias,
|
|
232
|
+
value,
|
|
233
|
+
breached: isBreached(check, value),
|
|
234
|
+
raw: raw.slice(0, 2000),
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* A human-readable rendering of one value.
|
|
240
|
+
*
|
|
241
|
+
* @param check - the check the value belongs to.
|
|
242
|
+
* @param value - the parsed value.
|
|
243
|
+
* @returns the display text.
|
|
244
|
+
*/
|
|
245
|
+
export function formatValue(check: Check, value: number | boolean | string | null): string {
|
|
246
|
+
if (value === null) return '未知'
|
|
247
|
+
if (typeof value === 'boolean') return value ? '正常' : '异常'
|
|
248
|
+
if (typeof value === 'string') return value
|
|
249
|
+
const rounded = Number.isInteger(value) ? String(value) : value.toFixed(1)
|
|
250
|
+
return check.unit === undefined ? rounded : `${rounded}${check.unit}`
|
|
251
|
+
}
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inspection runner and result retention.
|
|
3
|
+
*
|
|
4
|
+
* Runs the built-in checks across a selected set of hosts (reusing the
|
|
5
|
+
* engine's pool, so no extra connections are opened), stores each round as
|
|
6
|
+
* a JSON Lines record, and answers "what changed since last time".
|
|
7
|
+
*
|
|
8
|
+
* The comparison is the point: a raw dump of "disk 71%" says nothing, while
|
|
9
|
+
* "disk 71% (was 42% an hour ago)" says the fleet is filling up. Every round
|
|
10
|
+
* is therefore retained, and the diff is computed from the two newest
|
|
11
|
+
* rounds by default.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs'
|
|
15
|
+
import { dirname, join } from 'node:path'
|
|
16
|
+
import { dshHome } from '../dsh-home.ts'
|
|
17
|
+
import type { SshEngine } from '../engine.ts'
|
|
18
|
+
import type { ExecResult } from '../protocol.ts'
|
|
19
|
+
import {
|
|
20
|
+
BUILTIN_CHECKS,
|
|
21
|
+
type Check,
|
|
22
|
+
type CheckResult,
|
|
23
|
+
formatValue,
|
|
24
|
+
toCheckResult,
|
|
25
|
+
} from './checks.ts'
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* How many hosts one inspection round works on at a time.
|
|
29
|
+
*
|
|
30
|
+
* Chosen to finish a 50-host sweep inside the default 60-second interval while
|
|
31
|
+
* keeping the number of simultaneously open SSH connections in the same range
|
|
32
|
+
* the connection pool already uses for other fan-out (8). Raising it trades
|
|
33
|
+
* sockets for wall clock; lowering it risks a round that outlasts its own
|
|
34
|
+
* timer and reports "上一轮尚未结束" instead of a result.
|
|
35
|
+
*/
|
|
36
|
+
export const DEFAULT_INSPECT_CONCURRENCY = 8
|
|
37
|
+
|
|
38
|
+
/** One inspection round. */
|
|
39
|
+
export interface InspectionRound {
|
|
40
|
+
/** Epoch milliseconds when the round started. */
|
|
41
|
+
startedAt: number
|
|
42
|
+
/** How long the round took. */
|
|
43
|
+
durationMs: number
|
|
44
|
+
/** The hosts covered. */
|
|
45
|
+
targets: string[]
|
|
46
|
+
/** Every check result, flattened. */
|
|
47
|
+
results: CheckResult[]
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** One host's change between two rounds. */
|
|
51
|
+
export interface HostChange {
|
|
52
|
+
alias: string
|
|
53
|
+
checkId: string
|
|
54
|
+
title: string
|
|
55
|
+
/** The previous display value, or undefined when the check is new. */
|
|
56
|
+
before?: string
|
|
57
|
+
/** The current display value. */
|
|
58
|
+
after: string
|
|
59
|
+
/** Whether the breach state changed (and in which direction). */
|
|
60
|
+
stateChange?: 'new-breach' | 'recovered'
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** The inspection history file. */
|
|
64
|
+
export function historyPath(): string {
|
|
65
|
+
return join(dshHome(), 'dsh-ssh-hub', 'inspection.jsonl')
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The inspection runner.
|
|
70
|
+
*
|
|
71
|
+
* One instance per plugin apply. Holds no long-lived state beyond the file
|
|
72
|
+
* path: rounds are written as they complete, so a crash loses at most the
|
|
73
|
+
* round in flight.
|
|
74
|
+
*/
|
|
75
|
+
export class Inspector {
|
|
76
|
+
/** The history file this inspector appends to. */
|
|
77
|
+
readonly path: string
|
|
78
|
+
/** How many rounds to keep for comparison and display. */
|
|
79
|
+
readonly maxRounds: number
|
|
80
|
+
|
|
81
|
+
constructor(path?: string, maxRounds = 100) {
|
|
82
|
+
this.path = path ?? historyPath()
|
|
83
|
+
this.maxRounds = maxRounds
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Run one round of checks.
|
|
88
|
+
*
|
|
89
|
+
* @param engine - the SSH engine (the connection pool is reused).
|
|
90
|
+
* @param targets - the host aliases to inspect.
|
|
91
|
+
* @param checkIds - which checks to run (default: all built-in checks).
|
|
92
|
+
* @param timeoutMs - per-command timeout.
|
|
93
|
+
* @returns the completed round.
|
|
94
|
+
*/
|
|
95
|
+
async run(
|
|
96
|
+
engine: SshEngine,
|
|
97
|
+
targets: string[],
|
|
98
|
+
checkIds?: string[],
|
|
99
|
+
timeoutMs = 30_000,
|
|
100
|
+
concurrency = DEFAULT_INSPECT_CONCURRENCY,
|
|
101
|
+
): Promise<InspectionRound> {
|
|
102
|
+
const startedAt = Date.now()
|
|
103
|
+
const checks = selectChecks(checkIds)
|
|
104
|
+
// Each (host, check) pair is one command.
|
|
105
|
+
const results: CheckResult[] = []
|
|
106
|
+
/**
|
|
107
|
+
* Inspect one host: its checks run in order, because they share that
|
|
108
|
+
* host's connection and a per-host sequence reads as one coherent sample.
|
|
109
|
+
*/
|
|
110
|
+
const inspectHost = async (alias: string): Promise<void> => {
|
|
111
|
+
for (const check of checks) {
|
|
112
|
+
// One unreachable host must not abort the round.
|
|
113
|
+
//
|
|
114
|
+
// An inspection runs on a timer against a whole fleet, and a fleet
|
|
115
|
+
// always contains at least one machine that is rebooting, firewalled
|
|
116
|
+
// or powered off. Letting engine.exec's rejection escape here would
|
|
117
|
+
// take down the entire round — and, since the round is what advances
|
|
118
|
+
// the alert state, the whole monitoring loop with it. The failure is
|
|
119
|
+
// recorded as a blind result (value null) against this one check, so
|
|
120
|
+
// it shows up as "we could not see" rather than as "the fleet is
|
|
121
|
+
// fine", which is what a missing row would have implied.
|
|
122
|
+
try {
|
|
123
|
+
const execResult = await engine.exec(alias, check.command, timeoutMs)
|
|
124
|
+
results.push(toResult(check, alias, execResult))
|
|
125
|
+
} catch (error) {
|
|
126
|
+
results.push({
|
|
127
|
+
checkId: check.id,
|
|
128
|
+
alias,
|
|
129
|
+
value: null,
|
|
130
|
+
breached: false,
|
|
131
|
+
raw: '',
|
|
132
|
+
error: error instanceof Error ? error.message : String(error),
|
|
133
|
+
})
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
// Bounded fan-out across hosts.
|
|
138
|
+
//
|
|
139
|
+
// A serial sweep is O(hosts) round trips: at 50 hosts and six checks that
|
|
140
|
+
// is 300 sequential commands, which can outlast the monitoring interval
|
|
141
|
+
// and make "every 60 seconds" a lie. Unbounded fan-out is worse — it asks
|
|
142
|
+
// the machine to hold one connection per host, which is exactly what a
|
|
143
|
+
// consumer desktop cannot do at 50+ targets. So the hosts are walked by a
|
|
144
|
+
// fixed worker pool: enough parallelism to finish inside the interval,
|
|
145
|
+
// never more sockets than the pool limit.
|
|
146
|
+
const workers = Math.max(1, Math.min(concurrency, targets.length))
|
|
147
|
+
const queue = [...targets]
|
|
148
|
+
const drain = async (): Promise<void> => {
|
|
149
|
+
for (;;) {
|
|
150
|
+
const alias = queue.shift()
|
|
151
|
+
if (alias === undefined) return
|
|
152
|
+
await inspectHost(alias)
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
await Promise.all(Array.from({ length: workers }, () => drain()))
|
|
156
|
+
// Restore a deterministic order: with parallel workers the completion
|
|
157
|
+
// order is a race, and the panel's per-host grouping and the round diff
|
|
158
|
+
// both read better (and compare equal across runs) when the rows follow
|
|
159
|
+
// the target and check order rather than whichever host answered first.
|
|
160
|
+
const aliasRank = new Map(targets.map((alias, index) => [alias, index]))
|
|
161
|
+
const checkRank = new Map(checks.map((check, index) => [check.id, index]))
|
|
162
|
+
results.sort((a, b) =>
|
|
163
|
+
(aliasRank.get(a.alias) ?? 0) - (aliasRank.get(b.alias) ?? 0)
|
|
164
|
+
|| (checkRank.get(a.checkId) ?? 0) - (checkRank.get(b.checkId) ?? 0))
|
|
165
|
+
const round: InspectionRound = {
|
|
166
|
+
startedAt,
|
|
167
|
+
durationMs: Date.now() - startedAt,
|
|
168
|
+
targets: [...targets],
|
|
169
|
+
results,
|
|
170
|
+
}
|
|
171
|
+
this.append(round)
|
|
172
|
+
return round
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Append one round to the history file.
|
|
177
|
+
*
|
|
178
|
+
* @param round - the round to store.
|
|
179
|
+
*/
|
|
180
|
+
append(round: InspectionRound): void {
|
|
181
|
+
const dir = dirname(this.path)
|
|
182
|
+
if (!existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 })
|
|
183
|
+
appendFileSync(this.path, JSON.stringify(round) + '\n', { encoding: 'utf8', mode: 0o600 })
|
|
184
|
+
// Notify the observer after the round is durable: a subscriber that
|
|
185
|
+
// needs the history (the metric store) must not see a round that a
|
|
186
|
+
// failed write left out of the file.
|
|
187
|
+
try {
|
|
188
|
+
this.onRound?.(round)
|
|
189
|
+
} catch {
|
|
190
|
+
// An observer failure is not an inspection failure. Swallowing it
|
|
191
|
+
// here keeps the round valid for everyone else.
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Called after every round is durably appended.
|
|
197
|
+
*
|
|
198
|
+
* Used by the metric store to derive series from rounds, so the tiers stay
|
|
199
|
+
* current whether the round came from the timer or from a tool call.
|
|
200
|
+
*/
|
|
201
|
+
onRound?: (round: InspectionRound) => void
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Read the most recent rounds, oldest first.
|
|
205
|
+
*
|
|
206
|
+
* @param limit - how many rounds to return.
|
|
207
|
+
* @returns the rounds.
|
|
208
|
+
*/
|
|
209
|
+
read(limit = 20): InspectionRound[] {
|
|
210
|
+
if (!existsSync(this.path)) return []
|
|
211
|
+
let text: string
|
|
212
|
+
try {
|
|
213
|
+
text = readFileSync(this.path, 'utf8')
|
|
214
|
+
} catch {
|
|
215
|
+
return []
|
|
216
|
+
}
|
|
217
|
+
const lines = text.split('\n').filter(line => line.trim() !== '')
|
|
218
|
+
const rounds: InspectionRound[] = []
|
|
219
|
+
for (const line of lines.slice(-limit)) {
|
|
220
|
+
try {
|
|
221
|
+
rounds.push(JSON.parse(line) as InspectionRound)
|
|
222
|
+
} catch {
|
|
223
|
+
// A truncated line (a crash mid-write) must not hide the rest.
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
return rounds
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Compare the two newest rounds.
|
|
231
|
+
*
|
|
232
|
+
* @returns the changes, newest round against the one before it. An empty
|
|
233
|
+
* array means either fewer than two rounds exist or nothing changed — the
|
|
234
|
+
* caller distinguishes the two by looking at {@link read}.
|
|
235
|
+
*/
|
|
236
|
+
diff(): HostChange[] {
|
|
237
|
+
const rounds = this.read(2)
|
|
238
|
+
if (rounds.length < 2) return []
|
|
239
|
+
const [previous, current] = rounds as [InspectionRound, InspectionRound]
|
|
240
|
+
const before = indexResults(previous)
|
|
241
|
+
const changes: HostChange[] = []
|
|
242
|
+
for (const result of current.results) {
|
|
243
|
+
const check = checkOf(result.checkId)
|
|
244
|
+
const key = `${result.alias}\u0000${result.checkId}`
|
|
245
|
+
const prior = before.get(key)
|
|
246
|
+
const after = formatValue(check, result.value)
|
|
247
|
+
if (prior === undefined) {
|
|
248
|
+
// A host or check that was not in the previous round: report it as
|
|
249
|
+
// new rather than pretending the value appeared unchanged.
|
|
250
|
+
changes.push({ alias: result.alias, checkId: result.checkId, title: check.title, after })
|
|
251
|
+
continue
|
|
252
|
+
}
|
|
253
|
+
const beforeText = formatValue(check, prior.value)
|
|
254
|
+
const stateChange = prior.breached === result.breached
|
|
255
|
+
? undefined
|
|
256
|
+
: result.breached ? 'new-breach' as const : 'recovered' as const
|
|
257
|
+
if (beforeText === after && stateChange === undefined) continue
|
|
258
|
+
changes.push({
|
|
259
|
+
alias: result.alias,
|
|
260
|
+
checkId: result.checkId,
|
|
261
|
+
title: check.title,
|
|
262
|
+
before: beforeText,
|
|
263
|
+
after,
|
|
264
|
+
...(stateChange !== undefined ? { stateChange } : {}),
|
|
265
|
+
})
|
|
266
|
+
}
|
|
267
|
+
return changes
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* The checks selected by the given ids.
|
|
273
|
+
*
|
|
274
|
+
* An unknown id throws: silently skipping it would let a typo look like a
|
|
275
|
+
* clean inspection.
|
|
276
|
+
*
|
|
277
|
+
* @param checkIds - the requested ids, or undefined for all.
|
|
278
|
+
* @returns the checks to run.
|
|
279
|
+
*/
|
|
280
|
+
export function selectChecks(checkIds?: string[]): readonly Check[] {
|
|
281
|
+
if (checkIds === undefined || checkIds.length === 0) return BUILTIN_CHECKS
|
|
282
|
+
const selected: Check[] = []
|
|
283
|
+
for (const id of checkIds) {
|
|
284
|
+
const check = BUILTIN_CHECKS.find(candidate => candidate.id === id)
|
|
285
|
+
if (check === undefined) {
|
|
286
|
+
throw new Error(`unknown check '${id}' — available: ${BUILTIN_CHECKS.map(candidate => candidate.id).join(', ')}`)
|
|
287
|
+
}
|
|
288
|
+
selected.push(check)
|
|
289
|
+
}
|
|
290
|
+
return selected
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** Index a round's results by host+check for lookup. */
|
|
294
|
+
function indexResults(round: InspectionRound): Map<string, CheckResult> {
|
|
295
|
+
const map = new Map<string, CheckResult>()
|
|
296
|
+
for (const result of round.results) {
|
|
297
|
+
map.set(`${result.alias}\u0000${result.checkId}`, result)
|
|
298
|
+
}
|
|
299
|
+
return map
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** The check a stored result belongs to (falls back to a stub for a removed check). */
|
|
303
|
+
function checkOf(checkId: string): Check {
|
|
304
|
+
const found = BUILTIN_CHECKS.find(candidate => candidate.id === checkId)
|
|
305
|
+
if (found !== undefined) return found
|
|
306
|
+
// A check id that no longer exists (a check was renamed or removed):
|
|
307
|
+
// keep the stored value readable instead of throwing.
|
|
308
|
+
return {
|
|
309
|
+
id: checkId,
|
|
310
|
+
title: checkId,
|
|
311
|
+
kind: 'text',
|
|
312
|
+
direction: 'above',
|
|
313
|
+
command: '',
|
|
314
|
+
parse: () => null,
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/** Project one exec result onto a stored check result. */
|
|
319
|
+
function toResult(check: Check, alias: string, result: ExecResult): CheckResult {
|
|
320
|
+
return toCheckResult(check, alias, result)
|
|
321
|
+
}
|