dsh-mesh-node 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/core/cluster.js +341 -340
- package/src/core/hardware.js +57 -0
- package/src/core/sakura-api.js +103 -0
- package/src/core/tunnels.js +212 -0
- package/src/server.js +317 -313
- package/src/tools/core.js +2 -0
- package/src/tools/host.js +3 -1
package/src/core/cluster.js
CHANGED
|
@@ -1,340 +1,341 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Cluster view (M7 step 4).
|
|
3
|
-
*
|
|
4
|
-
* A single read-only snapshot of everything this node can currently reach:
|
|
5
|
-
* - local: every DSH instance behind a host bridge (id / pid / tool count / live agents)
|
|
6
|
-
* - remote: every paired peer's `/health` (version / tools / uptime / TLS pin)
|
|
7
|
-
*
|
|
8
|
-
* The point is to answer "is the cluster healthy, and if not, which hop is broken"
|
|
9
|
-
* from one call, instead of hand-running `host status` plus curl on each peer.
|
|
10
|
-
* Nothing here mutates state; all probes are bounded by `timeoutMs`.
|
|
11
|
-
*/
|
|
12
|
-
import { request as httpRequest } from 'node:http'
|
|
13
|
-
import { request as httpsRequest } from 'node:https'
|
|
14
|
-
import { createHostClient, hostInstances } from './host-agent.js'
|
|
15
|
-
import { MESH_VERSION } from './config.js'
|
|
16
|
-
import { callPeerTool } from './peer-client.js'
|
|
17
|
-
import { fingerprintOfSocket, normalizeFingerprint } from './tls.js'
|
|
18
|
-
|
|
19
|
-
const DEFAULT_TIMEOUT_MS = 4000
|
|
20
|
-
|
|
21
|
-
function normalizeUrl(url) {
|
|
22
|
-
return String(url || '').replace(/\/+$/, '')
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
function normalizePeerHealth(body) {
|
|
26
|
-
if (!body || body.service !== 'dsh-mesh') {
|
|
27
|
-
return { error: { code: 'E_PEER_BAD_RESPONSE', message: 'peer /health did not identify as dsh-mesh' } }
|
|
28
|
-
}
|
|
29
|
-
return {
|
|
30
|
-
version: body.version,
|
|
31
|
-
node: body.node,
|
|
32
|
-
tools: body.tools,
|
|
33
|
-
pid: body.pid ?? null,
|
|
34
|
-
uptimeSec: body.uptimeSec,
|
|
35
|
-
tls: body.tls === true,
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
const
|
|
52
|
-
|
|
53
|
-
settled
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
const
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
//
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
res.on('
|
|
80
|
-
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
req.
|
|
148
|
-
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
const
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
const
|
|
160
|
-
const
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
*
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
*
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
const
|
|
193
|
-
const
|
|
194
|
-
const
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
*
|
|
220
|
-
*
|
|
221
|
-
*
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
const
|
|
230
|
-
const
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
*
|
|
247
|
-
*
|
|
248
|
-
*
|
|
249
|
-
*
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
const
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
else if (peer.
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
//
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
const
|
|
313
|
-
const
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
}
|
|
340
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Cluster view (M7 step 4).
|
|
3
|
+
*
|
|
4
|
+
* A single read-only snapshot of everything this node can currently reach:
|
|
5
|
+
* - local: every DSH instance behind a host bridge (id / pid / tool count / live agents)
|
|
6
|
+
* - remote: every paired peer's `/health` (version / tools / uptime / TLS pin)
|
|
7
|
+
*
|
|
8
|
+
* The point is to answer "is the cluster healthy, and if not, which hop is broken"
|
|
9
|
+
* from one call, instead of hand-running `host status` plus curl on each peer.
|
|
10
|
+
* Nothing here mutates state; all probes are bounded by `timeoutMs`.
|
|
11
|
+
*/
|
|
12
|
+
import { request as httpRequest } from 'node:http'
|
|
13
|
+
import { request as httpsRequest } from 'node:https'
|
|
14
|
+
import { createHostClient, hostInstances } from './host-agent.js'
|
|
15
|
+
import { MESH_VERSION } from './config.js'
|
|
16
|
+
import { callPeerTool } from './peer-client.js'
|
|
17
|
+
import { fingerprintOfSocket, normalizeFingerprint } from './tls.js'
|
|
18
|
+
|
|
19
|
+
const DEFAULT_TIMEOUT_MS = 4000
|
|
20
|
+
|
|
21
|
+
function normalizeUrl(url) {
|
|
22
|
+
return String(url || '').replace(/\/+$/, '')
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function normalizePeerHealth(body) {
|
|
26
|
+
if (!body || body.service !== 'dsh-mesh') {
|
|
27
|
+
return { error: { code: 'E_PEER_BAD_RESPONSE', message: 'peer /health did not identify as dsh-mesh' } }
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
version: body.version,
|
|
31
|
+
node: body.node,
|
|
32
|
+
tools: body.tools,
|
|
33
|
+
pid: body.pid ?? null,
|
|
34
|
+
uptimeSec: body.uptimeSec,
|
|
35
|
+
tls: body.tls === true,
|
|
36
|
+
tunnels: body.tunnels || null,
|
|
37
|
+
pairing: body.pairing ? { enabled: body.pairing.enabled === true, codeActive: body.pairing.codeActive === true } : null
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Probe one paired peer over HTTPS. Uses TOFU pinning (peers store the cert
|
|
43
|
+
* fingerprint captured at pairing time): the trust chain is not judged, the
|
|
44
|
+
* certificate is read off the socket and compared against the stored pin.
|
|
45
|
+
*/
|
|
46
|
+
export function probePeer(peer, { timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
|
|
47
|
+
const base = normalizeUrl(peer?.url)
|
|
48
|
+
const expected = peer?.certFingerprint ? normalizeFingerprint(peer.certFingerprint) : null
|
|
49
|
+
return new Promise((resolve) => {
|
|
50
|
+
let settled = false
|
|
51
|
+
const startedAt = Date.now()
|
|
52
|
+
const finish = (value) => {
|
|
53
|
+
if (settled) return
|
|
54
|
+
settled = true
|
|
55
|
+
clearTimeout(timer)
|
|
56
|
+
resolve({ id: peer?.id || null, name: peer?.name || null, url: base, ...value })
|
|
57
|
+
}
|
|
58
|
+
if (!base) {
|
|
59
|
+
return finish({ up: false, latencyMs: 0, error: { code: 'E_PEER_BAD_RESPONSE', message: 'peer has no url' } })
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const isHttps = base.startsWith('https:')
|
|
63
|
+
const request = isHttps ? httpsRequest : httpRequest
|
|
64
|
+
const options = { method: 'GET', agent: false, headers: { accept: 'application/json' } }
|
|
65
|
+
if (isHttps) options.rejectUnauthorized = false
|
|
66
|
+
|
|
67
|
+
const req = request(`${base}/health`, options, (res) => {
|
|
68
|
+
// Grab the certificate *inside* the response callback: by the time `end`
|
|
69
|
+
// fires the socket may already be gone (same trap as company-check.mjs).
|
|
70
|
+
let got = null
|
|
71
|
+
if (isHttps) {
|
|
72
|
+
try {
|
|
73
|
+
got = fingerprintOfSocket(res.socket)
|
|
74
|
+
} catch {
|
|
75
|
+
got = null
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const chunks = []
|
|
79
|
+
res.on('data', (chunk) => chunks.push(chunk))
|
|
80
|
+
res.on('end', () => {
|
|
81
|
+
const latencyMs = Date.now() - startedAt
|
|
82
|
+
const text = Buffer.concat(chunks).toString('utf8')
|
|
83
|
+
let body
|
|
84
|
+
try {
|
|
85
|
+
body = JSON.parse(text)
|
|
86
|
+
} catch {
|
|
87
|
+
return finish({
|
|
88
|
+
up: false,
|
|
89
|
+
latencyMs,
|
|
90
|
+
fingerprint: got,
|
|
91
|
+
fingerprintMatch: got && expected ? normalizeFingerprint(got) === expected : null,
|
|
92
|
+
error: { code: 'E_PEER_BAD_RESPONSE', message: `peer /health returned non-JSON (${res.statusCode})` }
|
|
93
|
+
})
|
|
94
|
+
}
|
|
95
|
+
const match = got && expected ? normalizeFingerprint(got) === expected : null
|
|
96
|
+
if (res.statusCode !== 200) {
|
|
97
|
+
return finish({
|
|
98
|
+
up: false,
|
|
99
|
+
latencyMs,
|
|
100
|
+
status: res.statusCode,
|
|
101
|
+
fingerprint: got,
|
|
102
|
+
fingerprintMatch: match,
|
|
103
|
+
error: { code: 'E_PEER_BAD_RESPONSE', message: `peer /health returned ${res.statusCode}` }
|
|
104
|
+
})
|
|
105
|
+
}
|
|
106
|
+
if (match === false) {
|
|
107
|
+
const health = normalizePeerHealth(body)
|
|
108
|
+
return finish({
|
|
109
|
+
up: true,
|
|
110
|
+
latencyMs,
|
|
111
|
+
fingerprint: got,
|
|
112
|
+
fingerprintMatch: false,
|
|
113
|
+
...health,
|
|
114
|
+
error: {
|
|
115
|
+
code: 'E_PEER_PIN_MISMATCH',
|
|
116
|
+
message: 'peer certificate does not match the fingerprint stored at pairing time — refuse to trust it'
|
|
117
|
+
}
|
|
118
|
+
})
|
|
119
|
+
}
|
|
120
|
+
const health = normalizePeerHealth(body)
|
|
121
|
+
return finish({ up: !health.error, latencyMs, fingerprint: got, fingerprintMatch: match, ...health })
|
|
122
|
+
})
|
|
123
|
+
})
|
|
124
|
+
const timer = setTimeout(() => {
|
|
125
|
+
try {
|
|
126
|
+
req.destroy(new Error('timeout'))
|
|
127
|
+
} catch {
|
|
128
|
+
/* ignore */
|
|
129
|
+
}
|
|
130
|
+
finish({
|
|
131
|
+
up: false,
|
|
132
|
+
latencyMs: Date.now() - startedAt,
|
|
133
|
+
error: { code: 'E_PEER_TIMEOUT', message: `peer did not answer within ${timeoutMs} ms` }
|
|
134
|
+
})
|
|
135
|
+
}, timeoutMs)
|
|
136
|
+
req.on('error', (error) => {
|
|
137
|
+
finish({
|
|
138
|
+
up: false,
|
|
139
|
+
latencyMs: Date.now() - startedAt,
|
|
140
|
+
error: {
|
|
141
|
+
code: error?.code === 'ECONNRESET' || error?.code === 'ECONNREFUSED' ? 'E_PEER_OFFLINE' : 'E_PEER_TLS',
|
|
142
|
+
message: `${base} unreachable (${error?.code || error?.message || error})`,
|
|
143
|
+
detail: error?.code || null
|
|
144
|
+
}
|
|
145
|
+
})
|
|
146
|
+
})
|
|
147
|
+
req.setTimeout(timeoutMs, () => req.destroy(new Error('timeout')))
|
|
148
|
+
req.end()
|
|
149
|
+
})
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Probe one local DSH instance through its host bridge. */
|
|
153
|
+
export async function probeInstance({ config, home, log, entry, timeoutMs = DEFAULT_TIMEOUT_MS }) {
|
|
154
|
+
const startedAt = Date.now()
|
|
155
|
+
const base = { id: entry.id, label: entry.label || entry.id, url: entry.url, profileDir: entry.profileDir || '' }
|
|
156
|
+
try {
|
|
157
|
+
const client = createHostClient({ config, home, log, instance: entry.id })
|
|
158
|
+
await client.ping()
|
|
159
|
+
const [self, tools, agents] = await Promise.all([client.self(), client.tools(), client.agents()])
|
|
160
|
+
const toolList = tools?.tools || []
|
|
161
|
+
const agentList = agents?.agents || []
|
|
162
|
+
return {
|
|
163
|
+
...base,
|
|
164
|
+
up: true,
|
|
165
|
+
latencyMs: Date.now() - startedAt,
|
|
166
|
+
plugin: self?.plugin || null,
|
|
167
|
+
hostname: self?.node?.hostname || null,
|
|
168
|
+
platform: self?.node?.platform || null,
|
|
169
|
+
pid: self?.node?.pid || null,
|
|
170
|
+
toolCount: toolList.length,
|
|
171
|
+
agents: agentList.map((agent) => ({ id: agent.id, status: agent.status, title: agent.title || '' })),
|
|
172
|
+
runningAgents: agentList.filter((agent) => agent.status === 'running').length
|
|
173
|
+
}
|
|
174
|
+
} catch (error) {
|
|
175
|
+
return {
|
|
176
|
+
...base,
|
|
177
|
+
up: false,
|
|
178
|
+
latencyMs: Date.now() - startedAt,
|
|
179
|
+
error: { code: error?.code || 'E_HOST_OFFLINE', message: error?.message || String(error) }
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Probe this node's OWN serve endpoint (`/health`) — the hop every peer needs.
|
|
186
|
+
* The host-bridge probes below only prove the DSH instances are up; when the
|
|
187
|
+
* local mesh server dies they all still look fine and only the peers look
|
|
188
|
+
* offline. Reported 2026-10-23: the company endpoint had been down ~3 days
|
|
189
|
+
* while `self-update status --deep` read clean.
|
|
190
|
+
*/
|
|
191
|
+
export async function probeSelfEndpoint(config, { timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
|
|
192
|
+
const node = config?.node || {}
|
|
193
|
+
const scheme = node.tls?.enabled ? 'https' : 'http'
|
|
194
|
+
const url = `${scheme}://${node.bind || '127.0.0.1'}:${node.port || 8124}`
|
|
195
|
+
const probe = await probePeer({ id: 'self', name: 'self', url }, { timeoutMs })
|
|
196
|
+
return {
|
|
197
|
+
url,
|
|
198
|
+
up: probe.up === true,
|
|
199
|
+
version: probe.version || null,
|
|
200
|
+
pid: probe.pid ?? null,
|
|
201
|
+
tools: probe.tools ?? null,
|
|
202
|
+
uptimeSec: probe.uptimeSec ?? null,
|
|
203
|
+
error: probe.error || null
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Parse the JSON payload of an MCP tool result (remote tools return text content). */
|
|
208
|
+
function parsePeerToolPayload(result) {
|
|
209
|
+
const block = result?.content?.[0]
|
|
210
|
+
if (!block || block.type !== 'text') return null
|
|
211
|
+
try {
|
|
212
|
+
return JSON.parse(block.text)
|
|
213
|
+
} catch {
|
|
214
|
+
return null
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Ask a paired peer for its OWN host instances through the MCP `host_instances`
|
|
220
|
+
* tool, so the cluster view can show all machines' DSH instances in one place.
|
|
221
|
+
* This is best-effort: a peer that denies the tool (or has no token) still gets
|
|
222
|
+
* its /health row — we just record `instancesError` instead of failing the view.
|
|
223
|
+
*/
|
|
224
|
+
async function probePeerInstances(peer, timeoutMs) {
|
|
225
|
+
if (!peer?.token) {
|
|
226
|
+
return { error: { code: 'E_PEER_NO_TOKEN', message: 'peer has no outbound token in peers.json — cannot list its host instances' } }
|
|
227
|
+
}
|
|
228
|
+
try {
|
|
229
|
+
const result = await callPeerTool(peer, { tool: 'host_instances', args: {}, timeoutMs })
|
|
230
|
+
const body = parsePeerToolPayload(result.result)
|
|
231
|
+
const rows = Array.isArray(body?.instances) ? body.instances : []
|
|
232
|
+
return {
|
|
233
|
+
instances: rows.map((row) => ({
|
|
234
|
+
id: row.id,
|
|
235
|
+
label: row.label || row.id,
|
|
236
|
+
url: row.url,
|
|
237
|
+
alive: row.alive === true
|
|
238
|
+
}))
|
|
239
|
+
}
|
|
240
|
+
} catch (error) {
|
|
241
|
+
return { error: { code: error?.code || 'E_PEER_CALL_FAILED', message: error?.message || String(error) } }
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* @param {{ probeSelf?: boolean }} options `probeSelf: true` additionally probes
|
|
247
|
+
* this node's own serve endpoint, so a CLI running *outside* the server can tell
|
|
248
|
+
* "my host bridges are fine" apart from "my node is actually serving peers".
|
|
249
|
+
* Off by default: callers inside the server would only measure themselves, and
|
|
250
|
+
* tests use `node.port = 0`.
|
|
251
|
+
*/
|
|
252
|
+
export async function clusterReport({ config, home, log, peers = [], timeoutMs = DEFAULT_TIMEOUT_MS, probeSelf = false } = {}) {
|
|
253
|
+
const endpoint = probeSelf ? await probeSelfEndpoint(config, { timeoutMs }) : null
|
|
254
|
+
const instances = []
|
|
255
|
+
for (const entry of hostInstances(config)) {
|
|
256
|
+
if (entry.enabled === false) continue
|
|
257
|
+
instances.push(await probeInstance({ config, home, log, entry, timeoutMs }))
|
|
258
|
+
}
|
|
259
|
+
const peerReports = await Promise.all(
|
|
260
|
+
peers.map(async (peer) => {
|
|
261
|
+
const report = await probePeer(peer, { timeoutMs })
|
|
262
|
+
if (report.up === true && !report.error) {
|
|
263
|
+
const remote = await probePeerInstances(peer, timeoutMs)
|
|
264
|
+
report.instances = remote.instances || []
|
|
265
|
+
if (remote.error) report.instancesError = remote.error
|
|
266
|
+
}
|
|
267
|
+
return report
|
|
268
|
+
})
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
const issues = []
|
|
272
|
+
if (endpoint && !endpoint.up) {
|
|
273
|
+
issues.push(`local endpoint ${endpoint.url} down: ${endpoint.error?.code || 'E_ENDPOINT_OFFLINE'}`)
|
|
274
|
+
}
|
|
275
|
+
for (const instance of instances) {
|
|
276
|
+
if (!instance.up) issues.push(`instance ${instance.id} down: ${instance.error?.code || 'E_HOST_OFFLINE'}`)
|
|
277
|
+
else if (instance.runningAgents > 0) issues.push(`instance ${instance.id} has ${instance.runningAgents} running agent(s)`)
|
|
278
|
+
}
|
|
279
|
+
for (const peer of peerReports) {
|
|
280
|
+
if (!peer.up) issues.push(`peer ${peer.name || peer.id || peer.url} down: ${peer.error?.code || 'E_PEER_OFFLINE'}`)
|
|
281
|
+
else if (peer.fingerprintMatch === false) issues.push(`peer ${peer.name || peer.id} fingerprint mismatch`)
|
|
282
|
+
else if (peer.instancesError) {
|
|
283
|
+
issues.push(`peer ${peer.name || peer.id} host instances unavailable: ${peer.instancesError.code}`)
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Flatten remote instances next to local ones so one call shows the whole
|
|
288
|
+
// cluster (four DSH instances on two machines). `scope` says where a row lives.
|
|
289
|
+
const localRows = instances.map((instance) => ({
|
|
290
|
+
scope: 'local',
|
|
291
|
+
id: instance.id,
|
|
292
|
+
label: instance.label,
|
|
293
|
+
up: instance.up === true,
|
|
294
|
+
toolCount: instance.toolCount ?? null,
|
|
295
|
+
pid: instance.pid ?? null
|
|
296
|
+
}))
|
|
297
|
+
const remoteRows = []
|
|
298
|
+
for (const peer of peerReports) {
|
|
299
|
+
for (const row of peer.instances || []) {
|
|
300
|
+
remoteRows.push({
|
|
301
|
+
scope: peer.name || peer.id,
|
|
302
|
+
id: row.id,
|
|
303
|
+
label: row.label,
|
|
304
|
+
url: row.url,
|
|
305
|
+
up: row.alive === true,
|
|
306
|
+
remote: true
|
|
307
|
+
})
|
|
308
|
+
if (row.alive !== true) issues.push(`peer ${peer.name || peer.id} instance ${row.id} down`)
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const anyInstanceUp = instances.some((instance) => instance.up)
|
|
313
|
+
const remoteAllAlive = remoteRows.every((row) => row.up === true)
|
|
314
|
+
const allUp =
|
|
315
|
+
(!endpoint || endpoint.up) &&
|
|
316
|
+
instances.every((instance) => instance.up) &&
|
|
317
|
+
peerReports.every((peer) => peer.up && peer.fingerprintMatch !== false) &&
|
|
318
|
+
remoteAllAlive
|
|
319
|
+
|
|
320
|
+
return {
|
|
321
|
+
node: { version: MESH_VERSION, home, instances: instances.length, peers: peerReports.length },
|
|
322
|
+
summary: { local: localRows.length, remote: remoteRows.length, total: localRows.length + remoteRows.length },
|
|
323
|
+
endpoint,
|
|
324
|
+
instances,
|
|
325
|
+
remoteInstances: remoteRows,
|
|
326
|
+
cluster: [...localRows, ...remoteRows],
|
|
327
|
+
peers: peerReports,
|
|
328
|
+
verdict:
|
|
329
|
+
endpoint && !endpoint.up
|
|
330
|
+
? 'down'
|
|
331
|
+
: instances.length === 0
|
|
332
|
+
? 'unknown'
|
|
333
|
+
: allUp
|
|
334
|
+
? 'ok'
|
|
335
|
+
: anyInstanceUp
|
|
336
|
+
? 'degraded'
|
|
337
|
+
: 'down',
|
|
338
|
+
issues
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|