dsh-server-dashboard 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,718 @@
1
+ import z from 'schemastery';
2
+ import { credentialRef } from '@deepseek-ai/dsh-credentials';
3
+ import { collectSnapshot, discoverLogs, testConnection, disposeState } from './host/collector.js';
4
+ import { parseSshConfig } from './host/sshconfig.js';
5
+ import { TEMP_HOT } from './client/thresholds.js';
6
+ export const name = 'dsh-server-dashboard';
7
+ /** Host services this plugin waits for (registry, credentials vault, http carrier). */
8
+ export const inject = ['settings', 'credentials', 'webServer'];
9
+ const HostSchema = z.object({
10
+ id: z.string().required(),
11
+ name: z.string().required(),
12
+ host: z.string().required(),
13
+ port: z.number().step(1).min(1).max(65535).default(22),
14
+ username: z.string().required(),
15
+ authKind: z.union(['password', 'key', 'none']).default('none'),
16
+ credentialRef: z.string().default(''),
17
+ identityFile: z.string().default(''),
18
+ logPath: z.string().default(''),
19
+ pinned: z.boolean().default(false),
20
+ });
21
+ const DashboardConfigSchema = z.object({
22
+ hosts: z.array(HostSchema).default([]),
23
+ refreshIntervalS: z.number().step(1).min(10).max(300).default(30),
24
+ staleMinutes: z.number().step(1).min(1).max(1440).default(10),
25
+ });
26
+ // 0.1.5 起 dsh-settings 移除了 settingsNamespace 包装:命名空间直接以字面量传入
27
+ const NS = 'server-dashboard';
28
+ const CRED_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
29
+ async function readIdentityFile(path) {
30
+ const { readFile } = await import('node:fs/promises');
31
+ try {
32
+ return await readFile(path, 'utf8');
33
+ }
34
+ catch (err) {
35
+ throw new Error(`无法读取密钥文件 ${path}: ${err instanceof Error ? err.message : String(err)}`);
36
+ }
37
+ }
38
+ /** Server-derived credential name for a host row — one host maps to exactly one
39
+ * credential. Client-supplied credentialRef values are display-only and are
40
+ * never used to address the vault (prevents overwriting arbitrary existing
41
+ * credentials via PUT /dash/config). */
42
+ function derivedCredName(hostId) {
43
+ return `SD_${hostId.replace(/[^A-Za-z0-9_]/g, '_')}`;
44
+ }
45
+ async function resolveAuth(ctx, cfg, cache) {
46
+ const ref = derivedCredName(cfg.id);
47
+ const sig = `${cfg.authKind}|${cfg.username}@${cfg.host}:${cfg.port}`;
48
+ let identityMtimeMs = -1;
49
+ if (cfg.authKind === 'key' && cfg.identityFile) {
50
+ try {
51
+ const { stat } = await import('node:fs/promises');
52
+ identityMtimeMs = Math.round((await stat(cfg.identityFile)).mtimeMs);
53
+ }
54
+ catch {
55
+ identityMtimeMs = -1;
56
+ }
57
+ }
58
+ const hit = cache?.get(cfg.id);
59
+ if (hit && hit.credentialRef === ref && hit.sig === sig && hit.identityMtimeMs === identityMtimeMs)
60
+ return hit.auth;
61
+ const auth = { host: cfg.host, port: cfg.port, username: cfg.username };
62
+ // 1) credential vault (explicitly stored private key / password) — always
63
+ // addressed by the SERVER-DERIVED name, never by a client-supplied ref
64
+ if (cfg.authKind !== 'none') {
65
+ let cred = await ctx.credentials.resolve(credentialRef(ref)).catch(() => undefined);
66
+ // legacy migration: pre-derivation configs stored credentials under the
67
+ // client-supplied ref (SD_L40_KEY style). If the derived name is empty but
68
+ // the legacy ref holds a value, copy it across once — self-heals existing
69
+ // installs without weakening the derived-name model.
70
+ if (!cred?.value && cfg.credentialRef && CRED_NAME.test(cfg.credentialRef) && cfg.credentialRef !== ref) {
71
+ const legacy = await ctx.credentials.resolve(credentialRef(cfg.credentialRef)).catch(() => undefined);
72
+ if (legacy?.value) {
73
+ await ctx.credentials.set(credentialRef(ref), legacy.value).catch(() => undefined);
74
+ cred = legacy;
75
+ }
76
+ }
77
+ const value = cred?.value;
78
+ if (value) {
79
+ if (cfg.authKind === 'password')
80
+ auth.password = String(value);
81
+ else
82
+ auth.privateKey = String(value);
83
+ }
84
+ }
85
+ // 2) identity file referenced by an import from ~/.ssh/config
86
+ if (cfg.authKind === 'key' && !auth.privateKey && cfg.identityFile) {
87
+ auth.privateKey = await readIdentityFile(cfg.identityFile);
88
+ }
89
+ // 3) authKind 'none': mirror OpenSSH default identities
90
+ if (cfg.authKind === 'none' && !auth.privateKey && !auth.password) {
91
+ auth.privateKey = await readDefaultIdentity();
92
+ }
93
+ cache?.set(cfg.id, { credentialRef: ref, sig, identityMtimeMs, auth });
94
+ return auth;
95
+ }
96
+ /** OpenSSH default-identity order for authKind 'none' (unencrypted keys only). */
97
+ async function readDefaultIdentity() {
98
+ const home = process.env.USERPROFILE ?? process.env.HOME ?? '';
99
+ const { readFile } = await import('node:fs/promises');
100
+ for (const f of ['id_ed25519', 'id_ecdsa', 'id_rsa']) {
101
+ try {
102
+ return await readFile(`${home}/.ssh/${f}`, 'utf8');
103
+ }
104
+ catch {
105
+ // try the next default identity
106
+ }
107
+ }
108
+ return undefined;
109
+ }
110
+ /** Auth resolution for one-off route bodies (test / discover-logs): same chain
111
+ * as polling — inline secret → credential vault → identityFile → default key. */
112
+ async function resolveAuthFromBody(ctx, body) {
113
+ const auth = { host: body.host, port: body.port ?? 22, username: body.username ?? 'root' };
114
+ const kind = body.authKind ?? 'none';
115
+ if (kind === 'password') {
116
+ if (body.password)
117
+ auth.password = body.password;
118
+ else if (body.credentialRef && CRED_NAME.test(body.credentialRef)) {
119
+ const hit = await ctx.credentials.resolve(credentialRef(body.credentialRef));
120
+ if (hit?.value)
121
+ auth.password = String(hit.value);
122
+ }
123
+ }
124
+ else if (kind === 'key') {
125
+ if (body.privateKey)
126
+ auth.privateKey = body.privateKey;
127
+ else if (body.credentialRef && CRED_NAME.test(body.credentialRef)) {
128
+ const hit = await ctx.credentials.resolve(credentialRef(body.credentialRef));
129
+ if (hit?.value)
130
+ auth.privateKey = String(hit.value);
131
+ }
132
+ else if (body.identityFile)
133
+ auth.privateKey = await readIdentityFile(body.identityFile);
134
+ }
135
+ else {
136
+ auth.privateKey = await readDefaultIdentity();
137
+ }
138
+ return auth;
139
+ }
140
+ /**
141
+ * Refresh the observation cursors from one round of snapshots. Targets are
142
+ * keyed by log path; an mtime/size change (or a first observation, which gets
143
+ * a full-threshold grace period) bumps changeAt to `now`. Failed snapshots
144
+ * keep their previous cursors — a transient collect failure must not fake
145
+ * either freshness or staleness. Pure (unit-testable).
146
+ */
147
+ export function refreshLogActivity(snapshots, prev, now) {
148
+ const next = {};
149
+ for (const [hostId, snap] of Object.entries(snapshots)) {
150
+ if (!snap || !snap.ok) {
151
+ if (prev[hostId])
152
+ next[hostId] = prev[hostId];
153
+ continue;
154
+ }
155
+ const targets = [];
156
+ if (snap.log && snap.log.mtimeMs > 0)
157
+ targets.push(snap.log);
158
+ for (const g of snap.gpus ?? [])
159
+ if (g.log && g.log.mtimeMs > 0)
160
+ targets.push(g.log);
161
+ const prevRecord = prev[hostId] ?? {};
162
+ const record = {};
163
+ for (const t of targets) {
164
+ const p = prevRecord[t.path];
165
+ record[t.path] = p && p.mtimeMs === t.mtimeMs && p.size === t.size
166
+ ? { mtimeMs: t.mtimeMs, size: t.size, changeAt: p.changeAt } // unchanged — keep the last change moment
167
+ : { mtimeMs: t.mtimeMs, size: t.size, changeAt: now }; // changed or first sight — grace from now
168
+ }
169
+ next[hostId] = record;
170
+ }
171
+ return next;
172
+ }
173
+ /** Is a target fresh under the incremental rule? (pure) */
174
+ function activityFresh(record, path, now, staleMs) {
175
+ const a = record?.[path];
176
+ return !!a && now - a.changeAt < staleMs;
177
+ }
178
+ /**
179
+ * Host-level logPath lifecycle:
180
+ * - a configured path whose log shows no mtime/size CHANGE for ≥ staleMinutes
181
+ * (observation-based, immune to remote clock skew) is cleared — the
182
+ * experiment ended, stop flagging it;
183
+ * - an empty path adopts the freshest per-GPU log (still being written; ties
184
+ * broken by the largest remote mtime, comparable within one host) so the
185
+ * card-level log follows whatever experiment starts next.
186
+ * Pure logic (unit-testable); the caller persists the returned patches.
187
+ */
188
+ export function planLogPathUpdates(config, snapshots, activity, staleMinutes, now) {
189
+ const staleMs = staleMinutes * 60_000;
190
+ const out = [];
191
+ for (const hostCfg of config.hosts) {
192
+ const snap = snapshots[hostCfg.id];
193
+ if (!snap || !snap.ok)
194
+ continue;
195
+ const record = activity[hostCfg.id];
196
+ if (hostCfg.logPath && snap.log && snap.log.mtimeMs > 0 && snap.log.path === hostCfg.logPath) {
197
+ // stalled = no observed change within the threshold (first observation
198
+ // starts its grace period, so a genuinely old log is not cleared at once)
199
+ if (!activityFresh(record, hostCfg.logPath, now, staleMs))
200
+ out.push({ id: hostCfg.id, logPath: '' });
201
+ }
202
+ else if (!hostCfg.logPath) {
203
+ // adopt: freshest changed-recently per-GPU log; mtimeMs tiebreak compares
204
+ // clocks of the SAME remote host, so skew cannot reorder the candidates
205
+ let best;
206
+ for (const g of snap.gpus ?? []) {
207
+ const l = g.log;
208
+ if (!l || l.mtimeMs <= 0 || !activityFresh(record, l.path, now, staleMs))
209
+ continue;
210
+ if (!best || l.mtimeMs > best.mtimeMs)
211
+ best = l;
212
+ }
213
+ if (best)
214
+ out.push({ id: hostCfg.id, logPath: best.path });
215
+ }
216
+ }
217
+ return out;
218
+ }
219
+ export function apply(ctx) {
220
+ const scope = ctx.settings.register(NS, DashboardConfigSchema, {});
221
+ const hostState = new Map();
222
+ // last good snapshot cache, so a transient collect failure still renders
223
+ const cache = new Map();
224
+ // time-based exponential backoff per host: consecutive failures push the next
225
+ // probe 1min → 2min → 5min → 10min → 30min out; the cached error snapshot is
226
+ // served in between so a dead host never blocks healthy polls on SSH timeouts.
227
+ // A manual refresh (?force=1) clears the cooldown so a revived host is retried
228
+ // immediately instead of waiting out the full backoff.
229
+ const backoff = new Map();
230
+ const BACKOFF_MINUTES = [1, 2, 5, 10, 30];
231
+ // single-flight: concurrent snapshot requests share one polling round
232
+ let inflight = null;
233
+ let lastRun = 0;
234
+ // resolveAuth result cache per host (invalidated by PUT /dash/config; see
235
+ // AuthCacheEntry for the identity-file mtime part of the key)
236
+ const authCache = new Map();
237
+ // staleness observation cursors: hostId → log path → {mtime,size,changeAt};
238
+ // the incremental rule lives in refreshLogActivity/planLogPathUpdates
239
+ const logActivity = new Map();
240
+ // one-shot stall notices with a 2-minute grace: a client that connects shortly
241
+ // AFTER the clearing poll still gets the experiment-finished toast
242
+ const pendingNotices = new Map();
243
+ const cacheSnapshot = () => Object.fromEntries([...cache.entries()]);
244
+ /** drop ALL runtime state of hosts that are no longer in the config (covers
245
+ * both PUT /dash/config and direct settings edits); disposing the CollectState
246
+ * also closes its pooled SSH connection, so a removed host cannot pin one */
247
+ const sweepRemovedHosts = (live) => {
248
+ for (const id of [...hostState.keys()]) {
249
+ if (!live.has(id)) {
250
+ const st = hostState.get(id);
251
+ if (st)
252
+ disposeState(st);
253
+ hostState.delete(id);
254
+ }
255
+ }
256
+ for (const id of [...cache.keys()])
257
+ if (!live.has(id))
258
+ cache.delete(id);
259
+ for (const id of [...backoff.keys()])
260
+ if (!live.has(id))
261
+ backoff.delete(id);
262
+ for (const id of [...logActivity.keys()])
263
+ if (!live.has(id))
264
+ logActivity.delete(id);
265
+ for (const id of [...pendingNotices.keys()])
266
+ if (!live.has(id))
267
+ pendingNotices.delete(id);
268
+ for (const id of [...lastLevels.keys()])
269
+ if (!live.has(id))
270
+ lastLevels.delete(id);
271
+ };
272
+ async function runPoll(config) {
273
+ await Promise.all(config.hosts.map(async (hostCfg) => {
274
+ const back = backoff.get(hostCfg.id);
275
+ if (back && Date.now() < back.nextTryAt) {
276
+ // cooling down — serve the cached error with a fresh timestamp
277
+ const cached = cache.get(hostCfg.id);
278
+ if (cached)
279
+ cache.set(hostCfg.id, { ...cached, at: Date.now(), ok: false });
280
+ return;
281
+ }
282
+ let state = hostState.get(hostCfg.id);
283
+ if (!state) {
284
+ state = { series: new Map() };
285
+ hostState.set(hostCfg.id, state);
286
+ }
287
+ try {
288
+ const runtime = {
289
+ id: hostCfg.id,
290
+ auth: await resolveAuth(ctx, hostCfg, authCache),
291
+ logPath: hostCfg.logPath || undefined,
292
+ };
293
+ const snap = await collectSnapshot(runtime, state);
294
+ if (snap.ok) {
295
+ backoff.delete(hostCfg.id);
296
+ }
297
+ else {
298
+ const failures = (backoff.get(hostCfg.id)?.failures ?? 0) + 1;
299
+ backoff.set(hostCfg.id, {
300
+ failures,
301
+ nextTryAt: Date.now() + BACKOFF_MINUTES[Math.min(failures, BACKOFF_MINUTES.length) - 1] * 60_000,
302
+ });
303
+ }
304
+ cache.set(hostCfg.id, snap);
305
+ }
306
+ catch (err) {
307
+ const failures = (backoff.get(hostCfg.id)?.failures ?? 0) + 1;
308
+ backoff.set(hostCfg.id, {
309
+ failures,
310
+ nextTryAt: Date.now() + BACKOFF_MINUTES[Math.min(failures, BACKOFF_MINUTES.length) - 1] * 60_000,
311
+ });
312
+ cache.set(hostCfg.id, {
313
+ hostId: hostCfg.id,
314
+ at: Date.now(),
315
+ ok: false,
316
+ error: err instanceof Error ? err.message : String(err),
317
+ });
318
+ }
319
+ }));
320
+ // refresh the incremental staleness cursors from this round's snapshots
321
+ const snapsForPlan = Object.fromEntries([...cache.entries()]);
322
+ const nextActivity = refreshLogActivity(snapsForPlan, Object.fromEntries([...logActivity.entries()]), Date.now());
323
+ for (const [id, rec] of Object.entries(nextActivity))
324
+ logActivity.set(id, rec);
325
+ for (const id of [...logActivity.keys()])
326
+ if (!(id in nextActivity))
327
+ logActivity.delete(id);
328
+ // host-level logPath lifecycle: stalled path → auto-clear, empty path → adopt a fresh GPU log
329
+ try {
330
+ const patches = planLogPathUpdates(config, snapsForPlan, Object.fromEntries([...logActivity.entries()]), config.staleMinutes, Date.now());
331
+ if (patches.length > 0) {
332
+ const byId = new Map(patches.map((p) => [p.id, p.logPath]));
333
+ // re-read the config right before writing: a concurrent PUT during
334
+ // this poll must not be overwritten by our stale host list (lost update)
335
+ const fresh = scope.get();
336
+ const next = fresh.hosts.map((h) => (byId.has(h.id) ? { ...h, logPath: byId.get(h.id) } : h));
337
+ await scope.update({ hosts: next });
338
+ for (const p of patches) {
339
+ const st = hostState.get(p.id);
340
+ if (st) {
341
+ st.series.clear();
342
+ st.lastLine = undefined;
343
+ st.lastHeader = undefined;
344
+ st.path = undefined;
345
+ st.lastSize = undefined;
346
+ st.lastMtime = undefined;
347
+ }
348
+ if (!p.logPath) {
349
+ // the stalled log is being dropped from the snapshot — record a
350
+ // one-shot notice, otherwise the client can never see the
351
+ // "unchanged for N minutes" evidence its toast is built on
352
+ const clearedPath = config.hosts.find((h) => h.id === p.id)?.logPath ?? '';
353
+ pendingNotices.set(p.id, { path: clearedPath, minutes: config.staleMinutes, at: Date.now() });
354
+ emitStall(p.id, config.hosts.find((h) => h.id === p.id)?.name ?? p.id, config.staleMinutes);
355
+ const cached = cache.get(p.id);
356
+ if (cached)
357
+ cache.set(p.id, { ...cached, log: undefined, series: undefined });
358
+ }
359
+ console.log(`[dsh-server-dashboard] ${p.id} logPath ${p.logPath ? `自动接管 ${p.logPath}` : '已停滞,自动清除'}`);
360
+ }
361
+ }
362
+ }
363
+ catch (err) {
364
+ console.warn('[dsh-server-dashboard] logPath lifecycle update failed:', err instanceof Error ? err.message : String(err));
365
+ }
366
+ // recycle state (and pooled SSH connections) of hosts removed from the
367
+ // config outside our PUT handler (direct settings edits)
368
+ sweepRemovedHosts(new Set(config.hosts.map((h) => h.id)));
369
+ return cacheSnapshot();
370
+ }
371
+ async function pollAll(config, force = false, forceHost) {
372
+ if (inflight) {
373
+ // a poll round is already running — non-forced callers just wait for it
374
+ if (!force)
375
+ return inflight;
376
+ await inflight.catch(() => { });
377
+ }
378
+ // ?host=<id> alone must NOT bypass the backoff — only a real manual retry
379
+ // (force=1, optionally narrowed to one host) clears it
380
+ if (forceHost && force)
381
+ backoff.delete(forceHost);
382
+ else if (force)
383
+ backoff.clear();
384
+ else if (Date.now() - lastRun < 5_000)
385
+ return cacheSnapshot();
386
+ inflight = runPoll(config);
387
+ try {
388
+ return await inflight;
389
+ }
390
+ finally {
391
+ inflight = null;
392
+ lastRun = Date.now();
393
+ }
394
+ }
395
+ /* ---------- shared route helpers ---------- */
396
+ /** Loopback guard for every /dash/* route: only 127.0.0.1 / ::1 / IPv4-mapped
397
+ * loopback may talk to the dashboard API. Even when the web server is bound
398
+ * to 0.0.0.0 for LAN sharing, remote clients cannot reach the dashboard
399
+ * routes (config write / SSH probes / snapshots) directly. */
400
+ const isLoopback = (req) => {
401
+ const addr = req.socket.remoteAddress ?? '';
402
+ return addr === '127.0.0.1' || addr === '::1' || addr === '::ffff:127.0.0.1';
403
+ };
404
+ /** 403 unless loopback; returns false when the response is already sent */
405
+ const guard = (req, res) => {
406
+ if (!isLoopback(req)) {
407
+ sendJson(res, 403, { error: '仅允许本机(loopback)访问' });
408
+ return false;
409
+ }
410
+ return true;
411
+ };
412
+ const readBody = (req) => new Promise((resolve, reject) => {
413
+ // CSRF hardening: cross-origin simple requests cannot send JSON content-type
414
+ // without a CORS preflight, which this server never grants.
415
+ const ct = String(req.headers['content-type'] ?? '');
416
+ if (!ct.includes('application/json')) {
417
+ reject(new Error('请求必须是 application/json'));
418
+ return;
419
+ }
420
+ let data = '';
421
+ let bytes = 0;
422
+ let over = false;
423
+ req.on('data', (chunk) => {
424
+ if (over)
425
+ return;
426
+ data += chunk;
427
+ bytes += chunk.length; // count BYTES, not UTF-16 code units
428
+ if (bytes > 1_000_000) {
429
+ // destroy the stream the moment the cap is exceeded — otherwise a
430
+ // hostile client can keep feeding the body while we keep buffering it
431
+ over = true;
432
+ req.destroy();
433
+ reject(new Error('请求体过大'));
434
+ }
435
+ });
436
+ req.on('end', () => { if (!over)
437
+ resolve(data); });
438
+ req.on('error', reject);
439
+ });
440
+ const sendJson = (res, code, payload) => {
441
+ res.statusCode = code;
442
+ res.setHeader('content-type', 'application/json; charset=utf-8');
443
+ res.end(JSON.stringify(payload));
444
+ };
445
+ ctx.effect(() => ctx.webServer.register({
446
+ kind: 'exact',
447
+ path: '/dash/snapshots',
448
+ handler: async (req, res) => {
449
+ if (!guard(req, res))
450
+ return;
451
+ if (req.method !== 'GET')
452
+ return sendJson(res, 405, { error: 'method not allowed' });
453
+ const config = scope.get();
454
+ const hostsMeta = config.hosts.map((h) => ({ id: h.id, name: h.name, pinned: h.pinned }));
455
+ // ?force=1[&host=<id>] — manual refresh bypasses the failure backoff,
456
+ // optionally only for one host (the offline card's retry button)
457
+ const query = new URL(req.url ?? '', 'http://localhost').searchParams;
458
+ const force = query.get('force') === '1';
459
+ const forceHost = query.get('host') ?? undefined;
460
+ try {
461
+ const snapshots = await pollAll(config, force, forceHost);
462
+ // attach unexpired stall notices so late-connecting clients still toast
463
+ for (const [id, notice] of [...pendingNotices.entries()]) {
464
+ if (Date.now() - notice.at > 120_000 || !config.hosts.some((h) => h.id === id)) {
465
+ pendingNotices.delete(id);
466
+ continue;
467
+ }
468
+ if (snapshots[id] && !snapshots[id].stallNotice)
469
+ snapshots[id] = { ...snapshots[id], stallNotice: notice };
470
+ }
471
+ res.setHeader('content-type', 'application/json; charset=utf-8');
472
+ res.end(JSON.stringify({
473
+ hosts: hostsMeta,
474
+ snapshots,
475
+ refreshIntervalS: config.refreshIntervalS,
476
+ staleMinutes: config.staleMinutes,
477
+ }));
478
+ }
479
+ catch (err) {
480
+ res.statusCode = 500;
481
+ res.setHeader('content-type', 'application/json; charset=utf-8');
482
+ res.end(JSON.stringify({ error: err instanceof Error ? err.message : String(err) }));
483
+ }
484
+ },
485
+ }));
486
+ // config read (secrets never leave the host; only refs are returned)
487
+ ctx.effect(() => ctx.webServer.register({
488
+ kind: 'exact',
489
+ path: '/dash/config',
490
+ handler: async (req, res) => {
491
+ if (!guard(req, res))
492
+ return;
493
+ try {
494
+ if (req.method === 'GET') {
495
+ const config = scope.get();
496
+ sendJson(res, 200, { config });
497
+ return;
498
+ }
499
+ if (req.method === 'PUT') {
500
+ const body = JSON.parse(await readBody(req));
501
+ const hosts = body.hosts;
502
+ if (!Array.isArray(hosts))
503
+ return sendJson(res, 400, { error: 'hosts 必须是数组' });
504
+ // duplicate ids would share one CollectState and reconnect every poll
505
+ const seenIds = new Set();
506
+ for (const h of hosts) {
507
+ if (seenIds.has(h.id))
508
+ return sendJson(res, 400, { error: `主机 id 重复:${h.id}` });
509
+ seenIds.add(h.id);
510
+ }
511
+ // omitted fields keep their persisted values — a hosts-only PUT (host
512
+ // edit, import, log apply) must not reset the interval thresholds
513
+ const current = scope.get();
514
+ const refreshIntervalS = body.refreshIntervalS ?? current.refreshIntervalS;
515
+ const staleMinutes = body.staleMinutes ?? current.staleMinutes;
516
+ // validate against the schema before persisting
517
+ DashboardConfigSchema({ hosts, refreshIntervalS, staleMinutes });
518
+ // secrets are ALWAYS stored under the server-derived SD_<hostId> name;
519
+ // a client-supplied credentialRef is never used as a vault address
520
+ // (prevents overwriting arbitrary existing credentials). The config
521
+ // field is normalized for display only.
522
+ const secretIds = new Set(Object.entries(body.secrets ?? {})
523
+ .filter(([, s]) => s?.privateKey || s?.password)
524
+ .map(([id]) => id));
525
+ for (const row of hosts) {
526
+ if (secretIds.has(row.id))
527
+ row.credentialRef = derivedCredName(row.id);
528
+ }
529
+ await scope.update({ hosts, refreshIntervalS, staleMinutes });
530
+ // config changed: clear the failure backoff + poll throttle so the
531
+ // next request re-probes immediately ("fixed the credentials but the
532
+ // card stayed red for up to 30min" case), drop the auth cache, and
533
+ // re-arm the events watcher at the new interval
534
+ backoff.clear();
535
+ lastRun = 0;
536
+ authCache.clear();
537
+ scheduleWatcher(Math.max(WATCHER_MIN_MS, refreshIntervalS * 1000));
538
+ // drop runtime state of hosts that no longer exist (cache/backoff/state leak)
539
+ sweepRemovedHosts(new Set(hosts.map((h) => h.id)));
540
+ // persist the supplied secrets; ids not in the host list are ignored
541
+ // (no orphan credentials for removed hosts)
542
+ for (const [id, secret] of Object.entries(body.secrets ?? {})) {
543
+ if (!seenIds.has(id))
544
+ continue;
545
+ const name = derivedCredName(id);
546
+ if (secret?.privateKey)
547
+ await ctx.credentials.set(credentialRef(name), secret.privateKey);
548
+ else if (secret?.password)
549
+ await ctx.credentials.set(credentialRef(name), secret.password);
550
+ }
551
+ sendJson(res, 200, { ok: true, config: { hosts, refreshIntervalS, staleMinutes } });
552
+ return;
553
+ }
554
+ sendJson(res, 405, { error: 'method not allowed' });
555
+ }
556
+ catch (err) {
557
+ sendJson(res, 400, { error: err instanceof Error ? err.message : String(err) });
558
+ }
559
+ },
560
+ }));
561
+ // import hosts from ~/.ssh/config (host side runs as the local user)
562
+ ctx.effect(() => ctx.webServer.register({
563
+ kind: 'exact',
564
+ path: '/dash/import-ssh',
565
+ handler: async (req, res) => {
566
+ if (!guard(req, res))
567
+ return;
568
+ try {
569
+ const home = process.env.USERPROFILE ?? process.env.HOME ?? '';
570
+ const { readFile } = await import('node:fs/promises');
571
+ const text = await readFile(`${home}/.ssh/config`, 'utf8');
572
+ sendJson(res, 200, { hosts: parseSshConfig(text) });
573
+ }
574
+ catch (err) {
575
+ sendJson(res, 500, { error: err instanceof Error ? err.message : String(err) });
576
+ }
577
+ },
578
+ }));
579
+ // connection test (light SSH probe; accepts one-off secrets for untested entries)
580
+ ctx.effect(() => ctx.webServer.register({
581
+ kind: 'exact',
582
+ path: '/dash/test',
583
+ handler: async (req, res) => {
584
+ if (!guard(req, res))
585
+ return;
586
+ try {
587
+ const body = JSON.parse(await readBody(req));
588
+ if (!body.host)
589
+ return sendJson(res, 400, { error: 'host 必填' });
590
+ const result = await testConnection(await resolveAuthFromBody(ctx, body));
591
+ sendJson(res, 200, result);
592
+ }
593
+ catch (err) {
594
+ sendJson(res, 400, { error: err instanceof Error ? err.message : String(err) });
595
+ }
596
+ },
597
+ }));
598
+ // auto-discover training logs from running processes
599
+ ctx.effect(() => ctx.webServer.register({
600
+ kind: 'exact',
601
+ path: '/dash/discover-logs',
602
+ handler: async (req, res) => {
603
+ if (!guard(req, res))
604
+ return;
605
+ try {
606
+ const body = JSON.parse(await readBody(req));
607
+ if (!body.host)
608
+ return sendJson(res, 400, { error: 'host 必填' });
609
+ const candidates = await discoverLogs(await resolveAuthFromBody(ctx, body));
610
+ sendJson(res, 200, { candidates });
611
+ }
612
+ catch (err) {
613
+ sendJson(res, 400, { error: err instanceof Error ? err.message : String(err) });
614
+ }
615
+ },
616
+ }));
617
+ console.log('[dsh-server-dashboard] host half ready: settings ns + /dash/snapshots + /dash/config + /dash/test');
618
+ /* ---------- instant alerts (/dash/events, long-poll) ----------
619
+ * The browser panel polls snapshots; this endpoint pushes second-level
620
+ * alerts the moment the host sampler sees a transition:
621
+ * {type:'stall', hostId, hostName, minutes} — experiment log stalled
622
+ * {type:'host-status', hostId, hostName, level, previous}
623
+ * — offline / hot flips
624
+ * Hanging GET + seq cursor: request /dash/events?since=N and it resolves
625
+ * the instant an event lands (25s idle timeout). Sampling reuses pollAll
626
+ * (single-flight + per-host backoff) and only runs while a client is
627
+ * actually waiting, so no SSH load is added when the panel is closed. */
628
+ const eventLog = [];
629
+ let eventSeq = 0;
630
+ const eventWaiters = new Set();
631
+ const emitAlert = (payload) => {
632
+ eventLog.push({ seq: ++eventSeq, payload });
633
+ if (eventLog.length > 80)
634
+ eventLog.splice(0, eventLog.length - 80);
635
+ for (const wake of [...eventWaiters])
636
+ wake();
637
+ eventWaiters.clear();
638
+ };
639
+ const lastLevels = new Map();
640
+ const levelOf = (snap) => {
641
+ if (!snap || !snap.ok)
642
+ return 'offline';
643
+ return (snap.gpus ?? []).some((g) => g.tempC >= TEMP_HOT) ? 'warn' : 'ok';
644
+ };
645
+ // host-level stall: emit the moment planLogPathUpdates records a notice
646
+ const emitStall = (hostId, hostName, minutes) => emitAlert({ type: 'stall', hostId, hostName, minutes });
647
+ // sampling interval follows the configured refreshIntervalS (10s floor), so
648
+ // the watcher never polls SSH more often than the panel itself; rebuilt when
649
+ // the config (or its interval) changes
650
+ const WATCHER_MIN_MS = 10_000;
651
+ let watcherTimer = null;
652
+ let watcherIntervalMs = 0;
653
+ const scheduleWatcher = (intervalMs) => {
654
+ if (watcherTimer)
655
+ clearInterval(watcherTimer);
656
+ watcherIntervalMs = intervalMs;
657
+ watcherTimer = setInterval(watcherTick, intervalMs);
658
+ watcherTimer.unref();
659
+ };
660
+ async function watcherTick() {
661
+ if (eventWaiters.size === 0)
662
+ return; // nobody long-polling — skip SSH sampling
663
+ const config = scope.get();
664
+ // the interval may have changed via direct settings edit — re-arm
665
+ const want = Math.max(WATCHER_MIN_MS, config.refreshIntervalS * 1000);
666
+ if (want !== watcherIntervalMs)
667
+ scheduleWatcher(want);
668
+ const snaps = await pollAll(config).catch(() => cacheSnapshot());
669
+ for (const h of config.hosts) {
670
+ const level = levelOf(snaps[h.id]);
671
+ const prev = lastLevels.get(h.id);
672
+ lastLevels.set(h.id, level);
673
+ if (!prev || prev === level)
674
+ continue;
675
+ emitAlert({ type: 'host-status', hostId: h.id, hostName: h.name, level, previous: prev });
676
+ }
677
+ }
678
+ scheduleWatcher(Math.max(WATCHER_MIN_MS, (scope.get().refreshIntervalS || 30) * 1000));
679
+ ctx.effect(() => ctx.webServer.register({
680
+ kind: 'exact',
681
+ path: '/dash/events',
682
+ handler: async (req, res) => {
683
+ if (!guard(req, res))
684
+ return;
685
+ if (req.method !== 'GET')
686
+ return sendJson(res, 405, { error: 'method not allowed' });
687
+ const q = new URL(req.url ?? '/', 'http://localhost').searchParams;
688
+ const since = Number(q.get('since') ?? 0) || 0;
689
+ const deliver = () => {
690
+ res.setHeader('content-type', 'application/json; charset=utf-8');
691
+ res.end(JSON.stringify({
692
+ events: eventLog.filter((e) => e.seq > since).map((e) => e.payload),
693
+ last: eventSeq,
694
+ }));
695
+ };
696
+ // fresh client (since=0) only receives the cursor baseline — replaying the
697
+ // buffered history would re-toast hours-old alerts on every page reload
698
+ if (since === 0) {
699
+ res.setHeader('content-type', 'application/json; charset=utf-8');
700
+ res.end(JSON.stringify({ events: [], last: eventSeq }));
701
+ return;
702
+ }
703
+ if (eventLog.some((e) => e.seq > since))
704
+ return deliver();
705
+ const to = setTimeout(() => { eventWaiters.delete(wake); deliver(); }, 25_000);
706
+ const wake = () => { clearTimeout(to); eventWaiters.delete(wake); deliver(); };
707
+ eventWaiters.add(wake);
708
+ res.on('close', () => { clearTimeout(to); eventWaiters.delete(wake); });
709
+ },
710
+ }));
711
+ // close all pooled SSH connections when the plugin scope is disposed
712
+ ctx.effect(() => () => {
713
+ if (watcherTimer)
714
+ clearInterval(watcherTimer);
715
+ for (const st of hostState.values())
716
+ disposeState(st);
717
+ });
718
+ }