instar 1.3.710 → 1.3.711
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +228 -1
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +25 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/HttpLeaseTransport.d.ts +9 -0
- package/dist/core/HttpLeaseTransport.d.ts.map +1 -1
- package/dist/core/HttpLeaseTransport.js +28 -2
- package/dist/core/HttpLeaseTransport.js.map +1 -1
- package/dist/core/MultiMachineCoordinator.d.ts +44 -0
- package/dist/core/MultiMachineCoordinator.d.ts.map +1 -1
- package/dist/core/MultiMachineCoordinator.js +52 -0
- package/dist/core/MultiMachineCoordinator.js.map +1 -1
- package/dist/core/PeerEndpointResolver.d.ts +0 -0
- package/dist/core/PeerEndpointResolver.d.ts.map +1 -1
- package/dist/core/PeerEndpointResolver.js +0 -0
- package/dist/core/PeerEndpointResolver.js.map +1 -1
- package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
- package/dist/core/PostUpdateMigrator.js +19 -0
- package/dist/core/PostUpdateMigrator.js.map +1 -1
- package/dist/core/RopeRecoveryProber.d.ts +126 -0
- package/dist/core/RopeRecoveryProber.d.ts.map +1 -0
- package/dist/core/RopeRecoveryProber.js +266 -0
- package/dist/core/RopeRecoveryProber.js.map +1 -0
- package/dist/core/devGatedFeatures.d.ts.map +1 -1
- package/dist/core/devGatedFeatures.js +12 -0
- package/dist/core/devGatedFeatures.js.map +1 -1
- package/dist/core/ropeProbeContract.d.ts +56 -0
- package/dist/core/ropeProbeContract.d.ts.map +1 -0
- package/dist/core/ropeProbeContract.js +92 -0
- package/dist/core/ropeProbeContract.js.map +1 -0
- package/dist/core/tailscaleStatusParser.d.ts +46 -0
- package/dist/core/tailscaleStatusParser.d.ts.map +1 -0
- package/dist/core/tailscaleStatusParser.js +80 -0
- package/dist/core/tailscaleStatusParser.js.map +1 -0
- package/dist/core/types.d.ts +56 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/monitoring/RopeHealthMonitor.d.ts +243 -0
- package/dist/monitoring/RopeHealthMonitor.d.ts.map +1 -0
- package/dist/monitoring/RopeHealthMonitor.js +659 -0
- package/dist/monitoring/RopeHealthMonitor.js.map +1 -0
- package/dist/monitoring/guardManifest.d.ts.map +1 -1
- package/dist/monitoring/guardManifest.js +45 -0
- package/dist/monitoring/guardManifest.js.map +1 -1
- package/dist/scaffold/templates.d.ts.map +1 -1
- package/dist/scaffold/templates.js +9 -0
- package/dist/scaffold/templates.js.map +1 -1
- package/dist/server/AgentServer.d.ts +2 -0
- package/dist/server/AgentServer.d.ts.map +1 -1
- package/dist/server/AgentServer.js +1 -0
- package/dist/server/AgentServer.js.map +1 -1
- package/dist/server/CapabilityIndex.js +1 -1
- package/dist/server/CapabilityIndex.js.map +1 -1
- package/dist/server/routes.d.ts +3 -0
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +19 -0
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/scripts/lint-scrape-fixture-realness.js +20 -0
- package/src/data/builtin-manifest.json +65 -65
- package/src/data/state-coherence-registry.json +17 -0
- package/src/scaffold/templates/jobs/instar/rope-health-digest.md +42 -0
- package/src/scaffold/templates.ts +9 -0
- package/upgrades/1.3.711.md +79 -0
- package/upgrades/side-effects/u4-rope-probe-alerts.md +140 -0
|
@@ -0,0 +1,659 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RopeHealthMonitor — U4.5 rope-health alerts
|
|
3
|
+
* (docs/specs/u4-5-rope-health-alerts.md).
|
|
4
|
+
*
|
|
5
|
+
* Mesh transport degradation was silent: a Tailscale key expiry dropped a rope
|
|
6
|
+
* with no warning; a persistently-down rope was visible only to someone who
|
|
7
|
+
* went looking; an all-transports-down partition — the precondition for silent
|
|
8
|
+
* message loss — had no prompt alert at all. This monitor is the productized,
|
|
9
|
+
* in-server detector: it runs its OWN bounded 30s evaluation loop (constructed
|
|
10
|
+
* and torn down by the real server boot — R-r2-2), reads the U4.3
|
|
11
|
+
* `PeerEndpointResolver.snapshot()` seam (the HARD data dependency; no interim
|
|
12
|
+
* fallback exists), and classifies each peer deterministically:
|
|
13
|
+
*
|
|
14
|
+
* ok — silence (no digest line, nothing).
|
|
15
|
+
* degraded — a rope down while ≥1 other rope is healthy, or a Tailscale
|
|
16
|
+
* key expiring within `keyExpiryWarnDays`. Digest-only.
|
|
17
|
+
* peer-offline — ALL ropes down AND the peer's git-synced coarse heartbeat
|
|
18
|
+
* has STOPPED advancing (or the registry already marks it
|
|
19
|
+
* offline). Digest-only — a lid-close is NEVER urgent.
|
|
20
|
+
* urgent — ALL ropes down to a peer whose git-synced coarse heartbeat
|
|
21
|
+
* is still ADVANCING — advancement-since-onset semantics
|
|
22
|
+
* (R-r3-1): a beat NEWER than the all-down onset, observed
|
|
23
|
+
* after the onset. Freshness-window semantics are REJECTED
|
|
24
|
+
* (a just-lid-closed peer's last beat looks fresh for up to
|
|
25
|
+
* an hour). ONE HIGH attention item per episode.
|
|
26
|
+
*
|
|
27
|
+
* The urgent tier's honest detection latency is bounded by the heartbeat
|
|
28
|
+
* interval plus up to two git-sync cadences (~30-90 min); `urgentDebounceMs`
|
|
29
|
+
* is only the short-term flap filter on the all-down condition itself.
|
|
30
|
+
*
|
|
31
|
+
* Episode semantics: episodeKey = sorted machine pair + the onset quantized to
|
|
32
|
+
* 15 min — deterministically computable on BOTH sides without coordination;
|
|
33
|
+
* post-heal grouping matches ADJACENT quantization windows (R-r2-5). An episode
|
|
34
|
+
* ends only after `clearSustainMs` of continuous health (a blip cannot
|
|
35
|
+
* clear-then-re-fire). At most ONE item per side per episode; if a split-brain
|
|
36
|
+
* item is already open for the same peer, the monitor does not raise a second.
|
|
37
|
+
*
|
|
38
|
+
* Durable state (`state/rope-health.json`): transition-only writes with a short
|
|
39
|
+
* debounce (R-r2-4) — a steady-state evaluation never touches disk; counters
|
|
40
|
+
* lost to a restart are accepted (the safe direction: a restart re-debounces,
|
|
41
|
+
* never fabricates an episode).
|
|
42
|
+
*
|
|
43
|
+
* Content scrub (hard rule): alert/digest text carries rope KIND + machine
|
|
44
|
+
* NICKNAME + relative times ONLY — never raw IPs, URLs, tunnel hostnames,
|
|
45
|
+
* tailnet names, or account emails (the tailscale JSON's identifying fields
|
|
46
|
+
* never leave `parseTailscaleStatus`).
|
|
47
|
+
*/
|
|
48
|
+
import fs from 'node:fs';
|
|
49
|
+
import path from 'node:path';
|
|
50
|
+
import { execFile } from 'node:child_process';
|
|
51
|
+
import { parseTailscaleStatus, soonestKeyExpiry, } from '../core/tailscaleStatusParser.js';
|
|
52
|
+
export const ROPE_HEALTH_DEFAULTS = {
|
|
53
|
+
urgentEnabled: true,
|
|
54
|
+
urgentDebounceMs: 60_000,
|
|
55
|
+
clearSustainMs: 600_000,
|
|
56
|
+
keyExpiryWarnDays: 14,
|
|
57
|
+
evaluateIntervalMs: 30_000,
|
|
58
|
+
keyExpiryCheckIntervalMs: 3_600_000,
|
|
59
|
+
wakeGraceMaxMs: 300_000,
|
|
60
|
+
episodeQuantumMs: 900_000,
|
|
61
|
+
writeDebounceMs: 2_000,
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* Deterministic shared episode key: both sides compute the same value from the
|
|
65
|
+
* sorted machine pair + the onset's quantization window (spec §2).
|
|
66
|
+
*/
|
|
67
|
+
export function computeEpisodeKey(machineA, machineB, onsetMs, quantumMs = ROPE_HEALTH_DEFAULTS.episodeQuantumMs) {
|
|
68
|
+
const pair = [machineA, machineB].sort().join('+');
|
|
69
|
+
const win = Math.floor(onsetMs / quantumMs) * quantumMs;
|
|
70
|
+
return `${pair}:${win}`;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Post-heal grouping (R-r2-5): two episode keys group when they name the same
|
|
74
|
+
* machine pair and their quantization windows are ADJACENT (±1 quantum) — the
|
|
75
|
+
* two sides detect at different instants, so an onset straddling a boundary
|
|
76
|
+
* yields adjacent window keys. Beyond one quantum the keys honestly show as
|
|
77
|
+
* two groups (declared best-effort display degradation).
|
|
78
|
+
*/
|
|
79
|
+
export function episodeKeysGroup(a, b, quantumMs = ROPE_HEALTH_DEFAULTS.episodeQuantumMs) {
|
|
80
|
+
const pa = a.lastIndexOf(':');
|
|
81
|
+
const pb = b.lastIndexOf(':');
|
|
82
|
+
if (pa <= 0 || pb <= 0)
|
|
83
|
+
return false;
|
|
84
|
+
if (a.slice(0, pa) !== b.slice(0, pb))
|
|
85
|
+
return false;
|
|
86
|
+
const wa = Number(a.slice(pa + 1));
|
|
87
|
+
const wb = Number(b.slice(pb + 1));
|
|
88
|
+
if (!Number.isFinite(wa) || !Number.isFinite(wb))
|
|
89
|
+
return false;
|
|
90
|
+
return Math.abs(wa - wb) <= quantumMs;
|
|
91
|
+
}
|
|
92
|
+
/** Candidate tailscale CLI paths (same set as MeshUrlAdvertiser Decision 16). */
|
|
93
|
+
const TAILSCALE_CANDIDATES = [
|
|
94
|
+
'/opt/homebrew/bin/tailscale',
|
|
95
|
+
'/usr/local/bin/tailscale',
|
|
96
|
+
'/Applications/Tailscale.app/Contents/MacOS/Tailscale',
|
|
97
|
+
'tailscale',
|
|
98
|
+
];
|
|
99
|
+
/** Default bounded exec: 5s hard timeout; absent CLI / any failure ⇒ null. */
|
|
100
|
+
export function defaultExecTailscaleStatusJson() {
|
|
101
|
+
return new Promise((resolve) => {
|
|
102
|
+
const tryNext = (i) => {
|
|
103
|
+
if (i >= TAILSCALE_CANDIDATES.length) {
|
|
104
|
+
resolve(null);
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
const file = TAILSCALE_CANDIDATES[i];
|
|
108
|
+
if (file !== 'tailscale' && !fs.existsSync(file)) {
|
|
109
|
+
tryNext(i + 1);
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
execFile(file, ['status', '--json'], { timeout: 5_000, maxBuffer: 4 * 1024 * 1024 }, (err, stdout) => {
|
|
113
|
+
if (err || !stdout) {
|
|
114
|
+
tryNext(i + 1);
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
resolve(String(stdout));
|
|
118
|
+
});
|
|
119
|
+
};
|
|
120
|
+
tryNext(0);
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
export class RopeHealthMonitor {
|
|
124
|
+
d;
|
|
125
|
+
cfg;
|
|
126
|
+
peers = new Map();
|
|
127
|
+
timer = null;
|
|
128
|
+
lastEvaluatedAt = null;
|
|
129
|
+
evaluations = 0;
|
|
130
|
+
evaluating = false;
|
|
131
|
+
// Self-wake grace (bounded — see wakeGraceMaxMs hazard note / P1-A7).
|
|
132
|
+
ownWakeAt = null;
|
|
133
|
+
// Key-expiry tier state.
|
|
134
|
+
keyExpiryLastCheckedAt = null;
|
|
135
|
+
keyExpiryAvailable = false;
|
|
136
|
+
keyExpirySoonest = null;
|
|
137
|
+
keyExpiryAbsentLogged = false;
|
|
138
|
+
keyExpiryInFlight = false;
|
|
139
|
+
// Counters (observability only).
|
|
140
|
+
urgentEpisodes = 0;
|
|
141
|
+
suppressedBySleepGate = 0;
|
|
142
|
+
suppressedBySplitBrain = 0;
|
|
143
|
+
notNotifiedRetries = 0;
|
|
144
|
+
// Transition-only durable writes (R-r2-4).
|
|
145
|
+
writePending = false;
|
|
146
|
+
writeTimer = null;
|
|
147
|
+
constructor(deps, cfg) {
|
|
148
|
+
this.d = deps;
|
|
149
|
+
this.cfg = { ...ROPE_HEALTH_DEFAULTS, ...cfg };
|
|
150
|
+
this.loadState();
|
|
151
|
+
}
|
|
152
|
+
now() {
|
|
153
|
+
return (this.d.now ?? Date.now)();
|
|
154
|
+
}
|
|
155
|
+
log(m) {
|
|
156
|
+
this.d.logger?.(`[rope-health] ${m}`);
|
|
157
|
+
}
|
|
158
|
+
metric(e) {
|
|
159
|
+
try {
|
|
160
|
+
this.d.recordMetric?.(e);
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
// @silent-fallback-ok: metrics are observability, never authority — a
|
|
164
|
+
// recorder fault must not break the evaluation loop.
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/** Start the monitor's OWN bounded evaluation loop (R-r2-2). */
|
|
168
|
+
start() {
|
|
169
|
+
if (this.timer)
|
|
170
|
+
return;
|
|
171
|
+
this.evaluate();
|
|
172
|
+
this.timer = setInterval(() => this.evaluate(), this.cfg.evaluateIntervalMs);
|
|
173
|
+
if (typeof this.timer.unref === 'function')
|
|
174
|
+
this.timer.unref();
|
|
175
|
+
}
|
|
176
|
+
/** Torn down at server shutdown (R-r2-2). Flushes any pending state write. */
|
|
177
|
+
stop() {
|
|
178
|
+
if (this.timer) {
|
|
179
|
+
clearInterval(this.timer);
|
|
180
|
+
this.timer = null;
|
|
181
|
+
}
|
|
182
|
+
if (this.writeTimer) {
|
|
183
|
+
clearTimeout(this.writeTimer);
|
|
184
|
+
this.writeTimer = null;
|
|
185
|
+
}
|
|
186
|
+
if (this.writePending)
|
|
187
|
+
this.writeStateNow();
|
|
188
|
+
}
|
|
189
|
+
/** Is the evaluation loop currently armed? (e2e wiring-integrity read.) */
|
|
190
|
+
running() {
|
|
191
|
+
return this.timer !== null;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Own-machine wake note (SleepWakeDetector 'wake' — the ONE thing it can
|
|
195
|
+
* tell us). Post-wake, all snapshots are stale; urgent is suppressed until
|
|
196
|
+
* each (peer, kind) is re-observed post-wake — BOUNDED by wakeGraceMaxMs
|
|
197
|
+
* (P1-A7: false wake events must never become an open-ended veto).
|
|
198
|
+
*/
|
|
199
|
+
noteOwnWake(atMs) {
|
|
200
|
+
this.ownWakeAt = atMs ?? this.now();
|
|
201
|
+
}
|
|
202
|
+
/** One evaluation pass. Public for tests; the loop calls it on cadence. */
|
|
203
|
+
evaluate() {
|
|
204
|
+
if (this.evaluating)
|
|
205
|
+
return;
|
|
206
|
+
this.evaluating = true;
|
|
207
|
+
try {
|
|
208
|
+
this.evaluateInner();
|
|
209
|
+
}
|
|
210
|
+
catch (err) {
|
|
211
|
+
// Fail toward silence, never a crashed loop (signal, not authority).
|
|
212
|
+
this.log(`evaluation error: ${err instanceof Error ? err.message : String(err)}`);
|
|
213
|
+
}
|
|
214
|
+
finally {
|
|
215
|
+
this.evaluating = false;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
evaluateInner() {
|
|
219
|
+
const nowMs = this.now();
|
|
220
|
+
this.lastEvaluatedAt = nowMs;
|
|
221
|
+
this.evaluations += 1;
|
|
222
|
+
this.metric('evaluation');
|
|
223
|
+
const rows = this.safeSnapshot();
|
|
224
|
+
const rowsByPeer = new Map();
|
|
225
|
+
for (const r of rows) {
|
|
226
|
+
const list = rowsByPeer.get(r.peer) ?? [];
|
|
227
|
+
list.push(r);
|
|
228
|
+
rowsByPeer.set(r.peer, list);
|
|
229
|
+
}
|
|
230
|
+
for (const peer of this.safePeers()) {
|
|
231
|
+
const st = this.peers.get(peer.machineId) ?? freshPeerState();
|
|
232
|
+
this.peers.set(peer.machineId, st);
|
|
233
|
+
const peerRows = rowsByPeer.get(peer.machineId) ?? [];
|
|
234
|
+
const prev = st.condition;
|
|
235
|
+
// Absent-record semantics (R-r2-minor): a peer with NO snapshot rows is
|
|
236
|
+
// UNKNOWN, not down — fails toward NOT-urgent.
|
|
237
|
+
if (peerRows.length === 0) {
|
|
238
|
+
this.settleHealthy(st, nowMs);
|
|
239
|
+
st.condition = 'unknown';
|
|
240
|
+
this.recordTransition(prev, st.condition);
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
const deadRows = peerRows.filter((r) => r.dead);
|
|
244
|
+
const allDown = deadRows.length === peerRows.length;
|
|
245
|
+
if (!allDown) {
|
|
246
|
+
this.settleHealthy(st, nowMs);
|
|
247
|
+
st.condition = deadRows.length > 0 ? 'degraded' : 'ok';
|
|
248
|
+
this.recordTransition(prev, st.condition);
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
// ── ALL ropes down. ──
|
|
252
|
+
st.healthySince = null;
|
|
253
|
+
if (st.allDownSince === null) {
|
|
254
|
+
st.allDownSince = nowMs;
|
|
255
|
+
st.consecutiveObservations = 1;
|
|
256
|
+
st.episodeKey = computeEpisodeKey(this.d.selfMachineId, peer.machineId, nowMs, this.cfg.episodeQuantumMs);
|
|
257
|
+
st.postOnsetBeatObservedAt = null;
|
|
258
|
+
this.log(`all-down onset for ${peer.nickname} (episode ${st.episodeKey})`);
|
|
259
|
+
}
|
|
260
|
+
else {
|
|
261
|
+
st.consecutiveObservations += 1;
|
|
262
|
+
}
|
|
263
|
+
// The R-r3-1 discriminator: advancement-since-onset — a heartbeat NEWER
|
|
264
|
+
// than the onset, observed after the onset. Freshness-window semantics
|
|
265
|
+
// are rejected (a just-lid-closed peer's pre-sleep beat looks fresh).
|
|
266
|
+
const hbAt = this.safeHeartbeat(peer.machineId);
|
|
267
|
+
if (hbAt !== null && st.allDownSince !== null && hbAt > st.allDownSince && st.postOnsetBeatObservedAt === null) {
|
|
268
|
+
st.postOnsetBeatObservedAt = nowMs;
|
|
269
|
+
}
|
|
270
|
+
if (!peer.registryOnline) {
|
|
271
|
+
// WS4.2 'offline since <t>' — expected. Never urgent.
|
|
272
|
+
st.condition = 'peer-offline';
|
|
273
|
+
this.recordTransition(prev, st.condition);
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
if (st.postOnsetBeatObservedAt === null) {
|
|
277
|
+
// Heartbeat stopped (peer likely asleep/off) or no post-onset beat has
|
|
278
|
+
// landed YET — late-but-honest: a later post-onset beat upgrades this.
|
|
279
|
+
st.condition = 'peer-offline';
|
|
280
|
+
this.recordTransition(prev, st.condition);
|
|
281
|
+
continue;
|
|
282
|
+
}
|
|
283
|
+
// ── Urgent candidate: all-down + advancing heartbeat. ──
|
|
284
|
+
st.condition = 'urgent';
|
|
285
|
+
this.recordTransition(prev, st.condition);
|
|
286
|
+
if (!this.cfg.urgentEnabled)
|
|
287
|
+
continue;
|
|
288
|
+
// Time-pinned debounce (R-r2-2) + ≥2 consecutive observations: a spurious
|
|
289
|
+
// single-evaluation blip (or a spurious SLEEP signal coinciding with a
|
|
290
|
+
// real rope failure — the P1-A7 hazard test) never fires alone, and a
|
|
291
|
+
// REAL sustained partition always fires once the debounce elapses.
|
|
292
|
+
if (nowMs - (st.allDownSince ?? nowMs) < this.cfg.urgentDebounceMs)
|
|
293
|
+
continue;
|
|
294
|
+
if (st.consecutiveObservations < 2)
|
|
295
|
+
continue;
|
|
296
|
+
// Self-wake grace (BOUNDED — P1-A7): suppress only while (a) inside the
|
|
297
|
+
// wakeGraceMaxMs cap AND (b) some rope of this peer has not been
|
|
298
|
+
// re-observed since the wake. A false wake event therefore delays an
|
|
299
|
+
// urgent item by AT MOST wakeGraceMaxMs, never vetoes it.
|
|
300
|
+
if (this.sleepGateSuppresses(peerRows, nowMs)) {
|
|
301
|
+
this.suppressedBySleepGate += 1;
|
|
302
|
+
this.metric('suppressed-by-sleep-gate');
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
if (st.urgentRaisedAt !== null && !st.detectedNotNotified)
|
|
306
|
+
continue; // one item per episode
|
|
307
|
+
if (this.safeSplitBrainOpen(peer.machineId)) {
|
|
308
|
+
// One episode, one ask — the already-open split-brain item wins.
|
|
309
|
+
this.suppressedBySplitBrain += 1;
|
|
310
|
+
this.metric('suppressed-by-split-brain');
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
this.raiseUrgent(peer, st, nowMs);
|
|
314
|
+
}
|
|
315
|
+
// ── Key-expiry tier (own cadence, R-r2-3). ──
|
|
316
|
+
if (this.keyExpiryLastCheckedAt === null ||
|
|
317
|
+
nowMs - this.keyExpiryLastCheckedAt >= this.cfg.keyExpiryCheckIntervalMs) {
|
|
318
|
+
this.checkKeyExpiry(nowMs);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
settleHealthy(st, nowMs) {
|
|
322
|
+
if (st.allDownSince === null)
|
|
323
|
+
return;
|
|
324
|
+
// Episode still open: sustained-clear gate (a blip cannot clear-then-re-fire).
|
|
325
|
+
if (st.healthySince === null)
|
|
326
|
+
st.healthySince = nowMs;
|
|
327
|
+
if (nowMs - st.healthySince >= this.cfg.clearSustainMs) {
|
|
328
|
+
this.log(`episode ${st.episodeKey ?? '?'} ended after sustained health`);
|
|
329
|
+
st.allDownSince = null;
|
|
330
|
+
st.consecutiveObservations = 0;
|
|
331
|
+
st.episodeKey = null;
|
|
332
|
+
st.urgentRaisedAt = null;
|
|
333
|
+
st.detectedNotNotified = false;
|
|
334
|
+
st.postOnsetBeatObservedAt = null;
|
|
335
|
+
st.healthySince = null;
|
|
336
|
+
this.scheduleWrite();
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
sleepGateSuppresses(peerRows, nowMs) {
|
|
340
|
+
if (this.ownWakeAt === null)
|
|
341
|
+
return false;
|
|
342
|
+
if (nowMs - this.ownWakeAt > this.cfg.wakeGraceMaxMs)
|
|
343
|
+
return false; // bounded (P1-A7)
|
|
344
|
+
// Suppress only while some rope has NOT been re-observed post-wake.
|
|
345
|
+
return peerRows.some((r) => Math.max(r.lastOkAt, r.lastFailAt) <= this.ownWakeAt);
|
|
346
|
+
}
|
|
347
|
+
raiseUrgent(peer, st, nowMs) {
|
|
348
|
+
// The urgent-episode counter fires on the FIRST successful delivery for
|
|
349
|
+
// this episode (a delivery that only succeeded on a retry still counts once).
|
|
350
|
+
const firstDelivery = st.urgentRaisedAt === null;
|
|
351
|
+
const sinceMin = st.allDownSince ? Math.round((nowMs - st.allDownSince) / 60_000) : 0;
|
|
352
|
+
// Content scrub: kind + nickname + relative time ONLY.
|
|
353
|
+
const kinds = 'every mesh rope';
|
|
354
|
+
const item = {
|
|
355
|
+
id: `rope-health-urgent:${st.episodeKey ?? peer.machineId}`,
|
|
356
|
+
title: `Mesh partition: ${peer.nickname} unreachable but alive`,
|
|
357
|
+
body: `ALL mesh ropes (${kinds}) to ${peer.nickname} have been down for ~${sinceMin} min, ` +
|
|
358
|
+
`but its machine heartbeat is still advancing — it is alive yet unreachable (a genuine partition, ` +
|
|
359
|
+
`not a sleeping machine). Messages routed to it may be delayed until a rope recovers. ` +
|
|
360
|
+
`Episode ${st.episodeKey ?? 'n/a'}; this is the ONE alert for this episode.`,
|
|
361
|
+
};
|
|
362
|
+
try {
|
|
363
|
+
const r = this.d.raiseAttention(item);
|
|
364
|
+
if (r && typeof r.then === 'function') {
|
|
365
|
+
void r.then(() => this.settleNotified(st, nowMs, firstDelivery), () => this.markNotNotified(st));
|
|
366
|
+
}
|
|
367
|
+
else {
|
|
368
|
+
this.settleNotified(st, nowMs, firstDelivery);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
catch {
|
|
372
|
+
// Detected-but-silent must be impossible to lose silently: the failure is
|
|
373
|
+
// recorded and the next evaluation retries (spec "alert delivery honesty").
|
|
374
|
+
this.markNotNotified(st);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
settleNotified(st, nowMs, firstDelivery) {
|
|
378
|
+
st.urgentRaisedAt = nowMs;
|
|
379
|
+
st.detectedNotNotified = false;
|
|
380
|
+
if (firstDelivery) {
|
|
381
|
+
this.urgentEpisodes += 1;
|
|
382
|
+
this.metric('urgent-episode');
|
|
383
|
+
}
|
|
384
|
+
this.scheduleWrite();
|
|
385
|
+
}
|
|
386
|
+
markNotNotified(st) {
|
|
387
|
+
if (st.detectedNotNotified) {
|
|
388
|
+
this.notNotifiedRetries += 1;
|
|
389
|
+
this.metric('detected-not-notified-retry');
|
|
390
|
+
}
|
|
391
|
+
st.detectedNotNotified = true;
|
|
392
|
+
if (st.urgentRaisedAt === null)
|
|
393
|
+
st.urgentRaisedAt = null; // stays unraised — retried next evaluation
|
|
394
|
+
this.scheduleWrite();
|
|
395
|
+
}
|
|
396
|
+
checkKeyExpiry(nowMs) {
|
|
397
|
+
if (this.keyExpiryInFlight)
|
|
398
|
+
return;
|
|
399
|
+
this.keyExpiryInFlight = true;
|
|
400
|
+
this.keyExpiryLastCheckedAt = nowMs;
|
|
401
|
+
const exec = this.d.execTailscaleStatusJson ?? defaultExecTailscaleStatusJson;
|
|
402
|
+
void Promise.resolve()
|
|
403
|
+
.then(() => exec())
|
|
404
|
+
.then((raw) => {
|
|
405
|
+
if (raw === null) {
|
|
406
|
+
// Absent CLI ⇒ the expiry tier is silently absent (ONE debug line).
|
|
407
|
+
this.keyExpiryAvailable = false;
|
|
408
|
+
this.keyExpirySoonest = null;
|
|
409
|
+
if (!this.keyExpiryAbsentLogged) {
|
|
410
|
+
this.keyExpiryAbsentLogged = true;
|
|
411
|
+
this.log('tailscale CLI absent/unavailable — key-expiry tier silently absent');
|
|
412
|
+
}
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
const parse = parseTailscaleStatus(raw);
|
|
416
|
+
if (!parse.parsed) {
|
|
417
|
+
this.keyExpiryAvailable = false;
|
|
418
|
+
this.keyExpirySoonest = null;
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
this.keyExpiryAvailable = true;
|
|
422
|
+
this.keyExpirySoonest = soonestKeyExpiry(parse, this.now());
|
|
423
|
+
if (this.keyExpiryWarn())
|
|
424
|
+
this.metric('key-expiry-warning');
|
|
425
|
+
})
|
|
426
|
+
.catch((err) => {
|
|
427
|
+
// @silent-fallback-ok: the expiry tier degrades to absent on any exec
|
|
428
|
+
// fault (the rest of the monitor is unaffected — spec R-r2-3).
|
|
429
|
+
this.log(`key-expiry check failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
430
|
+
this.keyExpiryAvailable = false;
|
|
431
|
+
})
|
|
432
|
+
.finally(() => {
|
|
433
|
+
this.keyExpiryInFlight = false;
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
keyExpiryWarn() {
|
|
437
|
+
return (this.keyExpiryAvailable &&
|
|
438
|
+
this.keyExpirySoonest !== null &&
|
|
439
|
+
this.keyExpirySoonest.inDays <= this.cfg.keyExpiryWarnDays);
|
|
440
|
+
}
|
|
441
|
+
/** The GET /mesh/rope-health read surface (content-scrubbed). */
|
|
442
|
+
status() {
|
|
443
|
+
const rows = this.safeSnapshot();
|
|
444
|
+
const rowsByPeer = new Map();
|
|
445
|
+
for (const r of rows) {
|
|
446
|
+
const list = rowsByPeer.get(r.peer) ?? [];
|
|
447
|
+
list.push(r);
|
|
448
|
+
rowsByPeer.set(r.peer, list);
|
|
449
|
+
}
|
|
450
|
+
const peers = [];
|
|
451
|
+
for (const p of this.safePeers()) {
|
|
452
|
+
const st = this.peers.get(p.machineId);
|
|
453
|
+
peers.push({
|
|
454
|
+
machineId: p.machineId,
|
|
455
|
+
nickname: p.nickname,
|
|
456
|
+
condition: st?.condition ?? 'unknown',
|
|
457
|
+
kinds: (rowsByPeer.get(p.machineId) ?? []).map((r) => ({
|
|
458
|
+
kind: r.kind,
|
|
459
|
+
dead: r.dead,
|
|
460
|
+
consecutiveFailures: r.consecutiveFailures,
|
|
461
|
+
})),
|
|
462
|
+
allDownSince: st?.allDownSince ?? null,
|
|
463
|
+
episodeKey: st?.episodeKey ?? null,
|
|
464
|
+
urgentRaisedAt: st?.urgentRaisedAt ?? null,
|
|
465
|
+
detectedNotNotified: st?.detectedNotNotified ?? false,
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
return {
|
|
469
|
+
lastEvaluatedAt: this.lastEvaluatedAt,
|
|
470
|
+
evaluations: this.evaluations,
|
|
471
|
+
peers,
|
|
472
|
+
keyExpiry: {
|
|
473
|
+
available: this.keyExpiryAvailable,
|
|
474
|
+
...(this.keyExpirySoonest
|
|
475
|
+
? { soonest: { ...this.keyExpirySoonest, inDays: Math.round(this.keyExpirySoonest.inDays * 10) / 10 } }
|
|
476
|
+
: {}),
|
|
477
|
+
warn: this.keyExpiryWarn(),
|
|
478
|
+
lastCheckedAt: this.keyExpiryLastCheckedAt,
|
|
479
|
+
},
|
|
480
|
+
digest: this.composeDigest(),
|
|
481
|
+
counters: {
|
|
482
|
+
urgentEpisodes: this.urgentEpisodes,
|
|
483
|
+
suppressedBySleepGate: this.suppressedBySleepGate,
|
|
484
|
+
suppressedBySplitBrain: this.suppressedBySplitBrain,
|
|
485
|
+
notNotifiedRetries: this.notNotifiedRetries,
|
|
486
|
+
},
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
/**
|
|
490
|
+
* The consolidated daily-digest section (≤3 sentences, clamped,
|
|
491
|
+
* machine-named), or null when everything is ok — the rope-health-digest
|
|
492
|
+
* job's content source. Deterministic; content-scrubbed by construction.
|
|
493
|
+
*/
|
|
494
|
+
composeDigest() {
|
|
495
|
+
const nowMs = this.now();
|
|
496
|
+
const sentences = [];
|
|
497
|
+
for (const p of this.safePeers()) {
|
|
498
|
+
const st = this.peers.get(p.machineId);
|
|
499
|
+
if (!st)
|
|
500
|
+
continue;
|
|
501
|
+
if (st.condition === 'urgent') {
|
|
502
|
+
const min = st.allDownSince ? Math.round((nowMs - st.allDownSince) / 60_000) : 0;
|
|
503
|
+
sentences.push(`ALL mesh ropes to ${p.nickname} are down (~${min} min) while its heartbeat still advances — alive but unreachable.`);
|
|
504
|
+
}
|
|
505
|
+
else if (st.condition === 'peer-offline') {
|
|
506
|
+
const min = st.allDownSince ? Math.round((nowMs - st.allDownSince) / 60_000) : 0;
|
|
507
|
+
sentences.push(`${p.nickname} is offline (~${min} min) — expected (its heartbeat stopped).`);
|
|
508
|
+
}
|
|
509
|
+
else if (st.condition === 'degraded') {
|
|
510
|
+
const rows = this.safeSnapshot().filter((r) => r.peer === p.machineId && r.dead);
|
|
511
|
+
const kinds = rows.map((r) => r.kind).join(', ') || 'a rope';
|
|
512
|
+
sentences.push(`The ${kinds} rope to ${p.nickname} is down; another rope is carrying traffic.`);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
if (this.keyExpiryWarn() && this.keyExpirySoonest) {
|
|
516
|
+
const who = this.keyExpirySoonest.role === 'self' ? 'this machine' : 'a peer machine';
|
|
517
|
+
sentences.push(`A Tailscale key (${who}) expires in ${Math.max(0, Math.floor(this.keyExpirySoonest.inDays))} days — re-authenticate before it drops the rope.`);
|
|
518
|
+
}
|
|
519
|
+
if (sentences.length === 0)
|
|
520
|
+
return null;
|
|
521
|
+
return sentences.slice(0, 3).join(' ');
|
|
522
|
+
}
|
|
523
|
+
/** Record a digest emission (the digest job calls the route with ?digest=1). */
|
|
524
|
+
recordDigestEmission() {
|
|
525
|
+
this.metric('digest-emission');
|
|
526
|
+
}
|
|
527
|
+
// ── Safe dep reads (fail toward silence). ──
|
|
528
|
+
safeSnapshot() {
|
|
529
|
+
try {
|
|
530
|
+
return this.d.snapshot();
|
|
531
|
+
}
|
|
532
|
+
catch {
|
|
533
|
+
// @silent-fallback-ok: a throwing snapshot source yields an empty view this
|
|
534
|
+
// pass (UNKNOWN, fails toward NOT-urgent) — the next pass re-reads.
|
|
535
|
+
return [];
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
safePeers() {
|
|
539
|
+
try {
|
|
540
|
+
return this.d.listPeers().filter((p) => p.machineId !== this.d.selfMachineId);
|
|
541
|
+
}
|
|
542
|
+
catch {
|
|
543
|
+
// @silent-fallback-ok: same posture as safeSnapshot.
|
|
544
|
+
return [];
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
safeHeartbeat(machineId) {
|
|
548
|
+
try {
|
|
549
|
+
return this.d.readHeartbeatAtMs(machineId);
|
|
550
|
+
}
|
|
551
|
+
catch {
|
|
552
|
+
// @silent-fallback-ok: an unreadable heartbeat is "no evidence" — the
|
|
553
|
+
// classifier then reads peer-offline (fails toward NOT-urgent).
|
|
554
|
+
return null;
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
safeSplitBrainOpen(machineId) {
|
|
558
|
+
try {
|
|
559
|
+
return this.d.splitBrainItemOpen?.(machineId) ?? false;
|
|
560
|
+
}
|
|
561
|
+
catch {
|
|
562
|
+
// @silent-fallback-ok: an unreadable episode registry must not block the
|
|
563
|
+
// urgent item (the failure direction here is a possible duplicate ask,
|
|
564
|
+
// never a silent partition).
|
|
565
|
+
return false;
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
recordTransition(prev, next) {
|
|
569
|
+
if (prev === next)
|
|
570
|
+
return;
|
|
571
|
+
const evt = `transition-${next}`;
|
|
572
|
+
if (next !== 'unknown')
|
|
573
|
+
this.metric(evt);
|
|
574
|
+
this.scheduleWrite();
|
|
575
|
+
}
|
|
576
|
+
// ── Durable state (R-r2-4: transition-only writes + short debounce). ──
|
|
577
|
+
scheduleWrite() {
|
|
578
|
+
this.writePending = true;
|
|
579
|
+
if (this.writeTimer)
|
|
580
|
+
return;
|
|
581
|
+
if (this.cfg.writeDebounceMs <= 0) {
|
|
582
|
+
this.writeStateNow();
|
|
583
|
+
return;
|
|
584
|
+
}
|
|
585
|
+
this.writeTimer = setTimeout(() => {
|
|
586
|
+
this.writeTimer = null;
|
|
587
|
+
if (this.writePending)
|
|
588
|
+
this.writeStateNow();
|
|
589
|
+
}, this.cfg.writeDebounceMs);
|
|
590
|
+
if (typeof this.writeTimer.unref === 'function')
|
|
591
|
+
this.writeTimer.unref();
|
|
592
|
+
}
|
|
593
|
+
writeStateNow() {
|
|
594
|
+
this.writePending = false;
|
|
595
|
+
try {
|
|
596
|
+
const out = {};
|
|
597
|
+
for (const [id, st] of this.peers) {
|
|
598
|
+
out[id] = {
|
|
599
|
+
condition: st.condition,
|
|
600
|
+
allDownSince: st.allDownSince,
|
|
601
|
+
consecutiveObservations: st.consecutiveObservations,
|
|
602
|
+
episodeKey: st.episodeKey,
|
|
603
|
+
urgentRaisedAt: st.urgentRaisedAt,
|
|
604
|
+
detectedNotNotified: st.detectedNotNotified,
|
|
605
|
+
};
|
|
606
|
+
}
|
|
607
|
+
fs.mkdirSync(path.dirname(this.d.stateFilePath), { recursive: true });
|
|
608
|
+
const tmp = `${this.d.stateFilePath}.tmp`;
|
|
609
|
+
fs.writeFileSync(tmp, JSON.stringify({ version: 1, peers: out }, null, 2));
|
|
610
|
+
fs.renameSync(tmp, this.d.stateFilePath);
|
|
611
|
+
}
|
|
612
|
+
catch (err) {
|
|
613
|
+
// @silent-fallback-ok: state persistence is debounce memory, never
|
|
614
|
+
// authority — a failed write means a restart re-debounces (the declared
|
|
615
|
+
// safe direction); the alert path is unaffected.
|
|
616
|
+
this.log(`state write failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
loadState() {
|
|
620
|
+
try {
|
|
621
|
+
if (!fs.existsSync(this.d.stateFilePath))
|
|
622
|
+
return;
|
|
623
|
+
const raw = JSON.parse(fs.readFileSync(this.d.stateFilePath, 'utf-8'));
|
|
624
|
+
for (const [id, p] of Object.entries(raw.peers ?? {})) {
|
|
625
|
+
this.peers.set(id, {
|
|
626
|
+
...freshPeerState(),
|
|
627
|
+
condition: isCondition(p.condition) ? p.condition : 'unknown',
|
|
628
|
+
allDownSince: typeof p.allDownSince === 'number' ? p.allDownSince : null,
|
|
629
|
+
// Intra-episode counters lost to a restart are ACCEPTED (a restart
|
|
630
|
+
// re-debounces; it never fabricates an episode) — R-r2-4.
|
|
631
|
+
consecutiveObservations: 0,
|
|
632
|
+
episodeKey: typeof p.episodeKey === 'string' ? p.episodeKey : null,
|
|
633
|
+
urgentRaisedAt: typeof p.urgentRaisedAt === 'number' ? p.urgentRaisedAt : null,
|
|
634
|
+
detectedNotNotified: p.detectedNotNotified === true,
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
catch {
|
|
639
|
+
// @silent-fallback-ok: a corrupt state file is a missing one (the safe
|
|
640
|
+
// direction: re-debounce from scratch) — never a boot failure.
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
function freshPeerState() {
|
|
645
|
+
return {
|
|
646
|
+
condition: 'unknown',
|
|
647
|
+
allDownSince: null,
|
|
648
|
+
consecutiveObservations: 0,
|
|
649
|
+
episodeKey: null,
|
|
650
|
+
urgentRaisedAt: null,
|
|
651
|
+
detectedNotNotified: false,
|
|
652
|
+
healthySince: null,
|
|
653
|
+
postOnsetBeatObservedAt: null,
|
|
654
|
+
};
|
|
655
|
+
}
|
|
656
|
+
function isCondition(v) {
|
|
657
|
+
return v === 'ok' || v === 'degraded' || v === 'peer-offline' || v === 'urgent' || v === 'unknown';
|
|
658
|
+
}
|
|
659
|
+
//# sourceMappingURL=RopeHealthMonitor.js.map
|