instar 0.28.111 → 0.28.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,475 @@
1
+ #!/bin/bash
2
+ # Instar Fleet Watchdog
3
+ # Monitors all registered Instar agents and recovers them when they fail.
4
+ # Runs via launchd every 5 minutes.
5
+ #
6
+ # Recovery capabilities:
7
+ # 1. Reloads agents that launchd has unloaded entirely
8
+ # 2. Detects agents that are loaded but crash-looping (no PID, non-zero exit)
9
+ # 3. Self-heals common crash causes:
10
+ # - Missing shadow install → reinstalls it (PATH-resolved npm, works under launchd)
11
+ # - Missing node symlink → recreates it
12
+ # - Stale lifeline locks → removes them
13
+ # 4. Force-reloads services after self-healing
14
+ # 5. Escalates to user via healthy peer agent's /attention endpoint when
15
+ # self-heal fails N consecutive cycles for the same agent.
16
+ #
17
+ # Usage: ./instar-watchdog.sh [--dry-run] [--verbose]
18
+ #
19
+ # Source of truth: src/templates/scripts/instar-watchdog.sh in the instar repo.
20
+ # Installed/migrated by PostUpdateMigrator.migrateFleetWatchdog().
21
+
22
+ set -euo pipefail
23
+
24
+ # Paths can be overridden via env vars for testability. Defaults are production.
25
+ LAUNCH_AGENTS_DIR="${INSTAR_WATCHDOG_LAUNCH_AGENTS_DIR:-$HOME/Library/LaunchAgents}"
26
+ LOG_FILE="${INSTAR_WATCHDOG_LOG_FILE:-$HOME/.instar/watchdog.log}"
27
+ HEAL_STATE_DIR="${INSTAR_WATCHDOG_STATE_DIR:-$HOME/.instar/watchdog-state}"
28
+ ESCALATE_AFTER_FAILS="${INSTAR_WATCHDOG_ESCALATE_AFTER:-3}"
29
+ DRY_RUN=false
30
+ VERBOSE=false
31
+
32
+ for arg in "$@"; do
33
+ case "$arg" in
34
+ --dry-run) DRY_RUN=true ;;
35
+ --verbose) VERBOSE=true ;;
36
+ esac
37
+ done
38
+
39
+ mkdir -p "$(dirname "$LOG_FILE")" "$HEAL_STATE_DIR"
40
+
41
+ log() {
42
+ local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $1"
43
+ echo "$msg" >> "$LOG_FILE"
44
+ if $VERBOSE; then echo "$msg"; fi
45
+ }
46
+
47
+ # Resolve an executable node binary. PATH is often empty under launchd, so
48
+ # we cannot rely on `which` / `command -v` exclusively. Absolute paths first;
49
+ # fall back to PATH lookup so the script works on Linux/CI/non-macOS hosts
50
+ # where Homebrew paths don't exist. Caches result via global var to avoid
51
+ # re-probing.
52
+ RESOLVED_NODE=""
53
+ resolve_node() {
54
+ [ -n "$RESOLVED_NODE" ] && { echo "$RESOLVED_NODE"; return 0; }
55
+ # Explicit override always wins (used by tests and unusual deployments).
56
+ if [ -n "${INSTAR_WATCHDOG_NODE_BIN:-}" ] && [ -x "${INSTAR_WATCHDOG_NODE_BIN}" ]; then
57
+ RESOLVED_NODE="${INSTAR_WATCHDOG_NODE_BIN}"; echo "$RESOLVED_NODE"; return 0
58
+ fi
59
+ for cand in /opt/homebrew/bin/node /usr/local/bin/node /usr/bin/node; do
60
+ if [ -x "$cand" ]; then RESOLVED_NODE="$cand"; echo "$cand"; return 0; fi
61
+ done
62
+ # Last resort: ask the shell. May find nvm/asdf/system node.
63
+ local p
64
+ p=$(command -v node 2>/dev/null || true)
65
+ if [ -n "$p" ] && [ -x "$p" ]; then RESOLVED_NODE="$p"; echo "$p"; return 0; fi
66
+ return 1
67
+ }
68
+
69
+ # Resolve the npm-cli.js entry point so we can invoke it as `node npm-cli.js`,
70
+ # bypassing npm's `#!/usr/bin/env node` shebang (which fails under empty PATH).
71
+ # Same fallback order as resolve_node.
72
+ RESOLVED_NPM=""
73
+ resolve_npm() {
74
+ [ -n "$RESOLVED_NPM" ] && { echo "$RESOLVED_NPM"; return 0; }
75
+ if [ -n "${INSTAR_WATCHDOG_NPM_CLI:-}" ] && [ -r "${INSTAR_WATCHDOG_NPM_CLI}" ]; then
76
+ RESOLVED_NPM="${INSTAR_WATCHDOG_NPM_CLI}"; echo "$RESOLVED_NPM"; return 0
77
+ fi
78
+ for cand in \
79
+ /opt/homebrew/lib/node_modules/npm/bin/npm-cli.js \
80
+ /usr/local/lib/node_modules/npm/bin/npm-cli.js \
81
+ /usr/lib/node_modules/npm/bin/npm-cli.js \
82
+ /usr/share/npm/bin/npm-cli.js; do
83
+ if [ -r "$cand" ]; then RESOLVED_NPM="$cand"; echo "$cand"; return 0; fi
84
+ done
85
+ # Last resort: derive from `which npm` (npm itself is a shell script that
86
+ # exec's its own npm-cli.js — find that path).
87
+ local p npm_real
88
+ p=$(command -v npm 2>/dev/null || true)
89
+ if [ -n "$p" ]; then
90
+ # npm is usually a symlink to ../lib/node_modules/npm/bin/npm-cli.js
91
+ npm_real=$(readlink -f "$p" 2>/dev/null || readlink "$p" 2>/dev/null || echo "$p")
92
+ case "$npm_real" in
93
+ *.js) [ -r "$npm_real" ] && RESOLVED_NPM="$npm_real" && echo "$npm_real" && return 0 ;;
94
+ *)
95
+ # npm wrapper points to a different script — try its sibling lib path.
96
+ local guess="$(dirname "$p")/../lib/node_modules/npm/bin/npm-cli.js"
97
+ if [ -r "$guess" ]; then
98
+ RESOLVED_NPM=$(cd "$(dirname "$guess")" 2>/dev/null && pwd)/$(basename "$guess")
99
+ echo "$RESOLVED_NPM"
100
+ return 0
101
+ fi
102
+ ;;
103
+ esac
104
+ fi
105
+ return 1
106
+ }
107
+
108
+ # Extract the WorkingDirectory (project dir) from a launchd plist.
109
+ # macOS: PlistBuddy is fast and authoritative.
110
+ # Linux/CI: falls back to python3 XML parsing, then crude grep.
111
+ get_project_dir() {
112
+ local plist="$1"
113
+ if [ -x /usr/libexec/PlistBuddy ]; then
114
+ /usr/libexec/PlistBuddy -c "Print :WorkingDirectory" "$plist" 2>/dev/null || true
115
+ return
116
+ fi
117
+ if command -v python3 &>/dev/null; then
118
+ python3 - "$plist" 2>/dev/null <<'PYEOF' || true
119
+ import sys, xml.etree.ElementTree as ET
120
+ d = ET.parse(sys.argv[1]).getroot().find('dict')
121
+ els = list(d)
122
+ for i, el in enumerate(els):
123
+ if el.tag == 'key' and el.text == 'WorkingDirectory' and i + 1 < len(els):
124
+ print(els[i + 1].text)
125
+ break
126
+ PYEOF
127
+ return
128
+ fi
129
+ grep -A1 '<key>WorkingDirectory</key>' "$plist" 2>/dev/null \
130
+ | grep '<string>' | sed 's|.*<string>\(.*\)</string>.*|\1|' || true
131
+ }
132
+
133
+ # Try to fix common issues that prevent an agent from starting.
134
+ # Returns 0 if a fix was applied, 1 if nothing actionable.
135
+ try_self_heal() {
136
+ local project_dir="$1"
137
+ local label="$2"
138
+ local healed=1
139
+
140
+ if [ -z "$project_dir" ] || [ ! -d "$project_dir" ]; then
141
+ log "HEAL-SKIP: $label — project dir not found or empty"
142
+ return 1
143
+ fi
144
+
145
+ local state_dir="$project_dir/.instar"
146
+
147
+ # Heal 1: Missing shadow install
148
+ local shadow_dir="$state_dir/shadow-install"
149
+ local shadow_cli="$shadow_dir/node_modules/instar/dist/cli.js"
150
+ if [ ! -f "$shadow_cli" ]; then
151
+ log "HEAL: $label — shadow install missing, reinstalling"
152
+ if ! $DRY_RUN; then
153
+ local node_bin npm_cli
154
+ node_bin=$(resolve_node || true)
155
+ npm_cli=$(resolve_npm || true)
156
+
157
+ if [ -z "$node_bin" ] || [ -z "$npm_cli" ]; then
158
+ log "HEAL-FAIL: $label — no node/npm binary found (node='$node_bin' npm='$npm_cli')"
159
+ else
160
+ mkdir -p "$shadow_dir"
161
+ if [ ! -f "$shadow_dir/package.json" ]; then
162
+ cat > "$shadow_dir/package.json" <<'PKGEOF'
163
+ {
164
+ "name": "instar-shadow",
165
+ "private": true,
166
+ "dependencies": { "instar": "latest" }
167
+ }
168
+ PKGEOF
169
+ fi
170
+ if "$node_bin" "$npm_cli" install --no-audit --no-fund --silent --prefix "$shadow_dir" >> "$LOG_FILE" 2>&1; then
171
+ if [ -f "$shadow_cli" ]; then
172
+ log "HEAL-OK: $label — shadow install restored"
173
+ healed=0
174
+ else
175
+ log "HEAL-FAIL: $label — npm install ran but CLI still missing"
176
+ fi
177
+ else
178
+ log "HEAL-FAIL: $label — npm install exited non-zero"
179
+ fi
180
+ fi
181
+ else
182
+ log "DRY-RUN: Would reinstall shadow for $label"
183
+ healed=0
184
+ fi
185
+ fi
186
+
187
+ # Heal 2: Missing or broken node symlink
188
+ local node_symlink="$state_dir/bin/node"
189
+ if [ ! -x "$node_symlink" ] || ! "$node_symlink" --version &>/dev/null; then
190
+ log "HEAL: $label — node symlink missing or broken"
191
+ if ! $DRY_RUN; then
192
+ mkdir -p "$state_dir/bin"
193
+ local node_path
194
+ node_path=$(resolve_node || true)
195
+ if [ -n "$node_path" ]; then
196
+ ln -sf "$node_path" "$node_symlink"
197
+ log "HEAL-OK: $label — node symlink → $node_path"
198
+ healed=0
199
+ else
200
+ log "HEAL-FAIL: $label — no node binary found for symlink"
201
+ fi
202
+ else
203
+ log "DRY-RUN: Would fix node symlink for $label"
204
+ healed=0
205
+ fi
206
+ fi
207
+
208
+ # Heal 3: Stale lifeline lock
209
+ local lock_file="$state_dir/lifeline.lock"
210
+ if [ -f "$lock_file" ]; then
211
+ local lock_age
212
+ lock_age=$(( $(date +%s) - $(stat -f %m "$lock_file" 2>/dev/null || echo 0) ))
213
+ if [ "$lock_age" -gt 600 ]; then
214
+ log "HEAL: $label — stale lifeline lock (${lock_age}s old)"
215
+ if ! $DRY_RUN; then
216
+ rm -f "$lock_file"
217
+ log "HEAL-OK: $label — lock removed"
218
+ healed=0
219
+ else
220
+ log "DRY-RUN: Would remove stale lock for $label"
221
+ healed=0
222
+ fi
223
+ fi
224
+ fi
225
+
226
+ # Heal 4: Last-error context for diagnostics
227
+ local err_log="$state_dir/logs/lifeline-launchd.err"
228
+ if [ -f "$err_log" ] && [ "$healed" -ne 0 ]; then
229
+ local last_err
230
+ last_err=$(tail -1 "$err_log" 2>/dev/null || true)
231
+ if [ -n "$last_err" ]; then
232
+ log "HEAL-INFO: $label — last error: $last_err"
233
+ fi
234
+ fi
235
+
236
+ return $healed
237
+ }
238
+
239
+ # Rate-limit overall heal attempts per agent (in addition to the boot-wrapper's
240
+ # own 5-min debounce). Default: one attempt per 30 minutes.
241
+ should_attempt_heal() {
242
+ local label="$1"
243
+ local state_file="$HEAL_STATE_DIR/$label.last-heal"
244
+ [ ! -f "$state_file" ] && return 0
245
+ local last_heal
246
+ last_heal=$(cat "$state_file" 2>/dev/null || echo 0)
247
+ local now elapsed
248
+ now=$(date +%s)
249
+ elapsed=$(( now - last_heal ))
250
+ [ "$elapsed" -gt 1800 ] && return 0
251
+ return 1
252
+ }
253
+
254
+ mark_heal_attempt() {
255
+ local label="$1"
256
+ date +%s > "$HEAL_STATE_DIR/$label.last-heal"
257
+ }
258
+
259
+ # Increment consecutive-fail counter; return new value via stdout.
260
+ bump_fail_counter() {
261
+ local label="$1"
262
+ local fail_file="$HEAL_STATE_DIR/$label.consecutive-heal-fails"
263
+ local current=0
264
+ [ -r "$fail_file" ] && current=$(cat "$fail_file" 2>/dev/null || echo 0)
265
+ current=$((current + 1))
266
+ echo "$current" > "$fail_file"
267
+ echo "$current"
268
+ }
269
+
270
+ reset_fail_counter() {
271
+ local label="$1"
272
+ rm -f "$HEAL_STATE_DIR/$label.consecutive-heal-fails" 2>/dev/null || true
273
+ }
274
+
275
+ # Escalate via a healthy peer agent's /attention endpoint.
276
+ # The dead agent has no Telegram bot connection by definition; SOME peer's
277
+ # server has to make the actual Telegram call. We discover a peer by scanning
278
+ # launchctl labels for ai.instar.* and probing /health on each agent's port.
279
+ #
280
+ # The payload uses category="degradation" which triggers the isHealthAlert
281
+ # branch in routes.ts → MessagingToneGate with B12-B14 ruleset. If the gate
282
+ # blocks our message it falls back to SAFE_HEALTH_ALERT_TEMPLATE — either way
283
+ # the user gets a clean Telegram alert.
284
+ escalate_via_peer() {
285
+ local dead_label="$1"
286
+ local fail_count="$2"
287
+ local minutes_down=$((fail_count * 5))
288
+
289
+ # Strip ai.instar. prefix for the user-facing name
290
+ local dead_name="${dead_label#ai.instar.}"
291
+
292
+ # Find a healthy peer
293
+ local peer_plist peer_label peer_dir peer_port peer_auth health_code
294
+ for peer_plist in "$LAUNCH_AGENTS_DIR"/ai.instar.*.plist; do
295
+ [ -f "$peer_plist" ] || continue
296
+ case "$peer_plist" in *.DISABLED) continue ;; esac
297
+ peer_label=$(basename "$peer_plist" .plist)
298
+ [ "$peer_label" = "$dead_label" ] && continue
299
+ [ "$peer_label" = "ai.instar.watchdog" ] && continue
300
+
301
+ peer_dir=$(get_project_dir "$peer_plist")
302
+ [ -z "$peer_dir" ] && continue
303
+ [ ! -r "$peer_dir/.instar/config.json" ] && continue
304
+
305
+ # Read peer port + auth via node. Pass config path as argv (process.argv[1])
306
+ # rather than interpolating into the source string — defends against any
307
+ # weird character ever appearing in HOME or project paths.
308
+ local node_bin
309
+ node_bin=$(resolve_node || true)
310
+ [ -z "$node_bin" ] && continue
311
+ local peer_meta
312
+ peer_meta=$("$node_bin" -e "const c=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'));process.stdout.write((c.port||'')+'\t'+(c.authToken||''))" "$peer_dir/.instar/config.json" 2>/dev/null || true)
313
+ peer_port="${peer_meta%% *}"
314
+ peer_auth="${peer_meta##* }"
315
+ [ -z "$peer_port" ] && continue
316
+
317
+ health_code=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${peer_port}/health" 2>/dev/null || echo "000")
318
+ if [ "$health_code" = "200" ]; then
319
+ log "ESCALATE: $dead_label — via peer $peer_label (port $peer_port)"
320
+ if ! $DRY_RUN; then
321
+ # First attempt: our specific payload. If the tone gate (B12-B14)
322
+ # blocks it (422), createAttentionItem is NOT called — no Telegram
323
+ # topic, no user alert. We MUST retry with the canonical safe
324
+ # template to guarantee the user is paged.
325
+ local payload safe_payload resp_code
326
+ payload=$(cat <<JSONEOF
327
+ {
328
+ "id": "fleet-watchdog-heal-fail-${dead_label}-${fail_count}",
329
+ "title": "${dead_name} is offline",
330
+ "summary": "${dead_name} has been offline for about ${minutes_down} minutes and my repair attempts aren't working.",
331
+ "description": "Want me to dig in?",
332
+ "category": "degradation",
333
+ "priority": "HIGH"
334
+ }
335
+ JSONEOF
336
+ )
337
+ resp_code=$(curl -s -o /dev/null -w "%{http_code}" \
338
+ -X POST "http://localhost:${peer_port}/attention" \
339
+ -H "Authorization: Bearer $peer_auth" \
340
+ -H "Content-Type: application/json" \
341
+ -H "X-Instar-Request: 1" \
342
+ -d "$payload" 2>/dev/null || echo "000")
343
+ log "ESCALATE-RESULT: $dead_label — peer responded $resp_code"
344
+
345
+ if [ "$resp_code" = "201" ]; then
346
+ reset_fail_counter "$dead_label"
347
+ return 0
348
+ fi
349
+
350
+ if [ "$resp_code" = "422" ]; then
351
+ # Tone gate blocked our specific copy. Retry with canonical safe
352
+ # template — same wording as the in-process DegradationReporter
353
+ # fallback so it MUST pass the gate (or the gate itself is broken).
354
+ log "ESCALATE-RETRY: $dead_label — retrying with safe template"
355
+ safe_payload=$(cat <<JSONEOF
356
+ {
357
+ "id": "fleet-watchdog-heal-fail-${dead_label}-${fail_count}-safe",
358
+ "title": "Something stopped working",
359
+ "summary": "Something on my end stopped working and I haven't been able to fix it on my own.",
360
+ "description": "Want me to dig in?",
361
+ "category": "degradation",
362
+ "priority": "HIGH"
363
+ }
364
+ JSONEOF
365
+ )
366
+ resp_code=$(curl -s -o /dev/null -w "%{http_code}" \
367
+ -X POST "http://localhost:${peer_port}/attention" \
368
+ -H "Authorization: Bearer $peer_auth" \
369
+ -H "Content-Type: application/json" \
370
+ -H "X-Instar-Request: 1" \
371
+ -d "$safe_payload" 2>/dev/null || echo "000")
372
+ log "ESCALATE-RETRY-RESULT: $dead_label — peer responded $resp_code"
373
+ if [ "$resp_code" = "201" ]; then
374
+ reset_fail_counter "$dead_label"
375
+ return 0
376
+ fi
377
+ fi
378
+
379
+ # Any other response: counter STAYS. We will retry next cycle.
380
+ log "ESCALATE-INCOMPLETE: $dead_label — counter preserved for next cycle"
381
+ return 1
382
+ else
383
+ log "DRY-RUN: Would POST attention to $peer_label for $dead_label"
384
+ return 0
385
+ fi
386
+ fi
387
+ done
388
+ log "ESCALATE-FAIL: $dead_label — no healthy peer found to relay alert"
389
+ return 1
390
+ }
391
+
392
+ recovered=0
393
+ checked=0
394
+ for plist in "$LAUNCH_AGENTS_DIR"/ai.instar.*.plist; do
395
+ [ -f "$plist" ] || continue
396
+ case "$plist" in *.DISABLED) continue ;; esac
397
+
398
+ label=$(basename "$plist" .plist)
399
+ # Don't watchdog the watchdog
400
+ [ "$label" = "ai.instar.watchdog" ] && continue
401
+ checked=$((checked + 1))
402
+
403
+ if launchctl list "$label" &>/dev/null; then
404
+ pid=$(launchctl list "$label" 2>/dev/null | grep '"PID"' | grep -o '[0-9]*' || true)
405
+ exit_status=$(launchctl list "$label" 2>/dev/null | grep '"LastExitStatus"' | grep -o '[0-9]*' || true)
406
+
407
+ if [ -n "$pid" ] && [ "$pid" != "0" ]; then
408
+ # Healthy — clear heal state
409
+ rm -f "$HEAL_STATE_DIR/$label.last-heal" 2>/dev/null || true
410
+ reset_fail_counter "$label"
411
+ $VERBOSE && log "OK: $label (PID $pid)"
412
+ elif [ -n "$exit_status" ] && [ "$exit_status" != "0" ]; then
413
+ log "CRASH-LOOP: $label (exit $exit_status, no PID)"
414
+
415
+ if should_attempt_heal "$label"; then
416
+ project_dir=$(get_project_dir "$plist")
417
+ if try_self_heal "$project_dir" "$label"; then
418
+ log "RECOVERING: $label — self-heal applied, reloading"
419
+ if ! $DRY_RUN; then
420
+ mark_heal_attempt "$label"
421
+ launchctl bootout "gui/$(id -u)" "$plist" 2>/dev/null || launchctl unload "$plist" 2>/dev/null || true
422
+ sleep 1
423
+ launchctl bootstrap "gui/$(id -u)" "$plist" 2>/dev/null || launchctl load "$plist" 2>/dev/null || true
424
+ recovered=$((recovered + 1))
425
+ reset_fail_counter "$label"
426
+ log "RELOADED: $label (after self-heal)"
427
+ else
428
+ log "DRY-RUN: Would reload $label after self-heal"
429
+ fi
430
+ else
431
+ log "CRASH-LOOP: $label — no fixable issues found"
432
+ mark_heal_attempt "$label"
433
+ fail_count=$(bump_fail_counter "$label")
434
+ if [ "$fail_count" -ge "$ESCALATE_AFTER_FAILS" ]; then
435
+ escalate_via_peer "$label" "$fail_count" || true
436
+ else
437
+ $VERBOSE && log "FAIL-COUNT: $label — ${fail_count}/${ESCALATE_AFTER_FAILS}"
438
+ fi
439
+ fi
440
+ else
441
+ $VERBOSE && log "CRASH-LOOP: $label — heal attempted recently, waiting"
442
+ fi
443
+ else
444
+ $VERBOSE && log "LOADED-IDLE: $label (no PID, exit=$exit_status)"
445
+ fi
446
+ else
447
+ log "RECOVERING: $label (not loaded, reloading)"
448
+ if ! $DRY_RUN; then
449
+ project_dir=$(get_project_dir "$plist")
450
+ if [ -n "$project_dir" ]; then
451
+ try_self_heal "$project_dir" "$label" || true
452
+ fi
453
+ launchctl bootstrap "gui/$(id -u)" "$plist" 2>/dev/null || launchctl load "$plist" 2>/dev/null || true
454
+ recovered=$((recovered + 1))
455
+ log "RELOADED: $label"
456
+ else
457
+ log "DRY-RUN: Would reload $label"
458
+ fi
459
+ fi
460
+ done
461
+
462
+ if [ "$recovered" -gt 0 ]; then
463
+ log "Watchdog complete: $checked checked, $recovered recovered"
464
+ elif $VERBOSE; then
465
+ log "Watchdog complete: $checked checked, all healthy"
466
+ fi
467
+
468
+ # Trim log file
469
+ if [ -f "$LOG_FILE" ] && [ "$(wc -l < "$LOG_FILE")" -gt 1000 ]; then
470
+ tail -500 "$LOG_FILE" > "$LOG_FILE.tmp"
471
+ mv "$LOG_FILE.tmp" "$LOG_FILE"
472
+ fi
473
+
474
+ # Trim heal state files older than 24 hours
475
+ find "$HEAL_STATE_DIR" -name "*.last-heal" -mmin +1440 -delete 2>/dev/null || true
@@ -0,0 +1,28 @@
1
+ # Upgrade Guide — vNEXT
2
+
3
+ <!-- bump: patch -->
4
+
5
+ ## What Changed
6
+
7
+ Ships S-3 of Tier-3 from the self-healing remediator spec — the **commit-time** half of the promotion gate that pairs with the **PR-merge-time** half shipped in C-1 (PR #204).
8
+
9
+ A "proposal-derived runbook" is a runbook source file that the SystemReviewer's clustering pipeline (Tier-3 S-1, not yet shipped) emits when it spots a repeating failure pattern that warrants a new automated remediation. Those files carry two special markers: `__proposalDerivedFrom = '<proposalId>'` and `__producingAgentId = '<agentId>'`. Before this release, nothing checked those markers locally — the only enforcement happened in CI at PR review.
10
+
11
+ This release adds a second check at commit time, inside the `/instar-dev` skill's pre-commit hook:
12
+
13
+ - If you stage a runbook source under `src/remediation/runbooks/` that carries `__proposalDerivedFrom`, the hook reads the proposal JSON at `.instar/remediation/proposals-<machineId>/<proposalId>.json`, verifies the agent IDs match, and (if a signature and pubkey are present) verifies the Ed25519 signature.
14
+ - If anything is missing or mismatched, the commit is refused with a structured message pointing at the proposal pipeline as the responsible surface.
15
+ - Runbooks without the markers — every runbook on main today — pass through with no work.
16
+
17
+ This catches author mistakes before a PR is pushed. The CI gate (C-1) remains the authoritative check at PR-merge time because only CI can verify signing identity for the different-principal rule.
18
+
19
+ ## What to Tell Your User
20
+
21
+ No behavior change for users. This is a developer-side guardrail that prevents a malformed runbook from being committed in the first place. The user-visible promotion flow (proposal review and approval via the dashboard or Telegram) still works the same way.
22
+
23
+ ## Summary of New Capabilities
24
+
25
+ | Capability | How to Use |
26
+ |-----------|-----------|
27
+ | Commit-time promotion gate for proposal-derived runbooks | Automatic; runs inside `/instar-dev` pre-commit |
28
+ | `verify-proposal-derived-runbook.mjs` CLI | `node skills/instar-dev/scripts/verify-proposal-derived-runbook.mjs --files <csv>` |
@@ -0,0 +1,43 @@
1
+ # Upgrade Guide — vNEXT
2
+
3
+ <!-- bump: patch -->
4
+
5
+ ## What Changed
6
+
7
+ Three coordinated changes that close a fleet-wide outage class observed on 2026-05-17: AI Guy crash-looped for ~4 days after its `shadow-install/` directory vanished, and no alert reached the user because the fleet watchdog's self-heal was failing silently under launchd's empty PATH.
8
+
9
+ 1. **Boot wrapper self-heals missing shadow-install** (`src/commands/setup.ts`).
10
+ When `instar-boot.cjs` or `instar-boot.sh` finds no `shadow-install/node_modules/instar/dist/cli.js`, it now attempts ONE `npm install` (resolved via absolute `/opt/homebrew/bin/node` + `npm-cli.js` to survive launchd's empty PATH) before exiting. Debounced by a `.heal-attempted` marker file so launchd KeepAlive throttling can't trigger a reinstall storm. This alone would have recovered AI Guy within seconds.
11
+
12
+ 2. **Fleet watchdog ships from instar source** (`src/templates/scripts/instar-watchdog.sh`, new). The user-machine fleet watchdog at `~/.instar/instar-watchdog.sh` now ships from the repo, with two behavior fixes vs the prior hand-rolled version:
13
+ - Heal-step `npm install` uses absolute-path `node` + `npm-cli.js` resolution instead of bare `npm` (which silently fails under launchd PATH).
14
+ - When self-heal fails for 3 cycles in a row for the same agent (~15 min), the watchdog discovers a healthy peer agent and POSTs to that peer's `/attention` endpoint with `category: degradation`. The receiving server routes the alert through `MessagingToneGate` with the B12-B14 health-alert ruleset (the same authority `DegradationReporter` already uses in-process). On 422 (gate block) the watchdog retries with the canonical `SAFE_HEALTH_ALERT_TEMPLATE` so the user always learns SOMETHING when an agent stays dead.
15
+
16
+ 3. **Watchdog launchd plist sets PATH explicitly** (`src/core/PostUpdateMigrator.ts:migrateFleetWatchdog`, new). Belt-and-suspenders next to the absolute-path resolution inside the script.
17
+
18
+ The fleet watchdog and its launchd plist are now migrated to every existing agent on `instar update` and installed on every fresh agent setup.
19
+
20
+ ## What to Tell Your User
21
+
22
+ If an agent on your machine ever crashes in a way the watchdog can't auto-fix (network issues, disk full, etc.), you'll now get a plain-English Telegram message from one of your healthy agents within ~15 minutes: *"[name] is offline — repair attempts aren't working — want me to dig in?"* No more silent 4-day outages.
23
+
24
+ If you ONLY have one agent on a machine, peer escalation isn't available — you'd still need to notice via the dashboard or by trying to message the agent. Single-agent machines are a known gap; the long-term fix lives in the v3 Self-Healing Remediator's Tier-3 Fleet Intelligence.
25
+
26
+ ## Summary of New Capabilities
27
+
28
+ | Capability | How to Use |
29
+ |-----------|-----------|
30
+ | Boot-wrapper auto-reinstalls missing shadow-install | Automatic on next agent restart after this update |
31
+ | Fleet watchdog ships from source + migrates to existing agents | Automatic via `PostUpdateMigrator.migrateFleetWatchdog()` on every `instar update` |
32
+ | Cross-agent peer escalation on persistent agent failure | Automatic; configure threshold via `INSTAR_WATCHDOG_ESCALATE_AFTER` env (default 3 cycles = 15 min) |
33
+
34
+ ## Evidence
35
+
36
+ - Spec: `docs/specs/lifeline-shadow-install-self-heal.md` (with ELI16 companion `.eli16.md`)
37
+ - Side-effects review: `upgrades/side-effects/lifeline-shadow-install-self-heal.md`
38
+ - Tests: `tests/unit/lifeline-shadow-install-self-heal.test.ts` (14 tests), `tests/integration/fleet-watchdog-escalation.test.ts` (4 tests)
39
+ - Incident reference: AI Guy outage 2026-05-13 → 2026-05-17, topic 5447
40
+
41
+ ## Rollback
42
+
43
+ Pure code change. Revert the three source files (`src/commands/setup.ts`, `src/core/PostUpdateMigrator.ts`, `src/templates/scripts/instar-watchdog.sh`) and ship as a patch. No persistent state migrations needed; marker files and per-label fail counters are harmless if left behind.