aiur-cli 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/bin/aiur.js +235 -3
- package/libexec/aiur-engine.sh +1476 -73
- package/package.json +6 -6
- package/scripts/postinstall.mjs +5 -4
- package/share/aiur.tmux.conf +10 -0
package/libexec/aiur-engine.sh
CHANGED
|
@@ -23,6 +23,18 @@
|
|
|
23
23
|
|
|
24
24
|
set -euo pipefail
|
|
25
25
|
|
|
26
|
+
# Raise the soft open-file limit toward the hard maximum. High agent concurrency
|
|
27
|
+
# spawns many tmux/opencode/git subprocesses + sockets; on hosts with a low
|
|
28
|
+
# default (macOS ships 256) that exhausts file descriptors (:emfile) and crashes
|
|
29
|
+
# the node. Soft<=hard needs no privilege; best-effort, never fatal.
|
|
30
|
+
__aiur_hard_nofile="$(ulimit -Hn 2>/dev/null || echo)"
|
|
31
|
+
if [ "${__aiur_hard_nofile}" = "unlimited" ]; then
|
|
32
|
+
ulimit -Sn 65536 2>/dev/null || true
|
|
33
|
+
elif [ -n "${__aiur_hard_nofile}" ]; then
|
|
34
|
+
ulimit -Sn "${__aiur_hard_nofile}" 2>/dev/null || true
|
|
35
|
+
fi
|
|
36
|
+
unset __aiur_hard_nofile
|
|
37
|
+
|
|
26
38
|
die() {
|
|
27
39
|
echo "❌ $*" >&2
|
|
28
40
|
exit 1
|
|
@@ -30,19 +42,99 @@ die() {
|
|
|
30
42
|
|
|
31
43
|
engine_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
32
44
|
|
|
33
|
-
# --- distribution identity (
|
|
45
|
+
# --- distribution identity (per-instance: keyed by the aiur project root) -----
|
|
46
|
+
|
|
47
|
+
# The aiur project root used to key this instance. AIUR_REPO_ROOT (set by the dev
|
|
48
|
+
# shim) wins. Otherwise walk up from $PWD to the first dir holding a REPO-LOCAL
|
|
49
|
+
# config — but the walk STOPS at $HOME: the global config at ~/.aiur/config (legacy
|
|
50
|
+
# ~/.aiurconfig) is not a repo root, and treating it as one would collapse every
|
|
51
|
+
# project under $HOME onto one key (#443). When no repo-local config is found, the
|
|
52
|
+
# BEAM serves this run via the global config (mirroring its discovery order in
|
|
53
|
+
# src/lib/aiur/workflow.ex) — or via none — and in both cases the project being
|
|
54
|
+
# served is the cwd, so we key by realpath($PWD). That gives each global-config
|
|
55
|
+
# project a distinct identity instead of an empty key (legacy aiur-$USER@…) or $HOME.
|
|
56
|
+
#
|
|
57
|
+
# Caveat: a global-config run's key is cwd-derived (there is no repo root to
|
|
58
|
+
# converge on), so control commands (status/pause/stop) must be run from the SAME
|
|
59
|
+
# directory the run was launched from. A repo with a repo-local .aiur/config keeps
|
|
60
|
+
# the walk-up, so its control commands still resolve from any subdir.
|
|
61
|
+
aiur_project_root() {
|
|
62
|
+
if [ -n "${AIUR_REPO_ROOT:-}" ]; then printf '%s' "$AIUR_REPO_ROOT"; return; fi
|
|
63
|
+
|
|
64
|
+
# Canonicalize $PWD and $HOME so the home-boundary test below holds even when
|
|
65
|
+
# either is reached through a symlink.
|
|
66
|
+
local pwd_real home_real
|
|
67
|
+
pwd_real="$(pwd -P 2>/dev/null || printf '%s' "$PWD")"
|
|
68
|
+
# ${HOME:-} (not bare $HOME) so an unset HOME under `set -u` can't abort the
|
|
69
|
+
# script here; an empty home_real simply disables the boundary (walk to /).
|
|
70
|
+
home_real="$(cd "${HOME:-}" 2>/dev/null && pwd -P || printf '%s' "${HOME:-}")"
|
|
71
|
+
|
|
72
|
+
local d="$pwd_real"
|
|
73
|
+
while [ -n "$d" ] && [ "$d" != "/" ] && [ "$d" != "$home_real" ]; do
|
|
74
|
+
if [ -f "$d/.aiur/config" ] || [ -f "$d/.aiurconfig" ]; then
|
|
75
|
+
printf '%s' "$d"
|
|
76
|
+
return
|
|
77
|
+
fi
|
|
78
|
+
d="$(dirname "$d")"
|
|
79
|
+
done
|
|
80
|
+
|
|
81
|
+
printf '%s' "$pwd_real"
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
aiur_project_root_source() {
|
|
85
|
+
if [ -n "${AIUR_REPO_ROOT:-}" ]; then printf 'env'; return; fi
|
|
86
|
+
|
|
87
|
+
local pwd_real home_real
|
|
88
|
+
pwd_real="$(pwd -P 2>/dev/null || printf '%s' "$PWD")"
|
|
89
|
+
home_real="$(cd "${HOME:-}" 2>/dev/null && pwd -P || printf '%s' "${HOME:-}")"
|
|
90
|
+
|
|
91
|
+
local d="$pwd_real"
|
|
92
|
+
while [ -n "$d" ] && [ "$d" != "/" ] && [ "$d" != "$home_real" ]; do
|
|
93
|
+
if [ -f "$d/.aiur/config" ] || [ -f "$d/.aiurconfig" ]; then
|
|
94
|
+
printf 'repo'
|
|
95
|
+
return
|
|
96
|
+
fi
|
|
97
|
+
d="$(dirname "$d")"
|
|
98
|
+
done
|
|
99
|
+
|
|
100
|
+
printf 'cwd'
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
# Short, stable, node-name-legal (lowercase hex) key for the project root, so two
|
|
104
|
+
# aiur instances for the same user get distinct node/session/socket names and can't
|
|
105
|
+
# reap each other. Any real cwd now resolves a key (global-config runs key by
|
|
106
|
+
# realpath($PWD), #443); only a degenerate unreadable cwd yields empty, falling back
|
|
107
|
+
# to the legacy un-keyed name.
|
|
108
|
+
aiur_instance_key() {
|
|
109
|
+
local root
|
|
110
|
+
root="$(aiur_project_root)"
|
|
111
|
+
[ -n "$root" ] || return 0
|
|
112
|
+
# Canonicalize (resolve symlinks) so launch and control commands invoked via
|
|
113
|
+
# different logical paths to the same project agree on the key. Fall back to the
|
|
114
|
+
# literal path when the dir doesn't exist (e.g. a test-fixture root).
|
|
115
|
+
root="$(cd "$root" 2>/dev/null && pwd -P || printf '%s' "$root")"
|
|
116
|
+
printf '%s' "$root" | { shasum -a 256 2>/dev/null || sha256sum; } | cut -c1-10
|
|
117
|
+
}
|
|
34
118
|
|
|
35
119
|
aiur_resolve_identity() {
|
|
36
120
|
local config_home="${XDG_CONFIG_HOME:-$HOME/.config}"
|
|
121
|
+
AIUR_PROJECT_ROOT="$(aiur_project_root)"
|
|
122
|
+
AIUR_PROJECT_ROOT_SOURCE="$(aiur_project_root_source)"
|
|
37
123
|
|
|
38
124
|
: "${AIUR_BG_STATE_DIR:=$config_home/aiur}"
|
|
39
125
|
: "${AIUR_COOKIE_FILE:=$AIUR_BG_STATE_DIR/cookie}"
|
|
40
126
|
: "${AIUR_SESSION_PREFIX:=aiur}"
|
|
41
127
|
: "${AIUR_PROFILES_FILE:=$config_home/aiur/aiur.profiles}"
|
|
42
|
-
|
|
128
|
+
# Compute the per-instance key once. `${VAR+x}` so an explicit empty value (no
|
|
129
|
+
# project resolved) is honored and not recomputed on each call.
|
|
130
|
+
if [ -z "${AIUR_INSTANCE_KEY+x}" ]; then
|
|
131
|
+
AIUR_INSTANCE_KEY="$(aiur_instance_key)"
|
|
132
|
+
fi
|
|
133
|
+
: "${AIUR_RELEASE_NODE:=aiur-${USER}${AIUR_INSTANCE_KEY:+-$AIUR_INSTANCE_KEY}@127.0.0.1}"
|
|
43
134
|
|
|
44
135
|
export AIUR_BG_STATE_DIR AIUR_COOKIE_FILE AIUR_SESSION_PREFIX \
|
|
45
|
-
AIUR_PROFILES_FILE AIUR_RELEASE_NODE
|
|
136
|
+
AIUR_PROFILES_FILE AIUR_RELEASE_NODE AIUR_INSTANCE_KEY \
|
|
137
|
+
AIUR_PROJECT_ROOT AIUR_PROJECT_ROOT_SOURCE
|
|
46
138
|
}
|
|
47
139
|
|
|
48
140
|
aiur_print_identity() {
|
|
@@ -52,6 +144,9 @@ aiur_print_identity() {
|
|
|
52
144
|
printf 'AIUR_SESSION_PREFIX=%s\n' "$AIUR_SESSION_PREFIX"
|
|
53
145
|
printf 'AIUR_PROFILES_FILE=%s\n' "$AIUR_PROFILES_FILE"
|
|
54
146
|
printf 'AIUR_RELEASE_NODE=%s\n' "$AIUR_RELEASE_NODE"
|
|
147
|
+
printf 'AIUR_INSTANCE_KEY=%s\n' "$AIUR_INSTANCE_KEY"
|
|
148
|
+
printf 'AIUR_PROJECT_ROOT=%s\n' "$AIUR_PROJECT_ROOT"
|
|
149
|
+
printf 'AIUR_PROJECT_ROOT_SOURCE=%s\n' "$AIUR_PROJECT_ROOT_SOURCE"
|
|
55
150
|
printf 'AIUR_COOKIE_FILE=%s\n' "$AIUR_COOKIE_FILE"
|
|
56
151
|
}
|
|
57
152
|
|
|
@@ -172,12 +267,18 @@ build_init_cmd() {
|
|
|
172
267
|
|
|
173
268
|
usage() {
|
|
174
269
|
cat <<'EOF'
|
|
175
|
-
Usage: aiur [--interactive] [--logs-root <path>] [--port <port>] [--host <host>] [path-to-.aiurconfig]
|
|
270
|
+
Usage: aiur [--interactive] [--max-agents <n>] [--logs-root <path>] [--port <port>] [--host <host>] [path-to-.aiurconfig]
|
|
176
271
|
aiur init [--force] scaffold .aiurconfig (interactive setup wizard)
|
|
177
|
-
aiur --bg start in a detached tmux session
|
|
272
|
+
aiur --bg start in a lean, headless detached tmux session
|
|
178
273
|
aiur stop stop the running session
|
|
179
274
|
aiur status show agent status
|
|
275
|
+
aiur agents show each agent's state + current activity
|
|
276
|
+
aiur alerts [--needs-attention] show structured alert feed
|
|
277
|
+
aiur watch [--full|--changes] [--interval <secs>] server-side status board
|
|
278
|
+
aiur set max-agents <n> change the concurrent-agent cap at runtime
|
|
180
279
|
aiur pause <ids|--all> | resume <ids|--all>
|
|
280
|
+
aiur message <id> <text> send operator text to a running agent
|
|
281
|
+
aiur cleanup-stale [--dry-run] list/reap stale manual-smoke leftovers
|
|
181
282
|
aiur --version
|
|
182
283
|
EOF
|
|
183
284
|
}
|
|
@@ -265,19 +366,28 @@ run_session() {
|
|
|
265
366
|
|
|
266
367
|
init_argv_file
|
|
267
368
|
|
|
268
|
-
# Inject the flags a bare `aiur` needs: loopback bind,
|
|
269
|
-
# no-guardrails ack. Skip any the user already passed.
|
|
270
|
-
|
|
369
|
+
# Inject the flags a bare `aiur` needs: loopback bind, UI mode, and the
|
|
370
|
+
# no-guardrails ack. Skip any the user already passed. Foreground runs are
|
|
371
|
+
# interactive (tmux panes + dashboard); `--bg` runs lean/headless — no panes,
|
|
372
|
+
# no dashboard bind, no chat backfill — and is driven over the control RPC
|
|
373
|
+
# (status/agents/message/pause/set). `aiur --bg --interactive` opts back into
|
|
374
|
+
# the full interactive stack for an attachable background session.
|
|
375
|
+
local has_host=0 has_interactive=0 has_headless=0 has_ack=0 arg
|
|
271
376
|
for arg in "$@"; do
|
|
272
377
|
case "$arg" in
|
|
273
378
|
--host | --host=*) has_host=1 ;;
|
|
274
379
|
--interactive) has_interactive=1 ;;
|
|
380
|
+
--headless) has_headless=1 ;;
|
|
275
381
|
--i-understand-that-this-will-be-running-without-the-usual-guardrails) has_ack=1 ;;
|
|
276
382
|
esac
|
|
277
383
|
done
|
|
278
384
|
local injected=()
|
|
279
385
|
[ "$has_host" -eq 1 ] || injected+=(--host 127.0.0.1)
|
|
280
|
-
[ "$has_interactive" -eq
|
|
386
|
+
if [ "$mode" = "background" ] && [ "$has_interactive" -eq 0 ]; then
|
|
387
|
+
[ "$has_headless" -eq 1 ] || injected+=(--headless)
|
|
388
|
+
else
|
|
389
|
+
[ "$has_interactive" -eq 1 ] || injected+=(--interactive)
|
|
390
|
+
fi
|
|
281
391
|
[ "$has_ack" -eq 1 ] || injected+=(--i-understand-that-this-will-be-running-without-the-usual-guardrails)
|
|
282
392
|
|
|
283
393
|
write_argv "${injected[@]}" "$@"
|
|
@@ -291,12 +401,14 @@ run_session() {
|
|
|
291
401
|
export ELIXIR_ERL_OPTIONS="${ELIXIR_ERL_OPTIONS:-} +fnu"
|
|
292
402
|
fi
|
|
293
403
|
|
|
294
|
-
local session="${AIUR_SESSION_PREFIX}-${USER:-user}-default"
|
|
295
|
-
local socket="${AIUR_SESSION_PREFIX}-${USER:-user}"
|
|
404
|
+
local session="${AIUR_SESSION_PREFIX}-${USER:-user}${AIUR_INSTANCE_KEY:+-$AIUR_INSTANCE_KEY}-default"
|
|
405
|
+
local socket="${AIUR_SESSION_PREFIX}-${USER:-user}${AIUR_INSTANCE_KEY:+-$AIUR_INSTANCE_KEY}"
|
|
296
406
|
local conf
|
|
297
407
|
conf="$(resolve_tmux_conf)"
|
|
298
408
|
[ -f "$conf" ] || die "tmux conf not found at $conf"
|
|
299
409
|
|
|
410
|
+
preflight_stale_manual_smoke
|
|
411
|
+
|
|
300
412
|
mkdir -p "$AIUR_BG_STATE_DIR"
|
|
301
413
|
printf '%s\n' "$session" >"$AIUR_BG_STATE_DIR/state"
|
|
302
414
|
export AIUR_TMUX_SESSION="$session"
|
|
@@ -308,8 +420,37 @@ run_session() {
|
|
|
308
420
|
export AIUR_SESSION_TMPFILE="${session_root}/aiur-${$}-sessions"
|
|
309
421
|
: >"$AIUR_SESSION_TMPFILE"
|
|
310
422
|
|
|
423
|
+
# Agent pidfile: the BEAM appends one line per spawned agent (pane or headless
|
|
424
|
+
# os_pid) via Aiur.ProcessReaper. The BEAM-death watchdog and session_cleanup
|
|
425
|
+
# reap from it after the BEAM is gone — a crashed BEAM can kill nothing itself.
|
|
426
|
+
export AIUR_AGENT_TMPFILE="${session_root}/aiur-${$}-agents"
|
|
427
|
+
: >"$AIUR_AGENT_TMPFILE"
|
|
428
|
+
|
|
429
|
+
# Workspace-root handoff for the shell cwd-sweep backstop. The BEAM writes the
|
|
430
|
+
# resolved Aiur.Config.workspace_root() here after config loads.
|
|
431
|
+
export AIUR_WORKSPACE_ROOT_FILE="${session_root}/aiur-${$}-workspace-root"
|
|
432
|
+
: >"$AIUR_WORKSPACE_ROOT_FILE"
|
|
433
|
+
|
|
434
|
+
# Background runs persist under a known run log dir so (a) the BEAM-death
|
|
435
|
+
# watchdog, which outlives the BEAM, can drop a crash record next to aiur.log,
|
|
436
|
+
# and (b) the post-start boot capture is not lost to /tmp when the launcher
|
|
437
|
+
# removes its startup tempfile. Minting it here and exporting it makes the
|
|
438
|
+
# shell and the BEAM (Aiur.LogFile honors AIUR_LOGS_ROOT) agree on one dir.
|
|
439
|
+
if [ "$mode" = "background" ] && [ -z "${AIUR_LOGS_ROOT:-}" ]; then
|
|
440
|
+
AIUR_LOGS_ROOT="$(printf '%s/%s-%s' "$HOME/.aiur/logs" "$(date -u +%Y%m%dT%H%M%SZ)" "$$")"
|
|
441
|
+
export AIUR_LOGS_ROOT
|
|
442
|
+
fi
|
|
443
|
+
|
|
444
|
+
# Capture sink for BEAM startup (and, in background mode, the whole run's
|
|
445
|
+
# boot stdout/stderr). Foreground uses a throwaway tempfile; background points
|
|
446
|
+
# at a durable file in the run log dir. The dir/file is created lazily just
|
|
447
|
+
# before launch (below) so an idempotent early-return start leaves no empty dir.
|
|
311
448
|
local startup_capture
|
|
312
|
-
|
|
449
|
+
if [ "$mode" = "background" ] && [ -n "${AIUR_LOGS_ROOT:-}" ]; then
|
|
450
|
+
startup_capture="$AIUR_LOGS_ROOT/log/boot.out.log"
|
|
451
|
+
else
|
|
452
|
+
startup_capture="$(mktemp "${TMPDIR:-/tmp}/aiur-startup.XXXXXX")"
|
|
453
|
+
fi
|
|
313
454
|
|
|
314
455
|
# Inner pane script: tmux's server may pre-exist and not inherit our env, so
|
|
315
456
|
# re-export every var the BEAM needs. tee preserves a startup capture.
|
|
@@ -324,7 +465,8 @@ run_session() {
|
|
|
324
465
|
for v in AIUR_RELEASE_DIR AIUR_ARGV_FILE RELEASE_DISTRIBUTION RELEASE_NODE \
|
|
325
466
|
RELEASE_COOKIE ERL_AFLAGS ERL_EPMD_ADDRESS AIUR_NODE AIUR_ERLANG_COOKIE \
|
|
326
467
|
AIUR_TMUX_SESSION AIUR_TMUX_SOCKET AIUR_TMUX_CONF AIUR_BIN \
|
|
327
|
-
AIUR_SESSION_TMPFILE
|
|
468
|
+
AIUR_SESSION_TMPFILE AIUR_AGENT_TMPFILE AIUR_WORKSPACE_ROOT_FILE \
|
|
469
|
+
ELIXIR_ERL_OPTIONS AIUR_LOGS_ROOT AIUR_OPENCODE_BRIDGE_PORT AIUR_DEBUG; do
|
|
328
470
|
if [ -n "${!v:-}" ]; then printf 'export %s=%q\n' "$v" "${!v}"; fi
|
|
329
471
|
done
|
|
330
472
|
printf 'capture=%q\n' "$startup_capture"
|
|
@@ -341,48 +483,127 @@ run_session() {
|
|
|
341
483
|
_session_socket="$socket" _session_name="$session" _session_conf="$conf" \
|
|
342
484
|
_session_tmpfile="$AIUR_SESSION_TMPFILE" _session_capture="$startup_capture" \
|
|
343
485
|
_session_argv="$argv_file" _session_release="$release_dir" _session_tmux="$tmux_bin" \
|
|
344
|
-
_session_node="$AIUR_RELEASE_NODE"
|
|
486
|
+
_session_node="$AIUR_RELEASE_NODE" _session_pidfile="$AIUR_AGENT_TMPFILE" \
|
|
487
|
+
_session_workspace_root_file="$AIUR_WORKSPACE_ROOT_FILE"
|
|
345
488
|
install_foreground_traps
|
|
346
489
|
fi
|
|
347
490
|
|
|
348
|
-
#
|
|
349
|
-
#
|
|
350
|
-
#
|
|
351
|
-
|
|
491
|
+
# Background starts are intentionally idempotent. A prior live node should not
|
|
492
|
+
# fall through to tmux's opaque "duplicate session" failure, and a stale tmux
|
|
493
|
+
# session whose BEAM/control plane is gone should be reclaimed before retry.
|
|
494
|
+
if [ "$mode" = "background" ] && "$tmux_bin" -L "$socket" -f "$conf" has-session -t "$session" 2>/dev/null; then
|
|
495
|
+
if [ "$(probe_control_liveness)" = "up" ]; then
|
|
496
|
+
write_aiur_instance_record "$session" "$socket"
|
|
497
|
+
echo "aiur is already running in the background (tmux session ${session})." >&2
|
|
498
|
+
echo "Use: aiur status # inspect agents" >&2
|
|
499
|
+
echo "Use: aiur stop # stop it before starting a fresh session" >&2
|
|
500
|
+
rm -f "$startup_capture" "$argv_file" "$launcher" "$AIUR_SESSION_TMPFILE" "$AIUR_AGENT_TMPFILE" "$AIUR_WORKSPACE_ROOT_FILE" 2>/dev/null || true
|
|
501
|
+
return 0
|
|
502
|
+
fi
|
|
503
|
+
|
|
504
|
+
echo "aiur found stale tmux session ${session}; cleaning it up before restart" >&2
|
|
505
|
+
reap_aiur_agents "$socket" "$AIUR_AGENT_TMPFILE"
|
|
506
|
+
kill_beams_matching "-name ${AIUR_RELEASE_NODE}"
|
|
507
|
+
fi
|
|
508
|
+
|
|
509
|
+
# An orphaned BEAM (its tmux session gone) can still hold THIS instance's node
|
|
510
|
+
# name. With no live session on our (instance-keyed) socket, reap the name-holder
|
|
511
|
+
# so the launch isn't blocked by "name seems to be in use". The keyed name means
|
|
512
|
+
# this only ever reaps our own instance's orphan, never another live aiur.
|
|
352
513
|
if ! "$tmux_bin" -L "$socket" -f "$conf" has-session -t "$session" 2>/dev/null; then
|
|
353
514
|
kill_beams_matching "-name ${AIUR_RELEASE_NODE}"
|
|
354
515
|
fi
|
|
355
516
|
|
|
517
|
+
# Transition reclaim: a keyed instance also reaps a stale beam under the legacy
|
|
518
|
+
# un-keyed name (aiur-$USER@127.0.0.1) — but only when no live legacy session
|
|
519
|
+
# exists, so a pre-fix run still in progress is never killed. (Runs on every keyed
|
|
520
|
+
# launch, not latched; it's a cheap best-effort sweep guarded by has-session.)
|
|
521
|
+
if [ -n "${AIUR_INSTANCE_KEY:-}" ]; then
|
|
522
|
+
local legacy_socket="${AIUR_SESSION_PREFIX}-${USER:-user}"
|
|
523
|
+
if ! "$tmux_bin" -L "$legacy_socket" -f "$conf" has-session -t "${legacy_socket}-default" 2>/dev/null; then
|
|
524
|
+
kill_beams_matching "-name aiur-${USER}@127.0.0.1"
|
|
525
|
+
fi
|
|
526
|
+
fi
|
|
527
|
+
|
|
528
|
+
# Create the capture sink now (after the idempotent early-return checks above,
|
|
529
|
+
# so a no-op "already running" start never mints an empty run log dir). The
|
|
530
|
+
# launcher's `tee -a` needs the file's parent to exist when the pane runs.
|
|
531
|
+
mkdir -p "$(dirname "$startup_capture")" 2>/dev/null || true
|
|
532
|
+
: >"$startup_capture" 2>/dev/null || true
|
|
533
|
+
|
|
356
534
|
if ! "$tmux_bin" -L "$socket" -f "$conf" new-session -d -s "$session" \
|
|
357
535
|
-x "${COLUMNS:-200}" -y "${LINES:-50}" "$inner_cmd"; then
|
|
358
536
|
echo "❌ aiur failed to start; captured output:" >&2
|
|
359
537
|
tail -n 30 "$startup_capture" 2>/dev/null | sed 's/^/ /' >&2 || true
|
|
360
|
-
rm -f "$startup_capture" "$argv_file" 2>/dev/null || true
|
|
538
|
+
rm -f "$startup_capture" "$argv_file" "$launcher" "$AIUR_SESSION_TMPFILE" "$AIUR_AGENT_TMPFILE" "$AIUR_WORKSPACE_ROOT_FILE" 2>/dev/null || true
|
|
361
539
|
exit 1
|
|
362
540
|
fi
|
|
363
541
|
|
|
364
|
-
#
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
542
|
+
# Title the agent-list pane (the only pane in the fresh session). The conf's
|
|
543
|
+
# `pane-border-status`/`pane-border-format` render it; PaneManager titles the
|
|
544
|
+
# chat panes it opens. Best-effort — a missing title just shows the default.
|
|
545
|
+
"$tmux_bin" -L "$socket" -f "$conf" select-pane -t "$session" -T "AIUR Agents" 2>/dev/null || true
|
|
546
|
+
|
|
547
|
+
# Grace window: surface a boot crash instead of attaching to nothing. Prove
|
|
548
|
+
# the control plane can answer before declaring startup usable: background
|
|
549
|
+
# control commands depend on that RPC path, and foreground automation must not
|
|
550
|
+
# treat a pre-TUI application crash as a successful `tmux attach`.
|
|
551
|
+
local require_control=1
|
|
552
|
+
if ! wait_for_session_startup "$tmux_bin" "$socket" "$conf" "$session" "$startup_capture" "$require_control"; then
|
|
553
|
+
if [ "$mode" = "background" ]; then
|
|
554
|
+
"$tmux_bin" -L "$socket" -f "$conf" kill-session -t "$session" 2>/dev/null || true
|
|
555
|
+
reap_aiur_agents "$socket" "$AIUR_AGENT_TMPFILE"
|
|
556
|
+
kill_beams_matching "-name ${AIUR_RELEASE_NODE}"
|
|
373
557
|
fi
|
|
374
|
-
|
|
375
|
-
|
|
558
|
+
[ "$mode" = "foreground" ] && return 1
|
|
559
|
+
rm -f "$startup_capture" "$argv_file" "$launcher" "$AIUR_SESSION_TMPFILE" "$AIUR_AGENT_TMPFILE" "$AIUR_WORKSPACE_ROOT_FILE" 2>/dev/null || true
|
|
560
|
+
exit 1
|
|
561
|
+
fi
|
|
562
|
+
|
|
563
|
+
write_aiur_instance_record "$session" "$socket"
|
|
564
|
+
|
|
565
|
+
if [ "$mode" = "foreground" ]; then
|
|
566
|
+
echo "aiur foreground tmux socket ${socket}, session ${session}" >&2
|
|
567
|
+
fi
|
|
376
568
|
|
|
377
569
|
if [ "$mode" = "background" ]; then
|
|
378
|
-
|
|
379
|
-
|
|
570
|
+
# Fresh run: drop any stale crash/stop state from a prior dead instance so
|
|
571
|
+
# `status` doesn't report a phantom orphan and the watchdog starts clean.
|
|
572
|
+
rm -f "$(aiur_stop_sentinel_path)" "$(aiur_crash_marker_path)" 2>/dev/null || true
|
|
573
|
+
|
|
574
|
+
local background_watchdog_pid
|
|
575
|
+
background_watchdog_pid="$(start_beam_death_watchdog \
|
|
576
|
+
"-name ${AIUR_RELEASE_NODE}" "$socket" "$AIUR_AGENT_TMPFILE" 1 1 \
|
|
577
|
+
"$AIUR_RELEASE_NODE" "${AIUR_LOGS_ROOT:-}" \
|
|
578
|
+
"$(aiur_stop_sentinel_path)" "$(aiur_crash_marker_path)" "$AIUR_WORKSPACE_ROOT_FILE")"
|
|
579
|
+
disown "$background_watchdog_pid" 2>/dev/null || true
|
|
580
|
+
echo "aiur started in the background (tmux socket ${socket}, session ${session}). Attach with: aiur" >&2
|
|
581
|
+
# Keep $startup_capture (boot.out.log) for the run's lifetime; only the
|
|
582
|
+
# transient argv file is no longer needed.
|
|
583
|
+
rm -f "$argv_file" 2>/dev/null || true
|
|
380
584
|
return 0
|
|
381
585
|
fi
|
|
382
586
|
|
|
587
|
+
# Arm the BEAM-death watchdog before attaching. If the BEAM crashes
|
|
588
|
+
# (:emfile) mid-run, agent windows keep the orphaned session alive so the
|
|
589
|
+
# `tmux attach` below never returns and the EXIT trap never fires — the
|
|
590
|
+
# watchdog is the external reaper that survives the dead BEAM, kill-servers
|
|
591
|
+
# the session, and unblocks the attach. It polls the node name rather than a
|
|
592
|
+
# captured pid, so another Aiur instance from the same release cannot hold it open.
|
|
593
|
+
_session_watchdog_pid="$(start_beam_death_watchdog \
|
|
594
|
+
"-name ${AIUR_RELEASE_NODE}" "$socket" "$AIUR_AGENT_TMPFILE" 1 0 \
|
|
595
|
+
"" "" "" "" "$AIUR_WORKSPACE_ROOT_FILE")"
|
|
596
|
+
|
|
383
597
|
# Foreground: attach the UI. Do not exec — that would drop the teardown trap.
|
|
384
|
-
|
|
385
|
-
|
|
598
|
+
# Avoid process substitution here: some sandboxed non-TTY launchers reject
|
|
599
|
+
# opening /dev/fd/* during the real manual-test wrapper path.
|
|
600
|
+
local attach_stderr attach_code
|
|
601
|
+
attach_stderr="$(mktemp "${TMPDIR:-/tmp}/aiur-attach-stderr.XXXXXX")"
|
|
602
|
+
"$tmux_bin" -L "$socket" -f "$conf" attach -t "$session" 2>"$attach_stderr"
|
|
603
|
+
attach_code=$?
|
|
604
|
+
grep -v -F "[server exited]" "$attach_stderr" >&2 || true
|
|
605
|
+
rm -f "$attach_stderr" 2>/dev/null || true
|
|
606
|
+
return "$attach_code"
|
|
386
607
|
}
|
|
387
608
|
|
|
388
609
|
resolve_tmux_conf() {
|
|
@@ -417,37 +638,655 @@ kill_beams_matching() {
|
|
|
417
638
|
for pid in $pids; do kill -KILL "$pid" 2>/dev/null || true; done
|
|
418
639
|
}
|
|
419
640
|
|
|
641
|
+
pid_owner() {
|
|
642
|
+
ps -p "$1" -o user= 2>/dev/null | awk '{print $1}' || true
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
pid_command() {
|
|
646
|
+
ps -p "$1" -o command= 2>/dev/null || true
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
pid_ppid() {
|
|
650
|
+
ps -p "$1" -o ppid= 2>/dev/null | awk '{print $1}' || true
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
pid_cwd() {
|
|
654
|
+
local pid="$1" cwd
|
|
655
|
+
if [ -e "/proc/$pid/cwd" ]; then
|
|
656
|
+
readlink "/proc/$pid/cwd" 2>/dev/null || true
|
|
657
|
+
return 0
|
|
658
|
+
fi
|
|
659
|
+
if command -v lsof >/dev/null 2>&1; then
|
|
660
|
+
cwd="$(lsof -a -p "$pid" -d cwd -Fn 2>/dev/null | sed -n 's/^n//p' | head -n 1 || true)"
|
|
661
|
+
[ -n "$cwd" ] && printf '%s\n' "$cwd"
|
|
662
|
+
fi
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
kill_pid_with_escalation() {
|
|
666
|
+
local pid="$1" waited=0
|
|
667
|
+
[ -n "$pid" ] || return 0
|
|
668
|
+
kill -TERM "$pid" 2>/dev/null || true
|
|
669
|
+
while [ "$waited" -lt 20 ]; do
|
|
670
|
+
kill -0 "$pid" 2>/dev/null || return 0
|
|
671
|
+
sleep 0.1
|
|
672
|
+
waited=$((waited + 1))
|
|
673
|
+
done
|
|
674
|
+
kill -KILL "$pid" 2>/dev/null || true
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
aiur_node_from_command() {
|
|
678
|
+
local cmd="$1" node
|
|
679
|
+
node="$(printf '%s\n' "$cmd" | sed -n "s/.*--name[[:space:]]\\(aiur-${USER}[-A-Za-z0-9_]*@127\\.0\\.0\\.1\\).*/\\1/p" | head -n 1)"
|
|
680
|
+
[ -n "$node" ] && printf '%s\n' "$node"
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
aiur_release_root_from_command() {
|
|
684
|
+
local cmd="$1"
|
|
685
|
+
if [[ "$cmd" =~ --boot-var[[:space:]]+RELEASE_LIB[[:space:]]+([^[:space:]]+)/lib ]]; then
|
|
686
|
+
printf '%s\n' "${BASH_REMATCH[1]}"
|
|
687
|
+
return 0
|
|
688
|
+
fi
|
|
689
|
+
if [[ "$cmd" =~ ([^[:space:]]*/src/_build/dev/rel/aiur)/releases/ ]]; then
|
|
690
|
+
printf '%s\n' "${BASH_REMATCH[1]}"
|
|
691
|
+
return 0
|
|
692
|
+
fi
|
|
693
|
+
if [[ "$cmd" =~ ([^[:space:]]*/release)/releases/ ]]; then
|
|
694
|
+
printf '%s\n' "${BASH_REMATCH[1]}"
|
|
695
|
+
return 0
|
|
696
|
+
fi
|
|
697
|
+
return 1
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
workspace_root_from_aiur_release_root() {
|
|
701
|
+
local release_root="${1%/}"
|
|
702
|
+
case "$release_root" in
|
|
703
|
+
*/src/_build/dev/rel/aiur)
|
|
704
|
+
printf '%s\n' "${release_root%/src/_build/dev/rel/aiur}"
|
|
705
|
+
;;
|
|
706
|
+
*)
|
|
707
|
+
printf '%s\n' "$release_root"
|
|
708
|
+
;;
|
|
709
|
+
esac
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
aiur_node_tmux_session_alive() {
|
|
713
|
+
local node="$1" short tmux_bin
|
|
714
|
+
short="${node%@*}"
|
|
715
|
+
tmux_bin="$(command -v tmux || true)"
|
|
716
|
+
[ -n "$tmux_bin" ] || return 1
|
|
717
|
+
"$tmux_bin" -L "$short" has-session -t "${short}-default" 2>/dev/null
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
stale_manual_smoke_beam_inventory() {
|
|
721
|
+
local pid cmd node release_root workspace_root owner
|
|
722
|
+
aiur_resolve_identity
|
|
723
|
+
|
|
724
|
+
for pid in $(pgrep -f -- "beam\\.smp.*--name aiur-${USER}.*@127\\.0\\.0\\.1" 2>/dev/null || true); do
|
|
725
|
+
[ -n "$pid" ] || continue
|
|
726
|
+
owner="$(pid_owner "$pid")"
|
|
727
|
+
[ "$owner" = "$USER" ] || continue
|
|
728
|
+
|
|
729
|
+
cmd="$(pid_command "$pid")"
|
|
730
|
+
node="$(aiur_node_from_command "$cmd")"
|
|
731
|
+
[ -n "$node" ] || continue
|
|
732
|
+
|
|
733
|
+
release_root="$(aiur_release_root_from_command "$cmd" || true)"
|
|
734
|
+
[ -n "$release_root" ] || continue
|
|
735
|
+
workspace_root="$(workspace_root_from_aiur_release_root "$release_root")"
|
|
736
|
+
|
|
737
|
+
# Scope this broad cleanup to stale issue/manual-smoke workspaces. The
|
|
738
|
+
# current instance cleanup path handles the active daemon's own node.
|
|
739
|
+
case "$workspace_root" in
|
|
740
|
+
*/aiur-workspaces/*) ;;
|
|
741
|
+
*) continue ;;
|
|
742
|
+
esac
|
|
743
|
+
|
|
744
|
+
# A matching live aiur tmux session means this node may still be in use.
|
|
745
|
+
aiur_node_tmux_session_alive "$node" && continue
|
|
746
|
+
|
|
747
|
+
printf '%s\t%s\t%s\t%s\n' "$pid" "$node" "$workspace_root" "$release_root"
|
|
748
|
+
done
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
manual_smoke_wrapper_tmux_sockets() {
|
|
752
|
+
local tmux_bin sockdir path name panes current pane_path start manual_context all_sleep
|
|
753
|
+
tmux_bin="$(command -v tmux || true)"
|
|
754
|
+
[ -n "$tmux_bin" ] || return 0
|
|
755
|
+
sockdir="${TMUX_TMPDIR:-/tmp}/tmux-$(id -u)"
|
|
756
|
+
[ -d "$sockdir" ] || return 0
|
|
757
|
+
|
|
758
|
+
for path in "$sockdir"/*; do
|
|
759
|
+
[ -S "$path" ] || continue
|
|
760
|
+
name="${path##*/}"
|
|
761
|
+
case "$name" in
|
|
762
|
+
codex-driver-* | codex-aiur* | codex-[0-9]* | claude-driver*) ;;
|
|
763
|
+
*) continue ;;
|
|
764
|
+
esac
|
|
765
|
+
panes="$("$tmux_bin" -L "$name" list-panes -a -F '#{pane_current_command} #{pane_current_path} #{pane_start_command}' 2>/dev/null || true)"
|
|
766
|
+
[ -n "$panes" ] || continue
|
|
767
|
+
manual_context=0
|
|
768
|
+
all_sleep=1
|
|
769
|
+
while IFS=$'\t' read -r current pane_path start; do
|
|
770
|
+
case "$start" in
|
|
771
|
+
*aiurdev\ --test* | *scripts/aiurdev\ --test*) manual_context=1 ;;
|
|
772
|
+
esac
|
|
773
|
+
[ "$current" = "sleep" ] || all_sleep=0
|
|
774
|
+
done <<<"$panes"
|
|
775
|
+
[ "$manual_context" -eq 1 ] && [ "$all_sleep" -eq 1 ] && printf '%s\n' "$name"
|
|
776
|
+
done
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
orphaned_opencode_attach_inventory() {
|
|
780
|
+
local pid owner ppid cmd cwd
|
|
781
|
+
for pid in $(pgrep -f -- "opencode attach http://127\\.0\\.0\\.1:" 2>/dev/null || true); do
|
|
782
|
+
owner="$(pid_owner "$pid")"
|
|
783
|
+
[ "$owner" = "$USER" ] || continue
|
|
784
|
+
ppid="$(pid_ppid "$pid")"
|
|
785
|
+
[ "$ppid" = "1" ] || continue
|
|
786
|
+
cwd="$(pid_cwd "$pid")"
|
|
787
|
+
case "$cwd" in
|
|
788
|
+
*/aiur-workspaces/*) ;;
|
|
789
|
+
*) continue ;;
|
|
790
|
+
esac
|
|
791
|
+
cmd="$(pid_command "$pid")"
|
|
792
|
+
case "$cmd" in
|
|
793
|
+
*"opencode attach http://127.0.0.1:"*) printf '%s\t%s\t%s\n' "$pid" "$cwd" "$cmd" ;;
|
|
794
|
+
esac
|
|
795
|
+
done
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
report_stale_manual_smoke() {
|
|
799
|
+
local found=0 pid node workspace_root release_root socket cwd cmd
|
|
800
|
+
|
|
801
|
+
while IFS=$'\t' read -r pid node workspace_root release_root; do
|
|
802
|
+
[ -n "$pid" ] || continue
|
|
803
|
+
if [ "$found" -eq 0 ]; then
|
|
804
|
+
echo "aiur: stale manual-smoke leftovers detected:" >&2
|
|
805
|
+
found=1
|
|
806
|
+
fi
|
|
807
|
+
echo " BEAM pid=${pid} node=${node} workspace=${workspace_root} release=${release_root}" >&2
|
|
808
|
+
done < <(stale_manual_smoke_beam_inventory)
|
|
809
|
+
|
|
810
|
+
while IFS= read -r socket; do
|
|
811
|
+
[ -n "$socket" ] || continue
|
|
812
|
+
if [ "$found" -eq 0 ]; then
|
|
813
|
+
echo "aiur: stale manual-smoke leftovers detected:" >&2
|
|
814
|
+
found=1
|
|
815
|
+
fi
|
|
816
|
+
echo " wrapper tmux socket=${socket}" >&2
|
|
817
|
+
done < <(manual_smoke_wrapper_tmux_sockets)
|
|
818
|
+
|
|
819
|
+
while IFS=$'\t' read -r pid cwd cmd; do
|
|
820
|
+
[ -n "$pid" ] || continue
|
|
821
|
+
if [ "$found" -eq 0 ]; then
|
|
822
|
+
echo "aiur: stale manual-smoke leftovers detected:" >&2
|
|
823
|
+
found=1
|
|
824
|
+
fi
|
|
825
|
+
echo " orphan opencode attach pid=${pid} workspace=${cwd}" >&2
|
|
826
|
+
done < <(orphaned_opencode_attach_inventory)
|
|
827
|
+
|
|
828
|
+
if [ "$found" -eq 1 ]; then
|
|
829
|
+
echo "aiur: run 'aiur cleanup-stale' to TERM/KILL only these same-user Aiur leftovers." >&2
|
|
830
|
+
fi
|
|
831
|
+
|
|
832
|
+
return "$found"
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
preflight_stale_manual_smoke() {
|
|
836
|
+
report_stale_manual_smoke >/dev/null || true
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
reap_stale_manual_smoke() {
|
|
840
|
+
local include_wrappers="${1:-0}"
|
|
841
|
+
local pid node workspace_root release_root socket cwd cmd tmux_bin reaped=0
|
|
842
|
+
|
|
843
|
+
while IFS=$'\t' read -r pid node workspace_root release_root; do
|
|
844
|
+
[ -n "$pid" ] || continue
|
|
845
|
+
echo "aiur: reaping stale BEAM pid=${pid} node=${node} workspace=${workspace_root}" >&2
|
|
846
|
+
kill_pid_with_escalation "$pid"
|
|
847
|
+
reaped=$((reaped + 1))
|
|
848
|
+
done < <(stale_manual_smoke_beam_inventory)
|
|
849
|
+
|
|
850
|
+
tmux_bin="$(command -v tmux || true)"
|
|
851
|
+
if [ "$include_wrappers" = "1" ] && [ -n "$tmux_bin" ]; then
|
|
852
|
+
while IFS= read -r socket; do
|
|
853
|
+
[ -n "$socket" ] || continue
|
|
854
|
+
echo "aiur: reaping stale wrapper tmux socket=${socket}" >&2
|
|
855
|
+
"$tmux_bin" -L "$socket" kill-server 2>/dev/null || true
|
|
856
|
+
reaped=$((reaped + 1))
|
|
857
|
+
done < <(manual_smoke_wrapper_tmux_sockets)
|
|
858
|
+
fi
|
|
859
|
+
|
|
860
|
+
while IFS=$'\t' read -r pid cwd cmd; do
|
|
861
|
+
[ -n "$pid" ] || continue
|
|
862
|
+
echo "aiur: reaping orphan opencode attach pid=${pid}" >&2
|
|
863
|
+
kill_pid_with_escalation "$pid"
|
|
864
|
+
reaped=$((reaped + 1))
|
|
865
|
+
done < <(orphaned_opencode_attach_inventory)
|
|
866
|
+
|
|
867
|
+
if [ "$reaped" -eq 0 ]; then
|
|
868
|
+
echo "aiur: no stale manual-smoke leftovers found" >&2
|
|
869
|
+
fi
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
# Echo $1 and every descendant pid, depth-first. Mac-safe (`pgrep -P`, no
|
|
873
|
+
# /proc), mirroring Aiur.RemoteControl.collect_descendants so the launcher reaps
|
|
874
|
+
# the same tree the BEAM-side reaper would. The recorded agent pid is a bash
|
|
875
|
+
# `-lc` wrapper; the model process (claude --print / codex) is its child and
|
|
876
|
+
# reparents to init if only the wrapper is signalled — orphan-and-survive is
|
|
877
|
+
# exactly the bug — so the whole tree must be collected before any kill lands.
|
|
878
|
+
agent_pid_tree() {
|
|
879
|
+
local root="$1" child
|
|
880
|
+
printf '%s\n' "$root"
|
|
881
|
+
for child in $(pgrep -P "$root" 2>/dev/null || true); do
|
|
882
|
+
agent_pid_tree "$child"
|
|
883
|
+
done
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
# pid-reuse guard: succeeds when pid $1 is alive AND its command still contains
|
|
887
|
+
# the comm substring $2 the BEAM recorded. An empty comm kills unconditionally.
|
|
888
|
+
# Mirrors the BEAM-side cmdline guard (Aiur.ProcessReaper) but Mac-safe via
|
|
889
|
+
# `ps -o command=`, so a recycled pid whose command no longer matches is spared.
|
|
890
|
+
agent_pid_matches() {
|
|
891
|
+
local pid="$1" comm="$2" cmd
|
|
892
|
+
kill -0 "$pid" 2>/dev/null || return 1
|
|
893
|
+
[ -n "$comm" ] || return 0
|
|
894
|
+
cmd="$(ps -p "$pid" -o command= 2>/dev/null || true)"
|
|
895
|
+
case "$cmd" in *"$comm"*) return 0 ;; *) return 1 ;; esac
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
# Reap every aiur agent the BEAM can no longer reap itself.
|
|
899
|
+
#
|
|
900
|
+
# $1 socket aiur tmux socket (-L); kill-server nukes all panes (may be empty)
|
|
901
|
+
# $2 pidfile AIUR_AGENT_TMPFILE (BEAM-written agent refs; may be empty/missing)
|
|
902
|
+
#
|
|
903
|
+
# tmux runs on aiur's PRIVATE socket (-L aiur-$USER), so kill-server tears down
|
|
904
|
+
# every REPL/chat pane agent in one shot AND leaves no live aiur tmux server —
|
|
905
|
+
# never touching the operator's own default tmux. Headless agents (claude/codex
|
|
906
|
+
# app-servers spawned via Port) are bare OS processes that reparent to init on a
|
|
907
|
+
# BEAM crash; kill-server can't see them, so they're reaped from the pidfile by
|
|
908
|
+
# process tree, comm-guarded against pid reuse. Idempotent.
|
|
909
|
+
reap_aiur_agents() {
|
|
910
|
+
local socket="$1" pidfile="${2:-}"
|
|
911
|
+
local tmux_bin
|
|
912
|
+
tmux_bin="$(command -v tmux || true)"
|
|
913
|
+
|
|
914
|
+
if [ -n "$tmux_bin" ] && [ -n "$socket" ]; then
|
|
915
|
+
"$tmux_bin" -L "$socket" kill-server 2>/dev/null || true
|
|
916
|
+
fi
|
|
917
|
+
|
|
918
|
+
[ -n "$pidfile" ] && [ -r "$pidfile" ] || return 0
|
|
919
|
+
|
|
920
|
+
# Snapshot the full process tree of every still-matching headless agent before
|
|
921
|
+
# signalling, so descendants that reparent mid-reap are already on the list.
|
|
922
|
+
local kind pid comm tree=() p
|
|
923
|
+
while read -r kind pid comm; do
|
|
924
|
+
[ "$kind" = "pid" ] && [ -n "$pid" ] || continue
|
|
925
|
+
agent_pid_matches "$pid" "$comm" || continue
|
|
926
|
+
while IFS= read -r p; do tree+=("$p"); done < <(agent_pid_tree "$pid")
|
|
927
|
+
done <"$pidfile"
|
|
928
|
+
|
|
929
|
+
[ "${#tree[@]}" -gt 0 ] || return 0
|
|
930
|
+
|
|
931
|
+
for p in "${tree[@]}"; do kill -TERM "$p" 2>/dev/null || true; done
|
|
932
|
+
local waited=0
|
|
933
|
+
while [ "$waited" -lt 20 ]; do
|
|
934
|
+
local any=0
|
|
935
|
+
for p in "${tree[@]}"; do kill -0 "$p" 2>/dev/null && { any=1; break; }; done
|
|
936
|
+
[ "$any" -eq 0 ] && break
|
|
937
|
+
sleep 0.1
|
|
938
|
+
waited=$((waited + 1))
|
|
939
|
+
done
|
|
940
|
+
for p in "${tree[@]}"; do kill -KILL "$p" 2>/dev/null || true; done
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
# Stable, per-instance state paths the background BEAM-death machinery shares
|
|
944
|
+
# across `run`, `stop`, and `status`. Keyed by the (instance-keyed) release node
|
|
945
|
+
# rather than the tmux socket so `status` — which resolves the node but not the
|
|
946
|
+
# socket — agrees on the same files. AIUR_BG_STATE_DIR and AIUR_RELEASE_NODE are
|
|
947
|
+
# resolved by aiur_resolve_identity, which every one of those paths calls first.
|
|
948
|
+
aiur_state_slug() {
|
|
949
|
+
printf '%s' "${AIUR_RELEASE_NODE:-aiur}" | tr -c 'A-Za-z0-9._-' '_'
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
aiur_instances_dir() {
|
|
953
|
+
printf '%s/instances' "${AIUR_BG_STATE_DIR:?}"
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
aiur_instance_record_path() {
|
|
957
|
+
printf '%s/%s.instance' "$(aiur_instances_dir)" "$(aiur_state_slug)"
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
write_aiur_instance_record() {
|
|
961
|
+
local session="$1" socket="$2" record_dir record tmp root
|
|
962
|
+
record_dir="$(aiur_instances_dir)"
|
|
963
|
+
record="$(aiur_instance_record_path)"
|
|
964
|
+
root="$(canonical_workspace_root "${AIUR_PROJECT_ROOT:-}")"
|
|
965
|
+
mkdir -p "$record_dir" 2>/dev/null || return 0
|
|
966
|
+
tmp="$(mktemp "$record_dir/.${AIUR_INSTANCE_KEY:-aiur}.XXXXXX" 2>/dev/null)" || return 0
|
|
967
|
+
{
|
|
968
|
+
printf 'AIUR_RECORD_NODE=%q\n' "$AIUR_RELEASE_NODE"
|
|
969
|
+
printf 'AIUR_RECORD_INSTANCE_KEY=%q\n' "$AIUR_INSTANCE_KEY"
|
|
970
|
+
printf 'AIUR_RECORD_SESSION=%q\n' "$session"
|
|
971
|
+
printf 'AIUR_RECORD_SOCKET=%q\n' "$socket"
|
|
972
|
+
printf 'AIUR_RECORD_PROJECT_ROOT=%q\n' "$root"
|
|
973
|
+
printf 'AIUR_RECORD_PROJECT_ROOT_SOURCE=%q\n' "${AIUR_PROJECT_ROOT_SOURCE:-}"
|
|
974
|
+
printf 'AIUR_RECORD_WRITTEN_AT=%q\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || true)"
|
|
975
|
+
} >"$tmp" || { rm -f "$tmp" 2>/dev/null || true; return 0; }
|
|
976
|
+
chmod 0600 "$tmp" 2>/dev/null || true
|
|
977
|
+
mv "$tmp" "$record" 2>/dev/null || rm -f "$tmp" 2>/dev/null || true
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
# Marker `status` reads to tell "daemon crashed, agents may be orphaned" apart
|
|
981
|
+
# from "nothing was ever running". Written by the watchdog on an unexpected exit,
|
|
982
|
+
# cleared on a clean stop and on the next successful background start.
|
|
983
|
+
aiur_crash_marker_path() {
|
|
984
|
+
printf '%s/%s.last-crash' "${AIUR_BG_STATE_DIR:?}" "$(aiur_state_slug)"
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
# Sentinel `stop` drops before killing the BEAM so the watchdog knows the exit
|
|
988
|
+
# was intentional and does not record a false crash.
|
|
989
|
+
aiur_stop_sentinel_path() {
|
|
990
|
+
printf '%s/%s.stopping' "${AIUR_BG_STATE_DIR:?}" "$(aiur_state_slug)"
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
# Write a durable record that the background BEAM exited unexpectedly. Two sinks:
|
|
994
|
+
# the run log dir (full record next to aiur.log, for forensics) and the stable
|
|
995
|
+
# per-instance marker (for `status` to surface). Best-effort throughout — this
|
|
996
|
+
# runs in the disowned watchdog after the BEAM is already gone, so it must never
|
|
997
|
+
# fail loudly or block the reap that follows.
|
|
998
|
+
record_beam_crash() {
|
|
999
|
+
local node="$1" run_log_dir="$2" marker="$3"
|
|
1000
|
+
local ts boot_tail=""
|
|
1001
|
+
ts="$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || true)"
|
|
1002
|
+
if [ -n "$run_log_dir" ] && [ -r "$run_log_dir/log/boot.out.log" ]; then
|
|
1003
|
+
boot_tail="$(tail -n 20 "$run_log_dir/log/boot.out.log" 2>/dev/null || true)"
|
|
1004
|
+
fi
|
|
1005
|
+
|
|
1006
|
+
local body
|
|
1007
|
+
body="$(
|
|
1008
|
+
printf 'aiur background BEAM exited unexpectedly\n'
|
|
1009
|
+
printf 'timestamp: %s\n' "${ts:-unknown}"
|
|
1010
|
+
printf 'node: %s\n' "${node:-unknown}"
|
|
1011
|
+
printf 'run_log_dir: %s\n' "${run_log_dir:-unknown}"
|
|
1012
|
+
printf 'detected_by: background BEAM-death watchdog (no clean stop sentinel)\n'
|
|
1013
|
+
if [ -n "$boot_tail" ]; then
|
|
1014
|
+
printf -- '--- last 20 lines of boot.out.log ---\n%s\n' "$boot_tail"
|
|
1015
|
+
fi
|
|
1016
|
+
)"
|
|
1017
|
+
|
|
1018
|
+
if [ -n "$run_log_dir" ]; then
|
|
1019
|
+
mkdir -p "$run_log_dir/log" 2>/dev/null || true
|
|
1020
|
+
printf '%s\n' "$body" >>"$run_log_dir/log/aiur.crash" 2>/dev/null || true
|
|
1021
|
+
fi
|
|
1022
|
+
if [ -n "$marker" ]; then
|
|
1023
|
+
printf '%s\n' "$body" >"$marker" 2>/dev/null || true
|
|
1024
|
+
fi
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
canonical_workspace_root() {
|
|
1028
|
+
local root="$1"
|
|
1029
|
+
if [ -d "$root" ]; then
|
|
1030
|
+
(cd "$root" 2>/dev/null && pwd -P) || printf '%s\n' "$root"
|
|
1031
|
+
else
|
|
1032
|
+
printf '%s\n' "$root"
|
|
1033
|
+
fi
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
workspace_root_is_shallow() {
|
|
1037
|
+
local root="${1%/}" trimmed
|
|
1038
|
+
trimmed="${root#/}"
|
|
1039
|
+
[ -n "$trimmed" ] || return 0
|
|
1040
|
+
case "$trimmed" in
|
|
1041
|
+
*/*) return 1 ;;
|
|
1042
|
+
*) return 0 ;;
|
|
1043
|
+
esac
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
workspace_cwd_pids() {
|
|
1047
|
+
local root="${1%/}" proc_dir="${2:-/proc}" entry pid cwd
|
|
1048
|
+
[ -d "$proc_dir" ] || return 0
|
|
1049
|
+
|
|
1050
|
+
for entry in "$proc_dir"/[0-9]*; do
|
|
1051
|
+
[ -d "$entry" ] || continue
|
|
1052
|
+
pid="${entry##*/}"
|
|
1053
|
+
cwd="$(readlink "$entry/cwd" 2>/dev/null || true)"
|
|
1054
|
+
[ -n "$cwd" ] || continue
|
|
1055
|
+
cwd="${cwd%/}"
|
|
1056
|
+
if [ "$cwd" != "$root" ] && [[ "$cwd/" == "$root/"* ]]; then
|
|
1057
|
+
printf '%s\n' "$pid"
|
|
1058
|
+
fi
|
|
1059
|
+
done
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
shell_protected_pid_list() {
|
|
1063
|
+
local p
|
|
1064
|
+
printf '%s\n' "$$"
|
|
1065
|
+
[ "${BASHPID:-$$}" = "$$" ] || printf '%s\n' "$BASHPID"
|
|
1066
|
+
[ -n "${PPID:-}" ] && printf '%s\n' "$PPID"
|
|
1067
|
+
for p in $(agent_pid_tree "$$" 2>/dev/null || true); do
|
|
1068
|
+
printf '%s\n' "$p"
|
|
1069
|
+
done
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
reap_workspace_cwd_agents() {
|
|
1073
|
+
local root="${1:-}"
|
|
1074
|
+
[ -n "$root" ] || return 0
|
|
1075
|
+
[ -d /proc ] || return 0
|
|
1076
|
+
|
|
1077
|
+
root="$(canonical_workspace_root "$root")"
|
|
1078
|
+
if workspace_root_is_shallow "$root"; then
|
|
1079
|
+
echo "⚠️ refusing shallow workspace cwd sweep root: $root" >&2
|
|
1080
|
+
return 0
|
|
1081
|
+
fi
|
|
1082
|
+
|
|
1083
|
+
local max_sweeps="${AIUR_WORKSPACE_REAP_SWEEPS:-6}"
|
|
1084
|
+
case "$max_sweeps" in '' | *[!0-9]*) max_sweeps=6 ;; esac
|
|
1085
|
+
[ "$max_sweeps" -gt 0 ] || return 0
|
|
1086
|
+
|
|
1087
|
+
local protected pids=() filtered=() pid p sweep waited any
|
|
1088
|
+
protected=" $(shell_protected_pid_list | tr '\n' ' ') "
|
|
1089
|
+
|
|
1090
|
+
sweep=0
|
|
1091
|
+
while [ "$sweep" -lt "$max_sweeps" ]; do
|
|
1092
|
+
mapfile -t pids < <(workspace_cwd_pids "$root" /proc)
|
|
1093
|
+
filtered=()
|
|
1094
|
+
for pid in "${pids[@]}"; do
|
|
1095
|
+
case "$protected" in *" $pid "*) continue ;; esac
|
|
1096
|
+
kill -0 "$pid" 2>/dev/null && filtered+=("$pid")
|
|
1097
|
+
done
|
|
1098
|
+
|
|
1099
|
+
[ "${#filtered[@]}" -gt 0 ] || return 0
|
|
1100
|
+
|
|
1101
|
+
for p in "${filtered[@]}"; do kill -TERM "$p" 2>/dev/null || true; done
|
|
1102
|
+
waited=0
|
|
1103
|
+
while [ "$waited" -lt 10 ]; do
|
|
1104
|
+
any=0
|
|
1105
|
+
for p in "${filtered[@]}"; do kill -0 "$p" 2>/dev/null && { any=1; break; }; done
|
|
1106
|
+
[ "$any" -eq 0 ] && break
|
|
1107
|
+
sleep 0.1
|
|
1108
|
+
waited=$((waited + 1))
|
|
1109
|
+
done
|
|
1110
|
+
for p in "${filtered[@]}"; do kill -KILL "$p" 2>/dev/null || true; done
|
|
1111
|
+
|
|
1112
|
+
sweep=$((sweep + 1))
|
|
1113
|
+
sleep 0.1
|
|
1114
|
+
done
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
reap_workspace_cwd_from_file() {
|
|
1118
|
+
local root_file="${1:-}" root
|
|
1119
|
+
[ -n "$root_file" ] && [ -s "$root_file" ] || return 0
|
|
1120
|
+
IFS= read -r root <"$root_file" || root=""
|
|
1121
|
+
reap_workspace_cwd_agents "$root"
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
current_workspace_root() {
|
|
1125
|
+
resolve_release
|
|
1126
|
+
prepare_distribution >/dev/null 2>&1 || return 1
|
|
1127
|
+
|
|
1128
|
+
local output status line
|
|
1129
|
+
set +e
|
|
1130
|
+
output="$("$release_bin" rpc "IO.puts(Aiur.Config.workspace_root())" 2>/dev/null)"
|
|
1131
|
+
status=$?
|
|
1132
|
+
set -e
|
|
1133
|
+
[ "$status" -eq 0 ] || return 1
|
|
1134
|
+
|
|
1135
|
+
while IFS= read -r line; do
|
|
1136
|
+
case "$line" in
|
|
1137
|
+
"" | :ok) continue ;;
|
|
1138
|
+
*) printf '%s\n' "$line"; return 0 ;;
|
|
1139
|
+
esac
|
|
1140
|
+
done <<<"$output"
|
|
1141
|
+
return 1
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
# Background watchdog that survives the BEAM. Polls for the release BEAM by
|
|
1145
|
+
# command pattern and, once it has SEEN the BEAM and then the BEAM disappears
|
|
1146
|
+
# (orderly halt OR :emfile-style crash), reaps every agent. Polling the pattern
|
|
1147
|
+
# rather than a captured pid avoids two failure modes: a recycled BEAM pid the
|
|
1148
|
+
# watchdog would poll forever, and an empty pid at arm time that would silently
|
|
1149
|
+
# disarm the only crash reaper. The `seen` latch prevents a startup race from
|
|
1150
|
+
# reaping before the BEAM has come up. kill-server collapses the orphaned tmux
|
|
1151
|
+
# session, which returns the foreground `tmux attach` so the EXIT trap
|
|
1152
|
+
# (session_cleanup) runs its idempotent reap too.
|
|
1153
|
+
#
|
|
1154
|
+
# $1 beam_pattern $2 socket $3 pidfile $4 interval_s $5 initial_seen
|
|
1155
|
+
# $6 node $7 run_log_dir $8 stop_sentinel $9 crash_marker
|
|
1156
|
+
# $10 workspace_root_file
|
|
1157
|
+
# Args 6-9 arm crash recording (background mode). Omit them (foreground) and the
|
|
1158
|
+
# watchdog only reaps — a foreground BEAM death is already visible at the UI.
|
|
1159
|
+
# Prints the watchdog's own pid so the caller can kill it on a clean teardown.
|
|
1160
|
+
start_beam_death_watchdog() {
|
|
1161
|
+
local beam_pattern="$1" socket="$2" pidfile="$3" interval="${4:-1}" initial_seen="${5:-0}"
|
|
1162
|
+
local node="${6:-}" run_log_dir="${7:-}" stop_sentinel="${8:-}" crash_marker="${9:-}" workspace_root_file="${10:-}"
|
|
1163
|
+
# Redirect the subshell's stdout so a command-substitution caller
|
|
1164
|
+
# (`pid=$(start_beam_death_watchdog ...)`) returns immediately instead of
|
|
1165
|
+
# blocking on the still-open pipe until the watchdog finishes.
|
|
1166
|
+
(
|
|
1167
|
+
seen="$initial_seen"
|
|
1168
|
+
while :; do
|
|
1169
|
+
if pgrep -f -- "$beam_pattern" >/dev/null 2>&1; then
|
|
1170
|
+
seen=1
|
|
1171
|
+
elif [ "$seen" = 1 ]; then
|
|
1172
|
+
break
|
|
1173
|
+
fi
|
|
1174
|
+
sleep "$interval"
|
|
1175
|
+
done
|
|
1176
|
+
# The BEAM we had seen is gone. A clean `aiur stop` drops the sentinel first;
|
|
1177
|
+
# its presence means an intentional exit (consume it, no crash record).
|
|
1178
|
+
# Anything else is an unexpected exit worth a durable record before reaping.
|
|
1179
|
+
if [ -n "$stop_sentinel" ] && [ -f "$stop_sentinel" ]; then
|
|
1180
|
+
rm -f "$stop_sentinel" 2>/dev/null || true
|
|
1181
|
+
elif [ -n "$crash_marker" ] || [ -n "$run_log_dir" ]; then
|
|
1182
|
+
record_beam_crash "$node" "$run_log_dir" "$crash_marker"
|
|
1183
|
+
fi
|
|
1184
|
+
reap_aiur_agents "$socket" "$pidfile"
|
|
1185
|
+
reap_workspace_cwd_from_file "$workspace_root_file"
|
|
1186
|
+
) >/dev/null 2>&1 &
|
|
1187
|
+
printf '%s\n' "$!"
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
probe_control_liveness() {
|
|
1191
|
+
local expression output status
|
|
1192
|
+
expression='case Process.whereis(Aiur.Orchestrator) do pid when is_pid(pid) -> case Aiur.Orchestrator.status(Aiur.Orchestrator, 100) do statuses when is_list(statuses) -> IO.puts("__AIUR_CONTROL_READY__"); _ -> IO.puts("__AIUR_CONTROL_NOT_READY__") end; _ -> IO.puts("__AIUR_CONTROL_NOT_READY__") end'
|
|
1193
|
+
|
|
1194
|
+
set +e
|
|
1195
|
+
output="$("$release_bin" rpc "$expression" 2>&1)"
|
|
1196
|
+
status=$?
|
|
1197
|
+
set -e
|
|
1198
|
+
|
|
1199
|
+
if [ "$status" -eq 0 ] && [[ "$output" == *"__AIUR_CONTROL_READY__"* ]]; then
|
|
1200
|
+
printf 'up'
|
|
1201
|
+
else
|
|
1202
|
+
printf 'down'
|
|
1203
|
+
fi
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
wait_for_session_startup() {
|
|
1207
|
+
local tmux_bin="$1" socket="$2" conf="$3" session="$4" startup_capture="$5" require_control="$6"
|
|
1208
|
+
local max_ticks="${AIUR_TMUX_GRACE_TICKS:-30}" tick control_state
|
|
1209
|
+
if [ "$require_control" = "1" ]; then
|
|
1210
|
+
max_ticks="${AIUR_NODE_GRACE_TICKS:-100}"
|
|
1211
|
+
fi
|
|
1212
|
+
|
|
1213
|
+
for ((tick = 0; tick < max_ticks; tick++)); do
|
|
1214
|
+
if ! "$tmux_bin" -L "$socket" -f "$conf" has-session -t "$session" 2>/dev/null; then
|
|
1215
|
+
echo "❌ aiur exited during startup; captured output:" >&2
|
|
1216
|
+
tail -n 30 "$startup_capture" 2>/dev/null | sed 's/^/ /' >&2 || true
|
|
1217
|
+
return 1
|
|
1218
|
+
fi
|
|
1219
|
+
|
|
1220
|
+
if [ "$require_control" != "1" ]; then
|
|
1221
|
+
sleep 0.1
|
|
1222
|
+
continue
|
|
1223
|
+
fi
|
|
1224
|
+
|
|
1225
|
+
control_state="$(probe_control_liveness)"
|
|
1226
|
+
if [ "$control_state" = "up" ]; then
|
|
1227
|
+
return 0
|
|
1228
|
+
fi
|
|
1229
|
+
|
|
1230
|
+
sleep 0.1
|
|
1231
|
+
done
|
|
1232
|
+
|
|
1233
|
+
if [ "$require_control" = "1" ]; then
|
|
1234
|
+
echo "❌ aiur control plane did not become ready at ${RELEASE_NODE:-unknown} during startup; captured output:" >&2
|
|
1235
|
+
tail -n 30 "$startup_capture" 2>/dev/null | sed 's/^/ /' >&2 || true
|
|
1236
|
+
return 1
|
|
1237
|
+
fi
|
|
1238
|
+
|
|
1239
|
+
return 0
|
|
1240
|
+
}
|
|
1241
|
+
|
|
420
1242
|
# Foreground teardown: kill the session + BEAM and reap opencode sessions on exit.
|
|
421
1243
|
_session_socket="" _session_name="" _session_conf="" _session_tmpfile=""
|
|
422
1244
|
_session_capture="" _session_argv="" _session_release="" _session_tmux="" _session_node=""
|
|
1245
|
+
_session_pidfile="" _session_watchdog_pid="" _session_workspace_root_file=""
|
|
423
1246
|
_cleanup_ran=0
|
|
424
1247
|
session_cleanup() {
|
|
425
1248
|
[ "$_cleanup_ran" = 1 ] && return 0
|
|
426
1249
|
_cleanup_ran=1
|
|
427
1250
|
local code=$?
|
|
428
1251
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
1252
|
+
# Stop the BEAM-death watchdog: cleanup is running, so it has no work left and
|
|
1253
|
+
# a lingering poller would outlive this teardown.
|
|
1254
|
+
[ -n "$_session_watchdog_pid" ] && kill "$_session_watchdog_pid" 2>/dev/null || true
|
|
432
1255
|
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
[ -z "$beam_pids" ] && break
|
|
440
|
-
sleep 0.1
|
|
441
|
-
waited=$((waited + 1))
|
|
442
|
-
done
|
|
443
|
-
beam_pids="$(pgrep -f "$_session_release/.*erts.*beam.smp" 2>/dev/null || true)"
|
|
444
|
-
for pid in $beam_pids; do kill -KILL "$pid" 2>/dev/null || true; done
|
|
1256
|
+
# kill-session on the detached aiur session FIRST: this is what propagates
|
|
1257
|
+
# SIGHUP to the BEAM in that session so it begins its own orderly shutdown.
|
|
1258
|
+
# Without it the BEAM survives Ctrl+C, holding port 4000 + the node name and
|
|
1259
|
+
# breaking the next launch (regression test/aiur/regression/shutdown_cleanup_test.exs).
|
|
1260
|
+
if [ -n "$_session_tmux" ] && [ -n "$_session_socket" ] && [ -n "$_session_name" ]; then
|
|
1261
|
+
"$_session_tmux" -L "$_session_socket" kill-session -t "$_session_name" 2>/dev/null || true
|
|
445
1262
|
fi
|
|
446
1263
|
|
|
447
|
-
#
|
|
448
|
-
#
|
|
1264
|
+
# Then kill-server on aiur's private socket: tears down every pane agent across
|
|
1265
|
+
# all windows AND leaves no live aiur tmux server, then reaps headless agents
|
|
1266
|
+
# from the pidfile. Idempotent with the watchdog's own reap and the kill-session
|
|
1267
|
+
# above. reap_aiur_agents re-resolves tmux itself and reaps headless agents even
|
|
1268
|
+
# when tmux is absent, so it is not gated on $_session_tmux.
|
|
1269
|
+
reap_aiur_agents "$_session_socket" "$_session_pidfile"
|
|
1270
|
+
|
|
1271
|
+
# Reap only this instance's BEAM. Multiple keyed instances can share the same
|
|
1272
|
+
# release dir, so release-path pgrep would terminate sibling workflows.
|
|
449
1273
|
[ -n "$_session_node" ] && kill_beams_matching "-name ${_session_node}"
|
|
450
1274
|
|
|
1275
|
+
# Final cwd-sweep backstop after the BEAM is gone: catches workspace-rooted
|
|
1276
|
+
# agents or test children that registered too late, reparented during the
|
|
1277
|
+
# BEAM-side sweep, or survived a bounded in-BEAM cleanup.
|
|
1278
|
+
reap_workspace_cwd_from_file "$_session_workspace_root_file"
|
|
1279
|
+
|
|
1280
|
+
# Reap the epmd our BEAM spawned for distribution so it doesn't linger after
|
|
1281
|
+
# exit. The node is dead by now, so `epmd -kill` succeeds; it safely refuses
|
|
1282
|
+
# while any *other* Erlang node is alive, so a daemon shared with an unrelated
|
|
1283
|
+
# Erlang program is never disrupted. Use the release's bundled epmd and pin it
|
|
1284
|
+
# to the loopback address aiur starts it on.
|
|
1285
|
+
for _epmd in "$_session_release"/erts-*/bin/epmd; do
|
|
1286
|
+
[ -x "$_epmd" ] || continue
|
|
1287
|
+
ERL_EPMD_ADDRESS="${ERL_EPMD_ADDRESS:-127.0.0.1}" "$_epmd" -kill >/dev/null 2>&1 || true
|
|
1288
|
+
done
|
|
1289
|
+
|
|
451
1290
|
if command -v opencode >/dev/null 2>&1 && [ -s "$_session_tmpfile" ]; then
|
|
452
1291
|
local id
|
|
453
1292
|
while IFS= read -r id; do
|
|
@@ -460,7 +1299,13 @@ session_cleanup() {
|
|
|
460
1299
|
# Reap agent-driver sockets this run orphaned at close, not next launch.
|
|
461
1300
|
sweep_dead_tmux_sockets || true
|
|
462
1301
|
|
|
463
|
-
|
|
1302
|
+
# Reap stale aiur /tmp debris (per-run tempfiles, leaked test artifacts) so a
|
|
1303
|
+
# tmpfs /tmp can't fill across runs. Age-gated + pid-safe; our own still-present
|
|
1304
|
+
# tempfiles are spared (fresh + live pid) before the explicit rm below clears them.
|
|
1305
|
+
sweep_stale_tmp_artifacts || true
|
|
1306
|
+
|
|
1307
|
+
rm -f "$_session_tmpfile" "$_session_capture" "$_session_argv" "$_session_pidfile" "$_session_workspace_root_file" 2>/dev/null || true
|
|
1308
|
+
rm -f "$(aiur_instance_record_path)" 2>/dev/null || true
|
|
464
1309
|
return $code
|
|
465
1310
|
}
|
|
466
1311
|
install_foreground_traps() {
|
|
@@ -479,23 +1324,264 @@ trim() {
|
|
|
479
1324
|
printf '%s' "$s"
|
|
480
1325
|
}
|
|
481
1326
|
|
|
1327
|
+
# Classify our distribution node's liveness via epmd, which only advertises a
|
|
1328
|
+
# node while its BEAM holds the registration. Echoes exactly one word:
|
|
1329
|
+
# up — node is registered: it is alive, so an rpc failure is a real error
|
|
1330
|
+
# down — epmd answered and our node is absent: it genuinely is not running
|
|
1331
|
+
# unknown — epmd could not be queried (binary missing, or daemon unreachable),
|
|
1332
|
+
# so node state is indeterminate and callers must NOT assume "down"
|
|
1333
|
+
# and mask the real error
|
|
1334
|
+
# Distinguishing `unknown` from `down` matters: a live node whose epmd we cannot
|
|
1335
|
+
# reach must still surface its real rpc error rather than the "start aiur" hint.
|
|
1336
|
+
# Relies on RELEASE_NODE + ERL_EPMD_ADDRESS (prepare_distribution) and
|
|
1337
|
+
# release_dir (resolve_release), so call it only after both have run.
|
|
1338
|
+
probe_node_liveness() {
|
|
1339
|
+
local epmd names short="${RELEASE_NODE%@*}"
|
|
1340
|
+
for epmd in "$release_dir"/erts-*/bin/epmd; do
|
|
1341
|
+
[ -x "$epmd" ] || continue
|
|
1342
|
+
names="$(ERL_EPMD_ADDRESS="${ERL_EPMD_ADDRESS:-127.0.0.1}" "$epmd" -names 2>/dev/null)" \
|
|
1343
|
+
|| { printf 'unknown'; return; }
|
|
1344
|
+
case "$names" in
|
|
1345
|
+
*"name ${short} at port "*) printf 'up' ;;
|
|
1346
|
+
*) printf 'down' ;;
|
|
1347
|
+
esac
|
|
1348
|
+
return
|
|
1349
|
+
done
|
|
1350
|
+
printf 'unknown'
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
probe_named_node_liveness() {
|
|
1354
|
+
local node="$1" old_node="${RELEASE_NODE:-}" state
|
|
1355
|
+
RELEASE_NODE="$node"
|
|
1356
|
+
state="$(probe_node_liveness)"
|
|
1357
|
+
RELEASE_NODE="$old_node"
|
|
1358
|
+
printf '%s' "$state"
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
load_aiur_instance_record() {
|
|
1362
|
+
local file="$1"
|
|
1363
|
+
AIUR_RECORD_NODE=""
|
|
1364
|
+
AIUR_RECORD_INSTANCE_KEY=""
|
|
1365
|
+
AIUR_RECORD_SESSION=""
|
|
1366
|
+
AIUR_RECORD_SOCKET=""
|
|
1367
|
+
AIUR_RECORD_PROJECT_ROOT=""
|
|
1368
|
+
AIUR_RECORD_PROJECT_ROOT_SOURCE=""
|
|
1369
|
+
[ -r "$file" ] || return 1
|
|
1370
|
+
# Records are written by this engine as KEY=%q lines under the user's state dir.
|
|
1371
|
+
# shellcheck disable=SC1090
|
|
1372
|
+
source "$file" 2>/dev/null || return 1
|
|
1373
|
+
[ -n "$AIUR_RECORD_NODE" ] && [ -n "$AIUR_RECORD_SESSION" ] && \
|
|
1374
|
+
[ -n "$AIUR_RECORD_SOCKET" ] && [ -n "$AIUR_RECORD_PROJECT_ROOT" ]
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
path_is_within_root() {
|
|
1378
|
+
local root child
|
|
1379
|
+
root="$(canonical_workspace_root "$1")"
|
|
1380
|
+
child="$(canonical_workspace_root "$2")"
|
|
1381
|
+
[ "$child" = "$root" ] || [[ "$child/" == "$root/"* ]]
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
AIUR_CONTROL_ADOPTED_RECORD=0
|
|
1385
|
+
AIUR_CONTROL_CURRENT_NODE_STATE=""
|
|
1386
|
+
AIUR_CONTROL_HINT_ROOTS=""
|
|
1387
|
+
AIUR_CONTROL_CALLER_ROOT=""
|
|
1388
|
+
AIUR_CONTROL_CALLER_NODE=""
|
|
1389
|
+
AIUR_CONTROL_CALLER_ROOT_SOURCE=""
|
|
1390
|
+
|
|
1391
|
+
resolve_control_identity_from_records() {
|
|
1392
|
+
AIUR_CONTROL_ADOPTED_RECORD=0
|
|
1393
|
+
AIUR_CONTROL_CURRENT_NODE_STATE="$(probe_node_liveness)"
|
|
1394
|
+
AIUR_CONTROL_HINT_ROOTS=""
|
|
1395
|
+
AIUR_CONTROL_CALLER_ROOT="$(canonical_workspace_root "${AIUR_PROJECT_ROOT:-}")"
|
|
1396
|
+
AIUR_CONTROL_CALLER_NODE="$AIUR_RELEASE_NODE"
|
|
1397
|
+
AIUR_CONTROL_CALLER_ROOT_SOURCE="${AIUR_PROJECT_ROOT_SOURCE:-}"
|
|
1398
|
+
|
|
1399
|
+
[ "$AIUR_CONTROL_CURRENT_NODE_STATE" = "down" ] || return 0
|
|
1400
|
+
[ "${AIUR_PROJECT_ROOT_SOURCE:-}" = "cwd" ] || return 0
|
|
1401
|
+
|
|
1402
|
+
local dir file state root match_count=0
|
|
1403
|
+
local match_node="" match_key="" match_session="" match_socket="" match_root="" match_source=""
|
|
1404
|
+
dir="$(aiur_instances_dir)"
|
|
1405
|
+
[ -d "$dir" ] || return 0
|
|
1406
|
+
|
|
1407
|
+
for file in "$dir"/*.instance; do
|
|
1408
|
+
[ -e "$file" ] || continue
|
|
1409
|
+
load_aiur_instance_record "$file" || continue
|
|
1410
|
+
state="$(probe_named_node_liveness "$AIUR_RECORD_NODE")"
|
|
1411
|
+
[ "$state" = "up" ] || continue
|
|
1412
|
+
|
|
1413
|
+
root="$(canonical_workspace_root "$AIUR_RECORD_PROJECT_ROOT")"
|
|
1414
|
+
case "$AIUR_CONTROL_HINT_ROOTS" in
|
|
1415
|
+
*"
|
|
1416
|
+
$root
|
|
1417
|
+
"*) ;;
|
|
1418
|
+
*) AIUR_CONTROL_HINT_ROOTS="${AIUR_CONTROL_HINT_ROOTS}${root}
|
|
1419
|
+
" ;;
|
|
1420
|
+
esac
|
|
1421
|
+
|
|
1422
|
+
path_is_within_root "$root" "$AIUR_CONTROL_CALLER_ROOT" || continue
|
|
1423
|
+
match_count=$((match_count + 1))
|
|
1424
|
+
match_node="$AIUR_RECORD_NODE"
|
|
1425
|
+
match_key="$AIUR_RECORD_INSTANCE_KEY"
|
|
1426
|
+
match_session="$AIUR_RECORD_SESSION"
|
|
1427
|
+
match_socket="$AIUR_RECORD_SOCKET"
|
|
1428
|
+
match_root="$root"
|
|
1429
|
+
match_source="$AIUR_RECORD_PROJECT_ROOT_SOURCE"
|
|
1430
|
+
done
|
|
1431
|
+
|
|
1432
|
+
[ "$match_count" -eq 1 ] || return 0
|
|
1433
|
+
|
|
1434
|
+
AIUR_RELEASE_NODE="$match_node"
|
|
1435
|
+
AIUR_INSTANCE_KEY="$match_key"
|
|
1436
|
+
AIUR_PROJECT_ROOT="$match_root"
|
|
1437
|
+
AIUR_PROJECT_ROOT_SOURCE="$match_source"
|
|
1438
|
+
AIUR_ADOPTED_TMUX_SESSION="$match_session"
|
|
1439
|
+
AIUR_ADOPTED_TMUX_SOCKET="$match_socket"
|
|
1440
|
+
export AIUR_RELEASE_NODE AIUR_INSTANCE_KEY AIUR_PROJECT_ROOT AIUR_PROJECT_ROOT_SOURCE \
|
|
1441
|
+
AIUR_ADOPTED_TMUX_SESSION AIUR_ADOPTED_TMUX_SOCKET
|
|
1442
|
+
RELEASE_NODE="$AIUR_RELEASE_NODE"
|
|
1443
|
+
export RELEASE_NODE
|
|
1444
|
+
AIUR_CONTROL_ADOPTED_RECORD=1
|
|
1445
|
+
}
|
|
1446
|
+
|
|
1447
|
+
print_global_config_control_hint() {
|
|
1448
|
+
[ "${AIUR_CONTROL_CALLER_ROOT_SOURCE:-}" = "cwd" ] || return 0
|
|
1449
|
+
[ "${AIUR_CONTROL_CURRENT_NODE_STATE:-}" = "down" ] || return 0
|
|
1450
|
+
|
|
1451
|
+
echo "aiur: global-config control identity is keyed by cwd ${AIUR_CONTROL_CALLER_ROOT:-${AIUR_PROJECT_ROOT:-unknown}}" >&2
|
|
1452
|
+
echo "aiur: run control commands from the launch directory, or from a subdirectory of that launch directory" >&2
|
|
1453
|
+
if [ -n "${AIUR_CONTROL_HINT_ROOTS:-}" ]; then
|
|
1454
|
+
echo "aiur: live launch directory candidate(s):" >&2
|
|
1455
|
+
printf '%s' "$AIUR_CONTROL_HINT_ROOTS" | sed '/^$/d; s/^/ /' >&2
|
|
1456
|
+
fi
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
control_rpc_timeout_seconds() {
|
|
1460
|
+
local seconds="${AIUR_CONTROL_RPC_TIMEOUT_SECONDS:-10}"
|
|
1461
|
+
case "$seconds" in
|
|
1462
|
+
'' | *[!0-9]* | 0) seconds=10 ;;
|
|
1463
|
+
esac
|
|
1464
|
+
printf '%s' "$seconds"
|
|
1465
|
+
}
|
|
1466
|
+
|
|
1467
|
+
kill_control_rpc_process() {
|
|
1468
|
+
local pid="$1" grouped="$2" p tree=()
|
|
1469
|
+
[ -n "$pid" ] || return 0
|
|
1470
|
+
|
|
1471
|
+
if [ "$grouped" = "1" ]; then
|
|
1472
|
+
kill -TERM "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true
|
|
1473
|
+
sleep 0.2
|
|
1474
|
+
kill -KILL "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true
|
|
1475
|
+
return 0
|
|
1476
|
+
fi
|
|
1477
|
+
|
|
1478
|
+
while IFS= read -r p; do tree+=("$p"); done < <(agent_pid_tree "$pid")
|
|
1479
|
+
for p in "${tree[@]}"; do kill -TERM "$p" 2>/dev/null || true; done
|
|
1480
|
+
sleep 0.2
|
|
1481
|
+
for p in "${tree[@]}"; do kill -KILL "$p" 2>/dev/null || true; done
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
run_release_rpc_with_timeout() {
|
|
1485
|
+
local expression="$1" timeout output_file timeout_file pid watchdog_pid status grouped=0
|
|
1486
|
+
timeout="$(control_rpc_timeout_seconds)"
|
|
1487
|
+
output_file="$(mktemp "${TMPDIR:-/tmp}/aiur-control-rpc-output.XXXXXX")"
|
|
1488
|
+
timeout_file="$(mktemp "${TMPDIR:-/tmp}/aiur-control-rpc-timeout.XXXXXX")"
|
|
1489
|
+
rm -f "$timeout_file" 2>/dev/null || true
|
|
1490
|
+
|
|
1491
|
+
if command -v setsid >/dev/null 2>&1; then
|
|
1492
|
+
setsid "$release_bin" rpc "$expression" >"$output_file" 2>&1 &
|
|
1493
|
+
grouped=1
|
|
1494
|
+
else
|
|
1495
|
+
"$release_bin" rpc "$expression" >"$output_file" 2>&1 &
|
|
1496
|
+
fi
|
|
1497
|
+
pid=$!
|
|
1498
|
+
|
|
1499
|
+
(
|
|
1500
|
+
sleep "$timeout"
|
|
1501
|
+
if kill -0 "$pid" 2>/dev/null; then
|
|
1502
|
+
: >"$timeout_file"
|
|
1503
|
+
kill_control_rpc_process "$pid" "$grouped"
|
|
1504
|
+
fi
|
|
1505
|
+
) &
|
|
1506
|
+
watchdog_pid=$!
|
|
1507
|
+
|
|
1508
|
+
if wait "$pid"; then
|
|
1509
|
+
status=0
|
|
1510
|
+
else
|
|
1511
|
+
status=$?
|
|
1512
|
+
fi
|
|
1513
|
+
kill "$watchdog_pid" 2>/dev/null || true
|
|
1514
|
+
wait "$watchdog_pid" 2>/dev/null || true
|
|
1515
|
+
|
|
1516
|
+
AIUR_CONTROL_RPC_OUTPUT="$(cat "$output_file" 2>/dev/null || true)"
|
|
1517
|
+
if [ -f "$timeout_file" ]; then
|
|
1518
|
+
AIUR_CONTROL_RPC_TIMED_OUT=1
|
|
1519
|
+
status=124
|
|
1520
|
+
else
|
|
1521
|
+
AIUR_CONTROL_RPC_TIMED_OUT=0
|
|
1522
|
+
fi
|
|
1523
|
+
|
|
1524
|
+
rm -f "$output_file" "$timeout_file" 2>/dev/null || true
|
|
1525
|
+
return "$status"
|
|
1526
|
+
}
|
|
1527
|
+
|
|
482
1528
|
# RPC an expression into the running node. The control CLI prints a trailing
|
|
483
1529
|
# `__AIUR_CONTROL_EXIT__:<code>` marker we translate into the process exit code.
|
|
484
1530
|
run_control_rpc() {
|
|
485
1531
|
local expression="$1"
|
|
486
1532
|
resolve_release
|
|
487
1533
|
prepare_distribution || die "distribution setup failed; cannot contact aiur"
|
|
1534
|
+
resolve_control_identity_from_records
|
|
1535
|
+
if [ "${AIUR_CONTROL_ADOPTED_RECORD:-0}" -eq 1 ]; then
|
|
1536
|
+
prepare_distribution || die "distribution setup failed; cannot contact aiur"
|
|
1537
|
+
fi
|
|
488
1538
|
|
|
489
1539
|
local marker="__AIUR_CONTROL_EXIT__:"
|
|
490
1540
|
local output status exit_code=0 saw_marker=0 line
|
|
491
1541
|
|
|
492
1542
|
set +e
|
|
493
|
-
|
|
1543
|
+
run_release_rpc_with_timeout "$expression"
|
|
494
1544
|
status=$?
|
|
1545
|
+
output="$AIUR_CONTROL_RPC_OUTPUT"
|
|
495
1546
|
set -e
|
|
496
1547
|
|
|
1548
|
+
if [ "${AIUR_CONTROL_RPC_TIMED_OUT:-0}" -eq 1 ]; then
|
|
1549
|
+
[ -n "$output" ] && printf '%s\n' "$output" >&2
|
|
1550
|
+
echo "aiur: control rpc to ${RELEASE_NODE} timed out after $(control_rpc_timeout_seconds)s; helper process was terminated" >&2
|
|
1551
|
+
return 124
|
|
1552
|
+
fi
|
|
1553
|
+
|
|
497
1554
|
if [ "$status" -ne 0 ]; then
|
|
498
|
-
|
|
1555
|
+
# A non-zero exit here is the rpc transport failing — application-level
|
|
1556
|
+
# outcomes ride the marker path below. `bin/aiur rpc` (Elixir --rpc-eval)
|
|
1557
|
+
# reports a genuinely-down node and a live-but-unreachable one identically
|
|
1558
|
+
# (`:noconnection`), and prints any exception raised inside the expression.
|
|
1559
|
+
# So the reason string can't be trusted; classify via epmd instead. Only a
|
|
1560
|
+
# node epmd confirms is down earns the friendly "start aiur" hint; an `up`
|
|
1561
|
+
# node failed for a real reason, and an `unknown` probe must not be assumed
|
|
1562
|
+
# down — in both of those cases surface the actual rpc output, never mask it.
|
|
1563
|
+
case "$(probe_node_liveness)" in
|
|
1564
|
+
down)
|
|
1565
|
+
local crash_marker
|
|
1566
|
+
crash_marker="$(aiur_crash_marker_path)"
|
|
1567
|
+
if [ -f "$crash_marker" ]; then
|
|
1568
|
+
echo "aiur: background daemon at ${RELEASE_NODE} is DOWN after an unexpected exit; agents may be orphaned" >&2
|
|
1569
|
+
sed 's/^/ /' "$crash_marker" >&2 2>/dev/null || true
|
|
1570
|
+
echo "aiur: run 'aiur stop' to reap any orphaned agents, then start aiur again" >&2
|
|
1571
|
+
else
|
|
1572
|
+
echo "aiur: no running aiur node at ${RELEASE_NODE}; start aiur and try again" >&2
|
|
1573
|
+
print_global_config_control_hint
|
|
1574
|
+
fi
|
|
1575
|
+
;;
|
|
1576
|
+
up)
|
|
1577
|
+
[ -n "$output" ] && printf '%s\n' "$output" >&2
|
|
1578
|
+
echo "aiur: rpc to ${RELEASE_NODE} failed (node is running); see the error above" >&2
|
|
1579
|
+
;;
|
|
1580
|
+
*)
|
|
1581
|
+
[ -n "$output" ] && printf '%s\n' "$output" >&2
|
|
1582
|
+
echo "aiur: rpc to ${RELEASE_NODE} failed (could not query epmd to confirm node state); see the error above" >&2
|
|
1583
|
+
;;
|
|
1584
|
+
esac
|
|
499
1585
|
return 1
|
|
500
1586
|
fi
|
|
501
1587
|
|
|
@@ -510,7 +1596,11 @@ run_control_rpc() {
|
|
|
510
1596
|
esac
|
|
511
1597
|
done <<<"$output"
|
|
512
1598
|
|
|
513
|
-
[ "$saw_marker" -
|
|
1599
|
+
if [ "$saw_marker" -ne 1 ]; then
|
|
1600
|
+
echo "aiur: control rpc to ${RELEASE_NODE} returned no exit marker; command output may be incomplete" >&2
|
|
1601
|
+
return 1
|
|
1602
|
+
fi
|
|
1603
|
+
|
|
514
1604
|
return "$exit_code"
|
|
515
1605
|
}
|
|
516
1606
|
|
|
@@ -575,6 +1665,152 @@ cmd_pause_resume() {
|
|
|
575
1665
|
run_control_rpc "$expression"
|
|
576
1666
|
}
|
|
577
1667
|
|
|
1668
|
+
# `aiur message <issue> <text>` — deliver operator text to one running agent.
|
|
1669
|
+
# The text is base64-encoded for the RPC hop so arbitrary content (quotes,
|
|
1670
|
+
# backslashes, `#{}`, newlines) survives without Elixir-string escaping.
|
|
1671
|
+
cmd_message() {
|
|
1672
|
+
local usage="aiur: message expects an issue ID and text (e.g. aiur message 44 \"ship it\")"
|
|
1673
|
+
|
|
1674
|
+
local issue="${1:-}"
|
|
1675
|
+
if [ -z "$issue" ] || [[ ! "$issue" =~ ^[0-9]+$ ]]; then
|
|
1676
|
+
echo "$usage" >&2
|
|
1677
|
+
exit 64
|
|
1678
|
+
fi
|
|
1679
|
+
shift
|
|
1680
|
+
|
|
1681
|
+
local text="$*"
|
|
1682
|
+
if [ -z "$text" ]; then
|
|
1683
|
+
echo "$usage" >&2
|
|
1684
|
+
exit 64
|
|
1685
|
+
fi
|
|
1686
|
+
|
|
1687
|
+
local encoded
|
|
1688
|
+
encoded="$(printf '%s' "$text" | base64 | tr -d '\n')"
|
|
1689
|
+
run_control_rpc "Aiur.AgentControlCLI.message(\"$issue\", Base.decode64!(\"$encoded\"))"
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
# `aiur agents` — concise one-line-per-agent state + current activity from a
|
|
1693
|
+
# live node (the headless equivalent of the dashboard / aiur-status skill).
|
|
1694
|
+
cmd_agents() {
|
|
1695
|
+
[ "$#" -eq 0 ] || die "agents does not accept arguments"
|
|
1696
|
+
run_control_rpc "Aiur.AgentControlCLI.agents()"
|
|
1697
|
+
}
|
|
1698
|
+
|
|
1699
|
+
# `aiur alerts` — newline-delimited structured alert feed from persisted
|
|
1700
|
+
# per-agent logs. `--needs-attention` filters to operator-actionable alerts.
|
|
1701
|
+
cmd_alerts() {
|
|
1702
|
+
local needs_attention=0 arg
|
|
1703
|
+
for arg in "$@"; do
|
|
1704
|
+
case "$arg" in
|
|
1705
|
+
--needs-attention) needs_attention=1 ;;
|
|
1706
|
+
*)
|
|
1707
|
+
echo "aiur: alerts only accepts --needs-attention" >&2
|
|
1708
|
+
exit 64
|
|
1709
|
+
;;
|
|
1710
|
+
esac
|
|
1711
|
+
done
|
|
1712
|
+
|
|
1713
|
+
if [ "$needs_attention" -eq 1 ]; then
|
|
1714
|
+
run_control_rpc "Aiur.AgentControlCLI.alerts(needs_attention: true)"
|
|
1715
|
+
else
|
|
1716
|
+
run_control_rpc "Aiur.AgentControlCLI.alerts()"
|
|
1717
|
+
fi
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
# `aiur watch` — the server-side status board. Compiles one row per active
|
|
1721
|
+
# agent (state · complexity · activity-age · doing) plus an actionable section
|
|
1722
|
+
# from aiur's own state, with no GitHub round-trip. `--changes` (default) prints
|
|
1723
|
+
# only state-level deltas since the last call; `--full` prints every row;
|
|
1724
|
+
# `--interval N` re-renders every N seconds as a foreground watcher (the Elixir
|
|
1725
|
+
# call stays one-shot — the loop lives here).
|
|
1726
|
+
cmd_watch() {
|
|
1727
|
+
local mode="changes" interval="" arg
|
|
1728
|
+
|
|
1729
|
+
while [ "$#" -gt 0 ]; do
|
|
1730
|
+
arg="$1"
|
|
1731
|
+
case "$arg" in
|
|
1732
|
+
--full) mode="full" ;;
|
|
1733
|
+
--changes) mode="changes" ;;
|
|
1734
|
+
--once) : ;;
|
|
1735
|
+
--interval)
|
|
1736
|
+
shift
|
|
1737
|
+
interval="${1:-}"
|
|
1738
|
+
watch_validate_interval "$interval"
|
|
1739
|
+
;;
|
|
1740
|
+
--interval=*)
|
|
1741
|
+
interval="${arg#--interval=}"
|
|
1742
|
+
watch_validate_interval "$interval"
|
|
1743
|
+
;;
|
|
1744
|
+
*)
|
|
1745
|
+
echo "aiur: watch accepts --full, --changes, --once, --interval <secs>" >&2
|
|
1746
|
+
exit 64
|
|
1747
|
+
;;
|
|
1748
|
+
esac
|
|
1749
|
+
shift
|
|
1750
|
+
done
|
|
1751
|
+
|
|
1752
|
+
local expression="Aiur.AgentControlCLI.watch(mode: :${mode})"
|
|
1753
|
+
|
|
1754
|
+
if [ -n "$interval" ]; then
|
|
1755
|
+
while true; do
|
|
1756
|
+
run_control_rpc "$expression" || true
|
|
1757
|
+
sleep "$interval"
|
|
1758
|
+
done
|
|
1759
|
+
else
|
|
1760
|
+
run_control_rpc "$expression"
|
|
1761
|
+
fi
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1764
|
+
watch_validate_interval() {
|
|
1765
|
+
if ! [[ "$1" =~ ^[0-9]+$ ]] || [ "$1" -le 0 ]; then
|
|
1766
|
+
echo "aiur: watch --interval expects a positive integer (seconds)" >&2
|
|
1767
|
+
exit 64
|
|
1768
|
+
fi
|
|
1769
|
+
}
|
|
1770
|
+
|
|
1771
|
+
cmd_cleanup_stale() {
|
|
1772
|
+
local dry_run=0 arg
|
|
1773
|
+
for arg in "$@"; do
|
|
1774
|
+
case "$arg" in
|
|
1775
|
+
--dry-run) dry_run=1 ;;
|
|
1776
|
+
*)
|
|
1777
|
+
echo "aiur: cleanup-stale only accepts --dry-run" >&2
|
|
1778
|
+
exit 64
|
|
1779
|
+
;;
|
|
1780
|
+
esac
|
|
1781
|
+
done
|
|
1782
|
+
|
|
1783
|
+
aiur_resolve_identity
|
|
1784
|
+
if [ "$dry_run" -eq 1 ]; then
|
|
1785
|
+
report_stale_manual_smoke || true
|
|
1786
|
+
else
|
|
1787
|
+
report_stale_manual_smoke || true
|
|
1788
|
+
reap_stale_manual_smoke 1
|
|
1789
|
+
fi
|
|
1790
|
+
}
|
|
1791
|
+
|
|
1792
|
+
# `aiur set <key> <value>` — runtime config overrides without editing
|
|
1793
|
+
# `.aiur/config`. Currently: `aiur set max-agents N`.
|
|
1794
|
+
cmd_set() {
|
|
1795
|
+
local key="${1:-}"
|
|
1796
|
+
shift 2>/dev/null || true
|
|
1797
|
+
|
|
1798
|
+
case "$key" in
|
|
1799
|
+
max-agents)
|
|
1800
|
+
local n="${1:-}"
|
|
1801
|
+
if [ -z "$n" ] || [[ ! "$n" =~ ^[0-9]+$ ]] || [ "$n" -lt 1 ]; then
|
|
1802
|
+
echo "aiur: set max-agents expects a positive integer (e.g. aiur set max-agents 5)" >&2
|
|
1803
|
+
exit 64
|
|
1804
|
+
fi
|
|
1805
|
+
run_control_rpc "Aiur.AgentControlCLI.set_max_agents($n)"
|
|
1806
|
+
;;
|
|
1807
|
+
*)
|
|
1808
|
+
echo "aiur: unknown setting: ${key:-(none)} (supported: max-agents)" >&2
|
|
1809
|
+
exit 64
|
|
1810
|
+
;;
|
|
1811
|
+
esac
|
|
1812
|
+
}
|
|
1813
|
+
|
|
578
1814
|
sweep_dead_tmux_sockets() {
|
|
579
1815
|
local tmux_bin
|
|
580
1816
|
tmux_bin="$(command -v tmux || true)"
|
|
@@ -600,38 +1836,181 @@ sweep_dead_tmux_sockets() {
|
|
|
600
1836
|
return 0
|
|
601
1837
|
}
|
|
602
1838
|
|
|
603
|
-
#
|
|
1839
|
+
# Reap stale aiur /tmp debris so a tmpfs /tmp can't fill from accumulated per-run
|
|
1840
|
+
# tempfiles and leaked test artifacts. Runs alongside sweep_dead_tmux_sockets on
|
|
1841
|
+
# foreground teardown (session_cleanup) and on `aiur stop`. Bounded and safe:
|
|
1842
|
+
#
|
|
1843
|
+
# * Only exact top-level Aiur artifact families under the temp roots the engine
|
|
1844
|
+
# and BEAM write to. Arbitrary operator worktrees/checkouts like
|
|
1845
|
+
# `/tmp/aiur-pr490` are not candidates.
|
|
1846
|
+
# * Age-gated by AIUR_TMP_REAP_MINUTES (default 1440 = 24h). An entry is removed
|
|
1847
|
+
# only when NOTHING in its subtree was modified within the window, so a live
|
|
1848
|
+
# run's shared debug dir (aiur-rc / aiur-claude-hooks / aiur-debug) holding a
|
|
1849
|
+
# fresh file is spared even when the dir's own mtime is stale. A non-numeric or
|
|
1850
|
+
# 0 value disables the sweep.
|
|
1851
|
+
# * Ownership-gated: if anything in the tree is not owned by the effective user,
|
|
1852
|
+
# the whole tree is spared.
|
|
1853
|
+
# * A live run's `aiur-<pid>-sessions|-agents` bookkeeping is kept while its pid
|
|
1854
|
+
# is alive, regardless of age.
|
|
1855
|
+
is_aiur_tmp_artifact_candidate() {
|
|
1856
|
+
local base="$1" pid
|
|
1857
|
+
case "$base" in
|
|
1858
|
+
aiur-argv.* | aiur-startup.* | aiur-pane.* | aiur-launcher.* | aiur-capture.* | \
|
|
1859
|
+
aiur-trap.*.log | aiur-tree.*.json | aiur-wrapper.pid | aiur-*-frames.bin | \
|
|
1860
|
+
aiur-rc | aiur-claude-hooks | aiur-debug)
|
|
1861
|
+
return 0
|
|
1862
|
+
;;
|
|
1863
|
+
aiur-*-sessions | aiur-*-agents)
|
|
1864
|
+
pid="${base#aiur-}"
|
|
1865
|
+
pid="${pid%-*}"
|
|
1866
|
+
[ -n "$pid" ] && [ -z "${pid//[0-9]/}" ]
|
|
1867
|
+
return
|
|
1868
|
+
;;
|
|
1869
|
+
*)
|
|
1870
|
+
return 1
|
|
1871
|
+
;;
|
|
1872
|
+
esac
|
|
1873
|
+
}
|
|
1874
|
+
|
|
1875
|
+
sweep_stale_tmp_artifacts() {
|
|
1876
|
+
local minutes="${AIUR_TMP_REAP_MINUTES:-1440}"
|
|
1877
|
+
case "$minutes" in '' | *[!0-9]*) return 0 ;; esac
|
|
1878
|
+
[ "$minutes" -gt 0 ] || return 0
|
|
1879
|
+
|
|
1880
|
+
# The dirs the engine + BEAM target: ${TMPDIR:-/tmp} (argv/startup/pane tempfiles
|
|
1881
|
+
# plus the BEAM's System.tmp_dir debug dirs) and ${XDG_RUNTIME_DIR:-${TMPDIR:-/tmp}}
|
|
1882
|
+
# (the session/agent pidfiles' session_root). Deduped — a typical box scans one dir.
|
|
1883
|
+
local roots=() d seen
|
|
1884
|
+
for d in "${TMPDIR:-/tmp}" "${XDG_RUNTIME_DIR:-${TMPDIR:-/tmp}}"; do
|
|
1885
|
+
[ -d "$d" ] || continue
|
|
1886
|
+
seen=0
|
|
1887
|
+
local root
|
|
1888
|
+
if [ "${#roots[@]}" -gt 0 ]; then
|
|
1889
|
+
for root in "${roots[@]}"; do
|
|
1890
|
+
[ "$root" = "$d" ] && { seen=1; break; }
|
|
1891
|
+
done
|
|
1892
|
+
fi
|
|
1893
|
+
[ "$seen" -eq 1 ] || roots+=("$d")
|
|
1894
|
+
done
|
|
1895
|
+
[ "${#roots[@]}" -gt 0 ] || return 0
|
|
1896
|
+
|
|
1897
|
+
local removed=0 path base pid uid found
|
|
1898
|
+
uid="$(id -u)"
|
|
1899
|
+
for d in "${roots[@]}"; do
|
|
1900
|
+
while IFS= read -r -d '' path; do
|
|
1901
|
+
base="${path##*/}"
|
|
1902
|
+
is_aiur_tmp_artifact_candidate "$base" || continue
|
|
1903
|
+
|
|
1904
|
+
# Spare mixed-ownership trees; cleanup should never cross user boundaries.
|
|
1905
|
+
if ! found="$(find "$path" ! -user "$uid" -print -quit 2>/dev/null)"; then
|
|
1906
|
+
continue
|
|
1907
|
+
fi
|
|
1908
|
+
if [ -n "$found" ]; then
|
|
1909
|
+
continue
|
|
1910
|
+
fi
|
|
1911
|
+
# Spare anything modified within the window anywhere in its subtree.
|
|
1912
|
+
if ! found="$(find "$path" -mmin "-$minutes" -print -quit 2>/dev/null)"; then
|
|
1913
|
+
continue
|
|
1914
|
+
fi
|
|
1915
|
+
if [ -n "$found" ]; then
|
|
1916
|
+
continue
|
|
1917
|
+
fi
|
|
1918
|
+
# Spare a live run's session/agent bookkeeping (name is aiur-<pid>-<kind>).
|
|
1919
|
+
case "$base" in
|
|
1920
|
+
aiur-*-sessions | aiur-*-agents)
|
|
1921
|
+
pid="${base#aiur-}"
|
|
1922
|
+
pid="${pid%-*}"
|
|
1923
|
+
if [ -n "$pid" ] && [ -z "${pid//[0-9]/}" ] && kill -0 "$pid" 2>/dev/null; then
|
|
1924
|
+
continue
|
|
1925
|
+
fi
|
|
1926
|
+
;;
|
|
1927
|
+
esac
|
|
1928
|
+
if rm -rf -- "$path" 2>/dev/null; then
|
|
1929
|
+
removed=$((removed + 1))
|
|
1930
|
+
fi
|
|
1931
|
+
done < <(find "$d" -maxdepth 1 -mindepth 1 -name 'aiur-*' -print0 2>/dev/null)
|
|
1932
|
+
done
|
|
1933
|
+
|
|
1934
|
+
[ "$removed" -gt 0 ] && echo "🧹 swept $removed stale aiur tmp artifact(s)" >&2
|
|
1935
|
+
return 0
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1938
|
+
# Stop the running session: kill this instance's tmux + BEAM, then sweep.
|
|
604
1939
|
cmd_stop() {
|
|
605
1940
|
resolve_release
|
|
606
1941
|
aiur_resolve_identity
|
|
1942
|
+
RELEASE_NODE="$AIUR_RELEASE_NODE"
|
|
1943
|
+
ERL_EPMD_ADDRESS="${ERL_EPMD_ADDRESS:-127.0.0.1}"
|
|
1944
|
+
export RELEASE_NODE ERL_EPMD_ADDRESS
|
|
1945
|
+
resolve_control_identity_from_records
|
|
1946
|
+
|
|
1947
|
+
local workspace_root
|
|
1948
|
+
workspace_root="$(current_workspace_root 2>/dev/null || true)"
|
|
607
1949
|
|
|
608
1950
|
local tmux_bin
|
|
609
1951
|
tmux_bin="$(command -v tmux || true)"
|
|
610
|
-
local session="${AIUR_SESSION_PREFIX}-${USER:-user}-default"
|
|
611
|
-
local socket="${AIUR_SESSION_PREFIX}-${USER:-user}"
|
|
612
|
-
|
|
613
|
-
|
|
1952
|
+
local session="${AIUR_ADOPTED_TMUX_SESSION:-${AIUR_SESSION_PREFIX}-${USER:-user}${AIUR_INSTANCE_KEY:+-$AIUR_INSTANCE_KEY}-default}"
|
|
1953
|
+
local socket="${AIUR_ADOPTED_TMUX_SOCKET:-${AIUR_SESSION_PREFIX}-${USER:-user}${AIUR_INSTANCE_KEY:+-$AIUR_INSTANCE_KEY}}"
|
|
1954
|
+
|
|
1955
|
+
local has_session=0
|
|
1956
|
+
if [ -n "$tmux_bin" ] && "$tmux_bin" -L "$socket" has-session -t "$session" 2>/dev/null; then
|
|
1957
|
+
has_session=1
|
|
614
1958
|
fi
|
|
615
1959
|
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
1960
|
+
if [ "${AIUR_CONTROL_ADOPTED_RECORD:-0}" -ne 1 ] && \
|
|
1961
|
+
[ "${AIUR_CONTROL_CURRENT_NODE_STATE:-}" = "down" ] && \
|
|
1962
|
+
[ "${AIUR_PROJECT_ROOT_SOURCE:-}" = "cwd" ] && \
|
|
1963
|
+
[ "$has_session" -eq 0 ] && \
|
|
1964
|
+
[ ! -f "$(aiur_crash_marker_path)" ]; then
|
|
1965
|
+
echo "aiur: no running aiur node at ${AIUR_RELEASE_NODE}; nothing stopped" >&2
|
|
1966
|
+
print_global_config_control_hint
|
|
1967
|
+
return 1
|
|
1968
|
+
fi
|
|
1969
|
+
|
|
1970
|
+
# Tell the background BEAM-death watchdog this exit is intentional before we
|
|
1971
|
+
# kill the BEAM, so it consumes the sentinel instead of recording a crash. The
|
|
1972
|
+
# watchdog removes the sentinel when it fires; a fresh start also clears it.
|
|
1973
|
+
# Clear any prior crash marker too — `status` should report a clean stop, not
|
|
1974
|
+
# a stale orphan from an earlier dead run.
|
|
1975
|
+
mkdir -p "$AIUR_BG_STATE_DIR" 2>/dev/null || true
|
|
1976
|
+
: >"$(aiur_stop_sentinel_path)" 2>/dev/null || true
|
|
1977
|
+
rm -f "$(aiur_crash_marker_path)" 2>/dev/null || true
|
|
1978
|
+
|
|
1979
|
+
if [ -n "$tmux_bin" ]; then
|
|
1980
|
+
"$tmux_bin" -L "$socket" kill-session -t "$session" 2>/dev/null || true
|
|
628
1981
|
fi
|
|
629
1982
|
|
|
630
1983
|
# Reap any BEAM holding our node name regardless of which release dir launched
|
|
631
|
-
# it
|
|
1984
|
+
# it. Do not sweep by release dir: sibling instances share dev releases.
|
|
632
1985
|
kill_beams_matching "-name ${AIUR_RELEASE_NODE}"
|
|
633
1986
|
|
|
1987
|
+
# Final cwd-sweep backstop after the BEAM is dead. `cmd_stop` is a fresh shell,
|
|
1988
|
+
# so it captures the root from the live control node before the kill above.
|
|
1989
|
+
reap_workspace_cwd_agents "$workspace_root"
|
|
1990
|
+
|
|
1991
|
+
# The BEAM (alive until the TERM above) reaped its own headless agents through
|
|
1992
|
+
# ProcessReaper on Application.stop. kill-server is the guarantee the earlier
|
|
1993
|
+
# kill-session can't give for mid-turn agents: every pane agent across all
|
|
1994
|
+
# windows dies and no live aiur tmux server is left behind.
|
|
1995
|
+
if [ -n "$tmux_bin" ]; then
|
|
1996
|
+
"$tmux_bin" -L "$socket" kill-server 2>/dev/null || true
|
|
1997
|
+
fi
|
|
1998
|
+
|
|
1999
|
+
# Belt-and-suspenders for a mid-turn stop: sweep any headless agent (this run
|
|
2000
|
+
# or a prior crashed one) still recorded in a pidfile. comm-guarded, so a
|
|
2001
|
+
# recycled pid is spared. Empty socket: the kill-server above already ran.
|
|
2002
|
+
local session_root agentfile
|
|
2003
|
+
session_root="${XDG_RUNTIME_DIR:-${TMPDIR:-/tmp}}"
|
|
2004
|
+
for agentfile in "$session_root"/aiur-*-agents; do
|
|
2005
|
+
[ -e "$agentfile" ] || continue
|
|
2006
|
+
reap_aiur_agents "" "$agentfile"
|
|
2007
|
+
rm -f "$agentfile" 2>/dev/null || true
|
|
2008
|
+
done
|
|
2009
|
+
|
|
2010
|
+
reap_stale_manual_smoke 0
|
|
634
2011
|
sweep_dead_tmux_sockets
|
|
2012
|
+
sweep_stale_tmp_artifacts
|
|
2013
|
+
rm -f "$(aiur_instance_record_path)" 2>/dev/null || true
|
|
635
2014
|
}
|
|
636
2015
|
|
|
637
2016
|
# --- dispatch ----------------------------------------------------------------
|
|
@@ -663,10 +2042,34 @@ aiur_engine_main() {
|
|
|
663
2042
|
shift
|
|
664
2043
|
cmd_status "$@"
|
|
665
2044
|
;;
|
|
2045
|
+
agents)
|
|
2046
|
+
shift
|
|
2047
|
+
cmd_agents "$@"
|
|
2048
|
+
;;
|
|
2049
|
+
alerts)
|
|
2050
|
+
shift
|
|
2051
|
+
cmd_alerts "$@"
|
|
2052
|
+
;;
|
|
2053
|
+
watch)
|
|
2054
|
+
shift
|
|
2055
|
+
cmd_watch "$@"
|
|
2056
|
+
;;
|
|
2057
|
+
set)
|
|
2058
|
+
shift
|
|
2059
|
+
cmd_set "$@"
|
|
2060
|
+
;;
|
|
666
2061
|
pause | resume)
|
|
667
2062
|
shift
|
|
668
2063
|
cmd_pause_resume "$cmd" "$@"
|
|
669
2064
|
;;
|
|
2065
|
+
message)
|
|
2066
|
+
shift
|
|
2067
|
+
cmd_message "$@"
|
|
2068
|
+
;;
|
|
2069
|
+
cleanup-stale)
|
|
2070
|
+
shift
|
|
2071
|
+
cmd_cleanup_stale "$@"
|
|
2072
|
+
;;
|
|
670
2073
|
stop)
|
|
671
2074
|
cmd_stop
|
|
672
2075
|
;;
|