aiur-cli 0.0.2 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -23,6 +23,18 @@
23
23
 
24
24
  set -euo pipefail
25
25
 
26
+ # Raise the soft open-file limit toward the hard maximum. High agent concurrency
27
+ # spawns many tmux/opencode/git subprocesses + sockets; on hosts with a low
28
+ # default (macOS ships 256) that exhausts file descriptors (:emfile) and crashes
29
+ # the node. Soft<=hard needs no privilege; best-effort, never fatal.
30
+ __aiur_hard_nofile="$(ulimit -Hn 2>/dev/null || echo)"
31
+ if [ "${__aiur_hard_nofile}" = "unlimited" ]; then
32
+ ulimit -Sn 65536 2>/dev/null || true
33
+ elif [ -n "${__aiur_hard_nofile}" ]; then
34
+ ulimit -Sn "${__aiur_hard_nofile}" 2>/dev/null || true
35
+ fi
36
+ unset __aiur_hard_nofile
37
+
26
38
  die() {
27
39
  echo "❌ $*" >&2
28
40
  exit 1
@@ -30,19 +42,99 @@ die() {
30
42
 
31
43
  engine_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
32
44
 
33
- # --- distribution identity (fixed: the single `aiur` identity) ---------------
45
+ # --- distribution identity (per-instance: keyed by the aiur project root) -----
46
+
47
+ # The aiur project root used to key this instance. AIUR_REPO_ROOT (set by the dev
48
+ # shim) wins. Otherwise walk up from $PWD to the first dir holding a REPO-LOCAL
49
+ # config — but the walk STOPS at $HOME: the global config at ~/.aiur/config (legacy
50
+ # ~/.aiurconfig) is not a repo root, and treating it as one would collapse every
51
+ # project under $HOME onto one key (#443). When no repo-local config is found, the
52
+ # BEAM serves this run via the global config (mirroring its discovery order in
53
+ # src/lib/aiur/workflow.ex) — or via none — and in both cases the project being
54
+ # served is the cwd, so we key by realpath($PWD). That gives each global-config
55
+ # project a distinct identity instead of an empty key (legacy aiur-$USER@…) or $HOME.
56
+ #
57
+ # Caveat: a global-config run's key is cwd-derived (there is no repo root to
58
+ # converge on), so control commands (status/pause/stop) must be run from the SAME
59
+ # directory the run was launched from. A repo with a repo-local .aiur/config keeps
60
+ # the walk-up, so its control commands still resolve from any subdir.
61
+ aiur_project_root() {
62
+ if [ -n "${AIUR_REPO_ROOT:-}" ]; then printf '%s' "$AIUR_REPO_ROOT"; return; fi
63
+
64
+ # Canonicalize $PWD and $HOME so the home-boundary test below holds even when
65
+ # either is reached through a symlink.
66
+ local pwd_real home_real
67
+ pwd_real="$(pwd -P 2>/dev/null || printf '%s' "$PWD")"
68
+ # ${HOME:-} (not bare $HOME) so an unset HOME under `set -u` can't abort the
69
+ # script here; an empty home_real simply disables the boundary (walk to /).
70
+ home_real="$(cd "${HOME:-}" 2>/dev/null && pwd -P || printf '%s' "${HOME:-}")"
71
+
72
+ local d="$pwd_real"
73
+ while [ -n "$d" ] && [ "$d" != "/" ] && [ "$d" != "$home_real" ]; do
74
+ if [ -f "$d/.aiur/config" ] || [ -f "$d/.aiurconfig" ]; then
75
+ printf '%s' "$d"
76
+ return
77
+ fi
78
+ d="$(dirname "$d")"
79
+ done
80
+
81
+ printf '%s' "$pwd_real"
82
+ }
83
+
84
+ aiur_project_root_source() {
85
+ if [ -n "${AIUR_REPO_ROOT:-}" ]; then printf 'env'; return; fi
86
+
87
+ local pwd_real home_real
88
+ pwd_real="$(pwd -P 2>/dev/null || printf '%s' "$PWD")"
89
+ home_real="$(cd "${HOME:-}" 2>/dev/null && pwd -P || printf '%s' "${HOME:-}")"
90
+
91
+ local d="$pwd_real"
92
+ while [ -n "$d" ] && [ "$d" != "/" ] && [ "$d" != "$home_real" ]; do
93
+ if [ -f "$d/.aiur/config" ] || [ -f "$d/.aiurconfig" ]; then
94
+ printf 'repo'
95
+ return
96
+ fi
97
+ d="$(dirname "$d")"
98
+ done
99
+
100
+ printf 'cwd'
101
+ }
102
+
103
+ # Short, stable, node-name-legal (lowercase hex) key for the project root, so two
104
+ # aiur instances for the same user get distinct node/session/socket names and can't
105
+ # reap each other. Any real cwd now resolves a key (global-config runs key by
106
+ # realpath($PWD), #443); only a degenerate unreadable cwd yields empty, falling back
107
+ # to the legacy un-keyed name.
108
+ aiur_instance_key() {
109
+ local root
110
+ root="$(aiur_project_root)"
111
+ [ -n "$root" ] || return 0
112
+ # Canonicalize (resolve symlinks) so launch and control commands invoked via
113
+ # different logical paths to the same project agree on the key. Fall back to the
114
+ # literal path when the dir doesn't exist (e.g. a test-fixture root).
115
+ root="$(cd "$root" 2>/dev/null && pwd -P || printf '%s' "$root")"
116
+ printf '%s' "$root" | { shasum -a 256 2>/dev/null || sha256sum; } | cut -c1-10
117
+ }
34
118
 
35
119
  aiur_resolve_identity() {
36
120
  local config_home="${XDG_CONFIG_HOME:-$HOME/.config}"
121
+ AIUR_PROJECT_ROOT="$(aiur_project_root)"
122
+ AIUR_PROJECT_ROOT_SOURCE="$(aiur_project_root_source)"
37
123
 
38
124
  : "${AIUR_BG_STATE_DIR:=$config_home/aiur}"
39
125
  : "${AIUR_COOKIE_FILE:=$AIUR_BG_STATE_DIR/cookie}"
40
126
  : "${AIUR_SESSION_PREFIX:=aiur}"
41
127
  : "${AIUR_PROFILES_FILE:=$config_home/aiur/aiur.profiles}"
42
- : "${AIUR_RELEASE_NODE:=aiur-${USER}@127.0.0.1}"
128
+ # Compute the per-instance key once. `${VAR+x}` so an explicit empty value (no
129
+ # project resolved) is honored and not recomputed on each call.
130
+ if [ -z "${AIUR_INSTANCE_KEY+x}" ]; then
131
+ AIUR_INSTANCE_KEY="$(aiur_instance_key)"
132
+ fi
133
+ : "${AIUR_RELEASE_NODE:=aiur-${USER}${AIUR_INSTANCE_KEY:+-$AIUR_INSTANCE_KEY}@127.0.0.1}"
43
134
 
44
135
  export AIUR_BG_STATE_DIR AIUR_COOKIE_FILE AIUR_SESSION_PREFIX \
45
- AIUR_PROFILES_FILE AIUR_RELEASE_NODE
136
+ AIUR_PROFILES_FILE AIUR_RELEASE_NODE AIUR_INSTANCE_KEY \
137
+ AIUR_PROJECT_ROOT AIUR_PROJECT_ROOT_SOURCE
46
138
  }
47
139
 
48
140
  aiur_print_identity() {
@@ -52,6 +144,9 @@ aiur_print_identity() {
52
144
  printf 'AIUR_SESSION_PREFIX=%s\n' "$AIUR_SESSION_PREFIX"
53
145
  printf 'AIUR_PROFILES_FILE=%s\n' "$AIUR_PROFILES_FILE"
54
146
  printf 'AIUR_RELEASE_NODE=%s\n' "$AIUR_RELEASE_NODE"
147
+ printf 'AIUR_INSTANCE_KEY=%s\n' "$AIUR_INSTANCE_KEY"
148
+ printf 'AIUR_PROJECT_ROOT=%s\n' "$AIUR_PROJECT_ROOT"
149
+ printf 'AIUR_PROJECT_ROOT_SOURCE=%s\n' "$AIUR_PROJECT_ROOT_SOURCE"
55
150
  printf 'AIUR_COOKIE_FILE=%s\n' "$AIUR_COOKIE_FILE"
56
151
  }
57
152
 
@@ -172,12 +267,18 @@ build_init_cmd() {
172
267
 
173
268
  usage() {
174
269
  cat <<'EOF'
175
- Usage: aiur [--interactive] [--logs-root <path>] [--port <port>] [--host <host>] [path-to-.aiurconfig]
270
+ Usage: aiur [--interactive] [--max-agents <n>] [--logs-root <path>] [--port <port>] [--host <host>] [path-to-.aiurconfig]
176
271
  aiur init [--force] scaffold .aiurconfig (interactive setup wizard)
177
- aiur --bg start in a detached tmux session
272
+ aiur --bg start in a lean, headless detached tmux session
178
273
  aiur stop stop the running session
179
274
  aiur status show agent status
275
+ aiur agents show each agent's state + current activity
276
+ aiur alerts [--needs-attention] show structured alert feed
277
+ aiur watch [--full|--changes] [--interval <secs>] server-side status board
278
+ aiur set max-agents <n> change the concurrent-agent cap at runtime
180
279
  aiur pause <ids|--all> | resume <ids|--all>
280
+ aiur message <id> <text> send operator text to a running agent
281
+ aiur cleanup-stale [--dry-run] list/reap stale manual-smoke leftovers
181
282
  aiur --version
182
283
  EOF
183
284
  }
@@ -265,19 +366,28 @@ run_session() {
265
366
 
266
367
  init_argv_file
267
368
 
268
- # Inject the flags a bare `aiur` needs: loopback bind, interactive UI, and the
269
- # no-guardrails ack. Skip any the user already passed.
270
- local has_host=0 has_interactive=0 has_ack=0 arg
369
+ # Inject the flags a bare `aiur` needs: loopback bind, UI mode, and the
370
+ # no-guardrails ack. Skip any the user already passed. Foreground runs are
371
+ # interactive (tmux panes + dashboard); `--bg` runs lean/headless — no panes,
372
+ # no dashboard bind, no chat backfill — and is driven over the control RPC
373
+ # (status/agents/message/pause/set). `aiur --bg --interactive` opts back into
374
+ # the full interactive stack for an attachable background session.
375
+ local has_host=0 has_interactive=0 has_headless=0 has_ack=0 arg
271
376
  for arg in "$@"; do
272
377
  case "$arg" in
273
378
  --host | --host=*) has_host=1 ;;
274
379
  --interactive) has_interactive=1 ;;
380
+ --headless) has_headless=1 ;;
275
381
  --i-understand-that-this-will-be-running-without-the-usual-guardrails) has_ack=1 ;;
276
382
  esac
277
383
  done
278
384
  local injected=()
279
385
  [ "$has_host" -eq 1 ] || injected+=(--host 127.0.0.1)
280
- [ "$has_interactive" -eq 1 ] || injected+=(--interactive)
386
+ if [ "$mode" = "background" ] && [ "$has_interactive" -eq 0 ]; then
387
+ [ "$has_headless" -eq 1 ] || injected+=(--headless)
388
+ else
389
+ [ "$has_interactive" -eq 1 ] || injected+=(--interactive)
390
+ fi
281
391
  [ "$has_ack" -eq 1 ] || injected+=(--i-understand-that-this-will-be-running-without-the-usual-guardrails)
282
392
 
283
393
  write_argv "${injected[@]}" "$@"
@@ -291,12 +401,14 @@ run_session() {
291
401
  export ELIXIR_ERL_OPTIONS="${ELIXIR_ERL_OPTIONS:-} +fnu"
292
402
  fi
293
403
 
294
- local session="${AIUR_SESSION_PREFIX}-${USER:-user}-default"
295
- local socket="${AIUR_SESSION_PREFIX}-${USER:-user}"
404
+ local session="${AIUR_SESSION_PREFIX}-${USER:-user}${AIUR_INSTANCE_KEY:+-$AIUR_INSTANCE_KEY}-default"
405
+ local socket="${AIUR_SESSION_PREFIX}-${USER:-user}${AIUR_INSTANCE_KEY:+-$AIUR_INSTANCE_KEY}"
296
406
  local conf
297
407
  conf="$(resolve_tmux_conf)"
298
408
  [ -f "$conf" ] || die "tmux conf not found at $conf"
299
409
 
410
+ preflight_stale_manual_smoke
411
+
300
412
  mkdir -p "$AIUR_BG_STATE_DIR"
301
413
  printf '%s\n' "$session" >"$AIUR_BG_STATE_DIR/state"
302
414
  export AIUR_TMUX_SESSION="$session"
@@ -308,8 +420,37 @@ run_session() {
308
420
  export AIUR_SESSION_TMPFILE="${session_root}/aiur-${$}-sessions"
309
421
  : >"$AIUR_SESSION_TMPFILE"
310
422
 
423
+ # Agent pidfile: the BEAM appends one line per spawned agent (pane or headless
424
+ # os_pid) via Aiur.ProcessReaper. The BEAM-death watchdog and session_cleanup
425
+ # reap from it after the BEAM is gone — a crashed BEAM can kill nothing itself.
426
+ export AIUR_AGENT_TMPFILE="${session_root}/aiur-${$}-agents"
427
+ : >"$AIUR_AGENT_TMPFILE"
428
+
429
+ # Workspace-root handoff for the shell cwd-sweep backstop. The BEAM writes the
430
+ # resolved Aiur.Config.workspace_root() here after config loads.
431
+ export AIUR_WORKSPACE_ROOT_FILE="${session_root}/aiur-${$}-workspace-root"
432
+ : >"$AIUR_WORKSPACE_ROOT_FILE"
433
+
434
+ # Background runs persist under a known run log dir so (a) the BEAM-death
435
+ # watchdog, which outlives the BEAM, can drop a crash record next to aiur.log,
436
+ # and (b) the post-start boot capture is not lost to /tmp when the launcher
437
+ # removes its startup tempfile. Minting it here and exporting it makes the
438
+ # shell and the BEAM (Aiur.LogFile honors AIUR_LOGS_ROOT) agree on one dir.
439
+ if [ "$mode" = "background" ] && [ -z "${AIUR_LOGS_ROOT:-}" ]; then
440
+ AIUR_LOGS_ROOT="$(printf '%s/%s-%s' "$HOME/.aiur/logs" "$(date -u +%Y%m%dT%H%M%SZ)" "$$")"
441
+ export AIUR_LOGS_ROOT
442
+ fi
443
+
444
+ # Capture sink for BEAM startup (and, in background mode, the whole run's
445
+ # boot stdout/stderr). Foreground uses a throwaway tempfile; background points
446
+ # at a durable file in the run log dir. The dir/file is created lazily just
447
+ # before launch (below) so an idempotent early-return start leaves no empty dir.
311
448
  local startup_capture
312
- startup_capture="$(mktemp "${TMPDIR:-/tmp}/aiur-startup.XXXXXX")"
449
+ if [ "$mode" = "background" ] && [ -n "${AIUR_LOGS_ROOT:-}" ]; then
450
+ startup_capture="$AIUR_LOGS_ROOT/log/boot.out.log"
451
+ else
452
+ startup_capture="$(mktemp "${TMPDIR:-/tmp}/aiur-startup.XXXXXX")"
453
+ fi
313
454
 
314
455
  # Inner pane script: tmux's server may pre-exist and not inherit our env, so
315
456
  # re-export every var the BEAM needs. tee preserves a startup capture.
@@ -324,7 +465,8 @@ run_session() {
324
465
  for v in AIUR_RELEASE_DIR AIUR_ARGV_FILE RELEASE_DISTRIBUTION RELEASE_NODE \
325
466
  RELEASE_COOKIE ERL_AFLAGS ERL_EPMD_ADDRESS AIUR_NODE AIUR_ERLANG_COOKIE \
326
467
  AIUR_TMUX_SESSION AIUR_TMUX_SOCKET AIUR_TMUX_CONF AIUR_BIN \
327
- AIUR_SESSION_TMPFILE ELIXIR_ERL_OPTIONS AIUR_LOGS_ROOT AIUR_DEBUG; do
468
+ AIUR_SESSION_TMPFILE AIUR_AGENT_TMPFILE AIUR_WORKSPACE_ROOT_FILE \
469
+ ELIXIR_ERL_OPTIONS AIUR_LOGS_ROOT AIUR_OPENCODE_BRIDGE_PORT AIUR_DEBUG; do
328
470
  if [ -n "${!v:-}" ]; then printf 'export %s=%q\n' "$v" "${!v}"; fi
329
471
  done
330
472
  printf 'capture=%q\n' "$startup_capture"
@@ -341,48 +483,127 @@ run_session() {
341
483
  _session_socket="$socket" _session_name="$session" _session_conf="$conf" \
342
484
  _session_tmpfile="$AIUR_SESSION_TMPFILE" _session_capture="$startup_capture" \
343
485
  _session_argv="$argv_file" _session_release="$release_dir" _session_tmux="$tmux_bin" \
344
- _session_node="$AIUR_RELEASE_NODE"
486
+ _session_node="$AIUR_RELEASE_NODE" _session_pidfile="$AIUR_AGENT_TMPFILE" \
487
+ _session_workspace_root_file="$AIUR_WORKSPACE_ROOT_FILE"
345
488
  install_foreground_traps
346
489
  fi
347
490
 
348
- # An orphaned BEAM (its tmux session gone) can still hold the node name —
349
- # under the unified identity, even one from a different release dir. With no
350
- # live session on our socket, reap any name-holder so the launch isn't blocked
351
- # by "name seems to be in use".
491
+ # Background starts are intentionally idempotent. A prior live node should not
492
+ # fall through to tmux's opaque "duplicate session" failure, and a stale tmux
493
+ # session whose BEAM/control plane is gone should be reclaimed before retry.
494
+ if [ "$mode" = "background" ] && "$tmux_bin" -L "$socket" -f "$conf" has-session -t "$session" 2>/dev/null; then
495
+ if [ "$(probe_control_liveness)" = "up" ]; then
496
+ write_aiur_instance_record "$session" "$socket"
497
+ echo "aiur is already running in the background (tmux session ${session})." >&2
498
+ echo "Use: aiur status # inspect agents" >&2
499
+ echo "Use: aiur stop # stop it before starting a fresh session" >&2
500
+ rm -f "$startup_capture" "$argv_file" "$launcher" "$AIUR_SESSION_TMPFILE" "$AIUR_AGENT_TMPFILE" "$AIUR_WORKSPACE_ROOT_FILE" 2>/dev/null || true
501
+ return 0
502
+ fi
503
+
504
+ echo "aiur found stale tmux session ${session}; cleaning it up before restart" >&2
505
+ reap_aiur_agents "$socket" "$AIUR_AGENT_TMPFILE"
506
+ kill_beams_matching "-name ${AIUR_RELEASE_NODE}"
507
+ fi
508
+
509
+ # An orphaned BEAM (its tmux session gone) can still hold THIS instance's node
510
+ # name. With no live session on our (instance-keyed) socket, reap the name-holder
511
+ # so the launch isn't blocked by "name seems to be in use". The keyed name means
512
+ # this only ever reaps our own instance's orphan, never another live aiur.
352
513
  if ! "$tmux_bin" -L "$socket" -f "$conf" has-session -t "$session" 2>/dev/null; then
353
514
  kill_beams_matching "-name ${AIUR_RELEASE_NODE}"
354
515
  fi
355
516
 
517
+ # Transition reclaim: a keyed instance also reaps a stale beam under the legacy
518
+ # un-keyed name (aiur-$USER@127.0.0.1) — but only when no live legacy session
519
+ # exists, so a pre-fix run still in progress is never killed. (Runs on every keyed
520
+ # launch, not latched; it's a cheap best-effort sweep guarded by has-session.)
521
+ if [ -n "${AIUR_INSTANCE_KEY:-}" ]; then
522
+ local legacy_socket="${AIUR_SESSION_PREFIX}-${USER:-user}"
523
+ if ! "$tmux_bin" -L "$legacy_socket" -f "$conf" has-session -t "${legacy_socket}-default" 2>/dev/null; then
524
+ kill_beams_matching "-name aiur-${USER}@127.0.0.1"
525
+ fi
526
+ fi
527
+
528
+ # Create the capture sink now (after the idempotent early-return checks above,
529
+ # so a no-op "already running" start never mints an empty run log dir). The
530
+ # launcher's `tee -a` needs the file's parent to exist when the pane runs.
531
+ mkdir -p "$(dirname "$startup_capture")" 2>/dev/null || true
532
+ : >"$startup_capture" 2>/dev/null || true
533
+
356
534
  if ! "$tmux_bin" -L "$socket" -f "$conf" new-session -d -s "$session" \
357
535
  -x "${COLUMNS:-200}" -y "${LINES:-50}" "$inner_cmd"; then
358
536
  echo "❌ aiur failed to start; captured output:" >&2
359
537
  tail -n 30 "$startup_capture" 2>/dev/null | sed 's/^/ /' >&2 || true
360
- rm -f "$startup_capture" "$argv_file" 2>/dev/null || true
538
+ rm -f "$startup_capture" "$argv_file" "$launcher" "$AIUR_SESSION_TMPFILE" "$AIUR_AGENT_TMPFILE" "$AIUR_WORKSPACE_ROOT_FILE" 2>/dev/null || true
361
539
  exit 1
362
540
  fi
363
541
 
364
- # Grace window: surface a boot crash instead of attaching to nothing.
365
- local tick
366
- for ((tick = 0; tick < 30; tick++)); do
367
- if ! "$tmux_bin" -L "$socket" -f "$conf" has-session -t "$session" 2>/dev/null; then
368
- echo "❌ aiur exited during startup; captured output:" >&2
369
- tail -n 30 "$startup_capture" 2>/dev/null | sed 's/^/ /' >&2 || true
370
- [ "$mode" = "foreground" ] && return 1
371
- rm -f "$startup_capture" "$argv_file" 2>/dev/null || true
372
- exit 1
542
+ # Title the agent-list pane (the only pane in the fresh session). The conf's
543
+ # `pane-border-status`/`pane-border-format` render it; PaneManager titles the
544
+ # chat panes it opens. Best-effort — a missing title just shows the default.
545
+ "$tmux_bin" -L "$socket" -f "$conf" select-pane -t "$session" -T "AIUR Agents" 2>/dev/null || true
546
+
547
+ # Grace window: surface a boot crash instead of attaching to nothing. Prove
548
+ # the control plane can answer before declaring startup usable: background
549
+ # control commands depend on that RPC path, and foreground automation must not
550
+ # treat a pre-TUI application crash as a successful `tmux attach`.
551
+ local require_control=1
552
+ if ! wait_for_session_startup "$tmux_bin" "$socket" "$conf" "$session" "$startup_capture" "$require_control"; then
553
+ if [ "$mode" = "background" ]; then
554
+ "$tmux_bin" -L "$socket" -f "$conf" kill-session -t "$session" 2>/dev/null || true
555
+ reap_aiur_agents "$socket" "$AIUR_AGENT_TMPFILE"
556
+ kill_beams_matching "-name ${AIUR_RELEASE_NODE}"
373
557
  fi
374
- sleep 0.1
375
- done
558
+ [ "$mode" = "foreground" ] && return 1
559
+ rm -f "$startup_capture" "$argv_file" "$launcher" "$AIUR_SESSION_TMPFILE" "$AIUR_AGENT_TMPFILE" "$AIUR_WORKSPACE_ROOT_FILE" 2>/dev/null || true
560
+ exit 1
561
+ fi
562
+
563
+ write_aiur_instance_record "$session" "$socket"
564
+
565
+ if [ "$mode" = "foreground" ]; then
566
+ echo "aiur foreground tmux socket ${socket}, session ${session}" >&2
567
+ fi
376
568
 
377
569
  if [ "$mode" = "background" ]; then
378
- echo "aiur started in the background (tmux session ${session}). Attach with: aiur" >&2
379
- rm -f "$startup_capture" "$argv_file" 2>/dev/null || true
570
+ # Fresh run: drop any stale crash/stop state from a prior dead instance so
571
+ # `status` doesn't report a phantom orphan and the watchdog starts clean.
572
+ rm -f "$(aiur_stop_sentinel_path)" "$(aiur_crash_marker_path)" 2>/dev/null || true
573
+
574
+ local background_watchdog_pid
575
+ background_watchdog_pid="$(start_beam_death_watchdog \
576
+ "-name ${AIUR_RELEASE_NODE}" "$socket" "$AIUR_AGENT_TMPFILE" 1 1 \
577
+ "$AIUR_RELEASE_NODE" "${AIUR_LOGS_ROOT:-}" \
578
+ "$(aiur_stop_sentinel_path)" "$(aiur_crash_marker_path)" "$AIUR_WORKSPACE_ROOT_FILE")"
579
+ disown "$background_watchdog_pid" 2>/dev/null || true
580
+ echo "aiur started in the background (tmux socket ${socket}, session ${session}). Attach with: aiur" >&2
581
+ # Keep $startup_capture (boot.out.log) for the run's lifetime; only the
582
+ # transient argv file is no longer needed.
583
+ rm -f "$argv_file" 2>/dev/null || true
380
584
  return 0
381
585
  fi
382
586
 
587
+ # Arm the BEAM-death watchdog before attaching. If the BEAM crashes
588
+ # (:emfile) mid-run, agent windows keep the orphaned session alive so the
589
+ # `tmux attach` below never returns and the EXIT trap never fires — the
590
+ # watchdog is the external reaper that survives the dead BEAM, kill-servers
591
+ # the session, and unblocks the attach. It polls the node name rather than a
592
+ # captured pid, so another Aiur instance from the same release cannot hold it open.
593
+ _session_watchdog_pid="$(start_beam_death_watchdog \
594
+ "-name ${AIUR_RELEASE_NODE}" "$socket" "$AIUR_AGENT_TMPFILE" 1 0 \
595
+ "" "" "" "" "$AIUR_WORKSPACE_ROOT_FILE")"
596
+
383
597
  # Foreground: attach the UI. Do not exec — that would drop the teardown trap.
384
- "$tmux_bin" -L "$socket" -f "$conf" attach -t "$session" 2> >(grep -v -F "[server exited]" >&2)
385
- return $?
598
+ # Avoid process substitution here: some sandboxed non-TTY launchers reject
599
+ # opening /dev/fd/* during the real manual-test wrapper path.
600
+ local attach_stderr attach_code
601
+ attach_stderr="$(mktemp "${TMPDIR:-/tmp}/aiur-attach-stderr.XXXXXX")"
602
+ "$tmux_bin" -L "$socket" -f "$conf" attach -t "$session" 2>"$attach_stderr"
603
+ attach_code=$?
604
+ grep -v -F "[server exited]" "$attach_stderr" >&2 || true
605
+ rm -f "$attach_stderr" 2>/dev/null || true
606
+ return "$attach_code"
386
607
  }
387
608
 
388
609
  resolve_tmux_conf() {
@@ -417,37 +638,655 @@ kill_beams_matching() {
417
638
  for pid in $pids; do kill -KILL "$pid" 2>/dev/null || true; done
418
639
  }
419
640
 
641
+ pid_owner() {
642
+ ps -p "$1" -o user= 2>/dev/null | awk '{print $1}' || true
643
+ }
644
+
645
+ pid_command() {
646
+ ps -p "$1" -o command= 2>/dev/null || true
647
+ }
648
+
649
+ pid_ppid() {
650
+ ps -p "$1" -o ppid= 2>/dev/null | awk '{print $1}' || true
651
+ }
652
+
653
+ pid_cwd() {
654
+ local pid="$1" cwd
655
+ if [ -e "/proc/$pid/cwd" ]; then
656
+ readlink "/proc/$pid/cwd" 2>/dev/null || true
657
+ return 0
658
+ fi
659
+ if command -v lsof >/dev/null 2>&1; then
660
+ cwd="$(lsof -a -p "$pid" -d cwd -Fn 2>/dev/null | sed -n 's/^n//p' | head -n 1 || true)"
661
+ [ -n "$cwd" ] && printf '%s\n' "$cwd"
662
+ fi
663
+ }
664
+
665
+ kill_pid_with_escalation() {
666
+ local pid="$1" waited=0
667
+ [ -n "$pid" ] || return 0
668
+ kill -TERM "$pid" 2>/dev/null || true
669
+ while [ "$waited" -lt 20 ]; do
670
+ kill -0 "$pid" 2>/dev/null || return 0
671
+ sleep 0.1
672
+ waited=$((waited + 1))
673
+ done
674
+ kill -KILL "$pid" 2>/dev/null || true
675
+ }
676
+
677
+ aiur_node_from_command() {
678
+ local cmd="$1" node
679
+ node="$(printf '%s\n' "$cmd" | sed -n "s/.*--name[[:space:]]\\(aiur-${USER}[-A-Za-z0-9_]*@127\\.0\\.0\\.1\\).*/\\1/p" | head -n 1)"
680
+ [ -n "$node" ] && printf '%s\n' "$node"
681
+ }
682
+
683
+ aiur_release_root_from_command() {
684
+ local cmd="$1"
685
+ if [[ "$cmd" =~ --boot-var[[:space:]]+RELEASE_LIB[[:space:]]+([^[:space:]]+)/lib ]]; then
686
+ printf '%s\n' "${BASH_REMATCH[1]}"
687
+ return 0
688
+ fi
689
+ if [[ "$cmd" =~ ([^[:space:]]*/src/_build/dev/rel/aiur)/releases/ ]]; then
690
+ printf '%s\n' "${BASH_REMATCH[1]}"
691
+ return 0
692
+ fi
693
+ if [[ "$cmd" =~ ([^[:space:]]*/release)/releases/ ]]; then
694
+ printf '%s\n' "${BASH_REMATCH[1]}"
695
+ return 0
696
+ fi
697
+ return 1
698
+ }
699
+
700
+ workspace_root_from_aiur_release_root() {
701
+ local release_root="${1%/}"
702
+ case "$release_root" in
703
+ */src/_build/dev/rel/aiur)
704
+ printf '%s\n' "${release_root%/src/_build/dev/rel/aiur}"
705
+ ;;
706
+ *)
707
+ printf '%s\n' "$release_root"
708
+ ;;
709
+ esac
710
+ }
711
+
712
+ aiur_node_tmux_session_alive() {
713
+ local node="$1" short tmux_bin
714
+ short="${node%@*}"
715
+ tmux_bin="$(command -v tmux || true)"
716
+ [ -n "$tmux_bin" ] || return 1
717
+ "$tmux_bin" -L "$short" has-session -t "${short}-default" 2>/dev/null
718
+ }
719
+
720
+ stale_manual_smoke_beam_inventory() {
721
+ local pid cmd node release_root workspace_root owner
722
+ aiur_resolve_identity
723
+
724
+ for pid in $(pgrep -f -- "beam\\.smp.*--name aiur-${USER}.*@127\\.0\\.0\\.1" 2>/dev/null || true); do
725
+ [ -n "$pid" ] || continue
726
+ owner="$(pid_owner "$pid")"
727
+ [ "$owner" = "$USER" ] || continue
728
+
729
+ cmd="$(pid_command "$pid")"
730
+ node="$(aiur_node_from_command "$cmd")"
731
+ [ -n "$node" ] || continue
732
+
733
+ release_root="$(aiur_release_root_from_command "$cmd" || true)"
734
+ [ -n "$release_root" ] || continue
735
+ workspace_root="$(workspace_root_from_aiur_release_root "$release_root")"
736
+
737
+ # Scope this broad cleanup to stale issue/manual-smoke workspaces. The
738
+ # current instance cleanup path handles the active daemon's own node.
739
+ case "$workspace_root" in
740
+ */aiur-workspaces/*) ;;
741
+ *) continue ;;
742
+ esac
743
+
744
+ # A matching live aiur tmux session means this node may still be in use.
745
+ aiur_node_tmux_session_alive "$node" && continue
746
+
747
+ printf '%s\t%s\t%s\t%s\n' "$pid" "$node" "$workspace_root" "$release_root"
748
+ done
749
+ }
750
+
751
+ manual_smoke_wrapper_tmux_sockets() {
752
+ local tmux_bin sockdir path name panes current pane_path start manual_context all_sleep
753
+ tmux_bin="$(command -v tmux || true)"
754
+ [ -n "$tmux_bin" ] || return 0
755
+ sockdir="${TMUX_TMPDIR:-/tmp}/tmux-$(id -u)"
756
+ [ -d "$sockdir" ] || return 0
757
+
758
+ for path in "$sockdir"/*; do
759
+ [ -S "$path" ] || continue
760
+ name="${path##*/}"
761
+ case "$name" in
762
+ codex-driver-* | codex-aiur* | codex-[0-9]* | claude-driver*) ;;
763
+ *) continue ;;
764
+ esac
765
+ panes="$("$tmux_bin" -L "$name" list-panes -a -F '#{pane_current_command} #{pane_current_path} #{pane_start_command}' 2>/dev/null || true)"
766
+ [ -n "$panes" ] || continue
767
+ manual_context=0
768
+ all_sleep=1
769
+ while IFS=$'\t' read -r current pane_path start; do
770
+ case "$start" in
771
+ *aiurdev\ --test* | *scripts/aiurdev\ --test*) manual_context=1 ;;
772
+ esac
773
+ [ "$current" = "sleep" ] || all_sleep=0
774
+ done <<<"$panes"
775
+ [ "$manual_context" -eq 1 ] && [ "$all_sleep" -eq 1 ] && printf '%s\n' "$name"
776
+ done
777
+ }
778
+
779
+ orphaned_opencode_attach_inventory() {
780
+ local pid owner ppid cmd cwd
781
+ for pid in $(pgrep -f -- "opencode attach http://127\\.0\\.0\\.1:" 2>/dev/null || true); do
782
+ owner="$(pid_owner "$pid")"
783
+ [ "$owner" = "$USER" ] || continue
784
+ ppid="$(pid_ppid "$pid")"
785
+ [ "$ppid" = "1" ] || continue
786
+ cwd="$(pid_cwd "$pid")"
787
+ case "$cwd" in
788
+ */aiur-workspaces/*) ;;
789
+ *) continue ;;
790
+ esac
791
+ cmd="$(pid_command "$pid")"
792
+ case "$cmd" in
793
+ *"opencode attach http://127.0.0.1:"*) printf '%s\t%s\t%s\n' "$pid" "$cwd" "$cmd" ;;
794
+ esac
795
+ done
796
+ }
797
+
798
+ report_stale_manual_smoke() {
799
+ local found=0 pid node workspace_root release_root socket cwd cmd
800
+
801
+ while IFS=$'\t' read -r pid node workspace_root release_root; do
802
+ [ -n "$pid" ] || continue
803
+ if [ "$found" -eq 0 ]; then
804
+ echo "aiur: stale manual-smoke leftovers detected:" >&2
805
+ found=1
806
+ fi
807
+ echo " BEAM pid=${pid} node=${node} workspace=${workspace_root} release=${release_root}" >&2
808
+ done < <(stale_manual_smoke_beam_inventory)
809
+
810
+ while IFS= read -r socket; do
811
+ [ -n "$socket" ] || continue
812
+ if [ "$found" -eq 0 ]; then
813
+ echo "aiur: stale manual-smoke leftovers detected:" >&2
814
+ found=1
815
+ fi
816
+ echo " wrapper tmux socket=${socket}" >&2
817
+ done < <(manual_smoke_wrapper_tmux_sockets)
818
+
819
+ while IFS=$'\t' read -r pid cwd cmd; do
820
+ [ -n "$pid" ] || continue
821
+ if [ "$found" -eq 0 ]; then
822
+ echo "aiur: stale manual-smoke leftovers detected:" >&2
823
+ found=1
824
+ fi
825
+ echo " orphan opencode attach pid=${pid} workspace=${cwd}" >&2
826
+ done < <(orphaned_opencode_attach_inventory)
827
+
828
+ if [ "$found" -eq 1 ]; then
829
+ echo "aiur: run 'aiur cleanup-stale' to TERM/KILL only these same-user Aiur leftovers." >&2
830
+ fi
831
+
832
+ return "$found"
833
+ }
834
+
835
+ preflight_stale_manual_smoke() {
836
+ report_stale_manual_smoke >/dev/null || true
837
+ }
838
+
839
+ reap_stale_manual_smoke() {
840
+ local include_wrappers="${1:-0}"
841
+ local pid node workspace_root release_root socket cwd cmd tmux_bin reaped=0
842
+
843
+ while IFS=$'\t' read -r pid node workspace_root release_root; do
844
+ [ -n "$pid" ] || continue
845
+ echo "aiur: reaping stale BEAM pid=${pid} node=${node} workspace=${workspace_root}" >&2
846
+ kill_pid_with_escalation "$pid"
847
+ reaped=$((reaped + 1))
848
+ done < <(stale_manual_smoke_beam_inventory)
849
+
850
+ tmux_bin="$(command -v tmux || true)"
851
+ if [ "$include_wrappers" = "1" ] && [ -n "$tmux_bin" ]; then
852
+ while IFS= read -r socket; do
853
+ [ -n "$socket" ] || continue
854
+ echo "aiur: reaping stale wrapper tmux socket=${socket}" >&2
855
+ "$tmux_bin" -L "$socket" kill-server 2>/dev/null || true
856
+ reaped=$((reaped + 1))
857
+ done < <(manual_smoke_wrapper_tmux_sockets)
858
+ fi
859
+
860
+ while IFS=$'\t' read -r pid cwd cmd; do
861
+ [ -n "$pid" ] || continue
862
+ echo "aiur: reaping orphan opencode attach pid=${pid}" >&2
863
+ kill_pid_with_escalation "$pid"
864
+ reaped=$((reaped + 1))
865
+ done < <(orphaned_opencode_attach_inventory)
866
+
867
+ if [ "$reaped" -eq 0 ]; then
868
+ echo "aiur: no stale manual-smoke leftovers found" >&2
869
+ fi
870
+ }
871
+
872
+ # Echo $1 and every descendant pid, depth-first. Mac-safe (`pgrep -P`, no
873
+ # /proc), mirroring Aiur.RemoteControl.collect_descendants so the launcher reaps
874
+ # the same tree the BEAM-side reaper would. The recorded agent pid is a bash
875
+ # `-lc` wrapper; the model process (claude --print / codex) is its child and
876
+ # reparents to init if only the wrapper is signalled — orphan-and-survive is
877
+ # exactly the bug — so the whole tree must be collected before any kill lands.
878
+ agent_pid_tree() {
879
+ local root="$1" child
880
+ printf '%s\n' "$root"
881
+ for child in $(pgrep -P "$root" 2>/dev/null || true); do
882
+ agent_pid_tree "$child"
883
+ done
884
+ }
885
+
886
+ # pid-reuse guard: succeeds when pid $1 is alive AND its command still contains
887
+ # the comm substring $2 the BEAM recorded. An empty comm kills unconditionally.
888
+ # Mirrors the BEAM-side cmdline guard (Aiur.ProcessReaper) but Mac-safe via
889
+ # `ps -o command=`, so a recycled pid whose command no longer matches is spared.
890
+ agent_pid_matches() {
891
+ local pid="$1" comm="$2" cmd
892
+ kill -0 "$pid" 2>/dev/null || return 1
893
+ [ -n "$comm" ] || return 0
894
+ cmd="$(ps -p "$pid" -o command= 2>/dev/null || true)"
895
+ case "$cmd" in *"$comm"*) return 0 ;; *) return 1 ;; esac
896
+ }
897
+
898
+ # Reap every aiur agent the BEAM can no longer reap itself.
899
+ #
900
+ # $1 socket aiur tmux socket (-L); kill-server nukes all panes (may be empty)
901
+ # $2 pidfile AIUR_AGENT_TMPFILE (BEAM-written agent refs; may be empty/missing)
902
+ #
903
+ # tmux runs on aiur's PRIVATE socket (-L aiur-$USER), so kill-server tears down
904
+ # every REPL/chat pane agent in one shot AND leaves no live aiur tmux server —
905
+ # never touching the operator's own default tmux. Headless agents (claude/codex
906
+ # app-servers spawned via Port) are bare OS processes that reparent to init on a
907
+ # BEAM crash; kill-server can't see them, so they're reaped from the pidfile by
908
+ # process tree, comm-guarded against pid reuse. Idempotent.
909
+ reap_aiur_agents() {
910
+ local socket="$1" pidfile="${2:-}"
911
+ local tmux_bin
912
+ tmux_bin="$(command -v tmux || true)"
913
+
914
+ if [ -n "$tmux_bin" ] && [ -n "$socket" ]; then
915
+ "$tmux_bin" -L "$socket" kill-server 2>/dev/null || true
916
+ fi
917
+
918
+ [ -n "$pidfile" ] && [ -r "$pidfile" ] || return 0
919
+
920
+ # Snapshot the full process tree of every still-matching headless agent before
921
+ # signalling, so descendants that reparent mid-reap are already on the list.
922
+ local kind pid comm tree=() p
923
+ while read -r kind pid comm; do
924
+ [ "$kind" = "pid" ] && [ -n "$pid" ] || continue
925
+ agent_pid_matches "$pid" "$comm" || continue
926
+ while IFS= read -r p; do tree+=("$p"); done < <(agent_pid_tree "$pid")
927
+ done <"$pidfile"
928
+
929
+ [ "${#tree[@]}" -gt 0 ] || return 0
930
+
931
+ for p in "${tree[@]}"; do kill -TERM "$p" 2>/dev/null || true; done
932
+ local waited=0
933
+ while [ "$waited" -lt 20 ]; do
934
+ local any=0
935
+ for p in "${tree[@]}"; do kill -0 "$p" 2>/dev/null && { any=1; break; }; done
936
+ [ "$any" -eq 0 ] && break
937
+ sleep 0.1
938
+ waited=$((waited + 1))
939
+ done
940
+ for p in "${tree[@]}"; do kill -KILL "$p" 2>/dev/null || true; done
941
+ }
942
+
943
+ # Stable, per-instance state paths the background BEAM-death machinery shares
944
+ # across `run`, `stop`, and `status`. Keyed by the (instance-keyed) release node
945
+ # rather than the tmux socket so `status` — which resolves the node but not the
946
+ # socket — agrees on the same files. AIUR_BG_STATE_DIR and AIUR_RELEASE_NODE are
947
+ # resolved by aiur_resolve_identity, which every one of those paths calls first.
948
+ aiur_state_slug() {
949
+ printf '%s' "${AIUR_RELEASE_NODE:-aiur}" | tr -c 'A-Za-z0-9._-' '_'
950
+ }
951
+
952
+ aiur_instances_dir() {
953
+ printf '%s/instances' "${AIUR_BG_STATE_DIR:?}"
954
+ }
955
+
956
+ aiur_instance_record_path() {
957
+ printf '%s/%s.instance' "$(aiur_instances_dir)" "$(aiur_state_slug)"
958
+ }
959
+
960
+ write_aiur_instance_record() {
961
+ local session="$1" socket="$2" record_dir record tmp root
962
+ record_dir="$(aiur_instances_dir)"
963
+ record="$(aiur_instance_record_path)"
964
+ root="$(canonical_workspace_root "${AIUR_PROJECT_ROOT:-}")"
965
+ mkdir -p "$record_dir" 2>/dev/null || return 0
966
+ tmp="$(mktemp "$record_dir/.${AIUR_INSTANCE_KEY:-aiur}.XXXXXX" 2>/dev/null)" || return 0
967
+ {
968
+ printf 'AIUR_RECORD_NODE=%q\n' "$AIUR_RELEASE_NODE"
969
+ printf 'AIUR_RECORD_INSTANCE_KEY=%q\n' "$AIUR_INSTANCE_KEY"
970
+ printf 'AIUR_RECORD_SESSION=%q\n' "$session"
971
+ printf 'AIUR_RECORD_SOCKET=%q\n' "$socket"
972
+ printf 'AIUR_RECORD_PROJECT_ROOT=%q\n' "$root"
973
+ printf 'AIUR_RECORD_PROJECT_ROOT_SOURCE=%q\n' "${AIUR_PROJECT_ROOT_SOURCE:-}"
974
+ printf 'AIUR_RECORD_WRITTEN_AT=%q\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || true)"
975
+ } >"$tmp" || { rm -f "$tmp" 2>/dev/null || true; return 0; }
976
+ chmod 0600 "$tmp" 2>/dev/null || true
977
+ mv "$tmp" "$record" 2>/dev/null || rm -f "$tmp" 2>/dev/null || true
978
+ }
979
+
980
+ # Marker `status` reads to tell "daemon crashed, agents may be orphaned" apart
981
+ # from "nothing was ever running". Written by the watchdog on an unexpected exit,
982
+ # cleared on a clean stop and on the next successful background start.
983
+ aiur_crash_marker_path() {
984
+ printf '%s/%s.last-crash' "${AIUR_BG_STATE_DIR:?}" "$(aiur_state_slug)"
985
+ }
986
+
987
+ # Sentinel `stop` drops before killing the BEAM so the watchdog knows the exit
988
+ # was intentional and does not record a false crash.
989
+ aiur_stop_sentinel_path() {
990
+ printf '%s/%s.stopping' "${AIUR_BG_STATE_DIR:?}" "$(aiur_state_slug)"
991
+ }
992
+
993
+ # Write a durable record that the background BEAM exited unexpectedly. Two sinks:
994
+ # the run log dir (full record next to aiur.log, for forensics) and the stable
995
+ # per-instance marker (for `status` to surface). Best-effort throughout — this
996
+ # runs in the disowned watchdog after the BEAM is already gone, so it must never
997
+ # fail loudly or block the reap that follows.
998
+ record_beam_crash() {
999
+ local node="$1" run_log_dir="$2" marker="$3"
1000
+ local ts boot_tail=""
1001
+ ts="$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || true)"
1002
+ if [ -n "$run_log_dir" ] && [ -r "$run_log_dir/log/boot.out.log" ]; then
1003
+ boot_tail="$(tail -n 20 "$run_log_dir/log/boot.out.log" 2>/dev/null || true)"
1004
+ fi
1005
+
1006
+ local body
1007
+ body="$(
1008
+ printf 'aiur background BEAM exited unexpectedly\n'
1009
+ printf 'timestamp: %s\n' "${ts:-unknown}"
1010
+ printf 'node: %s\n' "${node:-unknown}"
1011
+ printf 'run_log_dir: %s\n' "${run_log_dir:-unknown}"
1012
+ printf 'detected_by: background BEAM-death watchdog (no clean stop sentinel)\n'
1013
+ if [ -n "$boot_tail" ]; then
1014
+ printf -- '--- last 20 lines of boot.out.log ---\n%s\n' "$boot_tail"
1015
+ fi
1016
+ )"
1017
+
1018
+ if [ -n "$run_log_dir" ]; then
1019
+ mkdir -p "$run_log_dir/log" 2>/dev/null || true
1020
+ printf '%s\n' "$body" >>"$run_log_dir/log/aiur.crash" 2>/dev/null || true
1021
+ fi
1022
+ if [ -n "$marker" ]; then
1023
+ printf '%s\n' "$body" >"$marker" 2>/dev/null || true
1024
+ fi
1025
+ }
1026
+
1027
+ canonical_workspace_root() {
1028
+ local root="$1"
1029
+ if [ -d "$root" ]; then
1030
+ (cd "$root" 2>/dev/null && pwd -P) || printf '%s\n' "$root"
1031
+ else
1032
+ printf '%s\n' "$root"
1033
+ fi
1034
+ }
1035
+
1036
+ workspace_root_is_shallow() {
1037
+ local root="${1%/}" trimmed
1038
+ trimmed="${root#/}"
1039
+ [ -n "$trimmed" ] || return 0
1040
+ case "$trimmed" in
1041
+ */*) return 1 ;;
1042
+ *) return 0 ;;
1043
+ esac
1044
+ }
1045
+
1046
+ workspace_cwd_pids() {
1047
+ local root="${1%/}" proc_dir="${2:-/proc}" entry pid cwd
1048
+ [ -d "$proc_dir" ] || return 0
1049
+
1050
+ for entry in "$proc_dir"/[0-9]*; do
1051
+ [ -d "$entry" ] || continue
1052
+ pid="${entry##*/}"
1053
+ cwd="$(readlink "$entry/cwd" 2>/dev/null || true)"
1054
+ [ -n "$cwd" ] || continue
1055
+ cwd="${cwd%/}"
1056
+ if [ "$cwd" != "$root" ] && [[ "$cwd/" == "$root/"* ]]; then
1057
+ printf '%s\n' "$pid"
1058
+ fi
1059
+ done
1060
+ }
1061
+
1062
+ shell_protected_pid_list() {
1063
+ local p
1064
+ printf '%s\n' "$$"
1065
+ [ "${BASHPID:-$$}" = "$$" ] || printf '%s\n' "$BASHPID"
1066
+ [ -n "${PPID:-}" ] && printf '%s\n' "$PPID"
1067
+ for p in $(agent_pid_tree "$$" 2>/dev/null || true); do
1068
+ printf '%s\n' "$p"
1069
+ done
1070
+ }
1071
+
1072
+ reap_workspace_cwd_agents() {
1073
+ local root="${1:-}"
1074
+ [ -n "$root" ] || return 0
1075
+ [ -d /proc ] || return 0
1076
+
1077
+ root="$(canonical_workspace_root "$root")"
1078
+ if workspace_root_is_shallow "$root"; then
1079
+ echo "⚠️ refusing shallow workspace cwd sweep root: $root" >&2
1080
+ return 0
1081
+ fi
1082
+
1083
+ local max_sweeps="${AIUR_WORKSPACE_REAP_SWEEPS:-6}"
1084
+ case "$max_sweeps" in '' | *[!0-9]*) max_sweeps=6 ;; esac
1085
+ [ "$max_sweeps" -gt 0 ] || return 0
1086
+
1087
+ local protected pids=() filtered=() pid p sweep waited any
1088
+ protected=" $(shell_protected_pid_list | tr '\n' ' ') "
1089
+
1090
+ sweep=0
1091
+ while [ "$sweep" -lt "$max_sweeps" ]; do
1092
+ mapfile -t pids < <(workspace_cwd_pids "$root" /proc)
1093
+ filtered=()
1094
+ for pid in "${pids[@]}"; do
1095
+ case "$protected" in *" $pid "*) continue ;; esac
1096
+ kill -0 "$pid" 2>/dev/null && filtered+=("$pid")
1097
+ done
1098
+
1099
+ [ "${#filtered[@]}" -gt 0 ] || return 0
1100
+
1101
+ for p in "${filtered[@]}"; do kill -TERM "$p" 2>/dev/null || true; done
1102
+ waited=0
1103
+ while [ "$waited" -lt 10 ]; do
1104
+ any=0
1105
+ for p in "${filtered[@]}"; do kill -0 "$p" 2>/dev/null && { any=1; break; }; done
1106
+ [ "$any" -eq 0 ] && break
1107
+ sleep 0.1
1108
+ waited=$((waited + 1))
1109
+ done
1110
+ for p in "${filtered[@]}"; do kill -KILL "$p" 2>/dev/null || true; done
1111
+
1112
+ sweep=$((sweep + 1))
1113
+ sleep 0.1
1114
+ done
1115
+ }
1116
+
1117
+ reap_workspace_cwd_from_file() {
1118
+ local root_file="${1:-}" root
1119
+ [ -n "$root_file" ] && [ -s "$root_file" ] || return 0
1120
+ IFS= read -r root <"$root_file" || root=""
1121
+ reap_workspace_cwd_agents "$root"
1122
+ }
1123
+
1124
+ current_workspace_root() {
1125
+ resolve_release
1126
+ prepare_distribution >/dev/null 2>&1 || return 1
1127
+
1128
+ local output status line
1129
+ set +e
1130
+ output="$("$release_bin" rpc "IO.puts(Aiur.Config.workspace_root())" 2>/dev/null)"
1131
+ status=$?
1132
+ set -e
1133
+ [ "$status" -eq 0 ] || return 1
1134
+
1135
+ while IFS= read -r line; do
1136
+ case "$line" in
1137
+ "" | :ok) continue ;;
1138
+ *) printf '%s\n' "$line"; return 0 ;;
1139
+ esac
1140
+ done <<<"$output"
1141
+ return 1
1142
+ }
1143
+
1144
+ # Background watchdog that survives the BEAM. Polls for the release BEAM by
1145
+ # command pattern and, once it has SEEN the BEAM and then the BEAM disappears
1146
+ # (orderly halt OR :emfile-style crash), reaps every agent. Polling the pattern
1147
+ # rather than a captured pid avoids two failure modes: a recycled BEAM pid the
1148
+ # watchdog would poll forever, and an empty pid at arm time that would silently
1149
+ # disarm the only crash reaper. The `seen` latch prevents a startup race from
1150
+ # reaping before the BEAM has come up. kill-server collapses the orphaned tmux
1151
+ # session, which returns the foreground `tmux attach` so the EXIT trap
1152
+ # (session_cleanup) runs its idempotent reap too.
1153
+ #
1154
+ # $1 beam_pattern $2 socket $3 pidfile $4 interval_s $5 initial_seen
1155
+ # $6 node $7 run_log_dir $8 stop_sentinel $9 crash_marker
1156
+ # $10 workspace_root_file
1157
+ # Args 6-9 arm crash recording (background mode). Omit them (foreground) and the
1158
+ # watchdog only reaps — a foreground BEAM death is already visible at the UI.
1159
+ # Prints the watchdog's own pid so the caller can kill it on a clean teardown.
1160
+ start_beam_death_watchdog() {
1161
+ local beam_pattern="$1" socket="$2" pidfile="$3" interval="${4:-1}" initial_seen="${5:-0}"
1162
+ local node="${6:-}" run_log_dir="${7:-}" stop_sentinel="${8:-}" crash_marker="${9:-}" workspace_root_file="${10:-}"
1163
+ # Redirect the subshell's stdout so a command-substitution caller
1164
+ # (`pid=$(start_beam_death_watchdog ...)`) returns immediately instead of
1165
+ # blocking on the still-open pipe until the watchdog finishes.
1166
+ (
1167
+ seen="$initial_seen"
1168
+ while :; do
1169
+ if pgrep -f -- "$beam_pattern" >/dev/null 2>&1; then
1170
+ seen=1
1171
+ elif [ "$seen" = 1 ]; then
1172
+ break
1173
+ fi
1174
+ sleep "$interval"
1175
+ done
1176
+ # The BEAM we had seen is gone. A clean `aiur stop` drops the sentinel first;
1177
+ # its presence means an intentional exit (consume it, no crash record).
1178
+ # Anything else is an unexpected exit worth a durable record before reaping.
1179
+ if [ -n "$stop_sentinel" ] && [ -f "$stop_sentinel" ]; then
1180
+ rm -f "$stop_sentinel" 2>/dev/null || true
1181
+ elif [ -n "$crash_marker" ] || [ -n "$run_log_dir" ]; then
1182
+ record_beam_crash "$node" "$run_log_dir" "$crash_marker"
1183
+ fi
1184
+ reap_aiur_agents "$socket" "$pidfile"
1185
+ reap_workspace_cwd_from_file "$workspace_root_file"
1186
+ ) >/dev/null 2>&1 &
1187
+ printf '%s\n' "$!"
1188
+ }
1189
+
1190
+ probe_control_liveness() {
1191
+ local expression output status
1192
+ expression='case Process.whereis(Aiur.Orchestrator) do pid when is_pid(pid) -> case Aiur.Orchestrator.status(Aiur.Orchestrator, 100) do statuses when is_list(statuses) -> IO.puts("__AIUR_CONTROL_READY__"); _ -> IO.puts("__AIUR_CONTROL_NOT_READY__") end; _ -> IO.puts("__AIUR_CONTROL_NOT_READY__") end'
1193
+
1194
+ set +e
1195
+ output="$("$release_bin" rpc "$expression" 2>&1)"
1196
+ status=$?
1197
+ set -e
1198
+
1199
+ if [ "$status" -eq 0 ] && [[ "$output" == *"__AIUR_CONTROL_READY__"* ]]; then
1200
+ printf 'up'
1201
+ else
1202
+ printf 'down'
1203
+ fi
1204
+ }
1205
+
1206
+ wait_for_session_startup() {
1207
+ local tmux_bin="$1" socket="$2" conf="$3" session="$4" startup_capture="$5" require_control="$6"
1208
+ local max_ticks="${AIUR_TMUX_GRACE_TICKS:-30}" tick control_state
1209
+ if [ "$require_control" = "1" ]; then
1210
+ max_ticks="${AIUR_NODE_GRACE_TICKS:-100}"
1211
+ fi
1212
+
1213
+ for ((tick = 0; tick < max_ticks; tick++)); do
1214
+ if ! "$tmux_bin" -L "$socket" -f "$conf" has-session -t "$session" 2>/dev/null; then
1215
+ echo "❌ aiur exited during startup; captured output:" >&2
1216
+ tail -n 30 "$startup_capture" 2>/dev/null | sed 's/^/ /' >&2 || true
1217
+ return 1
1218
+ fi
1219
+
1220
+ if [ "$require_control" != "1" ]; then
1221
+ sleep 0.1
1222
+ continue
1223
+ fi
1224
+
1225
+ control_state="$(probe_control_liveness)"
1226
+ if [ "$control_state" = "up" ]; then
1227
+ return 0
1228
+ fi
1229
+
1230
+ sleep 0.1
1231
+ done
1232
+
1233
+ if [ "$require_control" = "1" ]; then
1234
+ echo "❌ aiur control plane did not become ready at ${RELEASE_NODE:-unknown} during startup; captured output:" >&2
1235
+ tail -n 30 "$startup_capture" 2>/dev/null | sed 's/^/ /' >&2 || true
1236
+ return 1
1237
+ fi
1238
+
1239
+ return 0
1240
+ }
1241
+
420
1242
  # Foreground teardown: kill the session + BEAM and reap opencode sessions on exit.
421
1243
  _session_socket="" _session_name="" _session_conf="" _session_tmpfile=""
422
1244
  _session_capture="" _session_argv="" _session_release="" _session_tmux="" _session_node=""
1245
+ _session_pidfile="" _session_watchdog_pid="" _session_workspace_root_file=""
423
1246
  _cleanup_ran=0
424
1247
  session_cleanup() {
425
1248
  [ "$_cleanup_ran" = 1 ] && return 0
426
1249
  _cleanup_ran=1
427
1250
  local code=$?
428
1251
 
429
- if [ -n "$_session_tmux" ]; then
430
- "$_session_tmux" -L "$_session_socket" -f "$_session_conf" kill-session -t "$_session_name" 2>/dev/null || true
431
- fi
1252
+ # Stop the BEAM-death watchdog: cleanup is running, so it has no work left and
1253
+ # a lingering poller would outlive this teardown.
1254
+ [ -n "$_session_watchdog_pid" ] && kill "$_session_watchdog_pid" 2>/dev/null || true
432
1255
 
433
- local beam_pids pid waited=0
434
- beam_pids="$(pgrep -f "$_session_release/.*erts.*beam.smp" 2>/dev/null || true)"
435
- if [ -n "$beam_pids" ]; then
436
- for pid in $beam_pids; do kill -TERM "$pid" 2>/dev/null || true; done
437
- while [ $waited -lt 30 ]; do
438
- beam_pids="$(pgrep -f "$_session_release/.*erts.*beam.smp" 2>/dev/null || true)"
439
- [ -z "$beam_pids" ] && break
440
- sleep 0.1
441
- waited=$((waited + 1))
442
- done
443
- beam_pids="$(pgrep -f "$_session_release/.*erts.*beam.smp" 2>/dev/null || true)"
444
- for pid in $beam_pids; do kill -KILL "$pid" 2>/dev/null || true; done
1256
+ # kill-session on the detached aiur session FIRST: this is what propagates
1257
+ # SIGHUP to the BEAM in that session so it begins its own orderly shutdown.
1258
+ # Without it the BEAM survives Ctrl+C, holding port 4000 + the node name and
1259
+ # breaking the next launch (regression test/aiur/regression/shutdown_cleanup_test.exs).
1260
+ if [ -n "$_session_tmux" ] && [ -n "$_session_socket" ] && [ -n "$_session_name" ]; then
1261
+ "$_session_tmux" -L "$_session_socket" kill-session -t "$_session_name" 2>/dev/null || true
445
1262
  fi
446
1263
 
447
- # Belt-and-suspenders: reap any BEAM still holding our node name, even one
448
- # launched from a different release dir than this run.
1264
+ # Then kill-server on aiur's private socket: tears down every pane agent across
1265
+ # all windows AND leaves no live aiur tmux server, then reaps headless agents
1266
+ # from the pidfile. Idempotent with the watchdog's own reap and the kill-session
1267
+ # above. reap_aiur_agents re-resolves tmux itself and reaps headless agents even
1268
+ # when tmux is absent, so it is not gated on $_session_tmux.
1269
+ reap_aiur_agents "$_session_socket" "$_session_pidfile"
1270
+
1271
+ # Reap only this instance's BEAM. Multiple keyed instances can share the same
1272
+ # release dir, so release-path pgrep would terminate sibling workflows.
449
1273
  [ -n "$_session_node" ] && kill_beams_matching "-name ${_session_node}"
450
1274
 
1275
+ # Final cwd-sweep backstop after the BEAM is gone: catches workspace-rooted
1276
+ # agents or test children that registered too late, reparented during the
1277
+ # BEAM-side sweep, or survived a bounded in-BEAM cleanup.
1278
+ reap_workspace_cwd_from_file "$_session_workspace_root_file"
1279
+
1280
+ # Reap the epmd our BEAM spawned for distribution so it doesn't linger after
1281
+ # exit. The node is dead by now, so `epmd -kill` succeeds; it safely refuses
1282
+ # while any *other* Erlang node is alive, so a daemon shared with an unrelated
1283
+ # Erlang program is never disrupted. Use the release's bundled epmd and pin it
1284
+ # to the loopback address aiur starts it on.
1285
+ for _epmd in "$_session_release"/erts-*/bin/epmd; do
1286
+ [ -x "$_epmd" ] || continue
1287
+ ERL_EPMD_ADDRESS="${ERL_EPMD_ADDRESS:-127.0.0.1}" "$_epmd" -kill >/dev/null 2>&1 || true
1288
+ done
1289
+
451
1290
  if command -v opencode >/dev/null 2>&1 && [ -s "$_session_tmpfile" ]; then
452
1291
  local id
453
1292
  while IFS= read -r id; do
@@ -460,7 +1299,13 @@ session_cleanup() {
460
1299
  # Reap agent-driver sockets this run orphaned at close, not next launch.
461
1300
  sweep_dead_tmux_sockets || true
462
1301
 
463
- rm -f "$_session_tmpfile" "$_session_capture" "$_session_argv" 2>/dev/null || true
1302
+ # Reap stale aiur /tmp debris (per-run tempfiles, leaked test artifacts) so a
1303
+ # tmpfs /tmp can't fill across runs. Age-gated + pid-safe; our own still-present
1304
+ # tempfiles are spared (fresh + live pid) before the explicit rm below clears them.
1305
+ sweep_stale_tmp_artifacts || true
1306
+
1307
+ rm -f "$_session_tmpfile" "$_session_capture" "$_session_argv" "$_session_pidfile" "$_session_workspace_root_file" 2>/dev/null || true
1308
+ rm -f "$(aiur_instance_record_path)" 2>/dev/null || true
464
1309
  return $code
465
1310
  }
466
1311
  install_foreground_traps() {
@@ -479,23 +1324,264 @@ trim() {
479
1324
  printf '%s' "$s"
480
1325
  }
481
1326
 
1327
+ # Classify our distribution node's liveness via epmd, which only advertises a
1328
+ # node while its BEAM holds the registration. Echoes exactly one word:
1329
+ # up — node is registered: it is alive, so an rpc failure is a real error
1330
+ # down — epmd answered and our node is absent: it genuinely is not running
1331
+ # unknown — epmd could not be queried (binary missing, or daemon unreachable),
1332
+ # so node state is indeterminate and callers must NOT assume "down"
1333
+ # and mask the real error
1334
+ # Distinguishing `unknown` from `down` matters: a live node whose epmd we cannot
1335
+ # reach must still surface its real rpc error rather than the "start aiur" hint.
1336
+ # Relies on RELEASE_NODE + ERL_EPMD_ADDRESS (prepare_distribution) and
1337
+ # release_dir (resolve_release), so call it only after both have run.
1338
+ probe_node_liveness() {
1339
+ local epmd names short="${RELEASE_NODE%@*}"
1340
+ for epmd in "$release_dir"/erts-*/bin/epmd; do
1341
+ [ -x "$epmd" ] || continue
1342
+ names="$(ERL_EPMD_ADDRESS="${ERL_EPMD_ADDRESS:-127.0.0.1}" "$epmd" -names 2>/dev/null)" \
1343
+ || { printf 'unknown'; return; }
1344
+ case "$names" in
1345
+ *"name ${short} at port "*) printf 'up' ;;
1346
+ *) printf 'down' ;;
1347
+ esac
1348
+ return
1349
+ done
1350
+ printf 'unknown'
1351
+ }
1352
+
1353
+ probe_named_node_liveness() {
1354
+ local node="$1" old_node="${RELEASE_NODE:-}" state
1355
+ RELEASE_NODE="$node"
1356
+ state="$(probe_node_liveness)"
1357
+ RELEASE_NODE="$old_node"
1358
+ printf '%s' "$state"
1359
+ }
1360
+
1361
+ load_aiur_instance_record() {
1362
+ local file="$1"
1363
+ AIUR_RECORD_NODE=""
1364
+ AIUR_RECORD_INSTANCE_KEY=""
1365
+ AIUR_RECORD_SESSION=""
1366
+ AIUR_RECORD_SOCKET=""
1367
+ AIUR_RECORD_PROJECT_ROOT=""
1368
+ AIUR_RECORD_PROJECT_ROOT_SOURCE=""
1369
+ [ -r "$file" ] || return 1
1370
+ # Records are written by this engine as KEY=%q lines under the user's state dir.
1371
+ # shellcheck disable=SC1090
1372
+ source "$file" 2>/dev/null || return 1
1373
+ [ -n "$AIUR_RECORD_NODE" ] && [ -n "$AIUR_RECORD_SESSION" ] && \
1374
+ [ -n "$AIUR_RECORD_SOCKET" ] && [ -n "$AIUR_RECORD_PROJECT_ROOT" ]
1375
+ }
1376
+
1377
+ path_is_within_root() {
1378
+ local root child
1379
+ root="$(canonical_workspace_root "$1")"
1380
+ child="$(canonical_workspace_root "$2")"
1381
+ [ "$child" = "$root" ] || [[ "$child/" == "$root/"* ]]
1382
+ }
1383
+
1384
+ AIUR_CONTROL_ADOPTED_RECORD=0
1385
+ AIUR_CONTROL_CURRENT_NODE_STATE=""
1386
+ AIUR_CONTROL_HINT_ROOTS=""
1387
+ AIUR_CONTROL_CALLER_ROOT=""
1388
+ AIUR_CONTROL_CALLER_NODE=""
1389
+ AIUR_CONTROL_CALLER_ROOT_SOURCE=""
1390
+
1391
+ resolve_control_identity_from_records() {
1392
+ AIUR_CONTROL_ADOPTED_RECORD=0
1393
+ AIUR_CONTROL_CURRENT_NODE_STATE="$(probe_node_liveness)"
1394
+ AIUR_CONTROL_HINT_ROOTS=""
1395
+ AIUR_CONTROL_CALLER_ROOT="$(canonical_workspace_root "${AIUR_PROJECT_ROOT:-}")"
1396
+ AIUR_CONTROL_CALLER_NODE="$AIUR_RELEASE_NODE"
1397
+ AIUR_CONTROL_CALLER_ROOT_SOURCE="${AIUR_PROJECT_ROOT_SOURCE:-}"
1398
+
1399
+ [ "$AIUR_CONTROL_CURRENT_NODE_STATE" = "down" ] || return 0
1400
+ [ "${AIUR_PROJECT_ROOT_SOURCE:-}" = "cwd" ] || return 0
1401
+
1402
+ local dir file state root match_count=0
1403
+ local match_node="" match_key="" match_session="" match_socket="" match_root="" match_source=""
1404
+ dir="$(aiur_instances_dir)"
1405
+ [ -d "$dir" ] || return 0
1406
+
1407
+ for file in "$dir"/*.instance; do
1408
+ [ -e "$file" ] || continue
1409
+ load_aiur_instance_record "$file" || continue
1410
+ state="$(probe_named_node_liveness "$AIUR_RECORD_NODE")"
1411
+ [ "$state" = "up" ] || continue
1412
+
1413
+ root="$(canonical_workspace_root "$AIUR_RECORD_PROJECT_ROOT")"
1414
+ case "$AIUR_CONTROL_HINT_ROOTS" in
1415
+ *"
1416
+ $root
1417
+ "*) ;;
1418
+ *) AIUR_CONTROL_HINT_ROOTS="${AIUR_CONTROL_HINT_ROOTS}${root}
1419
+ " ;;
1420
+ esac
1421
+
1422
+ path_is_within_root "$root" "$AIUR_CONTROL_CALLER_ROOT" || continue
1423
+ match_count=$((match_count + 1))
1424
+ match_node="$AIUR_RECORD_NODE"
1425
+ match_key="$AIUR_RECORD_INSTANCE_KEY"
1426
+ match_session="$AIUR_RECORD_SESSION"
1427
+ match_socket="$AIUR_RECORD_SOCKET"
1428
+ match_root="$root"
1429
+ match_source="$AIUR_RECORD_PROJECT_ROOT_SOURCE"
1430
+ done
1431
+
1432
+ [ "$match_count" -eq 1 ] || return 0
1433
+
1434
+ AIUR_RELEASE_NODE="$match_node"
1435
+ AIUR_INSTANCE_KEY="$match_key"
1436
+ AIUR_PROJECT_ROOT="$match_root"
1437
+ AIUR_PROJECT_ROOT_SOURCE="$match_source"
1438
+ AIUR_ADOPTED_TMUX_SESSION="$match_session"
1439
+ AIUR_ADOPTED_TMUX_SOCKET="$match_socket"
1440
+ export AIUR_RELEASE_NODE AIUR_INSTANCE_KEY AIUR_PROJECT_ROOT AIUR_PROJECT_ROOT_SOURCE \
1441
+ AIUR_ADOPTED_TMUX_SESSION AIUR_ADOPTED_TMUX_SOCKET
1442
+ RELEASE_NODE="$AIUR_RELEASE_NODE"
1443
+ export RELEASE_NODE
1444
+ AIUR_CONTROL_ADOPTED_RECORD=1
1445
+ }
1446
+
1447
+ print_global_config_control_hint() {
1448
+ [ "${AIUR_CONTROL_CALLER_ROOT_SOURCE:-}" = "cwd" ] || return 0
1449
+ [ "${AIUR_CONTROL_CURRENT_NODE_STATE:-}" = "down" ] || return 0
1450
+
1451
+ echo "aiur: global-config control identity is keyed by cwd ${AIUR_CONTROL_CALLER_ROOT:-${AIUR_PROJECT_ROOT:-unknown}}" >&2
1452
+ echo "aiur: run control commands from the launch directory, or from a subdirectory of that launch directory" >&2
1453
+ if [ -n "${AIUR_CONTROL_HINT_ROOTS:-}" ]; then
1454
+ echo "aiur: live launch directory candidate(s):" >&2
1455
+ printf '%s' "$AIUR_CONTROL_HINT_ROOTS" | sed '/^$/d; s/^/ /' >&2
1456
+ fi
1457
+ }
1458
+
1459
+ control_rpc_timeout_seconds() {
1460
+ local seconds="${AIUR_CONTROL_RPC_TIMEOUT_SECONDS:-10}"
1461
+ case "$seconds" in
1462
+ '' | *[!0-9]* | 0) seconds=10 ;;
1463
+ esac
1464
+ printf '%s' "$seconds"
1465
+ }
1466
+
1467
+ kill_control_rpc_process() {
1468
+ local pid="$1" grouped="$2" p tree=()
1469
+ [ -n "$pid" ] || return 0
1470
+
1471
+ if [ "$grouped" = "1" ]; then
1472
+ kill -TERM "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true
1473
+ sleep 0.2
1474
+ kill -KILL "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true
1475
+ return 0
1476
+ fi
1477
+
1478
+ while IFS= read -r p; do tree+=("$p"); done < <(agent_pid_tree "$pid")
1479
+ for p in "${tree[@]}"; do kill -TERM "$p" 2>/dev/null || true; done
1480
+ sleep 0.2
1481
+ for p in "${tree[@]}"; do kill -KILL "$p" 2>/dev/null || true; done
1482
+ }
1483
+
1484
+ run_release_rpc_with_timeout() {
1485
+ local expression="$1" timeout output_file timeout_file pid watchdog_pid status grouped=0
1486
+ timeout="$(control_rpc_timeout_seconds)"
1487
+ output_file="$(mktemp "${TMPDIR:-/tmp}/aiur-control-rpc-output.XXXXXX")"
1488
+ timeout_file="$(mktemp "${TMPDIR:-/tmp}/aiur-control-rpc-timeout.XXXXXX")"
1489
+ rm -f "$timeout_file" 2>/dev/null || true
1490
+
1491
+ if command -v setsid >/dev/null 2>&1; then
1492
+ setsid "$release_bin" rpc "$expression" >"$output_file" 2>&1 &
1493
+ grouped=1
1494
+ else
1495
+ "$release_bin" rpc "$expression" >"$output_file" 2>&1 &
1496
+ fi
1497
+ pid=$!
1498
+
1499
+ (
1500
+ sleep "$timeout"
1501
+ if kill -0 "$pid" 2>/dev/null; then
1502
+ : >"$timeout_file"
1503
+ kill_control_rpc_process "$pid" "$grouped"
1504
+ fi
1505
+ ) &
1506
+ watchdog_pid=$!
1507
+
1508
+ if wait "$pid"; then
1509
+ status=0
1510
+ else
1511
+ status=$?
1512
+ fi
1513
+ kill "$watchdog_pid" 2>/dev/null || true
1514
+ wait "$watchdog_pid" 2>/dev/null || true
1515
+
1516
+ AIUR_CONTROL_RPC_OUTPUT="$(cat "$output_file" 2>/dev/null || true)"
1517
+ if [ -f "$timeout_file" ]; then
1518
+ AIUR_CONTROL_RPC_TIMED_OUT=1
1519
+ status=124
1520
+ else
1521
+ AIUR_CONTROL_RPC_TIMED_OUT=0
1522
+ fi
1523
+
1524
+ rm -f "$output_file" "$timeout_file" 2>/dev/null || true
1525
+ return "$status"
1526
+ }
1527
+
482
1528
  # RPC an expression into the running node. The control CLI prints a trailing
483
1529
  # `__AIUR_CONTROL_EXIT__:<code>` marker we translate into the process exit code.
484
1530
  run_control_rpc() {
485
1531
  local expression="$1"
486
1532
  resolve_release
487
1533
  prepare_distribution || die "distribution setup failed; cannot contact aiur"
1534
+ resolve_control_identity_from_records
1535
+ if [ "${AIUR_CONTROL_ADOPTED_RECORD:-0}" -eq 1 ]; then
1536
+ prepare_distribution || die "distribution setup failed; cannot contact aiur"
1537
+ fi
488
1538
 
489
1539
  local marker="__AIUR_CONTROL_EXIT__:"
490
1540
  local output status exit_code=0 saw_marker=0 line
491
1541
 
492
1542
  set +e
493
- output="$("$release_bin" rpc "$expression" 2>&1)"
1543
+ run_release_rpc_with_timeout "$expression"
494
1544
  status=$?
1545
+ output="$AIUR_CONTROL_RPC_OUTPUT"
495
1546
  set -e
496
1547
 
1548
+ if [ "${AIUR_CONTROL_RPC_TIMED_OUT:-0}" -eq 1 ]; then
1549
+ [ -n "$output" ] && printf '%s\n' "$output" >&2
1550
+ echo "aiur: control rpc to ${RELEASE_NODE} timed out after $(control_rpc_timeout_seconds)s; helper process was terminated" >&2
1551
+ return 124
1552
+ fi
1553
+
497
1554
  if [ "$status" -ne 0 ]; then
498
- echo "aiur: no running aiur node at ${RELEASE_NODE}; start aiur and try again" >&2
1555
+ # A non-zero exit here is the rpc transport failing — application-level
1556
+ # outcomes ride the marker path below. `bin/aiur rpc` (Elixir --rpc-eval)
1557
+ # reports a genuinely-down node and a live-but-unreachable one identically
1558
+ # (`:noconnection`), and prints any exception raised inside the expression.
1559
+ # So the reason string can't be trusted; classify via epmd instead. Only a
1560
+ # node epmd confirms is down earns the friendly "start aiur" hint; an `up`
1561
+ # node failed for a real reason, and an `unknown` probe must not be assumed
1562
+ # down — in both of those cases surface the actual rpc output, never mask it.
1563
+ case "$(probe_node_liveness)" in
1564
+ down)
1565
+ local crash_marker
1566
+ crash_marker="$(aiur_crash_marker_path)"
1567
+ if [ -f "$crash_marker" ]; then
1568
+ echo "aiur: background daemon at ${RELEASE_NODE} is DOWN after an unexpected exit; agents may be orphaned" >&2
1569
+ sed 's/^/ /' "$crash_marker" >&2 2>/dev/null || true
1570
+ echo "aiur: run 'aiur stop' to reap any orphaned agents, then start aiur again" >&2
1571
+ else
1572
+ echo "aiur: no running aiur node at ${RELEASE_NODE}; start aiur and try again" >&2
1573
+ print_global_config_control_hint
1574
+ fi
1575
+ ;;
1576
+ up)
1577
+ [ -n "$output" ] && printf '%s\n' "$output" >&2
1578
+ echo "aiur: rpc to ${RELEASE_NODE} failed (node is running); see the error above" >&2
1579
+ ;;
1580
+ *)
1581
+ [ -n "$output" ] && printf '%s\n' "$output" >&2
1582
+ echo "aiur: rpc to ${RELEASE_NODE} failed (could not query epmd to confirm node state); see the error above" >&2
1583
+ ;;
1584
+ esac
499
1585
  return 1
500
1586
  fi
501
1587
 
@@ -510,7 +1596,11 @@ run_control_rpc() {
510
1596
  esac
511
1597
  done <<<"$output"
512
1598
 
513
- [ "$saw_marker" -eq 1 ] || return 1
1599
+ if [ "$saw_marker" -ne 1 ]; then
1600
+ echo "aiur: control rpc to ${RELEASE_NODE} returned no exit marker; command output may be incomplete" >&2
1601
+ return 1
1602
+ fi
1603
+
514
1604
  return "$exit_code"
515
1605
  }
516
1606
 
@@ -575,6 +1665,152 @@ cmd_pause_resume() {
575
1665
  run_control_rpc "$expression"
576
1666
  }
577
1667
 
1668
+ # `aiur message <issue> <text>` — deliver operator text to one running agent.
1669
+ # The text is base64-encoded for the RPC hop so arbitrary content (quotes,
1670
+ # backslashes, `#{}`, newlines) survives without Elixir-string escaping.
1671
+ cmd_message() {
1672
+ local usage="aiur: message expects an issue ID and text (e.g. aiur message 44 \"ship it\")"
1673
+
1674
+ local issue="${1:-}"
1675
+ if [ -z "$issue" ] || [[ ! "$issue" =~ ^[0-9]+$ ]]; then
1676
+ echo "$usage" >&2
1677
+ exit 64
1678
+ fi
1679
+ shift
1680
+
1681
+ local text="$*"
1682
+ if [ -z "$text" ]; then
1683
+ echo "$usage" >&2
1684
+ exit 64
1685
+ fi
1686
+
1687
+ local encoded
1688
+ encoded="$(printf '%s' "$text" | base64 | tr -d '\n')"
1689
+ run_control_rpc "Aiur.AgentControlCLI.message(\"$issue\", Base.decode64!(\"$encoded\"))"
1690
+ }
1691
+
1692
+ # `aiur agents` — concise one-line-per-agent state + current activity from a
1693
+ # live node (the headless equivalent of the dashboard / aiur-status skill).
1694
+ cmd_agents() {
1695
+ [ "$#" -eq 0 ] || die "agents does not accept arguments"
1696
+ run_control_rpc "Aiur.AgentControlCLI.agents()"
1697
+ }
1698
+
1699
+ # `aiur alerts` — newline-delimited structured alert feed from persisted
1700
+ # per-agent logs. `--needs-attention` filters to operator-actionable alerts.
1701
+ cmd_alerts() {
1702
+ local needs_attention=0 arg
1703
+ for arg in "$@"; do
1704
+ case "$arg" in
1705
+ --needs-attention) needs_attention=1 ;;
1706
+ *)
1707
+ echo "aiur: alerts only accepts --needs-attention" >&2
1708
+ exit 64
1709
+ ;;
1710
+ esac
1711
+ done
1712
+
1713
+ if [ "$needs_attention" -eq 1 ]; then
1714
+ run_control_rpc "Aiur.AgentControlCLI.alerts(needs_attention: true)"
1715
+ else
1716
+ run_control_rpc "Aiur.AgentControlCLI.alerts()"
1717
+ fi
1718
+ }
1719
+
1720
+ # `aiur watch` — the server-side status board. Compiles one row per active
1721
+ # agent (state · complexity · activity-age · doing) plus an actionable section
1722
+ # from aiur's own state, with no GitHub round-trip. `--changes` (default) prints
1723
+ # only state-level deltas since the last call; `--full` prints every row;
1724
+ # `--interval N` re-renders every N seconds as a foreground watcher (the Elixir
1725
+ # call stays one-shot — the loop lives here).
1726
+ cmd_watch() {
1727
+ local mode="changes" interval="" arg
1728
+
1729
+ while [ "$#" -gt 0 ]; do
1730
+ arg="$1"
1731
+ case "$arg" in
1732
+ --full) mode="full" ;;
1733
+ --changes) mode="changes" ;;
1734
+ --once) : ;;
1735
+ --interval)
1736
+ shift
1737
+ interval="${1:-}"
1738
+ watch_validate_interval "$interval"
1739
+ ;;
1740
+ --interval=*)
1741
+ interval="${arg#--interval=}"
1742
+ watch_validate_interval "$interval"
1743
+ ;;
1744
+ *)
1745
+ echo "aiur: watch accepts --full, --changes, --once, --interval <secs>" >&2
1746
+ exit 64
1747
+ ;;
1748
+ esac
1749
+ shift
1750
+ done
1751
+
1752
+ local expression="Aiur.AgentControlCLI.watch(mode: :${mode})"
1753
+
1754
+ if [ -n "$interval" ]; then
1755
+ while true; do
1756
+ run_control_rpc "$expression" || true
1757
+ sleep "$interval"
1758
+ done
1759
+ else
1760
+ run_control_rpc "$expression"
1761
+ fi
1762
+ }
1763
+
1764
+ watch_validate_interval() {
1765
+ if ! [[ "$1" =~ ^[0-9]+$ ]] || [ "$1" -le 0 ]; then
1766
+ echo "aiur: watch --interval expects a positive integer (seconds)" >&2
1767
+ exit 64
1768
+ fi
1769
+ }
1770
+
1771
+ cmd_cleanup_stale() {
1772
+ local dry_run=0 arg
1773
+ for arg in "$@"; do
1774
+ case "$arg" in
1775
+ --dry-run) dry_run=1 ;;
1776
+ *)
1777
+ echo "aiur: cleanup-stale only accepts --dry-run" >&2
1778
+ exit 64
1779
+ ;;
1780
+ esac
1781
+ done
1782
+
1783
+ aiur_resolve_identity
1784
+ if [ "$dry_run" -eq 1 ]; then
1785
+ report_stale_manual_smoke || true
1786
+ else
1787
+ report_stale_manual_smoke || true
1788
+ reap_stale_manual_smoke 1
1789
+ fi
1790
+ }
1791
+
1792
+ # `aiur set <key> <value>` — runtime config overrides without editing
1793
+ # `.aiur/config`. Currently: `aiur set max-agents N`.
1794
+ cmd_set() {
1795
+ local key="${1:-}"
1796
+ shift 2>/dev/null || true
1797
+
1798
+ case "$key" in
1799
+ max-agents)
1800
+ local n="${1:-}"
1801
+ if [ -z "$n" ] || [[ ! "$n" =~ ^[0-9]+$ ]] || [ "$n" -lt 1 ]; then
1802
+ echo "aiur: set max-agents expects a positive integer (e.g. aiur set max-agents 5)" >&2
1803
+ exit 64
1804
+ fi
1805
+ run_control_rpc "Aiur.AgentControlCLI.set_max_agents($n)"
1806
+ ;;
1807
+ *)
1808
+ echo "aiur: unknown setting: ${key:-(none)} (supported: max-agents)" >&2
1809
+ exit 64
1810
+ ;;
1811
+ esac
1812
+ }
1813
+
578
1814
  sweep_dead_tmux_sockets() {
579
1815
  local tmux_bin
580
1816
  tmux_bin="$(command -v tmux || true)"
@@ -600,38 +1836,181 @@ sweep_dead_tmux_sockets() {
600
1836
  return 0
601
1837
  }
602
1838
 
603
- # Stop the running session: kill the tmux session + the release BEAM, then sweep.
1839
+ # Reap stale aiur /tmp debris so a tmpfs /tmp can't fill from accumulated per-run
1840
+ # tempfiles and leaked test artifacts. Runs alongside sweep_dead_tmux_sockets on
1841
+ # foreground teardown (session_cleanup) and on `aiur stop`. Bounded and safe:
1842
+ #
1843
+ # * Only exact top-level Aiur artifact families under the temp roots the engine
1844
+ # and BEAM write to. Arbitrary operator worktrees/checkouts like
1845
+ # `/tmp/aiur-pr490` are not candidates.
1846
+ # * Age-gated by AIUR_TMP_REAP_MINUTES (default 1440 = 24h). An entry is removed
1847
+ # only when NOTHING in its subtree was modified within the window, so a live
1848
+ # run's shared debug dir (aiur-rc / aiur-claude-hooks / aiur-debug) holding a
1849
+ # fresh file is spared even when the dir's own mtime is stale. A non-numeric or
1850
+ # 0 value disables the sweep.
1851
+ # * Ownership-gated: if anything in the tree is not owned by the effective user,
1852
+ # the whole tree is spared.
1853
+ # * A live run's `aiur-<pid>-sessions|-agents` bookkeeping is kept while its pid
1854
+ # is alive, regardless of age.
1855
+ is_aiur_tmp_artifact_candidate() {
1856
+ local base="$1" pid
1857
+ case "$base" in
1858
+ aiur-argv.* | aiur-startup.* | aiur-pane.* | aiur-launcher.* | aiur-capture.* | \
1859
+ aiur-trap.*.log | aiur-tree.*.json | aiur-wrapper.pid | aiur-*-frames.bin | \
1860
+ aiur-rc | aiur-claude-hooks | aiur-debug)
1861
+ return 0
1862
+ ;;
1863
+ aiur-*-sessions | aiur-*-agents)
1864
+ pid="${base#aiur-}"
1865
+ pid="${pid%-*}"
1866
+ [ -n "$pid" ] && [ -z "${pid//[0-9]/}" ]
1867
+ return
1868
+ ;;
1869
+ *)
1870
+ return 1
1871
+ ;;
1872
+ esac
1873
+ }
1874
+
1875
+ sweep_stale_tmp_artifacts() {
1876
+ local minutes="${AIUR_TMP_REAP_MINUTES:-1440}"
1877
+ case "$minutes" in '' | *[!0-9]*) return 0 ;; esac
1878
+ [ "$minutes" -gt 0 ] || return 0
1879
+
1880
+ # The dirs the engine + BEAM target: ${TMPDIR:-/tmp} (argv/startup/pane tempfiles
1881
+ # plus the BEAM's System.tmp_dir debug dirs) and ${XDG_RUNTIME_DIR:-${TMPDIR:-/tmp}}
1882
+ # (the session/agent pidfiles' session_root). Deduped — a typical box scans one dir.
1883
+ local roots=() d seen
1884
+ for d in "${TMPDIR:-/tmp}" "${XDG_RUNTIME_DIR:-${TMPDIR:-/tmp}}"; do
1885
+ [ -d "$d" ] || continue
1886
+ seen=0
1887
+ local root
1888
+ if [ "${#roots[@]}" -gt 0 ]; then
1889
+ for root in "${roots[@]}"; do
1890
+ [ "$root" = "$d" ] && { seen=1; break; }
1891
+ done
1892
+ fi
1893
+ [ "$seen" -eq 1 ] || roots+=("$d")
1894
+ done
1895
+ [ "${#roots[@]}" -gt 0 ] || return 0
1896
+
1897
+ local removed=0 path base pid uid found
1898
+ uid="$(id -u)"
1899
+ for d in "${roots[@]}"; do
1900
+ while IFS= read -r -d '' path; do
1901
+ base="${path##*/}"
1902
+ is_aiur_tmp_artifact_candidate "$base" || continue
1903
+
1904
+ # Spare mixed-ownership trees; cleanup should never cross user boundaries.
1905
+ if ! found="$(find "$path" ! -user "$uid" -print -quit 2>/dev/null)"; then
1906
+ continue
1907
+ fi
1908
+ if [ -n "$found" ]; then
1909
+ continue
1910
+ fi
1911
+ # Spare anything modified within the window anywhere in its subtree.
1912
+ if ! found="$(find "$path" -mmin "-$minutes" -print -quit 2>/dev/null)"; then
1913
+ continue
1914
+ fi
1915
+ if [ -n "$found" ]; then
1916
+ continue
1917
+ fi
1918
+ # Spare a live run's session/agent bookkeeping (name is aiur-<pid>-<kind>).
1919
+ case "$base" in
1920
+ aiur-*-sessions | aiur-*-agents)
1921
+ pid="${base#aiur-}"
1922
+ pid="${pid%-*}"
1923
+ if [ -n "$pid" ] && [ -z "${pid//[0-9]/}" ] && kill -0 "$pid" 2>/dev/null; then
1924
+ continue
1925
+ fi
1926
+ ;;
1927
+ esac
1928
+ if rm -rf -- "$path" 2>/dev/null; then
1929
+ removed=$((removed + 1))
1930
+ fi
1931
+ done < <(find "$d" -maxdepth 1 -mindepth 1 -name 'aiur-*' -print0 2>/dev/null)
1932
+ done
1933
+
1934
+ [ "$removed" -gt 0 ] && echo "🧹 swept $removed stale aiur tmp artifact(s)" >&2
1935
+ return 0
1936
+ }
1937
+
1938
+ # Stop the running session: kill this instance's tmux + BEAM, then sweep.
604
1939
  cmd_stop() {
605
1940
  resolve_release
606
1941
  aiur_resolve_identity
1942
+ RELEASE_NODE="$AIUR_RELEASE_NODE"
1943
+ ERL_EPMD_ADDRESS="${ERL_EPMD_ADDRESS:-127.0.0.1}"
1944
+ export RELEASE_NODE ERL_EPMD_ADDRESS
1945
+ resolve_control_identity_from_records
1946
+
1947
+ local workspace_root
1948
+ workspace_root="$(current_workspace_root 2>/dev/null || true)"
607
1949
 
608
1950
  local tmux_bin
609
1951
  tmux_bin="$(command -v tmux || true)"
610
- local session="${AIUR_SESSION_PREFIX}-${USER:-user}-default"
611
- local socket="${AIUR_SESSION_PREFIX}-${USER:-user}"
612
- if [ -n "$tmux_bin" ]; then
613
- "$tmux_bin" -L "$socket" kill-session -t "$session" 2>/dev/null || true
1952
+ local session="${AIUR_ADOPTED_TMUX_SESSION:-${AIUR_SESSION_PREFIX}-${USER:-user}${AIUR_INSTANCE_KEY:+-$AIUR_INSTANCE_KEY}-default}"
1953
+ local socket="${AIUR_ADOPTED_TMUX_SOCKET:-${AIUR_SESSION_PREFIX}-${USER:-user}${AIUR_INSTANCE_KEY:+-$AIUR_INSTANCE_KEY}}"
1954
+
1955
+ local has_session=0
1956
+ if [ -n "$tmux_bin" ] && "$tmux_bin" -L "$socket" has-session -t "$session" 2>/dev/null; then
1957
+ has_session=1
614
1958
  fi
615
1959
 
616
- local beam_pids pid waited=0
617
- beam_pids="$(pgrep -f "$release_dir/.*erts.*beam.smp" 2>/dev/null || true)"
618
- if [ -n "$beam_pids" ]; then
619
- for pid in $beam_pids; do kill -TERM "$pid" 2>/dev/null || true; done
620
- while [ $waited -lt 30 ]; do
621
- beam_pids="$(pgrep -f "$release_dir/.*erts.*beam.smp" 2>/dev/null || true)"
622
- [ -z "$beam_pids" ] && break
623
- sleep 0.1
624
- waited=$((waited + 1))
625
- done
626
- beam_pids="$(pgrep -f "$release_dir/.*erts.*beam.smp" 2>/dev/null || true)"
627
- for pid in $beam_pids; do kill -KILL "$pid" 2>/dev/null || true; done
1960
+ if [ "${AIUR_CONTROL_ADOPTED_RECORD:-0}" -ne 1 ] && \
1961
+ [ "${AIUR_CONTROL_CURRENT_NODE_STATE:-}" = "down" ] && \
1962
+ [ "${AIUR_PROJECT_ROOT_SOURCE:-}" = "cwd" ] && \
1963
+ [ "$has_session" -eq 0 ] && \
1964
+ [ ! -f "$(aiur_crash_marker_path)" ]; then
1965
+ echo "aiur: no running aiur node at ${AIUR_RELEASE_NODE}; nothing stopped" >&2
1966
+ print_global_config_control_hint
1967
+ return 1
1968
+ fi
1969
+
1970
+ # Tell the background BEAM-death watchdog this exit is intentional before we
1971
+ # kill the BEAM, so it consumes the sentinel instead of recording a crash. The
1972
+ # watchdog removes the sentinel when it fires; a fresh start also clears it.
1973
+ # Clear any prior crash marker too — `status` should report a clean stop, not
1974
+ # a stale orphan from an earlier dead run.
1975
+ mkdir -p "$AIUR_BG_STATE_DIR" 2>/dev/null || true
1976
+ : >"$(aiur_stop_sentinel_path)" 2>/dev/null || true
1977
+ rm -f "$(aiur_crash_marker_path)" 2>/dev/null || true
1978
+
1979
+ if [ -n "$tmux_bin" ]; then
1980
+ "$tmux_bin" -L "$socket" kill-session -t "$session" 2>/dev/null || true
628
1981
  fi
629
1982
 
630
1983
  # Reap any BEAM holding our node name regardless of which release dir launched
631
- # it — orphans from a dev build or a prior install share the unified name.
1984
+ # it. Do not sweep by release dir: sibling instances share dev releases.
632
1985
  kill_beams_matching "-name ${AIUR_RELEASE_NODE}"
633
1986
 
1987
+ # Final cwd-sweep backstop after the BEAM is dead. `cmd_stop` is a fresh shell,
1988
+ # so it captures the root from the live control node before the kill above.
1989
+ reap_workspace_cwd_agents "$workspace_root"
1990
+
1991
+ # The BEAM (alive until the TERM above) reaped its own headless agents through
1992
+ # ProcessReaper on Application.stop. kill-server is the guarantee the earlier
1993
+ # kill-session can't give for mid-turn agents: every pane agent across all
1994
+ # windows dies and no live aiur tmux server is left behind.
1995
+ if [ -n "$tmux_bin" ]; then
1996
+ "$tmux_bin" -L "$socket" kill-server 2>/dev/null || true
1997
+ fi
1998
+
1999
+ # Belt-and-suspenders for a mid-turn stop: sweep any headless agent (this run
2000
+ # or a prior crashed one) still recorded in a pidfile. comm-guarded, so a
2001
+ # recycled pid is spared. Empty socket: the kill-server above already ran.
2002
+ local session_root agentfile
2003
+ session_root="${XDG_RUNTIME_DIR:-${TMPDIR:-/tmp}}"
2004
+ for agentfile in "$session_root"/aiur-*-agents; do
2005
+ [ -e "$agentfile" ] || continue
2006
+ reap_aiur_agents "" "$agentfile"
2007
+ rm -f "$agentfile" 2>/dev/null || true
2008
+ done
2009
+
2010
+ reap_stale_manual_smoke 0
634
2011
  sweep_dead_tmux_sockets
2012
+ sweep_stale_tmp_artifacts
2013
+ rm -f "$(aiur_instance_record_path)" 2>/dev/null || true
635
2014
  }
636
2015
 
637
2016
  # --- dispatch ----------------------------------------------------------------
@@ -663,10 +2042,34 @@ aiur_engine_main() {
663
2042
  shift
664
2043
  cmd_status "$@"
665
2044
  ;;
2045
+ agents)
2046
+ shift
2047
+ cmd_agents "$@"
2048
+ ;;
2049
+ alerts)
2050
+ shift
2051
+ cmd_alerts "$@"
2052
+ ;;
2053
+ watch)
2054
+ shift
2055
+ cmd_watch "$@"
2056
+ ;;
2057
+ set)
2058
+ shift
2059
+ cmd_set "$@"
2060
+ ;;
666
2061
  pause | resume)
667
2062
  shift
668
2063
  cmd_pause_resume "$cmd" "$@"
669
2064
  ;;
2065
+ message)
2066
+ shift
2067
+ cmd_message "$@"
2068
+ ;;
2069
+ cleanup-stale)
2070
+ shift
2071
+ cmd_cleanup_stale "$@"
2072
+ ;;
670
2073
  stop)
671
2074
  cmd_stop
672
2075
  ;;