@zerwiz/ymir 0.1.3 → 0.1.5

package/.agents/tests/fm-task-inbox.test.sh

@@ -0,0 +1,513 @@
+#!/usr/bin/env bash
+# tests/fm-task-inbox.test.sh - the per-task steering inbox
+# (bin/fm-task-inbox-lib.sh) and the watcher's re-ring ladder.
+#
+# The inbox+doorbell design replaces typed steer payloads with durable
+# sequenced records acknowledged by an atomic mv into handled/; the terminal
+# carries only a constant doorbell line, and the watcher re-rings an
+# unacknowledged message before escalating once as an ordinary stale wake.
+# These tests pin the semantics with real processes:
+#   1. A message is written durably and appears in the inbox, byte-exact
+#      including newlines, with a doorbell naming the inbox glob, numeric order,
+#      and handled/.
+#   2. Sequencing dedups per worker lifetime: the handled mv retires a record,
+#      re-acking it is a no-op, and an acknowledged sequence is never reissued.
+#      The idempotent enqueue (the remote steer leg's primitive) additionally
+#      dedups an exact-body re-run onto the existing record, handled or not.
+#   3. Concurrent writers serialize on the sequence lock: no clobbered records.
+#   4. The re-ring ladder: within grace is quiet, past grace rings, ring
+#      spacing holds, a spent budget escalates exactly once, and an
+#      acknowledgement resets the ladder for the next message.
+#   5. A real fm-watch.sh subprocess re-rings the doorbell for an unhandled
+#      aged message on an idle pane WITHOUT waking firstmate, waits on a busy
+#      pane, stays silent on a healthy/empty inbox, surfaces unwritable ladder
+#      bookkeeping only while its record remains unhandled, and emits exactly
+#      one stale wake once the ring budget is spent.
+set -u
+
+# shellcheck source=tests/wake-helpers.sh
+. "$(dirname "${BASH_SOURCE[0]}")/wake-helpers.sh"
+
+WATCH="$ROOT/bin/fm-watch.sh"
+TMP_ROOT=$(fm_test_tmproot fm-task-inbox)
+# The doorbell line canonicalizes its paths, so keep the fixture root
+# canonical too (a trailing-slash TMPDIR otherwise yields a double slash).
+TMP_ROOT=$(cd "$TMP_ROOT" && pwd)
+
+# Run one library function against a state dir through a subshell that sources
+# the production library, so the tests exercise the executable surface rather
+# than re-implementing any format knowledge here.
+inbox_lib() {  # <state> <function> [args...]
+  local state=$1
+  shift
+  FM_STATE_OVERRIDE="$state" bash -c '
+    . "$1"
+    fn=$2
+    shift 2
+    "$fn" "$@"
+  ' _ "$ROOT/bin/fm-task-inbox-lib.sh" "$@"
+}
+
+# A fake tmux for the watcher cases: capture-pane replays FM_FAKE_TMUX_CAPTURE,
+# display-message yields a numeric cursor row, and every literal send-keys is
+# logged to FM_SEND_LOG so a doorbell ring is observable.
+make_watch_stubs() {  # <dir> -> echoes fakebin dir
+  local dir=$1 fb="$1/fakebin"
+  mkdir -p "$fb"
+  cat > "$fb/tmux" <<'SH'
+#!/usr/bin/env bash
+set -u
+case "${1:-}" in
+  send-keys)
+    shift
+    literal=0
+    while [ $# -gt 0 ]; do
+      case "$1" in
+        -t) shift 2 ;;
+        -l) literal=1; shift ;;
+        *) break ;;
+      esac
+    done
+    if [ "$literal" = 1 ]; then
+      printf '%s\n' "${1:-}" >> "${FM_SEND_LOG:-/dev/null}"
+      if [ -n "${FM_ACK_RECORD:-}" ] && [ -f "$FM_ACK_RECORD" ]; then
+        mv "$FM_ACK_RECORD" "${FM_ACK_RECORD%/*}/handled/"
+      fi
+    fi
+    exit 0 ;;
+  display-message)
+    for a in "$@"; do case "$a" in *cursor_y*) printf '1\n'; exit 0 ;; esac; done
+    printf 'fakepane\n'; exit 0 ;;
+  capture-pane)
+    if [ -n "${FM_FAKE_TMUX_CAPTURE:-}" ] && [ -f "$FM_FAKE_TMUX_CAPTURE" ]; then
+      cat "$FM_FAKE_TMUX_CAPTURE"
+    else
+      printf '╭────╮\n│    │\n╰────╯\n'
+    fi
+    exit 0 ;;
+  list-windows) exit 0 ;;
+esac
+exit 0
+SH
+  chmod +x "$fb/tmux"
+  make_fake_crew_state "$fb" >/dev/null
+  printf '%s\n' "$fb"
+}
+
+watch_bg() {  # <state> <fakebin> <out> [extra env assignments...]
+  local state=$1 fakebin=$2 out=$3
+  shift 3
+  PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" \
+    FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
+    FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)' \
+    FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \
+    FM_TASK_INBOX_GRACE_SECS=1 \
+    env "$@" "$WATCH" > "$out" 2>/dev/null &
+}
+
+wait_watcher_gone() {  # <pid> [limit-ticks]
+  local pid=$1 limit=${2:-120} i=0
+  while [ "$i" -lt "$limit" ]; do
+    kill -0 "$pid" 2>/dev/null || return 0
+    sleep 0.1
+    i=$((i + 1))
+  done
+  return 1
+}
+
+age_path() {  # <path>  (set mtime well past any grace under test)
+  touch -t 202001010000 "$1"
+}
+
+test_write_is_durable_and_exact() {
+  local state rec rec2 doorbell doorbell2 expected actual expected2 actual2 text
+  state="$TMP_ROOT/write/state"; mkdir -p "$state"
+  text=$'line one\nline two with  spaces\n/slash body\n\n'
+  rec=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "$text") \
+    || fail "inbox write failed"
+  [ -f "$rec" ] || fail "inbox write printed a path that does not exist: $rec"
+  case "$rec" in
+    "$state/t1.inbox/001.msg") : ;;
+    *) fail "first record should be 001.msg under the task inbox, got $rec" ;;
+  esac
+  expected="$state/expected.body"
+  actual="$state/actual.body"
+  printf '%s' "$text" > "$expected"
+  inbox_lib "$state" fm_task_inbox_body "$rec" > "$actual" \
+    || fail "record body could not be read"
+  cmp -s "$expected" "$actual" \
+    || fail "record body did not preserve trailing and blank-line bytes"
+  rec2=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "no trailing newline") \
+    || fail "second inbox write failed"
+  expected2="$state/expected-no-newline.body"
+  actual2="$state/actual-no-newline.body"
+  printf '%s' "no trailing newline" > "$expected2"
+  inbox_lib "$state" fm_task_inbox_body "$rec2" > "$actual2" \
+    || fail "second record body could not be read"
+  cmp -s "$expected2" "$actual2" \
+    || fail "record body added a trailing newline"
+  doorbell=$(inbox_lib "$state" fm_task_inbox_doorbell_line "$rec")
+  doorbell2=$(inbox_lib "$state" fm_task_inbox_doorbell_line "$rec2")
+  [ "$doorbell" = "$doorbell2" ] \
+    || fail "every record in one inbox should ring the same drain-all doorbell"
+  assert_contains "$doorbell" "$state/t1.inbox/*.msg" "doorbell should name all unhandled records"
+  assert_contains "$doorbell" "numeric order" "doorbell should require ordered processing"
+  assert_contains "$doorbell" "$state/t1.inbox/handled/" "doorbell should name the handled dir"
+  assert_contains "$doorbell" "Firstmate instruction waiting" "doorbell should be self-describing"
+  case "$doorbell" in
+    *$'\n'*) fail "the doorbell must be a single line" ;;
+  esac
+  pass "inbox: a steer is written durably and round-trips byte-exact with a self-describing doorbell"
+}
+
+test_idempotent_write_dedups_exact_body() {
+  local state r1 r2 r3 r4 count text
+  state="$TMP_ROOT/idem/state"; mkdir -p "$state"
+  text=$'re-runnable steer\nsecond line'
+  r1=$(inbox_lib "$state" fm_task_inbox_write_idempotent "$state" t1 "$text") \
+    || fail "idempotent write failed"
+  [ "$r1" = "$state/t1.inbox/001.msg" ] || fail "first idempotent write should create 001.msg, got $r1"
+  # Re-running the same enqueue (the safe recovery after an ambiguous remote
+  # transport failure) lands on the SAME record, never a duplicate.
+  r2=$(inbox_lib "$state" fm_task_inbox_write_idempotent "$state" t1 "$text") \
+    || fail "idempotent re-run failed"
+  [ "$r2" = "$r1" ] || fail "an identical re-run should return the existing record, got $r2"
+  count=$(find "$state/t1.inbox" -maxdepth 1 -name '*.msg' | wc -l | tr -d ' ')
+  [ "$count" = 1 ] || fail "an identical re-run must not enqueue a duplicate, found $count records"
+  # A different body - two logical requests differ at least by their embedded
+  # correlation token - still enqueues normally.
+  r3=$(inbox_lib "$state" fm_task_inbox_write_idempotent "$state" t1 $'re-runnable steer\nsecond line changed') \
+    || fail "idempotent write of a different body failed"
+  [ "$r3" = "$state/t1.inbox/002.msg" ] || fail "a different body should enqueue a new record, got $r3"
+  # A body the worker already acknowledged still dedups: the re-run reports
+  # the handled record rather than re-delivering an instruction that was
+  # already acted on.
+  mv "$r1" "$state/t1.inbox/handled/"
+  r4=$(inbox_lib "$state" fm_task_inbox_write_idempotent "$state" t1 "$text") \
+    || fail "idempotent re-run after the ack failed"
+  [ "$r4" = "$state/t1.inbox/handled/001.msg" ] \
+    || fail "a re-run of an acknowledged steer should land on the handled record, got $r4"
+  count=$(find "$state/t1.inbox" -maxdepth 1 -name '*.msg' | wc -l | tr -d ' ')
+  [ "$count" = 1 ] || fail "a re-run of an acknowledged steer must not re-enqueue it, found $count unhandled records"
+  pass "inbox: the idempotent enqueue dedups an exact re-run onto the same record, handled or not"
+}
+
+test_idempotent_write_follows_concurrent_ack() {
+  local state rec result count text
+  state="$TMP_ROOT/idem-ack-race/state"; mkdir -p "$state"
+  text="acknowledge while dedup scans"
+  rec=$(inbox_lib "$state" fm_task_inbox_write_idempotent "$state" t1 "$text") \
+    || fail "race fixture write failed"
+  result=$(FM_STATE_OVERRIDE="$state" bash -c '
+    . "$1"
+    eval "$(declare -f fm_task_inbox_body | sed "1s/fm_task_inbox_body/_original_fm_task_inbox_body/")"
+    fm_task_inbox_body() {
+      candidate=$1
+      case "$candidate" in
+        */handled/*) ;;
+        *) mv "$candidate" "${candidate%/*}/handled/" || return 1
+           candidate="${candidate%/*}/handled/${candidate##*/}" ;;
+      esac
+      _original_fm_task_inbox_body "$candidate"
+    }
+    fm_task_inbox_write_idempotent "$2" t1 "$3"
+  ' _ "$ROOT/bin/fm-task-inbox-lib.sh" "$state" "$text") \
+    || fail "idempotent enqueue failed while acknowledgement moved its candidate"
+  [ "$result" = "$state/t1.inbox/handled/${rec##*/}" ] \
+    || fail "dedup did not follow the concurrently acknowledged record: $result"
+  count=$(find "$state/t1.inbox" -name '*.msg' | wc -l | tr -d ' ')
+  [ "$count" = 1 ] || fail "acknowledgement racing dedup created a duplicate record"
+  pass "inbox: idempotent enqueue follows a record concurrently moved to handled"
+}
+
+test_handled_mv_dedups_by_sequence() {
+  local state r1 r2 oldest r3
+  state="$TMP_ROOT/dedup/state"; mkdir -p "$state"
+  r1=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "first")
+  r2=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "second")
+  [ "$r2" = "$state/t1.inbox/002.msg" ] || fail "second record should be 002.msg, got $r2"
+  oldest=$(inbox_lib "$state" fm_task_inbox_oldest_unhandled "$state" t1)
+  [ "$oldest" = "$r1" ] || fail "oldest unhandled should be 001, got $oldest"
+  mv "$r1" "$state/t1.inbox/handled/"
+  oldest=$(inbox_lib "$state" fm_task_inbox_oldest_unhandled "$state" t1)
+  [ "$oldest" = "$r2" ] || fail "after the ack mv the oldest should advance to 002, got $oldest"
+  # Re-acking the same message is a no-op: the record is already retired and
+  # nothing re-lists it as unhandled.
+  mv "$state/t1.inbox/001.msg" "$state/t1.inbox/handled/" 2>/dev/null \
+    && fail "a second mv of an acked record should find nothing to move"
+  mv "$r2" "$state/t1.inbox/handled/"
+  if inbox_lib "$state" fm_task_inbox_oldest_unhandled "$state" t1 >/dev/null; then
+    fail "a fully handled inbox should report no unhandled record"
+  fi
+  # An acknowledged sequence is never reissued, so a message is processed at
+  # most once per worker lifetime even if every doorbell is duplicated.
+  r3=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "third")
+  [ "$r3" = "$state/t1.inbox/003.msg" ] || fail "a handled sequence was reissued: $r3"
+  pass "inbox: the handled mv is the idempotent ack and sequences are never reissued"
+}
+
+test_concurrent_writers_never_clobber() {
+  local state i pids=() count
+  state="$TMP_ROOT/race/state"; mkdir -p "$state"
+  for i in 1 2 3 4 5 6; do
+    inbox_lib "$state" fm_task_inbox_write "$state" t1 "steer number $i" >/dev/null &
+    pids+=($!)
+  done
+  for i in "${pids[@]}"; do
+    wait "$i" || fail "a concurrent inbox write failed"
+  done
+  count=$(find "$state/t1.inbox" -maxdepth 1 -name '*.msg' | wc -l | tr -d ' ')
+  [ "$count" = 6 ] || fail "6 concurrent writes should yield 6 records, got $count:"$'\n'"$(ls "$state/t1.inbox")"
+  for i in 1 2 3 4 5 6; do
+    grep -rqF "steer number $i" "$state/t1.inbox" \
+      || fail "steer number $i was lost in the concurrent write race"
+  done
+  pass "inbox: concurrent writers serialize on the sequence lock and lose nothing"
+}
+
+test_ladder_writes_ignore_vanished_inbox() {
+  local state rec
+  state="$TMP_ROOT/vanished/state"; mkdir -p "$state"
+  rec=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "retired task")
+  rm -rf "$state/t1.inbox"
+  inbox_lib "$state" fm_task_inbox_record_ring "$state" t1 "$rec" \
+    || fail "ring bookkeeping should ignore a concurrently removed inbox"
+  inbox_lib "$state" fm_task_inbox_record_escalated "$state" t1 "$rec" \
+    || fail "escalation bookkeeping should ignore a concurrently removed inbox"
+  [ ! -e "$state/t1.inbox" ] || fail "bookkeeping recreated a retired task inbox"
+  pass "inbox: ladder bookkeeping ignores a concurrently removed inbox"
+}
+
+test_fire_and_forget_records_never_enter_the_ladder() {
+  local state fire tracked action
+  state="$TMP_ROOT/fire-and-forget/state"; mkdir -p "$state"
+  fire=$(inbox_lib "$state" fm_task_inbox_write_idempotent "$state" t1 "one-shot steer" fire-and-forget)
+  age_path "$fire"
+  action=$(FM_TASK_INBOX_GRACE_SECS=0 FM_TASK_INBOX_RING_MAX=0 \
+    inbox_lib "$state" fm_task_inbox_due_action "$state" t1)
+  [ "$action" = quiet ] || fail "a fire-and-forget record entered the re-ring ladder: $action"
+  tracked=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "tracked steer")
+  age_path "$tracked"
+  action=$(FM_TASK_INBOX_GRACE_SECS=0 FM_TASK_INBOX_RING_MAX=0 \
+    inbox_lib "$state" fm_task_inbox_due_action "$state" t1)
+  [ "$action" = "escalate $tracked 0" ] \
+    || fail "a fire-and-forget record hid the later tracked steer: $action"
+  [ -f "$fire" ] || fail "excluding fire-and-forget from escalation removed its durable record"
+  pass "inbox: fire-and-forget records stay durable and outside the ladder"
+}
+
+test_ring_ladder_policy() {
+  local state rec action
+  state="$TMP_ROOT/ladder/state"; mkdir -p "$state"
+  rec=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "do the thing")
+  # Within grace: quiet.
+  action=$(FM_TASK_INBOX_GRACE_SECS=3600 inbox_lib "$state" fm_task_inbox_due_action "$state" t1)
+  [ "$action" = quiet ] || fail "a fresh unhandled message inside grace should be quiet, got: $action"
+  # Past grace: one ring is due.
+  age_path "$rec"
+  action=$(FM_TASK_INBOX_GRACE_SECS=60 inbox_lib "$state" fm_task_inbox_due_action "$state" t1)
+  [ "$action" = "ring $rec" ] || fail "an aged unhandled message should be due a ring, got: $action"
+  # A just-recorded ring holds the spacing: quiet until another grace elapses.
+  inbox_lib "$state" fm_task_inbox_record_ring "$state" t1 "$rec"
+  action=$(FM_TASK_INBOX_GRACE_SECS=60 inbox_lib "$state" fm_task_inbox_due_action "$state" t1)
+  [ "$action" = quiet ] || fail "a ring within the spacing window should be quiet, got: $action"
+  # Backdate the ladder: the next ring becomes due, and at the budget the
+  # action turns into a single escalation.
+  printf '001.msg\t1\t100\n' > "$state/t1.inbox/.ring-state"
+  action=$(FM_TASK_INBOX_GRACE_SECS=60 FM_TASK_INBOX_RING_MAX=3 inbox_lib "$state" fm_task_inbox_due_action "$state" t1)
+  [ "$action" = "ring $rec" ] || fail "an aged ladder should ring again, got: $action"
+  printf '001.msg\t3\t100\n' > "$state/t1.inbox/.ring-state"
+  action=$(FM_TASK_INBOX_GRACE_SECS=60 FM_TASK_INBOX_RING_MAX=3 inbox_lib "$state" fm_task_inbox_due_action "$state" t1)
+  [ "$action" = "escalate $rec 3" ] || fail "a spent ring budget should escalate, got: $action"
+  # Escalation fires at most once per message.
+  inbox_lib "$state" fm_task_inbox_record_escalated "$state" t1 "$rec"
+  action=$(FM_TASK_INBOX_GRACE_SECS=60 FM_TASK_INBOX_RING_MAX=3 inbox_lib "$state" fm_task_inbox_due_action "$state" t1)
+  [ "$action" = quiet ] || fail "an escalated message should stay quiet for recovery, got: $action"
+  # The acknowledgement resets the ladder: the next message starts fresh.
+  mv "$rec" "$state/t1.inbox/handled/"
+  action=$(FM_TASK_INBOX_GRACE_SECS=60 inbox_lib "$state" fm_task_inbox_due_action "$state" t1)
+  [ "$action" = quiet ] || fail "a handled inbox should be quiet, got: $action"
+  [ ! -e "$state/t1.inbox/.escalated" ] || fail "the ack should clear the escalation marker"
+  rec=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "next thing")
+  age_path "$rec"
+  action=$(FM_TASK_INBOX_GRACE_SECS=60 FM_TASK_INBOX_RING_MAX=3 inbox_lib "$state" fm_task_inbox_due_action "$state" t1)
+  [ "$action" = "ring $rec" ] || fail "the next message should start a fresh ladder, got: $action"
+  pass "inbox: the re-ring ladder paces by grace, escalates once, and resets on ack"
+}
+
+setup_watch_case() {  # <name> -> echoes case dir; state in <dir>/state
+  local name=$1 dir
+  dir="$TMP_ROOT/$name"
+  mkdir -p "$dir/state"
+  make_watch_stubs "$dir" >/dev/null
+  fm_write_meta "$dir/state/t1.meta" "window=sess:fm-t1" "kind=ship" "harness=grok"
+  printf '%s\n' "$dir"
+}
+
+idle_capture() {  # <dir>
+  printf '╭────╮\n│    │\n╰────╯\n' > "$1/idle.capture"
+  printf '%s\n' "$1/idle.capture"
+}
+
+test_watcher_rerings_idle_pane_quietly() {
+  local dir state out log pid rec
+  dir=$(setup_watch_case rering)
+  state="$dir/state"; out="$dir/watch.out"; log="$dir/send.log"; : > "$log"
+  rec=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "please continue")
+  age_path "$rec"
+  watch_bg "$state" "$dir/fakebin" "$out" \
+    FM_SEND_LOG="$log" FM_FAKE_TMUX_CAPTURE="$(idle_capture "$dir")" \
+    FM_TASK_INBOX_RING_MAX=99
+  pid=$!
+  local i=0
+  while [ "$i" -lt 100 ]; do
+    grep -qF 'Firstmate instruction waiting' "$log" 2>/dev/null && break
+    kill -0 "$pid" 2>/dev/null || break
+    sleep 0.1
+    i=$((i + 1))
+  done
+  grep -qF "Firstmate instruction waiting: list $state/t1.inbox/*.msg" "$log" \
+    || { kill "$pid" 2>/dev/null; fail "the watcher never re-rang the doorbell:"$'\n'"$(cat "$log")"; }
+  kill -0 "$pid" 2>/dev/null \
+    || fail "a healthy re-ring must not wake firstmate (watcher exited):"$'\n'"$(cat "$out")"
+  [ ! -s "$state/.wake-queue" ] \
+    || { kill "$pid" 2>/dev/null; fail "a healthy re-ring queued a wake:"$'\n'"$(cat "$state/.wake-queue")"; }
+  # The acknowledgement silences the ladder: no further doorbells after the mv.
+  mv "$rec" "$state/t1.inbox/handled/"
+  sleep 2.5
+  : > "$log"
+  sleep 2.5
+  kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null
+  [ ! -s "$log" ] || fail "the watcher kept ringing after the ack:"$'\n'"$(cat "$log")"
+  pass "watcher: an unhandled aged message on an idle pane re-rings without waking firstmate, and the ack silences it"
+}
+
+test_watcher_waits_on_busy_pane() {
+  local dir state out log pid rec
+  dir=$(setup_watch_case busywait)
+  state="$dir/state"; out="$dir/watch.out"; log="$dir/send.log"; : > "$log"
+  printf 'some output\nBUSYTOKEN active\n' > "$dir/busy.capture"
+  rec=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "please continue")
+  age_path "$rec"
+  watch_bg "$state" "$dir/fakebin" "$out" \
+    FM_SEND_LOG="$log" FM_FAKE_TMUX_CAPTURE="$dir/busy.capture" \
+    FM_BUSY_REGEX=BUSYTOKEN FM_TASK_INBOX_RING_MAX=99
+  pid=$!
+  sleep 4
+  kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null
+  [ ! -s "$log" ] || fail "a busy pane should wait, not ring:"$'\n'"$(cat "$log")"
+  [ ! -s "$state/.wake-queue" ] || fail "a busy wait queued a wake:"$'\n'"$(cat "$state/.wake-queue")"
+  pass "watcher: a busy pane just waits - the record is durable and no doorbell is typed"
+}
+
+test_watcher_quiet_on_healthy_inbox() {
+  local dir state out log pid
+  dir=$(setup_watch_case healthy)
+  state="$dir/state"; out="$dir/watch.out"; log="$dir/send.log"; : > "$log"
+  mkdir -p "$state/t1.inbox/handled"
+  watch_bg "$state" "$dir/fakebin" "$out" \
+    FM_SEND_LOG="$log" FM_FAKE_TMUX_CAPTURE="$(idle_capture "$dir")" \
+    FM_TASK_INBOX_RING_MAX=99
+  pid=$!
+  sleep 4
+  kill -0 "$pid" 2>/dev/null || fail "the watcher exited on a healthy empty inbox:"$'\n'"$(cat "$out")"
+  kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null
+  [ ! -s "$log" ] || fail "an empty inbox rang a doorbell:"$'\n'"$(cat "$log")"
+  [ ! -s "$state/.wake-queue" ] || fail "an empty inbox queued a wake:"$'\n'"$(cat "$state/.wake-queue")"
+  pass "watcher: a healthy or empty inbox stays completely silent"
+}
+
+test_watcher_ack_silences_unwritable_ladder() {
+  local dir state out log pid rec rings i=0
+  dir=$(setup_watch_case ack-unwritable-ladder)
+  state="$dir/state"; out="$dir/watch.out"; log="$dir/send.log"; : > "$log"
+  rec=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "please continue")
+  age_path "$rec"
+  mkdir "$state/t1.inbox/.ring-state"
+  watch_bg "$state" "$dir/fakebin" "$out" \
+    FM_SEND_LOG="$log" FM_FAKE_TMUX_CAPTURE="$(idle_capture "$dir")" \
+    FM_ACK_RECORD="$rec" FM_TASK_INBOX_RING_MAX=99
+  pid=$!
+  while [ "$i" -lt 100 ]; do
+    [ -f "$state/t1.inbox/handled/001.msg" ] && break
+    kill -0 "$pid" 2>/dev/null || break
+    sleep 0.1
+    i=$((i + 1))
+  done
+  [ -f "$state/t1.inbox/handled/001.msg" ] \
+    || { kill "$pid" 2>/dev/null; fail "the doorbell stub did not acknowledge the record"; }
+  sleep 2
+  kill -0 "$pid" 2>/dev/null \
+    || fail "the watcher escalated ladder failure after the record was acknowledged:"$'\n'"$(cat "$out")"
+  kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null
+  rings=$(grep -cF 'Firstmate instruction waiting' "$log" || true)
+  [ "$rings" = 1 ] || fail "acknowledgement should silence retries, got $rings doorbells:"$'\n'"$(cat "$log")"
+  [ ! -s "$state/.wake-queue" ] \
+    || fail "an acknowledged record queued a bookkeeping wake:"$'\n'"$(cat "$state/.wake-queue")"
+  pass "watcher: acknowledgement silences an unwritable ladder without a stale wake"
+}
+
+test_watcher_surfaces_unwritable_ladder() {
+  local dir state out log pid rec rings wakes
+  dir=$(setup_watch_case unwritable-ladder)
+  state="$dir/state"; out="$dir/watch.out"; log="$dir/send.log"; : > "$log"
+  rec=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "please continue")
+  age_path "$rec"
+  mkdir "$state/t1.inbox/.ring-state"
+  watch_bg "$state" "$dir/fakebin" "$out" \
+    FM_SEND_LOG="$log" FM_FAKE_TMUX_CAPTURE="$(idle_capture "$dir")" \
+    FM_TASK_INBOX_RING_MAX=99
+  pid=$!
+  wait_watcher_gone "$pid" \
+    || { kill "$pid" 2>/dev/null; fail "the watcher silently retried with unwritable ladder bookkeeping"; }
+  rings=$(grep -cF 'Firstmate instruction waiting' "$log" || true)
+  [ "$rings" = 1 ] || fail "expected one doorbell before the bookkeeping wake, got $rings:"$'\n'"$(cat "$log")"
+  wakes=$(grep -cF 'steering-inbox ladder bookkeeping unwritable' "$state/.wake-queue" || true)
+  [ "$wakes" = 1 ] \
+    || fail "expected exactly one bookkeeping-unwritable stale wake, got $wakes:"$'\n'"$(cat "$state/.wake-queue" 2>/dev/null)"
+  grep -qF "$state/t1.inbox/.ring-state cannot be written" "$state/.wake-queue" \
+    || fail "the stale wake did not identify the unwritable ladder:"$'\n'"$(cat "$state/.wake-queue")"
+  [ -f "$rec" ] || fail "the unhandled record disappeared during bookkeeping failure"
+  grep -qF 'stale:' "$out" \
+    || fail "the watcher should exit through the ordinary stale wake:"$'\n'"$(cat "$out")"
+  pass "watcher: unwritable ladder bookkeeping surfaces a stale wake after the doorbell"
+}
+
+test_watcher_escalates_once_after_budget() {
+  local dir state out log pid rec rings
+  dir=$(setup_watch_case escalate)
+  state="$dir/state"; out="$dir/watch.out"; log="$dir/send.log"; : > "$log"
+  rec=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "please continue")
+  age_path "$rec"
+  watch_bg "$state" "$dir/fakebin" "$out" \
+    FM_SEND_LOG="$log" FM_FAKE_TMUX_CAPTURE="$(idle_capture "$dir")" \
+    FM_TASK_INBOX_RING_MAX=1
+  pid=$!
+  wait_watcher_gone "$pid" \
+    || { kill "$pid" 2>/dev/null; fail "the watcher never escalated a spent ring budget"; }
+  rings=$(grep -cF 'Firstmate instruction waiting' "$log" || true)
+  [ "$rings" = 1 ] || fail "expected exactly 1 doorbell before escalation, got $rings:"$'\n'"$(cat "$log")"
+  grep -qF 'unread firstmate instruction' "$state/.wake-queue" \
+    || fail "the escalation should queue a stale wake naming the unread instruction:"$'\n'"$(cat "$state/.wake-queue" 2>/dev/null)"
+  grep -qF "$rec" "$state/.wake-queue" \
+    || fail "the stale wake should name the record path:"$'\n'"$(cat "$state/.wake-queue")"
+  [ "$(grep -cF 'unread firstmate instruction' "$state/.wake-queue")" = 1 ] \
+    || fail "the escalation must fire exactly once:"$'\n'"$(cat "$state/.wake-queue")"
+  grep -qF 'stale:' "$out" || fail "the watcher should exit through the ordinary stale wake:"$'\n'"$(cat "$out")"
+  pass "watcher: a spent ring budget emits exactly one ordinary stale wake for recovery"
+}
+
+test_write_is_durable_and_exact
+test_idempotent_write_dedups_exact_body
+test_idempotent_write_follows_concurrent_ack
+test_handled_mv_dedups_by_sequence
+test_concurrent_writers_never_clobber
+test_ladder_writes_ignore_vanished_inbox
+test_fire_and_forget_records_never_enter_the_ladder
+test_ring_ladder_policy
+test_watcher_rerings_idle_pane_quietly
+test_watcher_waits_on_busy_pane
+test_watcher_quiet_on_healthy_inbox
+test_watcher_ack_silences_unwritable_ladder
+test_watcher_surfaces_unwritable_ladder
+test_watcher_escalates_once_after_budget