@zerwiz/ymir 0.1.3 → 0.1.5

package/.agents/tests/fm-backend-herdr-prune-safety-e2e.test.sh

@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+# tests/fm-backend-herdr-prune-safety-e2e.test.sh - isolated real-herdr
+# regression test for the 2026-07-02 self-kill incident and its fix
+# (bin/backends/herdr.sh's created-vs-adopted default-tab-prune gate; see
+# docs/herdr-backend.md "Default-tab prune" / the incident writeup there).
+#
+# Reproduces the exact collision shape against a private, throwaway
+# HERDR_SESSION (never the captain's default): a startup-workspace-shaped
+# layout - one tab labeled "1" in a pre-existing workspace labeled
+# "firstmate" - with a live long-running process in that pane, exactly as
+# the captain's own live crewmate session looked at incident time. Then
+# drives the real spawn-time container_ensure +
+# create_task path and asserts the live pane (and its live process) survive
+# untouched. Also exercises the normal happy path (a genuinely fresh
+# workspace's seeded default tab gets pruned, leaving exactly one clean
+# fm-<id> task tab), mirroring tests/fm-backend-herdr-smoke.test.sh's broader
+# coverage but scoped tightly to this one safety property.
+#
+# Safety (tests/herdr-test-safety.sh): cleanup uses ONLY
+# herdr_safe_stop_and_delete, never a bare/inline-prefixed `herdr server
+# stop` - the exact category of unscoped destructive call that caused the
+# 2026-07-02 incident in the first place.
+set -u
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; }
+pass() { printf 'ok - %s\n' "$1"; }
+
+command -v herdr >/dev/null 2>&1 || { echo "skip: herdr not found"; exit 0; }
+command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the herdr adapter)"; exit 0; }
+
+# shellcheck source=tests/herdr-test-safety.sh
+. "$ROOT/tests/herdr-test-safety.sh"
+
+# This suite runs against its own isolated lab session, so a Herdr pane
+# inherited from the terminal it was launched in must not follow spawn into it
+# as a cross-session parent identity (tests/herdr-test-safety.sh).
+herdr_forget_inherited_pane
+
+SESSION="fm-lab-prune-safety-e2e-$$"
+export HERDR_SESSION="$SESSION"
+SCRATCH=$(mktemp -d "${TMPDIR:-/tmp}/fm-herdr-prune-safety.XXXXXX")
+cleanup_all() {
+  herdr_safe_stop_and_delete "$SESSION"
+  rm -rf "$SCRATCH"
+}
+trap cleanup_all EXIT
+fm_herdr_lab_prepare "$SESSION" || fail "could not prepare isolated Herdr lab session"
+
+# shellcheck source=/dev/null
+. "$ROOT/bin/fm-backend.sh"
+fm_backend_source herdr || fail "fm_backend_source herdr failed"
+
+fm_backend_herdr_version_check || fail "version_check failed against the real installed herdr"
+
+# --- 1. reproduce the label-collision startup-workspace shape ---------------
+# Explicitly label the startup workspace "firstmate" to create the collision
+# deterministically. Herdr's unlabeled workspace-label derivation is not a
+# stable test contract, while the adopted-workspace state is the behavior
+# this regression must exercise. The seeded tab remains labeled "1".
+
+LIVE_CWD="$SCRATCH/firstmate"
+mkdir -p "$LIVE_CWD"
+
+fm_backend_herdr_server_ensure "$SESSION" || fail "could not start the isolated session's server"
+
+CREATE_OUT=$(fm_backend_herdr_cli "$SESSION" workspace create --cwd "$LIVE_CWD" --label firstmate --no-focus) \
+  || fail "could not create the label-collision startup workspace"
+LIVE_WSID=$(printf '%s' "$CREATE_OUT" | jq -r '.result.workspace.workspace_id // empty')
+LIVE_TAB_ID=$(printf '%s' "$CREATE_OUT" | jq -r '.result.tab.tab_id // empty')
+LIVE_PANE_ID=$(printf '%s' "$CREATE_OUT" | jq -r '.result.root_pane.pane_id // empty')
+if [ -z "$LIVE_WSID" ] || [ -z "$LIVE_TAB_ID" ] || [ -z "$LIVE_PANE_ID" ]; then
+  fail "could not parse the startup workspace's ids from workspace create: $CREATE_OUT"
+fi
+
+LIVE_LABEL=$(herdr workspace list --session "$SESSION" 2>&1 | jq -r --arg id "$LIVE_WSID" '.result.workspaces[]? | select(.workspace_id == $id) | .label')
+[ "$LIVE_LABEL" = firstmate ] || fail "the startup workspace label should be 'firstmate', got '$LIVE_LABEL' - repro setup is wrong"
+pass "repro setup: a pre-existing workspace labeled 'firstmate' collides with the primary home's own label"
+
+# Simulate a live long-running agent in that pane: a heartbeat loop that
+# appends to a marker file, so liveness is independently verifiable (not just
+# "the pane object still exists").
+PANE_READY=false
+READY_SAMPLES=0
+for _ in $(seq 1 100); do
+  PROCESS_INFO=$(fm_backend_herdr_cli "$SESSION" pane process-info --pane "$LIVE_PANE_ID" 2>/dev/null || true)
+  if printf '%s' "$PROCESS_INFO" | jq -e '
+    .result.process_info as $process
+    | ($process.foreground_processes | length == 1)
+      and ($process.foreground_processes[0].pid == $process.shell_pid)
+  ' >/dev/null 2>&1; then
+    READY_SAMPLES=$((READY_SAMPLES + 1))
+    if [ "$READY_SAMPLES" -ge 10 ]; then
+      PANE_READY=true
+      break
+    fi
+  else
+    READY_SAMPLES=0
+  fi
+  sleep 0.1
+done
+[ "$PANE_READY" = true ] || fail "the startup workspace's shell did not become ready"
+
+MARKER="$SCRATCH/heartbeat.log"
+fm_backend_herdr_cli "$SESSION" pane run "$LIVE_PANE_ID" \
+  "sh -c 'while true; do date +%s >> $MARKER; sleep 1; done'" >/dev/null 2>&1 \
+  || fail "could not start the live heartbeat process in the startup workspace's pane"
+sleep 2
+[ -s "$MARKER" ] || fail "the live heartbeat process did not start writing its marker file"
+BEFORE_COUNT=$(wc -l < "$MARKER" | tr -d '[:space:]')
+pass "repro setup: a live long-running process is running in the startup workspace's single tab (label '1'), heartbeating to a marker file"
+
+# --- 2. run the real spawn-time path: container_ensure adopts the startup --
+# workspace by label match; create_task must NOT prune its tab.
+
+RAW=$(fm_backend_herdr_container_ensure "$LIVE_CWD") || fail "container_ensure failed"
+CONTAINER=${RAW%%$'\t'*}
+SEEDED_TAB_ID=${RAW#*$'\t'}
+[ "$CONTAINER" = "$SESSION:$LIVE_WSID" ] || fail "container_ensure should have ADOPTED the pre-existing label-colliding workspace ($LIVE_WSID), got '$CONTAINER'"
+[ -z "$SEEDED_TAB_ID" ] || fail "an ADOPTED workspace must report an EMPTY seeded default tab id (the created-vs-adopted gate), got '$SEEDED_TAB_ID' - this is exactly what would reproduce the 2026-07-02 self-kill"
+pass "fixed: container_ensure adopts the label-colliding startup workspace and reports NO seeded default tab (never a prune candidate)"
+
+TASK_IDS=$(fm_backend_herdr_create_task "$CONTAINER" fm-prunesafety-e2e "$LIVE_CWD" "$SEEDED_TAB_ID") \
+  || fail "create_task failed"
+read -r NEW_TAB_ID NEW_PANE_ID <<EOF
+$TASK_IDS
+EOF
+if [ -z "$NEW_TAB_ID" ] || [ -z "$NEW_PANE_ID" ]; then
+  fail "create_task did not return tab/pane ids"
+fi
+
+# --- 3. assert the live pane survived untouched -----------------------------
+
+if ! herdr pane get "$LIVE_PANE_ID" --session "$SESSION" >/dev/null 2>&1; then
+  fail "REGRESSION (2026-07-02 self-kill): the live startup-workspace pane was CLOSED by create_task"
+fi
+sleep 2
+AFTER_COUNT=$(wc -l < "$MARKER" | tr -d '[:space:]')
+[ "$AFTER_COUNT" -gt "$BEFORE_COUNT" ] \
+  || fail "REGRESSION: the live heartbeat process stopped writing after create_task ran - it was killed even though its pane object survived"
+pass "fixed: the live pane (and its live process) survived create_task untouched - the exact 2026-07-02 self-kill incident does not reproduce"
+
+LIVE_TABS_AFTER=$(herdr tab list --workspace "$LIVE_WSID" --session "$SESSION" 2>&1)
+printf '%s' "$LIVE_TABS_AFTER" | jq -e --arg t "$LIVE_TAB_ID" '.result.tabs[] | select(.tab_id == $t)' >/dev/null 2>&1 \
+  || fail "REGRESSION: the startup workspace's original live tab is gone from tab list"
+pass "fixed: the startup workspace's original live tab is still present in tab list after the spawn"
+
+fm_backend_herdr_kill "$SESSION:$NEW_PANE_ID"
+fm_backend_herdr_kill "$SESSION:$LIVE_PANE_ID"
+
+# --- 4. happy path still works: a genuinely fresh workspace gets its seeded -
+# default tab pruned, leaving exactly one clean fm-<id> task tab -------------
+
+HAPPY_CWD="$SCRATCH/happy-project"
+mkdir -p "$HAPPY_CWD"
+HAPPY_RAW=$(fm_backend_herdr_container_ensure "$HAPPY_CWD") || fail "happy-path container_ensure failed"
+HAPPY_CONTAINER=${HAPPY_RAW%%$'\t'*}
+HAPPY_SEEDED=${HAPPY_RAW#*$'\t'}
+[ -n "$HAPPY_SEEDED" ] || fail "happy path: expected a genuinely fresh workspace with a non-empty seeded default tab id"
+
+HAPPY_TASK_IDS=$(fm_backend_herdr_create_task "$HAPPY_CONTAINER" fm-prunesafety-happy "$HAPPY_CWD" "$HAPPY_SEEDED") \
+  || fail "happy-path create_task failed"
+read -r _HAPPY_TAB HAPPY_PANE <<EOF
+$HAPPY_TASK_IDS
+EOF
+[ -n "$HAPPY_PANE" ] || fail "happy-path create_task did not return a pane id"
+
+HAPPY_WSID=${HAPPY_CONTAINER#*:}
+HAPPY_TABS=$(herdr tab list --workspace "$HAPPY_WSID" --session "$SESSION" 2>&1)
+HAPPY_COUNT=$(printf '%s' "$HAPPY_TABS" | jq -r '.result.tabs? // [] | length')
+[ "$HAPPY_COUNT" = 1 ] || fail "happy path: expected exactly 1 tab (seeded default pruned) after the first real task tab, got $HAPPY_COUNT: $HAPPY_TABS"
+printf '%s' "$HAPPY_TABS" | jq -e --arg t "$HAPPY_SEEDED" '.result.tabs[] | select(.tab_id == $t)' >/dev/null 2>&1 \
+  && fail "happy path: the seeded default tab should have been pruned but is still present: $HAPPY_TABS"
+pass "happy path: a genuinely fresh workspace's seeded default tab is still pruned, leaving exactly one clean fm-<id> task tab"
+
+fm_backend_herdr_kill "$SESSION:$HAPPY_PANE"
+
+cleanup_all
+trap - EXIT

package/.agents/tests/fm-backend-herdr-respawn-idem-e2e.test.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# tests/fm-backend-herdr-respawn-idem-e2e.test.sh - isolated real-herdr
+# regression test for firstmate-restart idempotency against herdr's
+# restored-layout husks (docs/herdr-backend.md "Known gaps" / "ID stability
+# across a server restart").
+#
+# herdr persists its whole session layout (workspaces/tabs/panes) and
+# restores it after a server restart, including a reboot. Before this fix, a
+# restored fm-<id> task tab came back a husk - a dead pane, or a plain
+# agent-less shell in the saved cwd - and bin/backends/herdr.sh's
+# fm_backend_herdr_create_task refused to spawn into it unconditionally,
+# because a same-labeled tab already existed. Every fleet respawn after a
+# real herdr server restart needed the operator to manually close each husk
+# pane first (this reproduced again on 2026-07-03).
+#
+# This test drives a REAL `herdr session stop` + fresh `herdr server` restart
+# (the same "ID stability" mechanism docs/herdr-backend.md already documents:
+# the pane survives alive, but agent_status resets and nothing is registered
+# in it - exactly the restored-plain-shell husk shape), then proves
+# fm_backend_herdr_create_task now closes-and-replaces the resulting husk
+# instead of refusing, while a GENUINELY live duplicate (a real registered
+# agent, via herdr's own `pane report-agent`) still refuses exactly as
+# before. Adapter-level (fm_backend_herdr_container_ensure/create_task), not
+# through the full bin/fm-spawn.sh + treehouse pipeline - mirrors
+# tests/fm-backend-herdr-prune-safety-e2e.test.sh's own style, and avoids any
+# question of whether treehouse itself supports re-acquiring a worktree for
+# an id that already has one checked out (a separate, out-of-scope concern).
+#
+# Safety (tests/herdr-test-safety.sh): cleanup uses ONLY
+# herdr_safe_stop_and_delete, never a bare/inline-prefixed `herdr server
+# stop` - the exact category of unscoped destructive call that caused the
+# 2026-07-02 incident. Every herdr lifecycle call in this file targets this
+# test's own isolated $SESSION explicitly via --session; the live `default`
+# session is never touched.
+set -u
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; }
+pass() { printf 'ok - %s\n' "$1"; }
+
+command -v herdr >/dev/null 2>&1 || { echo "skip: herdr not found"; exit 0; }
+command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the herdr adapter)"; exit 0; }
+
+# shellcheck source=tests/herdr-test-safety.sh
+. "$ROOT/tests/herdr-test-safety.sh"
+
+# This suite runs against its own isolated lab session, so a Herdr pane
+# inherited from the terminal it was launched in must not follow spawn into it
+# as a cross-session parent identity (tests/herdr-test-safety.sh).
+herdr_forget_inherited_pane
+
+SESSION="fm-lab-respawn-idem-e2e-$$"
+export HERDR_SESSION="$SESSION"
+SCRATCH=$(mktemp -d "${TMPDIR:-/tmp}/fm-herdr-respawn-idem.XXXXXX")
+cleanup_all() {
+  herdr_safe_stop_and_delete "$SESSION"
+  rm -rf "$SCRATCH"
+}
+trap cleanup_all EXIT
+fm_herdr_lab_prepare "$SESSION" || fail "could not prepare isolated Herdr lab session"
+
+# shellcheck source=/dev/null
+. "$ROOT/bin/fm-backend.sh"
+fm_backend_source herdr || fail "fm_backend_source herdr failed"
+
+fm_backend_herdr_version_check || fail "version_check failed against the real installed herdr"
+
+# --- 1. spawn two real task tabs (crewmate-shaped and secondmate-shaped) ----
+# fm_backend_herdr_create_task is the ONE function both bin/fm-spawn.sh's
+# ordinary crewmate/scout path and its --secondmate path call, so exercising
+# it directly here covers both paths identically - already proven distinct
+# only in FM_HOME-shadowing (tests/fm-backend-herdr-workspace-per-home-e2e.test.sh),
+# never in this duplicate-guard logic, which has no home-specific branching.
+
+PROJ_CWD="$SCRATCH/proj"
+mkdir -p "$PROJ_CWD"
+
+RAW=$(fm_backend_herdr_container_ensure "$PROJ_CWD") || fail "container_ensure failed"
+CONTAINER=${RAW%%$'\t'*}
+SEEDED_TAB_ID=${RAW#*$'\t'}
+WSID=${CONTAINER#*:}
+
+CREW_LABEL="fm-respawn-crew1"
+CREW_IDS=$(fm_backend_herdr_create_task "$CONTAINER" "$CREW_LABEL" "$PROJ_CWD" "$SEEDED_TAB_ID") \
+  || fail "initial crewmate-shaped task creation failed"
+read -r CREW_TAB_ID CREW_PANE_ID <<EOF
+$CREW_IDS
+EOF
+if [ -z "$CREW_TAB_ID" ] || [ -z "$CREW_PANE_ID" ]; then
+  fail "initial crewmate-shaped task did not return tab/pane ids"
+fi
+
+SM_LABEL="fm-respawn-sm1"
+SM_IDS=$(fm_backend_herdr_create_task "$CONTAINER" "$SM_LABEL" "$PROJ_CWD") \
+  || fail "initial secondmate-shaped task creation failed"
+read -r SM_TAB_ID SM_PANE_ID <<EOF
+$SM_IDS
+EOF
+if [ -z "$SM_TAB_ID" ] || [ -z "$SM_PANE_ID" ]; then
+  fail "initial secondmate-shaped task did not return tab/pane ids"
+fi
+
+pass "repro setup: two real fm-<id> task tabs exist (crewmate-shaped and secondmate-shaped), neither with a registered agent"
+
+# --- 2. a REAL herdr session restart - the actual root cause -----------------
+# `session stop` + fresh `herdr server` for the SAME named session: verified
+# in docs/herdr-backend.md "ID stability across a server restart" to preserve
+# every workspace/tab/pane id and label, while resetting each pane's
+# underlying process (a fresh shell) and its agent_status to unknown - the
+# exact husk shape a restored task tab comes back in.
+
+fm_herdr_lab_stop "$SESSION" >/dev/null 2>&1 \
+  || fail "could not stop the isolated session for the restart"
+sleep 0.5
+fm_backend_herdr_server_ensure "$SESSION" || fail "the isolated session's server did not come back up after the restart"
+
+if ! herdr pane get "$CREW_PANE_ID" --session "$SESSION" >/dev/null 2>&1; then
+  fail "repro setup is wrong: the crewmate-shaped pane should survive a session restart alive (docs/herdr-backend.md 'ID stability'), but it is gone"
+fi
+if herdr agent get "$CREW_PANE_ID" --session "$SESSION" >/dev/null 2>&1; then
+  fail "repro setup is wrong: the restored pane should have NO registered agent (agent_not_found expected)"
+fi
+pass "repro confirmed: after a real session restart, both task panes survive alive but with no registered agent - the restored-layout husk"
+
+# --- 3. BEFORE the fix this would refuse; now it closes-and-replaces -------
+
+RESPAWN_CREW_IDS=$(fm_backend_herdr_create_task "$CONTAINER" "$CREW_LABEL" "$PROJ_CWD") \
+  || fail "REGRESSION: create_task refused to respawn into the crewmate-shaped husk instead of closing-and-replacing it - this is the exact 2026-07-03 incident (manual pane close required)"
+read -r NEW_CREW_TAB_ID NEW_CREW_PANE_ID <<EOF
+$RESPAWN_CREW_IDS
+EOF
+if [ -z "$NEW_CREW_TAB_ID" ] || [ -z "$NEW_CREW_PANE_ID" ]; then
+  fail "husk respawn (crewmate-shaped) did not return new tab/pane ids"
+fi
+[ "$NEW_CREW_PANE_ID" != "$CREW_PANE_ID" ] || fail "husk respawn (crewmate-shaped) returned the SAME pane id - nothing was actually replaced"
+if herdr pane get "$CREW_PANE_ID" --session "$SESSION" >/dev/null 2>&1; then
+  fail "REGRESSION: the old crewmate-shaped husk pane should have been closed by close-and-replace, but it still exists"
+fi
+pass "fixed: create_task closes and replaces the crewmate-shaped restored husk instead of refusing - no manual pane close needed"
+
+RESPAWN_SM_IDS=$(fm_backend_herdr_create_task "$CONTAINER" "$SM_LABEL" "$PROJ_CWD") \
+  || fail "REGRESSION: create_task refused to respawn into the secondmate-shaped husk instead of closing-and-replacing it"
+read -r NEW_SM_TAB_ID NEW_SM_PANE_ID <<EOF
+$RESPAWN_SM_IDS
+EOF
+if [ -z "$NEW_SM_TAB_ID" ] || [ -z "$NEW_SM_PANE_ID" ]; then
+  fail "husk respawn (secondmate-shaped) did not return new tab/pane ids"
+fi
+[ "$NEW_SM_PANE_ID" != "$SM_PANE_ID" ] || fail "husk respawn (secondmate-shaped) returned the SAME pane id - nothing was actually replaced"
+if herdr pane get "$SM_PANE_ID" --session "$SESSION" >/dev/null 2>&1; then
+  fail "REGRESSION: the old secondmate-shaped husk pane should have been closed by close-and-replace, but it still exists"
+fi
+pass "fixed: create_task closes and replaces the secondmate-shaped restored husk instead of refusing - same fix, same function, both spawn shapes"
+
+WS_TABS=$(herdr tab list --workspace "$WSID" --session "$SESSION" 2>&1)
+WS_COUNT=$(printf '%s' "$WS_TABS" | jq -r '.result.tabs? // [] | length')
+[ "$WS_COUNT" = 2 ] || fail "expected exactly 2 tabs (the two replacements, husks closed, no leaks), got $WS_COUNT: $WS_TABS"
+pass "fixed: the workspace holds exactly the 2 replacement tabs after both respawns - no leaked husk tabs, no destroyed workspace"
+
+# --- 4. a GENUINELY live duplicate still refuses, unchanged -----------------
+# Register a real agent (herdr's own native registration primitive) on one of
+# the freshly-respawned panes, then confirm a further same-labeled spawn
+# attempt refuses exactly as before - the husk fix must never touch a pane
+# that actually has something registered in it.
+
+herdr pane report-agent "$NEW_CREW_PANE_ID" --source fm-respawn-e2e --agent fm-respawn-live-agent --state idle --session "$SESSION" >/dev/null 2>&1 \
+  || fail "could not register a live agent on the respawned crewmate-shaped pane"
+
+if fm_backend_herdr_create_task "$CONTAINER" "$CREW_LABEL" "$PROJ_CWD" >/dev/null 2>&1; then
+  fail "REGRESSION: create_task should refuse a same-labeled tab whose pane hosts a genuinely live registered agent"
+fi
+if ! herdr pane get "$NEW_CREW_PANE_ID" --session "$SESSION" >/dev/null 2>&1; then
+  fail "REGRESSION: the live-agent pane should have survived the refused create_task call untouched"
+fi
+pass "fixed: a genuinely live duplicate (a real registered agent) still refuses exactly as before - the husk fix never closes a live pane"
+
+fm_backend_herdr_kill "$SESSION:$NEW_CREW_PANE_ID"
+fm_backend_herdr_kill "$SESSION:$NEW_SM_PANE_ID"
+
+cleanup_all
+trap - EXIT