@chrono-meta/fh-gate 3.0.0 → 3.1.1

package/scripts/pipe_verdict_guard.sh

@@ -138,14 +138,54 @@ if ! printf '%s' "$NORM" | grep -qE 'set -o pipefail|set -[a-zA-Z]*o[a-zA-Z]* pi
   # Branch (b)'s optional `([^|&]*;)?` prefix keeps `| (sort; tail -5); echo $?` caught — the
   # filter need not be the group's FIRST command, only its LAST — while the `;` requirement keeps
   # the filter at a statement start, so `| (echo cat); rc=$?` does not match through a bare word.
+  # 🟥 «바로 다음 문장» 으로 좁힌다 (2026-09-06, 운영자 실사용 신고 + known-pair 재현).
+  #   종전 꼬리는 `[;&].*\$\?` 라 파이프 뒤 **어디든** `$?` 가 있으면 잡았다 — 그게 **다른
+  #   명령의** `$?` 여도. 한 호출에 문장이 여럿인 실사용 커맨드는 거의 항상 걸린다.
+  #   실측 4팔: ①`out=$(c); rc=$?; echo "$out"|tail` 안 터짐(정상) ②`c|tail; echo $?` 터짐
+  #   (known-positive) ③표시전용 안 터짐 ④`out=$(a); rc=$?; echo "$out"|tail; b; echo $?`
+  #   **터짐 = 오탐**. ④의 `$?` 는 `b` 의 것이라 안 잡는 것이 옳다.
+  #   ⇒ 꼬리를 `[;&]&?[^;&|]*\$\?` 로 — 파이프라인 **직후 한 문장** 안에서만 읽는다.
+  #   `&?` 는 `cmd | tail && echo $?`(진짜 결함)를 살리기 위한 것이다.
+  #   이 좁힘의 재현율 손실은 사실상 0 이다: 사이에 명령이 끼면 `$?` 는 그 명령의 것이므로
+  #   원래 **잡으면 안 되는** 자리다. 훅 자기 주석이 «100% FP 는 정작 중요한 한 건을
+  #   무시하도록 훈련시킨다» 고 적었고, 운영자 신고가 정확히 그 신호였다.
   _F='(tail|head|cat|less|more)'
   if printf '%s' "$NORM" \
-     | grep -qE "\|&?[[:space:]]*(${_F}([[:space:]][^|;&]*)?|[({]([^|&]*;)?[[:space:]]*${_F}([[:space:]][^|;&()}]*)?[[:space:]]*;?[[:space:]]*[)}])[[:space:]]*[;&].*\\\$\?"; then
+     | grep -qE "\|&?[[:space:]]*(${_F}([[:space:]][^|;&]*)?|[({]([^|&]*;)?[[:space:]]*${_F}([[:space:]][^|;&()}]*)?[[:space:]]*;?[[:space:]]*[)}])[[:space:]]*[;&]&?[^;&|]*\\\$\?"; then
     add "R2 \$? after a display filter" \
         "\$? holds the filter's status (tail/head/cat almost always succeed), not the command's — a FAILED check reads as 0. Capture first: \`out=\$(cmd 2>&1); rc=\$?\` then print \"\$out\" | tail."
   fi
 fi
 
+# ── R3 — a line that is ONLY redirections (`2>&1` alone on the line after a heredoc). ─────────
+# zsh runs a redirection-only line as `$NULLCMD` (= `cat` by default) — measured 2026-09-04 with
+# `zsh -x`: `+zsh:1> cat`. That `cat` reads STDIN until EOF. In this Bash tool, stdin is `/dev/null`
+# ONLY when the harness appends `< /dev/null`, and it does NOT append it when the top-level command
+# already carries a stdin redirect (`<` or a `<<` heredoc) — then stdin is a pipe held open for the
+# life of the command, and `cat` never returns. Measured 2026-09-04: 9 tasks in two sessions, all of
+# the shape `out=$(git commit -q -F - <<'CEOF' … CEOF ⏎ 2>&1)`: every commit landed within 4 s, no
+# push ever started, the shell sat in `$( )` with no git child, killed later (exit 144). The
+# pre-push hook never ran — it was blamed for a hang that happened before it.
+# Heredoc BODIES are stripped first: a one-word markdown blockquote line (`> 정본`) inside a commit
+# message is data, not a redirection. Deterministic on the remaining lines; zero-FP by construction
+# (a redirection-only line is never what the author meant outside a heredoc body).
+_R3_HIT=$(printf '%s\n' "$CMD" | awk '
+  BEGIN { inhd = 0; d = "" }
+  inhd { if ($0 == d || $0 ~ ("^[[:space:]]*" d "$")) { inhd = 0 }; next }
+  {
+    if (match($0, /<<-?[[:space:]]*["'"'"']?[A-Za-z_][A-Za-z_0-9]*["'"'"']?/)) {
+      d = substr($0, RSTART, RLENGTH); sub(/^<<-?[[:space:]]*/, "", d); gsub(/["'"'"']/, "", d); inhd = 1
+    }
+    # a STATEMENT at line start made only of redirections, ended by `)`, `;`, `}`, `&&`, `||` or EOL —
+    # the measured shape is `2>&1); rc=$?`, where `)` closes the `$( )` and the last statement inside
+    # it is the bare `2>&1`.
+    if ($0 ~ /^[[:space:]]*[0-9]*(>>?|<)(&[0-9]+|[[:space:]]*[^[:space:];|&<>()]+)?([[:space:]]+[0-9]*(>>?|<)(&[0-9]+|[[:space:]]*[^[:space:];|&<>()]+)?)*[[:space:]]*([;)}]|&&|\|\||$)/) { print NR ": " $0; exit }
+  }')
+if [ -n "$_R3_HIT" ]; then
+  add "R3 redirection-only line runs \`cat\` on stdin (line ${_R3_HIT})" \
+      "zsh executes a line that is only redirections as \$NULLCMD=cat, which reads stdin to EOF. This command carries a top-level \`<\`/\`<<\`, so the harness leaves stdin as an OPEN PIPE — that cat never returns and everything after it (the push) never starts. Put the redirection on the command line: \`out=\$(git commit -q -F - 2>&1 <<'"'"'CEOF'"'"'\`."
+fi
+
 [ -n "$hits" ] || exit 0
 
 if [ "${FH_PIPE_VERDICT_BLOCK:-0}" = "1" ]; then

package/scripts/probe_live_eval.sh

@@ -0,0 +1,240 @@
+#!/usr/bin/env bash
+# probe_live_eval.sh — the LIVE twin of /prompt-regression's static probe check.
+#
+# WHY THIS EXISTS. `plugins/fh-meta/skills/prompt-regression/SKILL.md` says this about itself,
+# verbatim, in its own §Step 4: "What it therefore cannot catch — a rule that is still present but
+# has stopped FIRING (salience loss, ordering, competition from another rule) — a trigger phrase
+# present in the file but shadowed by a higher-priority route — any behavior change that leaves the
+# source text identical." That gap is exactly what an eval-style harness closes (CLAUDE.md
+# §Autonomous Initiative Layer cites the shape: settings-changed PR + a scored run against real
+# tasks, not a source grep). This script IS that harness for FH's own golden probe set
+# (.claude/regression/probes.md): it sends each selected probe's literal utterance to a floor-tier
+# `claude -p` inside an ISOLATED, disposable clone (via scripts/sim_isolated_run.sh — never the live
+# repo, per that script's own header) and greps the ACTUAL RESPONSE, not the source.
+#
+# WHAT IT IS NOT. It does not replace /prompt-regression (source-correctness stays cheap and
+# instant) and it does not replace a blind sim-conductor persona run (which reads a whole artifact
+# for open-ended judgment). This is narrow, mechanical, known-answer, and cheap enough to run
+# nightly — the same trade CLAUDE.md's Measured-Loop memory entry describes for any recurring
+# measurement: sealed pre-registration (probes_live.yaml is authored BEFORE a run, not fit to one),
+# one variable per probe (ARM=primary utterance, CTRL=known-negative), a scorer that runs before
+# results are read, and the falsification condition (a probe whose control does not discriminate
+# is not scored PASS/FAIL — see §Scoring) executed exactly as it is written.
+#
+# SELECTION. Not every probes.md row is live-runnable — see .claude/regression/probes_live.yaml's
+# header for the 4-step mechanical rule (class filter, utterance-shape filter, INERT-ANCHOR filter,
+# hand-curated CLI-event filter) and scripts/probe_live_eval_lib.py for the implementation. Run
+# `--dry-run` to see the full 33-row breakdown: which probes are selected, which are excluded and
+# why, and which pass the mechanical rule but have no authored expect/control yet
+# (NOT-YET-AUTHORED — an honest gap, not a silent drop; G-LINT-01 is the current example, deferred
+# because scoring a full /harness-doctor Step 5 run needs more than a keyword regex).
+#
+# COST. Each selected probe costs TWO live `claude -p` calls (primary + control). Do not run the
+# full selected set casually — use --subset N or --ids P1,P2 for a spot-check, and read
+# sim_isolated_run.sh's own header before running unattended (isolation guarantees, what "observe"
+# mode does and does not prevent, the three-valued rc/bytes verdict for a timeout vs an empty
+# answer).
+#
+# 🟥 «미실행 ≠ 0» — a probe whose primary or control call produced 0 bytes (timeout, rate limit,
+# crash) is scored FAILED-TO-RUN, excluded from the pass-rate denominator, and its count is
+# reported separately. A FAILED-TO-RUN probe is not evidence the behavior is absent — it is
+# evidence nothing was measured (CLAUDE.md §Instrument-Calibration: "not found ≠ 0").
+#
+# SCORING (see scripts/probe_live_eval_lib.py:score_probe for the exact rule). Each probe declares
+# a `polarity` in probes_live.yaml:
+#   present — expect_re must appear in PRIMARY and must NOT appear in CONTROL
+#   absent  — expect_re must NOT appear in PRIMARY and MUST appear in CONTROL
+# Either direction of "the control disagrees with what polarity predicts" scores that probe
+# UNCALIBRATED, never PASS or FAIL — an instrument that cannot discriminate has not measured
+# anything, per this repo's own instrument-calibration discipline. If ANY probe in a run is
+# UNCALIBRATED, the WHOLE RUN's overall verdict is UNCALIBRATED (rc=2) — a pass rate computed
+# alongside a proven-blind probe is not trustworthy just because the other rows look fine.
+#
+# EXIT CODES: 0 = PASS (pass_rate >= threshold, no UNCALIBRATED). 1 = FAIL (pass_rate < threshold).
+# 2 = UNCALIBRATED, NO-PROBES-RAN, or a RUNNER PREFLIGHT FAILURE (sim_isolated_run.sh's own usage
+# guards exit 2 before ever calling `claude` — e.g. no `claude` on PATH, or — measured 2026-09-05 —
+# a launchd PATH with no `timeout(1)` on it). This script fails fast on the FIRST such rc=2 rather
+# than burning the remaining clones against an environment already known broken: no verdict
+# rendered either way — fix the instrument before trusting it. See sim_isolated_run.sh's own
+# §timeout(1) RESOLUTION header for why that preflight can fail even when `claude` itself is fine.
+#
+# USAGE
+#   bash scripts/probe_live_eval.sh --dry-run
+#   bash scripts/probe_live_eval.sh --ids G-GREET-01,G-TRIG-02 --model sonnet
+#   bash scripts/probe_live_eval.sh --subset 3
+#   bash scripts/probe_live_eval.sh --subset 3 --report-out /tmp/spot.md   # spot-check: keep the nightly record untouched
+#   bash scripts/probe_live_eval.sh                      # full selected set — nightly cron shape
+#
+# PORTABILITY: bash 3.2 (macOS) + bash 5.x (Linux CI). No associative arrays, no `${var,,}`,
+# heredocs only inside functions with quoted delimiters (see [[feedback_unquoted_heredoc_backtick_executes]]
+# — this script has none; the one heredoc-shaped block lives in probe_live_eval_lib.py, a real
+# file, not a bash heredoc).
+
+set -uo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+PROBES_MD="$REPO_ROOT/.claude/regression/probes.md"
+PROBES_LIVE="$REPO_ROOT/.claude/regression/probes_live.yaml"
+LIB="$REPO_ROOT/scripts/probe_live_eval_lib.py"
+# FH_SIM_RUNNER_BIN — same override name test_sim_isolated_run_lanes.sh already uses for the same
+# purpose (point at an alternate runner build). Here it also lets test_probe_live_eval_lanes.sh
+# swap in a stub runner (rc=2, no `claude` call, no network) to test the fail-fast behavior below
+# without spawning a live session. No-op when unset — default behavior is unchanged.
+SIM_RUNNER="${FH_SIM_RUNNER_BIN:-$REPO_ROOT/scripts/sim_isolated_run.sh}"
+
+# ── file-header constant — the "문턱" the design brief calls for. Change here, not per-invocation. ──
+THRESHOLD="0.8"
+
+MODEL="sonnet"
+# ── reps — 프로브당 반복 횟수. 기본 1(종전 행동 그대로), 무인 런은 plist 에서 3 을 준다. ──
+# 🟥 왜 1 이 기본이면서 야간은 3 인가 (2026-09-06 실측): 유효 런 3 개를 재채점하니 12 프로브 중
+#   5 개가 flaky 였고, **같은 코퍼스·15분 간격**의 두 런에서 4 개가 뒤집혔다. 관측 pass_rate 는
+#   0.50 / 0.67 / 0.67 — 단일 rep 의 노이즈 폭이 문턱 0.80 까지의 거리보다 넓다. 즉 reps=1 짜리
+#   pass_rate 로는 문턱을 정할 수 없다. 채점은 과반이고, 분산은 리포트의 `reps(pass/ran)` 칸에
+#   그대로 남긴다(3/3 과 2/3 은 다른 사실이다).
+#   ⚠️ 비용: reps=3 이면 프로브당 `claude -p` 호출이 2 → 6 이다(12 프로브 = 24 → 72).
+REPS=1
+DRYRUN=0
+SUBSET=""
+IDS=""
+OUTDIR=""
+REPORT_OUT=""   # --report-out: where the markdown report lands (default: tracks/_meta/live_eval_<date>.md)
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --subset)   SUBSET="${2:-}"; shift 2 ;;
+    --ids)      IDS="${2:-}"; shift 2 ;;
+    --model)    MODEL="${2:-sonnet}"; shift 2 ;;
+    --reps)     REPS="${2:-1}"; shift 2 ;;
+    --dry-run)  DRYRUN=1; shift ;;
+    --out)      OUTDIR="${2:-}"; shift 2 ;;
+    --report-out) REPORT_OUT="${2:-}"; shift 2 ;;   # lanes/spot-checks MUST pass this — never the live path
+    *) echo "unknown flag: $1" >&2; exit 2 ;;
+  esac
+done
+
+case "$REPS" in ''|*[!0-9]*) echo "FAIL: --reps must be a positive integer (got '$REPS')" >&2; exit 2 ;; esac
+[ "$REPS" -ge 1 ] || { echo "FAIL: --reps must be >= 1 (got '$REPS')" >&2; exit 2; }
+
+command -v python3 >/dev/null 2>&1 || { echo "FAIL: python3 required" >&2; exit 2; }
+[ -f "$PROBES_MD" ]   || { echo "FAIL: $PROBES_MD not found" >&2; exit 2; }
+[ -f "$PROBES_LIVE" ] || { echo "FAIL: $PROBES_LIVE not found" >&2; exit 2; }
+[ -f "$LIB" ]         || { echo "FAIL: $LIB not found" >&2; exit 2; }
+
+WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/fh-live-eval-XXXXXX")"
+SPEC_DIR="$WORKDIR/spec"
+SELECT_JSON="$WORKDIR/select.json"
+
+SELECT_ARGS=(select --probes-md "$PROBES_MD" --probes-live "$PROBES_LIVE"
+             --json-out "$SELECT_JSON" --spec-dir "$SPEC_DIR")
+[ -n "$SUBSET" ] && SELECT_ARGS+=(--subset "$SUBSET")
+[ -n "$IDS" ]     && SELECT_ARGS+=(--ids "$IDS")
+
+python3 "$LIB" "${SELECT_ARGS[@]}"
+select_rc=$?
+# select_rc nonzero = a DEAD-POINTER (probes_live.yaml names an id absent from probes.md). That is
+# an authoring bug in this repo's own asset, not a runtime condition — fail loudly rather than
+# silently running a smaller set than intended.
+if [ "$select_rc" -ne 0 ]; then
+  echo "" >&2
+  echo "❌ selection reported a dead pointer — fix .claude/regression/probes_live.yaml before running." >&2
+  rm -rf "$WORKDIR"
+  exit "$select_rc"
+fi
+
+if [ "$DRYRUN" -eq 1 ]; then
+  rm -rf "$WORKDIR"
+  exit 0
+fi
+
+[ -f "$SPEC_DIR/selected_ids.txt" ] || { echo "FAIL: selection produced no spec dir" >&2; rm -rf "$WORKDIR"; exit 2; }
+SELECTED_COUNT=$(wc -l < "$SPEC_DIR/selected_ids.txt" | tr -d ' ')
+if [ "$SELECTED_COUNT" -eq 0 ]; then
+  echo "❌ 0 probes selected after filtering — nothing to run (check --ids / --subset against the" >&2
+  echo "   SELECTED list printed above)." >&2
+  rm -rf "$WORKDIR"
+  exit 2
+fi
+
+# `claude` presence is the REAL runner's preflight, so it is checked only on that path. Under
+# FH_SIM_RUNNER_BIN the stub owns its own preflight — measured 2026-09-05 on CI (ubuntu, no
+# `claude` installed): this line fired BEFORE the stub was ever called, so FF2/FF3/FF4/FF6 went
+# red while FF1 passed by coincidence (both paths exit 2). Lane FF7 pins the seam; FF7b pins
+# that the real path still refuses to run without `claude`.
+if [ -z "${FH_SIM_RUNNER_BIN:-}" ]; then
+  command -v claude >/dev/null 2>&1 || { echo "FAIL: claude CLI not on PATH — cannot run live" >&2; rm -rf "$WORKDIR"; exit 2; }
+fi
+
+RUN_DATE="$(date +%Y-%m-%d)"
+OUTDIR="${OUTDIR:-$WORKDIR/run}"
+mkdir -p "$OUTDIR"
+
+echo ""
+echo "── live run: $SELECTED_COUNT probe(s), model=$MODEL, out=$OUTDIR ──────────────────────"
+
+while IFS= read -r id; do
+  [ -z "$id" ] && continue
+  input_text="$(cat "$SPEC_DIR/$id.input.txt")"
+  control_text="$(cat "$SPEC_DIR/$id.control.txt")"
+  probe_out="$OUTDIR/$id"
+  mkdir -p "$probe_out"
+
+  echo ""
+  echo "▶ $id — primary"
+  bash "$SIM_RUNNER" --arm primary --reps "$REPS" --prompt "$input_text" \
+       --mode observe --model "$MODEL" --out "$probe_out" \
+       > "$probe_out/_runner_primary.log" 2>&1
+  runner_rc=$?
+  tail -n 6 "$probe_out/_runner_primary.log"
+  # 🟥 fail-fast (2026-09-05) — rc=2 from the runner means its OWN usage/preflight guard tripped
+  # before `claude` was ever invoked (missing --arm/--prompt, bogus --mode, no `claude` on PATH,
+  # or — the incident this exists for — a launchd PATH with no `timeout(1)` resolvable). That
+  # condition is identical for every remaining probe in this run, so continuing would just burn
+  # the rest of the clones (up to 2*(N-1) more `claude -p` calls) to the same FAILED-TO-RUN wall.
+  # Abort loudly instead of quietly producing a 12/12 FAILED-TO-RUN report with no clue why.
+  if [ "$runner_rc" -eq 2 ]; then
+    echo "" >&2
+    echo "❌ $id primary runner call exited 2 (preflight failure, before \`claude\` ran)." >&2
+    echo "   Aborting the whole run rather than burning the remaining clones." >&2
+    echo "   Runner log:   $probe_out/_runner_primary.log" >&2
+    echo "   Partial run artifacts kept at: $OUTDIR" >&2
+    exit 2
+  fi
+
+  echo "▶ $id — control"
+  bash "$SIM_RUNNER" --arm control --reps "$REPS" --prompt "$control_text" \
+       --mode observe --model "$MODEL" --out "$probe_out" \
+       > "$probe_out/_runner_control.log" 2>&1
+  runner_rc=$?
+  tail -n 6 "$probe_out/_runner_control.log"
+  if [ "$runner_rc" -eq 2 ]; then
+    echo "" >&2
+    echo "❌ $id control runner call exited 2 (preflight failure, before \`claude\` ran)." >&2
+    echo "   Aborting the whole run rather than burning the remaining clones." >&2
+    echo "   Runner log:   $probe_out/_runner_control.log" >&2
+    echo "   Partial run artifacts kept at: $OUTDIR" >&2
+    exit 2
+  fi
+done < "$SPEC_DIR/selected_ids.txt"
+
+echo ""
+# 🟥 The live report path is the nightly RECORD. A lane or spot-check that reaches this line with the
+# default overwrote a real night's distribution once (2026-09-05 10:18: test_probe_live_eval_lanes.sh FF4
+# ran the real script with a stub runner and replaced the 02:30 report). Lanes pass --report-out.
+REPORT_PATH="${REPORT_OUT:-$REPO_ROOT/tracks/_meta/live_eval_${RUN_DATE}.md}"
+python3 "$LIB" score \
+  --probes-live "$PROBES_LIVE" \
+  --select-json "$SELECT_JSON" \
+  --ids-file "$SPEC_DIR/selected_ids.txt" \
+  --run-root "$OUTDIR" \
+  --threshold "$THRESHOLD" \
+  --model "$MODEL" \
+  --report-out "$REPORT_PATH" \
+  --run-date "$RUN_DATE" \
+  --reps "$REPS"
+score_rc=$?
+
+echo ""
+echo "run artifacts kept at: $OUTDIR"
+echo "(temp selection workspace $WORKDIR is NOT auto-deleted when --out was passed explicitly; ok to remove by hand)"
+
+exit "$score_rc"