dsh-aris-panel 0.1.0

package/tools/iteration_log.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""iteration_log.py — overnight-loop stall detection → forced structural pivot.
+
+Append-only per-iteration ledger for an unattended research loop. Each tick the
+orchestrator records how many NEW findings the iteration produced — where a "finding"
+is a concrete added entry (new evidence, a falsified hypothesis, a candidate direction),
+NOT a subjective "valuable result". Consecutive zero-finding iterations accumulate a
+stale_count, which drives a forced pivot:
+
+  stale_count >= 2  → pivot = "structural"  (change a STRUCTURAL constraint, not tactical params)
+  stale_count >= 4  → pivot = "human"        (flag for human attention)
+
+This is a **Type-A signal**: it COUNTS entries and changes *direction*; it does NOT judge
+quality — quality/correctness stays with the cross-model jury (shared-references/
+acceptance-gate.md). It only ever says "keep going / change direction," never "good enough".
+
+The ledger is a sidecar at `.aris/runs/<run_id>.iterations.jsonl`; it deliberately does
+NOT import or touch run_state.py's done/accepted state machine (only shares the `.aris/runs/`
+dir, with a distinct `.iterations.jsonl` suffix). An optional `direction` per record lets
+the loop's re-generation step reject candidates too close to a tried direction. See
+shared-references/external-cadence.md → "Stall detection & forced structural pivot".
+
+Usage:
+    python3 iteration_log.py note <root> <run_id> <phase> <new_findings> [--direction "..."]
+    python3 iteration_log.py show <root> <run_id>
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterator, Optional
+
+try:
+    import fcntl
+except ImportError:  # pragma: no cover - non-POSIX
+    fcntl = None  # type: ignore
+
+PIVOT_STRUCTURAL_AT = 2   # consecutive zero-finding iterations → force a structural pivot
+ESCALATE_HUMAN_AT = 4     # still stalled → flag for human attention
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _log_path(root: str, run_id: str) -> Path:
+    # Same run_id discipline as run_state.py: no path escape.
+    safe = "".join(c for c in run_id if c.isalnum() or c in "-_.")
+    if not safe or safe != run_id or run_id in (".", ".."):
+        raise ValueError(f"invalid run_id {run_id!r} (use [A-Za-z0-9-_.])")
+    return Path(root) / ".aris" / "runs" / f"{run_id}.iterations.jsonl"
+
+
+@contextmanager
+def _lock(path: Path) -> Iterator[None]:
+    """Best-effort advisory lock (single-orchestrator contract; guards a stray resumer)."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if fcntl is None:
+        yield
+        return
+    fh = open(path.with_suffix(".jsonl.lock"), "w")
+    try:
+        fcntl.flock(fh, fcntl.LOCK_EX)
+        yield
+    finally:
+        try:
+            fcntl.flock(fh, fcntl.LOCK_UN)
+        finally:
+            fh.close()
+
+
+def _last_stale(path: Path) -> int:
+    """Read the most recent stale_count from the append-only ledger (0 if none)."""
+    if not path.is_file():
+        return 0
+    last = 0
+    try:
+        for line in path.read_text(encoding="utf-8").splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                last = int(json.loads(line).get("stale_count", last))
+            except (json.JSONDecodeError, ValueError, TypeError):
+                continue  # tolerate a partial/garbled line, keep the last good count
+    except OSError:
+        return 0
+    return last
+
+
+def pivot_for(stale_count: int) -> str:
+    if stale_count >= ESCALATE_HUMAN_AT:
+        return "human"
+    if stale_count >= PIVOT_STRUCTURAL_AT:
+        return "structural"
+    return "none"
+
+
+def note(root: str, run_id: str, phase: str, new_findings: int,
+         direction: Optional[str] = None) -> dict:
+    """Record one iteration; return {stale_count, pivot}. Append-only; never blocks work."""
+    new_findings = int(new_findings)
+    if new_findings < 0:
+        raise ValueError(f"new_findings must be >= 0, got {new_findings}")
+    path = _log_path(root, run_id)
+    with _lock(path):
+        stale_count = 0 if new_findings > 0 else _last_stale(path) + 1
+        pivot = pivot_for(stale_count)
+        rec = {"ts": _now(), "phase": phase, "new_findings": new_findings,
+               "stale_count": stale_count, "pivot": pivot}
+        if direction is not None:
+            rec["direction"] = direction
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+    return {"stale_count": stale_count, "pivot": pivot}
+
+
+def show(root: str, run_id: str) -> str:
+    path = _log_path(root, run_id)
+    return path.read_text(encoding="utf-8") if path.is_file() else ""
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="overnight-loop stall detection → forced structural pivot")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    n = sub.add_parser("note")
+    n.add_argument("root"); n.add_argument("run_id"); n.add_argument("phase")
+    n.add_argument("new_findings", type=int); n.add_argument("--direction", default=None)
+    s = sub.add_parser("show"); s.add_argument("root"); s.add_argument("run_id")
+    a = ap.parse_args()
+    if a.cmd == "note":
+        print(json.dumps(note(a.root, a.run_id, a.phase, a.new_findings, a.direction)))
+    elif a.cmd == "show":
+        sys.stdout.write(show(a.root, a.run_id))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/tools/lint_skills_helpers.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# lint_skills_helpers.sh — Advisory lint for hardcoded `tools/<helper>` references.
+#
+# Per shared-references/integration-contract.md §2, SKILL.md files must
+# resolve helpers via the canonical strict-safe chain
+#   .aris/tools/<helper>  →  tools/<helper>  →  $ARIS_REPO/tools/<helper>
+#   →  $ARIS_REPO/tools/<helper> via the global pointer file ~/.aris/repo (#366)
+# (Codex mirror uses the mirror-side chain), NOT hardcode `python3 tools/foo.py`
+# or `bash tools/foo.sh` directly.
+#
+# This script is ADVISORY: it always exits 0 and only prints findings.
+# A future enforcement layer (issue #178) may fail CI on new violations,
+# but Phase 2 keeps the contract gentle so the maintainer is not blocked.
+#
+# Run from the ARIS repo root:
+#     bash tools/lint_skills_helpers.sh
+
+set -u
+
+# Patterns that indicate hardcoded helper invocation (no resolver).
+INVOCATION_PY='python3 tools/(verify_papers|extract_paper_style|paper_illustration_image2|figure_renderer|arxiv_fetch|semantic_scholar_fetch|deepxiv_fetch|exa_search|openalex_fetch|research_wiki|iteration_log)\.py'
+INVOCATION_SH='bash tools/(verify_paper_audits|save_trace|verify_wiki_coverage|overleaf_audit)\.sh'
+
+# Files exempted from the lint:
+#   - integration-contract.md (canonical docs include ❌ anti-pattern examples)
+#   - wiki-helper-resolution.md (defines the chain; layer-2 reference is intentional)
+#   - skills-codex/paper-writing/SKILL.md L525 hook JSON example (placeholder for user
+#     ~/.claude/settings.json or ~/.codex/config hook, not a SKILL bash block)
+EXEMPTIONS="\
+skills/shared-references/integration-contract.md
+skills/skills-codex/shared-references/integration-contract.md
+skills/shared-references/wiki-helper-resolution.md
+skills/skills-codex/shared-references/wiki-helper-resolution.md
+skills/skills-codex/paper-writing/SKILL.md"
+
+is_exempt() {
+  case "$EXEMPTIONS" in
+    *"$1"*) return 0 ;;
+  esac
+  return 1
+}
+
+violation_count=0
+violation_report=""
+
+while IFS= read -r f; do
+  if is_exempt "$f"; then
+    continue
+  fi
+  py_hits=$(grep -nE "$INVOCATION_PY" "$f" 2>/dev/null || true)
+  sh_hits=$(grep -nE "$INVOCATION_SH" "$f" 2>/dev/null || true)
+  if [ -n "$py_hits" ] || [ -n "$sh_hits" ]; then
+    violation_count=$((violation_count + 1))
+    violation_report="${violation_report}
+=== $f ==="
+    [ -n "$py_hits" ] && violation_report="${violation_report}
+${py_hits}"
+    [ -n "$sh_hits" ] && violation_report="${violation_report}
+${sh_hits}"
+  fi
+done < <(find skills -name '*.md' -type f 2>/dev/null)
+
+echo "ARIS helper-resolution lint (advisory)"
+echo "======================================="
+echo "Files with hardcoded \`tools/<helper>\` references: $violation_count"
+
+if [ "$violation_count" -gt 0 ]; then
+  printf '%s\n\n' "$violation_report"
+  echo "Resolution:"
+  echo "  Migrate each violating SKILL.md to the canonical strict-safe resolver"
+  echo "  per shared-references/integration-contract.md §2 (assign a semantic"
+  echo "  variable like \$AUDIT_VERIFIER / \$TRACE_HELPER / \$<NAME>_FETCHER from"
+  echo "  the four-layer chain, then invoke as \`python3 \"\$VAR\" ...\` or"
+  echo "  \`bash \"\$VAR\" ...\`)."
+  echo ""
+  echo "  Per-helper policy (Policy A gate / B side-effect / C forensic /"
+  echo "  D1 cascade / D2 multi-source / E diagnostic) is documented in the"
+  echo "  \"Per-helper policy assignments\" table of integration-contract.md §2."
+fi
+
+echo ""
+echo "Status: advisory (this script never fails CI; warnings only)."
+
+exit 0

package/tools/meta_opt/check_ready.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# ARIS Meta-Optimize: Readiness Check
+# Called by SessionEnd hook. If enough data has accumulated,
+# outputs a reminder to stdout (injected into Claude's context).
+#
+# Trigger: ≥5 skill invocations since last /meta-optimize run
+
+set -euo pipefail
+
+ARIS_META_DIR="${CLAUDE_PROJECT_DIR:-.}/.aris/meta"
+EVENTS_FILE="$ARIS_META_DIR/events.jsonl"
+LAST_RUN_FILE="$ARIS_META_DIR/.last_optimize"
+
+# No log = nothing to check
+[ -f "$EVENTS_FILE" ] || exit 0
+
+# Count skill invocations. Match the full event-field key so a stray
+# "skill_invoke" substring inside args/prompt values can't false-match.
+# Note: `grep -c` prints "0" then exits 1 on no match, so `... || echo 0`
+# would produce a two-line string "0\n0" that breaks the later integer
+# comparison. Use `|| true` to absorb the non-zero exit and keep the
+# captured "0" alone.
+TOTAL_SKILLS=$(grep -cE '"event": *"skill_invoke"' "$EVENTS_FILE" 2>/dev/null || true)
+TOTAL_SKILLS=${TOTAL_SKILLS:-0}
+
+# Check when meta-optimize was last run
+if [ -f "$LAST_RUN_FILE" ]; then
+    LAST_TS=$(cat "$LAST_RUN_FILE")
+    # Count skill invocations AFTER last run.
+    # WARNING: do NOT use `$0 > ts` here — every JSONL row starts with `{`
+    # (ASCII 0x7B), which sorts greater than every digit in an ISO 8601
+    # timestamp, so the comparison would degenerate to "always true" and
+    # SINCE_LAST would equal TOTAL_SKILLS. Extract the embedded "ts" value
+    # and compare that instead. log_event.sh emits Python json.dumps default
+    # format (`"ts": "..."` with a space after the colon), so the regex
+    # tolerates 0+ spaces between key and value to stay compatible if that
+    # ever changes.
+    SINCE_LAST=$(awk -v ts="$LAST_TS" '
+        /"event": *"skill_invoke"/ {
+            if (match($0, /"ts": *"[^"]+"/)) {
+                event_ts = substr($0, RSTART, RLENGTH)
+                sub(/^"ts": *"/, "", event_ts)
+                sub(/"$/, "", event_ts)
+                if (event_ts > ts) count++
+            }
+        }
+        END { print count + 0 }
+    ' "$EVENTS_FILE")
+else
+    SINCE_LAST=$TOTAL_SKILLS
+fi
+
+# Threshold: 5 skill invocations since last optimize
+if [ "$SINCE_LAST" -ge 5 ]; then
+    echo "📊 ARIS has logged $SINCE_LAST skill runs since last optimization. Run /meta-optimize to check for improvement opportunities."
+fi
+
+# Model-delta trigger (harness diet): a model bump makes existing scaffolding a
+# deletion candidate, independent of usage volume. /meta-optimize records the
+# session model at each run in .last_optimize_model; compare against the LATEST
+# session_start event's model. Absent files (older installs, no session_start
+# yet) silently skip.
+LAST_MODEL_FILE="$ARIS_META_DIR/.last_optimize_model"
+if [ -f "$LAST_MODEL_FILE" ]; then
+    CURRENT_MODEL=$(awk '
+        /"event": *"session_start"/ {
+            if (match($0, /"model": *"[^"]+"/)) {
+                m = substr($0, RSTART, RLENGTH)
+                sub(/^"model": *"/, "", m)
+                sub(/"$/, "", m)
+                latest = m
+            }
+        }
+        END { if (latest != "") print latest }
+    ' "$EVENTS_FILE")
+    LAST_MODEL=$(cat "$LAST_MODEL_FILE")
+    if [ -n "$CURRENT_MODEL" ] && [ -n "$LAST_MODEL" ] && [ "$CURRENT_MODEL" != "$LAST_MODEL" ]; then
+        echo "🔁 Model changed since last optimization ($LAST_MODEL → $CURRENT_MODEL). Run /meta-optimize — a model bump makes existing scaffolding a deletion candidate (harness diet)."
+    fi
+fi

package/tools/meta_opt/log_event.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# ARIS Meta-Optimize: Event Logger
+# Reads Claude Code hook JSON from stdin, extracts key fields,
+# appends structured event to BOTH project-level and global logs.
+#
+# Called automatically by Claude Code hooks (PostToolUse, UserPromptSubmit, etc.)
+# Input: JSON via stdin (Claude Code hook payload)
+# Output:
+#   Project: $CLAUDE_PROJECT_DIR/.aris/meta/events.jsonl  (project-specific details)
+#   Global:  ~/.aris/meta/events.jsonl                    (cross-project trends)
+
+set -euo pipefail
+
+PROJECT_META="${CLAUDE_PROJECT_DIR:-.}/.aris/meta"
+GLOBAL_META="$HOME/.aris/meta"
+mkdir -p "$PROJECT_META" "$GLOBAL_META"
+
+# Read stdin payload into env var (cannot use heredoc + herestring simultaneously)
+export ARIS_HOOK_PAYLOAD="$(cat)"
+
+python3 - "$PROJECT_META/events.jsonl" "$GLOBAL_META/events.jsonl" << 'PYEOF'
+import json, sys, os
+from datetime import datetime, timezone
+
+project_log = sys.argv[1]
+global_log = sys.argv[2]
+
+raw = os.environ.get("ARIS_HOOK_PAYLOAD", "").strip()
+if not raw:
+    sys.exit(0)
+try:
+    p = json.loads(raw)
+except json.JSONDecodeError:
+    sys.exit(0)
+
+event_name = p.get("hook_event_name", "unknown")
+session_id = p.get("session_id", "")
+project_dir = os.environ.get("CLAUDE_PROJECT_DIR", "")
+ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+record = {"ts": ts, "session": session_id, "event": event_name}
+
+if event_name in ("PostToolUse", "PostToolUseFailure"):
+    tool_name = p.get("tool_name", "")
+    tool_input = p.get("tool_input", {})
+    record["tool"] = tool_name
+
+    if event_name == "PostToolUseFailure":
+        record["event"] = "tool_failure"
+
+    if tool_name == "Skill":
+        record["event"] = "skill_invoke"
+        record["skill"] = tool_input.get("skill", "")
+        record["args"] = tool_input.get("args", "")
+    elif tool_name == "Bash":
+        record["input_summary"] = tool_input.get("command", "")[:200]
+    elif tool_name in ("Edit", "Write", "Read"):
+        record["input_summary"] = tool_input.get("file_path", "")
+    elif tool_name.startswith("mcp__codex__"):
+        record["event"] = "codex_call"
+        record["input_summary"] = tool_input.get("prompt", "")[:150]
+
+elif event_name == "UserPromptSubmit":
+    prompt = p.get("prompt", "")
+    if prompt.startswith("/"):
+        parts = prompt.split(None, 1)
+        record["event"] = "slash_command"
+        record["command"] = parts[0]
+        record["args"] = parts[1] if len(parts) > 1 else ""
+    else:
+        record["event"] = "user_prompt"
+        record["prompt_preview"] = prompt[:100]
+
+elif event_name == "SessionStart":
+    record["event"] = "session_start"
+    record["source"] = p.get("source", "")
+    record["model"] = p.get("model", "")
+
+elif event_name == "SessionEnd":
+    record["event"] = "session_end"
+
+# Write to project-level log
+with open(project_log, "a") as f:
+    f.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+# Write to global log with project tag
+global_record = record.copy()
+global_record["project"] = os.path.basename(project_dir) if project_dir else "unknown"
+with open(global_log, "a") as f:
+    f.write(json.dumps(global_record, ensure_ascii=False) + "\n")
+PYEOF

package/tools/meta_opt/trigger_eval.py

@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+"""trigger_eval.py — measure whether a skill's `description` actually triggers.
+
+ARIS's known pain: with 80+ skills installed, Claude Code sometimes fails to
+invoke the right skill for a query it should handle — and until now the only
+lever (the frontmatter `description`) was tuned by pure judgment, with zero
+measurement. This tool turns trigger behavior into a number.
+
+MEASURE-ONLY BY DESIGN. It never rewrites a description. Its report is
+EVIDENCE for a /meta-optimize proposal (which lands only via the human-gated
+/meta-apply) — "a loop can drive, never acquit" applies to description tuning
+too.
+
+How it works (adapted from Anthropic's Claude Science `skill-creator`
+run_eval.py — Apache-2.0; ported off its host.* runtime onto plain `claude -p`):
+- For each (skill, query), run `claude -p <query> --output-format stream-json
+  --max-turns 1 --permission-mode plan --disallowed-tools Bash Write Edit …`
+  as a subprocess FROM A NEUTRAL TEMP CWD. The user-level ~/.claude/skills
+  corpus is loaded as usual, so the measurement happens under the REALISTIC
+  long installed list — the exact condition under which omission happens (an
+  isolated one-skill sandbox would trivially inflate trigger rates).
+- Parse the stream for the first assistant turn's tool_use blocks. A `Skill`
+  tool call with input.skill == target counts as a TRIGGER; a Skill call for a
+  different skill is a CONFUSION (recorded by name — the confusion matrix is
+  the interesting part for the long-list problem); no Skill call is a MISS.
+  Reading the target's SKILL.md via the Read tool counts as a trigger too
+  (secondary signal).
+- SAFETY: `--permission-mode plan` blocks every side-effecting tool (Bash,
+  Write, Edit, …) from executing, so a probed skill's own commands (e.g.
+  check-gpu's ssh, vast-gpu's rentals) do NOT run — we observe only which tool
+  the model REACHED FOR. The read-only tools we score on (a `Skill` load, a
+  `Read` of a SKILL.md) may execute, and both are side-effect-free.
+  `--disallowed-tools` denies the stateful tools explicitly as belt-and-braces,
+  and `--no-session-persistence` avoids leaving session artifacts. This is a
+  measurement, not a sandbox — it does not stop the user's own SessionStart
+  hooks (their normal per-session behavior), it stops the PROBED WORK.
+
+Query-set methodology (see trigger_evals.sample.json): queries must PARAPHRASE
+user intent, never quote the description's own trigger phrases verbatim — a
+query containing the literal trigger string is trivially positive and measures
+nothing. Optional negative queries (expect: none) measure false-triggering.
+
+Usage:
+  python3 tools/meta_opt/trigger_eval.py --eval-file tools/meta_opt/trigger_evals.sample.json \\
+      [--skills check-gpu,research-lit] [--samples 2] [--model haiku] \\
+      [--out .aris/meta/trigger_report.json] [--timeout 120]
+
+Exit code: 0 on completed run (regardless of rates), 2 on setup error.
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+# ---------------------------------------------------------------- pure logic
+
+def parse_stream_tool_uses(stream_text: str):
+    """Extract (tool_name, tool_input) pairs from `claude -p` stream-json output.
+
+    Each line is a JSON event; assistant events carry message.content lists in
+    which tool_use blocks appear. Malformed lines are skipped (the stream can
+    interleave non-JSON stderr noise when things go wrong).
+    """
+    uses = []
+    for line in stream_text.splitlines():
+        line = line.strip()
+        if not line or not line.startswith("{"):
+            continue
+        try:
+            ev = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if ev.get("type") != "assistant":
+            continue
+        content = (ev.get("message") or {}).get("content") or []
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "tool_use":
+                uses.append((block.get("name") or "", block.get("input") or {}))
+    return uses
+
+
+def classify(tool_uses, target_skill: str):
+    """Classify one probe run: ('trigger'|'confusion'|'miss', detail).
+
+    Trigger: a Skill call for the target (exact id, or a `plugin:target`
+    namespaced form — the latter tagged in detail so a namespaced match is
+    never silently indistinguishable from an exact one), or a Read of the
+    target's SKILL.md.
+    Confusion: the FIRST Skill call named a different skill (detail = its name).
+    Miss: no skill engagement at all.
+    """
+    for name, inp in tool_uses:
+        if name == "Skill":
+            invoked = (inp.get("skill") or "").strip()
+            if invoked == target_skill:
+                return "trigger", invoked
+            # plugin-namespaced form "plugin:skill": a trigger only if the tail
+            # equals the target AND the target itself is bare (not namespaced),
+            # surfaced distinctly so a human can spot a plugin/bare collision.
+            if ":" in invoked and invoked.split(":")[-1] == target_skill \
+                    and ":" not in target_skill:
+                return "trigger", f"{invoked} (namespaced→{target_skill})"
+            return "confusion", invoked
+        if name == "Read":
+            path = str(inp.get("file_path") or "")
+            if f"/skills/{target_skill}/SKILL.md" in path:
+                return "trigger", path
+    return "miss", ""
+
+
+def aggregate(records):
+    """records: list of {skill, query, outcome, detail} → per-skill summary."""
+    out = {}
+    for r in records:
+        s = out.setdefault(r["skill"], {
+            "probes": 0, "triggers": 0, "misses": 0, "errors": 0,
+            "confusions": {}, "queries": {},
+        })
+        s["probes"] += 1
+        q = s["queries"].setdefault(r["query"], {"trigger": 0, "confusion": 0,
+                                                 "miss": 0, "error": 0})
+        q[r["outcome"]] += 1
+        if r["outcome"] == "trigger":
+            s["triggers"] += 1
+        elif r["outcome"] == "miss":
+            s["misses"] += 1
+        elif r["outcome"] == "error":
+            s["errors"] += 1
+        elif r["outcome"] == "confusion":
+            s["confusions"][r["detail"]] = s["confusions"].get(r["detail"], 0) + 1
+    for s in out.values():
+        graded = s["probes"] - s["errors"]
+        s["trigger_rate"] = round(s["triggers"] / graded, 3) if graded else None
+    return out
+
+
+# ------------------------------------------------------------------- probing
+
+# Stateful tools that must never execute during a probe (belt-and-braces on top
+# of --permission-mode plan, which already blocks side-effecting tools).
+_DENY_TOOLS = ["Bash", "Write", "Edit", "NotebookEdit", "WebFetch"]
+
+
+# `--max-turns 1` deliberately caps the probe at one turn, so the CLI ends with
+# result subtype `error_max_turns` and a NONZERO exit — that is the EXPECTED,
+# successful termination for a probe, NOT a failure. Only other errors (auth,
+# startup/hook failure, execution error) count as a real error.
+_EXPECTED_TERMINATION = "error_max_turns"
+
+
+def _stream_real_error(stream_text: str) -> bool:
+    """True iff the stream carries a genuine terminal error — an `is_error`
+    result whose subtype is NOT the expected max-turns cap. Auth/hook failures
+    that still emit JSON are caught here so they are graded `error`, never a
+    `miss` that would silently corrupt the trigger rate."""
+    for line in stream_text.splitlines():
+        line = line.strip()
+        if not line.startswith("{"):
+            continue
+        try:
+            ev = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if ev.get("type") == "result" and ev.get("is_error") \
+                and ev.get("subtype") != _EXPECTED_TERMINATION:
+            return True
+    return False
+
+
+def _stream_has_assistant(stream_text: str) -> bool:
+    """True iff the model produced at least one assistant turn — i.e. the probe
+    ran far enough to be gradeable (even if it then hit the max-turns cap)."""
+    for line in stream_text.splitlines():
+        line = line.strip()
+        if not line.startswith("{"):
+            continue
+        try:
+            if json.loads(line).get("type") == "assistant":
+                return True
+        except json.JSONDecodeError:
+            continue
+    return False
+
+
+def run_probe(query: str, model: str | None, timeout: int, cwd: str) -> str:
+    """One `claude -p` probe; returns raw stream-json text. Raises RuntimeError
+    on a REAL failure (genuine error event, or nonzero exit with no assistant
+    turn at all) so the caller records `error` rather than a rate-corrupting
+    `miss`. The expected max-turns termination (nonzero exit + assistant turn
+    present) is a normal, gradeable result."""
+    cmd = ["claude", "-p", "--output-format", "stream-json", "--verbose",
+           "--max-turns", "1", "--permission-mode", "plan",
+           "--no-session-persistence", "--disallowed-tools", *_DENY_TOOLS]
+    if model:
+        cmd += ["--model", model]
+    # Allow nesting claude -p inside a Claude Code session (same pattern as the
+    # Apache-2.0 source): the CLAUDECODE guard is for interactive terminals.
+    env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}
+    result = subprocess.run(cmd, input=query, capture_output=True, text=True,
+                            env=env, timeout=timeout, cwd=cwd)
+    if _stream_real_error(result.stdout):
+        raise RuntimeError("claude -p stream carried a terminal error result event")
+    if _stream_has_assistant(result.stdout):
+        return result.stdout                       # gradeable (max-turns cap is fine)
+    if result.returncode != 0:                     # no assistant turn AND failed = real
+        raise RuntimeError(f"claude -p exited {result.returncode} with no assistant "
+                           f"turn: {result.stderr.strip()[:300]}")
+    return result.stdout                            # clean, no tool call → graded miss
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="Measure skill-description trigger rates.")
+    ap.add_argument("--eval-file", required=True,
+                    help='JSON: {"<skill>": ["query", ...], ...}')
+    ap.add_argument("--skills", default="",
+                    help="comma-separated subset of skills to probe (default: all in file)")
+    ap.add_argument("--samples", type=int, default=3,
+                    help="probes per query (trigger behavior is stochastic; the "
+                         "default 3 matches the upstream eval — samples=1 is too "
+                         "noisy to act on)")
+    ap.add_argument("--model", default=None,
+                    help="model override for probes (default: claude CLI default). "
+                         "NB: trigger behavior is model-dependent — compare like with like.")
+    ap.add_argument("--timeout", type=int, default=120)
+    ap.add_argument("--out", default=".aris/meta/trigger_report.json")
+    args = ap.parse_args(argv)
+
+    try:
+        evals = json.loads(Path(args.eval_file).read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as e:
+        print(f"ERROR: cannot read eval file: {e}", file=sys.stderr)
+        return 2
+    subset = {s.strip() for s in args.skills.split(",") if s.strip()}
+    targets = {k: v for k, v in evals.items()
+               if (not subset or k in subset) and not k.startswith("_")}
+    if not targets:
+        print("ERROR: no skills selected", file=sys.stderr)
+        return 2
+
+    records = []
+    # Neutral cwd: no project-level .claude/, so probes see exactly the
+    # user-level installed corpus — the realistic long list.
+    with tempfile.TemporaryDirectory(prefix="trigger-eval-") as neutral_cwd:
+        for skill, queries in targets.items():
+            for query in queries:
+                for _ in range(args.samples):
+                    try:
+                        stream = run_probe(query, args.model, args.timeout, neutral_cwd)
+                        outcome, detail = classify(parse_stream_tool_uses(stream), skill)
+                    except (RuntimeError, subprocess.TimeoutExpired) as e:
+                        outcome, detail = "error", str(e)[:200]
+                    records.append({"skill": skill, "query": query,
+                                    "outcome": outcome, "detail": detail})
+                    print(f"  [{outcome:9}] {skill} ← {query[:60]!r}"
+                          + (f" → {detail}" if outcome == "confusion" else ""))
+
+    summary = aggregate(records)
+    report = {"model": args.model or "cli-default", "samples": args.samples,
+              "skills": summary, "records": records}
+    out = Path(args.out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+
+    print("\nskill                          rate   probes  confusions")
+    for name, s in sorted(summary.items()):
+        conf = ", ".join(f"{k}×{v}" for k, v in
+                         sorted(s["confusions"].items(), key=lambda kv: -kv[1])) or "-"
+        rate = "n/a " if s["trigger_rate"] is None else f"{s['trigger_rate']:.2f}"
+        print(f"{name:30} {rate}   {s['probes']:4}    {conf}")
+    print(f"\nreport → {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())