@malloy-publisher/skills 0.1.20 → 0.1.21

package/skills/eval-loop/scripts/run_baseline_test.py

@@ -4,11 +4,14 @@ credited with, which retriever answered, what was actually re-executed, and
 when an attempt is unjudgeable."""
 import argparse
 import json
+import os
 import pathlib
 import shutil
+import subprocess
 import sys
 import tempfile
 import unittest
+from unittest import mock
 
 sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
 import run_baseline as rb  # noqa: E402
@@ -260,6 +263,264 @@ class JudgeGate(unittest.TestCase):
         self.assertEqual(v["reason"], "no_saved_verdict")
 
 
+class VerdictObject(unittest.TestCase):
+    """Finding the judge's verdict in a reply that holds other braces.
+
+    The old read spanned from the FIRST brace in the document to the last, so a
+    judge that quoted a Malloy snippet on its way to deciding handed
+    `json.loads` a blob starting mid-query. The case then counted in no bucket
+    at all and left the denominator the pass rate is printed over.
+    """
+
+    GOOD = '{"why": "the figures agree", "verdict": "match", "confidence": 9}'
+
+    def test_a_quoted_malloy_query_does_not_break_the_read(self):
+        # The reproduction: a fenced query, then the verdict.
+        text = ("```malloy\nrun: orders -> { aggregate: n is count() }\n```\n"
+                f"Looks right.\n{self.GOOD}")
+        self.assertEqual(rb.parse_verdict(text)["verdict"], "match")
+
+    def test_the_last_verdict_object_wins(self):
+        # The judge is told to end with the object, and prose reasoning toward
+        # it may quote a draft on the way.
+        text = ('{"why": "draft", "verdict": "near_match", "confidence": 6}\n'
+                f"On reflection:\n{self.GOOD}")
+        v = rb.parse_verdict(text)
+        self.assertEqual(v["verdict"], "match")
+        self.assertEqual(v["reason"], "the figures agree")
+
+    def test_a_nested_object_still_parses(self):
+        text = ('{"why": "a", "verdict": "match", "confidence": 9, '
+                '"column_pairing": {"gold": "pred"}}')
+        self.assertEqual(rb.parse_verdict(text)["column_pairing"],
+                         {"gold": "pred"})
+
+    def test_prose_around_the_object_is_fine(self):
+        self.assertEqual(
+            rb.parse_verdict(f"Here you go:\n{self.GOOD}\nHope that helps"
+                             )["verdict"], "match")
+
+    def test_a_reply_with_no_object_is_unparseable(self):
+        # The shape five cases across three runs actually produced: the judge
+        # reasons to the point of deciding and the process ends.
+        v = rb.parse_verdict("I have what I need to decide.")
+        self.assertIsNone(v["verdict"])
+        self.assertEqual(v["reason"], "judge_unparseable")
+
+    def test_braces_without_a_verdict_are_not_a_verdict(self):
+        v = rb.parse_verdict('{"note": "no verdict here"}')
+        self.assertEqual(v["reason"], "judge_unparseable")
+
+    def test_the_retry_predicate_agrees_with_the_parser(self):
+        # `judge_unusable` reuses `parse_verdict` rather than re-deciding what
+        # parseable means, so a reply this now reads must not be re-rolled.
+        text = f"```malloy\nrun: x -> {{ aggregate: n is count() }}\n```\n{self.GOOD}"
+        self.assertFalse(rb.judge_unusable([{"type": "assistant"}], text))
+
+
+class JudgePrompt(unittest.TestCase):
+    """What the judge is SHOWN, and whether a run can say which version of it.
+
+    The tests cover the code paths; the defect that ran for three arms was in
+    the text the model reads. These are about that text.
+    """
+
+    def test_the_prompt_is_pinned_separately_from_the_skill(self):
+        # `rubricSha` hashes eval-judge/SKILL.md. The prompt template lives
+        # here and carries the golden rendering and the tie-breaking rules, so
+        # a run that names only the skill cannot say which judge it ran.
+        import ledger
+        self.assertIn("judgePromptSha", ledger.RUN_OPTIONAL)
+
+    def test_a_stale_rubric_figure_loses_to_the_golden(self):
+        # An answer holding the golden's own figures was failed against a
+        # rubric still quoting the previous morning's. Stated as a rule the
+        # judge applies at any precision, rather than a regex over prose.
+        self.assertIn("Where the CASE RUBRIC quotes a figure that the GOLDEN "
+                      "contradicts", rb.JUDGE_PROMPT)
+        self.assertIn("the rubric's figure is stale prose", rb.JUDGE_PROMPT)
+
+
+class JudgeCostAcrossARebuild(unittest.TestCase):
+    """A rebuild spends nothing on judging and must not erase what judging cost.
+
+    Found by writing a report off a rebuilt run: its cost line read $0.00 judge
+    for verdicts the original arm had paid $0.78 for.
+    """
+
+    def test_a_run_that_judged_records_what_it_spent(self):
+        self.assertEqual(rb.carry_judge_cost(0.78, None), (0.78, False))
+
+    def test_a_rebuild_keeps_the_earlier_figure_and_marks_it(self):
+        self.assertEqual(rb.carry_judge_cost(0.0, 0.78), (0.78, True))
+
+    def test_a_rejudge_wins_over_the_earlier_figure(self):
+        # It really did judge again, so the new spend is this run's own.
+        self.assertEqual(rb.carry_judge_cost(0.91, 0.78), (0.91, False))
+
+    def test_a_first_run_with_nothing_before_it_records_zero(self):
+        self.assertEqual(rb.carry_judge_cost(0.0, None), (0.0, False))
+
+    def test_a_prior_of_zero_is_not_carried(self):
+        # --no-judge then rebuild: there is no earlier spend to protect, and
+        # marking it carried would claim a judging that never happened.
+        self.assertEqual(rb.carry_judge_cost(0.0, 0.0), (0.0, False))
+
+    def test_the_prior_is_read_before_run_json_is_rewritten(self):
+        # The whole defect: read after `run_config` writes, it is always None
+        # and the fix silently does nothing.
+        src = pathlib.Path(rb.__file__).read_text()
+        self.assertLess(src.index("prior_judge = prior_judge_cost(a.out)"),
+                        src.index('(a.out / "run.json").write_text'))
+
+    def test_a_missing_or_unreadable_run_json_is_none(self):
+        tmp = pathlib.Path(tempfile.mkdtemp())
+        try:
+            self.assertIsNone(rb.prior_judge_cost(tmp))
+            (tmp / "run.json").write_text("{not json")
+            self.assertIsNone(rb.prior_judge_cost(tmp))
+        finally:
+            shutil.rmtree(tmp, ignore_errors=True)
+
+
+class TruncatedAttempt(unittest.TestCase):
+    """An attempt the turn cap cut off is not judged.
+
+    Measured on a 29-case customer arm: four attempts ended at exactly 31 turns
+    with `error_max_turns`, and each went to the judge as a complete answer.
+    One of them was a planning fragment -- "Let me build the correct monthly
+    trend query" -- which the judge scored `no_match` at confidence 9, where it
+    counted in the pass rate as if the model had got the question wrong.
+    """
+
+    def setUp(self):
+        self.tmp = pathlib.Path(tempfile.mkdtemp())
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    def judge(self, att, *, rebuild=True, rejudge=False, golden=None):
+        a = argparse.Namespace(rebuild=rebuild, rejudge=rejudge)
+        return rb.run_judge(
+            {"qid": "q", "question": "?",
+             "golden": golden if golden is not None else {"status": "verified",
+                                                          "value": 1}},
+            att, a, self.tmp, "", "", False)
+
+    def test_a_truncated_attempt_is_refused_and_says_so(self):
+        v = self.judge({"answer_text": "Let me build the correct query.",
+                        "submitted": True, "error": "error_max_turns"})
+        self.assertIsNone(v["verdict"])
+        self.assertEqual(v["reason"], "answerer_truncated")
+
+    def test_it_is_decided_before_a_saved_verdict_is_reused(self):
+        # The point of putting it first: `--rebuild` over the runs that
+        # motivated this re-derives the right ledger from their transcripts,
+        # rather than restoring the verdict the truncated attempt was given.
+        (self.tmp / "q").mkdir(parents=True)
+        (self.tmp / "q" / "judge.md").write_text(
+            '{"why": "trails off", "verdict": "no_match", "confidence": 9}')
+        v = self.judge({"answer_text": "frag", "submitted": True,
+                        "error": "error_max_turns"})
+        self.assertEqual(v["reason"], "answerer_truncated")
+        self.assertIsNone(v["verdict"])
+
+    def test_it_is_decided_before_the_golden_gate(self):
+        # Order matters the other way too: a truncated attempt against a
+        # provisional key is truncated. Both refuse, and the one that names
+        # the harness's own setting is the one a conductor can act on.
+        v = self.judge({"answer_text": "frag", "submitted": True,
+                        "error": "error_max_turns"},
+                       golden={"status": "provisional", "value": 1})
+        self.assertEqual(v["reason"], "answerer_truncated")
+
+    def test_any_other_harness_failure_is_also_refused(self):
+        # The four-strikes abort does NOT cover this: it stops the arm and
+        # still judges what it collected. Measured on a live run, four attempts
+        # whose text was the CLI's own "model not found" message were scored
+        # `no_match` for $0.29, and the run printed `passed 0 of 4 decided
+        # (0%)` about a model no answerer had reached.
+        v = self.judge({"answer_text": "Error: model not found",
+                        "submitted": False, "error": "error_during_execution"})
+        self.assertIsNone(v["verdict"])
+        self.assertTrue(v["reason"].startswith("environment_failure"))
+        # The cap keeps its own word, because the remedy differs: re-run those
+        # cases at a higher cap, versus fix the environment and re-run the arm.
+        self.assertNotIn("truncated", v["reason"])
+
+    def test_the_cli_reporting_subtype_success_on_an_error_still_refuses(self):
+        # Observed exactly: `is_error` true with subtype "success", so the
+        # attempt carried `run_error: "success"`. Testing the VALUE would have
+        # let every one of those through; the test is that an error was
+        # reported at all.
+        v = self.judge({"answer_text": "Error: model not found",
+                        "submitted": False, "error": "success"})
+        self.assertIsNone(v["verdict"])
+        self.assertTrue(v["reason"].startswith("environment_failure"))
+
+    def test_a_clean_attempt_is_unaffected(self):
+        v = self.judge({"answer_text": "4.2M", "submitted": True,
+                        "error": None})
+        self.assertEqual(v["reason"], "no_saved_verdict")
+
+
+class ContaminatedVerdict(unittest.TestCase):
+    """A voided verdict says why, and keeps what the judge said.
+
+    Nulling in silence left a score event carrying a correct `reason` and
+    `confidence` beside `verdict: null`, which is indistinguishable from the
+    field being dropped on the write path -- and was read as exactly that in a
+    real run report, where nine "lost" verdicts were recovered by re-parsing
+    the stored judge replies. They had been voided on purpose.
+    """
+
+    def void(self, verdict, breaches, *, vetoed=False):
+        """The real nulling, not a copy of it: `main` calls this same
+        function, so a change there cannot pass these tests by drifting."""
+        v = dict(verdict)
+        if vetoed:
+            # What the mustNotUse veto does just above the call site.
+            v["judge_verdict"] = v["verdict"]
+            v["verdict"] = "no_match"
+        return rb.void_contaminated(v, breaches)
+
+    def test_a_voided_verdict_names_contamination(self):
+        v = self.void({"verdict": "match", "reason": "the figures agree",
+                       "confidence": 9},
+                      ["host tool available to the answerer: Bash"])
+        self.assertIsNone(v["verdict"])
+        self.assertEqual(v["reason"], "contaminated")
+        # The schema documents this reason; before, the judge's own prose sat
+        # here and the null looked like a bug.
+        self.assertEqual(v["judge_verdict"], "match")
+
+    def test_a_veto_that_is_also_contaminated_keeps_the_judges_read(self):
+        # Two overrides on one case. `judge_verdict` is for what the JUDGE
+        # said, so the veto's own `no_match` must not displace it.
+        v = self.void({"verdict": "match", "reason": "ok", "confidence": 8},
+                      ["mcp server 'malloy' failed"], vetoed=True)
+        self.assertIsNone(v["verdict"])
+        self.assertEqual(v["judge_verdict"], "match")
+
+    def test_a_clean_attempt_keeps_its_verdict_and_reason(self):
+        v = self.void({"verdict": "match", "reason": "the figures agree",
+                       "confidence": 9}, [])
+        self.assertEqual(v["verdict"], "match")
+        self.assertEqual(v["reason"], "the figures agree")
+
+    def test_every_null_verdict_reason_is_in_the_documented_set(self):
+        # The generalisation of the report's own ask ("a score event with a
+        # reason also has a verdict, or an explicit unreadable marker"): a null
+        # verdict must always name which of the known causes it was, so no
+        # future nulling can be silent the way contamination was.
+        schema = (pathlib.Path(rb.__file__).parent.parent.parent
+                  / "eval-answer" / "reference" / "ledger-schema.md").read_text()
+        for reason in ("not_submitted", "contaminated", "answerer_truncated",
+                       "judge_unparseable", "no_saved_verdict"):
+            with self.subTest(reason):
+                self.assertIn(reason, schema)
+
+
 class GoldenStatusGate(unittest.TestCase):
     """A verdict is refused against a key nobody has established.
 
@@ -320,10 +581,14 @@ class GoldenStatusGate(unittest.TestCase):
         # `criteria` holds no value by design; its clauses are the comparison.
         # Refusing a value-less golden would drop every such case, and every
         # deliberately-unanswerable case with it.
+        # The clauses live in `rubric`, which is the documented field
+        # (reference/case-format.md); this fixture said `criteria` and nothing
+        # read it, so the drift was invisible until a renderer had to find them.
         v = self.judge({"status": "verified", "kind": "criteria",
-                        "criteria": ["breaks the total out by region"]})
+                        "rubric": "breaks the total out by region"})
         self.assertEqual(v["reason"], "no_saved_verdict")
 
+
     def test_refusal_is_decided_before_a_saved_verdict_is_reused(self):
         # `--rebuild` without `--rejudge` returns the saved judge.md. A saved
         # verdict against an unestablished key is not evidence either.
@@ -341,6 +606,242 @@ class GoldenStatusGate(unittest.TestCase):
                          ("provisional", "invalid", "ambiguous"))
 
 
+class GoldenForJudge(unittest.TestCase):
+    """What the GOLDEN line of the judge prompt says, per kind.
+
+    A `criteria` golden rendered as "unanswerable" is a contradiction the judge
+    can see, and one called it rather than scoring the case.
+    """
+
+    def test_criteria_never_says_unanswerable(self):
+        line = rb.golden_for_judge(
+            {"kind": "criteria", "rubric": "names the channels"})
+        self.assertNotIn("unanswerable", line)
+        self.assertIn("CASE RUBRIC", line)
+
+    def test_unanswerable_says_so(self):
+        self.assertIn("unanswerable",
+                      rb.golden_for_judge({"kind": "unanswerable"}))
+
+    def test_a_scalar_renders_its_value(self):
+        self.assertEqual(
+            rb.golden_for_judge({"kind": "scalar", "value": {"total": 12.5}}),
+            '{"total": 12.5}')
+
+    def test_rows_render_the_list(self):
+        self.assertEqual(
+            rb.golden_for_judge({"kind": "rows", "value": [{"a": 1}]}),
+            '[{"a": 1}]')
+
+    def test_an_explicit_null_value_is_a_refusal_case(self):
+        # How the ecommerce set's four refusal cases say there is no number.
+        self.assertIn("unanswerable",
+                      rb.golden_for_judge({"kind": "scalar", "value": None}))
+
+    def test_a_golden_with_no_kind_is_unchanged(self):
+        # Sets predating the field must read exactly as they did.
+        self.assertEqual(rb.golden_for_judge({"value": 7}), "7")
+        self.assertIn("unanswerable", rb.golden_for_judge({}))
+        self.assertIn("unanswerable", rb.golden_for_judge(None))
+
+
+class AnUnreadableVerdictIsNotLost(unittest.TestCase):
+    """A judge reply that will not parse carries no verdict, so it counts in
+    none of match, no_match, near_match or needs_human. It therefore left the
+    denominator without appearing anywhere, and the pass rate was printed over
+    the remainder. Two cases went that way in one hosted run."""
+
+    GOOD = '{"why":"y","verdict":"match","confidence":9}'
+
+    def test_the_retry_predicate_reaches_run_cli(self):
+        # `claude` accepted `retry_when` and then passed `no_events` literally,
+        # so the judge's predicate never arrived and the judge never retried.
+        with mock.patch.object(rb, "run_cli",
+                               return_value=([], "", "", 1, 0.0)) as run:
+            rb.claude("q", str(pathlib.Path(__file__).parent), "sonnet",
+                      mcp=None, retry_when=rb.judge_unusable)
+        self.assertIs(run.call_args.kwargs["retry_when"], rb.judge_unusable)
+
+    def test_the_answerer_still_defaults_to_no_events(self):
+        # Re-rolling a bad ANSWER would put a second sample where the run
+        # records one. Only the default was ever meant to be the answerer's.
+        with mock.patch.object(rb, "run_cli",
+                               return_value=([], "", "", 1, 0.0)) as run:
+            rb.claude("q", str(pathlib.Path(__file__).parent), "sonnet",
+                      mcp=None)
+        self.assertIs(run.call_args.kwargs["retry_when"], rb.no_events)
+
+    def test_unparseable_prose_is_retried(self):
+        self.assertTrue(rb.judge_unusable([{"type": "assistant"}],
+                                          "The answer looks right to me."))
+
+    def test_no_text_at_all_is_retried(self):
+        self.assertTrue(rb.judge_unusable([], ""))
+
+    def test_a_good_verdict_is_not_retried(self):
+        self.assertFalse(rb.judge_unusable([{"type": "assistant"}], self.GOOD))
+
+    def test_a_verdict_wrapped_in_prose_is_not_retried(self):
+        # `parse_verdict` finds the object inside prose, so this is readable
+        # and re-rolling it would just cost a judge call.
+        self.assertFalse(rb.judge_unusable(
+            [{"type": "assistant"}], f"Here you go:\n{self.GOOD}\nHope that helps"))
+
+    def summary(self, **kw):
+        base = dict(out=pathlib.Path("r"), set_dir=pathlib.Path("s"),
+                    events_n=0, attempted=10, decided=8, passed=4, near=0,
+                    human=0, doubted=[], vetoed=[], alt_path=0, unscorable=0,
+                    retrieval_mode="semantic",
+                    tally={"semantic": 1, "lexical": 0, "unreported": 0},
+                    rs={}, answerer_cost=0.0, judge_cost=0.0, publisher="",
+                    environment="e")
+        return "\n".join(rb.summary_lines(**{**base, **kw}))
+
+    def test_the_summary_names_the_cases_it_could_not_read(self):
+        body = self.summary(unparseable=["cq-07", "cq-02"])
+        self.assertIn("unreadable    2", body)
+        self.assertIn("cq-02, cq-07", body)   # sorted, so a diff is stable
+
+    def test_a_clean_run_still_says_zero(self):
+        # Printed even at zero, for the reason `unscorable` is: a reader must
+        # not have to know the line exists to notice it is missing.
+        body = self.summary()
+        self.assertIn("unreadable    0", body)
+
+    def test_an_aborted_arm_suppresses_the_pass_rate(self):
+        # The four-strikes abort fires, `run.json` says `aborted`, and the
+        # headline still printed a percentage. Live: `passed 0 of 4 decided
+        # (0%)` for an arm whose answerer never started.
+        body = self.summary(aborted=True)
+        self.assertIn("INCOMPLETE: the arm ABORTED", body)
+        self.assertNotIn("(50%)", body)
+
+    def test_a_truncated_case_suppresses_the_pass_rate(self):
+        # The stop rule, made mechanical. A directive did not hold against an
+        # arm that had already been paid for: on the run this comes from, the
+        # truncation was known before the cases were judged and the rate was
+        # quoted anyway.
+        body = self.summary(truncated=["cq-01", "cq-05"], max_turns=30)
+        self.assertIn("INCOMPLETE: 2 truncated", body)
+        self.assertIn("no pass rate until re-run", body)
+        self.assertNotIn("(50%)", body)   # 4 of 8 would have printed this
+
+    def test_it_names_the_command_that_finishes_the_arm(self):
+        # Four cases, not a whole arm: the answers already paid for are kept
+        # and only the truncated ones are re-run, at twice the cap.
+        body = self.summary(truncated=["cq-05", "cq-01"], max_turns=30)
+        self.assertIn("--only cq-01,cq-05", body)
+        self.assertIn("--max-turns 60", body)
+
+    def test_a_contaminated_case_suppresses_it_too(self):
+        body = self.summary(contaminated=["cq-03"])
+        self.assertIn("INCOMPLETE: 1 contaminated", body)
+        self.assertNotIn("(50%)", body)
+
+    def test_both_are_named_when_both_happened(self):
+        body = self.summary(truncated=["cq-01"], contaminated=["cq-03"])
+        self.assertIn("INCOMPLETE: 1 truncated, 1 contaminated", body)
+
+    def test_contamination_reasons_are_counted(self):
+        # One breach across every case is a harness misconfiguration to fix
+        # once; one breach on one case is that answerer going somewhere it
+        # should not have. The count is what tells them apart.
+        body = self.summary(
+            contaminated=["cq-03", "cq-04"],
+            contamination_reasons={"host tool available to the answerer: Bash": 2})
+        self.assertIn("2x host tool available to the answerer: Bash", body)
+
+    def test_a_complete_run_still_prints_its_rate(self):
+        # The suppression is narrow: neither `unscorable` (a dataset state) nor
+        # `unreadable` (which has a retry) withholds the number.
+        body = self.summary(unscorable=3, unparseable=["cq-02"])
+        self.assertIn("(50%)", body)
+        self.assertNotIn("INCOMPLETE", body)
+
+    def test_it_is_not_folded_into_unscorable(self):
+        # `unscorable` is a DATASET state no answerer can change. This is the
+        # harness failing to read its own judge. Counting them together would
+        # send someone to fix answer keys that are fine.
+        body = self.summary(unscorable=3, unparseable=["cq-02"])
+        self.assertIn("unscorable    3", body)
+        self.assertIn("unreadable    1", body)
+
+
+class HostedProbe(unittest.TestCase):
+    """What the hosted reachability probe concludes from its transcript.
+
+    Three outcomes that are three different problems: no tool granted (a login
+    that was never done), a tool that answered with an error (reachable and
+    authenticated, wrong call), and a payload.
+    """
+
+    TOOLS = ("mcp__hosted__get_context",)
+
+    def events(self, *blocks):
+        out = []
+        for kind, content in blocks:
+            out.append({"type": kind, "message": {"content": content}})
+        return out
+
+    def use(self, tid="t1", name="mcp__hosted__get_context"):
+        return {"type": "tool_use", "id": tid, "name": name, "input": {}}
+
+    def res(self, text, tid="t1", err=False):
+        return {"type": "tool_result", "tool_use_id": tid,
+                "content": text, "is_error": err}
+
+    def test_no_tool_call_at_all_is_not_granted(self):
+        ev = self.events(("assistant", [{"type": "text",
+                                         "text": "I have no such tool."}]))
+        self.assertEqual(rb.probe_outcome(ev, self.TOOLS)[0], "not_granted")
+
+    def test_an_errored_result_is_rejected_not_reached(self):
+        # The defect this replaces: a bare tool_use returned True without ever
+        # looking at its result, so a refused call passed the gate.
+        ev = self.events(("assistant", [self.use()]),
+                         ("user", [self.res("search_targets is required",
+                                            err=True)]))
+        outcome, said, payload = rb.probe_outcome(ev, self.TOOLS)
+        self.assertEqual(outcome, "rejected")
+        self.assertIn("search_targets is required", said)
+        self.assertIsNone(payload)
+
+    def test_a_payload_is_reached_and_comes_back(self):
+        ev = self.events(("assistant", [self.use()]),
+                         ("user", [self.res(json.dumps(
+                             {"retrieval": "semantic", "sources": []}))]))
+        outcome, _said, payload = rb.probe_outcome(ev, self.TOOLS)
+        self.assertEqual(outcome, "reached")
+        self.assertEqual(payload["retrieval"], "semantic")
+
+    def test_a_good_call_outweighs_an_earlier_refused_one(self):
+        ev = self.events(
+            ("assistant", [self.use("t1")]),
+            ("user", [self.res("nope", "t1", err=True)]),
+            ("assistant", [self.use("t2")]),
+            ("user", [self.res(json.dumps({"retrieval": "lexical"}), "t2")]))
+        self.assertEqual(rb.probe_outcome(ev, self.TOOLS)[0], "reached")
+
+    def test_a_call_with_no_result_is_not_a_missing_login(self):
+        ev = self.events(("assistant", [self.use()]))
+        self.assertEqual(rb.probe_outcome(ev, self.TOOLS)[0], "rejected")
+
+    def test_a_call_to_an_unallowed_tool_is_ignored(self):
+        ev = self.events(("assistant", [self.use(name="Bash")]),
+                         ("user", [self.res("ok")]))
+        self.assertEqual(rb.probe_outcome(ev, self.TOOLS)[0], "not_granted")
+
+    def test_the_probe_asks_for_the_arguments_the_tool_requires(self):
+        # A get_context with no arguments is a validation error on any server
+        # that enforces its required parameters, which is what the probe used
+        # to ask for.
+        a = argparse.Namespace(environment="examples", package="storefront")
+        args = rb.probe_arguments(a)
+        self.assertTrue(args["search_targets"])
+        self.assertEqual(args["scopes"],
+                         [{"environment": "examples", "package": "storefront"}])
+
+
 class UnscorablePreflight(unittest.TestCase):
     """A set of bare questions is a supported set, not a set to refuse.
 
@@ -532,6 +1033,123 @@ class PlatformMcpUrl(unittest.TestCase):
 
 
 
+class RetrievalPrecisionIsReportedHonestly(unittest.TestCase):
+    """Precision was computed and thrown away; the run printed recall alone.
+
+    It is worth printing, but it reads everything outside `acceptable` as
+    noise, so a set that never authored one scores every legitimate extra as a
+    miss. Measured on a real run: 3% to 11%, on a set whose eight cases all had
+    an empty `acceptable`. The caveat ships with the number.
+    """
+
+    def rs(self, **over):
+        base = {"retrieval_scored": 8, "mean_recall": 0.938,
+                "complete_retrievals": 7, "mean_precision": 0.073,
+                "mean_returned": 25.6, "mean_required": 1.8,
+                "cases_with_acceptable": 0, "failures_by_where_to_fix": {}}
+        base.update(over)
+        return base
+
+    def test_precision_prints_with_its_caveat_when_nothing_authored_acceptable(self):
+        text = "\n".join(self.lines(rs=self.rs()))
+        self.assertIn("entity precision mean 7.3%", text)
+        self.assertIn("no case authored `acceptable`", text)
+        self.assertIn("breadth, not as a verdict", text)
+
+    def test_breadth_does_not_depend_on_authoring(self):
+        text = "\n".join(self.lines(rs=self.rs()))
+        self.assertIn("26 returned per attempt for 2 the answer named", text)
+
+    def test_a_fully_authored_set_gets_the_number_without_the_caveat(self):
+        text = "\n".join(self.lines(rs=self.rs(cases_with_acceptable=8)))
+        self.assertIn("entity precision mean 7.3%", text)
+        self.assertNotIn("no case authored", text)
+        self.assertNotIn("only 8 of 8", text)
+
+    def test_a_partly_authored_set_says_it_is_uneven(self):
+        text = "\n".join(self.lines(rs=self.rs(cases_with_acceptable=3)))
+        self.assertIn("only 3 of 8", text)
+
+    def lines(self, **over):
+        return RunSummary.lines(self, **over)
+
+
+class SkillsActuallyOpened(unittest.TestCase):
+    """A run names the skills it granted; only the ones opened shaped anything.
+
+    Measured on a real run: every attempt invoked zero of its 11 skills, so an
+    edit to one could not have changed the answers and nothing in the report
+    said so.
+    """
+
+    def test_zero_says_the_run_does_not_measure_the_skills(self):
+        lines = rb.skill_lines({"attempts": 8, "with_skill": 0, "skills": []})
+        text = "\n".join(lines)
+        self.assertIn("0 of 8", text)
+        self.assertIn("cannot be credited or blamed", text)
+
+    def test_some_usage_names_the_skills_and_does_not_warn(self):
+        lines = rb.skill_lines({"attempts": 8, "with_skill": 3,
+                                "skills": ["malloy-phrase-detection"]})
+        text = "\n".join(lines)
+        self.assertIn("3 of 8", text)
+        self.assertIn("malloy-phrase-detection", text)
+        self.assertNotIn("cannot be credited", text)
+
+    def test_no_attempts_prints_nothing(self):
+        self.assertEqual(rb.skill_lines({"attempts": 0, "with_skill": 0}), [])
+        self.assertEqual(rb.skill_lines(None), [])
+
+
+class NoLocalShadowsAnImportedModule(unittest.TestCase):
+    """A local named after an imported module breaks every call to that module
+    in the same function, and only at runtime.
+
+    `ledger = verify_definitions.load_ledger(...)` in main() made `ledger` a
+    local for the whole function, so `ledger.run_config(...)` 260 lines earlier
+    raised UnboundLocalError on EVERY run. The unit tests all call helpers, so
+    nothing noticed until a real arm was run end to end. This is the cheap
+    structural guard that would have.
+    """
+
+    def test_no_function_rebinds_a_module_this_file_imports(self):
+        import ast
+        src = pathlib.Path(rb.__file__).read_text()
+        tree = ast.parse(src)
+        modules = {n.names[0].asname or n.names[0].name.split(".")[0]
+                   for n in ast.walk(tree) if isinstance(n, ast.Import)}
+        bad = []
+        for fn in ast.walk(tree):
+            if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                continue
+            for node in ast.walk(fn):
+                if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) \
+                        and node.id in modules:
+                    bad.append(f"{fn.name}() rebinds the module name "
+                               f"{node.id!r} at line {node.lineno}")
+        self.assertEqual(bad, [], "; ".join(bad))
+
+
+class UsageFields(unittest.TestCase):
+    """The ledger must be able to reprice a run from its own token columns."""
+
+    def test_all_four_token_counts_are_captured(self):
+        u = {"input_tokens": 224, "output_tokens": 31507,
+             "cache_read_input_tokens": 2435273,
+             "cache_creation_input_tokens": 900000}
+        self.assertEqual(rb.usage_fields(u), {
+            "input_tokens": 224, "output_tokens": 31507,
+            "cache_read_tokens": 2435273, "cache_write_tokens": 900000})
+
+    def test_cache_writes_were_the_missing_column(self):
+        # The one the ledger never held. On one analysed run it was 44% of the
+        # agent's cost; cost_usd carried it and the breakdown could not.
+        self.assertIn("cache_write_tokens", rb.usage_fields({}))
+
+    def test_absent_usage_is_all_null_not_a_crash(self):
+        self.assertEqual(set(rb.usage_fields(None).values()), {None})
+
+
 class RunSummary(unittest.TestCase):
     """The end-of-run report is three layers, and the order is the point.
 
@@ -559,6 +1177,53 @@ class RunSummary(unittest.TestCase):
     def index_of(self, lines, needle):
         return next(i for i, l in enumerate(lines) if needle in l)
 
+    def test_a_consumed_coverage_report_is_named_not_pointed_at(self):
+        # With a report given, "not measured here" is false and must not print.
+        lines = self.lines(coverage_report={
+            "path": "cov.json", "version": "0.0.58", "agentModel": "sonnet",
+            "decided": 45, "cases": 49})
+        text = "\n".join(lines)
+        self.assertIn("cov.json", text)
+        self.assertIn("45 of 49", text)
+        self.assertNotIn("not measured here", text)
+        # 4 undecided: the report did not settle every case, and the reader
+        # has to know which rows fell back to what.
+        self.assertIn("undecided cases fell back", text)
+
+    def test_a_report_that_decided_everything_carries_no_fallback_warning(self):
+        lines = self.lines(coverage_report={
+            "path": "cov.json", "version": "1", "agentModel": "m",
+            "decided": 49, "cases": 49})
+        self.assertNotIn("undecided cases fell back", "\n".join(lines))
+
+    def test_no_report_keeps_the_pointer(self):
+        self.assertIn("not measured here", "\n".join(self.lines()))
+
+    def test_the_cascade_reads_as_a_funnel_with_owners(self):
+        lines = self.lines(cascade={
+            "total": 49, "not covered": 6, "unmeasured": 2,
+            "no entities named": 0, "not retrieved": 5,
+            "delivered, wrong": 6, "delivered, right": 30, "not scored": 0})
+        text = "\n".join(lines)
+        self.assertIn("cascade       49 cases", text)
+        self.assertIn("covered?      41 yes, 6 no (model gap), 2 unmeasured", text)
+        # The rung must not assert the docs: a miss is the docs OR the search
+        # wording, and only diagnose separates them. This assertion is here
+        # because the label was renamed in score_retrieval and the display line
+        # in this file was missed, so the two disagreed in a shipped commit.
+        self.assertIn("retrieved?    36 yes, 5 no (the entity exists and did "
+                      "not come back", text)
+        self.assertNotIn("(documentation", text)
+        # And a delivered-but-wrong answer names no owner until diagnose runs.
+        self.assertIn("correct?      30 yes, 6 no (delivered, wrong", text)
+        self.assertIn("diagnose decides", text)
+        # It heads the COVERAGE & RETRIEVAL layer, above the retrieval-mode line.
+        self.assertLess(self.index_of(lines, "cascade"),
+                        self.index_of(lines, "  retrieval "))
+
+    def test_no_cascade_prints_nothing(self):
+        self.assertNotIn("cascade", "\n".join(self.lines()))
+
     def test_the_three_layers_appear_in_order(self):
         lines = self.lines()
         self.assertLess(self.index_of(lines, "RESULTS"),
@@ -729,5 +1394,138 @@ class RebuildCaseList(unittest.TestCase):
         finally:
             shutil.rmtree(tmp, ignore_errors=True)
 
+
+class ContaminatedAttemptsLeaveTheAggregates(unittest.TestCase):
+    """A flagged attempt must not reach the printed score.
+
+    The ledger has nulled a contaminated verdict since 2026-09-01, but the
+    nulling landed on the copy bound for events.jsonl while the summary read
+    the judge's original. A 33-case run in which EVERY attempt was flagged
+    printed `12 of 21 decided (57%)` over a ledger whose every score event said
+    `verdict: null`. That is the contamination check working and the line a
+    human reads disagreeing with it.
+    """
+
+    def test_a_flagged_attempt_is_not_counted_as_a_pass(self):
+        verdicts = {"q1": {"verdict": "match"}, "q2": {"verdict": "match"}}
+        attempts = {"q1": {"breaches": ["host tool available: TaskCreate"]},
+                    "q2": {"breaches": []}}
+        for qid, v in verdicts.items():
+            if attempts[qid].get("breaches"):
+                v["verdict"] = None
+        decided = [v for v in verdicts.values()
+                   if v.get("verdict") in ("match", "no_match")]
+        self.assertEqual(len(decided), 1, "the flagged attempt still counted")
+        self.assertIsNone(verdicts["q1"]["verdict"])
+        self.assertEqual(verdicts["q2"]["verdict"], "match")
+
+    def test_a_fully_contaminated_run_decides_nothing(self):
+        verdicts = {f"q{i}": {"verdict": "match"} for i in range(21)}
+        attempts = {q: {"breaches": ["host tool available: TaskCreate"]}
+                    for q in verdicts}
+        for qid, v in verdicts.items():
+            if attempts[qid].get("breaches"):
+                v["verdict"] = None
+        decided = [v for v in verdicts.values()
+                   if v.get("verdict") in ("match", "no_match")]
+        self.assertEqual(len(decided), 0,
+                         "a run with no clean attempt reported a score")
+
+class GitSha(unittest.TestCase):
+    """The pin has to mark a dirty model repo, and it did not.
+
+    The path is used BOTH as git's working directory (`-C`) and as its
+    pathspec. A relative one means different things in those two positions:
+    git runs IN the directory and then resolves the pathspec against it, so
+    `--model-repo sub` looks for `sub/sub`, matches nothing, and every tree
+    reads clean. The sibling in `ledger.skills_git_sha` was fixed for exactly
+    this; `git_sha` was not, and had no test at all.
+    """
+
+    def setUp(self):
+        self.tmp = tempfile.mkdtemp()
+        self.repo = pathlib.Path(self.tmp) / "repo"
+        (self.repo / "sub").mkdir(parents=True)
+        (self.repo / "sub" / "model.malloy").write_text("source: a is x\n")
+        run = lambda *a: subprocess.run(a, cwd=self.repo, capture_output=True)
+        run("git", "init", "-q")
+        run("git", "config", "user.email", "t@t")
+        run("git", "config", "user.name", "t")
+        run("git", "add", "-A")
+        run("git", "commit", "-qm", "init")
+        self.cwd = os.getcwd()
+        os.chdir(self.repo)
+
+    def tearDown(self):
+        os.chdir(self.cwd)
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    def dirty(self):
+        (self.repo / "sub" / "model.malloy").write_text("source: a is CHANGED\n")
+
+    def test_a_clean_tree_has_no_marker(self):
+        self.assertFalse(
+            rb.git_sha(pathlib.Path("sub"), scope=pathlib.Path("sub"))
+            .endswith("-dirty"))
+
+    def test_a_relative_path_still_marks_dirt(self):
+        self.dirty()
+        self.assertTrue(
+            rb.git_sha(pathlib.Path("sub"), scope=pathlib.Path("sub"))
+            .endswith("-dirty"),
+            "a relative --model-repo silently reported a dirty model as clean")
+
+    def test_an_absolute_path_agrees_with_the_relative_one(self):
+        self.dirty()
+        rel = rb.git_sha(pathlib.Path("sub"), scope=pathlib.Path("sub"))
+        abs_ = rb.git_sha(self.repo / "sub", scope=self.repo / "sub")
+        self.assertEqual(rel, abs_, "the pin depends on how it was spelled")
+
+    def test_the_scope_still_narrows(self):
+        # Dirt OUTSIDE the scope is not the model's, which is the whole reason
+        # the scope exists. This must not regress into "any dirt anywhere".
+        (self.repo / "scratch.txt").write_text("unrelated")
+        self.assertFalse(
+            rb.git_sha(pathlib.Path("sub"), scope=pathlib.Path("sub"))
+            .endswith("-dirty"))
+
+    def test_not_a_repo_is_none(self):
+        self.assertIsNone(rb.git_sha(pathlib.Path(self.tmp)))
+
+
+class GoldenCheckScope(unittest.TestCase):
+    """The pre-arm audit covers the cases the arm RUNS, not the whole file.
+
+    A set is built incrementally, so one un-derivable golden elsewhere in
+    cases.jsonl blocked an arm over a subset that did not include it. The only
+    way past was --skip-golden-check, which switches the audit off for the
+    cases that would have passed and stamps "skipped" into run.json.
+    """
+
+    SRC = (pathlib.Path(__file__).resolve().parent / "run_baseline.py").read_text()
+
+    def test_the_call_passes_the_selected_qids(self):
+        # A source pin, like test_kinds_by_target_matches_the_server: the call
+        # sits inside main() between a server probe and an arm, so the cheap
+        # way to stop it silently losing the argument again is to read it.
+        call = self.SRC.split("verify_goldens.verify(")[1][:400]
+        self.assertIn("qids=checked_qids", call,
+                      "the golden check audits the whole set again")
+
+    def test_the_qids_come_from_the_narrowed_case_list(self):
+        self.assertIn('checked_qids = {c["qid"] for c in cases}', self.SRC)
+
+    def test_verify_actually_filters_on_qids(self):
+        # The other half: the argument has to mean something.
+        import inspect
+        import sys as _sys
+        _sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent.parent
+                                / "eval-answer" / "scripts"))
+        import verify_goldens
+        src = inspect.getsource(verify_goldens.verify)
+        self.assertIn("qids", src)
+        self.assertIn('c["qid"] in qids', src)
+
+
 if __name__ == "__main__":
     unittest.main()