@malloy-publisher/skills 0.1.20 → 0.1.21

package/skills/eval-answer/scripts/verify_goldens_test.py

@@ -16,8 +16,9 @@ import unittest.mock
 sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
 import verify_goldens  # noqa: E402
 from verify_goldens import (  # noqa: E402
-    model_text, promotion_blocker, question_drift_findings,
-    truth_isolation_findings, unknown_name_findings, verify)
+    check_value, close_enough, model_text, promotion_blocker,
+    question_drift_findings, truth_isolation_findings,
+    unknown_name_findings, verify)
 
 MODEL = """
 source: order_items is duckdb.table('data/order_items.parquet') extend {
@@ -531,6 +532,90 @@ class AValueFreeSetNeedsNoTruthServer(unittest.TestCase):
         self.assertTrue(verify(self.tmp, None, "samples", quiet=True)["skipped"])
 
 
+class TheCompositionRuleAtTheGate(unittest.TestCase):
+    """A truth server re-derives values; a ledger validates the definitions a
+    value rests on. Either is evidence. Neither present is a check that did not
+    happen, and the gate says which."""
+
+    def setUp(self):
+        self.tmp = pathlib.Path(tempfile.mkdtemp())
+        (self.tmp / "set.json").write_text('{"name": "s"}')   # no truthPackage
+        self.led = self.tmp / "led.jsonl"
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    def write_case(self, status="verified", ids=("measure:s:m",),
+                   verification=None):
+        g = {"status": status, "kind": "scalar", "value": {"v": 1}}
+        if verification:
+            g["verification"] = verification
+        (self.tmp / "cases.jsonl").write_text(json.dumps(
+            {"qid": "q1", "question": "x", "split": "dev", "golden": g,
+             "expectedEntities": {"required": list(ids)}}) + "\n")
+
+    def write_ledger(self, **verdicts):
+        self.led.write_text("".join(json.dumps(
+            {"entityId": e, "verdict": v, "exprSha": "s"}) + "\n"
+            for e, v in verdicts.items()))
+
+    def test_validated_definitions_lift_the_skip(self):
+        self.write_case()
+        self.write_ledger(**{"measure:s:m": "agrees"})
+        r = verify(self.tmp, None, "samples", quiet=True, definitions=self.led)
+        self.assertTrue(r["skipped"], "still no truth server")
+        self.assertTrue(r["ledgerValidated"], "but the definitions are checked")
+        self.assertEqual(r["unvalidated"], [])
+
+    def test_an_unchecked_definition_names_the_case(self):
+        self.write_case()
+        self.write_ledger(**{"measure:s:m": "unchecked"})
+        r = verify(self.tmp, None, "samples", quiet=True, definitions=self.led)
+        self.assertFalse(r["ledgerValidated"])
+        self.assertEqual(r["unvalidated"], ["q1 (unchecked)"])
+
+    def test_a_definition_missing_from_the_ledger_does_not_validate(self):
+        self.write_case()
+        self.write_ledger(**{"measure:s:other": "agrees"})
+        r = verify(self.tmp, None, "samples", quiet=True, definitions=self.led)
+        self.assertFalse(r["ledgerValidated"])
+
+    def test_an_empty_ledger_validates_nothing(self):
+        # Zero rows is not "every definition agrees"; it is no evidence at all.
+        self.write_case()
+        self.led.write_text("")
+        r = verify(self.tmp, None, "samples", quiet=True, definitions=self.led)
+        self.assertFalse(r["ledgerValidated"])
+
+    def test_promotion_through_the_ledger_still_needs_the_second_derivation(self):
+        # Validated definitions say the pieces are right; two derivations
+        # agreeing say the VALUE is. `verified` needs both.
+        self.write_case(status="provisional")
+        self.write_ledger(**{"measure:s:m": "agrees"})
+        r = verify(self.tmp, None, "samples", quiet=True, definitions=self.led,
+                   promote=True)
+        self.assertEqual(r["promoted"], [])
+        self.assertTrue(any("q1" in n for n in r["promotionNotes"]),
+                        r["promotionNotes"])
+
+    def test_promotion_through_the_ledger_names_the_ledger(self):
+        self.write_case(status="provisional",
+                        verification={"primaryAxis": "a", "variesAxis": "b"})
+        self.write_ledger(**{"measure:s:m": "agrees"})
+        r = verify(self.tmp, None, "samples", quiet=True, definitions=self.led,
+                   promote=True)
+        self.assertEqual(r["promoted"], ["q1"])
+        g = json.loads((self.tmp / "cases.jsonl").read_text())["golden"]
+        self.assertEqual(g["status"], "verified")
+        self.assertIn("definition ledger", g["verifiedBy"])
+
+    def test_no_ledger_keeps_the_old_behaviour(self):
+        self.write_case()
+        r = verify(self.tmp, None, "samples", quiet=True)
+        self.assertTrue(r["skipped"])
+        self.assertFalse(r["ledgerValidated"])
+
+
 class Promotion(unittest.TestCase):
     """`--promote` is the ONLY thing that writes `golden.status`.
 
@@ -760,5 +845,179 @@ class RubricFigures(unittest.TestCase):
         self.assertEqual(len(f), 1)
         self.assertIn("99999.99", f[0])
 
+    def test_a_four_digit_stale_figure_is_now_caught(self):
+        # The floor was five significant digits, so a four-digit quantity the
+        # golden does not hold passed silently. Measured over the local sets,
+        # moving it to three digits at or above 100 took the findings from 14
+        # to 17 over 54 rubrics.
+        f = verify_goldens.rubric_number_findings(self.case(
+            "Right: 2672 returned line items.", returned=1830))
+        self.assertEqual(len(f), 1)
+        self.assertIn("2672", f[0])
+
+    def test_a_bare_three_digit_figure_is_NOT_caught(self):
+        # Deliberate, and the limit of this check. Catching it means
+        # tokenising every three-digit integer in rubric prose, which was
+        # measured over the same sets: 17 findings become 552, because a
+        # rubric quoting a list of ids contributes one per id. A check that
+        # reports 552 things is not read at all.
+        #
+        # So the "615 and 502 over goldens holding 747 and 370" case -- a
+        # correct answer failed against prose one re-derivation out of date --
+        # is NOT closed here. It is closed in the judge prompt, which is told
+        # that where the rubric and the golden disagree about a figure, the
+        # golden is the key. That rule holds at any precision and needs no
+        # regex. This test exists so nobody re-opens the floor without
+        # re-measuring.
+        f = verify_goldens.rubric_number_findings(self.case(
+            "Right: monthly page views of 615 and 502.",
+            first=747, second=370))
+        self.assertEqual(f, [])
+
+    def test_a_year_is_not_a_figure(self):
+        f = verify_goldens.rubric_number_findings(self.case(
+            "Right: the 2024 total, 747.", total=747))
+        self.assertEqual(f, [])
+
+    def test_a_small_count_is_not_a_figure(self):
+        # Below 100 stays out: "the top 12 categories", "5 rows", an ordinal.
+        f = verify_goldens.rubric_number_findings(self.case(
+            "Right: the top 12 categories, over 5 regions.", total=747))
+        self.assertEqual(f, [])
+
+    def test_a_one_decimal_average_is_checked(self):
+        # "about 740.5" is how a rubric quotes an average, and the tokeniser
+        # required TWO decimal places, so that whole shape was invisible.
+        # Admitting one-decimal figures added zero findings across every set
+        # available locally, so it is free.
+        f = verify_goldens.rubric_number_findings(self.case(
+            "Right: an average flight distance of about 688.2.",
+            average=740.48))
+        self.assertEqual(len(f), 1)
+        self.assertIn("688.2", f[0])
+
+    def test_a_small_decimal_is_still_excluded(self):
+        # The floor still applies after the tokeniser: a value under 100 with
+        # few digits is a rate or a ratio, not a quoted result.
+        f = verify_goldens.rubric_number_findings(self.case(
+            "Right: a ratio of 3.5.", total=747))
+        self.assertEqual(f, [])
+
+    def test_a_figure_the_golden_holds_is_not_reported(self):
+        f = verify_goldens.rubric_number_findings(self.case(
+            "Right: 747 page views.", views=747))
+        self.assertEqual(f, [])
+
+
+
+class ScalarValueShape(unittest.TestCase):
+    """A bare string where an object belongs is one finding, not a blackout.
+
+    `"value": "Ecommerce"` instead of `{"answer": "Ecommerce"}` is the obvious
+    authoring mistake for a benchmark whose answers ARE strings. It raised
+    AttributeError out of `verify()` and killed the sweep, so the other
+    thirty-nine cases went unaudited and the script's own closing line --
+    "this says NOTHING about the goldens" -- was right.
+    """
+
+    def args(self):
+        return argparse.Namespace(
+            publisher="http://x", environment="truth", truth_package="t",
+            truth_model="truth.malloy", rewrite=False)
+
+    def check(self, value, rows):
+        c = {"qid": "q", "golden": {"kind": "scalar", "value": value,
+                                    "canonicalQuery": "run: t -> { ... }"}}
+        with unittest.mock.patch("verify_goldens.try_query",
+                                 return_value=(rows, None)):
+            return check_value(c, self.args())
+
+    def test_a_bare_string_is_an_error_not_a_crash(self):
+        status, detail, _ = self.check("Ecommerce", [{"answer": "Ecommerce"}])
+        self.assertEqual(status, "error")
+        self.assertIn("value is str", detail)
+
+    def test_the_error_shows_the_shape_it_wanted(self):
+        _, detail, _ = self.check("Ecommerce", [{"answer": "Ecommerce"}])
+        self.assertIn('{"answer": "Ecommerce"}', detail)
+
+    def test_a_list_under_scalar_is_also_caught(self):
+        status, detail, _ = self.check([1, 2], [{"answer": 1}])
+        self.assertEqual(status, "error")
+        self.assertIn("value is list", detail)
+
+    def test_the_proper_shape_still_works(self):
+        status, _, _ = self.check({"answer": "Ecommerce"},
+                                  [{"answer": "Ecommerce"}])
+        self.assertEqual(status, "ok")
+
+
+class YoungGoldens(unittest.TestCase):
+    """`eval-import` produces "a number alone" as a legitimate outcome, and
+    the audit called it a hard finding -- so the import skill's own output
+    refused the check that runs before every arm."""
+
+    def args(self):
+        return argparse.Namespace(
+            publisher="http://x", environment="truth", truth_package="t",
+            truth_model="truth.malloy", rewrite=False)
+
+    def check(self, status_in):
+        c = {"qid": "q", "golden": {"kind": "scalar", "value": {"n": 1},
+                                    "status": status_in}}
+        return check_value(c, self.args())
+
+    def test_provisional_with_no_query_is_not_a_finding(self):
+        status, detail, _ = self.check("provisional")
+        self.assertEqual(status, "unrederivable")
+        self.assertIn("no canonicalQuery yet", detail)
+
+    def test_an_absent_status_reads_as_provisional(self):
+        c = {"qid": "q", "golden": {"kind": "scalar", "value": {"n": 1}}}
+        self.assertEqual(check_value(c, self.args())[0], "unrederivable")
+
+    def test_VERIFIED_with_no_query_is_still_an_error(self):
+        # `verified` is a claim that it re-derived. That claim needs the query.
+        status, detail, _ = self.check("verified")
+        self.assertEqual(status, "error")
+        self.assertIn("cannot be re-derived", detail)
+
+    def test_unrederivable_does_not_count_as_drift(self):
+        # `drifted` sums diff + error, and the arm gate reads `drifted`.
+        self.assertNotIn("unrederivable", ("diff", "error"))
+
+
+class NumericRendering(unittest.TestCase):
+    """Malloy renders a number in its shortest form, so a golden stated to two
+    decimals comes back as `75.7`. Comparing the rendered strings called that
+    drift, which blocks an arm -- and the workaround was padding expressions
+    in the truth queries whose only job was to make one string look like the
+    other."""
+
+    def test_numeric_strings_compare_as_numbers(self):
+        self.assertTrue(close_enough("75.70", "75.7", None))
+
+    def test_a_number_against_its_string_matches(self):
+        self.assertTrue(close_enough(75.70, "75.7", None))
+
+    def test_a_thousands_separator_is_read(self):
+        self.assertTrue(close_enough("1,234", "1234", None))
+
+    def test_genuine_text_still_compares_exactly(self):
+        self.assertTrue(close_enough("Ecommerce", "Ecommerce", None))
+        self.assertFalse(close_enough("Ecommerce", "Retail", None))
+
+    def test_a_bool_is_not_the_number_one(self):
+        # `True == 1` in Python, and a boolean golden must not match "1".
+        self.assertFalse(close_enough(True, "1", None))
+
+    def test_a_real_difference_is_still_drift(self):
+        self.assertFalse(close_enough("75.70", "76.1", None))
+
+    def test_a_compound_string_is_left_alone(self):
+        # Out of scope on purpose: it parses as no single number, so it stays
+        # an exact compare rather than being guessed at.
+        self.assertFalse(close_enough("C: 75.70", "C: 75.7", None))
+
 if __name__ == "__main__":
     unittest.main()

package/skills/eval-diagnose/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-diagnose
 description: 'Diagnose why a scored answer failed and who owns the fix, then cluster the failures by shared root cause. Walk dataset, agent-call, get_context/model, get_context/retrieval, construction, then model-definition. Append issue events to the file ledger, linked by traceId, one per cluster. Use after eval-answer, when triaging a run, or before changing a model. Does not edit the model (eval-improve).'
-version: 0.1.20
+version: 0.1.21
 ---
 
 # Diagnose One Answer
@@ -24,6 +24,35 @@ repair closes that. Take the stable list `flip_table.py` prints and pass
 `--only <qids> --verdicts near_match`. Never diagnose a one-armed `near_match`;
 that is noise, and it sends an agent to fix a model that is already right.
 
+**A correct answer can still carry a finding.** A case that answered right
+while a required entity never reached it is diagnosed too, for the retrieval
+miss alone, and `diagnose.py` selects it automatically. The answer was right by
+another route, and naming that route is the job: on the run this rule comes
+from it was always the same one, the agent rebuilding the model's own measure
+inline. That held while the measure was `count()` and failed the moment one
+carried a grain rule, producing the run's only wrong answer. Three of the four
+findings in that run sat on passing cases and, before this, produced nothing.
+
+**"It worked anyway" is not a reason to leave the model or the skills
+unfixed.** An answer that is right without the model's own entity is right for
+now, not right by design. Write the issue against the miss, record that the
+answer was correct so nobody reads it as a wrong number, and do not soften the
+finding because the number came out right. `--no-retrieval-misses` opts out for
+a run that only wants answer failures.
+
+Holdout is withheld so the acceptance check keeps something the improve step
+never saw. A **measure-only** run never reaches improve, so it is holding those
+cases back from nothing: pass `--include-holdout` there. The script refuses it
+on a run that already carries a `candidate`, because that run's holdout is the
+only thing left that can falsify the edit.
+
+**Say what the clusters do not cover.** `diagnose.py` prints a coverage account
+of every non-passing case and which bucket it fell in -- holdout, contaminated,
+a verdict outside `--verdicts`, unscored, or selected and not diagnosed. Quote
+it whenever you report clusters. One run's six clusters were read as covering
+its failures; they covered 8 of 18, and each individual exclusion had been
+correct and added up nowhere.
+
 ## Components, in order
 
 Walk **in this order** and stop at the first with positive evidence. A later
@@ -35,7 +64,7 @@ never "C1" / "C2" / "C3":
 | `dataset` | Bad question, bad or missing golden, or environment drift? |
 | `agent-call` | Did the agent ask for the needed concepts, with the right type and scope? |
 | `get_context/model` | Is the needed entity absent, undocumented, weakly labeled, duplicated, or missing guidance? |
-| `get_context/retrieval` | Was an on-target request against a well-described entity ranked or grouped wrong? |
+| `get_context/retrieval` | Was an on-target, in-scope request against a well-described entity ranked or grouped wrong? **Check the call's `scopes` first**: a call pinned to one source cannot return another source's entity, and that miss is `agent-call`. |
 | `construction` | Did sufficient context arrive, and the agent still built the wrong query? |
 | `model-definition` | Is a measure, join, filter convention, or source semantically wrong? |
 
@@ -316,6 +345,22 @@ Say what you considered merging and chose not to. A cluster is a claim that one
 change fixes N cases, and the near-misses are what a reviewer needs to falsify
 it.
 
+**Falsify a behavioural cluster against the passes.** This skill reads failures
+only, so any behaviour common to the whole run looks causal from inside it.
+`diagnose.py` hands the clustering step a `CONTROLS` block: the same
+measurements -- retrieval calls, targets carrying no `search_text`, queries,
+skills opened, turns -- taken on the cases that PASSED. Before claiming a
+behaviour explains a cluster, compare it there. If it occurs at a similar rate
+in the passes it does not separate the groups: mark the cluster `contributing`
+rather than `primary` and do not route it to an edit as the root cause.
+Measured, on the run this comes from: the largest cluster said the agent
+"substitutes broad enumeration for targeted retrieval", and bare targets were
+25% of all targets in the failures against 26% in the passes. What actually
+separated them was volume -- failures made about 50% more retrieval calls --
+and question difficulty explains that at least as well as call style does. With
+no controls at all, a behavioural cluster is unfalsified, which is a different
+claim from confirmed.
+
 Still no patch. Naming the shared root cause precisely enough that someone else
 can design the edit is the whole job here; the edit itself is
 `skill:eval-improve`, working from this. A cluster carrying an honest open

package/skills/eval-diagnose/scripts/cluster_failures.py

@@ -41,15 +41,52 @@ from typing import Any
 
 sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent.parent
                        / "eval-answer" / "scripts"))
-from score_retrieval import load, score_case  # noqa: E402
-
-# Which artifact a cluster's fix would land in. eval-improve needs this to know
-# what kind of edit it is even proposing.
-LEVER_BY_WHERE = {
-    "query construction": "skill",
-    "retrieval ranking": "ranking",
-    "model coverage": "model",
-    "refusal behaviour": "skill",
+from score_retrieval import (DELIVERED, MODEL, NEVER_ASKED,  # noqa: E402
+                             NOT_RETURNED, REFUSAL, UNMEASURED,
+                             load, score_case)
+
+# The labels are imported, never spelled again here. They were spelled out
+# once, and when `score_retrieval` renamed them -- "query construction" to
+# `delivered, wrong`, "retrieval ranking" to `not retrieved` -- every branch
+# below silently stopped matching: the two commonest failure kinds fell to the
+# catch-all `other:` group and lost their lever, while this file's tests, of
+# which there were none, stayed green.
+WHERE_DELIVERED = DELIVERED[2]
+WHERE_NOT_RETRIEVED = NOT_RETURNED[2]
+WHERE_NEVER_ASKED = NEVER_ASKED[2]
+WHERE_MODEL = MODEL[2]
+WHERE_REFUSAL = REFUSAL[2]
+WHERE_UNMEASURED = UNMEASURED[2]
+
+# A DIAGNOSED cluster arrives labelled by eval-diagnose's OWNER rather than by
+# a retrieval outcome, and both land in the same `where_to_fix` column of the
+# run package. These two maps are the translation, and they live here, in
+# eval-diagnose, because the owners they key on are eval-diagnose's vocabulary
+# (`diagnose.py`'s OWNERS): `retrieval` and `dataset` are values the retrieval
+# scorer never assigns, so holding them over there left eval-answer carrying
+# keys only this package can trigger. `diagnose.py` imports them from here.
+#
+# One map rather than two, because `diagnose.py` and this file both write
+# `clusters.jsonl` and each used to spell its own -- so the package served two
+# vocabularies under one column name and the doc on it matched neither. A
+# cluster's label is the nearest shared bucket, not a new one; its `component`
+# and cause code carry the precise finding.
+WHERE_BY_OWNER = {
+    "model": MODEL[2],
+    "retrieval": NOT_RETURNED[2],
+    "agent-skill": DELIVERED[2],
+    "dataset": "dataset",
+}
+# Which artifact the edit lands in. An owner nobody has named yet -- eval-
+# diagnose has not run, or it ran and declined (`undecided` for a
+# delivered-wrong or an unreturned entity, `unknown` for unmeasured coverage)
+# -- yields no lever rather than a guessed one, because a guess here is the
+# default blame this taxonomy exists to remove.
+LEVER_BY_OWNER = {
+    "model": "model",
+    "retrieval": "retrieval",
+    "agent-skill": "skill",
+    "dataset": "dataset",
 }
 
 
@@ -63,7 +100,7 @@ def cluster_key(row: dict[str, Any], case: dict[str, Any]) -> tuple[str, str]:
     where = row["where_to_fix"]
     missing = row.get("missing") or []
 
-    if where == "model coverage":
+    if where == WHERE_MODEL:
         note = (case.get("coverageNote") or "").strip()
         if note:
             # Notes are already phrased as an absence ("no measure; ..."), so
@@ -73,14 +110,20 @@ def cluster_key(row: dict[str, Any], case: dict[str, Any]) -> tuple[str, str]:
             return (f"model:{note[:60]}", f"the model has {lead}{note[:80]}")
         return ("model:unspecified", "the model does not cover what was asked")
 
-    if where == "retrieval ranking" and missing:
+    if where == WHERE_NOT_RETRIEVED and missing:
         # One entity that several questions needed and none received.
         return (f"retrieval:{missing[0]}", f"{missing[0]} is never returned")
 
-    if where == "refusal behaviour":
+    if where == WHERE_NEVER_ASKED and missing:
+        # A different edit from the one above, so a different group: the
+        # entity was reachable and no search of its kind went out.
+        return (f"unasked:{missing[0]}",
+                f"{missing[0]} is never searched for")
+
+    if where == WHERE_REFUSAL:
         return ("refusal", "answered a question the model cannot answer")
 
-    if where == "query construction":
+    if where == WHERE_DELIVERED:
         tags = [t for t in (case.get("tags") or [])
                 if t in ("window", "time-series", "ratio", "join", "distinct",
                          "cohort", "cumulative", "rank")]
@@ -114,7 +157,7 @@ def main(argv: list[str] | None = None) -> int:
         g = groups.setdefault(key, {
             "clusterId": key, "label": label,
             "whereToFix": r["where_to_fix"], "component": r["component"],
-            "owner": r["owner"], "lever": LEVER_BY_WHERE.get(r["where_to_fix"]),
+            "owner": r["owner"], "lever": LEVER_BY_OWNER.get(r["owner"]),
             "qids": [], "evidence": [], "proposedEdit": "", "confidence": None})
         g["qids"].append(r["qid"])
         why = (c.get("golden") or {}).get("rubric") or ""

package/skills/eval-diagnose/scripts/cluster_failures_test.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Tests for cluster_failures.py. Stdlib only: python3 cluster_failures_test.py
+
+What these guard is the seam this file has no other protection against: it
+CONSUMES `score_retrieval`'s `where_to_fix` vocabulary and used to spell every
+value again in its own branches. When the vocabulary was renamed -- "query
+construction" to `delivered, wrong`, "retrieval ranking" to `not retrieved` --
+every branch silently stopped matching, the two commonest failure kinds fell to
+the catch-all `other:` group, and their lever came back null. Nothing failed,
+because this file had no tests at all.
+
+So the rule these pin is: every label `score_retrieval` can emit reaches a
+branch here, and the levers come from the owner it already assigned.
+"""
+from __future__ import annotations
+
+import ast
+import pathlib
+import sys
+import unittest
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+sys.path.insert(0, str(HERE.parent.parent / "eval-answer" / "scripts"))
+
+import cluster_failures as cf  # noqa: E402
+import score_retrieval as sr  # noqa: E402
+
+# Every attribution `score_retrieval.attribute()` can return. A new one added
+# there without a branch here fails `test_every_label_reaches_a_branch`.
+ATTRIBUTIONS = (sr.DELIVERED, sr.NOT_RETURNED, sr.NEVER_ASKED,
+                sr.MODEL, sr.REFUSAL, sr.UNMEASURED)
+
+
+def row(attribution, missing=("measure:m:total",), **kw):
+    component, owner, where = attribution
+    return {"qid": "q", "where_to_fix": where, "owner": owner,
+            "component": component, "missing": list(missing), **kw}
+
+
+class EveryLabelIsHandled(unittest.TestCase):
+    def test_every_label_reaches_a_branch(self):
+        """No emitted label may fall through to `other:`, except the one that
+        genuinely has no specific edit behind it."""
+        fell_through = []
+        for att in ATTRIBUTIONS:
+            key, _ = cf.cluster_key(row(att), {"tags": ["ratio"]})
+            if key.startswith("other:"):
+                fell_through.append(att[2])
+        # `coverage not measured` is not an edit, it is the absence of a
+        # measurement, so it has no group of its own by design.
+        self.assertEqual(fell_through, [sr.UNMEASURED[2]])
+
+    def test_no_label_is_spelled_as_a_literal_in_the_code(self):
+        """The comments may quote the old names to explain the history; the
+        CODE may not spell any label, or the next rename breaks it again."""
+        tree = ast.parse(pathlib.Path(cf.__file__).read_text())
+        literals = {n.value for n in ast.walk(tree)
+                    if isinstance(n, ast.Constant)
+                    and isinstance(n.value, str)}
+        spelled = literals & {a[2] for a in ATTRIBUTIONS}
+        self.assertEqual(spelled, set())
+        # And they arrive from the one place that decides them.
+        imported = {alias.name for n in ast.walk(tree)
+                    if isinstance(n, ast.ImportFrom)
+                    and n.module == "score_retrieval" for alias in n.names}
+        self.assertLessEqual(
+            {"DELIVERED", "NOT_RETURNED", "NEVER_ASKED", "MODEL", "REFUSAL"},
+            imported)
+
+
+class TheGroupAFailureLandsIn(unittest.TestCase):
+    def key(self, attribution, case=None, **kw):
+        return cf.cluster_key(row(attribution, **kw), case or {})[0]
+
+    def test_a_delivered_wrong_failure_groups_by_its_construction_tag(self):
+        self.assertEqual(
+            self.key(sr.DELIVERED, {"tags": ["ratio"]}), "construction:ratio")
+
+    def test_a_delivered_wrong_failure_without_a_tag_still_groups(self):
+        self.assertEqual(self.key(sr.DELIVERED, {}), "construction:other")
+
+    def test_an_unreturned_entity_groups_by_the_entity(self):
+        self.assertEqual(self.key(sr.NOT_RETURNED),
+                         "retrieval:measure:m:total")
+
+    def test_never_asked_and_unreturned_are_different_groups(self):
+        # Same entity, different edit: one is the docs, the other is the search
+        # the agent never issued. Grouping them together proposes one fix for
+        # two problems.
+        self.assertNotEqual(self.key(sr.NEVER_ASKED),
+                            self.key(sr.NOT_RETURNED))
+
+    def test_a_model_gap_groups_on_its_coverage_note(self):
+        self.assertEqual(
+            self.key(sr.MODEL, {"coverageNote": "no measure; shipped_at"}),
+            "model:no measure; shipped_at")
+
+    def test_a_refusal_is_its_own_group(self):
+        self.assertEqual(self.key(sr.REFUSAL), "refusal")
+
+
+class TheLeverComesFromTheOwner(unittest.TestCase):
+    """One decision about who owns a failure, not two that drift."""
+
+    def lever(self, attribution):
+        return cf.LEVER_BY_OWNER.get(attribution[1])
+
+    def test_a_model_gap_is_fixed_in_the_model(self):
+        self.assertEqual(self.lever(sr.MODEL), "model")
+
+    def test_a_mechanical_agent_miss_is_fixed_in_the_skill(self):
+        self.assertEqual(self.lever(sr.NEVER_ASKED), "skill")
+        self.assertEqual(self.lever(sr.REFUSAL), "skill")
+
+    def test_an_undecided_owner_names_no_lever(self):
+        # Naming one would put back the default blame the taxonomy removed:
+        # eval-diagnose decides these, sufficiency first.
+        self.assertIsNone(self.lever(sr.DELIVERED))
+        self.assertIsNone(self.lever(sr.NOT_RETURNED))
+
+    def test_an_unknown_owner_names_no_lever(self):
+        self.assertIsNone(self.lever(sr.UNMEASURED))
+
+
+if __name__ == "__main__":
+    unittest.main()