@malloy-publisher/skills 0.1.20 → 0.1.21

package/skills/eval-diagnose/scripts/diagnose_test.py

@@ -211,5 +211,426 @@ class OffVocabularyValuesDoNotLoseTheRun(unittest.TestCase):
                       improve_py)
 
 
+class BehaviourStats(unittest.TestCase):
+    """The measurements a behavioural root cause gets stated in.
+
+    Diagnosis reads failures only, so a behaviour common to the whole run looks
+    causal from inside it. On one measured run the largest cluster was built on
+    "substitutes broad enumeration for targeted retrieval", and the enumeration
+    rate was 25% of targets in the failures against 26% in the passes.
+
+    THE FIXTURES HERE ARE THE SHAPES THE HARNESS ACTUALLY WRITES. An earlier
+    version of this class invented a `{"target_type", "search_text"}` dict that
+    no writer emits, and the code passed those tests while reading
+    `targetsWithoutSearchText: 0` on every real attempt -- because `targets`
+    holds `"<type>: <text>"` STRINGS and a target with no text is dropped
+    before the ledger sees it.
+    """
+
+    def events(self, qid, tool_call, *, verdict="match", turns=9):
+        return [
+            {"kind": "attempt", "qid": qid, "n_get_context": 2, "n_execute": 5,
+             "n_execute_errors": 1, "num_turns": turns,
+             "skills_invoked": ["malloy-analysis"]},
+            {"kind": "score", "qid": qid, "verdict": verdict},
+            {"kind": "tool_call", "qid": qid, "tool": "get_context",
+             **tool_call},
+            {"kind": "tool_call", "qid": qid, "tool": "execute_query"},
+        ]
+
+    def shapes(self, *pairs):
+        return {"target_shapes": [{"type": t, "has_text": h} for t, h in pairs]}
+
+    def test_a_bare_target_is_counted_from_target_shapes(self):
+        s = diagnose.behaviour_stats("q1", self.events("q1", self.shapes(
+            ("dimension", False), ("measure", True), ("view", False))))
+        self.assertEqual(s["searchTargets"], 3)
+        self.assertEqual(s["targetsWithoutSearchText"], 2)
+        self.assertTrue(s["targetsMeasured"])
+
+    def test_it_records_which_target_types_were_asked_for(self):
+        # A type-classification error is the most common agent-call defect, and
+        # a falsifier that cannot reach it is not much of a falsifier -- which
+        # a clustering agent said, in those terms, about the first version.
+        s = diagnose.behaviour_stats("q1", self.events("q1", self.shapes(
+            ("measure", True), ("dimension", True), ("dimension", False))))
+        self.assertEqual(s["targetTypes"], {"measure": 1, "dimension": 2})
+
+    def test_a_legacy_run_reads_types_but_abstains_on_the_bare_count(self):
+        # Runs written before `target_shapes` existed. The types survive in the
+        # `"<type>: <text>"` strings; the bare count does not, and reporting it
+        # as 0 would be a falsifier quietly asserting the opposite of the truth.
+        s = diagnose.behaviour_stats("q1", self.events(
+            "q1", {"targets": ["measure: total revenue", "source: flights"]}))
+        self.assertEqual(s["targetTypes"], {"measure": 1, "source": 1})
+        self.assertIsNone(s["targetsWithoutSearchText"])
+        self.assertFalse(s["targetsMeasured"])
+
+    def test_the_two_shapes_do_not_get_mixed(self):
+        # If both are present the measured one wins; the legacy strings are a
+        # lossy view of the same call.
+        s = diagnose.behaviour_stats("q1", self.events("q1", {
+            "targets": ["measure: total revenue"],
+            **self.shapes(("measure", True), ("dimension", False))}))
+        self.assertEqual(s["searchTargets"], 2)
+        self.assertEqual(s["targetsWithoutSearchText"], 1)
+
+    def test_it_carries_the_verdict_so_the_groups_are_comparable(self):
+        s = diagnose.behaviour_stats("q1", self.events("q1", self.shapes(),
+                                                       verdict="match"))
+        self.assertEqual(s["verdict"], "match")
+        self.assertEqual(s["skillsInvoked"], ["malloy-analysis"])
+        self.assertEqual(s["numTurns"], 9)
+
+    def test_only_this_case_is_measured(self):
+        ev = self.events("q1", self.shapes(("dimension", True))) + \
+            self.events("q2", self.shapes(("a", True), ("b", True)))
+        self.assertEqual(
+            diagnose.behaviour_stats("q1", ev)["searchTargets"], 1)
+
+    def test_an_attempt_with_no_calls_is_still_a_row(self):
+        s = diagnose.behaviour_stats("q9", [])
+        self.assertEqual(s["qid"], "q9")
+        self.assertEqual(s["searchTargets"], 0)
+
+    def test_the_clustering_prompt_asks_for_the_falsification(self):
+        self.assertIn("{controls}", diagnose.CLUSTER_PROMPT)
+
+
+class ControlsBlock(unittest.TestCase):
+    """The CONTROLS section sizes itself, because the instruction that is right
+    for twenty passing cases is wrong for one and meaningless for none."""
+
+    def block(self, n):
+        return diagnose.controls_block(
+            [{"qid": f"q{i}", "verdict": "match"} for i in range(n)])
+
+    def test_a_run_where_everything_failed_says_there_is_nothing_to_compare(self):
+        # 10 of 10 failing is a real shape, and asking the agent to compare
+        # rates against an empty list invites it to read [] as evidence.
+        b = self.block(0)
+        self.assertIn("CONTROLS: none", b)
+        self.assertIn("unfalsified", b)
+        self.assertIn("Do NOT read the absence of controls", b)
+        # A model-fact cluster is not affected by having no controls, and the
+        # block has to say so or every cluster gets downgraded.
+        self.assertIn("unaffected", b)
+
+    def test_one_or_two_controls_is_not_a_rate(self):
+        for n in (1, 2):
+            with self.subTest(n=n):
+                b = self.block(n)
+                self.assertIn("is NOT a rate", b)
+                self.assertIn("Do not compute a percentage", b)
+
+    def test_three_or_more_gets_the_real_comparison(self):
+        b = self.block(3)
+        self.assertIn("similar rate in the passes", b)
+        self.assertNotIn("is NOT a rate", b)
+
+    def test_the_prompt_renders_at_every_size(self):
+        # The failure this guards: an unrendered {controls} shipping to the
+        # agent, or a KeyError at the one moment the run cannot be redone.
+        for n in (0, 1, 2, 3, 20):
+            with self.subTest(n=n):
+                out = diagnose.CLUSTER_PROMPT.format(
+                    issues="[]", controls=self.block(n))
+                self.assertNotIn("{controls}", out)
+                self.assertNotIn("{issues}", out)
+
+    def test_every_size_carries_the_rows_it_has(self):
+        self.assertIn('"qid": "q0"', self.block(1))
+        self.assertNotIn('"qid"', self.block(0))
+
+
+class RetrievalMissOnAPassingCase(unittest.TestCase):
+    """A correct answer that never received a required entity is a finding.
+
+    It was silently dropped in three places: `attribute()` short-circuited on
+    `passed`, `summarise()` dropped the empty label, and `diagnose.py` routed
+    `match` to a control group. Measured on one run, 3 of 4 findings sat on
+    passing cases and produced nothing.
+    """
+
+    def case(self, qid="q", required=("measure:m:total",)):
+        return {"qid": qid, "coverage": "covered",
+                "expectedEntities": {"required": list(required)}}
+
+    def events(self, qid, returned, targets):
+        return [
+            {"kind": "attempt", "qid": qid, "sample": None, "phase": "baseline"},
+            {"kind": "score", "qid": qid, "sample": None, "phase": "baseline",
+             "verdict": "match", "reason": "ok"},
+            {"kind": "tool_call", "qid": qid, "sample": None, "phase": "baseline",
+             "tool": "get_context",
+             "rankedSummary": {"entityIds": list(returned)},
+             "target_shapes": [{"type": t, "has_text": True} for t in targets]},
+        ]
+
+    def test_a_pass_that_missed_an_entity_yields_a_finding(self):
+        ev = self.events("q", [], ["source", "dimension"])
+        row = diagnose.retrieval_finding(self.case(), ev,
+                                         ("q", None, "baseline"), "match")
+        self.assertIsNotNone(row)
+        # Deterministic: it needed a measure and sent no measure target.
+        self.assertIn("never asked", row["where_to_fix"])
+        # And the answer is still not a failure.
+        self.assertFalse(row["failed"])
+
+    def test_a_pass_that_received_everything_yields_nothing(self):
+        ev = self.events("q", ["measure:m:total"], ["measure"])
+        self.assertIsNone(diagnose.retrieval_finding(
+            self.case(), ev, ("q", None, "baseline"), "match"))
+
+    def test_it_calls_the_real_scorer_rather_than_restating_the_rule(self):
+        # Two definitions of "a miss" would drift, and the run summary and the
+        # diagnosis would then disagree about what happened.
+        src = pathlib.Path(diagnose.__file__).read_text()
+        self.assertIn("score_retrieval.score_case", src)
+
+    def test_the_prompt_says_the_answer_was_correct(self):
+        # Without it the diagnoser reads the evidence as a wrong answer and
+        # goes looking for a number that is not wrong.
+        src = pathlib.Path(diagnose.__file__).read_text()
+        self.assertIn("THIS ANSWER WAS CORRECT", src)
+        self.assertIn("answered_correctly", src)
+
+    def test_there_is_an_opt_out(self):
+        src = pathlib.Path(diagnose.__file__).read_text()
+        self.assertIn("--no-retrieval-misses", src)
+
+    def test_the_coverage_denominator_excludes_them(self):
+        # They are passes. Counting a diagnosed pass against a non-passing
+        # denominator printed "2 of 1 non-passing case(s) diagnosed (200%)".
+        src = pathlib.Path(diagnose.__file__).read_text()
+        self.assertIn("good_failures", src)
+
+
+class SelectingWhatToDiagnose(unittest.TestCase):
+    """Every scored case lands in exactly one bucket, and narrowing what gets
+    diagnosed does not change what the run failed."""
+
+    def cases(self, n, split=None):
+        return {f"q{i}": {"qid": f"q{i}", "coverage": "covered",
+                          **({"split": split} if split else {})}
+                for i in range(n)}
+
+    def scores(self, n, verdict="no_match"):
+        return [{"kind": "score", "qid": f"q{i}", "sample": None,
+                 "phase": "baseline", "verdict": verdict, "reason": "r"}
+                for i in range(n)]
+
+    def select(self, events, cases, **kw):
+        """(failed, passed, retrieval_only, excluded) -- the four buckets most
+        of these assert on. `excluded_passes` has its own tests below."""
+        return diagnose.select_cases(events, cases, ("no_match",), **kw)[:4]
+
+    def test_limit_records_what_it_dropped_as_an_exclusion(self):
+        # Truncating `failed` in place moved numerator and denominator
+        # together: 23 failures with --limit 5 printed "5 of 5 (100%)".
+        failed, _, _, excluded = self.select(
+            self.scores(23), self.cases(23), limit=5)
+        self.assertEqual(len(failed), 5)
+        not_passing = len(failed) + sum(len(v) for v in excluded.values())
+        self.assertEqual(not_passing, 23)
+        self.assertEqual(len(excluded["beyond --limit 5"]), 18)
+
+    def test_limit_zero_selects_nothing_rather_than_everything(self):
+        """`if limit:` read 0 as "no limit" and diagnosed all 23, one billable
+        agent each -- failing OPEN on the one axis where that costs money.
+        0 means zero; unlimited is spelled by omitting the flag."""
+        failed, _, _, excluded = self.select(
+            self.scores(23), self.cases(23), limit=0)
+        self.assertEqual(failed, [])
+        self.assertEqual(len(excluded["beyond --limit 0"]), 23)
+
+    def test_omitting_the_limit_still_means_unlimited(self):
+        failed, _, _, excluded = self.select(self.scores(23), self.cases(23))
+        self.assertEqual(len(failed), 23)
+        self.assertEqual(excluded, {})
+
+    def test_a_holdout_PASS_is_not_a_non_passing_case(self):
+        """The invariant, for the branch that still broke it.
+
+        The holdout test sits above the verdict checks on purpose, so the
+        split cannot depend on the score. That sent a holdout case that
+        PASSED into `excluded`, which `not_passing` sums: a real run with one
+        failure and four holdout passes printed "1 of 5 non-passing case(s)
+        diagnosed (20%)" when it had diagnosed the only failure there was.
+        """
+        events = self.scores(1, verdict="no_match") + [
+            {"kind": "score", "qid": f"h{i}", "sample": None,
+             "phase": "baseline", "verdict": "match", "reason": "r"}
+            for i in range(4)]
+        cases = dict(self.cases(1))
+        for i in range(4):
+            cases[f"h{i}"] = {"qid": f"h{i}", "split": "holdout"}
+        failed, _, _, excluded, excluded_passes = diagnose.select_cases(
+            events, cases, ("no_match",))
+        self.assertEqual(failed, ["q0"])
+        not_passing = len(failed) + sum(len(v) for v in excluded.values())
+        self.assertEqual(not_passing, 1, "four holdout PASSES inflated it to 5")
+        self.assertEqual(
+            len(excluded_passes["holdout, withheld from diagnosis"]), 4)
+
+    def test_a_holdout_FAILURE_is_still_a_non_passing_case(self):
+        events = [{"kind": "score", "qid": "h0", "sample": None,
+                   "phase": "baseline", "verdict": "no_match", "reason": "r"}]
+        cases = {"h0": {"qid": "h0", "split": "holdout"}}
+        failed, _, _, excluded, excluded_passes = diagnose.select_cases(
+            events, cases, ("no_match",))
+        self.assertEqual(failed, [])
+        self.assertEqual(excluded_passes, {})
+        self.assertEqual(
+            len(excluded["holdout, withheld from diagnosis"]), 1,
+            "a holdout failure is still withheld, and still non-passing")
+
+    def test_only_records_what_it_dropped_as_an_exclusion(self):
+        failed, _, _, excluded = self.select(
+            self.scores(4), self.cases(4), only="q0,q1")
+        self.assertEqual(sorted(failed), ["q0", "q1"])
+        self.assertEqual(
+            len(failed) + sum(len(v) for v in excluded.values()), 4)
+
+    def test_an_unnarrowed_run_excludes_nothing_for_narrowing(self):
+        failed, _, _, excluded = self.select(self.scores(3), self.cases(3))
+        self.assertEqual(len(failed), 3)
+        self.assertEqual(excluded, {})
+
+    def test_a_passing_holdout_with_a_retrieval_miss_stays_withheld(self):
+        """The leak: `verdict == "match"` was tested BEFORE the split, so a
+        holdout case that answered correctly with an incomplete retrieval went
+        into `retrieval_only` and on to a diagnosis call on every run, with no
+        flag able to prevent it. Holdout exists so improve has something
+        diagnosis never saw."""
+        cases = {"q0": {"qid": "q0", "coverage": "covered", "split": "holdout",
+                        "expectedEntities": {"required": ["measure:m:total"]}}}
+        events = [
+            {"kind": "score", "qid": "q0", "sample": None, "phase": "baseline",
+             "verdict": "match", "reason": "ok"},
+            {"kind": "tool_call", "qid": "q0", "sample": None,
+             "phase": "baseline", "tool": "get_context",
+             "rankedSummary": {"entityIds": []},
+             "target_shapes": [{"type": "dimension", "has_text": True}]},
+        ]
+        (failed, passed, retrieval_only, excluded,
+         excluded_passes) = diagnose.select_cases(events, cases, ("no_match",))
+        self.assertEqual(retrieval_only, [])
+        self.assertEqual(passed, [])
+        self.assertEqual(failed, [])
+        # Withheld, which is what this test is about -- and in the PASSES
+        # bucket, because it matched and `not_passing` sums the other one.
+        self.assertIn("q0",
+                      excluded_passes["holdout, withheld from diagnosis"])
+        self.assertEqual(excluded, {})
+
+    def test_include_holdout_lets_that_same_case_through(self):
+        cases = {"q0": {"qid": "q0", "coverage": "covered", "split": "holdout",
+                        "expectedEntities": {"required": ["measure:m:total"]}}}
+        events = [
+            {"kind": "score", "qid": "q0", "sample": None, "phase": "baseline",
+             "verdict": "match", "reason": "ok"},
+            {"kind": "tool_call", "qid": "q0", "sample": None,
+             "phase": "baseline", "tool": "get_context",
+             "rankedSummary": {"entityIds": []},
+             "target_shapes": [{"type": "dimension", "has_text": True}]},
+        ]
+        _, passed, retrieval_only, _ = self.select(
+            events, cases, include_holdout=True)
+        self.assertEqual(passed, ["q0"])
+        self.assertEqual(retrieval_only, ["q0"])
+
+    def passing_with_a_miss(self):
+        """Three cases that all PASSED, two of them on an incomplete
+        retrieval."""
+        cases, ev = {}, []
+        for q in ("q1", "q2", "q3"):
+            cases[q] = {"qid": q, "coverage": "covered",
+                        "expectedEntities": {"required": ["measure:m:total"]}}
+            got = [] if q in ("q1", "q2") else ["measure:m:total"]
+            ev += [{"kind": "score", "qid": q, "sample": None,
+                    "phase": "baseline", "verdict": "match", "reason": "ok"},
+                   {"kind": "tool_call", "qid": q, "sample": None,
+                    "phase": "baseline", "tool": "get_context",
+                    "rankedSummary": {"entityIds": got},
+                    "target_shapes": [{"type": "measure", "has_text": True}]}]
+        return cases, ev
+
+    def test_a_pass_kept_out_of_diagnosis_is_not_a_non_passing_case(self):
+        """`--no-retrieval-misses` used to push these into `excluded`, which
+        `not_passing` sums, so a run where every case passed printed
+        "coverage: 0 of 2 non-passing case(s) diagnosed (0%)"."""
+        cases, ev = self.passing_with_a_miss()
+        f, p, r, excluded, excluded_passes = diagnose.select_cases(
+            ev, cases, ("no_match",), no_retrieval_misses=True)
+        self.assertEqual(f, [])
+        self.assertEqual(sorted(p), ["q1", "q2", "q3"])
+        self.assertEqual(excluded, {})
+        not_passing = len(f) + sum(len(v) for v in excluded.values())
+        self.assertEqual(not_passing, 0)
+        # Still reported, just not as a failure.
+        self.assertEqual(
+            excluded_passes["passed with a retrieval miss "
+                            "(--no-retrieval-misses)"], ["q1", "q2"])
+
+    def test_without_the_flag_those_passes_are_diagnosed(self):
+        cases, ev = self.passing_with_a_miss()
+        f, p, r, excluded, excluded_passes = diagnose.select_cases(
+            ev, cases, ("no_match",))
+        self.assertEqual(sorted(r), ["q1", "q2"])
+        self.assertEqual(excluded_passes, {})
+        self.assertEqual(len(f) + sum(len(v) for v in excluded.values()), 0)
+
+    def test_contamination_still_outranks_the_split(self):
+        cases = self.cases(1, split="holdout")
+        events = [{"kind": "score", "qid": "q0", "sample": None,
+                   "phase": "baseline", "verdict": "no_match", "reason": "r",
+                   "contaminated": True}]
+        _, _, _, excluded = self.select(events, cases)
+        self.assertIn("contaminated", excluded)
+        self.assertNotIn("holdout, withheld from diagnosis", excluded)
+
+class MatchedPair(unittest.TestCase):
+    """A case that passed in another arm is the richest evidence in the run,
+    and it was the one thing the diagnoser never saw.
+
+    diagnose.py reads one run. A flip lives across two, so a case that passed
+    once and failed once was diagnosed from the failing side alone -- with the
+    matched pair, same question and same model, sitting unread in the other
+    run directory.
+    """
+
+    def evidence(self, passed_elsewhere=None):
+        events = [
+            {"kind": "attempt", "qid": "q1",
+             "final_query": "run: wrong -> { x }", "answer_text": "2 rows"},
+            {"kind": "score", "qid": "q1", "verdict": "no_match",
+             "reason": "expected 443"},
+        ]
+        return diagnose.evidence_for("q1", {"question": "how many?"}, events,
+                                     passed_elsewhere)
+
+    def test_absent_by_default(self):
+        self.assertIsNone(self.evidence()["passedInAnotherArm"])
+
+    def test_the_other_arms_query_reaches_the_agent(self):
+        e = self.evidence({"arm": "aa-2", "verdict": "match",
+                           "finalQuery": "run: right -> { y }"})
+        self.assertEqual(e["passedInAnotherArm"]["finalQuery"],
+                         "run: right -> { y }")
+        self.assertEqual(e["passedInAnotherArm"]["arm"], "aa-2")
+
+    def test_this_arms_query_is_still_there_to_diff_against(self):
+        e = self.evidence({"arm": "aa-2", "finalQuery": "run: right -> { y }"})
+        self.assertIn("run: wrong -> { x }", e["queriesRun"])
+
+    def test_the_prompt_tells_the_agent_what_to_do_with_it(self):
+        self.assertIn("passedInAnotherArm", diagnose.DIAGNOSE_PROMPT)
+        self.assertIn("matched pair", diagnose.DIAGNOSE_PROMPT)
+
+    def test_the_prompt_says_a_flip_is_not_noise(self):
+        # The doctrine change, pinned: a flip is a model-quality finding.
+        self.assertIn("not noise", diagnose.DIAGNOSE_PROMPT)
+
 if __name__ == "__main__":
     unittest.main()

package/skills/eval-import/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-import
 description: 'Turn a list of questions into an eval set, whatever shape it arrived in: a JSONL a customer sent, a CSV, a spreadsheet export, a markdown doc, an email thread, or a pull from production logs. Classify each item by what came WITH the question (a query, a number, prose criteria, or nothing), write cases.jsonl and set.json per reference/case-format.md, keep the file as it arrived, and seal each question so a later edit is detectable. Never marks a value-bearing golden verified and never invents a question. Use when questions arrive from outside and need to become a set, or before the first run against a set nobody here authored.'
-version: 0.1.20
+version: 0.1.21
 ---
 
 # Import questions into an eval set
@@ -95,6 +95,134 @@ back until somebody derives keys. A set of bare questions already measures
 whether the model can express an answer at all, and the answers it produces
 are what the keys get derived from.
 
+### Deriving a key yourself, when nothing arrived with the question
+
+The table above is about what ARRIVED. Sometimes nothing did, nobody knows the
+right number, and the set still needs keys. You may derive one, and the rules
+are the ones already stated rather than new ones:
+
+- Use your full model access and query until you can defend the number. The
+  agent that gets tested later holds only `get_context` and `execute_query`;
+  you are not that agent, and the gap is the point.
+- Record the query as `canonicalQuery`. A number with no query is somebody's
+  guess, which is the row above it in the table.
+- Record the entities you used to get there as `expectedEntities.required`, in
+  the `kind:source:name` form: the measures, dimensions and views the answer
+  cannot be produced without. `reference/case-format.md` says to leave this
+  field empty at import because guessing it invents a retrieval expectation
+  nobody stated; you are not guessing, you just used them. That is the only
+  trustworthy producer this field has, and it is what makes retrieval
+  measurable on a set nobody sent keys for. Where the model offers two
+  legitimate routes, write a `requiredAnyOf` group rather than pick one.
+  `verify_goldens.py` check 5 audits every id against the model, so a wrong one
+  is a hard finding rather than a silent retrieval miss.
+- **Derive the list with the model in front of you, not from memory, and not
+  with grep.** The model knows what each field IS; an author does not reliably.
+  Ask the question of the model: which entities does an answer to this depend
+  on, and is each a measure or a dimension. Confirm every id with `get_context`
+  before writing it. A list written from recollection is the most common way a
+  set acquires a retrieval expectation nobody can satisfy.
+
+  Measured: an agent given the model and one question found alternatives a
+  hand-written key had missed on all three cases tried -- carrier name against
+  nickname against code, `destination_count` against its underlying
+  `destination.airport_count`. At roughly $0.13 a question. It is better than a
+  person working from notes at finding what the model actually offers.
+
+  It is worse at deciding what to REQUIRE, and the two rules below are why. On
+  the same three cases it proposed demoting two named measures on the grounds
+  that a raw expression answered identically. One of those equivalences was
+  false; the other was true today and is not a reason to demote anything.
+
+- **But VERIFY every "this is not strictly required" claim by running both
+  sides.** This is the rule that makes the step above safe, and it is not
+  optional.
+
+  The same probe got one badly wrong. Told that a named measure is not required
+  when a raw column plus a plain aggregate answers identically, it wrote that
+  `average_plane_size` was "equally answerable by
+  `avg(aircraft.aircraft_models.seats)`". Executed, those are 229 and 196.93.
+  The measure is `aircraft.avg(...)`, an average over distinct aircraft, and the
+  inline version averages over flights. The claim was false for exactly the
+  measure whose entire purpose is to carry that distinction -- and it is the
+  same mistake, on the same field, that produced the only wrong answer in the
+  run this set came from.
+
+  Had that proposal been written into the key, the case would have stopped
+  testing the thing it exists to test.
+
+  So: an equivalence is a claim about DATA, and reading a definition is not
+  evidence. Run both expressions and compare the rows.
+
+  **But agreeing today is not a reason to demote the model's entity, and this
+  is the part the probe got backwards.** `airport_count is count()` returns the
+  same 984 as a bare `count()` right now, and stops doing so the moment anyone
+  adds a null filter or a deduplication to it. The key would not notice: it
+  would go on passing while testing nothing. That applies to every measure that
+  is currently a thin wrapper, which is most of them until the day one is not.
+
+  **`requiredAnyOf` is for two routes the MODEL defines** -- a carrier's `name`
+  and its `nickname`, `destination_count` and the `destination.airport_count`
+  it wraps. Both are entities, both are retrievable, and an answer using either
+  has gone through the semantic layer. That is a real choice and the group
+  records it.
+
+  It is NOT for "the model's entity, or a raw expression that happens to
+  agree". Those are not two routes: one is the route, the other is a bypass
+  that currently works. Keeping the named entity in `required` is what makes
+  the eval measure what the semantic layer is for, and finding it IS the
+  retrieval behaviour under test -- an agent that searches for the pre-defined
+  entity inherits its definition and its rules, while one that rebuilds from
+  raw columns inherits whatever it thought of.
+
+  What the equivalence check is FOR is reading a run afterwards. When an answer
+  came out right without the entity, comparing the two says whether it got
+  lucky or got it wrong, and a set whose passes are merely lucky is a set of
+  near-misses.
+
+- **Then prove each one is retrievable, and record what finds it.**
+  `check_findable.py --set <set> --mcp-url <mcp> --publisher <rest>
+  --environment <env> --package <pkg>` runs two checks and needs no model call.
+  With `--publisher` it reads the COMPILED model and settles whether each field
+  exists and what KIND it is; then it searches for each entity to confirm the
+  index can actually deliver it.
+
+  This matters more than it looks, because `required` is what BOTH retrieval
+  numbers are computed against: whether the agent asked for the entity, and
+  whether it came back. An id retrieval cannot deliver makes both fiction, and
+  the case reports a retrieval miss on every run, which reads as a defect in
+  the model or the agent rather than in the key.
+
+  **Prefer the compiled model over a grep, and prefer it in both directions.**
+  Measured on one package: `dimension:flights:flight_count` names a field the
+  model really has, so grepping the text passes it, and the compiled model says
+  `flight_count` is a MEASURE, which no `dimension` request can return.
+  Conversely `dimension:airports:own_type` appears zero times in the `.malloy`
+  and the compiled model declares it, because the source exposes it implicitly
+  from the data; the grep calls it missing and acting on that deletes a good
+  entity. `verify_goldens.py` check 5 is the grep, it is free and needs no
+  server, and it is not the authority.
+
+  A pass is a floor, not a verdict on the docs: each entity is searched by its
+  OWN NAME, the easiest query that could find it. An entity that answers to its
+  identifier and not to the words a question uses still fails at run time, as
+  `not retrieved`.
+
+  Writing the list forces the question an author has to answer anyway: what
+  would a reasonable agent search for, and of what type, to find this? If you
+  cannot state that, the agent cannot be expected to guess it.
+- `status: provisional`, never `verified`, for the reason stated above: you
+  derived it THROUGH the model under test, so a model bug would certify its
+  own key. `verify_goldens.py --promote` is still the only way out.
+- **Where two readings are both defensible from the model, do not pick one.**
+  A model that documents two conventions for the same population produces two
+  honest numbers, and choosing quietly is how a confident wrong key gets
+  written. Go to Hold an ambiguous golden in `skill:eval-loop`'s
+  `reference/golden-side-door.md`: record the competing candidates rather than
+  a new number.
+- What no query can settle is not a failure to derive. It is step 4's second
+  kind, and those clauses score on day one.
+
 ## Step 3: run their query, if they gave one
 
 Do it at import, before any run. It is the cheapest finding in the whole loop.
@@ -151,6 +279,57 @@ Where you cannot decide, mark the case and report it rather than guessing. A
 criterion nobody could classify is a question for its author, and
 `skill:eval-loop`'s golden side door is where it waits.
 
+### Four rules for writing the rubric itself
+
+Each of these cost a scored case on a real set, and none of them is obvious
+while you are writing one.
+
+**The golden rows are the figures. The rubric's prose is a gloss on them, and
+where the two disagree the prose is what is stale.** On one set 26 of 29
+rubrics quoted a figure that appears nowhere in their own golden rows: the
+goldens were re-derived the next morning, the prose was not, and an agent that
+computed 747 and 370 -- the exact numbers in the golden JSON -- was failed
+against a rubric still saying 615 and 502. Say how to derive the figure, not
+what it equalled. `verify_goldens.py` check 2 reports figures in the accepting
+clause that are absent from the rows, but it reads only figures specific enough
+to be a quoted result -- a bare three-digit number is invisible to it, and 615
+is exactly that. So the check is a help, not a guarantee, and the rule above is
+yours to keep. The judge is told the same thing from the other side: where a
+rubric and a golden disagree about a figure, it scores against the golden.
+
+**Do not assert the model's current behaviour.** "`contract_terms` cannot be
+used here at all -- it returns ZERO rows" was true when written and false four
+hours later, when a commit unblocked that view. Nothing linked the two, and the
+judge then reasoned from the stale claim against an answer that was right.
+Co-locating the set with the model makes such drift visible in a diff; it does
+not detect it. A rubric that describes a bug is a rubric with an expiry date:
+write the requirement, not the defect.
+
+**When a question admits two honest populations, accept either and require the
+answer to name which.** "Products in a category" can mean listed-in or
+primary-category, and on one set the two readings differed by 4,070 against
+2,707. Six or more cases turned on it and one rubric had been written to accept
+both. That one was right. The rest were repaired afterwards, having failed
+correct answers in the meantime.
+
+**If the rubric accepts an alternative, the entity list must too.** The two
+halves of a key are read by different things: the rubric is prose for the judge,
+`expectedEntities.required` is ids for retrieval scoring. They can disagree
+without anything noticing. A rubric saying "either the full carrier name or the
+nickname is fine" beside a required list naming only `dimension:carriers:name`
+scores an answer that used the nickname CORRECT and docks it recall in the same
+run, and that lost recall then reads as a retrieval failure. Use a
+`requiredAnyOf` group, which is satisfied when any member is delivered.
+`verify_goldens.py` reports the mismatch as a review item, but the rule is
+yours: it is a heuristic over prose and cannot catch every phrasing.
+
+**A trap note is not a requirement.** Notes that arrive beside the questions
+describe what their author thought was hard, and they mention things the
+question never asked for. Turning one into a rubric clause invents a
+requirement the answerer was never given, and it happened: a term appearing
+nowhere in any question became a clause an answer was marked down for missing.
+Convert a note only where it constrains the answer to the question as asked.
+
 ## Step 5: never change a question, and seal it
 
 The question is the stimulus. It is what a human asked, and it is not yours to

package/skills/eval-import/reference/case-format.md

@@ -18,7 +18,7 @@ On the case:
 | `split` | rarely present | you choose, and freeze it at import |
 | `source` | how it arrived: the filename, or the log query | say `unknown` rather than guessing a provenance |
 | `golden` | see the classification table in `SKILL.md` | absent. A case with no golden is legitimate |
-| `expectedEntities` | almost never present | leave it out. Guessing which fields an answer needs invents a retrieval expectation nobody stated, and it scores as a retrieval miss forever |
+| `expectedEntities` | almost never present | leave it out. Guessing which fields an answer needs invents a retrieval expectation nobody stated, and it scores as a retrieval miss forever. The one exception is a key you derived yourself (`SKILL.md`, Deriving a key yourself): then you are not guessing, you are recording what you used |
 
 `expectedEntities` is the one most worth leaving empty. A wrong `required` id
 cannot be delivered by any run, so it reads as a model failure on every case

package/skills/eval-improve/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-improve
 description: 'Make the smallest safe Malloy model edit that closes a diagnosed model-owned gap, with a probe receipt for every factual claim. Use after eval-diagnose, or when asked to fix a model so an agent can discover the right answer. Never accepts its own edit; the acceptance check belongs to eval-loop. Does not decide whether an answer was wrong (eval-answer) or why (eval-diagnose).'
-version: 0.1.20
+version: 0.1.21
 ---
 
 # Improve the Model