@malloy-publisher/skills 0.1.19 → 0.1.21

package/skills/eval-loop/scripts/flip_table.py

@@ -59,13 +59,43 @@ def outcome(verdict: str | None) -> str:
     return "neither"
 
 
+# A golden a person has established is WRONG. Its verdict says nothing about
+# the model -- the answer was scored against a key that does not hold -- so it
+# leaves every aggregate.
+EXCLUDED_GOLD = {"verified_wrong"}
+
+
+def counts_toward_score(gold_status: str | None) -> bool:
+    """Does this score belong in the run's aggregates?
+
+    Exported for the same reason `outcome` is. `run_baseline.py` dropped
+    `verified_wrong` before counting, `eval_run.malloy` did not, and the
+    package's own doc comment on `gold_status` said it did -- so one
+    `verified_wrong` golden made the printed pass rate and the notebook's
+    `pass_rate` disagree (92.31% against 91.67% on a real 13-case run) with
+    nothing to say which was right. One rule, three readers: this function,
+    the `counts` column `build_run_package.py` writes from it, and the
+    measures in `eval_run.malloy` that filter on that column.
+    """
+    return gold_status not in EXCLUDED_GOLD
+
+
 def verdicts(run: Path) -> dict[str, dict[str, Any]]:
     """qid -> the scored outcome, for cases this run actually scored."""
     out: dict[str, dict[str, Any]] = {}
+    queries: dict[str, str | None] = {}
     for line in (run / "events.jsonl").read_text().splitlines():
         if not line.strip():
             continue
         e = json.loads(line)
+        if e.get("kind") == "attempt":
+            # The query the arm actually ran. A flip is a matched pair -- same
+            # question, same model, one right answer and one wrong one -- and
+            # the diff between the two queries is what isolates the cause. It
+            # was not read here, so the richest evidence in the run was the one
+            # thing the flip table did not print.
+            queries[e["qid"]] = e.get("final_query")
+            continue
         if e.get("kind") != "score":
             continue
         v = e.get("verdict")
@@ -73,6 +103,7 @@ def verdicts(run: Path) -> dict[str, dict[str, Any]]:
             "verdict": v,
             "confidence": e.get("confidence"),
             "reason": (e.get("reason") or "")[:200],
+            "final_query": queries.get(e["qid"]),
             # near_match and needs_human are neither: counting either as a fail
             # would manufacture a flip every time the judge hedged in one run
             # and not the other.
@@ -82,6 +113,30 @@ def verdicts(run: Path) -> dict[str, dict[str, Any]]:
     return out
 
 
+def query_diff(a: dict[str, Any], b: dict[str, Any], la: str, lb: str) -> str:
+    """The two queries a flipped case ran, side by side.
+
+    The count says a case moved; this says what moved it. On a real set five
+    flips over 28 cases read as an 18% churn rate, and the five turned out to
+    be one question asked two ways -- two discrete query idioms, one returning
+    443 rows and the other 2. Reading the diff would have handed over the
+    finding; averaging the five hid it. The queries were already in the ledger
+    and this table had never printed them.
+
+    Identical queries are worth saying too: then the flip is downstream of the
+    query, in the judge or in the data, and that is a different search.
+    """
+    qa, qb = (a.get("final_query") or "").strip(), (b.get("final_query") or "").strip()
+    if not qa and not qb:
+        return "     (neither arm recorded a final query)"
+    if qa == qb:
+        return ("     both arms ran the SAME query, so the flip is downstream "
+                "of it:\n     the judge, the rubric, or non-determinism in the "
+                "data.")
+    return (f"     {la} ran:\n       " + qa.replace("\n", "\n       ") +
+            f"\n     {lb} ran:\n       " + qb.replace("\n", "\n       "))
+
+
 def cost(run: Path) -> dict[str, float]:
     tot: dict[str, float] = {"usd": 0.0, "turns": 0.0, "seconds": 0.0}
     for line in (run / "events.jsonl").read_text().splitlines():
@@ -114,6 +169,47 @@ COMPARABLE = ("datasetVersion", "datasetSha", "judgeVersion", "rubricSha",
               "retrievalMode", "modelGitSha", "targetVersion")
 
 
+def completeness_note(ca: dict, cb: dict, la: str, lb: str,
+                      A: dict, B: dict) -> int:
+    """Say which cases one arm excluded and the other scored. Never refuses.
+
+    This REPLACED a refusal, and the refusal was wrong. It rested on the claim
+    that an excluded case "reads as a flip", and it does not: a flip requires a
+    real pass or fail on BOTH sides (`a_only` / `b_only` below), so a case
+    carrying `verdict: null` on either side lands in `unscored` and contributes
+    to no flip count. The pairing already handles this, and the header already
+    prints "N cases, M scored in both, K not". Refusing a 99-versus-100 pair
+    threw away 99 good comparisons to avoid a distortion that was not there.
+
+    What IS worth saying is the asymmetry, which nothing named before. The
+    excluded cases are not a random sample: an attempt truncates BECAUSE it ran
+    long, so the cases one arm drops are its hard ones, and the surviving
+    comparison is over an easier subset than the case list suggests. That is a
+    caveat to carry into the number, not a reason to withhold it.
+
+    An `aborted` arm needs nothing here either: it stopped early, so its cases
+    are absent rather than unscored, and the existing "the runs do not cover
+    the same cases" check already exits 1 on it.
+    """
+    for label, cfg, mine, theirs in ((la, ca, A, B), (lb, cb, B, A)):
+        excluded = [q for q in sorted(set(mine) & set(theirs))
+                    if mine[q]["passed"] is None
+                    and theirs[q]["passed"] is not None]
+        if not excluded:
+            continue
+        why = ", ".join(
+            [f"{len(cfg.get('truncated') or [])} truncated"]
+            * bool(cfg.get("truncated"))
+            + [f"{len(cfg.get('contaminated') or [])} contaminated"]
+            * bool(cfg.get("contaminated"))) or "unscored"
+        print(f"\n  ! {label} left {len(excluded)} case(s) unscored that "
+              f"the other arm scored ({why}): {', '.join(excluded)}")
+        print("    They are out of the flips above, so the comparison holds --"
+              " but an attempt truncates BECAUSE it ran long, so these are not"
+              " a random sample of the set.")
+    return 0
+
+
 def retrieval_gate(ca: dict, cb: dict, la: str, lb: str,
                    allow: bool) -> int:
     """Refuse a pair whose runs used different retrievers.
@@ -402,13 +498,21 @@ def main() -> int:
               f"  ({', '.join(f'{v} {k}' for k, v in sorted(tally.items()))})")
 
     if a_only or b_only:
-        print(f"\nthe flips\n---------")
+        print(f"\nthe disagreement set\n--------------------")
+        print("  Read these before averaging them. A flip is a case where the "
+              "agent found\n  two paths and the model did not make one of them "
+              "obviously right, which is\n  a model-quality signal and a "
+              "matched pair: same question, same model, one\n  right answer "
+              "and one wrong one. The band says how much movement there is;\n"
+              "  these say what it IS.\n")
         for q in a_only:
             print(f"  {q}\n     {la}: {A[q]['verdict']}  ->  "
                   f"{lb}: {B[q]['verdict']}\n     {B[q]['reason'][:150]}")
+            print(query_diff(A[q], B[q], la, lb))
         for q in b_only:
             print(f"  {q}\n     {la}: {A[q]['verdict']}  ->  "
-                  f"{lb}: {B[q]['verdict']}\n     {B[q]['reason'][:150]}")
+                  f"{lb}: {B[q]['verdict']}\n     {A[q]['reason'][:150]}")
+            print(query_diff(A[q], B[q], la, lb))
 
     if a_args.targets:
         targeted_report(a_args, A, B, la, lb,
@@ -443,6 +547,7 @@ def main() -> int:
 
     gate = retrieval_gate(cfg_a, cfg_b, la, lb,
                           a_args.allow_retrieval_mismatch)
+    completeness_note(cfg_a, cfg_b, la, lb, A, B)
 
     ca, cb = cost(a_args.a), cost(a_args.b)
     print(f"\ncost\n----")

package/skills/eval-loop/scripts/flip_table_test.py

@@ -146,6 +146,70 @@ class Calibration(unittest.TestCase):
         self.assertIn("sonnet / opus", block)
 
 
+class CompletenessNote(unittest.TestCase):
+    """The asymmetry between two arms' unscored cases, reported not refused.
+
+    This replaced a refusal that was based on a false claim: that a case one
+    arm excluded "reads as a flip". It cannot. A flip needs a real pass or fail
+    on BOTH sides, so a null verdict on either lands in `unscored` and counts
+    toward no flip. Refusing a 99-versus-100 pair discarded 99 good
+    comparisons to avoid a distortion that was not there.
+    """
+
+    def verdicts(self, **qids):
+        return {q: {"verdict": v, "passed": p, "outcome": "x", "reason": "",
+                    "confidence": None}
+                for q, (v, p) in qids.items()}
+
+    def test_an_excluded_case_cannot_become_a_flip(self):
+        # The claim the refusal rested on, tested directly against the pairing
+        # rule the report uses.
+        A = self.verdicts(q1=("match", True), q2=(None, None))
+        B = self.verdicts(q1=("match", True), q2=("no_match", False))
+        shared = sorted(A)
+        a_only = [q for q in shared if A[q]["passed"] and B[q]["passed"] is False]
+        b_only = [q for q in shared if A[q]["passed"] is False and B[q]["passed"]]
+        unscored = [q for q in shared
+                    if A[q]["passed"] is None or B[q]["passed"] is None]
+        self.assertEqual(a_only + b_only, [])
+        self.assertEqual(unscored, ["q2"])
+
+    def test_it_never_refuses(self):
+        A = self.verdicts(q1=("match", True), q2=(None, None))
+        B = self.verdicts(q1=("match", True), q2=("no_match", False))
+        self.assertEqual(
+            ft.completeness_note({"status": "incomplete", "truncated": ["q2"]},
+                                 {"status": "complete"}, "a", "b", A, B), 0)
+
+    def test_two_complete_arms_say_nothing(self):
+        A = B = self.verdicts(q1=("match", True))
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            ft.completeness_note({}, {}, "a", "b", A, B)
+        self.assertEqual(out.getvalue(), "")
+
+    def test_it_names_the_cases_and_why(self):
+        A = self.verdicts(q1=("match", True), q2=(None, None))
+        B = self.verdicts(q1=("match", True), q2=("no_match", False))
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            ft.completeness_note({"truncated": ["q2"]}, {}, "arm-a", "arm-b",
+                                 A, B)
+        body = out.getvalue()
+        self.assertIn("arm-a left 1 case(s) unscored", body)
+        self.assertIn("1 truncated", body)
+        self.assertIn("q2", body)
+        # The caveat that is actually true: the dropped cases are the long ones.
+        self.assertIn("BECAUSE it ran long", body)
+
+    def test_the_answerer_model_is_already_a_pin(self):
+        # Not this note's job, and worth pinning so nobody adds a second
+        # mechanism: an arm on one model against another is already reported
+        # as a differing pin. That was the largest uncontrolled variable in the
+        # run this came from.
+        self.assertIn("answererModel", ft.COMPARABLE)
+
+
 class Outcome(unittest.TestCase):
     def test_near_match_is_neither(self):
         self.assertEqual(ft.outcome("near_match"), "neither")
@@ -153,6 +217,107 @@ class Outcome(unittest.TestCase):
     def test_an_unknown_verdict_is_never_a_fail(self):
         self.assertEqual(ft.outcome("invented"), "neither")
 
+class CountsTowardScore(unittest.TestCase):
+    """One rule, three readers, because two of them had drifted.
+
+    `run_baseline.py` dropped a `verified_wrong` golden before counting and
+    `eval_run.malloy` did not, while the package's own doc comment on
+    `gold_status` said it did. On a real 13-case run with one such golden the
+    printed rate read 91.67% and the notebook's `pass_rate` read 92.31%, off
+    the same rows, with nothing to say which was right.
+    """
+
+    def test_a_verified_wrong_golden_leaves_the_aggregates(self):
+        self.assertFalse(ft.counts_toward_score("verified_wrong"))
+
+    def test_every_other_status_counts(self):
+        for s in ("verified", "verified_benign", "suspect", "provisional",
+                  "ambiguous", "invalid"):
+            with self.subTest(status=s):
+                self.assertTrue(ft.counts_toward_score(s))
+
+    def test_an_absent_status_counts(self):
+        # A run predating gold_status must not silently lose every case.
+        self.assertTrue(ft.counts_toward_score(None))
+
+    def test_run_baseline_reads_this_rule_rather_than_restating_it(self):
+        src = (pathlib.Path(__file__).resolve().parent
+               / "run_baseline.py").read_text()
+        # The AGGREGATE site reads the helper. The other `verified_wrong`
+        # test in that file is a different question -- whether a case is
+        # scorable at all, from the golden's own status, before an attempt --
+        # and is correctly separate.
+        self.assertIn('counts_toward_score(v.get("gold_status"))', src)
+        self.assertNotIn('v.get("gold_status") == "verified_wrong"', src)
+
+    def test_the_flattener_writes_it_as_a_column(self):
+        src = (pathlib.Path(__file__).resolve().parent
+               / "build_run_package.py").read_text()
+        self.assertIn('"counts": counts_toward_score(', src)
+        self.assertIn('"counts", "reason"', src)
+
+    def test_the_malloy_measures_filter_on_that_column(self):
+        malloy = (pathlib.Path(__file__).resolve().parent.parent
+                  / "templates" / "eval-run-package" / "eval_run.malloy").read_text()
+        for m in ("confident_count", "passed", "failed", "near_matches",
+                  "needs_human"):
+            line = next(l for l in malloy.splitlines()
+                        if l.strip().startswith(f"{m} is count()"))
+            self.assertIn("counts = 'true'", line, f"{m} does not exclude it")
+        # Written as a string because flatten() serialises a bool that way.
+        self.assertIn("public: counts", malloy)
+
+class DisagreementSet(unittest.TestCase):
+    """A flip is a matched pair, and the table has to print the pair.
+
+    The count says a case moved; the two queries say what moved it. On a real
+    set five flips over 28 cases read as an 18% churn rate and were one
+    question asked two ways, the wrong idiom returning 2 rows against 443.
+    That took a day of transcript reading because this table printed only the
+    verdicts.
+    """
+
+    def test_it_prints_both_queries(self):
+        out = ft.query_diff({"final_query": "run: a -> { x }"},
+                            {"final_query": "run: b -> { y }"}, "arm1", "arm2")
+        self.assertIn("run: a -> { x }", out)
+        self.assertIn("run: b -> { y }", out)
+        self.assertIn("arm1", out)
+        self.assertIn("arm2", out)
+
+    def test_an_identical_query_is_called_out_as_a_different_finding(self):
+        q = "run: same -> { x }"
+        out = ft.query_diff({"final_query": q}, {"final_query": q}, "a", "b")
+        self.assertIn("SAME query", out)
+        self.assertIn("downstream", out)
+
+    def test_a_missing_query_says_so_rather_than_printing_nothing(self):
+        out = ft.query_diff({}, {}, "a", "b")
+        self.assertIn("neither arm recorded", out)
+
+    def test_one_side_missing_still_shows_the_other(self):
+        out = ft.query_diff({"final_query": "run: a"}, {}, "a", "b")
+        self.assertIn("run: a", out)
+
+    def test_verdicts_carries_the_final_query(self):
+        run = pathlib.Path(tempfile.mkdtemp()) / "r"
+        run.mkdir()
+        (run / "events.jsonl").write_text(
+            json.dumps({"kind": "attempt", "qid": "q1",
+                        "final_query": "run: flights -> { x }"}) + "\n" +
+            json.dumps({"kind": "score", "qid": "q1", "verdict": "match"}) + "\n")
+        v = ft.verdicts(run)
+        self.assertEqual(v["q1"]["final_query"], "run: flights -> { x }")
+
+    def test_a_run_with_no_attempt_event_still_scores(self):
+        # Older runs, and a --rebuild that wrote scores only.
+        run = pathlib.Path(tempfile.mkdtemp()) / "r"
+        run.mkdir()
+        (run / "events.jsonl").write_text(
+            json.dumps({"kind": "score", "qid": "q1", "verdict": "match"}) + "\n")
+        v = ft.verdicts(run)
+        self.assertTrue(v["q1"]["passed"])
+        self.assertIsNone(v["q1"]["final_query"])
 
 if __name__ == "__main__":
     unittest.main()