@malloy-publisher/skills 0.1.19 → 0.1.21

package/skills/eval-answer/scripts/mcp_payload_test.py

@@ -6,11 +6,12 @@ regression they exist for: an extractor written against the flat `results`
 shape returned [] for the nested one, which scored attempts that had received
 everything at 0% recall and moved three baseline failures off "query
 construction" onto "retrieval ranking" and "model coverage" -- the wrong team,
-for a bug in the eval.
+for a bug in the eval. Those are the labels as they read then; the first two
+are now `delivered, wrong` and `not retrieved`.
 """
 import unittest
 
-from mcp_payload import entity_ids, search_terms
+from mcp_payload import entity_hits, entity_ids, search_terms
 
 FLAT = {"results": [
     {"kind": "measure", "name": "total_sales_2022", "source": "order_items",
@@ -215,5 +216,75 @@ class EntityIds(unittest.TestCase):
             ["measure:s:m"])
 
 
+class EntityHits(unittest.TestCase):
+    """The provenance `entity_ids` throws away.
+
+    Without it, a required entity that never came back is indistinguishable
+    from one the agent never asked for -- a retrieval defect from a phrase
+    detection defect -- which is the reason retrieval scoring could not
+    attribute a miss.
+    """
+
+    PAYLOAD = {"sources": [{
+        "source_info": {"resource_id": {"source": "flights"}},
+        "entities": [
+            {"entity_id": "measure:flights:flight_count", "relevance": 0.81,
+             "matched_targets": [{"search_text": "flight count",
+                                  "relevance": 0.81}]},
+            {"name": "carrier", "entity_type": "dimension", "relevance": 0.44,
+             "matched_targets": [{"search_text": "carrier", "relevance": 0.44}]},
+        ]}]}
+
+    def test_it_agrees_with_entity_ids_about_identity(self):
+        # Two readers of one response that disagree about what an entity is
+        # called would make every comparison meaningless.
+        self.assertEqual([h["entity_id"] for h in entity_hits(self.PAYLOAD)],
+                         entity_ids(self.PAYLOAD))
+
+    def test_it_carries_relevance_and_the_matching_targets(self):
+        h = {x["entity_id"]: x for x in entity_hits(self.PAYLOAD)}
+        m = h["measure:flights:flight_count"]
+        self.assertEqual(m["relevance"], 0.81)
+        self.assertEqual(m["matched_targets"],
+                         [{"search_text": "flight count", "relevance": 0.81}])
+
+    def test_a_source_hit_has_no_attribution_and_says_null(self):
+        # The server discards a source card's target attribution before an
+        # entity object exists, so there is none to report. `null` is that
+        # claim; `[]` would claim the server attributed it to no target.
+        h = {x["entity_id"]: x for x in entity_hits(self.PAYLOAD)}
+        self.assertIsNone(h["source:flights:flights"]["matched_targets"])
+
+    def test_a_lexical_response_reports_no_attribution_not_an_empty_one(self):
+        # The lexical path withholds matched_targets by design: a lunr score is
+        # relative to its own query. Reading that as "matched no target" would
+        # charge the agent for never asking.
+        lex = {"sources": [{"source_info": {"resource_id": {"source": "f"}},
+                            "entities": [{"entity_id": "measure:f:x"}]}]}
+        h = entity_hits(lex)[1]
+        self.assertIsNone(h["matched_targets"])
+        self.assertIsNone(h["relevance"])
+
+    def test_position_is_not_a_rank(self):
+        # Kept because it is occasionally useful for reading a response back,
+        # never because it orders by relevance: the response sorts globally
+        # and then buckets into source cards.
+        hits = entity_hits(self.PAYLOAD)
+        self.assertEqual([h["position"] for h in hits], [1, 2, 3])
+        self.assertEqual([h["position_in_source"] for h in hits], [0, 1, 2])
+
+    def test_a_duplicate_entity_is_taken_once(self):
+        dup = {"sources": [
+            {"source_info": {"resource_id": {"source": "f"}},
+             "entities": [{"entity_id": "measure:f:x", "relevance": 0.9}]},
+            {"source_info": {"resource_id": {"source": "g"}},
+             "entities": [{"entity_id": "measure:f:x", "relevance": 0.1}]}]}
+        ids = [h["entity_id"] for h in entity_hits(dup)]
+        self.assertEqual(ids.count("measure:f:x"), 1)
+
+    def test_an_empty_payload_is_no_hits(self):
+        self.assertEqual(entity_hits({}), [])
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)

package/skills/eval-answer/scripts/publisher_rest.py

@@ -24,9 +24,15 @@ to the judge as the re-executed evidence for a verdict, so a nested result was
 judged against a golden it could not match.
 
 The recursive uncelling is therefore the one kept here, and both callers get it.
-It does not recover the field names INSIDE a nest -- the envelope only names the
-top level -- so nested records come back keyed `c0`, `c1`. That was true of the
-recursive version before this move too; it is a limit, not a regression.
+
+The request also asks for `compactJson`, and that is what recovers the field
+names INSIDE a nest. The typed envelope names only the top level, so a nested
+record came back keyed `c0`, `c1` -- and a golden written the way its author
+ran the query, with the real column names, then read as DRIFT, which blocks an
+arm. That was called a limit here; it was not. Publisher has answered plain
+rows all along, and the request simply never asked. `rows_from_result` handles
+both shapes (compact answers with `result` as a JSON string), so the envelope
+walk stays as the fallback for a server that ignores the flag.
 
 TWO ERROR CONVENTIONS, ON PURPOSE
 
@@ -103,7 +109,7 @@ def query(base: str, environment: str, package: str, model: str, malloy: str,
     """Run a Malloy query. Raises on transport, HTTP or parse failure."""
     req = urllib.request.Request(
         _query_url(base, environment, package, model),
-        data=json.dumps({"query": malloy}).encode(),
+        data=json.dumps({"query": malloy, "compactJson": True}).encode(),
         headers={"Content-Type": "application/json"})
     with urllib.request.urlopen(req, timeout=timeout) as r:
         return rows_from_result(json.loads(r.read().decode()))

package/skills/eval-answer/scripts/publisher_rest_test.py

@@ -1,8 +1,11 @@
 #!/usr/bin/env python3
 """Tests for publisher_rest. Stdlib only: python3 publisher_rest_test.py"""
+import json
 import os
 import unittest
+import unittest.mock
 
+import publisher_rest
 from publisher_rest import rows_from_result, uncell
 
 
@@ -73,6 +76,59 @@ class RowsFromResultTests(unittest.TestCase):
                          [{"brand": "Levi's", "by_year": [{"c0": 2022}]}])
 
 
+class CompactJson(unittest.TestCase):
+    """The request asks for compact rows, which is what recovers the names
+    inside a nest.
+
+    The typed envelope does not repeat field names below the top level, so a
+    nested record came back keyed `c0`, `c1`. A golden written the way its
+    author ran the query -- with the real column names -- then read as DRIFT,
+    and drift blocks an arm. Publisher has answered this all along; the
+    request just never asked.
+    """
+
+    def send(self, body):
+        """Capture the request `query()` builds, and hand back `body`."""
+        seen = {}
+
+        class Resp:
+            def read(self_): return json.dumps(body).encode()
+            def __enter__(self_): return self_
+            def __exit__(self_, *a): return False
+
+        def fake(req, timeout=None):
+            seen["payload"] = json.loads(req.data.decode())
+            return Resp()
+
+        with unittest.mock.patch.object(publisher_rest.urllib.request,
+                                        "urlopen", fake):
+            rows = publisher_rest.query("http://x", "e", "p", "m.malloy",
+                                        "run: x -> { ... }")
+        return seen["payload"], rows
+
+    def test_the_request_asks_for_compact_rows(self):
+        payload, _ = self.send({"result": "[]"})
+        self.assertIs(payload["compactJson"], True)
+        self.assertEqual(payload["query"], "run: x -> { ... }")
+
+    def test_a_nest_keeps_its_field_names(self):
+        # What the server returns under compactJson: `result` is a JSON
+        # STRING of plain rows, nested names intact.
+        body = {"result": json.dumps(
+            [{"carrier": "WN", "n": 2,
+              "top_dest": [{"destination_code": "PHX", "d": 1}]}])}
+        _, rows = self.send(body)
+        self.assertEqual(rows[0]["top_dest"], [{"destination_code": "PHX",
+                                                "d": 1}])
+        self.assertNotIn("c0", rows[0]["top_dest"][0])
+
+    def test_the_envelope_path_still_works(self):
+        # Kept as a fallback: a server that ignores the flag still parses.
+        body = envelope(["n"], [record({"number_value": 7})])
+        _, rows = self.send(body)
+        self.assertEqual(rows, [{"n": 7}])
+
+
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))
     unittest.main()