@malloy-publisher/skills 0.1.19 → 0.1.21

package/skills/eval-answer/scripts/check_findable_test.py

@@ -0,0 +1,326 @@
+#!/usr/bin/env python3
+"""Tests for the findability audit. No server: `get_context` is substituted.
+
+The audit exists because `expectedEntities.required` is what "did the agent ask
+for it" and "was it retrieved" are both computed against, so an id retrieval
+cannot deliver makes both numbers fiction -- and the case then reports a
+retrieval miss forever, which reads as the model's fault rather than the key's.
+"""
+from __future__ import annotations
+
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import check_findable  # noqa: E402
+
+MODEL = {
+    ("measure", "flight count"): "measure:flights:flight_count",
+    ("measure", "average plane size"): "measure:flights:average_plane_size",
+    ("dimension", "state"): "dimension:airports:state",
+    ("source", "flights"): "source:flights:flights",
+}
+
+
+def fake_get_context(_url, targets, _env, _pkg, timeout=60):
+    """A model that answers only a right-typed search for a name it has."""
+    t = targets[0]
+    eid = MODEL.get((t["target_type"], t["search_text"]))
+    ents = ([{"entity_id": eid, "relevance": 1.0}]
+            if eid and not eid.startswith("source:") else [])
+    return {"sources": [{"source_info": {"resource_id": {"source": "flights"}},
+                         "entities": ents}]}
+
+
+def case(qid, required=(), any_of=()):
+    return {"qid": qid, "expectedEntities": {"required": list(required),
+                                             "requiredAnyOf": [list(g) for g in any_of]}}
+
+
+class Findable(unittest.TestCase):
+    def setUp(self):
+        self._real = check_findable.get_context
+        check_findable.get_context = fake_get_context
+
+    def tearDown(self):
+        check_findable.get_context = self._real
+
+    def run_check(self, cases):
+        return check_findable.check(cases, "http://x/mcp", "e", "p")
+
+    def test_a_reachable_entity_is_no_finding(self):
+        f, rows = self.run_check([case("q", ["measure:flights:flight_count"])])
+        self.assertEqual(f, [])
+        self.assertTrue(rows[0]["found"])
+
+    def test_an_entity_the_model_does_not_have_is_caught(self):
+        f, _ = self.run_check([case("q", ["measure:flights:revenue_per_mile"])])
+        self.assertEqual(len(f), 1)
+        self.assertIn("retrieval miss forever", f[0])
+
+    def test_a_real_name_under_the_wrong_kind_is_caught(self):
+        # The case a text grep cannot catch: `flight_count` IS in the model, so
+        # checking the name against the source passes it. Only a search of the
+        # declared kind shows that no `dimension` request can return it.
+        f, _ = self.run_check([case("q", ["dimension:flights:flight_count"])])
+        self.assertEqual(len(f), 1)
+        self.assertIn("`dimension` search", f[0])
+
+    def test_a_malformed_id_is_caught_before_any_search(self):
+        f, rows = self.run_check([case("q", ["flight_count"])])
+        self.assertEqual(len(f), 1)
+        self.assertIn("malformed", f[0])
+        self.assertEqual(rows, [])
+
+    def test_an_unknown_kind_is_caught(self):
+        f, _ = self.run_check([case("q", ["metric:flights:x"])])
+        self.assertIn("unknown entity kind", f[0])
+
+    def test_requiredanyof_members_are_checked_too(self):
+        # A group is satisfied by any member, so a dead member is not fatal to
+        # the case -- but it is still an id nothing can deliver, and a group
+        # whose every member is dead is a case that can never score.
+        f, rows = self.run_check([case("q", any_of=[
+            ["measure:flights:flight_count", "measure:flights:gone"]])])
+        # Both members are searched; only the dead one is a finding.
+        self.assertEqual(len(rows), 2)
+        self.assertEqual(len(f), 1)
+        self.assertIn("gone", f[0])
+
+    def test_one_entity_required_by_six_cases_is_searched_once(self):
+        cases = [case(f"q{i}", ["measure:flights:flight_count"]) for i in range(6)]
+        _, rows = self.run_check(cases)
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(len(rows[0]["requiredBy"]), 6)
+
+    def test_findings_never_outnumber_the_required_ids(self):
+        # `rows` holds only the ids that reached a search; a malformed id is a
+        # finding that never becomes a row. Reporting `len(rows) - len(f)`
+        # subtracted findings that were never in the total and printed
+        # "-1 of 2 required entities are retrievable".
+        cases = [case("q", ["flight_count",                  # malformed
+                            "measure:flights:gone",          # genuine miss
+                            "measure:flights:also_gone"])]   # genuine miss
+        f, rows = self.run_check(cases)
+        total = len(check_findable.required_ids(cases))
+        self.assertEqual(total, 3)
+        self.assertEqual(len(rows), 2)
+        self.assertEqual(len(f), 3)
+        self.assertGreaterEqual(total - len(f), 0)
+
+    def test_the_finding_names_every_case_that_would_be_affected(self):
+        f, _ = self.run_check([case("q1", ["measure:flights:gone"]),
+                               case("q2", ["measure:flights:gone"])])
+        self.assertIn("q1, q2", f[0])
+
+
+class Phrase(unittest.TestCase):
+    def test_an_identifier_becomes_the_easiest_possible_query(self):
+        self.assertEqual(check_findable.phrase_for("average_plane_size"),
+                         "average plane size")
+
+    def test_a_join_path_is_flattened(self):
+        self.assertEqual(
+            check_findable.phrase_for("aircraft.aircraft_models.seats"),
+            "aircraft aircraft models seats")
+
+
+class KindMap(unittest.TestCase):
+    def test_it_inverts_the_forward_map_the_server_pins(self):
+        # `score_retrieval.KINDS_BY_TARGET` is pinned against the server's own
+        # table. This is its inverse and must not drift from it.
+        import score_retrieval
+        for target, kinds in score_retrieval.KINDS_BY_TARGET.items():
+            for kind in kinds:
+                self.assertEqual(check_findable.TARGET_FOR_KIND.get(kind),
+                                 target,
+                                 f"{kind} should be searched as {target}")
+
+
+class CompiledModel(unittest.TestCase):
+    """The compiled model is the authority on existence AND kind.
+
+    A grep over the `.malloy` text is neither, and is wrong in both directions:
+    it passes a real name under the wrong kind, and it fails a column the
+    source exposes implicitly.
+    """
+
+    DECLARED = {
+        "flights": {"source:flights", "measure:flight_count",
+                    "dimension:distance"},
+        "airports": {"source:airports", "measure:airport_count",
+                     "dimension:state", "dimension:own_type"},
+    }
+
+    def find(self, *ids):
+        cases = [{"qid": "q", "expectedEntities": {"required": list(ids)}}]
+        return check_findable.declared_findings(cases, self.DECLARED)
+
+    def test_a_real_name_under_the_wrong_kind_names_the_real_kind(self):
+        f = self.find("dimension:flights:flight_count")
+        self.assertEqual(len(f), 1)
+        self.assertIn("declared measure, not dimension", f[0])
+        self.assertIn("hard filter", f[0])
+
+    def test_an_implicit_column_is_NOT_a_finding(self):
+        # The grep false positive: `own_type` appears zero times in the
+        # .malloy and is exposed by the source. Acting on that finding deletes
+        # a good entity from the key.
+        self.assertEqual(self.find("dimension:airports:own_type"), [])
+
+    def test_a_field_that_does_not_exist_is_a_finding(self):
+        f = self.find("measure:flights:revenue_per_mile")
+        self.assertIn("declares no field", f[0])
+
+    def test_an_unknown_source_is_a_finding(self):
+        f = self.find("measure:gone:x")
+        self.assertIn("no source", f[0])
+
+    def test_a_correct_id_is_silent(self):
+        self.assertEqual(self.find("measure:flights:flight_count",
+                                   "dimension:airports:state"), [])
+
+    def test_a_malformed_id_is_left_to_the_other_check(self):
+        # `check` reports it before any search; reporting it twice reads as
+        # two defects.
+        self.assertEqual(self.find("flight_count"), [])
+
+    def test_a_finding_is_identified_by_its_whole_entity_id(self):
+        # Splitting on the bare colon returns the KIND, and deduplicating the
+        # search findings against the compiled-model ones on that basis
+        # silently deleted an unrelated entity's genuine finding -- which can
+        # flip the exit code from 1 to 0.
+        self.assertEqual(
+            check_findable.finding_id(
+                "measure:flights:distinct_planes: a `measure` search for "
+                "'distinct planes' does not return it (required by q2)"),
+            "measure:flights:distinct_planes")
+
+    def test_dedup_keeps_a_finding_that_only_shares_a_kind(self):
+        declared = ["measure:orders:total_sales: the compiled model has no "
+                    "source 'orders' (required by q1)"]
+        found = ["measure:flights:distinct_planes: a `measure` search does "
+                 "not return it (required by q2)"]
+        already = {check_findable.finding_id(f) for f in declared}
+        kept = declared + [f for f in found
+                           if check_findable.finding_id(f) not in already]
+        self.assertEqual(len(kept), 2)
+
+    def test_an_unreachable_server_is_not_read_as_an_empty_model(self):
+        # None means "not checked". Treating it as {} would report every id in
+        # the set as missing.
+        self.assertIsNone(check_findable.compiled_entities(
+            "http://127.0.0.1:9", "e", "p"))
+
+
+def fake_model_doc(queries=(), fields=(("carrier", "dimension"),)):
+    """One model path, one source, with whatever fields and queries are asked
+    for. Mirrors the REST shape: `queries` is a SIBLING of `sourceInfos`."""
+    return {"sourceInfos": [{"name": "flights",
+                             "schema": {"fields": [{"name": n, "kind": k}
+                                                   for n, k in fields]}}],
+            "queries": [dict(q) for q in queries]}
+
+
+class CompiledQueries(unittest.TestCase):
+    """A model-level named query is not a field and never appears under
+    `sourceInfos`; `FieldInfoType` has no `query`. Read from the response's own
+    `queries` array or every `query:` id reads as a field that does not exist --
+    the exact false positive `compiled_entities` exists to end."""
+
+    def declared_from(self, doc):
+        """`compiled_entities`' per-doc walk, with the two HTTP calls stubbed."""
+        import json as _json
+        import urllib.request as _u
+        real = _u.urlopen
+
+        class Resp:
+            def __init__(self, payload): self.payload = payload
+            def read(self): return _json.dumps(self.payload).encode()
+            def __enter__(self): return self
+            def __exit__(self, *a): return False
+
+        def fake(req, timeout=30):
+            url = req.full_url if hasattr(req, "full_url") else str(req)
+            return Resp([{"path": "m.malloy"}] if url.endswith("/models")
+                        else doc)
+
+        _u.urlopen = fake
+        try:
+            return check_findable.compiled_entities("http://x", "e", "p")
+        finally:
+            _u.urlopen = real
+
+    def test_a_named_query_is_declared(self):
+        declared = self.declared_from(fake_model_doc(
+            queries=[{"name": "top_carriers", "sourceName": "flights"}]))
+        self.assertIn("query:top_carriers", declared["flights"])
+
+    def test_a_named_query_raises_no_finding(self):
+        declared = self.declared_from(fake_model_doc(
+            queries=[{"name": "top_carriers", "sourceName": "flights"}]))
+        cases = [case("q1", required=["query:flights:top_carriers"])]
+        self.assertEqual(check_findable.declared_findings(cases, declared), [])
+
+    def test_without_the_queries_array_it_would_have_been_a_false_finding(self):
+        # The regression this pins: reading only `sourceInfos` reports a real,
+        # retrievable query as a field the source does not declare, and that
+        # finding survives main()'s dedup and exits 1.
+        declared = self.declared_from(fake_model_doc(queries=[]))
+        cases = [case("q1", required=["query:flights:top_carriers"])]
+        out = check_findable.declared_findings(cases, declared)
+        self.assertEqual(len(out), 1)
+        self.assertIn("declares no field", out[0])
+
+    def test_a_query_over_an_inline_source_stays_undeclared(self):
+        # It has no source name, and the server excludes it from the index for
+        # that reason, so an id naming it is genuinely unreachable.
+        declared = self.declared_from(fake_model_doc(
+            queries=[{"name": "adhoc"}]))
+        self.assertNotIn("query:adhoc", declared["flights"])
+
+
+class Staleness(unittest.TestCase):
+    """A stale package answers the models endpoint normally while describing
+    the compile BEFORE the last save, so the compiled model is not the
+    authority and must not be read as one."""
+
+    def status(self, payload):
+        import json as _json
+        import urllib.request as _u
+        real = _u.urlopen
+
+        class Resp:
+            def read(self): return _json.dumps(payload).encode()
+        _u.urlopen = lambda req, timeout=30: Resp()
+        try:
+            return check_findable.stale_packages("http://x")
+        finally:
+            _u.urlopen = real
+
+    def test_a_stale_package_is_named(self):
+        self.assertEqual(
+            self.status({"loadErrors": [{"environment": "e", "package": "p",
+                                         "stale": True}]}),
+            {("e", "p")})
+
+    def test_an_ordinary_load_failure_is_not_staleness(self):
+        # No `stale` flag: the package did not load at all, which is a
+        # different fact and is visible by the package simply being absent.
+        self.assertEqual(
+            self.status({"loadErrors": [{"environment": "e", "package": "p",
+                                         "message": "boom"}]}),
+            set())
+
+    def test_a_healthy_server_omits_the_field(self):
+        self.assertEqual(self.status({"operationalState": "serving"}), set())
+
+    def test_an_unreadable_status_is_not_read_as_healthy(self):
+        # None means "could not tell", which main() reports rather than
+        # silently treating the model as current.
+        self.assertIsNone(check_findable.stale_packages("http://127.0.0.1:9"))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/skills/eval-answer/scripts/json_scan.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""Pull JSON objects out of a model's prose reply. Stdlib only.
+
+Both judges in this loop ask an agent for a JSON verdict and get back prose
+with the object somewhere in it, and both must survive the agent quoting a
+Malloy snippet on the way: `re.search(r"\\{.*\\}", re.S)` is greedy, spans the
+FIRST brace in the document to the LAST, and hands `json.loads` a blob that
+starts mid-query. A good reply then reads as unparseable, and the case leaves
+every bucket -- including the denominator the pass rate is printed over.
+
+It lives in its own module because the two callers cannot import each other:
+`check_coverage` already reaches into `run_baseline` for `BLOCKED_TOOLS`, so
+`run_baseline` importing `check_coverage` back is a cycle. They each had their
+own scanner, written months apart, differing in algorithm and agreeing only by
+luck -- the shape of drift the rest of this harness spent several commits
+removing. One definition, imported by both, is the fix.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def json_objects(text: str) -> list[dict[str, Any]]:
+    """Every JSON object in `text`, in the order they appear.
+
+    Decoded from each `{` with `raw_decode` rather than matched with a regex,
+    so a brace that opens no valid object costs one character of scanning
+    instead of corrupting the match. A caller that wants the verdict takes the
+    LAST object naming one: the judge is told to end with it, and prose
+    reasoning toward it may quote a fragment on the way.
+    """
+    dec, out, i = json.JSONDecoder(), [], 0
+    while (i := text.find("{", i)) >= 0:
+        try:
+            v, end = dec.raw_decode(text, i)
+        except json.JSONDecodeError:
+            i += 1
+            continue
+        if isinstance(v, dict):
+            out.append(v)
+        i = max(end, i + 1)
+    return out

package/skills/eval-answer/scripts/ledger.py

@@ -67,6 +67,7 @@ EVENTS: dict[str, dict[str, set[str]]] = {
                      "mcp_tool_uses", "final_query_source",
                      "reported_calls", "contaminated", "contamination_reasons",
                      "input_tokens", "output_tokens", "cache_read_tokens",
+                     "cache_write_tokens", "skills_invoked",
                      "cost_usd", "num_turns", "wall_seconds", "run_error", "at"},
     },
     "tool_call": {
@@ -85,7 +86,13 @@ EVENTS: dict[str, dict[str, set[str]]] = {
         # errors, while a replay under another report's filter values returns
         # real rows for the wrong population. Keeping all three on the event
         # is what lets a re-execution be audited from the ledger at all.
-        "optional": {"targets", "rankedSummary", "error", "traceId",
+        # `target_shapes` is one row per search target -- its type, and whether
+        # it carried search text -- beside `targets`, which is the terms
+        # searched for and drops a target that carried none. Without it the
+        # bare-target rate is not recomputable from a run directory, only from
+        # transcripts, and transcripts get pruned.
+        "optional": {"targets", "target_shapes", "scopes", "rankedSummary",
+                     "error", "traceId",
                      "query", "modelPath", "filterParams", "retrieval_mode",
                      "at"},
     },
@@ -97,7 +104,14 @@ EVENTS: dict[str, dict[str, set[str]]] = {
         "optional": {"judge_version", "rubric_sha", "golden_revision",
                      "artifactPath", "confidence", "column_pairing",
                      "contaminated", "gold_status", "gold_note",
-                     "must_not_use_hits", "judge_verdict", "at"},
+                     "must_not_use_hits", "judge_verdict",
+                     # What this case's judge call cost. Collected per case and
+                     # discarded on the write path until now, so the run could
+                     # report what judging cost in total and never which case
+                     # was expensive -- and a case that spent its whole turn
+                     # budget without producing a verdict is the one worth
+                     # finding.
+                     "judge_cost_usd", "at"},
     },
     "retrieval_score": {
         "required": {"intentId", "term"},
@@ -157,17 +171,44 @@ RUN_RECOMMENDED = {"judgeModel", "judgeVersion", "datasetVersion", "modelSha",
 RUN_OPTIONAL = {"label", "effort", "environment", "package", "modelPath",
                 "modelGitSha", "mcpUrl", "publisher", "predictionsReExecuted",
                 "serverVersion", "diagnoserModel", "improverModel",
-                "rubricSha", "setName", "targetVersion", "scope", "mode",
+                "rubricSha",
+                # The prompt the judge was SHOWN, hashed. `rubricSha` covers
+                # the judge's skill file and not the harness's own prompt
+                # template, so two runs whose prompts differed compared as the
+                # same judge -- the blind spot that let a golden-rendering
+                # defect run for three arms with every test passing.
+                "judgePromptSha", "setName", "targetVersion", "scope", "mode",
                 "traceMode",
+                # The answerer's cap and timeout, as the run actually ran them.
+                # `eval-loop` step 6 lists "call budget" among the pins to
+                # freeze for a whole arm, and nothing wrote one, so no two runs
+                # could be compared on the setting that decides whether a case
+                # got to finish at all. `callBudget` stays for runs written
+                # before this, and is not the same field: it was never
+                # populated by this harness.
+                "maxTurns", "answererTimeout",
+                # How many attempts never reached a verdict for a reason that
+                # is the HARNESS's, not the model's. Both leave the denominator
+                # and both suppress the pass rate (`incomplete` below).
+                "truncated", "contaminated",
                 "callBudget", "status", "answererSkills",
                 "answererCostUsd", "judgeCostUsd", "goldenCheck",
+                # The run whose attempts this run's answerer cost was COPIED
+                # from, when it did not spawn an answerer at all (a re-judge, a
+                # rebuild, a `--from`). Null when this run paid. Summing
+                # `answererCostUsd` across run files double-counts without it:
+                # one re-judge reported $17.56 of answering that never happened.
+                "answererCostCopiedFrom",
+                # Set when a rebuild reused saved verdicts, so `judgeCostUsd`
+                # is the ORIGINAL judging spend rather than this run's zero.
+                "judgeCostCopiedFrom",
                 "skillsRoot", "harnessVersion",
                 "judgeSkills", "diagnoserManifest",
                 "improverManifest", "doubtedGoldens",
                 "packageSha", "servedRevision", "datasetSha",
                 "staleEntityNames",
                 "retrievalMode", "retrievalCalls", "retrievalGate",
-                "reExecution",
+                "reExecution", "coverageReport",
                 "modelRepo"} | RUN_RECOMMENDED
 
 
@@ -306,14 +347,29 @@ def skills_git_sha(root: pathlib.Path | None = None) -> str | None:
     the `skillsVersion` pin. Defaults to this checkout; pass the external
     --skills-root when the doctrine came from elsewhere (a Publisher checkout).
     The skills ARE the doctrine the agents load, so a run that cannot name
-    their version cannot take part in a comparison."""
-    d = pathlib.Path(root) if root else pathlib.Path(__file__).resolve().parent
+    their version cannot take part in a comparison.
+
+    `-dirty` is decided over the SKILLS PATH, not the whole repository. It used
+    to be the whole tree, so every run ever taken read `-dirty` -- including
+    runs taken before anything was edited -- because a checkout picks up stray
+    untracked files as a matter of course. A marker that cannot distinguish
+    "someone left a scratch file in the repo root" from "the judge prompt was
+    rewritten between these two arms" is not a pin, and the runs that carry it
+    cannot be told apart on the one thing it exists to record."""
+    # Resolved, always: the path is used BOTH as git's working directory and as
+    # its pathspec, and a relative one means different things in those two
+    # positions. Passed `skills`, git ran in `skills/` and then looked for
+    # `skills/skills`, which matches nothing -- so every tree read clean and
+    # the marker silently stopped working in the other direction.
+    d = (pathlib.Path(root) if root
+         else pathlib.Path(__file__).resolve().parent).resolve()
     try:
         head = subprocess.run(["git", "-C", str(d), "rev-parse", "HEAD"],
                               capture_output=True, text=True, timeout=10)
         if head.returncode != 0:
             return None
-        dirty = subprocess.run(["git", "-C", str(d), "status", "--porcelain"],
+        dirty = subprocess.run(["git", "-C", str(d), "status", "--porcelain",
+                                "--", str(d)],
                                capture_output=True, text=True, timeout=10)
         return head.stdout.strip() + ("-dirty" if dirty.stdout.strip() else "")
     except Exception:  # noqa: BLE001

package/skills/eval-answer/scripts/ledger_test.py

@@ -178,6 +178,13 @@ class WriteContract(unittest.TestCase):
                          filterParams={"report_id": "123"})
         self.assertEqual(e["filterParams"], {"report_id": "123"})
 
+    def test_an_attempt_may_carry_cache_write_tokens(self):
+        # The fourth token column. Without it the ledger held a cache-inclusive
+        # cost_usd and a breakdown that could not reproduce it.
+        e = ledger.event("attempt", **ATTEMPT, cache_read_tokens=2435273,
+                         cache_write_tokens=900000)
+        self.assertEqual(e["cache_write_tokens"], 900000)
+
     def test_run_config_requires_identity(self):
         with self.assertRaises(ValueError) as cm:
             ledger.run_config(**{k: v for k, v in RUN.items()

package/skills/eval-answer/scripts/mcp_payload.py

@@ -73,14 +73,123 @@ def search_terms(tool_input: dict[str, Any]) -> list[str]:
     return terms
 
 
+def entity_hits(payload: dict[str, Any]) -> list[dict[str, Any]]:
+    """Every returned entity with the provenance the response carries.
+
+    `entity_ids` answers "what came back". This answers "how", and the two
+    facts it adds are the ones the scorer needs and never had:
+
+    - `relevance`, the server's own score. On the semantic path it is a cosine
+      and IS comparable across targets, which is exactly why the server
+      publishes it there and withholds it on the lexical path, where a lunr
+      score is relative to its own query.
+    - `matched_targets`, naming WHICH of the caller's search targets matched
+      this entity and how well. Without it a required entity that never came
+      back is indistinguishable from one the agent never asked for -- the
+      distinction between a retrieval defect and a phrase-detection defect,
+      and the whole reason retrieval scoring could not attribute a miss.
+
+    `position` is the entity's index in the flattened response and is NOT a
+    rank. The response is sorted globally by relevance and then bucketed into
+    source cards, so flattening card by card interleaves. The harness used to
+    write `range(1, n+1)` here and call it a rank; read `relevance` instead.
+
+    Entities are the only thing that carries this. A source card's own target
+    attribution is discarded by the server before an entity object exists, so
+    a `source:` hit has none and says so with `matched_targets: None`.
+    """
+    hits: list[dict[str, Any]] = []
+    seen: set[str] = set()
+
+    def add(ident: str, source: str, ent: dict[str, Any] | None,
+            in_card: int | None) -> None:
+        if ident in seen:
+            return
+        seen.add(ident)
+        mt = (ent or {}).get("matched_targets")
+        hits.append({
+            "entity_id": ident,
+            "source": source,
+            "relevance": (ent or {}).get("relevance"),
+            # None and [] are different claims: None is "this path publishes
+            # no attribution" (a source hit, or a lexical run), [] is "the
+            # server attributed it to no target".
+            "matched_targets": ([{"search_text": m.get("search_text"),
+                                  "relevance": m.get("relevance")}
+                                 for m in mt if isinstance(m, dict)]
+                                if isinstance(mt, list) else None),
+            "position": len(hits) + 1,
+            "position_in_source": in_card,
+        })
+
+    for node in payload.get("sources") or []:
+        if not isinstance(node, dict):
+            continue
+        info = node.get("source_info")
+        rid = info.get("resource_id") if isinstance(info, dict) else None
+        src = rid.get("source") if isinstance(rid, dict) else None
+        if not isinstance(src, str) or not src:
+            continue
+        add(f"source:{src}:{src}", src, None, 0)
+        for i, ent in enumerate(node.get("entities") or [], 1):
+            if not isinstance(ent, dict):
+                continue
+            # The same id rule `entity_ids` uses, so the two cannot disagree
+            # about what an entity is called.
+            eid = ent.get("entity_id")
+            if not (isinstance(eid, str) and eid):
+                name, kind = ent.get("name"), ent.get("entity_type")
+                if not (isinstance(name, str) and name and isinstance(kind, str)):
+                    continue
+                eid = f"{kind.lower()}:{src}:{name}"
+            add(eid, src, ent, i)
+    return hits
+
+
+def target_shapes(tool_input: dict[str, Any]) -> list[dict[str, Any]]:
+    """Every search target's TYPE, and whether it carried search text.
+
+    Separate from `search_terms` because that function answers a different
+    question -- what did the answerer search FOR -- and to answer it, it drops
+    a target with no `search_text`: there is no term to record. A target with
+    no text is not a search, it is an enumeration of that type over the scope,
+    and dropping it means the ledger cannot express the single statistic the
+    "the agent enumerates instead of searching" argument is made of. Measured
+    on a real 10-case run, every attempt read `targetsWithoutSearchText: 0`
+    because a bare target had already been discarded before the ledger saw it.
+
+    So this keeps one row per target, text or not, and nothing else. Reading
+    the bare-target rate off transcripts instead works and is what has been
+    done; it means the number cannot be recomputed from a run directory after
+    the transcripts are pruned.
+    """
+    out: list[dict[str, Any]] = []
+    for t in tool_input.get("search_targets") or []:
+        if not isinstance(t, dict):
+            continue
+        text = t.get("search_text") or t.get("text")
+        out.append({"type": t.get("target_type") or "?",
+                    "has_text": bool(isinstance(text, str) and text.strip())})
+    return out
+
+
+def entity_id(kind: str, source: str | None, name: str) -> str:
+    """The `kind:source:name` id, minted in one place.
+
+    This file was already the only place ids are BUILT; naming the format as a
+    function keeps it that way now that the definition ledger needs to build
+    them too. `score_retrieval.split_entity()` is the matching reader.
+    """
+    return f"{kind}:{source}:{name}" if source else f"{kind}:{name}"
+
+
 def _ident(node: dict[str, Any]) -> str | None:
     eid = node.get("entityId")
     if isinstance(eid, str) and eid:
         return eid
     kind, name = node.get("kind"), node.get("name")
     if kind in ENTITY_KINDS and isinstance(name, str) and name:
-        src = node.get("source")
-        return f"{kind}:{src}:{name}" if src else f"{kind}:{name}"
+        return entity_id(kind, node.get("source"), name)
     return None