@malloy-publisher/skills 0.1.21 → 0.1.23

package/skills/eval-loop/scripts/run_baseline.py

@@ -465,6 +465,33 @@ def format_rows(rows: list[dict], limit: int = 60) -> str:
     return "\n".join(out)
 
 
+# A tool result too large for the model's context is not returned inline: the
+# host writes it to a file and hands back a notice naming the path. The
+# ANSWERER is unaffected -- it reads the file and carries on -- but the notice
+# is not JSON, so this module parsed nothing and recorded an empty entity list.
+# An empty list is not "nothing came back"; it scores as a total retrieval miss
+# on a call that returned a full response, and a run then reports a recall it
+# did not measure. Seen twice in one arm: 86.4% printed against 95% actual, and
+# diagnose went on to explain the phantom miss with an index-readiness story
+# that was not true.
+OFFLOADED = re.compile(r"(?:saved to|written to)\s+(/[^\s'\"]+)")
+
+
+def offloaded_json(text: str) -> dict[str, Any] | None:
+    """The response body a host spilled to a file, read back, or None.
+
+    Only ever reads a path the host itself named in the result it returned.
+    """
+    m = OFFLOADED.search(text or "")
+    if not m:
+        return None
+    try:
+        body = pathlib.Path(m.group(1)).read_text()
+    except OSError:
+        return None
+    return resource_json(body)
+
+
 def resource_json(text: str) -> dict[str, Any] | None:
     """The machine-readable part of a tool result. Publisher wraps it in a
     '[Resource from publisher at ...]' preamble; a hosted MCP may return
@@ -914,6 +941,52 @@ def resolve_target_version(scope: str | None, target_version: str | None
     return _strip_v(target_version, "--target-version")
 
 
+def measure_coverage(a: argparse.Namespace) -> str | None:
+    """Run check_coverage.py for this model version, into the run directory.
+
+    Coverage is the first of the four things a run reports -- can the model
+    express an answer at all, did retrieval deliver it, did the agent get it
+    right, what did it cost -- and the only one that says whether a failure
+    was ever winnable. A run that skips it cannot tell a model gap from a
+    retrieval miss from a bad answer, so it is measured rather than left to a
+    follow-up command nobody runs.
+
+    It reads the MODEL through the REST API, not the answers and not a local
+    checkout, so it needs no `--model-repo` and is valid for every arm against
+    this model version: keep the file and pass `--coverage` next time.
+
+    A failure here does not stop the arm. Coverage is one rung of the report,
+    and losing it is worth saying loudly and continuing; killing a run over a
+    side measurement would cost the answerers instead.
+    """
+    out = a.out / "coverage.json"
+    script = (pathlib.Path(__file__).resolve().parent.parent.parent
+              / "eval-answer" / "scripts" / "check_coverage.py")
+    cmd = [sys.executable, str(script), "--set", str(a.set_dir),
+           "--publisher", a.publisher, "--environment", a.environment,
+           "--package", a.package, "--out", str(out),
+           "--parallel", str(a.parallel)]
+    # `--only` narrows the arm, so it must narrow this too: without it a
+    # one-case re-run paid a claude -p call for every case in the set.
+    if a.only:
+        cmd += ["--only", a.only]
+    print("measuring coverage: can the model express each answer at all "
+          "(reads the model; no answerer, no judge, no warehouse)", flush=True)
+    try:
+        r = subprocess.run(cmd, timeout=a.timeout * 2)
+    except Exception as e:  # noqa: BLE001 -- any failure is the same call here
+        print(f"  ! coverage not measured ({e}); the covered? rung will be "
+              f"blank and no failure can be attributed to a model gap",
+              flush=True)
+        return None
+    if r.returncode != 0 or not out.exists():
+        print("  ! coverage not measured (check_coverage.py exited "
+              f"{r.returncode}); the covered? rung will be blank and no "
+              "failure can be attributed to a model gap", flush=True)
+        return None
+    return str(out)
+
+
 def run_retrieval_gate(a: argparse.Namespace) -> str:
     """Hold the arm until retrieval is steady, and say what happened.
 
@@ -1225,7 +1298,15 @@ def cascade_lines(c: dict | None) -> list[str]:
         return []
     covered = (c["total"] - c["not covered"] - c["unmeasured"]
                - c["no entities named"])
-    retrieved = covered - c["not retrieved"]
+    # Retrieval does NOT inherit coverage's denominator, and it must not be
+    # derived from the funnel either. `cascade()` is an elif chain: a row whose
+    # coverage was never measured stops at the first rung, so `not retrieved` is
+    # structurally 0 whenever coverage did not run, and computing the rung from
+    # it reports every retrieval as a success. `recall_scored` / `recall_short`
+    # are tallied outside the chain, over the rows whose recall was actually
+    # computed, which is what this rung is supposed to say.
+    retrieved = c.get("recall_scored", 0) - c.get("recall_short", 0)
+    not_retrieved = c.get("recall_short", 0)
     # A pass that stops on an earlier rung is reported there. Otherwise the
     # last rung reads as the pass count and disagrees with the headline.
     anyway = lambda n: f"; {n} answered correctly anyway" if n else ""
@@ -1245,7 +1326,7 @@ def cascade_lines(c: dict | None) -> list[str]:
     lines = [f"  cascade       {c['total']} cases",
              f"    covered?      {covered} yes, {c['not covered']} no (model "
              f"gap{anyway(c.get('passed_not_covered', 0))})" + covered_tail,
-             f"    retrieved?    {retrieved} yes, {c['not retrieved']} no "
+             f"    retrieved?    {retrieved} yes, {not_retrieved} no "
              f"(the entity exists and did not come back: the docs, or the "
              f"search wording; diagnose decides"
              f"{anyway(c.get('passed_not_retrieved', 0))})",
@@ -1875,6 +1956,18 @@ def run_answerer(case: dict[str, Any], a: argparse.Namespace,
                     calls.append({**info, "error": text[:300] if failed else None,
                                   "rankedSummary": None})
                 else:
+                    # The host may have spilled the body to a file. Read it
+                    # back rather than scoring the notice as an empty response.
+                    if payload is None:
+                        payload = offloaded_json(text)
+                    if payload is None and OFFLOADED.search(text or ""):
+                        # Named a file we could not read. Record NO summary
+                        # rather than an empty one: unmeasured is the truth,
+                        # and a zero here is a miss the run did not observe.
+                        calls.append({**info,
+                                      "error": text[:300] if failed else None,
+                                      "rankedSummary": None})
+                        continue
                     ids = entity_ids(payload or {})
                     # Which retriever answered, from the response that answered
                     # it: "semantic", "lexical" when the embedding path is down,
@@ -2645,6 +2738,11 @@ def main(argv: list[str] | None = None) -> int:
                          "in retrieval attribution, and run.json records which "
                          "report was read. Without it an unlabelled case is "
                          "attributed to nobody, not to the model")
+    ap.add_argument("--no-coverage", action="store_true",
+                    help="skip measuring coverage. The covered? rung goes "
+                         "blank and no failure can be attributed to a model "
+                         "gap, so a low score cannot be told from an "
+                         "unanswerable set")
     ap.add_argument("--parallel", type=int, default=4)
     ap.add_argument("--max-turns", type=int, default=30)
     ap.add_argument("--timeout", type=int, default=900)
@@ -3025,15 +3123,28 @@ def main(argv: list[str] | None = None) -> int:
 
     retrieval_gate = run_retrieval_gate(a)
 
-    # Coverage is a read of the MODEL and costs a judge call per case, so the
-    # run does not measure it; it consumes a report made separately and says
-    # which one. A path that does not exist is refused here, not discovered as
-    # a traceback after the answerers have been paid for.
+    # Coverage is one of the four things a run measures -- whether the model
+    # can express an answer AT ALL, before any question of whether retrieval
+    # found it or the agent got it right. A model that cannot express an answer
+    # cannot succeed at it, so a run without coverage cannot say whether a
+    # failure was ever winnable. It is measured by default for that reason.
+    #
+    # It reads the MODEL, not the answers, so it is the same for every arm
+    # against one model version: pass `--coverage` to reuse a report you
+    # already have, and it is used as given. Otherwise the run measures it
+    # once, here, before the answerers are paid for -- a `claude -p` per case
+    # with no warehouse, no judge and no answerer. `--no-coverage` skips it.
     if a.coverage and not pathlib.Path(a.coverage).exists():
         raise SystemExit(
             f"--coverage {a.coverage} does not exist. It should be a "
             f"check_coverage.py --out report for the model version this run "
             f"answers from.")
+    # `--rejudge` re-scores saved answers and calls no answerer, so it must
+    # not re-measure coverage either; it does not set `rebuild` (only
+    # `--from` does), so it needs naming here.
+    if not a.coverage and not a.no_coverage and not a.rebuild \
+            and not getattr(a, 'rejudge', False):
+        a.coverage = measure_coverage(a)
     coverage_report = coverage_report_summary(a.coverage) if a.coverage else None
 
     # BEFORE the fresh run.json overwrites it: a rebuild reuses saved verdicts

package/skills/eval-loop/scripts/run_baseline_test.py

@@ -1199,28 +1199,63 @@ class RunSummary(unittest.TestCase):
     def test_no_report_keeps_the_pointer(self):
         self.assertIn("not measured here", "\n".join(self.lines()))
 
+    def rows(self, spec):
+        """Cascade input built the way the harness builds it.
+
+        The fixtures here were hand-written dicts, and a hand-written cascade
+        can describe a state `cascade()` cannot produce -- which is how a
+        display bug survived its own test. Build the rows, let `cascade()`
+        count them.
+        """
+        out = []
+        for coverage, recall, failed, n in spec:
+            out += [{"coverage": coverage, "recall": recall, "failed": failed,
+                     "verdict": "no_match" if failed else "match"}] * n
+        return rb.cascade(out)
+
     def test_the_cascade_reads_as_a_funnel_with_owners(self):
-        lines = self.lines(cascade={
-            "total": 49, "not covered": 6, "unmeasured": 2,
-            "no entities named": 0, "not retrieved": 5,
-            "delivered, wrong": 6, "delivered, right": 30, "not scored": 0})
+        lines = self.lines(cascade=self.rows([
+            ("absent", None, True, 6),       # a known coverage gap
+            ("unknown", 1.0, True, 2),       # coverage never measured
+            ("covered", 0.5, True, 5),       # retrieved short
+            ("covered", 1.0, True, 6),       # delivered, wrong
+            ("covered", 1.0, False, 30),     # delivered, right
+        ]))
         text = "\n".join(lines)
         self.assertIn("cascade       49 cases", text)
         self.assertIn("covered?      41 yes, 6 no (model gap), 2 unmeasured", text)
-        # The rung must not assert the docs: a miss is the docs OR the search
-        # wording, and only diagnose separates them. This assertion is here
-        # because the label was renamed in score_retrieval and the display line
-        # in this file was missed, so the two disagreed in a shipped commit.
-        self.assertIn("retrieved?    36 yes, 5 no (the entity exists and did "
+        # 43 rows carried a recall (49 less the 6 coverage gaps), 5 fell short.
+        # The rung counts what recall MEASURED, not what the funnel let through.
+        self.assertIn("retrieved?    38 yes, 5 no (the entity exists and did "
                       "not come back", text)
         self.assertNotIn("(documentation", text)
-        # And a delivered-but-wrong answer names no owner until diagnose runs.
         self.assertIn("correct?      30 yes, 6 no (delivered, wrong", text)
         self.assertIn("diagnose decides", text)
-        # It heads the COVERAGE & RETRIEVAL layer, above the retrieval-mode line.
         self.assertLess(self.index_of(lines, "cascade"),
                         self.index_of(lines, "  retrieval "))
 
+
+    def test_unmeasured_coverage_does_not_zero_the_retrieval_rung(self):
+        """check_coverage is a separate spend, so most runs have none.
+
+        `cascade()` is an elif chain: a row whose coverage was never measured
+        stops at the first rung and never reaches the recall check, so
+        `not retrieved` is structurally 0. A display that subtracted it from a
+        denominator therefore reported EVERY retrieval as a success -- 12
+        cases, three at recall 0.5, printing "retrieved? 12 yes, 0 no". The
+        rung reads a tally kept outside the chain.
+        """
+        text = "\n".join(self.lines(cascade=self.rows([
+            ("unknown", 1.0, False, 10),
+            ("unknown", 0.5, False, 1),
+            ("unknown", None, False, 1),
+        ])))
+        self.assertIn("covered?      0 yes, 0 no (model gap), 12 unmeasured", text)
+        self.assertIn("retrieved?    10 yes, 1 no", text)
+        self.assertNotIn("retrieved?    12 yes", text)
+        self.assertNotIn("retrieved?    0 yes", text)
+
+
     def test_no_cascade_prints_nothing(self):
         self.assertNotIn("cascade", "\n".join(self.lines()))
 
@@ -1526,6 +1561,86 @@ class GoldenCheckScope(unittest.TestCase):
         self.assertIn("qids", src)
         self.assertIn('c["qid"] in qids', src)
 
+class CoverageByDefault(unittest.TestCase):
+    """Coverage is measured unless asked not to.
+
+    It answers the first of the four questions a run reports -- can the model
+    express an answer at all -- and it is the one that says whether a failure
+    was ever winnable. Left to a follow-up command it went unrun, and the
+    covered? rung came back blank on every real run.
+    """
+
+    def ns(self, **kw):
+        a = argparse.Namespace(
+            coverage=None, no_coverage=False, rebuild=False,
+            out=pathlib.Path("/tmp/does-not-matter"),
+            set_dir=pathlib.Path("evals/e"), publisher="http://p",
+            environment="env", package="pkg", timeout=60,
+            parallel=4, only=None, rejudge=False)
+        for k, v in kw.items():
+            setattr(a, k, v)
+        return a
+
+    def test_it_runs_when_no_report_was_given(self):
+        a = self.ns()
+        with mock.patch.object(rb, "subprocess") as sp, \
+             mock.patch.object(pathlib.Path, "exists", return_value=True):
+            sp.run.return_value = argparse.Namespace(returncode=0)
+            out = rb.measure_coverage(a)
+        self.assertIsNotNone(out)
+        cmd = sp.run.call_args[0][0]
+        self.assertIn("--publisher", cmd)
+        self.assertIn("http://p", cmd)
+        # Through REST, so it needs no local checkout and stays valid for every
+        # arm against this model version.
+        self.assertNotIn("--model", cmd)
+        self.assertIn("--parallel", cmd)
+
+    def test_only_narrows_coverage_as_it_narrows_the_arm(self):
+        """A one-case re-run paid a claude -p call for every case in the set."""
+        a = self.ns(only="q1,q2")
+        with mock.patch.object(rb, "subprocess") as sp, \
+             mock.patch.object(pathlib.Path, "exists", return_value=True):
+            sp.run.return_value = argparse.Namespace(returncode=0)
+            rb.measure_coverage(a)
+        cmd = sp.run.call_args[0][0]
+        self.assertIn("--only", cmd)
+        self.assertIn("q1,q2", cmd)
+
+    def test_a_failure_does_not_kill_the_arm(self):
+        """Losing one rung must not cost the answerers."""
+        a = self.ns()
+        with mock.patch.object(rb, "subprocess") as sp:
+            sp.run.return_value = argparse.Namespace(returncode=1)
+            self.assertIsNone(rb.measure_coverage(a))
+
+
+class OffloadedToolResult(unittest.TestCase):
+    """A response spilled to a file is not an empty response.
+
+    A get_context result too large for the model's context is written to a
+    file and replaced by a notice naming the path. The answerer reads the file
+    and is unaffected; the ledger used to parse the notice, find no JSON, and
+    record an empty entity list -- a total retrieval miss on a call that
+    returned in full. One arm printed 86.4% recall against an actual 95%, and
+    diagnose then explained the phantom miss with an index-readiness story
+    that was false.
+    """
+
+    def test_the_body_is_read_back_from_the_named_file(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = pathlib.Path(d) / "big.txt"
+            p.write_text(json.dumps({"sources": [], "retrieval": "semantic"}))
+            out = rb.offloaded_json(
+                f"Error: result exceeds maximum allowed tokens. "
+                f"Output has been saved to {p}")
+        self.assertEqual(out, {"sources": [], "retrieval": "semantic"})
+
+    def test_an_unreadable_path_is_none_not_empty(self):
+        self.assertIsNone(rb.offloaded_json("saved to /nope/does-not-exist.txt"))
+
+    def test_an_ordinary_result_is_not_mistaken_for_an_offload(self):
+        self.assertIsNone(rb.offloaded_json('{"sources": []}'))
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()

package/skills/eval-loop/templates/eval-run-package/public/app.js

@@ -92,9 +92,19 @@ function pill(v) {
   return `<span class="pill ${esc(v)}">${esc(verdictLabel[v] || v.replace('_', ' '))}</span>`;
 }
 
+/* The entities this answer needed, one dot each, and the count in front.
+   The dots alone made a reader count them to learn the one thing that
+   matters -- how many of the needed entities came back -- and the four
+   shades read as four verdicts rather than as one binary with three ways
+   of arriving at it. The fraction says it outright; the shades stay for
+   HOW each one was delivered, which the legend explains. */
 function dots(required) {
   if (!required || !required.length) return '';
-  return '<span class="dots">' + required.map(e =>
+  const got = required.filter(e => e.status !== 'missing').length;
+  const all = got === required.length;
+  return `<span class="dots"><span class="recall${all ? '' : ' short'}"`
+    + ` title="${got} of ${required.length} needed entities came back">`
+    + `${got}/${required.length}</span>` + required.map(e =>
     `<i class="dot ${esc(e.status)}" title="${esc(e.entity_id)} — ${esc(e.status)}"></i>`)
     .join('') + '</span>';
 }
@@ -145,7 +155,18 @@ function predictionBlock(p) {
       cells(l).map((c, i) => [cols[i] || `c${i}`, isFinite(c) && c !== '' ? Number(c) : c])));
     return rowsTable(rows);
   }
-  return `<div class="box mute">${esc(t)}</div>`;
+  // Malloy renders a result with no grouping as `name ------ value`, one line
+  // per field and not a pipe in sight, so it used to miss both branches above
+  // and land in the raw fallback: a lone number shown as
+  // `check_sum --------- 213939.33999999976`, full width, dashes and all.
+  // Through rowsTable it is a one-row table with the column named and the
+  // number formatted, which is what every other result here looks like.
+  const scalar = t.split('\n').map(l => l.match(/^\s*([^\s|]+)\s+-{3,}\s+(.*\S)\s*$/))
+    .filter(Boolean);
+  if (scalar.length && scalar.length === t.split('\n').filter(l => l.trim()).length)
+    return rowsTable([Object.fromEntries(scalar.map(([, k, v]) =>
+      [k, isFinite(v) && v !== '' ? Number(v) : v]))]);
+  return `<div class="box mute rawtext">${esc(t)}</div>`;
 }
 
 /* Enough markdown for an agent's answer: fences, tables, bold, code, headings, bullets. */
@@ -280,7 +301,8 @@ async function loadCase(qid) {
   const d0 = detail[0];
   let html = `<div><h3>Reference answer</h3>${goldenBlock(d0)}`;
   if (d0.rubric || d0.must_state)
-    html += `<details class="sub" style="margin-top:8px"><summary>Rubric</summary>
+    html += `<details class="sub" style="margin-top:8px"><summary><span
+      class="caret">&#9654;</span>Rubric &mdash; how this answer was judged</summary>
       <div class="box" style="margin-top:6px">${d0.rubric ? esc(d0.rubric) : ''}${
         d0.must_state ? `<br><br><b>Must state.</b> ${esc(d0.must_state)}` : ''}</div></details>`;
   html += '</div>';
@@ -318,10 +340,13 @@ async function loadCase(qid) {
         than on what the answer claims.</p>${predictionBlock(d.prediction)}`
         : `<h3>Re-executed rows</h3><p class="cap">None. The judge scored this
         attempt from its answer text and the reference answer alone.</p>`}
-      <h3>The attempt, step by step</h3>
+      <details class="trace"><summary><span class="caret">&#9654;</span>The attempt,
+        step by step<span class="mute"> &middot; ${mine.length || 1} step${
+          (mine.length || 1) === 1 ? '' : 's'}</span></summary>
       <div class="timeline">${mine.length ? mine.map(stepHtml).join('')
         : `<div class="step text"><div class="prose">${md(d.answer)}</div></div>${
             d.final_query ? `<div class="step execute_query"><span class="k">final query</span><pre>${esc(d.final_query)}</pre></div>` : ''}`}</div>
+      </details>
       ${d.transcript ? `<div class="qid" style="margin-top:10px">transcript: ${esc(d.transcript)}</div>` : ''}
     </div>`;
   }

package/skills/eval-loop/templates/eval-run-package/public/index.html

@@ -96,6 +96,9 @@
   .kind { font:400 12px "IBM Plex Mono",monospace; color:var(--mute); }
   .armcell { display:flex; align-items:center; justify-content:flex-end; gap:10px; }
   .dots { display:flex; gap:4px; align-items:center; }
+  .recall { font-family:"IBM Plex Mono",monospace; font-size:11.5px;
+            color:var(--mute); margin-right:3px; white-space:nowrap; }
+  .recall.short { color:var(--fail); font-weight:600; }
   .dot { width:10px; height:10px; border-radius:50%; display:inline-block; background:var(--na); }
   .dot.exact,.dot.alias { background:var(--pass); }
   .dot.alias { background:color-mix(in srgb,var(--pass) 60%,var(--surface)); }
@@ -166,8 +169,30 @@
   td.num { text-align:right; }
   tr:last-child td { border-bottom:0; }
   .more { font-size:12px; color:var(--mute); padding:6px 10px; }
-  details.sub summary { cursor:pointer; font-size:13px; color:var(--accent); list-style:none; }
-  details.sub summary::-webkit-details-marker { display:none; }
+  /* A bare coloured word did not read as clickable: it looks like a label, and
+     the rubric went unopened. Give every disclosure the same affordance the
+     case rows have -- a caret that turns, a border, and a hover state. */
+  details.sub summary, details.trace summary {
+    cursor:pointer; font-size:13px; color:var(--accent); list-style:none;
+    display:inline-flex; align-items:center; gap:7px;
+    border:1px solid var(--rule); border-radius:6px; padding:5px 10px;
+    background:var(--surface); user-select:none; }
+  details.sub summary:hover, details.trace summary:hover { border-color:var(--accent); }
+  details.sub summary::-webkit-details-marker,
+  details.trace summary::-webkit-details-marker { display:none; }
+  details.sub summary .caret, details.trace summary .caret { font-size:9px; }
+  details.sub[open] summary .caret, details.trace[open] summary .caret {
+    transform:rotate(90deg); }
+  /* The trace is the longest thing in the drawer and the least often wanted:
+     a reader opens a case for the verdict and the rows, and only then asks how
+     the agent got there. Collapsed, the judge's reasoning and the re-executed
+     rows sit above the fold. */
+  details.trace { margin-top:18px; }
+  details.trace > .timeline { margin-top:10px; }
+  /* Raw text the parsers could not turn into a table: keep it small and
+     wrapped rather than letting a long line set the drawer width. */
+  .rawtext { font-family:"IBM Plex Mono",monospace; font-size:12.5px;
+             white-space:pre-wrap; word-break:break-word; }
   .empty { color:var(--mute); font-style:italic; padding:20px; text-align:center; }
   .err { background:var(--fail-soft); border:1px solid var(--fail); color:var(--fail);
          padding:14px 16px; border-radius:6px; font-size:13.5px; }
@@ -190,11 +215,14 @@
     <span class="count" id="count"></span>
   </div>
   <div class="legend">
-    <span>Dots: the entities the golden answer depends on —</span>
-    <span><i class="dot exact"></i> returned as a ranked entity</span>
-    <span><i class="dot alias"></i> returned under a sibling source</span>
-    <span><i class="dot in_docs"></i> in a returned source's docs</span>
-    <span><i class="dot missing"></i> not returned</span>
+    <span><b>Retrieval</b>, per case: how many of the entities the right answer
+      needs came back from <code>get_context</code>. One dot each, and the
+      count in front &mdash; <span class="recall">3/3</span> means all of them.</span>
+    <span class="mute">The shade says how each one arrived:</span>
+    <span><i class="dot exact"></i> ranked directly</span>
+    <span><i class="dot alias"></i> under a sibling source</span>
+    <span><i class="dot in_docs"></i> named in a returned source's docs</span>
+    <span><i class="dot missing"></i> <b>did not come back</b></span>
   </div>
   <!-- Every case row carries two words from the SET, describing the question
        rather than how it went. Both were read as verdicts until they were

package/skills/eval-report/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-report
 description: 'Write the summary a person reads after an evaluation run: what ran, what it scored, which failures were the MODEL and which were the EVAL itself (out of turns, contaminated, unestablished golden, unreadable judge), what was learned, and what to do next. Every claim links to the artifact behind it. Use at the end of any run, before quoting a number to anyone. Never scores an answer (eval-answer) or decides who owns a failure (eval-diagnose).'
-version: 0.1.21
+version: 0.1.23
 ---
 
 # Report a Run
@@ -26,6 +26,30 @@ one exception is anything that qualifies THIS run's number -- a truncated
 attempt, a contaminated one, an unestablished answer key -- which the "Eval
 failures" section exists for.
 
+**Do not audit the harness. You were asked to measure a model.** This is the
+most common way this job goes wrong, and it does not look like going wrong: a
+run turns up something odd in a script, the odd thing is genuinely a bug, and
+the reply comes back as a critique of the tooling with the model's score
+somewhere underneath. The reader asked what their model scored. Answer that.
+
+So, unless the user asked you to work on the harness:
+
+- Do not read harness source to satisfy your own curiosity about a number.
+  Read it when a number you must report cannot be explained any other way, and
+  stop when it can.
+- Do not propose harness fixes, refactors, flags or "while I was in there"
+  improvements. Not in the report, not in the chat reply.
+- When a harness defect DID change this run's number, the report gets one
+  sentence: what the number should be and why. Not the mechanism, not the
+  file, not the fix.
+- Keep a defect that changed nothing out of the report entirely. Mention it
+  once in chat, in a line, and let the user decide whether they want it
+  chased.
+
+A harness bug you found and did not chase is not a loose end. It is the job
+being done. If the user wants it fixed they will say so, and then it is a
+different task with its own turn.
+
 ## Step 1: build the artifacts, before writing a word
 
 ```bash
@@ -64,6 +88,29 @@ not in `~/Downloads`, both of which have lost a findings document before.
 from the console.** A figure retyped from scrollback is a figure nobody can
 check, and the console rounds.
 
+### Keep it under a page and a half
+
+**Budget: about 80 lines, and 120 is the ceiling for a run with several
+distinct failures.** Reports have shipped at 200 and the length is not
+thoroughness -- it is the artifact links restated as prose, the cascade
+explained twice, and a paragraph apologising for a figure nobody disputed.
+A reader who wants the detail opens the case matrix, which is why step 1 builds
+it. What cannot be recovered from the artifacts is your judgement: what broke,
+why, and what to do. Spend the lines there.
+
+What earns its place: the headline, one row per case, one short entry per
+failure, the retrieval numbers, and the next steps. What does not: restating a
+number you already gave, explaining what a cascade is before showing it,
+defending a decision nobody questioned, or any section whose content is that
+nothing happened -- except "Eval failures", where "None." is the point.
+
+**Being brief is not being terse with the vocabulary.** The reader does not
+know what `near_match`, a cluster, recall or a holdout is, so the first time
+one appears, say what it means in the same sentence -- "`near_match`, which is
+excluded from the pass rate" -- and then use it. Cutting the explanations is
+how a short report becomes an unreadable one; cutting the restatements is how
+it becomes a good one.
+
 ### The template
 
 ```markdown
@@ -77,7 +124,7 @@ check, and the console rounds.
 - Target `<env>/<package>`, model `<modelPath>`, pinned `<modelSha or targetVersion>`
 - Answerer `<model>`, judge `<model>`, cap `<maxTurns>` turns
 - Steps run: scrape/run, eval<, diagnose><, improve>
-- Cost $X answerer + $Y judge
+- Cost $X answerer + $Y judge, N turns median / N p90, N entities delivered per answer
 
 ## The result
 
@@ -88,9 +135,15 @@ check, and the console rounds.
 
 [case matrix](<url>) - [aggregate tables](<url>)
 
-## Retrieval and coverage
+## Coverage, retrieval, accuracy, cost
 
-Covered? -> Retrieved? -> Correct?, with the per-arm numbers under each.
+**Coverage N of M** -- how many questions the model can express an answer to at
+all. **Entity recall N%** -- one number, first, before the cascade. It is the share
+of the entities an answer needed that retrieval actually handed the agent, and
+it is the headline of this section for the same reason the pass rate is the
+headline of the last one. Then the cascade: Covered? -> Retrieved? -> Correct?,
+with the per-arm numbers under each. Say in one clause what recall counts, then
+give the number.
 
 ## Model failures
 
@@ -119,10 +172,10 @@ these in order and put every one that fires into the list, with its command:
 
 | If the run shows | Then the next step is |
 |---|---|
-| `coverage: unmeasured` | run `check_coverage.py --set <set> --model <pkg> --out coverage.json`, then re-run with `--coverage` so it charges the failures |
+| `coverage: unmeasured` | the run was given `--no-coverage`, or the measurement failed and said so. It is measured by default, so say in "Eval failures" that this score cannot tell a model gap from a bad answer, and re-run without the flag |
 | `goldenCheck: skipped` or the set names no `truthPackage` | build one with `init_truth_package.py`; until then the goldens were derived through the model under test and certify themselves |
 | any golden still `provisional` | re-derive and `verify_goldens.py --promote` |
-| a stale entity name warning | fix `expectedEntities`; it scores as a retrieval miss on every run until you do |
+| a stale entity name warning | fix `expectedEntities`; it scores as a retrieval miss on every run until you do. A next step, not a section: it goes in this list and nowhere else in the report |
 | a passing case with recall below 1.0 | check whether `required` over-specifies one path |
 | `truncated` non-empty | re-run those cases at a higher cap with `--from` |
 | diagnose did not run | run it, or say the failures have no owner yet |
@@ -165,8 +218,23 @@ wrong answer. One entry each: what the answer said, what was right, and the
 mechanism in one sentence. Do not write a cluster id here; write what it got
 wrong.
 
-**Eval failures** are everything that stopped the run measuring the model.
-Report every one that occurred, with its count and its qids, and say plainly
+**Eval failures** are the things that stopped the run measuring the model, and
+**only** those. The test is one question: *did this cost a verdict, or make one
+untrustworthy?* If no, it does not appear in the report at any length.
+
+That rules out most of what is tempting to put here, and all of it has been put
+here on a real run: a `-dirty` model pin, a stale-entity-name warning that cost
+no case, a defect you hit in the harness and worked around, a setup step that
+took two tries, anything you would open with "worth knowing, though it changes
+no verdict". A reader wants to know what their model scored. Harness defects are
+real and belong in a harness issue, filed against the harness -- that is the
+skill's opening rule, and this section is where it gets broken.
+
+The section is usually two words. "**None.**" is a complete and good answer, and
+a reader who sees it learns exactly what they need to. Do not pad it into a
+paragraph explaining the absence.
+
+Report every one that DID occur, with its count and its qids, and say plainly
 that these are NOT evidence about the model:
 
 | What happened | How it reads in the ledger | Who fixes it |
@@ -185,11 +253,32 @@ harness prints `INCOMPLETE` and withholds the percentage; the report does the
 same. Give the counts and the re-run command instead of a number with a caveat,
 because the number is what gets repeated and the caveat is what gets dropped.
 
-## Retrieval and coverage belong in every report
+## The four measurements, and why a report carries all of them
+
+A pass rate says an answer was wrong. It does not say where, and on its own it
+makes every failure look like the model's. A run measures four things, in this
+order, and each one tells you whether the next is even a fair question:
 
-A pass rate says an answer was wrong. It does not say WHERE, and the three
-metrics that do are already in the run summary and the notebook. Report them,
-because a report that omits them makes every failure look like the model's:
+| | Measures | A failure here means |
+|---|---|---|
+| **a. Coverage** | can the data and the model express an answer at all | nothing downstream was winnable. Fix the model |
+| **b. Retrieval** | did `get_context` hand the agent the entities it needed | the model has it and the agent never saw it: the docs, or the search wording |
+| **c. Accuracy** | did the agent get the answer right | it had what it needed and still missed: the agent, or a doc that misleads |
+| **d. Cost** | dollars, turns, wall-clock, entities per answer | it works and cannot be afforded, which is its own kind of not working |
+
+**Coverage first, and a report without it is incomplete.** A model that cannot
+express an answer can never succeed at that question, so a pass rate quoted
+without coverage cannot distinguish a bad model from an unanswerable set.
+`run_baseline.py` measures it by default; if a run skipped it, say so in
+"Eval failures" and treat the score as provisional.
+
+**Cost is a result, not an aside.** Report dollars, the turn and call counts,
+and the entity payload per answer. An agent that answers correctly in 40 turns
+and 75 delivered entities per question is a different product from one that
+does it in 8 and 5, and only the report says which you have.
+
+Report them all, because a report that omits them makes every failure look
+like the model's:
 
 **Copy the cascade the run printed. Do not re-derive it.** `cascade_lines()`
 in `run_baseline.py` already renders every rung, and it carries one field a