@malloy-publisher/skills 0.1.20 → 0.1.21

package/skills/eval-improve/scripts/improve.py

@@ -175,9 +175,17 @@ def verify_goldens(a: argparse.Namespace, d: pathlib.Path,
     # and its failure was indistinguishable from a golden the edit really did
     # invalidate. `--environment` too: the verifier's own default is `samples`,
     # which silently verified against the wrong environment on any other set.
+    # The TRUTH server's environment, which is not the model server's. It is a
+    # separate server and names its environments however it likes, so passing
+    # `--environment` here 404'd every case ("Environment 'local' could not be
+    # resolved to a path"), the audit exited non-zero, and the acceptance
+    # check read that as "this edit may have invalidated a golden" and BLOCKED
+    # every cluster. No edit could ever be accepted. `run_baseline.py` grew
+    # `--truth-environment` for exactly this; this caller had not.
     cmd = [sys.executable, str(script.resolve()),
            "--set", str(a.set_dir.resolve()),
-           "--environment", a.environment]
+           "--environment",
+           getattr(a, "truth_environment", None) or a.environment]
     # Appended only when set: passing None here is a TypeError inside
     # subprocess.run, and the two excepts below catch OSError and a timeout,
     # not that -- so it would have crashed improve.py AFTER the model edit,
@@ -185,6 +193,9 @@ def verify_goldens(a: argparse.Namespace, d: pathlib.Path,
     # exits 3, which `couldNotRun` already reads correctly.
     if a.truth_publisher:
         cmd += ["--publisher", a.truth_publisher]
+    # getattr: older callers and the tests build this Namespace by hand.
+    if getattr(a, "definitions", None):
+        cmd += ["--definitions", str(a.definitions)]
     if model:
         cmd += ["--model", str(model)]
     # The isolation guard needs to know what is under test; without it the
@@ -331,11 +342,24 @@ def main(argv: list[str] | None = None) -> int:
                          "whole point of the second server. Without it the "
                          "value check does not run and the acceptance check "
                          "blocks")
+    ap.add_argument("--truth-environment", default=None,
+                    help="the environment name on the TRUTH server, when it "
+                         "differs from --environment. That server is separate "
+                         "and names its environments independently; without "
+                         "this the audit 404s on every case and the "
+                         "acceptance check blocks every cluster. Same flag, "
+                         "same reason, as run_baseline.py")
+    ap.add_argument("--definitions", type=pathlib.Path, default=None,
+                    help="a definition ledger (verify_definitions.py). Passed to "
+                         "the golden audit so a set with no truth package can "
+                         "pass the acceptance check when every tested definition "
+                         "is validated")
     ap.add_argument("--max-turns", type=int, default=60)
     ap.add_argument("--timeout", type=int, default=1800)
     ap.add_argument("--retries", type=int, default=1)
     ap.add_argument("--only", default=None, help="comma-separated issue_ids")
-    ap.add_argument("--limit", type=int, default=None)
+    ap.add_argument("--limit", type=int, default=None,
+                    help="how many to process; each one spawns a real agent. Omit for no limit. 0 means zero, not unlimited.")
     ap.add_argument("--force", action="store_true",
                     help="redo clusters that already have a result")
     ap.add_argument("--no-isolate", dest="isolate", action="store_false",
@@ -392,7 +416,9 @@ def main(argv: list[str] | None = None) -> int:
         want = {x.strip() for x in a.only.split(",")}
         issues = [i for i in issues if i["issue_id"] in want]
     issues.sort(key=lambda i: -len(i.get("qids", [])))
-    if a.limit:
+    # `is not None`: see diagnose.py's select_cases. 0 means zero issues,
+    # not every issue -- each one spawns a real agent that edits the model.
+    if a.limit is not None:
         issues = issues[:a.limit]
     if not issues:
         print("no open model-owned issues; run diagnose.py first, or every "

package/skills/eval-improve/scripts/improve_test.py

@@ -13,6 +13,7 @@ import subprocess
 import sys
 import tempfile
 import unittest
+from unittest import mock
 
 HERE = pathlib.Path(__file__).resolve().parent
 sys.path.insert(0, str(HERE))
@@ -42,6 +43,28 @@ class Invocation(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree(self.tmp, ignore_errors=True)
 
+    def capture(self):
+        seen = {}
+
+        def fake_run(cmd, **kw):
+            seen["cmd"] = cmd
+            return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
+        with mock.patch.object(improve.subprocess, "run", fake_run):
+            improve.verify_goldens(self.a, self.art, "a diff")
+        return seen["cmd"]
+
+    def test_definitions_reaches_the_verifier_command(self):
+        # The acceptance check can pass on a set with no truth package only if
+        # the ledger gets as far as the verifier.
+        self.a.definitions = pathlib.Path("/tmp/led.jsonl")
+        cmd = self.capture()
+        self.assertIn("--definitions", cmd)
+        self.assertIn("/tmp/led.jsonl", cmd)
+
+    def test_a_namespace_without_definitions_is_still_accepted(self):
+        # Older callers and these fixtures build the Namespace by hand.
+        self.assertNotIn("--definitions", self.capture())
+
     def test_the_verifier_accepts_the_arguments_improve_sends(self):
         r = improve.verify_goldens(self.a, self.art, "a diff")
         # Exit 2 is argparse refusing the command. Whatever else happens, the
@@ -177,5 +200,33 @@ class VerifierContract(unittest.TestCase):
         self.assertEqual(p.returncode, 0)
 
 
+class TruthEnvironment(unittest.TestCase):
+    """The truth server is separate and names its environments independently.
+
+    Passing the MODEL server's environment to the audit 404'd every case, the
+    audit exited non-zero, and the acceptance check read that as "this edit
+    may have invalidated a golden" -- so it BLOCKED every cluster and no edit
+    could ever be accepted. run_baseline.py grew --truth-environment for
+    exactly this; this caller had not.
+    """
+
+    SRC = (pathlib.Path(__file__).resolve().parent / "improve.py").read_text()
+
+    def test_the_flag_exists(self):
+        self.assertIn('ap.add_argument("--truth-environment"', self.SRC)
+
+    def test_the_audit_is_given_the_truth_environment(self):
+        self.assertIn('getattr(a, "truth_environment", None) or a.environment',
+                      self.SRC)
+
+    def test_it_falls_back_to_the_model_environment(self):
+        # One server serving both is the ordinary local case, and it must keep
+        # working without the new flag.
+        import argparse as _a
+        ns = _a.Namespace(environment="samples", truth_environment=None)
+        self.assertEqual(
+            getattr(ns, "truth_environment", None) or ns.environment, "samples")
+
+
 if __name__ == "__main__":
     unittest.main()

package/skills/eval-judge/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-judge
 description: 'Decide whether ONE answer matches its golden, and say whether you believe the golden. Read this before emitting any verdict. Covers containment, column pairing, near_match, refusals, and the gold_status judgement. Use when scoring an attempt in an evaluation run; never to conduct a run (eval-loop), diagnose a failure (eval-diagnose) or edit a model (eval-improve).'
-version: 0.1.20
+version: 0.1.21
 ---
 
 # The judge

package/skills/eval-loop/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-loop
 description: 'Conduct a local Publisher evaluation loop in five steps: scrape/run, eval, diagnose, improve, checkpoint. You are the conductor: import cases into the file ledger, spawn a blind answerer, then run eval-answer, eval-diagnose, and eval-improve. Persistence is plain files under the model package''s evals/ directory; checkpoints are git commits of the model repo. Use to score a model, diagnose failures, improve behind an acceptance check, or roll back a bad direction.'
-version: 0.1.20
+version: 0.1.21
 ---
 
 # The Evaluation Loop
@@ -41,7 +41,8 @@ all of them for every run is how a skill stops being read.
 | a golden is wrong, doubted, or out of step with the model | `reference/golden-side-door.md` |
 | auditing a key you doubt, or a set you did not author | `reference/auditing-an-answer-key.md` |
 | deciding whether an edit stays | `reference/acceptance-check.md` |
-| about to quote a number, or set the noise band | `reference/measurement.md` |
+| about to quote a number, set the band, or read a set's flips | `reference/measurement.md` |
+| the run finished and someone has to read it | `skill:eval-report` |
 | you changed judge doctrine or its inputs | `reference/checking-the-judge.md` |
 
 Read the file, do not work from the summary here. The acceptance-check rules and
@@ -58,10 +59,22 @@ produces a confident wrong answer rather than an error.
 | **d. improve** | One smallest model edit, then the acceptance check | improve writes `candidate`; you write `acceptance_check`. Revert on reject. |
 | **e. checkpoint** | Git commit after an accepted acceptance check | `checkpoint` event, then the commit |
 
+Every run then **reports** (below). A run whose result exists only as JSONL and
+a scrolled-away console summary has not been delivered.
+
 **scrape** and **run** share a letter but are not the same job. Scrape writes
 cases. Run writes attempts. Do not invent questions and score
 them in one breath.
 
+**Step d is `skill:eval-improve`, not a hand edit plus another arm.** The
+failure mode is specific and it has happened: the edits were made by hand,
+published, and validated by re-running the whole arm -- with the answer key
+repaired in the same window. No `candidate`, `acceptance_check` or `checkpoint`
+event exists in that run's ledger, and the resulting move cannot be attributed
+to the model, the key, or the answerer. Re-running a whole arm after changing
+two things is the one thing the acceptance check exists to replace: it scores
+the edit against dev AND holdout, and it is cheaper than the arm.
+
 ### Scrape, minimally
 
 Importing an existing corpus IS the scrape step: copy the set from its home
@@ -292,6 +305,29 @@ the run measure something other than what it names:
    under a published version, which is what makes it a pin. Check which you have
    before deciding how much of this you need.
 
+   **Look for a set and for prior runs before you author either.** A minute of
+   `find . -name cases.jsonl`, a glance at `evals/*/runs/` and at your host's
+   own transcripts for this repo. Two sessions fourteen minutes apart built the
+   same 29-case answer key from scratch, because the first had committed
+   nothing before it was deleted and the second had no way to know it existed.
+   Roughly a working day was spent twice, and three specific things were
+   rediscovered at cost: the MCP login flow, the retrieval gate's 401, and a
+   judge-rendering bug that had already cost $17 of arm once. The two keys,
+   independently authored from the same questions, differ by about ten points
+   on comparable answers -- which is the available measure of how much a key
+   depends on its author, and a reason to reuse one rather than rebuild it.
+
+   **Audit the set's entity ids before the first arm.** `verify_goldens.py`
+   checks that each id names something in the model; `check_findable.py` checks
+   that a search of its own kind actually returns it. Both are free of model
+   calls. An id that fails either scores a retrieval miss on every run, and the
+   miss reads as the model's fault.
+
+   **Commit the set before you spend money on an arm**, and keep durable
+   outputs in the repository. A findings document in `~/Downloads` is gone the
+   first time somebody tidies up; the set, the run directory and the write-up
+   belong in git beside the model.
+
 2. The server must be up with retrieval tracing on, so a call's ranked results
    can be recovered afterwards (open-source Publisher: `PUBLISHER_MCP_TRACE=retrieval`).
    Confirm a trace lookup is available (absent means tracing is off).
@@ -332,6 +368,44 @@ the run measure something other than what it names:
    whole run. Raising a call budget mid-run moved mean outcomes on an unchanged
    model.
 
+   The call budget is `--max-turns`, written to `run.json` as `maxTurns`.
+   **Size it from a pilot rather than taking the default of 30.** Run the three
+   cheapest cases uncapped (`--max-turns 100`) and set the cap at twice their
+   maximum. A set whose questions need two or three sources joined does not fit
+   a cap sized for single-source lookups: on one such set the completed
+   attempts had a median of 17 turns and a 90th percentile of 26 against a cap
+   of 30, and four cases died at it. A cap 15% above the 90th percentile of
+   completed work is not a safety margin. Note also that `malloy-analysis` tells
+   the answerer to persist -- retry a phrasing, let a small query settle whether
+   a field exists -- so a tight cap and that instruction are in direct conflict.
+
+   **Do not change the model and the measuring instrument in the same step.**
+   Those are two different axes and only one of them is cheap to separate.
+
+   Batching MODEL edits is fine and expected. Clustering exists so that one
+   edit closes several cases, and an arm per fix does not survive contact with
+   arithmetic: 20 fixes over 100 cases is 2,000 answers, and at the measured
+   $0.33 a case on a proxied warehouse that is $660 of answering to attribute
+   what the acceptance check attributes for the price of the affected cases
+   plus holdout. Budget five arms for a defensible claim (a baseline, two for
+   the A/A, two post-edit), not one per edit, and let `skill:eval-improve`
+   carry each cluster.
+
+   The instrument is the other axis: the answer key, the judge and its prompt,
+   the answerer model, the skills. Move one of those together with the model
+   and there is nothing left holding still, so the result measures neither. On
+   the run this comes from, nine model commits and a re-derived answer key
+   landed between two arms, and the move from 20% to 39% belongs to no one --
+   not because two model edits were batched, but because the ruler changed at
+   the same time as the thing being measured. A key repair mid-improve is not
+   forbidden; it ends that comparison, so re-baseline rather than quoting a
+   delta across it.
+
+   The answerer model is instrument too, and it is the one most often left
+   unstated. The same 23-case set read 12 match / 11 near / 0 no_match on one
+   model and 8 / 4 / 15 on a smaller one. No statement about "the agent's"
+   capability means anything until two arms name the same answerer.
+
 7. Generate every answerer prompt from the stored case in `cases.jsonl`.
    Never retype the question. A truncated retype is indistinguishable from a
    real question downstream.
@@ -352,6 +426,31 @@ the run measure something other than what it names:
 6. `skill:eval-improve` only when this run includes improve, and only for
    `owner: model`. Then run the acceptance check. On accept, checkpoint.
 
+## Report the run, or nobody can read it
+
+A run directory is JSONL. It is a record, not a result, and the console summary
+scrolls away. **Every run ends by producing something a person can open**, and
+that job is `skill:eval-report`: it builds the servable run package (the case
+matrix app and the aggregate notebook) and gives the template for the write-up.
+
+Read it at the end of every run, including a run that failed. The standing
+complaint about this loop is that "a bunch of stuff happens and it is hard to
+know the actual results", and a ledger nobody renders is why.
+
+Two rules from it are worth repeating here, because they are the ones a
+conductor skips:
+
+- **Separate a MODEL failure from an EVAL failure.** A wrong answer and a
+  broken measurement look identical in a pass rate and have nothing else in
+  common. An arm holding a truncated, contaminated or environment-failed
+  attempt has no rate to quote at all.
+- **Say when a step did not run, and why.** `improve` not running because every
+  cluster came back `owner: agent-skill` is a RESULT, and it reads identically
+  to having forgotten unless it is written down.
+
+Keep the write-up in the repository beside the set, not in a chat log and not
+in `~/Downloads`.
+
 ## Checkpoint
 
 A checkpoint is a git commit of the model repository, taken after an acceptance check
@@ -401,7 +500,16 @@ conductor. Do not:
 
 - The model is the only thing improve edits. No question text, qids, or
   expected values in any name, doc, or comment.
-- When the environment misbehaves, stop. Never diagnose a sick system.
+- When the environment misbehaves, stop. Never diagnose a sick system. The
+  harness is part of the environment: a known-broken measurement does not
+  become quotable by being finished. Measured against this directive, an arm
+  already paid for gets quoted anyway -- it happened three times in one run,
+  once with the confound stated in the same message that started the arm -- so
+  the harness now enforces the parts it can. A run with a truncated or
+  contaminated attempt prints no pass rate and records `status: incomplete`,
+  and `flip_table.py` names what one arm left unscored that the other did not.
+  Read `run_error` on the attempts
+  and the INCOMPLETE line before quoting any number.
 - When a subagent disagrees with you, probe. Do not win by authority.
 - When a rule here is wrong, change this file and note it on the run.
 
@@ -412,5 +520,6 @@ conductor. Do not:
   the judge.
 - `skill:eval-diagnose`: component, owner, issue events. No edit.
 - `skill:eval-improve`: smallest model edit, probe receipts, no self-accept.
+- `skill:eval-report`: the run package and the write-up a person reads.
 - The `malloy-analysis` skill: what the blind answerer follows. It is installed
   from the `analysis` manifest group, not the `eval` group.

package/skills/eval-loop/reference/measurement.md

@@ -1,4 +1,4 @@
-<!-- Sampling, the flip-count bar, the A/A noise band, and targeted fixes. Read this before quoting any number. -->
+<!-- Sampling, the flip-count bar, the A/A band, reading the disagreement set, and targeted fixes. Read this before quoting any number. -->
 
 # Measurement
 
@@ -33,9 +33,49 @@ measure a baseline and diagnose failures but cannot defend an edit.
 This table was asserted, not measured, and the number it needs is a property of
 your harness and your set -- not of this skill. Measure it with an **A/A run**:
 the same model, same config, same set, twice, compared with
-`scripts/flip_table.py`. Every flip it reports is noise by construction, since
-nothing changed. Record the result with the set, in `CALIBRATION.md`, and cite
-that file when you quote a band.
+`scripts/flip_table.py`. Record the result with the set, in `CALIBRATION.md`,
+and cite that file when you quote a band.
+
+An A/A flip is unattributable to a CHANGE, because there was no change. That
+is not the same as saying it carries no information, and reading it as "noise
+by construction" throws away the most useful evidence in the run. See below.
+
+## Read the disagreement set before you average it
+
+**First move on a flip set: read the flipping cases. Second move: establish
+the band.** Both, in that order. The band alone is half the information.
+
+A case that flips is a case where the agent found two paths and the model did
+not make one of them obviously right. That is a MODEL-quality signal, and
+often a cheaper one than a stable failure, because two runs hand you a matched
+pair: same question, same model, one right answer and one wrong one, with the
+diff between them isolating the cause.
+
+Measured, on a real set. Two arms of one model over 28 cases disagreed on
+five. Read as a floor, that is "this set has 18% churn, plan around it". Read
+as evidence, it was a single named model defect:
+
+- all five were the same question -- does a blank field on a fee rule match
+  every value?
+- the two arms produced two discrete query idioms, not scattered answers
+- the wrong idiom returned 2 fee IDs where the answer has 443, and 1 where it
+  has 390
+- the two runs swapped: run 1 was right on one case and wrong on the other,
+  run 2 the reverse
+
+A model that documents a rule in prose while leaving an inviting wrong path
+reachable produces exactly this: a bimodal coin flip that averages to "noise".
+The variance WAS the finding. Averaging the five would have hidden it, and
+reading them took a day of manual transcript work that the table now does --
+`flip_table.py` prints the two final queries side by side for every flipped
+case, and `diagnose.py --compare-run <other arm>` hands the pair to the
+diagnosing agent.
+
+Some flips really are noise, and a set with no band cannot tell an improvement
+from a wobble, so keep the band. What changes is which you do first, and what
+you call the thing before you have looked at it. Prefer **disagreement set**
+or **unstable cases** to "churn": the first two name a question, the last
+names an answer you have not earned yet.
 
 Re-measure whenever the model, judge, or set changes. This is not a formality:
 observed bands have moved by a factor of three across a fortnight of ordinary

package/skills/eval-loop/reference/running-a-run.md

@@ -30,7 +30,11 @@ python3 skills/eval-loop/scripts/serve.py --publisher-dir <publisher>/packages/s
 #     every run. Pass --model on a platform target too, from a checkout of the
 #     served version, since the harness cannot see the model text there.
 #     Exits 3 when set.json names no truthPackage: the audits ran, no
-#     golden was re-derived, and 0 would have claimed otherwise.
+#     golden was re-derived, and 0 would have claimed otherwise. With
+#     --definitions <ledger> (verify_definitions.py), a set whose every
+#     value-bearing case rests on validated definitions exits 0 instead:
+#     the composition rule, values not re-derived but their definitions
+#     checked.
 python3 skills/eval-answer/scripts/verify_goldens.py \
   --set <repo>/evals/ecommerce --publisher http://localhost:4881 \
   --model <repo>/ecommerce --target-package ecommerce

package/skills/eval-loop/scripts/agent_harness.py

@@ -50,10 +50,15 @@ import re
 import shutil
 import subprocess
 import tempfile
+import sys
 import time
 from dataclasses import dataclass, field
 from typing import Any, Iterable
 
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent.parent
+                       / "eval-answer" / "scripts"))
+from json_scan import json_objects  # noqa: E402
+
 SKILL_REF = re.compile(r"skill:([a-z0-9][a-z0-9-]*)")
 
 # Never available to a spawned agent, whatever its role. The network tools
@@ -81,6 +86,10 @@ ALWAYS_BLOCKED = (
     # does); seen 2026-09-01 on the first platform smoke, where they alone
     # flagged the attempt contaminated.
     "ListMcpResourcesTool", "ReadMcpResourceDirTool", "ReadMcpResourceTool",
+    # The same shape, one host surface later: granted when the runner is
+    # launched detached, and they alone flagged every attempt in a 37-case run.
+    # `Task` was blocked; the newer task-management tools were not.
+    "TaskCreate", "TaskGet", "TaskList", "TaskUpdate",
 )
 
 # The role-dependent half, and the reason it cannot join ALWAYS_BLOCKED:
@@ -370,26 +379,18 @@ def build_workspace(skills: Iterable[str],
 
 
 def last_json_object(text: str) -> dict[str, Any] | None:
-    """The last balanced {...} in the text; agents narrate before the payload."""
-    depth, start, best = 0, None, None
-    for i, ch in enumerate(text):
-        if ch == "{":
-            if depth == 0:
-                start = i
-            depth += 1
-        elif ch == "}":
-            if depth == 0:
-                continue
-            depth -= 1
-            if depth == 0 and start is not None:
-                best = text[start:i + 1]
-    if best is None:
-        return None
-    try:
-        obj = json.loads(best)
-    except json.JSONDecodeError:
-        return None
-    return obj if isinstance(obj, dict) else None
+    """The last JSON object in the text; agents narrate before the payload.
+
+    Delegates to `json_scan`, the one scanner in this harness, rather than
+    counting braces here. Depth counting cannot see that a brace is inside a
+    string or a comment, so one stray `{` in a quoted Malloy snippet -- which
+    is what `eval-diagnose` asks an agent to read and reason over -- left the
+    depth permanently above zero and made a perfectly good trailing verdict
+    read as no object at all. The diagnosis then came back `unparseable` and
+    the case left the denominator.
+    """
+    objs = json_objects(text)
+    return objs[-1] if objs else None
 
 
 def spawn_agent(prompt: str, *, skills: Iterable[str],

package/skills/eval-loop/scripts/agent_harness_test.py

@@ -13,6 +13,7 @@ import inspect
 import pathlib
 import sys
 import unittest
+import urllib.error
 from unittest import mock
 
 HERE = pathlib.Path(__file__).resolve().parent
@@ -362,6 +363,84 @@ class TheRetrievalGateIsWired(unittest.TestCase):
             self.assertEqual(rb.run_retrieval_gate(self.ns(rejudge=True)),
                              "not run (no answering phase)")
 
+    def test_a_401_is_named_not_retried(self):
+        # The probe carries no credentials and no CLI login reaches it, so
+        # waiting buys another 401. Retrying one spends twelve tries at ten
+        # seconds each to arrive at "retrieval is not ready", which blames the
+        # retriever for an auth failure.
+        def denied(_a):
+            raise rb.AuthRequired(401, "https://hosted/mcp")
+        with mock.patch.object(rb, "wait_retrieval_ready", denied):
+            note = rb.run_retrieval_gate(self.ns())
+        self.assertIn("not run", note)
+        self.assertIn("401", note)
+        self.assertIn("NOT confirmed", note)
+
+    def test_a_401_does_not_abort_the_run(self):
+        # It is the gate that cannot run, not the arm that must not.
+        def denied(_a):
+            raise rb.AuthRequired(403, "https://hosted/mcp")
+        with mock.patch.object(rb, "wait_retrieval_ready", denied):
+            rb.run_retrieval_gate(self.ns())      # no SystemExit
+
+    def test_the_probes_reply_is_a_confirmation_and_says_it_is_only_one(self):
+        # The reachability probe's call went through the CLI, so it WAS
+        # authenticated. A local run gets two confirmations; this gets one, and
+        # a weaker check must not read back as the same check.
+        def boom(_a):
+            raise AssertionError("made a second, unauthenticated probe")
+        with mock.patch.object(rb, "wait_retrieval_ready", boom):
+            note = rb.run_retrieval_gate(
+                self.ns(probe_payload={"retrieval": "semantic"}))
+        self.assertIn("ready:", note)
+        self.assertIn("1 confirmation", note)
+
+    def test_a_lexical_probe_reply_still_waits(self):
+        # One lexical read is exactly the cold-start case the gate exists for.
+        with mock.patch.object(rb, "wait_retrieval_ready",
+                               lambda _a: (True, "semantic retrieval ready")):
+            note = rb.run_retrieval_gate(
+                self.ns(probe_payload={"retrieval": "lexical"}))
+        self.assertNotIn("1 confirmation", note)
+
+
+class AuthIsNotAColdIndex(unittest.TestCase):
+    """`retrieval_probe` separates "refused me" from "not warm yet"."""
+
+    def probe(self, code):
+        a = argparse.Namespace(mcp_url="http://x/mcp", environment="e",
+                               package="p")
+        err = urllib.error.HTTPError("http://x/mcp", code, "no", {}, None)
+
+        def boom(*_a, **_kw):
+            raise err
+        with mock.patch.object(rb, "mcp_call", boom):
+            return rb.retrieval_probe(a)
+
+    def test_401_and_403_are_auth(self):
+        for code in (401, 403):
+            with self.subTest(code=code):
+                with self.assertRaises(rb.AuthRequired) as e:
+                    self.probe(code)
+                self.assertEqual(e.exception.code, code)
+
+    def test_another_http_error_is_left_alone(self):
+        # A 503 IS worth retrying, so it must not take the auth path.
+        with self.assertRaises(urllib.error.HTTPError):
+            self.probe(503)
+
+    def test_waiting_does_not_swallow_it(self):
+        # A bare `except Exception` in `wait_retrieval_ready` would catch it
+        # and record "probe failed", which reads as a warming index.
+        def denied(_a):
+            raise rb.AuthRequired(401, "http://x/mcp")
+        a = argparse.Namespace(mcp_url="http://x/mcp", environment="e",
+                               package="p")
+        with mock.patch.object(rb, "retrieval_probe", denied), \
+             mock.patch.object(rb.time, "sleep", lambda _s: None):
+            with self.assertRaises(rb.AuthRequired):
+                rb.wait_retrieval_ready(a, tries=12, pause=0)
+
 
 class ReadOnlyRoles(unittest.TestCase):
     def test_diagnose_and_cluster_block_the_shell_too(self):
@@ -374,5 +453,62 @@ class ReadOnlyRoles(unittest.TestCase):
             self.assertIn(tool, diagnose.READ_ONLY)
 
 
+
+class TaskToolsAreBlocked(unittest.TestCase):
+    """The newer task-management surface joins the blocklist.
+
+    `--allowedTools` grants permission without restricting availability, so the
+    denylist is the only lever. `Task` was on it; TaskCreate/Get/List/Update
+    were not, and alone flagged every attempt in a 37-case run.
+    """
+
+    def test_the_task_management_tools_are_always_blocked(self):
+        for name in ("Task", "TaskCreate", "TaskGet", "TaskList", "TaskUpdate",
+                     "TaskOutput", "TaskStop"):
+            self.assertIn(name, ah.ALWAYS_BLOCKED, name)
+
+
+class VerdictScanner(unittest.TestCase):
+    """`last_json_object` is how a diagnosis reaches `diagnose.py`.
+
+    It counted braces by hand, which cannot see that a brace sits inside a
+    string or a comment. `eval-diagnose` asks its agent to READ MALLOY SOURCE
+    and reason over it before emitting the verdict last, so one stray `{` in a
+    quoted snippet left the depth counter above zero for the rest of the reply
+    and a perfectly good trailing verdict read as no object at all. The case
+    then came back `unparseable` and left the denominator. It routes through
+    `json_scan` now, the one scanner in this harness.
+    """
+
+    REPLY = (
+        'Looking at the model:\n\n```malloy\n'
+        'source: orders is duckdb.table(\'o\') extend {\n'
+        '  # a brace in a comment: "{"\n'
+        '  measure: total is sum(amount)\n'
+        '}\n```\n\n'
+        'Verdict:\n{"code": "NOT-RETURNED", "owner": "model"}\n')
+
+    def test_a_brace_in_a_quoted_snippet_does_not_eat_the_verdict(self):
+        self.assertEqual(ah.last_json_object(self.REPLY),
+                         {"code": "NOT-RETURNED", "owner": "model"})
+
+    def test_it_takes_the_LAST_object_because_the_verdict_comes_last(self):
+        text = '{"draft": 1} thinking... {"final": 2}'
+        self.assertEqual(ah.last_json_object(text), {"final": 2})
+
+    def test_prose_with_no_object_is_none(self):
+        self.assertIsNone(ah.last_json_object("no verdict here {not json"))
+
+    def test_a_json_array_is_not_a_verdict(self):
+        self.assertIsNone(ah.last_json_object("[1, 2, 3]"))
+
+    def test_it_is_the_shared_scanner_not_a_second_copy(self):
+        # The drift this harness spent several commits removing. If a local
+        # brace-counter comes back, this is what says so.
+        import json_scan
+        self.assertEqual(ah.last_json_object(self.REPLY),
+                         json_scan.json_objects(self.REPLY)[-1])
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=1)

package/skills/eval-loop/scripts/build_run_package.py

@@ -37,7 +37,7 @@ TEMPLATE = pathlib.Path(__file__).resolve().parent.parent / "templates" / "eval-
 sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent.parent
                        / "eval-answer" / "scripts"))
 from score_retrieval import delivery, groups, score_case  # noqa: E402
-from flip_table import outcome  # noqa: E402  (same directory)
+from flip_table import counts_toward_score, outcome  # noqa: E402  (same directory)
 
 
 def read_jsonl(path: pathlib.Path) -> list[dict[str, Any]]:
@@ -364,6 +364,11 @@ def build(run_dirs: list[pathlib.Path], set_dir: pathlib.Path,
                            # uses, so the package cannot disagree with the
                            # flip table about what a flip is.
                            "outcome": outcome(s.get("verdict")),
+                           # Whether this score belongs in the aggregates, by
+                           # the same function run_baseline.py uses before it
+                           # prints one. Written as a column so the measures
+                           # read the rule rather than restating it.
+                           "counts": counts_toward_score(s.get("gold_status")),
                            **s})
 
             mine = tool_calls.get(kk, [])
@@ -474,7 +479,8 @@ def build(run_dirs: list[pathlib.Path], set_dir: pathlib.Path,
         "answer_text", "n_get_context", "n_execute", "n_execute_errors",
         "host_tool_uses", "mcp_tool_uses", "reported_calls", "contaminated",
         "final_query_source", "servedRevision",
-        "input_tokens", "output_tokens", "cache_read_tokens", "cost_usd",
+        "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens",
+        "cost_usd",
         "num_turns", "wall_seconds", "run_error", "transcriptPath",
         "n_steps", "prediction"])
     write_csv(data / "steps.csv", steps_rows, [
@@ -484,7 +490,7 @@ def build(run_dirs: list[pathlib.Path], set_dir: pathlib.Path,
         "attempt_key", "run_id", "qid", "entity_id", "status"])
     write_csv(data / "scores.csv", scores, [
         "attempt_key", "run_id", "qid", "sample", "phase", "verdict", "outcome",
-        "reason", "confidence",
+        "counts", "reason", "confidence",
         "judge_version", "rubric_sha", "golden_revision", "gold_status",
         "contaminated", "judge_verdict", "must_not_use_hits", "artifactPath"])
     write_csv(data / "retrieval.csv", retr, [