@malloy-publisher/server 0.8.5 → 0.9.0

package/dist/server.mjs

@@ -206074,13 +206074,69 @@ var init_path_safety = __esm(() => {
 
 // src/pg_helpers.ts
 function redactPgSecrets(s2) {
-  return s2.replace(/([a-z][a-z0-9+.-]*:\/\/[^:/?#\s]*):([^/\s]+)@/gi, "$1:***@").replace(/((?:postgres|postgresql):\/\/[^:/?#\s]*):([^@\s]+)@/gi, "$1:***@").replace(/password=('(?:\\.|[^'\\])*'|"[^"]*"|\S+)/gi, "password=***");
+  return redactUserinfoPasswords(redactUserinfoPasswords(s2, ANY_SCHEME_LAST_AT), PG_SCHEME_FIRST_AT).replace(/password=('(?:\\.|[^'\\])*'|"[^"]*"|\S+)/gi, "password=***");
+}
+function runEnd(run, s2, from) {
+  run.lastIndex = from;
+  run.exec(s2);
+  return run.lastIndex;
+}
+function redactUserinfoPasswords(s2, pass) {
+  let out = "";
+  let copied = 0;
+  let stop = -1;
+  let sep2 = s2.indexOf("://");
+  while (sep2 !== -1) {
+    const userEnd = runEnd(USERNAME_RUN, s2, sep2 + 3);
+    let at = -1;
+    if (s2[userEnd] === ":" && pass.hasScheme(s2, sep2, copied)) {
+      const from = userEnd + 1;
+      if (stop < from)
+        stop = runEnd(pass.passwordRun, s2, from);
+      at = pass.passwordEnd(s2, from, stop);
+    }
+    if (at === -1) {
+      sep2 = s2.indexOf("://", sep2 + 1);
+      continue;
+    }
+    out += s2.slice(copied, userEnd + 1) + "***@";
+    copied = at + 1;
+    sep2 = s2.indexOf("://", copied);
+  }
+  return out + s2.slice(copied);
 }
 function redactConnectionSecretShapes(s2) {
   return redactPgSecrets(s2.replace(PEM_TERMINATED_PATTERN, "***").replace(PEM_UNTERMINATED_PATTERN, "***").replace(SECRET_QUOTED_PATTERN, "$1***").replace(SECRET_BARE_PATTERN, "$1***"));
 }
-var SECRET_FIELD_NAMES, NAME_ALTERNATION, SECRET_QUOTED_PATTERN, SECRET_BARE_PATTERN, PEM_TERMINATED_PATTERN, PEM_UNTERMINATED_PATTERN;
+var USERNAME_RUN, SCHEME_CHAR, SCHEME_LETTER, PG_SCHEMES, ANY_SCHEME_LAST_AT, PG_SCHEME_FIRST_AT, SECRET_FIELD_NAMES, NAME_ALTERNATION, SECRET_QUOTED_PATTERN, SECRET_BARE_PATTERN, PEM_TERMINATED_PATTERN, PEM_UNTERMINATED_PATTERN;
 var init_pg_helpers = __esm(() => {
+  USERNAME_RUN = /[^:/?#\s]*/y;
+  SCHEME_CHAR = /[a-z0-9+.-]/i;
+  SCHEME_LETTER = /[a-z]/i;
+  PG_SCHEMES = ["postgres", "postgresql"];
+  ANY_SCHEME_LAST_AT = {
+    hasScheme(s2, sep2, floor) {
+      for (let i2 = sep2 - 1;i2 >= floor && SCHEME_CHAR.test(s2[i2]); i2--) {
+        if (SCHEME_LETTER.test(s2[i2]))
+          return true;
+      }
+      return false;
+    },
+    passwordRun: /[^/\s]*/y,
+    passwordEnd(s2, from, stop) {
+      const at = s2.slice(from, stop).lastIndexOf("@");
+      return at > 0 ? from + at : -1;
+    }
+  };
+  PG_SCHEME_FIRST_AT = {
+    hasScheme(s2, sep2, floor) {
+      return PG_SCHEMES.some((scheme) => sep2 - scheme.length >= floor && s2.slice(sep2 - scheme.length, sep2).toLowerCase() === scheme);
+    },
+    passwordRun: /[^@\s]*/y,
+    passwordEnd(s2, from, stop) {
+      return stop > from && s2[stop] === "@" ? stop : -1;
+    }
+  };
   SECRET_FIELD_NAMES = [
     "password",
     "privateKey",
@@ -308322,72 +308378,21 @@ function onMotlyRoute(text) {
 function hasEnvReference(annotation) {
   return annotation.includes("@env.");
 }
-function pollutionTargets() {
-  const targets = [Object.prototype];
-  for (const key of Object.getOwnPropertyNames(Object.prototype)) {
-    const descriptor = Object.getOwnPropertyDescriptor(Object.prototype, key);
-    if (!descriptor)
-      continue;
-    let value;
-    if ("value" in descriptor) {
-      value = descriptor.value;
-    } else {
-      if (BUILT_IN_PROTO_GETTER === undefined || descriptor.get !== BUILT_IN_PROTO_GETTER) {
-        return;
-      }
-      continue;
-    }
-    if ((typeof value === "object" || typeof value === "function") && value !== null && !targets.includes(value)) {
-      targets.push(value);
-    }
-  }
-  return targets;
-}
-function parseGuarded(texts) {
-  const targets = pollutionTargets();
-  if (targets === undefined) {
-    return { tag: undefined, messages: [UNSAFE_TO_PARSE] };
-  }
-  let unreadable = false;
-  const ownNames = (target) => {
-    try {
-      return Object.getOwnPropertyNames(target);
-    } catch {
-      unreadable = true;
-      return [];
-    }
-  };
-  const before = targets.map(ownNames);
-  if (unreadable)
-    return { tag: undefined, messages: [UNSAFE_TO_PARSE] };
-  const undoPollution = () => {
-    let polluted = false;
-    targets.forEach((target, index) => {
-      for (const key of ownNames(target)) {
-        if (before[index].includes(key))
-          continue;
-        polluted = true;
-        try {
-          delete target[key];
-        } catch {}
-      }
-    });
-    return polluted || unreadable;
-  };
+function parseBounded(texts) {
   if (texts.some((text) => text.length > MAX_ANNOTATION_CHARS)) {
     return { tag: undefined, messages: [ANNOTATION_TOO_LONG] };
   }
   try {
     const result = parseAnnotation([...texts]);
-    if (undoPollution())
-      return { tag: undefined, messages: [UNSAFE_TO_PARSE] };
     return {
       tag: result.tag,
       messages: result.log.map((error2) => error2.message)
     };
-  } catch {
-    undoPollution();
-    return { tag: undefined, messages: [UNSAFE_TO_PARSE] };
+  } catch (error2) {
+    return {
+      tag: undefined,
+      messages: [error2 instanceof Error ? error2.message : String(error2)]
+    };
   }
 }
 function docCommentText(texts) {
@@ -308519,16 +308524,16 @@ function parseMotly(texts) {
   const envErrors = envDropped > 0 ? [ENV_REFERENCE_DROPPED] : [];
   if (motly.length === 0)
     return { tag: undefined, errors: envErrors };
-  const direct = parseGuarded(motly);
+  const direct = parseBounded(motly);
   if (direct.messages.length === 0)
     return { tag: direct.tag, errors: envErrors };
   const rescued = motly.map((text) => {
-    if (parseGuarded([text]).messages.length === 0)
+    if (parseBounded([text]).messages.length === 0)
       return text;
     const rewritten = quoteFilterLiterals(text);
-    return parseGuarded([rewritten]).messages.length === 0 ? rewritten : text;
+    return parseBounded([rewritten]).messages.length === 0 ? rewritten : text;
   });
-  const after = parseGuarded(rescued);
+  const after = parseBounded(rescued);
   return { tag: after.tag, errors: [...envErrors, ...after.messages] };
 }
 function tagText(tag, ...path7) {
@@ -308594,9 +308599,8 @@ function readStartingGivens(tag, declaredType) {
   }
   return Object.keys(collected).length > 0 ? collected : undefined;
 }
-var BUILT_IN_PROTO_GETTER, MAX_ANNOTATION_CHARS = 8192, ANNOTATION_TOO_LONG, ENV_REFERENCE_DROPPED = "annotation dropped for carrying an @env. reference", UNSAFE_TO_PARSE = "annotation could not be parsed safely";
+var MAX_ANNOTATION_CHARS = 8192, ANNOTATION_TOO_LONG, ENV_REFERENCE_DROPPED = "annotation dropped for carrying an @env. reference";
 var init_motly = __esm(() => {
-  BUILT_IN_PROTO_GETTER = Object.getOwnPropertyDescriptor(Object.prototype, "__proto__")?.get;
   ANNOTATION_TOO_LONG = `annotation exceeds ${MAX_ANNOTATION_CHARS} characters and was not parsed`;
 });
 
@@ -316329,9 +316333,6 @@ function lintDashboard(facts, manifest) {
   return findings;
 }
 function describeParseFailure(message) {
-  if (message === UNSAFE_TO_PARSE) {
-    return `was refused rather than parsed (${message}), which can be caused by something outside this file`;
-  }
   if (message === ENV_REFERENCE_DROPPED) {
     return `was dropped rather than parsed (${message}), so nothing on that line took effect`;
   }
@@ -376723,7 +376724,7 @@ ${source}` : source ?? "";
   }
 }
 // package.json
-var version2 = "0.8.5";
+var version2 = "0.9.0";
 
 // src/version.ts
 var SERVER_VERSION = version2;
@@ -383628,7 +383629,8 @@ otherwise a modelling agent is dispatched to fix a model that is already right.
 
 ## Step 6: Append events, then stop
 
-Append to \`evals/<set>/runs/<runId>/events.jsonl\` with \`caseId\` set. Shapes
+Append to \`<workdir>/runs/<runId>/events.jsonl\` with \`caseId\` set (the run
+directory \`eval.py run\` printed; the workdir is the set's, from \`eval.toml\`). Shapes
 live in \`reference/ledger-schema.md\`.
 
 1. \`attempt\`: qid, sample, phase, question_sha, submitted, final_query,
@@ -384203,7 +384205,7 @@ a permanent parallel mechanism.`
     {
       name: "eval-answer/ledger-schema",
       description: "The eval ledger: files and events. Reference detail for the eval-answer skill.",
-      body: "# The eval ledger: files and events\n\nThe ledger is plain files in the model package's git repository. There is no\neval API and no eval database. The conductor (`skill:eval-loop`) reads and\nwrites these files directly; the stages share them as their contract.\n`eval-answer` writes `attempt`, `tool_call`, and `score`. An engine-side\n`eval-retrieval` skill, which does not ship here, writes `retrieval_score` and\n`probe`; both stay in this contract so one validator covers every run\ndirectory.\n`eval-diagnose` writes `issue` and `issue_status`. `eval-improve` writes\n`candidate`. `eval-loop` writes `acceptance_check`, further `issue_status`, and\n`checkpoint`.\n\n**The contract is code: `eval-answer/scripts/ledger.py`.** Every script that\nwrites `run.json` or `events.jsonl` builds its lines through that module, so a\nfield rename that reaches only one writer fails at write time. This document\nis the human-readable rendering; where the two disagree, the module is right\nand this file is the bug. Check any run directory with:\n\n```\npython3 skills/eval-answer/scripts/ledger.py validate <runDir>\n```\n\nErrors are broken identity (missing `run.json`, a missing required field, two\nscores for one attempt); warnings are missing comparison pins (a run without\n`skillsVersion` still measures, but cannot anchor an A/B on that pin) and\ngrandfathered unknown fields on old runs.\n\n## Layout\n\n```\nevals/<set>/\n  set.json                  # set metadata (below)\n  cases.jsonl               # one case per line\n  judge-regressions.jsonl   # judge verdicts a human overruled\n  runs/<runId>/\n    run.json                # run config, the attribution pins\n    events.jsonl            # append-only event lines\n    artifacts/              # prediction CSVs, judge outputs, transcripts\n```\n\nThe set directory lives in the SAME git repository as the model it evaluates,\nso a checkpoint (a git commit) pins the model and the ledger together. Never\nplace `evals/` inside the directory tree the answerer's package serves: gold\nin the served tree is a contamination path.\n\nRules that make the ledger trustworthy:\n\n- `events.jsonl` is append-only. Never edit a line. To void one, append a new\n  event whose payload marks the old one `voided`. One sanctioned exception: a\n  re-runnable stage (diagnose, improve) replaces ITS OWN prior events, keyed by\n  issue id, through `ledger.replace_events()` -- so a crash-and-rerun does not\n  duplicate issues. A stage never touches another stage's lines.\n- A stage never rewrites another stage's fields.\n- One artifact directory per attempt (`qid` plus `sample`). Never overwrite a\n  previous attempt's prediction CSV.\n- End-of-run numbers come from counting event lines (`jq` over\n  `events.jsonl`), never from arithmetic recalled in prose.\n\n## `set.json`\n\n| Field | Notes |\n|---|---|\n| `name` | Set name; also the directory name. |\n| `description` | |\n| `datasetVersion` | Integer. Bump on any golden repair or case change. Runs record the version they scored against. |\n| `targetModelPath` | Model path within the package. |\n| `truthPackage` | Name of the package holding the semantics-free sources every golden is re-derived from -- a package NAME, not an object. Without it `verify_goldens.py` re-derives nothing: it reports \"nothing to re-derive against\" and **exits 3**, which a caller must read as unverified rather than as a pass. The checks that need no truth server -- rubric figures, verification axis, stale rubric claims, set names -- still run and still report. `init_truth_package.py` scaffolds the package. |\n| `truthModel` | Model file inside that package. Default `truth.malloy`. |\n| `targetPackage` | Name of the package UNDER TEST. `verify_goldens.py` refuses a \"truth\" server that also serves it, because an answerer on that server can retrieve the raw truth sources beside the model and quietly change what is under test. `run_baseline.py` and `improve.py` pass the package they were given, so this field is the fallback for a direct call; without either the guard cannot fire. |\n| `truthTableRewrite` | Boolean, default false. Rewrites `duckdb.table('data/x.parquet')` refs to bare `x` in canonical queries, for a truth package whose tables are registered rather than read from files. |\n\n## `cases.jsonl`\n\nOne JSON object per line:\n\n| Field | Notes |\n|---|---|\n| `qid` | Stable case id. |\n| `question` | Exact text the answerer will see. |\n| `split` | `dev` or `holdout`. Frozen at import. Diagnose and improve read `dev` only; the acceptance check runs both. |\n| `tags` | list |\n| `state` | `candidate` / `selected` / `excluded`. |\n| `source` | Where the case came from. |\n| `golden` | `status` (`verified` / `provisional` / `invalid` / `ambiguous` / `verified_wrong`, the last being a key a person has established is wrong, which `run_baseline.py` drops from every aggregate), `kind`, `value` or `path` (artifact under the set directory), `canonicalQuery` (runs against the **truth** package, never the model under test), `verifiedBy`, and `verification` -- `{primaryAxis, variesAxis, note}` naming what the second derivation varied. **A golden is written only after two differently shaped derivations agree** (the truth query and a second one that varies a different axis -- group by something else, sum a different way, take a different route through the raw tables); `gold/<qid>.json` holds both (`truthRows`, `verifyRows`, `agreement`). Two derivations that vary the same axis share every other blind spot: that is how a golden that summed three overlapping time slices, 3x too high, passed its check. `verify_goldens.py` re-derives the value, reads the accepting clause of the rubric against the rows, and flags a second derivation that names no axis. **The question asks for data, never for an interpretation of it**, and either fixes the grain or the rubric accepts a correct figure at any stated grain (`skill:eval-judge` rules 10 and 11). |\n| `golden.kind` | What the key IS. A value (`scalar`, rows), `unanswerable`, or `criteria`: prose with no value, where the author's criteria are the whole key. `criteria` is the one kind that is legitimately `verified` on arrival, because there is no derivation to perform and nothing that can be numerically wrong; an imported key holding a VALUE is `provisional` until re-derived (`skill:eval-import`). **`verify_goldens.py --promote` is the only thing that writes `golden.status`**: it marks a `provisional` golden `verified` when its value re-derived cleanly from the truth package AND a second derivation exists. `--refresh` rewrites a drifted VALUE and never changes a status; `invalid` and `ambiguous` are settled by a person through the golden side door. |\n| `golden.rubric` | **On the golden, not on the case.** The prose the judge is shown as `CASE RUBRIC`, naming what counts as correct and what does not. `run_baseline.py` reads `golden.rubric`; a rubric written at the case's top level is silently not passed, the judge is told `CASE RUBRIC: none`, and it then scores from the golden value and the answer alone -- which reads as a judge that ignores its instructions. |\n| `golden.mustState` | **On the golden, not on the case.** What the answer has to say out loud. Read from `golden.mustState` by the run-package builder. |\n| `goldenRevision` | Integer. Bump on every golden change; `score` events stamp the revision they compared against. |\n| `expectedEntities` | `required`: entity ids (`kind:source:name`) the answer cannot be produced without. `requiredAnyOf`: a list of **groups**, each a list of ids of which any one suffices -- for a case the model can answer through more than one route. Naming only one route scores the other as a retrieval miss and steers the fix to the docs for an entity whose docs were never the problem. `acceptable`: ids that are not noise if returned. An entity counts as delivered when returned under its exact id, under the same type and name on a sibling source, or by name in a returned source's documentation; `score_retrieval.py` records the route per entity as `delivery`. |\n\n## Term files (`eval-retrieval` only)\n\nA customer set has no term file. The engine-side `eval-retrieval` skill keeps\n`intents.jsonl` fixtures -- search term, entity type, a description of what the\nsearcher meant, a validity flag -- beside its own scripts, for evaluating the\nengine against fixed inputs. Their shape and the `retrieval_score` events they\nproduce are documented there and validated here.\n\n## `run.json`\n\nThe attribution pins. Two runs are comparable only when these match where it\nmatters. Two runs with different `target` values are comparable on answer\nverdicts only if the same model version reached both, which is rarely worth\nassuming:\n\nRequired fields are the run's **identity** -- every run ever written carries\nthem. The rest are optional in the schema, but the **comparison pins** are\nwarned about when absent, because a run missing one cannot take part in an A/B\non that axis.\n\n| Field | Notes |\n|---|---|\n| `runId` *(required)* | Directory name. |\n| `target` *(required)* | Which server answered, and how it reached the data: `local`, `local-proxied`, or `platform`. Decides what an improve step is even allowed to do (see `skill:eval-loop`). |\n| `answererModel` *(required)* | |\n| `phase` / `started` *(required)* | |\n| `judgeModel` / `judgeVersion` / `rubricSha` *(pins)* | The judge's model, and the version + content sha of `skill:eval-judge`. |\n| `judgePromptSha` *(pin)* | Content sha of the prompt the judge is SHOWN, which is the harness's own template and not the skill file `rubricSha` covers. The template carries the golden rendering, the evidence blocks and the tie-breaking rules, so two runs whose prompts differed compared as the same judge -- the blind spot that let a golden-rendering defect run through three arms with every test passing, because the tests covered the code paths and not the text the model sees. Absent on runs written before it existed. |\n| `datasetVersion` *(pin)* | From `set.json` at run time. |\n| `modelSha` *(pin)* | Content sha of the `model.malloy` snapshot in the run directory -- the bytes the answerer actually queried. A git sha is not enough: a snapshot host serves a copy, often of a dirty tree no commit names. |\n| `skillsVersion` *(pin)* | HEAD of the checkout the agents' skills were loaded from, dirty-marked (`ledger.skills_git_sha(root)`). The skills are the doctrine the agents load; a run that cannot name their version cannot anchor a skills A/B. |\n| `skillsRoot` / `harnessVersion` | Which checkout supplied the doctrine (`--skills-root`, e.g. a Publisher checkout for the open-source skills; default this one), and this checkout's own HEAD. The eval-* skills always come from the harness checkout, whatever `skillsRoot` says. Two runs whose `skillsRoot` differ are a skills A/B only if their manifests name the same skills. |\n| `diagnoserModel` / `improverModel` | Written by `diagnose.py` / `improve.py` when those stages run, so the run names every LLM that touched it. Absent on a run that was only answered and judged. |\n| `modelGitSha` | Commit of the model repo, from `--model-repo`; `-dirty` suffix when `--model-dir` (or the whole repo, without it) had uncommitted changes -- fine for a band measurement, not for an A/B pin. Null when git cannot answer for the scope (a path outside the repo): absent beats wrong. **Null unless `--model-repo` names the repo.** It cannot be inferred: Publisher serves a COPY under `publisher_data/`, so the tree around the served file is the server's storage, not the model's history, and the field named that until 2026-09-03. `modelSha` above is the pin that always works; this is the pointer to where those bytes are versioned, and absent beats wrong. |\n| `modelRepo` | The path `--model-repo` named, so a reader can tell an absent pin from an unrecorded one. |\n| `modelDir` | The package directory `--model-dir` named, when it was: the path the `-dirty` marker on `modelGitSha` was decided over. Absent means the whole repo, where any unrelated untracked file marks the model dirty. |\n| `environment` / `package` / `modelPath` | What was served, and from where. On a platform target `environment` is the organization and `package` is the workspace the MCP URL is scoped to. |\n| `scope` | Platform target only: the `environment/package[@version]` the answerer was told to pass as an explicit `scopes` entry on every `get_context` / `execute_query` call. A workspace can serve many packages (a personal workspace serves every package the user can read), so without this the run also measures whether retrieval picks the right package -- a different measurement. Absent means unscoped. |\n| `targetVersion` *(pin)* | Platform target only: the published version the answers came from. Both hosted tools default an omitted version to the PINNED one, meaning whatever the workspace serves at the moment of the call, so this only describes the run when `--scope` carries it to every call. `--target-version` supplies it when the scope omits it; a scope naming a different version is refused rather than treated as an override, because a run cannot say which build answered if two pins disagree. |\n| `mcpUrl` / `publisher` | The endpoints the answerer and the re-execution used. |\n| `predictionsReExecuted` | Whether re-execution was POSSIBLE: the server was serving the bytes this run is pinned to. It is one bool for the whole run and says nothing about any given query, so it reads stronger than it is; `reExecution` below is what actually happened. |\n| `reExecution` | `{attempted, ok, failed, noQuery, notReExecuted, notJudged, missing}` over the run's predictions. `failed` means the query ran and the server returned an error, which is a different thing from an answer with no query at all. `notJudged` is a case the judge was never handed: a golden refused before judging (`provisional`, `invalid`, `ambiguous`, `verified_wrong`, missing), an attempt that submitted nothing, or every case under `--no-judge`. Those used to land in `missing` and read as corrupt artifacts on a freshly imported set. `ok`, `failed`, `noQuery`, `notReExecuted`, `notJudged` and `missing` partition the cases, so they sum to the case count; `attempted` is `ok + failed` and sits outside that sum. |\n| `retrievalMode` | Which retriever answered: `semantic`, `lexical`, `mixed` (it changed mid-run, so nothing can be averaged across it) or `unreported` (nothing ranked, which on a configured server means no embedding provider). Only a RANKING call names a retriever, so a call that enumerates a type or looks a name up is `unreported` on a fully semantic server and must not make the run read as `mixed`. Local retrieval degrades to lexical SILENTLY without an embedding key, and a pair compared across that reads as a model change, so `flip_table.py` refuses the pair unless both sides match. |\n| `retrievalCalls` | `{semantic, lexical, unreported}` call counts behind `retrievalMode`. `unreported` counts the calls that did not rank, which is a different thing from calls that fell back. |\n| `retrievalGate` | What the pre-arm retrieval gate found, or why it did not run. A run is held until `get_context` answers semantic twice in a row, because the first calls after a restart are lexical by design and an arm started immediately measures two retrievers. `not run (no answering phase)` under `--rebuild`/`--rejudge`, and `skipped by --no-retrieval-gate` when the operator opted out -- which is a different fact from a gate that passed, and the field is what tells them apart. |\n| `coverageReport` | The `check_coverage.py --out` report this run read, when `--coverage` named one: its path, the model version it was stamped with, its `agentModel`, and `decided` of `cases`. Coverage is a read of the MODEL, so it is measured separately and costs a judge call per case; the run does not run it, it consumes it. Its per-case verdict then beats the authored `coverage` label in retrieval attribution, and each `score_retrieval` row records `coverage_source` (`measured`, `authored`, `none`) so nobody has to guess which one charged the failure. Absent when no report was given, which is a different fact from a report that found everything covered. |\n| `label` | `<set>-<phase>-<nn>`, assigned by `run_baseline.py` from the set name, the run's phase and the next free number beside it (`ecommerce-baseline-01`, `ecommerce-baseline-02`, `ecommerce-blind_gate-01`). The A/A pair is two runs of the same phase; the post-edit arms are two runs of `blind_gate`. Hand-typed names do not survive one afternoon of runs -- `base`, `rejudged2`, `r3`, `post1` sort wrongly, group not at all, and cannot be matched to an arm. `--label` overrides for a run that genuinely needs a human name. |\n| `effort` | |\n| `maxTurns` / `answererTimeout` *(pins)* | The answerer's turn cap and per-attempt timeout, as the arm ran them. `skill:eval-loop` step 6 lists the call budget among the pins to freeze for a whole run, and nothing wrote one, so no two runs could be compared on the setting that decides whether a case got to finish at all. Distinct from `callBudget`, which no harness has ever written. Size the cap from a pilot rather than taking the default: on a 29-case arm at the default 30, completed attempts had a median of 17 turns and a 90th percentile of 26, and the four cases that ran past it were recorded as wrong answers. |\n| `truncated` / `contaminated` | The qids excluded for a HARNESS-owned reason: the cap cut the answer off, or the attempt breached isolation. Both leave the denominator, both are named here rather than only in the console, and either one makes `status` `incomplete`. |\n| `answererCostUsd` / `judgeCostUsd` | What the arm cost, split by role. The judge's half was discarded until 2026-09-02, so every \"cost per arm\" quoted before then was the answerer alone. Per-case judge cost is on each `score` event. |\n| `answererCostCopiedFrom` | The run whose attempts `answererCostUsd` was COPIED from, when this run spawned no answerer -- a `--rejudge`, a `--rebuild`, a `--from`. Those copy the source's attempt events verbatim, `cost_usd` included, so the figure is right about the attempts and wrong as this run's spend: one re-judge reported $17.56 of answering that never happened. Null when this run paid, so a total over several run files can skip the copies. |\n| `goldenCheck` | What `verify_goldens.py` said before the run started: `N ok, M drifted, K other finding(s)`, or why it did not run (no truth server on a platform target; `--skip-golden-check`; rebuild). A run that started on a drifted set says so here rather than pretending its verdicts mean something. `run_baseline` calls `verify()` without a model, so the set-name audit (check 5) does not run in-arm; the trailing `N stale entity name(s)` is the in-arm lint's own count, and `entity-name lint not run` says there was no model text to lint against. Without it the field read `0 other finding(s)` for a check that never happened. |\n| `staleEntityNames` | The `expectedEntities` names that appear nowhere in the served model, from the in-arm lint. Each one depresses the run's own recall on every case listing it, so a stale set reads as a retrieval failure until someone checks by hand. `null` when there was no model text to lint against (a platform target), which is a different fact from `[]`. |\n| `status` | `complete`; `aborted` when four consecutive attempts errored or found the server dead and the harness stopped rather than spend the rest of the budget on attempts nobody will trust; or `incomplete` when the arm ran to the end and still cannot report a rate, because `truncated` or `contaminated` is non-empty. `incomplete` is a third state, not a soft `complete`: the arm did everything asked of it and the number is still unearned. It does NOT stop the run being compared: `flip_table.py` scores the cases both arms did, and names the ones this arm left unscored -- an excluded case carries no verdict, so it can never be counted as a flip. |\n| `packageSha` / `servedRevision` *(pins)* | Taken from the server, not recomputed: `sourceContentSha` is a content hash over EVERY model path in the package, so an edit to an imported file moves it where a sha of the one `--model-path` does not; `servedRevision` is minted per load, so it identifies a load rather than content and is a poor pin alone. Measured: `publisher.json`'s `version` moves neither, and nothing in Publisher reads it -- it is not in the Package API schema and never returned, so it pins nothing. |\n| `datasetSha` *(pin)* | Content hash of `set.json` + `cases.jsonl`. Deliberately SEPARATE from the model's sha: a golden repair is not a model change, and one pin covering both would make every answer-key fix read as an edit to the model, which is the distinction an A/B rests on. Automatic, so nobody has to remember it; `datasetVersion` stays beside it as the human-readable sequence. **Local targets only.** A hosted target that publishes IMMUTABLE versions needs none of this: the set rides inside the version, and immutability -- not hashing -- is what makes a pin trustworthy. There, `targetVersion` alone identifies model and set together. |\n| `doubtedGoldens` | The cases whose golden is not believed: `qid`, `gold_status` (`suspect` or `verified_wrong`), `gold_note`, and `declaredBy` (`judge` or `set`). A status the SET declared is a key somebody already settled, not an opinion the judge formed this run, and attributing both to the judge claimed a judgement that never happened. **Read this before diagnose.** These are dataset issues, not model failures, and they go through the golden side door in `skill:eval-loop`. Empty list when no key is in doubt from either source. Written from the same scan that prints the end-of-run warning, because a warning that lives only in console text is one scrollback away from sending a modelling agent at a model that is already right. |\n| `mode` / `setName` / `serverVersion` / `traceMode` / `callBudget` | Defined and accepted, **not yet written by any harness** -- kept in the schema for the platform target and the conductor, which need them. (`targetVersion` and `status` were in this list and are both written now.) |\n\n## Events\n\nEvery line in `events.jsonl` is **flat**: `{ \"kind\": ..., <fields> }`, one\nJSON object per line, with an optional `at` ISO timestamp. Case-scoped kinds\n(`attempt`, `tool_call`, `score`) carry `qid`, `sample`, `phase` on the line;\nrun-level kinds do not. (An earlier draft of this document nested fields under\na `payload` with a `caseId`; no writer ever did that, and 23 run directories\nexist in the flat shape, so the flat shape is the contract.) `kind` is one of:\n`attempt`, `tool_call`, `score`, `retrieval_score`, `issue`, `issue_status`,\n`candidate`, `acceptance_check`, `checkpoint`.\n\n### `attempt`\n\n| Field | Type | Notes |\n|---|---|---|\n| `qid` | string | |\n| `sample` | int or null | Which repeat. Required even when null. |\n| `phase` | string | `baseline` / `loop` / `blind_gate` / `canary` / `final`. `phase` lives here, on the attempt, not in run config. |\n| `question_sha` | string | Hash of the exact text the answerer saw, so two runs can be compared on whether they asked the same thing. A case may carry `questionSha`, the seal stamped at conversion, and that wins when present; the ecommerce set writes a 16-char prefix of it, and a set without one left this null. Neither form proves the question was not edited, since both follow the case as it now reads: that comparison is check 6 of `verify_goldens.py`. |\n| `submitted` | bool | False when there was no final query. Not a wrong answer. |\n| `final_query` | string or null | Required to replay. A named view is recorded as the Malloy it stands for, `run: <source> -> <view>`, so every consumer sees one shape and the query re-executes. |\n| `final_query_source` | string or null | How `final_query` was chosen: `declared` (the answer printed it), `last_ok` (the last call the server answered) or `last`. `last` is a warning: a trailing sanity probe may be standing in for the answer's own query. |\n| `servedRevision` | string or null | From the package actually queried. |\n| `n_get_context` / `n_execute` / `n_execute_errors` | int | |\n| `host_tool_uses` | int | EVERY tool use the host logged, MCP calls included. It counted only the non-MCP ones until 2026-09-03, which made the under-report check below true of almost every clean attempt. |\n| `mcp_tool_uses` | int or null | The MCP subset: `get_context` plus `execute_query`. Null on a run written before the split. |\n| `reported_calls` | int | MCP calls the answerer claimed. `reported_calls > host_tool_uses` is the under-report floor from `skill:eval-answer`: the answerer cannot have made more calls than the host logged. `validate_run` warns on it, and only for runs that carry `mcp_tool_uses`, because the field meant something narrower before. |\n| `contaminated` | bool or `\"unknown\"` | `\"unknown\"` when no host log exists. Read it with `ledger.is_contaminated`, never for truthiness: runs written before 2026-09-03 carry the strings `\"true\"`/`\"false\"`, and `bool(\"false\")` is True. `ledger.event` now rejects those strings on write; `validate_run` grandfathers them on read as a warning. |\n| `contamination_reasons` | list | Empty when clean. |\n| `input_tokens` / `output_tokens` / `cache_read_tokens` / `cache_write_tokens` | int or null | Answerer token usage, the four counts that reprice a run from its own ledger. Null when the host does not report it. `cost_usd` beside them comes from the CLI's `total_cost_usd`, which prices cache reads and writes at their rates, so it was always cache-inclusive; what was missing was the breakdown that reproduces it. `cache_write_tokens` (`cache_creation_input_tokens` upstream) was never captured before 2026-09-15, and on one analysed run cache writes were 44% of the agent's cost, the single largest line. **Read `input_tokens` with care:** it counts only the tokens after the last cache breakpoint. One run recorded 224 for 24 questions beside 2.4M cache reads; the 224 is not the context volume, and a token total that omits the cache columns is not a size. |\n| `cost_usd` | float or null | Answerer cost for this attempt. |\n| `num_turns` / `wall_seconds` | int, float or null | |\n| `answer_text` | string or null | The answer the judge scored. Kept so a verdict can be re-read without the transcript. |\n| `skills_invoked` | list of string | The skills the answerer invoked through the `Skill` tool, deduplicated and sorted, counted from its own transcript. Always a list on a run that recorded it, so an EMPTY list is the finding: the arm granted a manifest and the agent reached for none of it. Absent on runs written before 2026-09-15, which is why every reader takes `or []`. Only explicit `Skill` calls are counted, and only skills the arm granted -- one outside `--answerer-skills` is a contamination breach and lands in `contamination_reasons` instead. What it is for: a manifest names what an arm GRANTED, and comparing two arms' doctrine is meaningless if the doctrine never loaded. |\n| `transcriptPath` | string | The answerer's transcript under `artifacts/`. |\n\nToken counts sit here rather than being derived later because the claim a\nsemantic model makes is about cost as well as correctness -- that a documented\nmodel reaches a good answer in fewer turns and tokens than working from raw\nschema. A ledger that counts calls but not tokens can state half of that.\n\n### `tool_call`\n\nOne event per MCP `get_context` or `execute_query` the attempt made.\n\n| Field | Type | Notes |\n|---|---|---|\n| `tool` | string | `get_context` or `execute_query`. |\n| `traceId` | string or null | `get_context` only; look up in your host's trace store. |\n| `targets` | string, list or null | **What the answerer asked for**: the search terms it sent to `get_context`. Null for `execute_query`. |\n| `rankedSummary` | object | Copied at capture so evidence survives trace eviction: `entityIds` (identity, in response order), `hits` (one row per entity: `entity_id`, `source`, `relevance`, `matched_targets`, `position`, `position_in_source`), `resultCount`, `docTokens`. **There is no rank.** This field previously documented `ranks` and \"per-target `targets` with within-target ranks\"; no writer ever wrote either, and what `ranks` did hold was `range(1, n+1)` over a list already flattened source-card by source-card -- a position, not a rank, and not even a relevance order, since the response sorts globally and then buckets into cards. Ask relevance questions of `hits[].relevance`. |\n| `rankedSummary.hits[].matched_targets` | list or null | Which of the caller's search targets matched this entity, each with the server's own relevance. **`null` means no attribution was published**, which is not the same as `[]`: the server attaches it to entities only (a source card's attribution is discarded before an entity object exists) and only on the semantic path, because a lexical score is relative to its own query and comparing two of them means nothing. A lexical run therefore cannot say which target found what, and a reader must report that as unavailable rather than as \"never asked\". |\n| `retrieval_mode` | string or null | `get_context` only: the `retrieval` field of the response that answered, `semantic` or `lexical`. Null when the server named none, which means no embedding provider. Recorded per call because the semantic path can fall over partway through a run. |\n| `query` | string or null | `execute_query` only: the Malloy the call sent, so the final query can be chosen by which call the server actually answered. |\n| `modelPath` | string or null | `execute_query` only: the model file the call named. A source does not resolve outside the file that declares it, so re-execution needs this. |\n| `error` | string or null | |\n\nNever persist `execute_query` result rows, givens, or credentials.\n\n`targets` records the request; `rankedSummary` records the response. Without\nboth, a low per-attempt recall has two readings that cannot be told apart: the\nanswerer searched for the wrong thing, or it searched well and retrieval ranked\nthe right entity too low. Those have opposite owners -- `agent-skill` and\n`retrieval` -- so a ledger holding only the response cannot attribute the\nfailure, and any recall computed from it is a blend of answerer behaviour and\nretrieval quality.\n\nThat blend is also why per-attempt recall is **not** comparable across arms: a\nstronger answerer searches better and scores higher retrieval recall without\nretrieval having changed. Use it for attribution within an arm. Comparing\nretrieval itself between engine versions is `eval-retrieval`'s job, with fixed\nterms, and is not something a customer run reports.\n\n### `score`\n\nThe answer judge's verdict for one attempt (protocol in\n`skill:eval-judge`). Every attempt in a scored run gets exactly one.\n\n| Field | Type | Notes |\n|---|---|---|\n| `verdict` | string or null | `match` / `near_match` / `no_match` / `needs_human`; null when the attempt is not scorable. Only `match` and `no_match` are decisions; see below. |\n| `reason` | string | Why, from the judge; for a null verdict, why not scorable, and it is ALWAYS one of these: `not_submitted`, `answerer_truncated`, `contaminated`, `judge_unparseable`, `no_saved_verdict`, `judge_error: <text>`, or `golden_<status>` -- `golden_missing`, `golden_provisional`, `golden_invalid`, `golden_ambiguous`, `golden_verified_wrong`. A null verdict never arrives without one: a nulling that does not say which of these it was is indistinguishable from the field being dropped on the write path, and was read as exactly that from a real run. The `golden_*` reasons and `answerer_truncated` are decided before the judge is called, so neither an unestablished key nor a cut-off attempt costs a judge call. |\n| `confidence` | int or null | 1 to 10. Confidence of 5 or lower forces `needs_human`. |\n| `column_pairing` | object or null | The judge's named gold-to-prediction column correspondence. |\n| `judge_version` / `rubric_sha` | string | Pins which rubric produced this verdict. |\n| `golden_revision` | int | From the case at score time. |\n| `contaminated` | bool or `\"unknown\"` | Copied from the attempt; true or unknown means `verdict: null`. |\n| `must_not_use_hits` | list or null | Entries of `golden.mustNotUse` a script found in the final query. A hit forces `verdict: no_match`. Decided by `check_must_not_use.py`, never by the judge: it is a question about query text. Only a BARE name vetoes; an entry with a connective (`X as ...`, `X through ...`) objects to a use of X rather than to X, and goes to the judge with the prose entries and a path's bare leaf. |\n| `judge_verdict` | string or null | What the judge said when a MECHANICAL override replaced it: a `must_not_use_hits` veto, or contamination voiding the verdict. Null otherwise, so the judge's own agreement rate stays measurable across both. When a case is vetoed AND contaminated this still holds the judge's read, not the veto's `no_match`. |\n| `artifactPath` | string | The full judge output under `artifacts/`. |\n| `gold_status` | string | `verified` / `verified_benign` / `suspect` / `verified_wrong`, and never anything else: the case file's `golden.status` is a DIFFERENT vocabulary that shares only `verified` and `verified_wrong`, and only those two fall back into this field. A `provisional` / `invalid` / `ambiguous` key is reported through the verdict's `reason` (`golden_<status>`) instead, so nothing downstream has to match on two vocabularies in one field. **From the judge**, which scored against the golden as written and reports separately whether it believes it; falls back to the case's standing `golden.status` when the judge does not say, written onto the verdict itself so the run summary and the aggregates see it too. `verified_wrong` excludes the case from run aggregates -- from the judge OR from the case file; assigning the fallback into a copy of the verdict meant a key the set had marked wrong stayed in the aggregates unless the judge independently agreed. `suspect` and `verified_wrong` route to the golden side door as `dataset` issues, never to improve. |\n| `gold_note` | string or null | The judge's evidence for a non-`verified` status: the two values, or the model line against the rubric sentence. Null when `verified`. |\n\nA `golden_*` reason means the ANSWER KEY is not something a verdict can be\nissued against. A golden the set stamps `provisional`, `invalid` or `ambiguous`\nis refused, as is a case with no golden at all and one the set marks\n`verified_wrong`; a golden with NO `status` is unguarded rather than\nunscorable, because sets that predate the field would otherwise score zero of\nzero. A golden holding no value is not refused either: that is how a\n`kind: unanswerable` case and a `kind: criteria` golden are written. This\nmatters most for an imported set, because `skill:eval-import` stamps\n`provisional` on every golden it writes and enforces it -- so a set is\nunscorable by construction until its keys are re-derived, and `run_baseline`\nrefuses to spend at all when every case in a set is in that state.\n\n`not_submitted` means the attempt produced NOTHING to judge: no answer text and\nno query. An attempt that wrote prose without querying **is judged**, and\nagainst a golden that holds a value an answer containing none of it is\n`no_match` (`skill:eval-judge`, `reference/refusal.md` rule 10). Excusing those\nas unscorable dropped a confident refusal on an answerable case out of the pass\nrate, which is the one thing the answerable-sounds-unanswerable cases exist to\nmeasure. A refusal is exempt only where `golden.kind` is `unanswerable`, and\nthere naming the gap is the pass and a confident number is the fail.\n\n`answerer_truncated` means the answerer hit the run's `maxTurns` cap and the\nCLI returned whatever it had, which is usually a sentence stopping mid-thought.\nIt is refused BEFORE the golden gate and before a saved verdict is reused, so\n`--rebuild` over a run made without this re-derives the right ledger from its\ntranscripts. A cap is a harness setting, so a case cut off by one says nothing\nabout whether the model could answer it: measured on one 29-case arm, four\nattempts ended at exactly 31 turns and each was judged as a wrong answer, one\nof them on the text \"Let me build the correct monthly trend query.\"\n\n`contaminated` means the attempt breached isolation (`contamination_reasons`\nnames how) and the verdict is withheld rather than lost. The judge's own read\nmoves to `judge_verdict`. Before this reason was written, a voided verdict kept\nthe judge's prose in `reason` beside `verdict: null`, and a run report\nreasonably concluded the ledger was dropping the field -- it recovered nine\n\"lost\" verdicts by re-parsing the stored judge replies, every one of which had\nbeen voided on purpose.\n\nBoth are HARNESS-owned, not model-owned, and a run containing either reports\n`status: incomplete` in `run.json` and **prints no pass rate**. `unscorable`\n(no established golden) and `judge_unparseable` do not suppress the rate: the\nfirst is a dataset state no arm can fix, and the second already has a retry.\n\nAggregates count decided verdicts only. `match` and `no_match` are the pass and\nthe fail; **`near_match`, `needs_human` and null are none of the above** and stay\nout of acceptance arithmetic. `near_match` is excluded because it means \"defensibly\ndifferent\", and an arguable verdict that moves a pass rate is a measurement\nartefact rather than a result (`skill:eval-judge` rule 7). Report its count:\nit rising is how a set tells you its rubrics are going vague.\n\nThis applies to `score`. On `retrieval_score` below, `near_match` **is** counted\ntowards recall and precision, because an overlapping entity is a genuine\nretrieval success. The two are different questions that share a word.\n\n### `retrieval_score` (written by `eval-retrieval`, not by a customer run)\n\nOne per term judged in a fixed-term replay (run-level; no `caseId`).\n\n| Field | Type | Notes |\n|---|---|---|\n| `intentId` / `term` / `entityType` | string | From the term file. |\n| `in_scope` | bool | Does THIS model version represent the concept? False is a coverage gap, not a retrieval failure. |\n| `judgments` | list | Per returned entity: `entityId`, `rank`, `level` (`match` / `near_match` / `no_match`), `confidence`, `why`. Empty when nothing returned. |\n| `judge_version` / `rubric_sha` | string | |\n| `traceId` | string or null | The `get_context` call judged. |\n\nRun-level metrics fall out by counting:\n\n- `coverage` = in-scope terms / valid terms -- a property of the model and its\n  data, reported on its own, never as a retrieval number.\n- `recall` (on in-scope terms) = fraction whose judgments contain a `match`.\n- `precision@N` = `match` judgments / judged, with N stated.\n\n### `issue`\n\n| Field | Type | Notes |\n|---|---|---|\n| `issue_id` | string | Stable across status events. |\n| `qids` | list | Affected cases. |\n| `primary_code` / `contributing_codes` | string / list | From `skill:eval-diagnose`, verbatim. An event written before the codes were renamed may carry a retired one: `COVERAGE` is now `MISSING`, `NO-DISAMBIG` is `AMBIGUOUS`, and `CONVENTION` is `RULE_UNWRITTEN`. Read those as the new name; nothing rewrites old events. |\n| `component` | string | `dataset` / `agent-call` / `get_context/model` / `get_context/retrieval` / `construction` / `model-definition`. |\n| `owner` | string | `model` / `retrieval` / `agent-skill` / `dataset`. Environment failures stop the run; they are never diagnosed, so there is no environment owner. |\n| `severity` / `confidence` | string | |\n| `sufficiency` | string | `sufficient` / `insufficient` / `unknown`. |\n| `traceIds` | list | |\n| `diagnosis` | string | Written before any edit exists. |\n\n### `issue_status`\n\n`issue_id` plus `status`: `open` / `batched` / `fixed` / `rejected` /\n`deferred`. Readers take the latest event for that `issue_id`. Do not invent\na status column.\n\n### `candidate`\n\nWritten by `eval-improve`, for every proposed edit, accepted or not. A\nrejected direction keeps its record.\n\n| Field | Notes |\n|---|---|\n| `issue_ids` | |\n| `files` | Paths the edit touched. |\n| `diffSummary` | One line per file. |\n| `probes` | Query and result for each factual claim. |\n| `meaningChanged` | Entities whose *meaning* the edit changed; `[]` for a docs-only edit. |\n| `goldenSuspect` | Each `{qid, entity, stored, rederived}`: a golden this edit may have invalidated. Reported by the improver, never repaired by it. **Non-empty halts the acceptance check** until adjudicated through the golden side door. |\n| `goldenAudit` | The set's `verify_goldens.py` run against the edited model: `{ran, clean, model, tail}`. Catches drift and rubric-vs-model contradictions only; a golden whose value silently moved is invisible to it, which is why `goldenSuspect` exists alongside. |\n\n### `acceptance_check`\n\nCalled `gate` before 2026-09-03. `ledger.py` reads the old kind as this one, so\nrun directories written earlier still validate and nothing rewrites them.\n\nWritten by `eval-loop`, one per acceptance check decision, BEFORE any checkpoint commit.\n\n| Field | Notes |\n|---|---|\n| `issue_ids` | |\n| `decision` | `accepted` / `rejected`. |\n| `class` | `docs` / `definition` / `retrieval` / `justified`. |\n| `baselineRunId` / `finalRunIds` | Plural: acceptance needs two independent runs. |\n| `regressions` | Case ids whose verdict got worse vs baseline. Must be empty to accept. |\n| `holdoutDelta` | Confident-verdict delta on the holdout slice. |\n| `reason` | Including independent deterministic justification when that is the basis. |\n\n### `checkpoint`\n\nWritten by `eval-loop` after an accepted acceptance check, or when a restore runs. The\nmodel bytes live in git, not in this payload.\n\n| Field | Notes |\n|---|---|\n| `action` | `created` / `restored`. |\n| `label` | |\n| `modelGitSha` | The commit this checkpoint names (create), or the commit restored to. |\n| `issueIds` | Issues the accepted edit closed. Empty on restore. |\n\n## `judge-regressions.jsonl`\n\nAppend a line whenever a human overrules a judge verdict: the case or intent,\nthe judge's verdict, the human's, and why. Re-run this file against the judge\nwhenever `skill:eval-judge` or the judge model changes; a rubric change that\nflips old human-settled verdicts is a judge regression, not new truth."
+      body: "# The eval ledger: files and events\n\nThe ledger is plain files in the model package's git repository. There is no\neval API and no eval database. The conductor (`skill:eval-loop`) reads and\nwrites these files directly; the stages share them as their contract.\n`eval-answer` writes `attempt`, `tool_call`, and `score`. An engine-side\n`eval-retrieval` skill, which does not ship here, writes `retrieval_score` and\n`probe`; both stay in this contract so one validator covers every run\ndirectory.\n`eval-diagnose` writes `issue` and `issue_status`. `eval-improve` writes\n`candidate`. `eval-loop` writes `acceptance_check`, further `issue_status`, and\n`checkpoint`.\n\n**The contract is code: `eval-answer/scripts/ledger.py`.** Every script that\nwrites `run.json` or `events.jsonl` builds its lines through that module, so a\nfield rename that reaches only one writer fails at write time. This document\nis the human-readable rendering; where the two disagree, the module is right\nand this file is the bug. Check any run directory with:\n\n```\npython3 skills/eval-answer/scripts/ledger.py validate <runDir>\n```\n\nErrors are broken identity (missing `run.json`, a missing required field, two\nscores for one attempt); warnings are missing comparison pins (a run without\n`skillsVersion` still measures, but cannot anchor an A/B on that pin) and\ngrandfathered unknown fields on old runs.\n\n## Layout\n\n```\nevals/<set>/                # the set, in the model's repository\n  set.json                  # set metadata (below)\n  eval.toml                 # servers and paths for eval.py (config.py)\n  cases.jsonl               # one case per line\n  judge-regressions.jsonl   # judge verdicts a human overruled\n\n<workdir>/                  # [paths] workdir, default ~/.malloy-eval/<set>-<hash>/\n  runs/<runId>/\n    run.json                # run config, the attribution pins\n    events.jsonl            # append-only event lines\n    artifacts/              # prediction CSVs, judge outputs, transcripts\n  packages/                 # built report packages\n  servers/<role>/           # server roots for eval.py serve\n```\n\nThe set directory lives in the SAME git repository as the model it evaluates,\nso a checkpoint (a git commit) pins the model and the answer key together.\nThe workdir is never inside the model package: a run holds a `model.malloy`\nsnapshot and a report is a Malloy package, and either one nested there puts\nthe package into `loadErrors`. For a checkpoint to pin the ledger as well,\npoint the workdir at a directory in the repository outside the package, and\ngitignore its `servers/` and `packages/`.\n\nRules that make the ledger trustworthy:\n\n- `events.jsonl` is append-only. Never edit a line. To void one, append a new\n  event whose payload marks the old one `voided`. One sanctioned exception: a\n  re-runnable stage (diagnose, improve) replaces ITS OWN prior events, keyed by\n  issue id, through `ledger.replace_events()` -- so a crash-and-rerun does not\n  duplicate issues. A stage never touches another stage's lines.\n- A stage never rewrites another stage's fields.\n- One artifact directory per attempt (`qid` plus `sample`). Never overwrite a\n  previous attempt's prediction CSV.\n- End-of-run numbers come from counting event lines (`jq` over\n  `events.jsonl`), never from arithmetic recalled in prose.\n\n## `set.json`\n\n| Field | Notes |\n|---|---|\n| `name` | Set name; also the directory name. |\n| `description` | |\n| `datasetVersion` | Integer. Bump on any golden repair or case change. Runs record the version they scored against. |\n| `targetModelPath` | Model path within the package. |\n| `truthPackage` | Name of the package holding the semantics-free sources every golden is re-derived from -- a package NAME, not an object. Without it `verify_goldens.py` re-derives nothing: it reports \"nothing to re-derive against\" and **exits 3**, which a caller must read as unverified rather than as a pass. The checks that need no truth server -- rubric figures, verification axis, stale rubric claims, set names -- still run and still report. `init_truth_package.py` scaffolds the package. |\n| `truthModel` | Model file inside that package. Default `truth.malloy`. |\n| `targetPackage` | Name of the package UNDER TEST. `verify_goldens.py` refuses a \"truth\" server that also serves it, because an answerer on that server can retrieve the raw truth sources beside the model and quietly change what is under test. `run_baseline.py` and `improve.py` pass the package they were given, so this field is the fallback for a direct call; without either the guard cannot fire. |\n| `truthTableRewrite` | Boolean, default false. Rewrites `duckdb.table('data/x.parquet')` refs to bare `x` in canonical queries, for a truth package whose tables are registered rather than read from files. |\n\n## `cases.jsonl`\n\nOne JSON object per line:\n\n| Field | Notes |\n|---|---|\n| `qid` | Stable case id. |\n| `question` | Exact text the answerer will see. |\n| `split` | `dev` or `holdout`. Frozen at import. Diagnose and improve read `dev` only; the acceptance check runs both. |\n| `tags` | list |\n| `state` | `candidate` / `selected` / `excluded`. |\n| `source` | Where the case came from. |\n| `golden` | `status` (`verified` / `provisional` / `invalid` / `ambiguous` / `verified_wrong`, the last being a key a person has established is wrong, which `run_baseline.py` drops from every aggregate), `kind`, `value` or `path` (artifact under the set directory; a `scalar` value is an object naming the canonical query's column, `{\"net_sales\": 834216}`, never a bare number), `canonicalQuery` (runs against the **truth** package, never the model under test), `verifiedBy`, and `verification` -- `{primaryAxis, variesAxis, note}` naming what the second derivation varied. **A golden is written only after two differently shaped derivations agree** (the truth query and a second one that varies a different axis -- group by something else, sum a different way, take a different route through the raw tables); `gold/<qid>.json` holds both (`truthRows`, `verifyRows`, `agreement`). Two derivations that vary the same axis share every other blind spot: that is how a golden that summed three overlapping time slices, 3x too high, passed its check. `verify_goldens.py` re-derives the value, reads the accepting clause of the rubric against the rows, and flags a second derivation that names no axis. **The question asks for data, never for an interpretation of it**, and either fixes the grain or the rubric accepts a correct figure at any stated grain (`skill:eval-judge` rules 10 and 11). |\n| `golden.kind` | What the key IS. A value (`scalar`, rows), `unanswerable`, or `criteria`: prose with no value, where the author's criteria are the whole key. `criteria` is the one kind that is legitimately `verified` on arrival, because there is no derivation to perform and nothing that can be numerically wrong; an imported key holding a VALUE is `provisional` until re-derived (`skill:eval-import`). **`verify_goldens.py --promote` is the only thing that writes `golden.status`**: it marks a `provisional` golden `verified` when its value re-derived cleanly from the truth package AND a second derivation exists. `--refresh` rewrites a drifted VALUE and never changes a status; `invalid` and `ambiguous` are settled by a person through the golden side door. |\n| `golden.rubric` | **On the golden, not on the case.** The prose the judge is shown as `CASE RUBRIC`, naming what counts as correct and what does not. `run_baseline.py` reads `golden.rubric`; a rubric written at the case's top level is silently not passed, the judge is told `CASE RUBRIC: none`, and it then scores from the golden value and the answer alone -- which reads as a judge that ignores its instructions. |\n| `golden.mustState` | **On the golden, not on the case.** What the answer has to say out loud. Read from `golden.mustState` by the run-package builder. |\n| `goldenRevision` | Integer. Bump on every golden change; `score` events stamp the revision they compared against. |\n| `expectedEntities` | `required`: entity ids (`kind:source:name`) the answer cannot be produced without. `requiredAnyOf`: a list of **groups**, each a list of ids of which any one suffices -- for a case the model can answer through more than one route. Naming only one route scores the other as a retrieval miss and steers the fix to the docs for an entity whose docs were never the problem. `acceptable`: ids that are not noise if returned. An entity counts as delivered when returned under its exact id, under the same type and name on a sibling source, or by name in a returned source's documentation; `score_retrieval.py` records the route per entity as `delivery`. |\n\n## Term files (`eval-retrieval` only)\n\nA customer set has no term file. The engine-side `eval-retrieval` skill keeps\n`intents.jsonl` fixtures -- search term, entity type, a description of what the\nsearcher meant, a validity flag -- beside its own scripts, for evaluating the\nengine against fixed inputs. Their shape and the `retrieval_score` events they\nproduce are documented there and validated here.\n\n## `run.json`\n\nThe attribution pins. Two runs are comparable only when these match where it\nmatters. Two runs with different `target` values are comparable on answer\nverdicts only if the same model version reached both, which is rarely worth\nassuming:\n\nRequired fields are the run's **identity** -- every run ever written carries\nthem. The rest are optional in the schema, but the **comparison pins** are\nwarned about when absent, because a run missing one cannot take part in an A/B\non that axis.\n\n| Field | Notes |\n|---|---|\n| `runId` *(required)* | Directory name. |\n| `target` *(required)* | Which server answered, and how it reached the data: `local`, `local-proxied`, or `platform`. Decides what an improve step is even allowed to do (see `skill:eval-loop`). |\n| `answererModel` *(required)* | |\n| `phase` / `started` *(required)* | |\n| `judgeModel` / `judgeVersion` / `rubricSha` *(pins)* | The judge's model, and the version + content sha of `skill:eval-judge`. |\n| `judgePromptSha` *(pin)* | Content sha of the prompt the judge is SHOWN, which is the harness's own template and not the skill file `rubricSha` covers. The template carries the golden rendering, the evidence blocks and the tie-breaking rules, so two runs whose prompts differed compared as the same judge -- the blind spot that let a golden-rendering defect run through three arms with every test passing, because the tests covered the code paths and not the text the model sees. Absent on runs written before it existed. |\n| `datasetVersion` *(pin)* | From `set.json` at run time. |\n| `modelSha` *(pin)* | Content sha of the `model.malloy` snapshot in the run directory -- the bytes the answerer actually queried. A git sha is not enough: a snapshot host serves a copy, often of a dirty tree no commit names. |\n| `skillsVersion` *(pin)* | HEAD of the checkout the agents' skills were loaded from, dirty-marked (`ledger.skills_git_sha(root)`). The skills are the doctrine the agents load; a run that cannot name their version cannot anchor a skills A/B. |\n| `skillsRoot` / `harnessVersion` | Which checkout supplied the doctrine (`--skills-root`, e.g. a Publisher checkout for the open-source skills; default this one), and this checkout's own HEAD. The eval-* skills always come from the harness checkout, whatever `skillsRoot` says. Two runs whose `skillsRoot` differ are a skills A/B only if their manifests name the same skills. |\n| `diagnoserModel` / `improverModel` | Written by `diagnose.py` / `improve.py` when those stages run, so the run names every LLM that touched it. Absent on a run that was only answered and judged. |\n| `modelGitSha` | Commit of the model repo, from `--model-repo`; `-dirty` suffix when `--model-dir` (or the whole repo, without it) had uncommitted changes -- fine for a band measurement, not for an A/B pin. Null when git cannot answer for the scope (a path outside the repo): absent beats wrong. **Null unless `--model-repo` names the repo.** It cannot be inferred: Publisher serves a COPY under `publisher_data/`, so the tree around the served file is the server's storage, not the model's history, and the field named that until 2026-09-03. `modelSha` above is the pin that always works; this is the pointer to where those bytes are versioned, and absent beats wrong. |\n| `modelRepo` | The path `--model-repo` named, so a reader can tell an absent pin from an unrecorded one. |\n| `modelDir` | The package directory `--model-dir` named, when it was: the path the `-dirty` marker on `modelGitSha` was decided over. Absent means the whole repo, where any unrelated untracked file marks the model dirty. |\n| `environment` / `package` / `modelPath` | What was served, and from where. On a platform target `environment` is the organization and `package` is the workspace the MCP URL is scoped to. |\n| `scope` | Platform target only: the `environment/package[@version]` the answerer was told to pass as an explicit `scopes` entry on every `get_context` / `execute_query` call. A workspace can serve many packages (a personal workspace serves every package the user can read), so without this the run also measures whether retrieval picks the right package -- a different measurement. Absent means unscoped. |\n| `targetVersion` *(pin)* | Platform target only: the published version the answers came from. Both hosted tools default an omitted version to the PINNED one, meaning whatever the workspace serves at the moment of the call, so this only describes the run when `--scope` carries it to every call. `--target-version` supplies it when the scope omits it; a scope naming a different version is refused rather than treated as an override, because a run cannot say which build answered if two pins disagree. |\n| `mcpUrl` / `publisher` | The endpoints the answerer and the re-execution used. |\n| `evalConfig` | The set's `eval.toml` (or `eval.json`) the run read, as `{path, values}`; `path` is null when the set has none. |\n| `predictionsReExecuted` | Whether re-execution was POSSIBLE: the server was serving the bytes this run is pinned to. It is one bool for the whole run and says nothing about any given query, so it reads stronger than it is; `reExecution` below is what actually happened. |\n| `reExecution` | `{attempted, ok, failed, noQuery, notReExecuted, notJudged, missing}` over the run's predictions. `failed` means the query ran and the server returned an error, which is a different thing from an answer with no query at all. `notJudged` is a case the judge was never handed: a golden refused before judging (`provisional`, `invalid`, `ambiguous`, `verified_wrong`, missing), an attempt that submitted nothing, or every case under `--no-judge`. Those used to land in `missing` and read as corrupt artifacts on a freshly imported set. `ok`, `failed`, `noQuery`, `notReExecuted`, `notJudged` and `missing` partition the cases, so they sum to the case count; `attempted` is `ok + failed` and sits outside that sum. |\n| `retrievalMode` | Which retriever answered: `semantic`, `lexical`, `mixed` (it changed mid-run, so nothing can be averaged across it) or `unreported` (nothing ranked, which on a configured server means no embedding provider). Only a RANKING call names a retriever, so a call that enumerates a type or looks a name up is `unreported` on a fully semantic server and must not make the run read as `mixed`. Local retrieval degrades to lexical SILENTLY without an embedding key, and a pair compared across that reads as a model change, so `flip_table.py` refuses the pair unless both sides match. |\n| `retrievalCalls` | `{semantic, lexical, unreported}` call counts behind `retrievalMode`. `unreported` counts the calls that did not rank, which is a different thing from calls that fell back. |\n| `retrievalGate` | What the pre-arm retrieval gate found, or why it did not run. A run is held until `get_context` answers semantic twice in a row, because the first calls after a restart are lexical by design and an arm started immediately measures two retrievers. `not run (no answering phase)` under `--rebuild`/`--rejudge`, and `skipped by --no-retrieval-gate` when the operator opted out -- which is a different fact from a gate that passed, and the field is what tells them apart. |\n| `coverageReport` | The `check_coverage.py --out` report this run read, when `--coverage` named one: its path, the model version it was stamped with, its `agentModel`, and `decided` of `cases`. Coverage is a read of the MODEL, so it is measured separately and costs a judge call per case; the run does not run it, it consumes it. Its per-case verdict then beats the authored `coverage` label in retrieval attribution, and each `score_retrieval` row records `coverage_source` (`measured`, `authored`, `none`) so nobody has to guess which one charged the failure. Absent when no report was given, which is a different fact from a report that found everything covered. |\n| `label` | `<set>-<phase>-<nn>`, assigned by `run_baseline.py` from the set name, the run's phase and the next free number beside it (`ecommerce-baseline-01`, `ecommerce-baseline-02`, `ecommerce-blind_gate-01`). The A/A pair is two runs of the same phase; the post-edit arms are two runs of `blind_gate`. Hand-typed names do not survive one afternoon of runs -- `base`, `rejudged2`, `r3`, `post1` sort wrongly, group not at all, and cannot be matched to an arm. `--label` overrides for a run that genuinely needs a human name. |\n| `effort` | |\n| `maxTurns` / `answererTimeout` *(pins)* | The answerer's turn cap and per-attempt timeout, as the arm ran them. `skill:eval-loop` step 6 lists the call budget among the pins to freeze for a whole run, and nothing wrote one, so no two runs could be compared on the setting that decides whether a case got to finish at all. Distinct from `callBudget`, which no harness has ever written. Size the cap from a pilot rather than taking the default: on a 29-case arm at the default 30, completed attempts had a median of 17 turns and a 90th percentile of 26, and the four cases that ran past it were recorded as wrong answers. |\n| `truncated` / `contaminated` | The qids excluded for a HARNESS-owned reason: the cap cut the answer off, or the attempt breached isolation. Both leave the denominator, both are named here rather than only in the console, and either one makes `status` `incomplete`. |\n| `answererCostUsd` / `judgeCostUsd` | What the arm cost, split by role. The judge's half was discarded until 2026-09-02, so every \"cost per arm\" quoted before then was the answerer alone. Per-case judge cost is on each `score` event. |\n| `answererCostCopiedFrom` | The run whose attempts `answererCostUsd` was COPIED from, when this run spawned no answerer -- a `--rejudge`, a `--rebuild`, a `--from`. Those copy the source's attempt events verbatim, `cost_usd` included, so the figure is right about the attempts and wrong as this run's spend: one re-judge reported $17.56 of answering that never happened. Null when this run paid, so a total over several run files can skip the copies. |\n| `goldenCheck` | What `verify_goldens.py` said before the run started: `N ok, M drifted, K other finding(s)`, or why it did not run (no truth server on a platform target; `--skip-golden-check`; rebuild). A run that started on a drifted set says so here rather than pretending its verdicts mean something. `run_baseline` calls `verify()` without a model, so the set-name audit (check 5) does not run in-arm; the trailing `N stale entity name(s)` is the in-arm lint's own count, and `entity-name lint not run` says there was no model text to lint against. Without it the field read `0 other finding(s)` for a check that never happened. |\n| `staleEntityNames` | The `expectedEntities` ids the compiled served model does not declare, from the in-arm lint (bare names from a search of the model text when the compiled model could not be read). Each one depresses the run's own recall on every case listing it, so a stale set reads as a retrieval failure until someone checks by hand. `null` when there was no model text to lint against (a platform target), which is a different fact from `[]`. |\n| `status` | `complete`; `aborted` when four consecutive attempts errored or found the server dead and the harness stopped rather than spend the rest of the budget on attempts nobody will trust; or `incomplete` when the arm ran to the end and still cannot report a rate, because `truncated` or `contaminated` is non-empty. `incomplete` is a third state, not a soft `complete`: the arm did everything asked of it and the number is still unearned. It does NOT stop the run being compared: `flip_table.py` scores the cases both arms did, and names the ones this arm left unscored -- an excluded case carries no verdict, so it can never be counted as a flip. |\n| `packageSha` / `servedRevision` *(pins)* | Taken from the server, not recomputed: `sourceContentSha` is a content hash over EVERY model path in the package, so an edit to an imported file moves it where a sha of the one `--model-path` does not; `servedRevision` is minted per load, so it identifies a load rather than content and is a poor pin alone. Measured: `publisher.json`'s `version` moves neither, and nothing in Publisher reads it -- it is not in the Package API schema and never returned, so it pins nothing. |\n| `datasetSha` *(pin)* | Content hash of `set.json` + `cases.jsonl`. Deliberately SEPARATE from the model's sha: a golden repair is not a model change, and one pin covering both would make every answer-key fix read as an edit to the model, which is the distinction an A/B rests on. Automatic, so nobody has to remember it; `datasetVersion` stays beside it as the human-readable sequence. **Local targets only.** A hosted target that publishes IMMUTABLE versions needs none of this: the set rides inside the version, and immutability -- not hashing -- is what makes a pin trustworthy. There, `targetVersion` alone identifies model and set together. |\n| `doubtedGoldens` | The cases whose golden is not believed: `qid`, `gold_status` (`suspect` or `verified_wrong`), `gold_note`, and `declaredBy` (`judge` or `set`). A status the SET declared is a key somebody already settled, not an opinion the judge formed this run, and attributing both to the judge claimed a judgement that never happened. **Read this before diagnose.** These are dataset issues, not model failures, and they go through the golden side door in `skill:eval-loop`. Empty list when no key is in doubt from either source. Written from the same scan that prints the end-of-run warning, because a warning that lives only in console text is one scrollback away from sending a modelling agent at a model that is already right. |\n| `mode` / `setName` / `serverVersion` / `traceMode` / `callBudget` | Defined and accepted, **not yet written by any harness** -- kept in the schema for the platform target and the conductor, which need them. (`targetVersion` and `status` were in this list and are both written now.) |\n\n## Events\n\nEvery line in `events.jsonl` is **flat**: `{ \"kind\": ..., <fields> }`, one\nJSON object per line, with an optional `at` ISO timestamp. Case-scoped kinds\n(`attempt`, `tool_call`, `score`) carry `qid`, `sample`, `phase` on the line;\nrun-level kinds do not. (An earlier draft of this document nested fields under\na `payload` with a `caseId`; no writer ever did that, and 23 run directories\nexist in the flat shape, so the flat shape is the contract.) `kind` is one of:\n`attempt`, `tool_call`, `score`, `retrieval_score`, `issue`, `issue_status`,\n`candidate`, `acceptance_check`, `checkpoint`.\n\n### `attempt`\n\n| Field | Type | Notes |\n|---|---|---|\n| `qid` | string | |\n| `sample` | int or null | Which repeat. Required even when null. |\n| `phase` | string | `baseline` / `loop` / `blind_gate` / `canary` / `final`. `phase` lives here, on the attempt, not in run config. |\n| `question_sha` | string | Hash of the exact text the answerer saw, so two runs can be compared on whether they asked the same thing. A case may carry `questionSha`, the seal stamped at conversion, and that wins when present; the ecommerce set writes a 16-char prefix of it, and a set without one left this null. Neither form proves the question was not edited, since both follow the case as it now reads: that comparison is check 6 of `verify_goldens.py`. |\n| `submitted` | bool | False when there was no final query. Not a wrong answer. |\n| `final_query` | string or null | Required to replay. A named view is recorded as the Malloy it stands for, `run: <source> -> <view>`, so every consumer sees one shape and the query re-executes. |\n| `final_query_source` | string or null | How `final_query` was chosen: `declared` (the answer printed it), `last_ok` (the last call the server answered) or `last`. `last` is a warning: a trailing sanity probe may be standing in for the answer's own query. |\n| `servedRevision` | string or null | From the package actually queried. |\n| `n_get_context` / `n_execute` / `n_execute_errors` | int | |\n| `host_tool_uses` | int | EVERY tool use the host logged, MCP calls included. It counted only the non-MCP ones until 2026-09-03, which made the under-report check below true of almost every clean attempt. |\n| `mcp_tool_uses` | int or null | The MCP subset: `get_context` plus `execute_query`. Null on a run written before the split. |\n| `reported_calls` | int | MCP calls the answerer claimed. `reported_calls > host_tool_uses` is the under-report floor from `skill:eval-answer`: the answerer cannot have made more calls than the host logged. `validate_run` warns on it, and only for runs that carry `mcp_tool_uses`, because the field meant something narrower before. |\n| `contaminated` | bool or `\"unknown\"` | `\"unknown\"` when no host log exists. Read it with `ledger.is_contaminated`, never for truthiness: runs written before 2026-09-03 carry the strings `\"true\"`/`\"false\"`, and `bool(\"false\")` is True. `ledger.event` now rejects those strings on write; `validate_run` grandfathers them on read as a warning. |\n| `contamination_reasons` | list | Empty when clean. |\n| `input_tokens` / `output_tokens` / `cache_read_tokens` / `cache_write_tokens` | int or null | Answerer token usage, the four counts that reprice a run from its own ledger. Null when the host does not report it. `cost_usd` beside them comes from the CLI's `total_cost_usd`, which prices cache reads and writes at their rates, so it was always cache-inclusive; what was missing was the breakdown that reproduces it. `cache_write_tokens` (`cache_creation_input_tokens` upstream) was never captured before 2026-09-15, and on one analysed run cache writes were 44% of the agent's cost, the single largest line. **Read `input_tokens` with care:** it counts only the tokens after the last cache breakpoint. One run recorded 224 for 24 questions beside 2.4M cache reads; the 224 is not the context volume, and a token total that omits the cache columns is not a size. |\n| `cost_usd` | float or null | Answerer cost for this attempt. |\n| `num_turns` / `wall_seconds` | int, float or null | |\n| `answer_text` | string or null | The answer the judge scored. Kept so a verdict can be re-read without the transcript. |\n| `skills_invoked` | list of string | The skills the answerer invoked through the `Skill` tool, deduplicated and sorted, counted from its own transcript. Always a list on a run that recorded it, so an EMPTY list is the finding: the arm granted a manifest and the agent reached for none of it. Absent on runs written before 2026-09-15, which is why every reader takes `or []`. Only explicit `Skill` calls are counted, and only skills the arm granted -- one outside `--answerer-skills` is a contamination breach and lands in `contamination_reasons` instead. What it is for: a manifest names what an arm GRANTED, and comparing two arms' doctrine is meaningless if the doctrine never loaded. |\n| `transcriptPath` | string | The answerer's transcript under `artifacts/`. |\n\nToken counts sit here rather than being derived later because the claim a\nsemantic model makes is about cost as well as correctness -- that a documented\nmodel reaches a good answer in fewer turns and tokens than working from raw\nschema. A ledger that counts calls but not tokens can state half of that.\n\n### `tool_call`\n\nOne event per MCP `get_context` or `execute_query` the attempt made.\n\n| Field | Type | Notes |\n|---|---|---|\n| `tool` | string | `get_context` or `execute_query`. |\n| `traceId` | string or null | `get_context` only; look up in your host's trace store. |\n| `targets` | string, list or null | **What the answerer asked for**: the search terms it sent to `get_context`. Null for `execute_query`. |\n| `rankedSummary` | object | Copied at capture so evidence survives trace eviction: `entityIds` (identity, in response order), `hits` (one row per entity: `entity_id`, `source`, `relevance`, `matched_targets`, `position`, `position_in_source`), `resultCount`, `docTokens`. **There is no rank.** This field previously documented `ranks` and \"per-target `targets` with within-target ranks\"; no writer ever wrote either, and what `ranks` did hold was `range(1, n+1)` over a list already flattened source-card by source-card -- a position, not a rank, and not even a relevance order, since the response sorts globally and then buckets into cards. Ask relevance questions of `hits[].relevance`. |\n| `rankedSummary.hits[].matched_targets` | list or null | Which of the caller's search targets matched this entity, each with the server's own relevance. **`null` means no attribution was published**, which is not the same as `[]`: the server attaches it to entities only (a source card's attribution is discarded before an entity object exists) and only on the semantic path, because a lexical score is relative to its own query and comparing two of them means nothing. A lexical run therefore cannot say which target found what, and a reader must report that as unavailable rather than as \"never asked\". |\n| `retrieval_mode` | string or null | `get_context` only: the `retrieval` field of the response that answered, `semantic` or `lexical`. Null when the server named none, which means no embedding provider. Recorded per call because the semantic path can fall over partway through a run. |\n| `query` | string or null | `execute_query` only: the Malloy the call sent, so the final query can be chosen by which call the server actually answered. |\n| `modelPath` | string or null | `execute_query` only: the model file the call named. A source does not resolve outside the file that declares it, so re-execution needs this. |\n| `error` | string or null | |\n\nNever persist `execute_query` result rows, givens, or credentials.\n\n`targets` records the request; `rankedSummary` records the response. Without\nboth, a low per-attempt recall has two readings that cannot be told apart: the\nanswerer searched for the wrong thing, or it searched well and retrieval ranked\nthe right entity too low. Those have opposite owners -- `agent-skill` and\n`retrieval` -- so a ledger holding only the response cannot attribute the\nfailure, and any recall computed from it is a blend of answerer behaviour and\nretrieval quality.\n\nThat blend is also why per-attempt recall is **not** comparable across arms: a\nstronger answerer searches better and scores higher retrieval recall without\nretrieval having changed. Use it for attribution within an arm. Comparing\nretrieval itself between engine versions is `eval-retrieval`'s job, with fixed\nterms, and is not something a customer run reports.\n\n### `score`\n\nThe answer judge's verdict for one attempt (protocol in\n`skill:eval-judge`). Every attempt in a scored run gets exactly one.\n\n| Field | Type | Notes |\n|---|---|---|\n| `verdict` | string or null | `match` / `near_match` / `no_match` / `needs_human`; null when the attempt is not scorable. Only `match` and `no_match` are decisions; see below. |\n| `reason` | string | Why, from the judge; for a null verdict, why not scorable, and it is ALWAYS one of these: `not_submitted`, `answerer_truncated`, `contaminated`, `judge_unparseable`, `no_saved_verdict`, `judge_error: <text>`, or `golden_<status>` -- `golden_missing`, `golden_provisional`, `golden_invalid`, `golden_ambiguous`, `golden_verified_wrong`. A null verdict never arrives without one: a nulling that does not say which of these it was is indistinguishable from the field being dropped on the write path, and was read as exactly that from a real run. The `golden_*` reasons and `answerer_truncated` are decided before the judge is called, so neither an unestablished key nor a cut-off attempt costs a judge call. |\n| `confidence` | int or null | 1 to 10. Confidence of 5 or lower forces `needs_human`. |\n| `column_pairing` | object or null | The judge's named gold-to-prediction column correspondence. |\n| `judge_version` / `rubric_sha` | string | Pins which rubric produced this verdict. |\n| `golden_revision` | int | From the case at score time. |\n| `contaminated` | bool or `\"unknown\"` | Copied from the attempt; true or unknown means `verdict: null`. |\n| `must_not_use_hits` | list or null | Entries of `golden.mustNotUse` a script found in the final query. A hit forces `verdict: no_match`. Decided by `check_must_not_use.py`, never by the judge: it is a question about query text. Only a BARE name vetoes; an entry with a connective (`X as ...`, `X through ...`) objects to a use of X rather than to X, and goes to the judge with the prose entries and a path's bare leaf. |\n| `judge_verdict` | string or null | What the judge said when a MECHANICAL override replaced it: a `must_not_use_hits` veto, or contamination voiding the verdict. Null otherwise, so the judge's own agreement rate stays measurable across both. When a case is vetoed AND contaminated this still holds the judge's read, not the veto's `no_match`. |\n| `artifactPath` | string | The full judge output under `artifacts/`. |\n| `gold_status` | string | `verified` / `verified_benign` / `suspect` / `verified_wrong`, and never anything else: the case file's `golden.status` is a DIFFERENT vocabulary that shares only `verified` and `verified_wrong`, and only those two fall back into this field. A `provisional` / `invalid` / `ambiguous` key is reported through the verdict's `reason` (`golden_<status>`) instead, so nothing downstream has to match on two vocabularies in one field. **From the judge**, which scored against the golden as written and reports separately whether it believes it; falls back to the case's standing `golden.status` when the judge does not say, written onto the verdict itself so the run summary and the aggregates see it too. `verified_wrong` excludes the case from run aggregates -- from the judge OR from the case file; assigning the fallback into a copy of the verdict meant a key the set had marked wrong stayed in the aggregates unless the judge independently agreed. `suspect` and `verified_wrong` route to the golden side door as `dataset` issues, never to improve. |\n| `gold_note` | string or null | The judge's evidence for a non-`verified` status: the two values, or the model line against the rubric sentence. Null when `verified`. |\n\nA `golden_*` reason means the ANSWER KEY is not something a verdict can be\nissued against. A golden the set stamps `provisional`, `invalid` or `ambiguous`\nis refused, as is a case with no golden at all and one the set marks\n`verified_wrong`; a golden with NO `status` is unguarded rather than\nunscorable, because sets that predate the field would otherwise score zero of\nzero. A golden holding no value is not refused either: that is how a\n`kind: unanswerable` case and a `kind: criteria` golden are written. This\nmatters most for an imported set, because `skill:eval-import` stamps\n`provisional` on every golden it writes and enforces it -- so a set is\nunscorable by construction until its keys are re-derived, and `run_baseline`\nrefuses to spend at all when every case in a set is in that state.\n\n`not_submitted` means the attempt produced NOTHING to judge: no answer text and\nno query. An attempt that wrote prose without querying **is judged**, and\nagainst a golden that holds a value an answer containing none of it is\n`no_match` (`skill:eval-judge`, `reference/refusal.md` rule 10). Excusing those\nas unscorable dropped a confident refusal on an answerable case out of the pass\nrate, which is the one thing the answerable-sounds-unanswerable cases exist to\nmeasure. A refusal is exempt only where `golden.kind` is `unanswerable`, and\nthere naming the gap is the pass and a confident number is the fail.\n\n`answerer_truncated` means the answerer hit the run's `maxTurns` cap and the\nCLI returned whatever it had, which is usually a sentence stopping mid-thought.\nIt is refused BEFORE the golden gate and before a saved verdict is reused, so\n`--rebuild` over a run made without this re-derives the right ledger from its\ntranscripts. A cap is a harness setting, so a case cut off by one says nothing\nabout whether the model could answer it: measured on one 29-case arm, four\nattempts ended at exactly 31 turns and each was judged as a wrong answer, one\nof them on the text \"Let me build the correct monthly trend query.\"\n\n`contaminated` means the attempt breached isolation (`contamination_reasons`\nnames how) and the verdict is withheld rather than lost. The judge's own read\nmoves to `judge_verdict`. Before this reason was written, a voided verdict kept\nthe judge's prose in `reason` beside `verdict: null`, and a run report\nreasonably concluded the ledger was dropping the field -- it recovered nine\n\"lost\" verdicts by re-parsing the stored judge replies, every one of which had\nbeen voided on purpose.\n\nBoth are HARNESS-owned, not model-owned, and a run containing either reports\n`status: incomplete` in `run.json` and **prints no pass rate**. `unscorable`\n(no established golden) and `judge_unparseable` do not suppress the rate: the\nfirst is a dataset state no arm can fix, and the second already has a retry.\n\nAggregates count decided verdicts only. `match` and `no_match` are the pass and\nthe fail; **`near_match`, `needs_human` and null are none of the above** and stay\nout of acceptance arithmetic. `near_match` is excluded because it means \"defensibly\ndifferent\", and an arguable verdict that moves a pass rate is a measurement\nartefact rather than a result (`skill:eval-judge` rule 7). Report its count:\nit rising is how a set tells you its rubrics are going vague.\n\nThis applies to `score`. On `retrieval_score` below, `near_match` **is** counted\ntowards recall and precision, because an overlapping entity is a genuine\nretrieval success. The two are different questions that share a word.\n\n### `retrieval_score` (written by `eval-retrieval`, not by a customer run)\n\nOne per term judged in a fixed-term replay (run-level; no `caseId`).\n\n| Field | Type | Notes |\n|---|---|---|\n| `intentId` / `term` / `entityType` | string | From the term file. |\n| `in_scope` | bool | Does THIS model version represent the concept? False is a coverage gap, not a retrieval failure. |\n| `judgments` | list | Per returned entity: `entityId`, `rank`, `level` (`match` / `near_match` / `no_match`), `confidence`, `why`. Empty when nothing returned. |\n| `judge_version` / `rubric_sha` | string | |\n| `traceId` | string or null | The `get_context` call judged. |\n\nRun-level metrics fall out by counting:\n\n- `coverage` = in-scope terms / valid terms -- a property of the model and its\n  data, reported on its own, never as a retrieval number.\n- `recall` (on in-scope terms) = fraction whose judgments contain a `match`.\n- `precision@N` = `match` judgments / judged, with N stated.\n\n### `issue`\n\n| Field | Type | Notes |\n|---|---|---|\n| `issue_id` | string | Stable across status events. |\n| `qids` | list | Affected cases. |\n| `primary_code` / `contributing_codes` | string / list | From `skill:eval-diagnose`, verbatim. An event written before the codes were renamed may carry a retired one: `COVERAGE` is now `MISSING`, `NO-DISAMBIG` is `AMBIGUOUS`, and `CONVENTION` is `RULE_UNWRITTEN`. Read those as the new name; nothing rewrites old events. |\n| `component` | string | `dataset` / `agent-call` / `get_context/model` / `get_context/retrieval` / `construction` / `model-definition`. |\n| `owner` | string | `model` / `retrieval` / `agent-skill` / `dataset`. Environment failures stop the run; they are never diagnosed, so there is no environment owner. |\n| `severity` / `confidence` | string | |\n| `sufficiency` | string | `sufficient` / `insufficient` / `unknown`. |\n| `traceIds` | list | |\n| `diagnosis` | string | Written before any edit exists. |\n\n### `issue_status`\n\n`issue_id` plus `status`: `open` / `batched` / `fixed` / `rejected` /\n`deferred`. Readers take the latest event for that `issue_id`. Do not invent\na status column.\n\n### `candidate`\n\nWritten by `eval-improve`, for every proposed edit, accepted or not. A\nrejected direction keeps its record.\n\n| Field | Notes |\n|---|---|\n| `issue_ids` | |\n| `files` | Paths the edit touched. |\n| `diffSummary` | One line per file. |\n| `probes` | Query and result for each factual claim. |\n| `meaningChanged` | Entities whose *meaning* the edit changed; `[]` for a docs-only edit. |\n| `goldenSuspect` | Each `{qid, entity, stored, rederived}`: a golden this edit may have invalidated. Reported by the improver, never repaired by it. **Non-empty halts the acceptance check** until adjudicated through the golden side door. |\n| `goldenAudit` | The set's `verify_goldens.py` run against the edited model: `{ran, clean, model, tail}`. Catches drift and rubric-vs-model contradictions only; a golden whose value silently moved is invisible to it, which is why `goldenSuspect` exists alongside. |\n\n### `acceptance_check`\n\nCalled `gate` before 2026-09-03. `ledger.py` reads the old kind as this one, so\nrun directories written earlier still validate and nothing rewrites them.\n\nWritten by `eval-loop`, one per acceptance check decision, BEFORE any checkpoint commit.\n\n| Field | Notes |\n|---|---|\n| `issue_ids` | |\n| `decision` | `accepted` / `rejected`. |\n| `class` | `docs` / `definition` / `retrieval` / `justified`. |\n| `baselineRunId` / `finalRunIds` | Plural: acceptance needs two independent runs. |\n| `regressions` | Case ids whose verdict got worse vs baseline. Must be empty to accept. |\n| `holdoutDelta` | Confident-verdict delta on the holdout slice. |\n| `reason` | Including independent deterministic justification when that is the basis. |\n\n### `checkpoint`\n\nWritten by `eval-loop` after an accepted acceptance check, or when a restore runs. The\nmodel bytes live in git, not in this payload.\n\n| Field | Notes |\n|---|---|\n| `action` | `created` / `restored`. |\n| `label` | |\n| `modelGitSha` | The commit this checkpoint names (create), or the commit restored to. |\n| `issueIds` | Issues the accepted edit closed. Empty on restore. |\n\n## `judge-regressions.jsonl`\n\nAppend a line whenever a human overrules a judge verdict: the case or intent,\nthe judge's verdict, the human's, and why. Re-run this file against the judge\nwhenever `skill:eval-judge` or the judge model changes; a rubric change that\nflips old human-settled verdicts is a judge regression, not new truth."
     },
     {
       name: "eval-diagnose",
@@ -384526,7 +384528,7 @@ model-owned finding deserves a probe of its own before it justifies an edit.
 
 ## Step 4: Append issue events, then stop
 
-Append to \`evals/<set>/runs/<runId>/events.jsonl\` with \`kind: issue\`
+Append to \`<workdir>/runs/<runId>/events.jsonl\` with \`kind: issue\`
 (shapes in \`skill:eval-answer\` \`reference/ledger-schema.md\`):
 
 - \`issue_id\`, affected \`qids\`, \`primary_code\`, \`contributing_codes\`
@@ -385237,7 +385239,7 @@ Yours, after applying step 3 and step 4:
 \`\`\`json
 {"qid": "ecom-q-014", "question": "How did denim sell last quarter versus the one before?",
  "questionSha": "<stamped>", "split": "dev", "source": "questions.jsonl",
- "golden": {"status": "provisional", "kind": "scalar", "value": 1830000,
+ "golden": {"status": "provisional", "kind": "scalar", "value": {"denim_change": 1830000},
             "canonicalQuery": "SELECT ...", "verifiedBy": "authored_query",
             "rubric": "Reports the quarter-over-quarter change for denim, and breaks the figure out by channel."},
  "sourceFields": {"difficulty": "hard"}}
@@ -386031,547 +386033,8 @@ can never fire.`
     },
     {
       name: "eval-loop",
-      description: "Conduct a local Publisher evaluation loop in five steps: scrape/run, eval, diagnose, improve, checkpoint. You are the conductor: import cases into the file ledger, spawn a blind answerer, then run eval-answer, eval-diagnose, and eval-improve. Persistence is plain files under the model package''s evals/ directory; checkpoints are git commits of the model repo. Use to score a model, diagnose failures, improve behind an acceptance check, or roll back a bad direction.",
-      body: `# The Evaluation Loop
-
-You conduct this loop. There is no batch orchestrator to start, no eval API,
-and no eval MCP tools. The ledger is plain files in the model package's git
-repository (\`reference/ledger-schema.md\` in \`skill:eval-answer\` defines every
-file and event). Scoring is an LLM judge you spawn per case. There is no
-scripted scorer, and there will not be one: a script that can pass a wrong
-answer is worse than none. The scripts under \`scripts/\` run the loop -- they
-answer, re-execute, spawn the judge, compare runs, and write the ledger -- but
-none of them decides whether an answer was right.
-
-\`\`\`
-scrape/run  ->  eval  ->  diagnose  ->  improve  ->  checkpoint
-\`\`\`
-
-**This skill conducts; it does not restate.** Scoring lives in
-\`skill:eval-answer\`. Components and owners live in \`skill:eval-diagnose\`.
-Edit rules live in \`skill:eval-improve\`.
-
-Do not merge **eval** into **diagnose**. A conductor who scores while
-explaining writes the explanation into the score. Do not skip the **acceptance
-check** inside improve. The acceptance check decides whether *this* edit
-stays. **Checkpoint** decides whether a *sequence* of accepted edits can be
-undone.
-
-## Where the rest of this lives
-
-This file is the procedure. Five things it used to carry inline are files beside
-it now, because each is needed at one moment rather than every run, and loading
-all of them for every run is how a skill stops being read.
-
-| When | Read |
-|---|---|
-| about to run one | \`reference/running-a-run.md\` |
-| a golden is wrong, doubted, or out of step with the model | \`reference/golden-side-door.md\` |
-| auditing a key you doubt, or a set you did not author | \`reference/auditing-an-answer-key.md\` |
-| deciding whether an edit stays | \`reference/acceptance-check.md\` |
-| about to quote a number, set the band, or read a set's flips | \`reference/measurement.md\` |
-| the run finished and someone has to read it | \`skill:eval-report\` |
-| you changed judge doctrine or its inputs | \`reference/checking-the-judge.md\` |
-
-Read the file, do not work from the summary here. The acceptance-check rules and
-the golden side door are both places where acting on a half-memory of the rule
-produces a confident wrong answer rather than an error.
-
-## The five steps
-
-| Step | Job | Writes |
-|---|---|---|
-| **a. scrape / run** | Put cases in the ledger; spawn a blind answerer | cases; \`attempt\`, \`tool_call\` |
-| **b. eval** | Judge the answer; score which required entities retrieval delivered | \`score\` |
-| **c. diagnose** | Why it failed, who owns it | \`issue\` / \`issue_status\`. Stop. Do not edit. |
-| **d. improve** | One smallest model edit, then the acceptance check | improve writes \`candidate\`; you write \`acceptance_check\`. Revert on reject. |
-| **e. checkpoint** | Git commit after an accepted acceptance check | \`checkpoint\` event, then the commit |
-
-Every run then **reports** (below). A run whose result exists only as JSONL and
-a scrolled-away console summary has not been delivered.
-
-**scrape** and **run** share a letter but are not the same job. Scrape writes
-cases. Run writes attempts. Do not invent questions and score
-them in one breath.
-
-**Step d is \`skill:eval-improve\`, not a hand edit plus another arm.** The
-failure mode is specific and it has happened: the edits were made by hand,
-published, and validated by re-running the whole arm -- with the answer key
-repaired in the same window. No \`candidate\`, \`acceptance_check\` or \`checkpoint\`
-event exists in that run's ledger, and the resulting move cannot be attributed
-to the model, the key, or the answerer. Re-running a whole arm after changing
-two things is the one thing the acceptance check exists to replace: it scores
-the edit against dev AND holdout, and it is cheaper than the arm.
-
-### Scrape, minimally
-
-Importing an existing corpus IS the scrape step: copy the set from its home
-(for example a benchmarks checkout) into \`evals/<set>/\` and convert to the
-ledger shapes. **\`skill:eval-import\` is that job in full** -- how to classify
-what arrived with each question, why nothing imports as a verified value, and
-the seal that makes a later edit to a question detectable. Read it whenever the
-questions came from outside, which is most of the time. While importing:
-
-- Freeze each case's \`split\`: \`dev\` or \`holdout\`. Diagnose and improve read
-  dev cases only; the acceptance check runs both. A set that is all dev cannot defend an
-  accept.
-- Later, each diagnosed-and-fixed failure becomes a new frozen dev case, so a
-  fixed bug cannot silently return.
-
-Scraping from production logs (chat transcripts, retrieval traces) is the
-other supported source, and usually the better one: real traffic asks what
-people actually ask. Where your logs physically live is a host concern; look
-for a host-specific log-fetching skill. \`skill:eval-import\` takes over once
-you have the text, and its \`reference/case-format.md\` covers what a log pull
-needs that a question list does not.
-
-Prefer variety over volume when you sample, from either source. Cases that
-differ in grain, source, filter shape, and phrasing are what move a
-measurement; a second sample of the same case is nearly free of new
-information.
-
-### Mode aliases
-
-Older mode names still work as aliases for how far one run walks:
-
-| Alias | Steps |
-|---|---|
-| \`measure\` | scrape/run + eval |
-| \`triage\` | plus diagnose |
-| \`improve\` | plus improve + acceptance check + checkpoint on accept |
-
-Say which alias (or which steps) you are running before the first question.
-Record it in \`run.json\`. Do not mix steps in a way that lets the answerer see
-gold, issues, or the model file.
-
-Most runs should stop after eval. Diagnose when you need a histogram of
-components and owners. Improve only for diagnosed *model* gaps, one batch at
-a time. Checkpoint only after the acceptance check **accepts**.
-
-## Roles
-
-| Role | Sees |
-|---|---|
-| **Answerer** | The question and the Malloy tools. Never the golden, \`evals/\`, the model file, or any hint it is being evaluated. |
-| **Judge** | The golden and the prediction. Never conducts, never answers, never edits. One fresh subagent per verdict (\`skill:eval-judge\`). |
-| **You (conductor / improver)** | Everything, including goldens and traces. |
-| **Acceptance check** | The edit and the evidence. Never the improver's self-assessment alone. |
-
-The answerer stays blind. That is not optional. A grader-visible answerer
-writes toward the expected answer, and the score is fiction.
-
-There are no eval MCP tools on purpose. The answerer inherits your tools,
-including Shell and Read, so any eval convenience surface would also be a
-gold path for it. Blindness is prevention plus detection, not a guarantee:
-\`eval-answer\` runs the contamination checklist on every attempt, which is
-why you keep a host-side tool-use log per answerer.
-
-## Pick the target first
-
-Both a local model server and a hosted platform expose the same two tools the
-answerer needs, \`get_context\` and \`execute_query\`, so the loop runs against
-either. What differs is which model is answering and whose data it reads, and
-those are two separate axes:
-
-| Target | Model under test | Data | Can edit and re-test? |
-|---|---|---|---|
-| **Local (direct)** | your working files | local (for example duckdb), or a direct warehouse connection | yes |
-| **Local (proxied)** | your working files | the platform's connection, through a proxy connection type | yes |
-| **Remote** | the published version, through the platform's hosted \`get_context\`/\`execute_query\` | the platform's | no, publishing is not an eval action |
-
-The middle row is the one worth knowing about: it decouples the two axes, so you
-can evaluate a model you are still editing against the customer's real data. It
-is a connection configuration, not a feature.
-
-### Three ways to reach the tools
-
-That table is about which MODEL answers. A second, independent choice is where
-the answerer's MCP tools come from, and \`--mcp-url\` is the whole of it:
-
-| | \`--target\` | \`--mcp-url\` | Auth |
-|---|---|---|---|
-| **1. Local Publisher** | \`local\` | \`http://localhost:4040/mcp\` (default) | none |
-| **2. Hosted, through an editor extension's local bridge** | \`platform\` | the localhost URL the extension prints | none: the extension holds the credential |
-| **3. Hosted, directly** | \`platform\` | the host's \`https\` endpoint, scoped if it offers one | a cached OAuth login, once, interactively |
-
-\`\`\`bash
-# 1. local
---target local            # --mcp-url defaults to the local Publisher
-
-# 2. hosted via the extension's bridge -- no OAuth, but check what it exposes:
-#    the same proxy may front a local Publisher instead
---target platform --mcp-url http://localhost:<port-the-extension-prints>/mcp \\
-  --hosted-mcp-server <name> --target-version <v> --scope <env>/<pkg>@<v>
-
-# 3. hosted directly -- authenticate first, under the SAME server name
-claude mcp add --transport http <name> <scoped-url>
-claude   # /mcp -> <name> -> Authenticate
---target platform --mcp-url <scoped-url> \\
-  --hosted-mcp-server <name> --target-version <v> --scope <env>/<pkg>@<v>
-\`\`\`
-
-All three hand the answerer the same three capabilities (\`get_context\`,
-\`execute_query\`, and the docs search), so a comparison between them is between
-agents that could do the same things; \`test_the_two_arms_hold_the_same_capabilities\`
-pins it. Modes 2 and 3 take those names from \`--hosted-tools\`, which defaults to
-the bare trio; pass it only if this host names them differently.
-
-A platform run left on the local default \`--mcp-url\` is refused rather than
-probed, because pointing every answerer at a local Publisher measures a
-different model over different data than the run claims.
-
-Two rules follow, and both are the kind of mistake that produces confident
-nonsense rather than an error:
-
-- **The answerer and the conductor must hit the same target.** If the answerer
-  queries the published model and you re-execute its query against your edited
-  local copy, the score describes neither. Decide the target before the first
-  question and record it.
-- **Pin the version the target actually served, not the one you happen to have.**
-  A local target pins a commit; a platform target pins the published version.
-  Recording a local commit for a run that queried a published model is a pin
-  that means nothing.
-
-Which target for which job:
-
-- **Baseline what customers experience:** Remote. It is the deployed model
-  through the deployed engine, which is the thing they actually hit. The judge
-  sees no re-executed rows on a Remote run (there is no local copy of the
-  bytes), so its verdicts rest on the answer text and the golden; say so.
-- **Improve and accept:** local, because the acceptance check needs compile,
-  reload, and a fresh re-answer between edits. Publishing to a customer
-  environment to score an edit is not something this loop does. Where the host
-  offers draft execution, that counts as local for this purpose.
-- **Measure real data without touching production:** local proxied.
-
-So a measure-only run can use any target; a run that includes **improve** needs
-a local one.
-
-Two things to check before a platform run, because neither errors and both make
-the run measure something other than what it names:
-
-- **The answerer's skills must be written for THIS host.** A shared skill names
-  an MCP tool by its bare name (\`get_context\`) so it reads correctly anywhere,
-  but a host/router skill names its own host's tools directly. Install the
-  latter for the wrong host and the answerer is told to call tools it does not
-  have. \`run_baseline.py\` warns when the manifest it loaded names Publisher-only
-  tools on a platform target; point \`--answerer-manifest\`, or \`--skills-root\`,
-  at the checkout that ships this host's manifest.
-- **The tool names are configuration.** \`--hosted-mcp-server\` is both the
-  \`mcp__<server>__<tool>\` prefix and the OAuth cache key, so it has to match the
-  name the answerer authenticated under, and \`--hosted-tools\` lists the bare
-  tools that host exposes.
-- **Get the hosted tools in front of a headless answerer, one of two ways.**
-  A spawned answerer cannot complete an OAuth flow, so the tools have to be
-  reachable before the run starts. \`run_baseline.py\` proves it with one cheap
-  probe and refuses to spend an arm otherwise -- a run whose answerers have no
-  tools does not error, it reads as a terrible model.
-
-  1. **Authenticate once, interactively.** Works anywhere, including a plain
-     CLI install, and is the route to assume unless you know otherwise. The
-     token is cached per server NAME, so authenticate under the same name the
-     run passes to \`--hosted-mcp-server\`:
-
-     \`\`\`bash
-     claude mcp add --transport http <name> <scoped-url>
-     claude          # then /mcp -> <name> -> Authenticate
-     \`\`\`
-
-     Then come back and run. This is a hand-off to a person; there is no
-     headless equivalent, so plan for it rather than discovering it mid-run.
-
-  2. **A local proxy that already holds the credential.** Some hosts ship an
-     editor extension whose local MCP proxy can expose the hosted
-     \`get_context\` / \`execute_query\` -- often behind a setting that is off by
-     default. Where that exists, point \`--mcp-url\` at the proxy on localhost
-     and no OAuth step is needed, because the extension holds it. Check what
-     the proxy actually exposes before relying on it: the same proxy may serve
-     a local Publisher's tools instead, and then \`--hosted-tools\` is
-     naming tools that are not there. This route is not available to someone
-     running the CLI alone.
-
-- **Prefer a SCOPED endpoint URL over asking for scope.** A hosted MCP is
-  usually reachable two ways: a global endpoint where every call carries an
-  organization and workspace, and a scoped one where the URL itself is the
-  scope. \`--scope\` and the prompt can only ASK an answerer to stay in one
-  package; a scoped URL enforces it. For an agent being measured that is the
-  difference between a case answered against the package it names and one
-  answered against whatever else the account can see. Authenticate once
-  interactively (\`claude\`, \`/mcp\`) under the same server name the run will use;
-  the token is cached per name, and a spawned headless answerer cannot complete
-  an OAuth flow.
-
-- **Pin the VERSION in the scope, not just the package.** Write
-  \`--scope <env>/<package>@<version>\`. Both hosted tools take a version and both
-  document the same default for an omitted one: the PINNED version, which is
-  whatever the workspace serves at the moment of the call. So an unversioned run
-  records \`targetVersion\` in \`run.json\` and then answers from whatever is
-  current, and the two part company the moment anyone publishes -- including
-  mid-run, which measures two builds under one label. \`--target-version\` fills
-  the version in when the scope omits it, so a platform run is pinned without
-  opting in; a scope naming a different version is refused rather than taken as
-  an override.
-
-## Before you start
-
-1. The model package under evaluation must live in a git repository, with
-   \`evals/<set>/\` in the package, beside the model files. Git is the checkpoint
-   mechanism; without it there is no rollback and no run can include improve.
-
-   Keeping the set IN the package is what stops a model edit and its answer key
-   drifting apart: they move in one commit, so fixing a measure and forgetting
-   the golden that depended on it stops being possible. It is safe -- measured
-   on a running server, a \`cases.jsonl\` inside a package appears in no model
-   listing, no notebook listing, no package resource, and 404s over HTTP, so an
-   MCP-only answerer has no route to it.
-
-   What it buys differs by target. On a LOCAL Publisher it does not get you free
-   versioning -- \`sourceContentSha\` hashes model paths only, so the set needs
-   its own \`datasetSha\`. On a hosted target that publishes the whole package
-   directory as an IMMUTABLE version, the set rides inside that version and
-   \`targetVersion\` pins model and answer key together; nothing can be edited
-   under a published version, which is what makes it a pin. Check which you have
-   before deciding how much of this you need.
-
-   **Look for a set and for prior runs before you author either.** A minute of
-   \`find . -name cases.jsonl\`, a glance at \`evals/*/runs/\` and at your host's
-   own transcripts for this repo. Two sessions fourteen minutes apart built the
-   same 29-case answer key from scratch, because the first had committed
-   nothing before it was deleted and the second had no way to know it existed.
-   Roughly a working day was spent twice, and three specific things were
-   rediscovered at cost: the MCP login flow, the retrieval gate's 401, and a
-   judge-rendering bug that had already cost $17 of arm once. The two keys,
-   independently authored from the same questions, differ by about ten points
-   on comparable answers -- which is the available measure of how much a key
-   depends on its author, and a reason to reuse one rather than rebuild it.
-
-   **Audit the set's entity ids before the first arm.** \`verify_goldens.py\`
-   checks that each id names something in the model; \`check_findable.py\` checks
-   that a search of its own kind actually returns it. Both are free of model
-   calls. An id that fails either scores a retrieval miss on every run, and the
-   miss reads as the model's fault.
-
-   **Commit the set before you spend money on an arm**, and keep durable
-   outputs in the repository. A findings document in \`~/Downloads\` is gone the
-   first time somebody tidies up; the set, the run directory and the write-up
-   belong in git beside the model.
-
-2. The server must be up. Where your host offers retrieval tracing, turn it on
-   and confirm a trace lookup answers, so a call's ranked results can be
-   recovered afterwards.
-
-   **Open-source Publisher has none.** \`PUBLISHER_MCP_TRACE=retrieval\` is set by
-   \`serve.py\` and read by nothing: the trace store was written and never merged,
-   so there is no trace tool and \`traceId\` is null on every local attempt. This
-   does not block a scored run, because attribution never depended on it:
-   \`rankedSummary\` is copied onto the \`tool_call\` event at capture, precisely so
-   the evidence survives without a store. So do not refuse a local run for want
-   of tracing -- an earlier version of this rule did, and it refused every local
-   run there has ever been. Refuse one whose \`tool_call\` events carry no
-   \`rankedSummary\`, which is what attribution actually reads.
-
-3. Health-check: your host's status check until it reports serving, and inspect
-   \`loadErrors\`. A dead database that still answers HTTP is an environment
-   failure, not a model failure. Stop and fix it. Four consecutive
-   environment or no-result attempts means stop the run.
-
-4. Load the set: scrape/import as above, or reuse an existing \`evals/<set>/\`.
-   Never keep two live copies of one set; the set directory in the model repo
-   is the single source of truth, versioned by \`datasetVersion\` in
-   \`set.json\`.
-
-5. Review goldens before you score. **Check how many cases can take a verdict
-   at all**, not just how many cases there are: a golden the set stamps
-   \`provisional\`, \`invalid\` or \`ambiguous\`, or a case with no golden, scores
-   \`verdict: null\` and stays out of the pass rate. An imported set is
-   \`provisional\` throughout by design (\`skill:eval-import\`), and the only thing
-   that changes that is \`verify_goldens.py --promote\` after a re-derivation
-   through the truth package. A run whose every key is underived is refused
-   rather than spent; a set of bare questions runs, because its answers are
-   what keys get derived from. A verified golden that holds a value but
-   no local artifact stays verified by provenance and is not scorable until you
-   have rows or a scalar to compare (the judge needs both sides). A \`criteria\`
-   golden is the exception: it holds no value, so its clauses are both sides. If diagnosis later marks
-   \`BAD-REFERENCE\` or \`AMBIGUOUS-REFERENCE\`, follow
-   \`reference/golden-side-door.md\`. Both are expected in the wild; both
-   are the golden side door below, not improve, and not a sixth step.
-
-6. Create \`runs/<runId>/run.json\` with the attribution pins
-   (\`reference/ledger-schema.md\`): mode, dataset version, **the target and the
-   version it served** (a local target pins a commit, so commit or stash first;
-   answering from a dirty tree pins nothing), server version, judge version and
-   rubric sha, answerer model, call budget, trace mode. Freeze those for the
-   whole run. Raising a call budget mid-run moved mean outcomes on an unchanged
-   model.
-
-   The call budget is \`--max-turns\`, written to \`run.json\` as \`maxTurns\`.
-   **Size it from a pilot rather than taking the default of 30.** Run the three
-   cheapest cases uncapped (\`--max-turns 100\`) and set the cap at twice their
-   maximum. A set whose questions need two or three sources joined does not fit
-   a cap sized for single-source lookups: on one such set the completed
-   attempts had a median of 17 turns and a 90th percentile of 26 against a cap
-   of 30, and four cases died at it. A cap 15% above the 90th percentile of
-   completed work is not a safety margin. Note also that \`malloy-analysis\` tells
-   the answerer to persist -- retry a phrasing, let a small query settle whether
-   a field exists -- so a tight cap and that instruction are in direct conflict.
-
-   **Do not change the model and the measuring instrument in the same step.**
-   Those are two different axes and only one of them is cheap to separate.
-
-   Batching MODEL edits is fine and expected. Clustering exists so that one
-   edit closes several cases, and an arm per fix does not survive contact with
-   arithmetic: 20 fixes over 100 cases is 2,000 answers, and at the measured
-   $0.33 a case on a proxied warehouse that is $660 of answering to attribute
-   what the acceptance check attributes for the price of the affected cases
-   plus holdout. Budget five arms for a defensible claim (a baseline, two for
-   the A/A, two post-edit), not one per edit, and let \`skill:eval-improve\`
-   carry each cluster.
-
-   The instrument is the other axis: the answer key, the judge and its prompt,
-   the answerer model, the skills. Move one of those together with the model
-   and there is nothing left holding still, so the result measures neither. On
-   the run this comes from, nine model commits and a re-derived answer key
-   landed between two arms, and the move from 20% to 39% belongs to no one --
-   not because two model edits were batched, but because the ruler changed at
-   the same time as the thing being measured. A key repair mid-improve is not
-   forbidden; it ends that comparison, so re-baseline rather than quoting a
-   delta across it.
-
-   The answerer model is instrument too, and it is the one most often left
-   unstated. The same 23-case set read 12 match / 11 near / 0 no_match on one
-   model and 8 / 4 / 15 on a smaller one. No statement about "the agent's"
-   capability means anything until two arms name the same answerer.
-
-7. Generate every answerer prompt from the stored case in \`cases.jsonl\`.
-   Never retype the question. A truncated retype is indistinguishable from a
-   real question downstream.
-
-## Per question
-
-1. Health-check again.
-2. Spawn a *fresh* blind subagent. Give it only the question text and the
-   Malloy analysis tools. Tell it to follow the \`malloy-analysis\` skill. Do not
-   mention eval, gold, scoring, or this skill.
-3. Keep a host-side tool-use log for that subagent (name, input path or
-   command, MCP tool name). Publisher traces see MCP only; a Read of a gold
-   CSV is invisible server-side.
-4. \`skill:eval-answer\`: contamination first, then re-execute, then the judge,
-   then events.
-5. \`skill:eval-diagnose\` only when this run includes diagnose, only on dev
-   cases, and only after the score event exists.
-6. \`skill:eval-improve\` only when this run includes improve, and only for
-   \`owner: model\`. Then run the acceptance check. On accept, checkpoint.
-
-## Report the run, or nobody can read it
-
-A run directory is JSONL. It is a record, not a result, and the console summary
-scrolls away. **Every run ends by producing something a person can open**, and
-that job is \`skill:eval-report\`: it builds the servable run package (the case
-matrix app and the aggregate notebook) and gives the template for the write-up.
-
-Read it at the end of every run, including a run that failed. The standing
-complaint about this loop is that "a bunch of stuff happens and it is hard to
-know the actual results", and a ledger nobody renders is why.
-
-Two rules from it are worth repeating here, because they are the ones a
-conductor skips:
-
-- **Separate a MODEL failure from an EVAL failure.** A wrong answer and a
-  broken measurement look identical in a pass rate and have nothing else in
-  common. An arm holding a truncated, contaminated or environment-failed
-  attempt has no rate to quote at all.
-- **Say when a step did not run, and why.** \`improve\` not running because every
-  cluster came back \`owner: agent-skill\` is a RESULT, and it reads identically
-  to having forgotten unless it is written down.
-
-Keep the write-up in the repository beside the set, not in a chat log and not
-in \`~/Downloads\`.
-
-## Checkpoint
-
-A checkpoint is a git commit of the model repository, taken after an acceptance check
-accepts, so a bad improve direction can be rolled back. It is not a report,
-and it is not a remote publish.
-
-1. Commit the model files AND the set's ledger in one commit; put the label
-   and the closed issue ids in the message.
-2. Append the \`checkpoint\` event (\`action: created\`, label, \`modelGitSha\`
-   from the commit you just made, issueIds). The event line itself rides in
-   the next commit; append-only logs trail by one commit and that is fine.
-3. Confirm \`git status\` is clean for the model files.
-
-**Restore**: \`git checkout <sha> -- <model files>\` (or \`git revert\` the
-checkpoint commits), then reload the package, then append a \`checkpoint\`
-event with \`action: restored\` and the sha. Readers return to the model that
-existed before the bad direction.
-
-Take a checkpoint of the current model *before* the first improve batch if no
-commit pins it yet. Rolling back by hand is guesswork.
-
-If reload reports \`mode: reinstalled\`, the package was re-fetched from its
-install location and may have overwritten the restored files. Prefer in-place
-/ watch-mounted packages for this loop.
-
-## Out of scope
-
-This loop is local. The ledger is files, the checkpoints are git, you are the
-conductor. Do not:
-
-- publish the model to a hosted platform as a "true" checkpoint or learning
-  curve
-- start a Python orchestrator (\`loop.py\`, \`run_all.py\`, \`improve_batch.py\`)
-  that runs the five steps end to end unattended. You conduct; the scripts are
-  the steps, not the sequencing. There are more than twenty of them and they
-  are not the exception to this: each does one step you invoke and hands back a
-  result you read. What is forbidden is a script that decides what to do next,
-  because every judgement this loop protects lives in that decision
-- score by string-diffing rows instead of judging them, or reintroduce a
-  scripted row oracle: one that can pass a wrong answer is worse than none
-- wait for a bigger gold set before the loop can run; dev/holdout on what
-  exists beats waiting
-- register eval MCP tools or stand up an eval API
-- encode unsettled goldens into the model
-
-## Prime directives
-
-- The model is the only thing improve edits. No question text, qids, or
-  expected values in any name, doc, or comment.
-- **You are measuring a model, not reviewing this harness.** Read a script when
-  a number you have to report cannot be explained otherwise, and stop there. Do
-  not audit the scripts, propose fixes to them, or hand back tooling critique
-  in place of a result -- a run that ends in harness feedback has not answered
-  the question it was asked. A defect that changed a number gets one sentence
-  in the report; a defect that changed nothing gets one line in chat, once.
-  Fixing it is a different task, and the user starts it.
-- When the environment misbehaves, stop. Never diagnose a sick system. The
-  harness is part of the environment: a known-broken measurement does not
-  become quotable by being finished. Measured against this directive, an arm
-  already paid for gets quoted anyway -- it happened three times in one run,
-  once with the confound stated in the same message that started the arm -- so
-  the harness now enforces the parts it can. A run with a truncated or
-  contaminated attempt prints no pass rate and records \`status: incomplete\`,
-  and \`flip_table.py\` names what one arm left unscored that the other did not.
-  Read \`run_error\` on the attempts
-  and the INCOMPLETE line before quoting any number.
-- When a subagent disagrees with you, probe. Do not win by authority.
-- When a rule here is wrong, change this file and note it on the run.
-
-## Related skills
-
-- \`skill:eval-answer\`: contamination, judge protocol, events. Its
-  \`reference/ledger-schema.md\` is the file contract; \`skill:eval-judge\` is
-  the judge.
-- \`skill:eval-diagnose\`: component, owner, issue events. No edit.
-- \`skill:eval-improve\`: smallest model edit, probe receipts, no self-accept.
-- \`skill:eval-report\`: the run package and the write-up a person reads.
-- The \`malloy-analysis\` skill: what the blind answerer follows. It is installed
-  from the \`analysis\` manifest group, not the \`eval\` group.
-
-## Reference files over MCP
-
-This skill's \`reference/\` files are served as separate prompts, one per file, fetched only when you ask for them. Where the text above says to read \`reference/<name>.md\`, get the prompt named \`eval-loop/<name>\` instead.
-
-Available: acceptance-check, auditing-an-answer-key, checking-the-judge, golden-side-door, measurement, running-a-run.`
+      description: "Conduct a local Publisher evaluation loop in five steps: scrape/run, eval, diagnose, improve, checkpoint. You are the conductor: import cases into the file ledger, spawn a blind answerer, then run eval-answer, eval-diagnose, and eval-improve. Persistence is plain files: the set in the model package''s evals/ directory, runs in the set''s workdir; checkpoints are git commits of the model repo. `eval.py` runs each step from the set''s eval.toml. Use to score a model, diagnose failures, improve behind an acceptance check, or roll back a bad direction.",
+      body: "# The Evaluation Loop\n\nYou conduct this loop. There is no batch orchestrator to start, no eval API,\nand no eval MCP tools. The ledger is plain files: the set in the model\npackage's git repository, and each run in the set's workdir\n(`reference/ledger-schema.md` in `skill:eval-answer` defines every file and\nevent). `scripts/eval.py` runs each step from the set's `eval.toml`;\n`reference/running-a-run.md` has the commands. Scoring is an LLM judge you spawn per case. There is no\nscripted scorer, and there will not be one: a script that can pass a wrong\nanswer is worse than none. The scripts under `scripts/` run the loop -- they\nanswer, re-execute, spawn the judge, compare runs, and write the ledger -- but\nnone of them decides whether an answer was right.\n\n```\nscrape/run  ->  eval  ->  diagnose  ->  improve  ->  checkpoint\n```\n\n**This skill conducts; it does not restate.** Scoring lives in\n`skill:eval-answer`. Components and owners live in `skill:eval-diagnose`.\nEdit rules live in `skill:eval-improve`.\n\nDo not merge **eval** into **diagnose**. A conductor who scores while\nexplaining writes the explanation into the score. Do not skip the **acceptance\ncheck** inside improve. The acceptance check decides whether *this* edit\nstays. **Checkpoint** decides whether a *sequence* of accepted edits can be\nundone.\n\n## Where the rest of this lives\n\nThis file is the procedure. The things it used to carry inline are files beside\nit now, because each is needed at one moment rather than every run, and loading\nall of them for every run is how a skill stops being read.\n\n| When | Read |\n|---|---|\n| the set does not exist yet, or has never run | `reference/setting-up-a-set.md` |\n| about to run one | `reference/running-a-run.md` |\n| a golden is wrong, doubted, or out of step with the model | `reference/golden-side-door.md` |\n| auditing a key you doubt, or a set you did not author | `reference/auditing-an-answer-key.md` |\n| deciding whether an edit stays | `reference/acceptance-check.md` |\n| about to quote a number, set the band, or read a set's flips | `reference/measurement.md` |\n| the run finished and someone has to read it | `skill:eval-report` |\n| you changed judge doctrine or its inputs | `reference/checking-the-judge.md` |\n\nRead the file, do not work from the summary here. The acceptance-check rules and\nthe golden side door are both places where acting on a half-memory of the rule\nproduces a confident wrong answer rather than an error.\n\n## The five steps\n\n| Step | Job | Writes |\n|---|---|---|\n| **a. scrape / run** | Put cases in the ledger; spawn a blind answerer | cases; `attempt`, `tool_call` |\n| **b. eval** | Judge the answer; score which required entities retrieval delivered | `score` |\n| **c. diagnose** | Why it failed, who owns it | `issue` / `issue_status`. Stop. Do not edit. |\n| **d. improve** | One smallest model edit, then the acceptance check | improve writes `candidate`; you write `acceptance_check`. Revert on reject. |\n| **e. checkpoint** | Git commit after an accepted acceptance check | `checkpoint` event, then the commit |\n\nEvery run then **reports** (below). A run whose result exists only as JSONL and\na scrolled-away console summary has not been delivered.\n\n**scrape** and **run** share a letter but are not the same job. Scrape writes\ncases. Run writes attempts. Do not invent questions and score\nthem in one breath.\n\n**Step d is `skill:eval-improve`, not a hand edit plus another arm.** The\nfailure mode is specific and it has happened: the edits were made by hand,\npublished, and validated by re-running the whole arm -- with the answer key\nrepaired in the same window. No `candidate`, `acceptance_check` or `checkpoint`\nevent exists in that run's ledger, and the resulting move cannot be attributed\nto the model, the key, or the answerer. Re-running a whole arm after changing\ntwo things is the one thing the acceptance check exists to replace: it scores\nthe edit against dev AND holdout, and it is cheaper than the arm.\n\n### Scrape, minimally\n\nImporting an existing corpus IS the scrape step: copy the set from its home\n(for example a benchmarks checkout) into `evals/<set>/` and convert to the\nledger shapes. **`skill:eval-import` is that job in full** -- how to classify\nwhat arrived with each question, why nothing imports as a verified value, and\nthe seal that makes a later edit to a question detectable. Read it whenever the\nquestions came from outside, which is most of the time. While importing:\n\n- Freeze each case's `split`: `dev` or `holdout`. Diagnose and improve read\n  dev cases only; the acceptance check runs both. A set that is all dev cannot defend an\n  accept.\n- Later, each diagnosed-and-fixed failure becomes a new frozen dev case, so a\n  fixed bug cannot silently return.\n\nScraping from production logs (chat transcripts, retrieval traces) is the\nother supported source, and usually the better one: real traffic asks what\npeople actually ask. Where your logs physically live is a host concern; look\nfor a host-specific log-fetching skill. `skill:eval-import` takes over once\nyou have the text, and its `reference/case-format.md` covers what a log pull\nneeds that a question list does not.\n\nPrefer variety over volume when you sample, from either source. Cases that\ndiffer in grain, source, filter shape, and phrasing are what move a\nmeasurement; a second sample of the same case is nearly free of new\ninformation.\n\n### Mode aliases\n\nOlder mode names still work as aliases for how far one run walks:\n\n| Alias | Steps |\n|---|---|\n| `measure` | scrape/run + eval (`eval.py package` then needs `--without-diagnosis`) |\n| `triage` | plus diagnose |\n| `improve` | plus improve + acceptance check + checkpoint on accept |\n\nSay which alias (or which steps) you are running before the first question.\nRecord it in `run.json`. Do not mix steps in a way that lets the answerer see\ngold, issues, or the model file.\n\nMost runs should stop after eval. Diagnose when you need a histogram of\ncomponents and owners. Improve only for diagnosed *model* gaps, one batch at\na time. Checkpoint only after the acceptance check **accepts**.\n\n## Roles\n\n| Role | Sees |\n|---|---|\n| **Answerer** | The question and the Malloy tools. Never the golden, `evals/`, the model file, or any hint it is being evaluated. |\n| **Judge** | The golden and the prediction. Never conducts, never answers, never edits. One fresh subagent per verdict (`skill:eval-judge`). |\n| **You (conductor / improver)** | Everything, including goldens and traces. |\n| **Acceptance check** | The edit and the evidence. Never the improver's self-assessment alone. |\n\nThe answerer stays blind. That is not optional. A grader-visible answerer\nwrites toward the expected answer, and the score is fiction.\n\nThere are no eval MCP tools on purpose. The answerer inherits your tools,\nincluding Shell and Read, so any eval convenience surface would also be a\ngold path for it. Blindness is prevention plus detection, not a guarantee:\n`eval-answer` runs the contamination checklist on every attempt, which is\nwhy you keep a host-side tool-use log per answerer.\n\n## Pick the target first\n\nBoth a local model server and a hosted platform expose the same two tools the\nanswerer needs, `get_context` and `execute_query`, so the loop runs against\neither. What differs is which model is answering and whose data it reads, and\nthose are two separate axes:\n\n| Target | Model under test | Data | Can edit and re-test? |\n|---|---|---|---|\n| **Local (direct)** | your working files | local (for example duckdb), or a direct warehouse connection | yes |\n| **Local (proxied)** | your working files | the platform's connection, through a proxy connection type | yes |\n| **Remote** | the published version, through the platform's hosted `get_context`/`execute_query` | the platform's | no, publishing is not an eval action |\n\nThe middle row is the one worth knowing about: it decouples the two axes, so you\ncan evaluate a model you are still editing against the customer's real data. It\nis a connection configuration, not a feature.\n\n### Three ways to reach the tools\n\nThat table is about which MODEL answers. A second, independent choice is where\nthe answerer's MCP tools come from, and `--mcp-url` is the whole of it:\n\n| | `--target` | `--mcp-url` | Auth |\n|---|---|---|---|\n| **1. Local Publisher** | `local` | `http://localhost:4040/mcp` (default) | none |\n| **2. Hosted, through an editor extension's local bridge** | `platform` | the localhost URL the extension prints | none: the extension holds the credential |\n| **3. Hosted, directly** | `platform` | the host's `https` endpoint, scoped if it offers one | a cached OAuth login, once, interactively |\n\n```bash\n# 1. local\n--target local            # --mcp-url defaults to the local Publisher\n\n# 2. hosted via the extension's bridge -- no OAuth, but check what it exposes:\n#    the same proxy may front a local Publisher instead\n--target platform --mcp-url http://localhost:<port-the-extension-prints>/mcp \\\n  --hosted-mcp-server <name> --target-version <v> --scope <env>/<pkg>@<v>\n\n# 3. hosted directly -- authenticate first, under the SAME server name\nclaude mcp add --transport http <name> <scoped-url>\nclaude   # /mcp -> <name> -> Authenticate\n--target platform --mcp-url <scoped-url> \\\n  --hosted-mcp-server <name> --target-version <v> --scope <env>/<pkg>@<v>\n```\n\nAll three hand the answerer the same three capabilities (`get_context`,\n`execute_query`, and the docs search), so a comparison between them is between\nagents that could do the same things; `test_the_two_arms_hold_the_same_capabilities`\npins it. Modes 2 and 3 take those names from `--hosted-tools`, which defaults to\nthe bare trio; pass it only if this host names them differently.\n\nA platform run left on the local default `--mcp-url` is refused rather than\nprobed, because pointing every answerer at a local Publisher measures a\ndifferent model over different data than the run claims.\n\nTwo rules follow, and both are the kind of mistake that produces confident\nnonsense rather than an error:\n\n- **The answerer and the conductor must hit the same target.** If the answerer\n  queries the published model and you re-execute its query against your edited\n  local copy, the score describes neither. Decide the target before the first\n  question and record it.\n- **Pin the version the target actually served, not the one you happen to have.**\n  A local target pins a commit; a platform target pins the published version.\n  Recording a local commit for a run that queried a published model is a pin\n  that means nothing.\n\nWhich target for which job:\n\n- **Baseline what customers experience:** Remote. It is the deployed model\n  through the deployed engine, which is the thing they actually hit. The judge\n  sees no re-executed rows on a Remote run (there is no local copy of the\n  bytes), so its verdicts rest on the answer text and the golden; say so.\n- **Improve and accept:** local, because the acceptance check needs compile,\n  reload, and a fresh re-answer between edits. Publishing to a customer\n  environment to score an edit is not something this loop does. Where the host\n  offers draft execution, that counts as local for this purpose.\n- **Measure real data without touching production:** local proxied.\n\nSo a measure-only run can use any target; a run that includes **improve** needs\na local one.\n\nTwo things to check before a platform run, because neither errors and both make\nthe run measure something other than what it names:\n\n- **The answerer's skills must be written for THIS host.** A shared skill names\n  an MCP tool by its bare name (`get_context`) so it reads correctly anywhere,\n  but a host/router skill names its own host's tools directly. Install the\n  latter for the wrong host and the answerer is told to call tools it does not\n  have. `run_baseline.py` warns when the manifest it loaded names Publisher-only\n  tools on a platform target; point `--answerer-manifest`, or `--skills-root`,\n  at the checkout that ships this host's manifest.\n- **The tool names are configuration.** `--hosted-mcp-server` is both the\n  `mcp__<server>__<tool>` prefix and the OAuth cache key, so it has to match the\n  name the answerer authenticated under, and `--hosted-tools` lists the bare\n  tools that host exposes.\n- **Get the hosted tools in front of a headless answerer, one of two ways.**\n  A spawned answerer cannot complete an OAuth flow, so the tools have to be\n  reachable before the run starts. `run_baseline.py` proves it with one cheap\n  probe and refuses to spend an arm otherwise -- a run whose answerers have no\n  tools does not error, it reads as a terrible model.\n\n  1. **Authenticate once, interactively.** Works anywhere, including a plain\n     CLI install, and is the route to assume unless you know otherwise. The\n     token is cached per server NAME, so authenticate under the same name the\n     run passes to `--hosted-mcp-server`:\n\n     ```bash\n     claude mcp add --transport http <name> <scoped-url>\n     claude          # then /mcp -> <name> -> Authenticate\n     ```\n\n     Then come back and run. This is a hand-off to a person; there is no\n     headless equivalent, so plan for it rather than discovering it mid-run.\n\n  2. **A local proxy that already holds the credential.** Some hosts ship an\n     editor extension whose local MCP proxy can expose the hosted\n     `get_context` / `execute_query` -- often behind a setting that is off by\n     default. Where that exists, point `--mcp-url` at the proxy on localhost\n     and no OAuth step is needed, because the extension holds it. Check what\n     the proxy actually exposes before relying on it: the same proxy may serve\n     a local Publisher's tools instead, and then `--hosted-tools` is\n     naming tools that are not there. This route is not available to someone\n     running the CLI alone.\n\n- **Prefer a SCOPED endpoint URL over asking for scope.** A hosted MCP is\n  usually reachable two ways: a global endpoint where every call carries an\n  organization and workspace, and a scoped one where the URL itself is the\n  scope. `--scope` and the prompt can only ASK an answerer to stay in one\n  package; a scoped URL enforces it. For an agent being measured that is the\n  difference between a case answered against the package it names and one\n  answered against whatever else the account can see. Authenticate once\n  interactively (`claude`, `/mcp`) under the same server name the run will use;\n  the token is cached per name, and a spawned headless answerer cannot complete\n  an OAuth flow.\n\n- **Pin the VERSION in the scope, not just the package.** Write\n  `--scope <env>/<package>@<version>`. Both hosted tools take a version and both\n  document the same default for an omitted one: the PINNED version, which is\n  whatever the workspace serves at the moment of the call. So an unversioned run\n  records `targetVersion` in `run.json` and then answers from whatever is\n  current, and the two part company the moment anyone publishes -- including\n  mid-run, which measures two builds under one label. `--target-version` fills\n  the version in when the scope omits it, so a platform run is pinned without\n  opting in; a scope naming a different version is refused rather than taken as\n  an override.\n\n## Before you start\n\n1. The model package under evaluation must live in a git repository, with\n   `evals/<set>/` in the package, beside the model files. Git is the checkpoint\n   mechanism; without it there is no rollback and no run can include improve.\n\n   Keeping the set IN the package is what stops a model edit and its answer key\n   drifting apart: they move in one commit, so fixing a measure and forgetting\n   the golden that depended on it stops being possible. It is safe -- measured\n   on a running server, a `cases.jsonl` inside a package appears in no model\n   listing, no notebook listing, no package resource, and 404s over HTTP, so an\n   MCP-only answerer has no route to it.\n\n   What it buys differs by target. On a LOCAL Publisher it does not get you free\n   versioning -- `sourceContentSha` hashes model paths only, so the set needs\n   its own `datasetSha`. On a hosted target that publishes the whole package\n   directory as an IMMUTABLE version, the set rides inside that version and\n   `targetVersion` pins model and answer key together; nothing can be edited\n   under a published version, which is what makes it a pin. Check which you have\n   before deciding how much of this you need.\n\n   **Look for a set and for prior runs before you author either.** A minute of\n   `find . -name cases.jsonl`, a glance at the set's workdir (`<workdir>/runs/`,\n   `~/.malloy-eval/<set>-<hash>/runs/` unless `eval.toml` moves it) and at your host's\n   own transcripts for this repo. Two sessions fourteen minutes apart built the\n   same 29-case answer key from scratch, because the first had committed\n   nothing before it was deleted and the second had no way to know it existed.\n   Roughly a working day was spent twice, and three specific things were\n   rediscovered at cost: the MCP login flow, the retrieval gate's 401, and a\n   judge-rendering bug that had already cost $17 of arm once. The two keys,\n   independently authored from the same questions, differ by about ten points\n   on comparable answers -- which is the available measure of how much a key\n   depends on its author, and a reason to reuse one rather than rebuild it.\n\n   **Audit the set's entity ids before the first arm.** `verify_goldens.py`\n   checks that each id names something in the model; `check_findable.py` checks\n   that a search of its own kind actually returns it. Both are free of model\n   calls. An id that fails either scores a retrieval miss on every run, and the\n   miss reads as the model's fault.\n\n   **Commit the set before you spend money on an arm**, and keep durable\n   outputs in the repository. A findings document in `~/Downloads` is gone the\n   first time somebody tidies up; the set and the write-up belong in git\n   beside the model.\n\n   **Runs go in the set's workdir, never inside the model package.** A run\n   directory holds a `model.malloy` snapshot, and a built report is a Malloy\n   package; inside the package under test, either puts that package into\n   `loadErrors`. The default workdir, `~/.malloy-eval/<set>-<hash>/`, is outside git,\n   which suits a measure-only run. A run that will improve and checkpoint needs\n   its ledger kept: set `[paths] workdir` in `eval.toml` to a directory in the\n   model's repository but outside the package, and gitignore its `servers/`\n   and `packages/`, which are a server's database and rebuilt reports.\n\n2. The server must be up. Where your host offers retrieval tracing, turn it on\n   and confirm a trace lookup answers, so a call's ranked results can be\n   recovered afterwards.\n\n   **Open-source Publisher has none.** `PUBLISHER_MCP_TRACE=retrieval` is set by\n   `serve.py` and read by nothing: the trace store was written and never merged,\n   so there is no trace tool and `traceId` is null on every local attempt. This\n   does not block a scored run, because attribution never depended on it:\n   `rankedSummary` is copied onto the `tool_call` event at capture, precisely so\n   the evidence survives without a store. So do not refuse a local run for want\n   of tracing -- an earlier version of this rule did, and it refused every local\n   run there has ever been. Refuse one whose `tool_call` events carry no\n   `rankedSummary`, which is what attribution actually reads.\n\n3. Health-check: your host's status check until it reports serving, and inspect\n   `loadErrors`. A dead database that still answers HTTP is an environment\n   failure, not a model failure. Stop and fix it. Four consecutive\n   environment or no-result attempts means stop the run.\n\n4. Load the set: scrape/import as above, or reuse an existing `evals/<set>/`.\n   `eval.py check --set <set>` names every gap in it before anything starts.\n   Never keep two live copies of one set; the set directory in the model repo\n   is the single source of truth, versioned by `datasetVersion` in\n   `set.json`.\n\n5. Review goldens before you score. **Check how many cases can take a verdict\n   at all**, not just how many cases there are: a golden the set stamps\n   `provisional`, `invalid` or `ambiguous`, or a case with no golden, scores\n   `verdict: null` and stays out of the pass rate. An imported set is\n   `provisional` throughout by design (`skill:eval-import`), and the only thing\n   that changes that is `verify_goldens.py --promote` after a re-derivation\n   through the truth package. A run whose every key is underived is refused\n   rather than spent; a set of bare questions runs, because its answers are\n   what keys get derived from. A verified golden that holds a value but\n   no local artifact stays verified by provenance and is not scorable until you\n   have rows or a scalar to compare (the judge needs both sides). A `criteria`\n   golden is the exception: it holds no value, so its clauses are both sides. If diagnosis later marks\n   `BAD-REFERENCE` or `AMBIGUOUS-REFERENCE`, follow\n   `reference/golden-side-door.md`. Both are expected in the wild; both\n   are the golden side door below, not improve, and not a sixth step.\n\n6. `eval.py run` creates `<workdir>/runs/<runId>/run.json` with the attribution pins\n   (`reference/ledger-schema.md`): mode, dataset version, **the target and the\n   version it served** (a local target pins a commit, so commit or stash first;\n   answering from a dirty tree pins nothing), server version, judge version and\n   rubric sha, answerer model, call budget, trace mode. Freeze those for the\n   whole run. Raising a call budget mid-run moved mean outcomes on an unchanged\n   model.\n\n   The call budget is `--max-turns`, written to `run.json` as `maxTurns`.\n   **Size it from a pilot rather than taking the default of 30.** Run the three\n   cheapest cases uncapped (`--max-turns 100`) and set the cap at twice their\n   maximum. A set whose questions need two or three sources joined does not fit\n   a cap sized for single-source lookups: on one such set the completed\n   attempts had a median of 17 turns and a 90th percentile of 26 against a cap\n   of 30, and four cases died at it. A cap 15% above the 90th percentile of\n   completed work is not a safety margin. Note also that `malloy-analysis` tells\n   the answerer to persist -- retry a phrasing, let a small query settle whether\n   a field exists -- so a tight cap and that instruction are in direct conflict.\n\n   **Do not change the model and the measuring instrument in the same step.**\n   Those are two different axes and only one of them is cheap to separate.\n\n   Batching MODEL edits is fine and expected. Clustering exists so that one\n   edit closes several cases, and an arm per fix does not survive contact with\n   arithmetic: 20 fixes over 100 cases is 2,000 answers, and at the measured\n   $0.33 a case on a proxied warehouse that is $660 of answering to attribute\n   what the acceptance check attributes for the price of the affected cases\n   plus holdout. Budget five arms for a defensible claim (a baseline, two for\n   the A/A, two post-edit), not one per edit, and let `skill:eval-improve`\n   carry each cluster.\n\n   The instrument is the other axis: the answer key, the judge and its prompt,\n   the answerer model, the skills. Move one of those together with the model\n   and there is nothing left holding still, so the result measures neither. On\n   the run this comes from, nine model commits and a re-derived answer key\n   landed between two arms, and the move from 20% to 39% belongs to no one --\n   not because two model edits were batched, but because the ruler changed at\n   the same time as the thing being measured. A key repair mid-improve is not\n   forbidden; it ends that comparison, so re-baseline rather than quoting a\n   delta across it.\n\n   The answerer model is instrument too, and it is the one most often left\n   unstated. The same 23-case set read 12 match / 11 near / 0 no_match on one\n   model and 8 / 4 / 15 on a smaller one. No statement about \"the agent's\"\n   capability means anything until two arms name the same answerer.\n\n7. Generate every answerer prompt from the stored case in `cases.jsonl`.\n   Never retype the question. A truncated retype is indistinguishable from a\n   real question downstream.\n\n## Per question\n\n1. Health-check again.\n2. Spawn a *fresh* blind subagent. Give it only the question text and the\n   Malloy analysis tools. Tell it to follow the `malloy-analysis` skill. Do not\n   mention eval, gold, scoring, or this skill.\n3. Keep a host-side tool-use log for that subagent (name, input path or\n   command, MCP tool name). Publisher traces see MCP only; a Read of a gold\n   CSV is invisible server-side.\n4. `skill:eval-answer`: contamination first, then re-execute, then the judge,\n   then events.\n5. `skill:eval-diagnose` only when this run includes diagnose, only on dev\n   cases, and only after the score event exists.\n6. `skill:eval-improve` only when this run includes improve, and only for\n   `owner: model`. Then run the acceptance check. On accept, checkpoint.\n\n## Report the run, or nobody can read it\n\nA run directory is JSONL. It is a record, not a result, and the console summary\nscrolls away. **Every run ends by producing something a person can open**, and\nthat job is `skill:eval-report`: it builds the servable run package (the case\nmatrix app and the aggregate notebook) and gives the template for the write-up.\n\nRead it at the end of every run, including a run that failed. The standing\ncomplaint about this loop is that \"a bunch of stuff happens and it is hard to\nknow the actual results\", and a ledger nobody renders is why.\n\nTwo rules from it are worth repeating here, because they are the ones a\nconductor skips:\n\n- **Separate a MODEL failure from an EVAL failure.** A wrong answer and a\n  broken measurement look identical in a pass rate and have nothing else in\n  common. An arm holding a truncated, contaminated or environment-failed\n  attempt has no rate to quote at all.\n- **Say when a step did not run, and why.** `improve` not running because every\n  cluster came back `owner: agent-skill` is a RESULT, and it reads identically\n  to having forgotten unless it is written down.\n\nKeep the write-up in the repository beside the set, not in a chat log and not\nin `~/Downloads`.\n\n## Checkpoint\n\nA checkpoint is a git commit of the model repository, taken after an acceptance check\naccepts, so a bad improve direction can be rolled back. It is not a report,\nand it is not a remote publish.\n\n1. Commit the model files AND the set's ledger in one commit; put the label\n   and the closed issue ids in the message. The run's ledger is in that commit\n   only when the workdir is in the repository (Before you start, item 1).\n2. Append the `checkpoint` event (`action: created`, label, `modelGitSha`\n   from the commit you just made, issueIds). The event line itself rides in\n   the next commit; append-only logs trail by one commit and that is fine.\n3. Confirm `git status` is clean for the model files.\n\n**Restore**: `git checkout <sha> -- <model files>` (or `git revert` the\ncheckpoint commits), then reload the package, then append a `checkpoint`\nevent with `action: restored` and the sha. Readers return to the model that\nexisted before the bad direction.\n\nTake a checkpoint of the current model *before* the first improve batch if no\ncommit pins it yet. Rolling back by hand is guesswork.\n\nIf reload reports `mode: reinstalled`, the package was re-fetched from its\ninstall location and may have overwritten the restored files. Prefer in-place\n/ watch-mounted packages for this loop.\n\n## Out of scope\n\nThis loop is local. The ledger is files, the checkpoints are git, you are the\nconductor. Do not:\n\n- publish the model to a hosted platform as a \"true\" checkpoint or learning\n  curve\n- start a Python orchestrator (`loop.py`, `run_all.py`, `improve_batch.py`)\n  that runs the five steps end to end unattended. You conduct; the scripts are\n  the steps, not the sequencing. There are more than twenty of them and they\n  are not the exception to this: each does one step you invoke and hands back a\n  result you read. What is forbidden is a script that decides what to do next,\n  because every judgement this loop protects lives in that decision\n- score by string-diffing rows instead of judging them, or reintroduce a\n  scripted row oracle: one that can pass a wrong answer is worse than none\n- wait for a bigger gold set before the loop can run; dev/holdout on what\n  exists beats waiting\n- register eval MCP tools or stand up an eval API\n- encode unsettled goldens into the model\n\n## Prime directives\n\n- The model is the only thing improve edits. No question text, qids, or\n  expected values in any name, doc, or comment.\n- **You are measuring a model, not reviewing this harness.** Read a script when\n  a number you have to report cannot be explained otherwise, and stop there. Do\n  not audit the scripts, propose fixes to them, or hand back tooling critique\n  in place of a result -- a run that ends in harness feedback has not answered\n  the question it was asked. A defect that changed a number gets one sentence\n  in the report; a defect that changed nothing gets one line in chat, once.\n  Fixing it is a different task, and the user starts it.\n- When the environment misbehaves, stop. Never diagnose a sick system. The\n  harness is part of the environment: a known-broken measurement does not\n  become quotable by being finished. Measured against this directive, an arm\n  already paid for gets quoted anyway -- it happened three times in one run,\n  once with the confound stated in the same message that started the arm -- so\n  the harness now enforces the parts it can. A run with a truncated or\n  contaminated attempt prints no pass rate and records `status: incomplete`,\n  and `flip_table.py` names what one arm left unscored that the other did not.\n  Read `run_error` on the attempts\n  and the INCOMPLETE line before quoting any number.\n- When a subagent disagrees with you, probe. Do not win by authority.\n- When a rule here is wrong, change this file and note it on the run.\n\n## Related skills\n\n- `skill:eval-answer`: contamination, judge protocol, events. Its\n  `reference/ledger-schema.md` is the file contract; `skill:eval-judge` is\n  the judge.\n- `skill:eval-diagnose`: component, owner, issue events. No edit.\n- `skill:eval-improve`: smallest model edit, probe receipts, no self-accept.\n- `skill:eval-report`: the run package and the write-up a person reads.\n- The `malloy-analysis` skill: what the blind answerer follows. It is installed\n  from the `analysis` manifest group, not the `eval` group.\n\n## Reference files over MCP\n\nThis skill's `reference/` files are served as separate prompts, one per file, fetched only when you ask for them. Where the text above says to read `reference/<name>.md`, get the prompt named `eval-loop/<name>` instead.\n\nAvailable: acceptance-check, auditing-an-answer-key, checking-the-judge, golden-side-door, measurement, running-a-run, setting-up-a-set."
     },
     {
       name: "eval-loop/acceptance-check",
@@ -387328,25 +386791,58 @@ model score.`
 one fresh answerer per case with only the Publisher MCP tools, a contamination
 check, a judge, and a conformant \`events.jsonl\`.
 
+Every step reads the set's \`eval.toml\`: the model server's environment,
+package, repo and ports, and the truth server's. Write it once per set, beside
+\`set.json\`. Relative paths resolve against the file. A set that does not exist
+yet starts at \`setting-up-a-set.md\`, beside this file.
+
+\`\`\`toml
+[model]                  # the Publisher the answerer queries
+environment = "<env>"
+package     = "<pkg>"
+repo        = "<path to the model package>"
+port        = 4811
+mcp_port    = 4812
+
+[truth]                  # a second Publisher, serving only the truth package
+environment = "truth"
+port        = 4881
+mcp_port    = 4882
+\`\`\`
+
+No script defaults to any particular set's environment or package. A missing
+one stops with the key to add. Runs, built packages and server roots go under
+\`~/.malloy-eval/<set>-<hash>/\` unless \`[paths] workdir\` says otherwise, never inside
+the repository: a run holds a \`model.malloy\` snapshot and a built package is a
+Malloy package, and nested in the package under test either can put that
+package into \`loadErrors\`. The hash is of the set's path, so two sets with one
+name never share a \`runs/\`. Each command below is \`python3 skills/eval-loop/scripts/eval.py\`,
+and any flag the underlying script takes passes through.
+
 \`\`\`bash
-# 1. serve the model under test -- in its own session, so the shell's exit
-#    cannot take it down, and returning only once it answers a query.
+# 0. every gap in the set at once, before anything starts: missing files, a
+#    case the importer refuses, a truth package the model server would serve,
+#    a table path that resolves to nothing once served, a port already held.
+#    Free, and it starts nothing. Exits 1 until nothing blocks a run.
+eval.py check --set <set>
+
+# 1. serve the model under test, and the TRUTH package on a second server the
+#    answerer has no route to. Each runs in its own session, so the shell's
+#    exit cannot take it down, and returns only once it answers a query. The
+#    script writes each server's publisher.config.json from eval.toml, and
+#    refuses a truth server on the model server's ports.
 #    --warm-retrieval also waits for the embedding index to settle and exits 3
 #    if it does not reach \`ready\`: the sync is lazy, so without it the first
 #    cases are answered LEXICALLY and the run reports that as the model's
 #    number.
 #    Semantic retrieval needs EMBEDDING_API_KEY in the server's environment
-#    (the Publisher does not read OPENAI_API_KEY); without it every ranking is
-#    lexical, the warm-up reports \`cooldown\`, and the provider's own error is
-#    only in publisher.log (grep Embedding).
-python3 skills/eval-loop/scripts/serve.py --publisher-dir <publisher>/packages/server \\
-  --server-root <root> --port 4811 --mcp-port 4040 --trace-retrieval \\
-  --warm-retrieval --environment <env> --package <pkg> \\
+#    (the Publisher does not read OPENAI_API_KEY). The script warns when it is
+#    unset; without it every ranking is lexical, the warm-up reports
+#    \`cooldown\`, and the provider's own error is only in publisher.log
+#    (grep Embedding).
+eval.py serve model --set <set> --warm-retrieval \\
   [--allow-proxy]   # required for a \`publisher\`-type (proxied) connection
-#    a second server for the TRUTH package, on other ports, that the answerer
-#    has no route to:
-python3 skills/eval-loop/scripts/serve.py --publisher-dir <publisher>/packages/server \\
-  --server-root <truthroot> --port 4881 --mcp-port 4882 [--allow-proxy]
+eval.py serve truth --set <set> [--allow-proxy]
 
 # 2a. audit the set against the package it is about to be scored on. Free, and
 #     it catches the failure that reads as a model regression: an entity id
@@ -387358,10 +386854,9 @@ python3 skills/eval-loop/scripts/serve.py --publisher-dir <publisher>/packages/s
 #     --definitions <ledger> (verify_definitions.py), a set whose every
 #     value-bearing case rests on validated definitions exits 0 instead:
 #     the composition rule, values not re-derived but their definitions
-#     checked.
-python3 skills/eval-answer/scripts/verify_goldens.py \\
-  --set <repo>/evals/ecommerce --publisher http://localhost:4881 \\
-  --model <repo>/ecommerce --target-package ecommerce
+#     checked. --model adds the entity-id audit; it greps the model text, so a
+#     column a source exposes implicitly reads as missing.
+eval.py verify --set <set> [--model <package>]
 
 # 2b. ONLY on a set whose keys are still provisional -- an imported one is,
 #     throughout, by design. Nothing else in this toolchain writes
@@ -387370,59 +386865,50 @@ python3 skills/eval-answer/scripts/verify_goldens.py \\
 #     AND carries a second derivation, and prints the reason for every golden
 #     it left alone. \`--refresh\` is NOT this: it rewrites a drifted value and
 #     never touches a status.
-python3 skills/eval-answer/scripts/verify_goldens.py \\
-  --set <repo>/evals/ecommerce --publisher http://localhost:4881 \\
-  --target-package ecommerce --promote
+eval.py verify --set <set> --promote
 
 # 2. smoke one case first ($0.13), then the arm. Goldens are re-derived from
 #    the truth server before either starts; a drifted set refuses to run.
-python3 skills/eval-loop/scripts/run_baseline.py \\
-  --set <repo>/evals/ecommerce --out results/smoke --only <qid> --no-judge \\
-  --truth-publisher http://localhost:4881
-python3 skills/eval-loop/scripts/run_baseline.py \\
-  --set <repo>/evals/ecommerce --out results/<arm> \\
-  --parallel 4 --truth-publisher http://localhost:4881 \\
-  --model-repo <repo> --model-dir <package>   # the checkout the MODEL is
-                        # versioned in, recorded as modelGitSha, and the package
-                        # directory inside it the -dirty marker is decided over
-                        # (relative to the repo). Without --model-dir any
-                        # untracked file anywhere in the repo marks the model
-                        # dirty. The repo cannot be inferred: Publisher serves a
-                        # copy under publisher_data/, whose surrounding tree is
-                        # the server's storage, not the model's history.
+#    [model] repo is the model PACKAGE directory. The run records the commit
+#    of the checkout it sits in as modelGitSha, and decides the -dirty marker
+#    over that package directory only, so an untracked file elsewhere in the
+#    repository does not mark the model dirty. It cannot be inferred:
+#    Publisher serves a copy under publisher_data/, whose surrounding tree is
+#    the server's storage, not the model's history.
+eval.py run --set <set> --label smoke --only <qid> --no-judge
+eval.py run --set <set> --parallel 4
 #    the run names itself <set>-<phase>-<nn> (ecommerce-baseline-01, then -02
-#    for the second arm of the A/A). Pass --label only for a run that needs a
-#    human name; hand-typed arm names stop being readable within an afternoon.
+#    for the second arm of the A/A) and prints its directory. Pass --label
+#    only for a run that needs a human name; hand-typed arm names stop being
+#    readable within an afternoon.
 
 # 3. compare two arms, or two runs of one arm. It exits 2 when the arms used
 #    different retrievers, because those flips measure the retriever.
-python3 skills/eval-loop/scripts/flip_table.py --a results/<a> --b results/<b>
+python3 skills/eval-loop/scripts/flip_table.py --a <run a> --b <run b>
 
 # 3a. on an A/A, write the band into the set's calibration record. Nothing
 #     else may quote a band, and a band only covers runs whose pins match.
 python3 skills/eval-loop/scripts/flip_table.py \\
-  --a results/aa-1 --b results/aa-2 --calibration \\
-  >> <repo>/evals/ecommerce/CALIBRATION.md
+  --a <aa-1 run> --b <aa-2 run> --calibration >> <set>/CALIBRATION.md
 
 # 3b. check the JUDGE, not the model, after any change to the judge skill, a
 #     rubric, or what the judge is shown. It also reports which fixtures are
 #     unpinned and which decision classes nothing covers.
-python3 skills/eval-loop/scripts/check_judge.py \\
-  --set <repo>/evals/ecommerce --repeat 3
+python3 skills/eval-loop/scripts/check_judge.py --set <set> --repeat 3
 
 # 4. FIRST: any golden in doubt, from the judge or the set. \`jq .doubtedGoldens
-#    results/<arm>/run.json\` -- non-empty means settle those through the golden
-#    side door before diagnosing, or you send a modelling agent at a model that
-#    is already right.
-python3 skills/eval-diagnose/scripts/diagnose.py \\
-  --run results/<arm> --set <repo>/evals/ecommerce --model-dir <package>
+#    <run>/run.json\` -- non-empty means settle those through the golden side
+#    door before diagnosing, or you send a modelling agent at a model that is
+#    already right. It probes the server the run recorded.
+eval.py diagnose --set <set> --label <label>
 #    (cluster_failures.py gives a free mechanical first look, as
 #     clusters-mechanical.jsonl; it groups by retrieval outcome and is not a
 #     diagnosis)
 
-# 5. build the browsable package
-python3 skills/eval-loop/scripts/build_run_package.py \\
-  --run results/<a> --run results/<b> --set <repo>/evals/ecommerce --out <pkg>
+# 5. build the browsable package. Refused until every run has a diagnosis
+#    (--without-diagnosis overrides); a run where everything passed records an
+#    empty one. Pass --run twice for an A/B.
+eval.py package --set <set> --label <label>
 \`\`\`
 
 ### What the run prints at the end
@@ -387442,12 +386928,12 @@ Three layers, in this order, and the order is what makes it readable:
    report, so this layer is the two commands that turn it into something you
    read: \`build_run_package.py\` (a Malloy model over the run's CSVs,
    \`notebooks/eval_run.malloy\` for the aggregate tables, and an in-package HTML app for
-   the case matrix and its per-case drawer), then a \`POST .../packages\` that
-   registers it on the Publisher already running, with no restart. It prints
-   the resulting URL with the run's own paths filled in.
-
-   The POST lands the package in the environment the run used. Point it at
-   another if you would rather that package listing stay untouched.
+   the case matrix and its per-case drawer), registered on a Publisher that
+   is already running, with no restart. \`eval.py package\` builds it,
+   registers it on the TRUTH server, and prints both URLs; the package holds
+   the answer key, and that is the server the answerer has no route to. The
+   registration command is also written into the package's README.md, for
+   serving it again later; \`--no-register\` builds without registering.
 
 Order of magnitude for planning, **calibrated on ecommerce over local duckdb**:
 a Sonnet arm over a few dozen cases costs single-digit dollars and finishes in
@@ -387485,6 +386971,163 @@ Two failure modes worth pre-empting, because both produce a clean-looking run:
 - **Check the served revision is the one you edited.** Publisher serves a
   snapshot copy, so a model fix can be absent from the run that is supposed to
   measure it. Query the changed measure once before spending an arm on it.`
+    },
+    {
+      name: "eval-loop/setting-up-a-set",
+      description: "Setting up a set on your own model. Reference detail for the eval-loop skill.",
+      body: `<!-- The first run of your own set, in order. Read it when a set does not exist yet. -->
+
+# Setting up a set on your own model
+
+\`examples/storefront/evals/storefront-tour\` is a finished set. This is the path
+to one of your own: which files a set needs, what writes each one, and the
+order they go in. Every command is \`python3 skills/eval-loop/scripts/eval.py\`,
+run from the Publisher clone, unless it names another script.
+
+\`eval.py check --set <set>\` reads everything below and names every gap at once,
+without starting a server or calling a model. Run it after each step; the set is
+ready when it prints \`ready: nothing blocks a run\`.
+
+## What you need first
+
+- A Malloy package with a \`publisher.json\`, in a git repository. Git is how a
+  run pins the model it measured and how an improve step is rolled back.
+- A built Publisher clone (\`bun install && bun run build\`), Python 3.11+, and a
+  Java runtime for the SDK build.
+- \`EMBEDDING_API_KEY\` exported, for semantic retrieval. Without it the model
+  server ranks lexically and says so on start.
+
+## 1. Make the set directory
+
+One directory per set, in the same repository as the model package.
+\`skill:eval-import\` "Where the set goes" has the rule about how close to the
+package it may sit. It holds:
+
+| File | What it is | Written by |
+|---|---|---|
+| \`set.json\` | the set's name, the package and model under test, the truth package | you |
+| \`cases.jsonl\` | one case per question: text, split, golden | \`skill:eval-import\` |
+| \`as-received/\` | the questions exactly as they arrived | you, untouched |
+| \`eval.toml\` | where each step finds its servers | you |
+| \`gold/<qid>.json\` | a golden's second derivation | you, when you derive one |
+
+\`set.json\`, at minimum (\`reference/ledger-schema.md\` in \`skill:eval-answer\`
+has every field):
+
+\`\`\`json
+{"name": "<set>", "description": "<what it measures>", "datasetVersion": 1,
+ "targetPackage": "<package name>", "targetModelPath": "<model>.malloy"}
+\`\`\`
+
+## 2. Import the questions
+
+\`skill:eval-import\` turns questions in any shape into \`cases.jsonl\` and says what
+each one is worth. A set with no questions yet has nothing to import: write the
+questions a person would actually ask into \`as-received/\` first, then import
+them, rather than writing cases by hand. Then stamp and validate:
+
+\`\`\`bash
+python3 skills/eval-import/scripts/import_cases.py --set <set> --stamp
+\`\`\`
+
+It refuses what later steps would fail on, with the fix: no \`split\`, a value
+claiming \`verified\` on arrival, a \`scalar\` value that is a bare number rather
+than \`{"<column>": <value>}\`. A set of bare questions is valid. It runs and
+measures coverage, and cannot score accuracy until keys exist.
+
+## 3. Build the truth package, if any case holds a value
+
+A golden holding a value is re-derived before every run from a truth package:
+the raw tables your model reads, with no modelling, served on a second server
+the answerer cannot reach. Scaffold it OUTSIDE the model package, because
+Publisher serves every \`.malloy\` under a package directory:
+
+\`\`\`bash
+python3 skills/eval-answer/scripts/init_truth_package.py \\
+    --package <model package> --out <outside it>/<set>-truth --name <set>-truth
+\`\`\`
+
+It writes one source per raw table and links the model's data directory in, so
+file paths still resolve once Publisher serves its copy of the package. The link
+holds this machine's path; to commit the truth package, copy the data in
+instead, as \`examples/storefront-tour-truth/\` does. Add the package's scope
+filters by hand, as its header says, and nothing else. Then set
+\`"truthPackage": "<set>-truth"\` in \`set.json\`.
+
+A set of criteria goldens or bare questions needs no truth package; \`check\`
+notes its absence and moves on.
+
+## 4. Write eval.toml
+
+\`\`\`toml
+[model]                  # the Publisher the answerer queries
+environment = "<env>"
+package     = "<package name>"
+repo        = "<path to the model package>"   # relative to this file
+port        = 4811
+mcp_port    = 4812
+
+[truth]                  # only with a truth package; an empty [truth] takes 4881/4882
+environment = "truth"
+package_dir = "<path to the truth package>"
+port        = 4881
+mcp_port    = 4882
+\`\`\`
+
+\`package\` here and \`targetPackage\` in \`set.json\` must name the same package.
+Where both are set, \`eval.toml\` wins, and \`check\` refuses the set until they
+agree.
+
+Pick ports nothing else on the machine holds; \`check\` names any that are taken.
+Every later step reads them from this file, so change them here rather than
+passing \`--port\`, which \`serve\` refuses when it disagrees with the file.
+
+## 5. Check, serve, verify
+
+\`\`\`bash
+eval.py check --set <set>                  # until: ready
+eval.py serve model --set <set> --warm-retrieval
+eval.py serve truth --set <set>            # with a truth package
+eval.py verify --set <set>                 # free: re-derives every golden
+\`\`\`
+
+\`verify\` exits 0 when every golden re-derives, 1 when one drifted, and 3 when it
+could not check (no truth package, or no truth server). Read 3 as "not run",
+never as a pass.
+
+## 6. Turn provisional keys into scorable ones
+
+An imported value is \`provisional\`, and a provisional golden takes no verdict.
+\`--promote\` marks one \`verified\` only when it re-derived cleanly through the
+truth package AND a second, differently shaped derivation exists: a
+\`golden.verification\` block, or \`gold/<qid>.json\` holding \`verifyRows\`
+(\`reference/ledger-schema.md\`, the \`golden\` row, says what the second one must
+vary).
+
+\`\`\`bash
+eval.py verify --set <set> --promote
+\`\`\`
+
+It prints why it left each golden alone. \`check\` shows the scorable count.
+Where a person has checked a value by hand instead of deriving it a second way,
+\`--promote --attest "<who checked, when, how>"\` promotes it and writes that text
+on the golden, so a reader sees a person stood behind it rather than a query.
+
+## 7. The first run
+
+\`\`\`bash
+eval.py run --set <set> --label smoke --only <qid>[,<qid>...] --max-turns 40   # a case or two first
+eval.py run --set <set> --label baseline-01 --max-turns 40
+eval.py diagnose --set <set> --label baseline-01
+eval.py package --set <set> --label baseline-01
+\`\`\`
+
+Then \`reference/running-a-run.md\` for what the run prints and how to read it,
+and \`skill:eval-report\` for the write-up.
+
+Measured on the tour's local DuckDB data, with sonnet answering and judging:
+about $0.35 a case to run, and $0.60 and three minutes to diagnose one failed
+case. A warehouse over a proxy costs several times more per case.`
     },
     {
       name: "eval-report",
@@ -387538,15 +387181,23 @@ different task with its own turn.
 ## Step 1: build the artifacts, before writing a word
 
 \`\`\`bash
-python3 skills/eval-loop/scripts/build_run_package.py \\
-    --run <run-dir> --set <set-dir> --out /tmp/eval-<label>
-curl -sS -X POST http://<publisher>/api/v0/environments/<env>/packages \\
-    -H 'content-type: application/json' \\
-    -d '{"name":"eval-<label>","location":"/tmp/eval-<label>"}'
+python3 skills/eval-loop/scripts/eval.py package --set <set-dir> --label <label>
 \`\`\`
 
-That builds a Malloy package over the run's own CSVs and registers it with no
-restart. It gives you two things to link:
+That builds a Malloy package over the run's own CSVs, registers it with no
+restart, and prints the two URLs below. It registers on the TRUTH server,
+because the package holds the answer key and the answerer must not reach it.
+If the truth server is not running it says so and prints the \`curl\` to run
+once it is. A set with no truth server gets no registration: the only
+Publisher is the answerer's. Add a \`[truth]\` section and
+\`eval.py serve truth\`, or pass \`--on-model-server\`, which prints the \`curl\`
+and the \`DELETE\` to run before the next run.
+
+It refuses a run with no diagnosis, since the report's cluster views would be
+empty. Run \`eval.py diagnose\` first; when this run skipped diagnosis on purpose,
+pass \`--without-diagnosis\` and say in the report that it has no clusters.
+
+It gives you two things to link:
 
 | Artifact | What it is | URL |
 |---|---|---|