@malloy-publisher/server 0.2.2 → 0.2.3

package/dist/server.mjs

@@ -156066,6 +156066,10 @@ function assertDuckDBResourceConfig() {
   if (memoryLimit !== undefined && !/^\d+(\.\d+)?\s*(B|KB|KIB|MB|MIB|GB|GIB|TB|TIB)$/i.test(memoryLimit)) {
     throw new Error(`Invalid value for PUBLISHER_DUCKDB_MEMORY_LIMIT: expected a size like ` + `"1GB" or "512MB" (or "off" to disable), got "${memoryLimit}"`);
   }
+  const rowGroupSizeBytes = getDuckLakeRowGroupSizeBytes();
+  if (rowGroupSizeBytes !== undefined && !/^\d+(\.\d+)?\s*(B|KB|KIB|MB|MIB|GB|GIB)$/i.test(rowGroupSizeBytes)) {
+    throw new Error(`Invalid value for PUBLISHER_DUCKLAKE_ROW_GROUP_SIZE_BYTES: expected a ` + `size like "16MB", got "${rowGroupSizeBytes}"`);
+  }
   const tempDirectory = getDuckDBTempDirectory();
   if (tempDirectory !== undefined) {
     try {
@@ -156225,6 +156229,9 @@ var BUNDLED_DEFAULT_CONFIG_PATH, PER_MODE_COLOR_KEYS, DEFAULT_HIGH_WATER_FRACTIO
     return;
   }
   return raw;
+}, getDuckLakeRowGroupSizeBytes = () => {
+  const raw = process.env.PUBLISHER_DUCKLAKE_ROW_GROUP_SIZE_BYTES?.trim();
+  return raw === undefined || raw === "" ? undefined : raw;
 }, getDuckDBTempDirectory = () => {
   const raw = process.env.PUBLISHER_DUCKDB_TEMP_DIRECTORY?.trim();
   return raw === undefined || raw === "" ? undefined : raw;
@@ -213188,7 +213195,7 @@ var require_utils74 = __commonJS((exports, module) => {
 
 // ../../node_modules/ssh2/lib/protocol/crypto/build/Release/sshcrypto.node
 var require_sshcrypto = __commonJS((exports, module) => {
-  module.exports = __require("./sshcrypto-xqan60jb.node");
+  module.exports = __require("./sshcrypto-vd2k5hq9.node");
 });
 
 // ../../node_modules/ssh2/lib/protocol/crypto/poly1305.js
@@ -233453,6 +233460,19 @@ import {
   MalloyConfig
 } from "@malloydata/malloy";
 import fs3 from "fs/promises";
+async function applyDuckLakeRowGroupBound(connection, dbName) {
+  const bytes = getDuckLakeRowGroupSizeBytes();
+  if (bytes === undefined) {
+    return;
+  }
+  try {
+    await connection.runSQL("SET preserve_insertion_order=false");
+    await connection.runSQL(`CALL ${dbName}.set_option('parquet_row_group_size_bytes', '${escapeSQL(bytes)}')`);
+    logger.info(`DuckLake row group bound applied to ${dbName}: ${bytes}`);
+  } catch (error) {
+    logger.warn(`Could not set the DuckLake row group bound on ${dbName}; the lake keeps ` + `its existing value: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
 async function applySessionResourceLimits(connection, { tempDirectory } = {}) {
   const memoryLimit = getDuckDBMemoryLimit();
   const temp = tempDirectory ?? getDuckDBTempDirectory();
@@ -233801,6 +233821,9 @@ async function attachDuckLakeWithMode(connection, dbName, ducklakeConfig, option
   try {
     await connection.runSQL(attachCommand);
     logger.info(`Successfully attached DuckLake database in ${mode} mode: ${dbName}`);
+    if (!options.readOnly) {
+      await applyDuckLakeRowGroupBound(connection, dbName);
+    }
   } catch (error) {
     if (error instanceof Error && (error.message.includes("already exists") || error.message.includes("already attached"))) {
       logger.info(`DuckLake database ${dbName} is already attached, skipping`);
@@ -294772,6 +294795,1896 @@ function registerGetStatusTool(mcpServer, environmentStore) {
 // src/mcp/skills/skills_bundle.json
 var skills_bundle_default = {
   skills: [
+    {
+      name: "eval-answer",
+      description: "Score one analytical answer against a verified golden, and score which of the entities the golden depends on retrieval delivered to the answerer. Run the contamination checklist, re-execute the submitted query yourself, then spawn a judge subagent per skill:eval-judge. Append attempt, tool_call, score, and retrieval_score events to the file ledger (reference/ledger-schema.md). Never explain the failure (eval-diagnose) or edit the model (eval-improve). Use when asked whether an answer was correct, to score a run, or to baseline a model.",
+      body: `# Evaluate One Answer
+
+One user intent, answered once. This skill decides whether that answer was
+correct, records the evidence, and stops.
+
+**Scope boundary:** verdict and events only. No diagnosis, no model edit.
+
+## The unit
+
+A chat is not the unit. Segment by user intent. Feedback ("break it out by
+region") is a revision inside the same answer; grade the final accepted
+revision.
+
+Take the question from the stored case (\`evals/<set>/cases.jsonl\`), never from
+memory or a truncated console line. Record \`question_sha\` of the exact text
+the answerer saw. Record \`servedRevision\` from \`get_context\` or reload, not
+the package name: a same-named decoy has been measured for hours.
+
+## Step 1: Contamination check, before any score
+
+The answerer can Read or Shell its way to gold. Publisher traces do not see
+that, so the check runs on the HOST-side tool-use log you kept for the
+answerer subagent (every tool name and its path or command), plus the MCP
+call counts the answerer reported.
+
+The checklist. An attempt is contaminated when its log shows any of:
+
+1. a Read, Shell, or any file tool touching \`evals/\` or a gold artifact path;
+2. any access to the model file under test through a file tool (the
+   \`modelPath\` argument on an MCP \`execute_query\` is NOT contamination; the
+   server resolves it, the answerer never reads the file);
+3. \`reported_calls\` greater than \`host_tool_uses\` (the detectable
+   under-report floor is reported at most total tool uses).
+
+\`skills/eval-answer/scripts/check_contamination.py\` is a reference aid that
+mechanizes the same checklist over a JSON log; your reading of the transcript
+is the check, the script is a second pair of eyes.
+
+Contaminated attempts get \`verdict: null\` and \`contaminated: true\`. They are
+excluded from the run aggregates. They are not "wrong answers."
+
+If you cannot produce a host log, mark \`contaminated: "unknown"\` on both the
+attempt and its score event, and do not treat the attempt as a clean pass.
+
+## Step 2: Re-run the submitted query yourself
+
+Never score the agent's reported rows. Take its final query, execute it with
+\`execute_query\`, and write a prediction CSV under the run's
+\`artifacts/\` directory.
+
+\`submitted: false\` when there is no final query. That is not a wrong answer.
+No verdict can be issued (\`verdict: null\`, with the reason) when the attempt
+is not submitted, when the golden is missing, provisional, invalid, or
+ambiguous, or when a verified golden has no local artifact to compare.
+
+## Step 3: Judge the answer
+
+Spawn one fresh judge subagent per attempt, following
+\`skill:eval-judge\` (the rubric, the anchors, and the output shape live
+there; this skill does not restate them). Give it the question, the golden,
+your re-executed prediction rows, the canonical query when present, and the
+relevant source and field definitions from the model. It returns
+\`{verdict, confidence, why, column_pairing}\`.
+
+- The judge sees gold. It is therefore never the answerer, and its verdict
+  never leaks back to any answerer.
+- Confidence 5 or lower records as \`needs_human\`: neither a pass nor a fail,
+  excluded from acceptance arithmetic, queued for a human look.
+- \`near_match\` is also neither. It means defensibly different, not "nearly a
+  pass", and it stays out of the pass rate and the acceptance check for the same reason
+  \`needs_human\` does. Report the count; do not fold it into either column.
+- When a human overrules a verdict, append the case to
+  \`evals/<set>/judge-regressions.jsonl\`.
+- For a scalar golden, the same protocol applies to a one-value prediction.
+  For \`unanswerable\`, a refusal that names the gap is the pass; a confident
+  numeric answer is the fail.
+- Large row sets are still the judge's job. There is no scripted row oracle:
+  a script that can pass a wrong answer is worse than none, and the rubric's
+  containment and column-pairing rules are what the comparison needs.
+
+## Step 4: Score what retrieval delivered
+
+Per attempt, mechanically, from the ledger -- \`scripts/score_retrieval.py\`. Each
+case names the entities its answer depends on (\`expectedEntities.required\`, and
+\`requiredAnyOf\` groups where the model offers more than one route). An entity
+was delivered if the attempt's \`get_context\` calls returned it as a ranked entity
+under its id, under the same type and name on a sibling source, or by name inside
+a returned source's documentation -- text the answerer reads and acts on. Only
+\`missing\` is a retrieval miss; the route per entity is recorded so the strict
+count is still there.
+
+Recall 1.0 with a wrong answer exonerates retrieval: the failure is in the query.
+Recall below 1.0 and \`coverage: covered\` means the entity existed and search did
+not surface it; \`derivable\` or \`absent\` means there was nothing to surface. Those
+look identical in an answer score and have opposite owners, which is what makes
+this number worth having. It uses the search terms the answerer chose, so it
+attributes a failure *within* an arm and does not compare retrieval across arms
+-- that is the engine-side \`eval-retrieval\` skill, which does not ship here.
+
+## Step 5: Distrust the golden
+
+A reference answer can be wrong (parent-column fanout, a join on a shared
+non-identifying key, or a rubric describing a model that has since been fixed).
+Fanout is not automatically a defect: \`AVG\` / \`STDDEV\` / \`MIN\` / \`MAX\` survive
+uniform duplication. Classify \`verified_wrong\` (exclude from scoring) vs
+\`verified_benign\` (keep).
+
+**The judge produces this, not you.** It is the only station holding the golden,
+the re-executed rows and the model source at once, so it is the only one that can
+see the key contradict any of them; the rules and the four values are in
+\`skill:eval-judge\`. Carry its \`gold_status\` and \`gold_note\` onto the score
+event unchanged, and where it says nothing, fall back to the case's standing
+\`golden.status\`.
+
+Do not encode a rewrite of a bad golden into the model. A \`suspect\` or
+\`verified_wrong\`, or a no_match whose why indicts the golden rather than the
+prediction, routes to the golden side door in \`skill:eval-loop\` as a **dataset**
+issue. It is never a model failure, and it must be settled before improve runs --
+otherwise a modelling agent is dispatched to fix a model that is already right.
+
+## Step 6: Append events, then stop
+
+Append to \`evals/<set>/runs/<runId>/events.jsonl\` with \`caseId\` set. Shapes
+live in \`reference/ledger-schema.md\`.
+
+1. \`attempt\`: qid, sample, phase, question_sha, submitted, final_query,
+   served revision, call counts, contamination verdict, transcript path.
+2. \`tool_call\`: one per MCP \`get_context\` / \`execute_query\`, with \`traceId\`
+   and the \`rankedSummary\` copied from the trace (per-target ranks included).
+   Do not copy full traces into the event; the trace store holds the body.
+3. \`score\`: the judge's verdict object plus \`judge_version\`, \`rubric_sha\`,
+   \`golden_revision\`, \`contaminated\`, \`gold_status\`, and the judge output's
+   artifact path.
+
+A stage never rewrites another stage's fields. End-of-run numbers come from
+counting events, not from your arithmetic in prose.
+
+Sample each case once. Breadth across cases beats repeats of one case; the
+comparison rule for a before/after is the flip count in \`skill:eval-loop\`
+Measurement, not a mean over samples.
+
+## Re-score after a golden repair
+
+When \`eval-loop\` has repaired a golden and opened a new run, this skill runs
+again **without a new answerer**: same stored \`final_query\` (or its saved
+prediction CSV), new gold artifact, fresh judge, new \`golden_revision\` on the
+\`score\` event. Contamination does not need to be re-litigated if the attempt
+was already clean. If you must re-execute, do it yourself; do not ask the
+original answerer to "try again" with the new key in context.
+
+## Related skills
+
+- \`skill:eval-diagnose\`: why it failed, after this record exists.
+- \`skill:eval-improve\`: smallest model edit, model-owned issues only.
+- The \`malloy-analysis-pitfalls\` skill: checks before you trust a result you ran.
+  The judge loads it by name; it is not part of the \`eval\` group.
+
+## Reference files over MCP
+
+This skill's \`reference/\` files are served as separate prompts, one per file, fetched only when you ask for them. Where the text above says to read \`reference/<name>.md\`, get the prompt named \`eval-answer/<name>\` instead.
+
+Available: ledger-schema.`
+    },
+    {
+      name: "eval-answer/ledger-schema",
+      description: "The eval ledger: files and events. Reference detail for the eval-answer skill.",
+      body: "# The eval ledger: files and events\n\nThe ledger is plain files in the model package's git repository. There is no\neval API and no eval database. The conductor (`skill:eval-loop`) reads and\nwrites these files directly; the stages share them as their contract.\n`eval-answer` writes `attempt`, `tool_call`, and `score`. An engine-side\n`eval-retrieval` skill, which does not ship here, writes `retrieval_score` and\n`probe`; both stay in this contract so one validator covers every run\ndirectory.\n`eval-diagnose` writes `issue` and `issue_status`. `eval-improve` writes\n`candidate`. `eval-loop` writes `acceptance_check`, further `issue_status`, and\n`checkpoint`.\n\n**The contract is code: `eval-answer/scripts/ledger.py`.** Every script that\nwrites `run.json` or `events.jsonl` builds its lines through that module, so a\nfield rename that reaches only one writer fails at write time. This document\nis the human-readable rendering; where the two disagree, the module is right\nand this file is the bug. Check any run directory with:\n\n```\npython3 skills/eval-answer/scripts/ledger.py validate <runDir>\n```\n\nErrors are broken identity (missing `run.json`, a missing required field, two\nscores for one attempt); warnings are missing comparison pins (a run without\n`skillsVersion` still measures, but cannot anchor an A/B on that pin) and\ngrandfathered unknown fields on old runs.\n\n## Layout\n\n```\nevals/<set>/\n  set.json                  # set metadata (below)\n  cases.jsonl               # one case per line\n  judge-regressions.jsonl   # judge verdicts a human overruled\n  runs/<runId>/\n    run.json                # run config, the attribution pins\n    events.jsonl            # append-only event lines\n    artifacts/              # prediction CSVs, judge outputs, transcripts\n```\n\nThe set directory lives in the SAME git repository as the model it evaluates,\nso a checkpoint (a git commit) pins the model and the ledger together. Never\nplace `evals/` inside the directory tree the answerer's package serves: gold\nin the served tree is a contamination path.\n\nRules that make the ledger trustworthy:\n\n- `events.jsonl` is append-only. Never edit a line. To void one, append a new\n  event whose payload marks the old one `voided`. One sanctioned exception: a\n  re-runnable stage (diagnose, improve) replaces ITS OWN prior events, keyed by\n  issue id, through `ledger.replace_events()` -- so a crash-and-rerun does not\n  duplicate issues. A stage never touches another stage's lines.\n- A stage never rewrites another stage's fields.\n- One artifact directory per attempt (`qid` plus `sample`). Never overwrite a\n  previous attempt's prediction CSV.\n- End-of-run numbers come from counting event lines (`jq` over\n  `events.jsonl`), never from arithmetic recalled in prose.\n\n## `set.json`\n\n| Field | Notes |\n|---|---|\n| `name` | Set name; also the directory name. |\n| `description` | |\n| `datasetVersion` | Integer. Bump on any golden repair or case change. Runs record the version they scored against. |\n| `targetModelPath` | Model path within the package. |\n| `truthPackage` | Name of the package holding the semantics-free sources every golden is re-derived from -- a package NAME, not an object. `verify_goldens.py` skips every check without it, reporting \"nothing to re-derive against\", so a set that omits it silently has no golden verification at all. `init_truth_package.py` scaffolds the package. |\n| `truthModel` | Model file inside that package. Default `truth.malloy`. |\n| `truthTableRewrite` | Boolean, default false. Rewrites `duckdb.table('data/x.parquet')` refs to bare `x` in canonical queries, for a truth package whose tables are registered rather than read from files. |\n\n## `cases.jsonl`\n\nOne JSON object per line:\n\n| Field | Notes |\n|---|---|\n| `qid` | Stable case id. |\n| `question` | Exact text the answerer will see. |\n| `split` | `dev` or `holdout`. Frozen at import. Diagnose and improve read `dev` only; the acceptance check runs both. |\n| `tags` | list |\n| `state` | `candidate` / `selected` / `excluded`. |\n| `source` | Where the case came from. |\n| `golden` | `status` (`verified` / `provisional` / `invalid` / `ambiguous`), `kind`, `value` or `path` (artifact under the set directory), `canonicalQuery` (runs against the **truth** package, never the model under test), `verifiedBy`, and `verification` -- `{primaryAxis, variesAxis, note}` naming what the second derivation varied. **A golden is written only after two differently shaped derivations agree** (the truth query and a second one that varies a different axis -- group by something else, sum a different way, take a different route through the raw tables); `gold/<qid>.json` holds both (`truthRows`, `verifyRows`, `agreement`). Two derivations that vary the same axis share every other blind spot: that is how a golden that summed three overlapping time slices, 3x too high, passed its check. `verify_goldens.py` re-derives the value, reads the accepting clause of the rubric against the rows, and flags a second derivation that names no axis. **The question asks for data, never for an interpretation of it**, and either fixes the grain or the rubric accepts a correct figure at any stated grain (`judge.md` rules 9 and 10). |\n| `golden.rubric` | **On the golden, not on the case.** The prose the judge is shown as `CASE RUBRIC`, naming what counts as correct and what does not. `run_baseline.py` reads `golden.rubric`; a rubric written at the case's top level is silently not passed, the judge is told `CASE RUBRIC: none`, and it then scores from the golden value and the answer alone -- which reads as a judge that ignores its instructions. |\n| `golden.mustState` | **On the golden, not on the case.** What the answer has to say out loud. Read from `golden.mustState` by the run-package builder. |\n| `goldenRevision` | Integer. Bump on every golden change; `score` events stamp the revision they compared against. |\n| `expectedEntities` | `required`: entity ids (`kind:source:name`) the answer cannot be produced without. `requiredAnyOf`: a list of **groups**, each a list of ids of which any one suffices -- for a case the model can answer through more than one route. Naming only one route scores the other as a retrieval miss and steers diagnosis to \"retrieval ranking\" for a failure that was never retrieval's. `acceptable`: ids that are not noise if returned. An entity counts as delivered when returned under its exact id, under the same type and name on a sibling source, or by name in a returned source's documentation; `score_retrieval.py` records the route per entity as `delivery`. |\n\n## Term files (`eval-retrieval` only)\n\nA customer set has no term file. The engine-side `eval-retrieval` skill keeps\n`intents.jsonl` fixtures -- search term, entity type, a description of what the\nsearcher meant, a validity flag -- beside its own scripts, for evaluating the\nengine against fixed inputs. Their shape and the `retrieval_score` events they\nproduce are documented there and validated here.\n\n## `run.json`\n\nThe attribution pins. Two runs are comparable only when these match where it\nmatters. Two runs with different `target` values are comparable on answer\nverdicts only if the same model version reached both, which is rarely worth\nassuming:\n\nRequired fields are the run's **identity** -- every run ever written carries\nthem. The rest are optional in the schema, but the **comparison pins** are\nwarned about when absent, because a run missing one cannot take part in an A/B\non that axis.\n\n| Field | Notes |\n|---|---|\n| `runId` *(required)* | Directory name. |\n| `target` *(required)* | Which server answered, and how it reached the data: `local`, `local-proxied`, or `platform`. Decides what an improve step is even allowed to do (see `skill:eval-loop`). |\n| `answererModel` *(required)* | |\n| `phase` / `started` *(required)* | |\n| `judgeModel` / `judgeVersion` / `rubricSha` *(pins)* | The judge's model, and the version + content sha of `skill:eval-judge`. |\n| `datasetVersion` *(pin)* | From `set.json` at run time. |\n| `modelSha` *(pin)* | Content sha of the `model.malloy` snapshot in the run directory -- the bytes the answerer actually queried. A git sha is not enough: a snapshot host serves a copy, often of a dirty tree no commit names. |\n| `skillsVersion` *(pin)* | HEAD of the checkout the agents' skills were loaded from, dirty-marked (`ledger.skills_git_sha(root)`). The skills are the doctrine the agents load; a run that cannot name their version cannot anchor a skills A/B. |\n| `skillsRoot` / `harnessVersion` | Which checkout supplied the doctrine (`--skills-root`, e.g. a Publisher checkout for the open-source skills; default this one), and this checkout's own HEAD. The eval-* skills always come from the harness checkout, whatever `skillsRoot` says. Two runs whose `skillsRoot` differ are a skills A/B only if their manifests name the same skills. |\n| `diagnoserModel` / `improverModel` | Written by `diagnose.py` / `improve.py` when those stages run, so the run names every LLM that touched it. Absent on a run that was only answered and judged. |\n| `modelGitSha` | Commit of the model repo, when the target served working files; `-dirty` suffix when the tree had uncommitted changes -- fine for a band measurement, not for an A/B pin. Omit for a platform target, where a local commit pins nothing. |\n| `environment` / `package` / `modelPath` | What was served, and from where. On a platform target `environment` is the organization and `package` is the workspace the MCP URL is scoped to. |\n| `scope` | Platform target only: the `environment/package` the answerer was told to pass as an explicit `scopes` entry on every `get_context` / `execute_query` call. A workspace can serve many packages (a personal workspace serves every package the user can read), so without this the run also measures whether retrieval picks the right package -- a different measurement. Absent means unscoped. |\n| `mcpUrl` / `publisher` | The endpoints the answerer and the re-execution used. |\n| `predictionsReExecuted` | Whether the judge saw re-executed rows (false when the served bytes no longer match the pinned snapshot). |\n| `label` | `<set>-<phase>-<nn>`, assigned by `run_baseline.py` from the set name, the run's phase and the next free number beside it (`ecommerce-baseline-01`, `ecommerce-baseline-02`, `ecommerce-blind_gate-01`). The A/A pair is two runs of the same phase; the post-edit arms are two runs of `blind_gate`. Hand-typed names do not survive one afternoon of runs -- `base`, `rejudged2`, `r3`, `post1` sort wrongly, group not at all, and cannot be matched to an arm. `--label` overrides for a run that genuinely needs a human name. |\n| `effort` | |\n| `answererCostUsd` / `judgeCostUsd` | What the arm cost, split by role. The judge's half was discarded until 2026-09-02, so every \"cost per arm\" quoted before then was the answerer alone. |\n| `goldenCheck` | What `verify_goldens.py` said before the run started: `N ok, M drifted, K other finding(s)`, or why it did not run (no truth server on a platform target; `--skip-golden-check`; rebuild). A run that started on a drifted set says so here rather than pretending its verdicts mean something. |\n| `status` | `complete`, or `aborted` when four consecutive attempts errored or found the server dead and the harness stopped rather than spend the rest of the budget on attempts nobody will trust. |\n| `packageSha` / `servedRevision` *(pins)* | Taken from the server, not recomputed: `sourceContentSha` is a content hash over EVERY model path in the package, so an edit to an imported file moves it where a sha of the one `--model-path` does not; `servedRevision` is minted per load, so it identifies a load rather than content and is a poor pin alone. Measured: `publisher.json`'s `version` moves neither, and nothing in Publisher reads it -- it is not in the Package API schema and never returned, so it pins nothing. |\n| `datasetSha` *(pin)* | Content hash of `set.json` + `cases.jsonl`. Deliberately SEPARATE from the model's sha: a golden repair is not a model change, and one pin covering both would make every answer-key fix read as an edit to the model, which is the distinction an A/B rests on. Automatic, so nobody has to remember it; `datasetVersion` stays beside it as the human-readable sequence. **Local targets only.** A hosted target that publishes IMMUTABLE versions needs none of this: the set rides inside the version, and immutability -- not hashing -- is what makes a pin trustworthy. There, `targetVersion` alone identifies model and set together. |\n| `doubtedGoldens` | The cases whose golden the judge did not believe: `qid`, `gold_status` (`suspect` or `verified_wrong`), `gold_note`. **Read this before diagnose.** These are dataset issues, not model failures, and they go through the golden side door in `skill:eval-loop`. Empty list when the judge believed every key. Written from the same scan that prints the end-of-run warning, because a warning that lives only in console text is one scrollback away from sending a modelling agent at a model that is already right. |\n| `mode` / `setName` / `targetVersion` / `serverVersion` / `traceMode` / `callBudget` / `status` | Defined and accepted, **not yet written by any harness** -- kept in the schema for the platform target and the conductor, which need them. |\n\n## Events\n\nEvery line in `events.jsonl` is **flat**: `{ \"kind\": ..., <fields> }`, one\nJSON object per line, with an optional `at` ISO timestamp. Case-scoped kinds\n(`attempt`, `tool_call`, `score`) carry `qid`, `sample`, `phase` on the line;\nrun-level kinds do not. (An earlier draft of this document nested fields under\na `payload` with a `caseId`; no writer ever did that, and 23 run directories\nexist in the flat shape, so the flat shape is the contract.) `kind` is one of:\n`attempt`, `tool_call`, `score`, `retrieval_score`, `issue`, `issue_status`,\n`candidate`, `acceptance_check`, `checkpoint`.\n\n### `attempt`\n\n| Field | Type | Notes |\n|---|---|---|\n| `qid` | string | |\n| `sample` | int or null | Which repeat. Required even when null. |\n| `phase` | string | `baseline` / `loop` / `blind_gate` / `canary` / `final`. `phase` lives here, on the attempt, not in run config. |\n| `question_sha` | string | Hash of the exact text the answerer saw. |\n| `submitted` | bool | False when there was no final query. Not a wrong answer. |\n| `final_query` | string or null | Required to replay. |\n| `servedRevision` | string or null | From the package actually queried. |\n| `n_get_context` / `n_execute` / `n_execute_errors` | int | |\n| `host_tool_uses` | int | Host-side count, including Read and Shell. |\n| `reported_calls` | int | MCP calls the answerer claimed. |\n| `contaminated` | bool or `\"unknown\"` | `\"unknown\"` when no host log exists. Read it with `ledger.is_contaminated`, never for truthiness: runs written before 2026-09-03 carry the strings `\"true\"`/`\"false\"`, and `bool(\"false\")` is True. `ledger.event` now rejects those strings on write; `validate_run` grandfathers them on read as a warning. |\n| `contamination_reasons` | list | Empty when clean. |\n| `input_tokens` / `output_tokens` / `cache_read_tokens` | int or null | Answerer token usage. Null when the host does not report it. |\n| `cost_usd` | float or null | Answerer cost for this attempt. |\n| `num_turns` / `wall_seconds` | int, float or null | |\n| `answer_text` | string or null | The answer the judge scored. Kept so a verdict can be re-read without the transcript. |\n| `transcriptPath` | string | The answerer's transcript under `artifacts/`. |\n\nToken counts sit here rather than being derived later because the claim a\nsemantic model makes is about cost as well as correctness -- that a documented\nmodel reaches a good answer in fewer turns and tokens than working from raw\nschema. A ledger that counts calls but not tokens can state half of that.\n\n### `tool_call`\n\nOne event per MCP `get_context` or `execute_query` the attempt made.\n\n| Field | Type | Notes |\n|---|---|---|\n| `tool` | string | `get_context` or `execute_query`. |\n| `traceId` | string or null | `get_context` only; look up in your host's trace store. |\n| `targets` | string, list or null | **What the answerer asked for**: the search terms it sent to `get_context`. Null for `execute_query`. |\n| `rankedSummary` | object | Copied at capture from the trace so evidence survives trace eviction: `entityIds`, `ranks`, `resultCount`, and per-target `targets` with within-target ranks. |\n| `error` | string or null | |\n\nNever persist `execute_query` result rows, givens, or credentials.\n\n`targets` records the request; `rankedSummary` records the response. Without\nboth, a low per-attempt recall has two readings that cannot be told apart: the\nanswerer searched for the wrong thing, or it searched well and retrieval ranked\nthe right entity too low. Those have opposite owners -- `agent-skill` and\n`retrieval` -- so a ledger holding only the response cannot attribute the\nfailure, and any recall computed from it is a blend of answerer behaviour and\nretrieval quality.\n\nThat blend is also why per-attempt recall is **not** comparable across arms: a\nstronger answerer searches better and scores higher retrieval recall without\nretrieval having changed. Use it for attribution within an arm. Comparing\nretrieval itself between engine versions is `eval-retrieval`'s job, with fixed\nterms, and is not something a customer run reports.\n\n### `score`\n\nThe answer judge's verdict for one attempt (protocol in\n`skill:eval-judge`). Every attempt in a scored run gets exactly one.\n\n| Field | Type | Notes |\n|---|---|---|\n| `verdict` | string or null | `match` / `near_match` / `no_match` / `needs_human`; null when the attempt is not scorable. Only `match` and `no_match` are decisions; see below. |\n| `reason` | string | Why, from the judge; for a null verdict, why not scorable (`not_submitted`, `golden_missing`, `golden_ambiguous`, `contaminated`). |\n| `confidence` | int or null | 1 to 10. Confidence of 5 or lower forces `needs_human`. |\n| `column_pairing` | object or null | The judge's named gold-to-prediction column correspondence. |\n| `judge_version` / `rubric_sha` | string | Pins which rubric produced this verdict. |\n| `golden_revision` | int | From the case at score time. |\n| `contaminated` | bool or `\"unknown\"` | Copied from the attempt; true or unknown means `verdict: null`. |\n| `artifactPath` | string | The full judge output under `artifacts/`. |\n| `gold_status` | string | `verified` / `verified_benign` / `suspect` / `verified_wrong`. **From the judge**, which scored against the golden as written and reports separately whether it believes it; falls back to the case's standing `golden.status` when the judge does not say. `verified_wrong` excludes the case from run aggregates. `suspect` and `verified_wrong` route to the golden side door as `dataset` issues, never to improve. |\n| `gold_note` | string or null | The judge's evidence for a non-`verified` status: the two values, or the model line against the rubric sentence. Null when `verified`. |\n\nA `submitted: false` attempt gets `verdict: null, reason: \"not_submitted\"`,\nexcept for an `unanswerable` golden, where a refusal that names the gap is the\npass and a confident numeric answer is the fail.\n\nAggregates count decided verdicts only. `match` and `no_match` are the pass and\nthe fail; **`near_match`, `needs_human` and null are none of the above** and stay\nout of acceptance arithmetic. `near_match` is excluded because it means \"defensibly\ndifferent\", and an arguable verdict that moves a pass rate is a measurement\nartefact rather than a result (`skill:eval-judge` rule 7). Report its count:\nit rising is how a set tells you its rubrics are going vague.\n\nThis applies to `score`. On `retrieval_score` below, `near_match` **is** counted\ntowards recall and precision, because an overlapping entity is a genuine\nretrieval success. The two are different questions that share a word.\n\n### `retrieval_score` (written by `eval-retrieval`, not by a customer run)\n\nOne per term judged in a fixed-term replay (run-level; no `caseId`).\n\n| Field | Type | Notes |\n|---|---|---|\n| `intentId` / `term` / `entityType` | string | From the term file. |\n| `in_scope` | bool | Does THIS model version represent the concept? False is a coverage gap, not a retrieval failure. |\n| `judgments` | list | Per returned entity: `entityId`, `rank`, `level` (`match` / `near_match` / `no_match`), `confidence`, `why`. Empty when nothing returned. |\n| `judge_version` / `rubric_sha` | string | |\n| `traceId` | string or null | The `get_context` call judged. |\n\nRun-level metrics fall out by counting:\n\n- `coverage` = in-scope terms / valid terms -- a property of the model and its\n  data, reported on its own, never as a retrieval number.\n- `recall` (on in-scope terms) = fraction whose judgments contain a `match`.\n- `precision@N` = `match` judgments / judged, with N stated.\n\n### `issue`\n\n| Field | Type | Notes |\n|---|---|---|\n| `issue_id` | string | Stable across status events. |\n| `qids` | list | Affected cases. |\n| `primary_code` / `contributing_codes` | string / list | From `skill:eval-diagnose`, verbatim. |\n| `component` | string | `dataset` / `agent-call` / `get_context/model` / `get_context/retrieval` / `construction` / `model-definition`. |\n| `owner` | string | `model` / `retrieval` / `agent-skill` / `dataset`. Environment failures stop the run; they are never diagnosed, so there is no environment owner. |\n| `severity` / `confidence` | string | |\n| `sufficiency` | string | `sufficient` / `insufficient` / `unknown`. |\n| `traceIds` | list | |\n| `diagnosis` | string | Written before any edit exists. |\n\n### `issue_status`\n\n`issue_id` plus `status`: `open` / `batched` / `fixed` / `rejected` /\n`deferred`. Readers take the latest event for that `issue_id`. Do not invent\na status column.\n\n### `candidate`\n\nWritten by `eval-improve`, for every proposed edit, accepted or not. A\nrejected direction keeps its record.\n\n| Field | Notes |\n|---|---|\n| `issue_ids` | |\n| `files` | Paths the edit touched. |\n| `diffSummary` | One line per file. |\n| `probes` | Query and result for each factual claim. |\n| `meaningChanged` | Entities whose *meaning* the edit changed; `[]` for a docs-only edit. |\n| `goldenSuspect` | Each `{qid, entity, stored, rederived}`: a golden this edit may have invalidated. Reported by the improver, never repaired by it. **Non-empty halts the acceptance check** until adjudicated through the golden side door. |\n| `goldenAudit` | The set's `verify_goldens.py` run against the edited model: `{ran, clean, model, tail}`. Catches drift and rubric-vs-model contradictions only; a golden whose value silently moved is invisible to it, which is why `goldenSuspect` exists alongside. |\n\n### `acceptance_check`\n\nCalled `gate` before 2026-09-03. `ledger.py` reads the old kind as this one, so\nrun directories written earlier still validate and nothing rewrites them.\n\nWritten by `eval-loop`, one per acceptance check decision, BEFORE any checkpoint commit.\n\n| Field | Notes |\n|---|---|\n| `issue_ids` | |\n| `decision` | `accepted` / `rejected`. |\n| `class` | `docs` / `definition` / `retrieval` / `justified`. |\n| `baselineRunId` / `finalRunIds` | Plural: acceptance needs two independent runs. |\n| `regressions` | Case ids whose verdict got worse vs baseline. Must be empty to accept. |\n| `holdoutDelta` | Confident-verdict delta on the holdout slice. |\n| `reason` | Including independent deterministic justification when that is the basis. |\n\n### `checkpoint`\n\nWritten by `eval-loop` after an accepted acceptance check, or when a restore runs. The\nmodel bytes live in git, not in this payload.\n\n| Field | Notes |\n|---|---|\n| `action` | `created` / `restored`. |\n| `label` | |\n| `modelGitSha` | The commit this checkpoint names (create), or the commit restored to. |\n| `issueIds` | Issues the accepted edit closed. Empty on restore. |\n\n## `judge-regressions.jsonl`\n\nAppend a line whenever a human overrules a judge verdict: the case or intent,\nthe judge's verdict, the human's, and why. Re-run this file against the judge\nwhenever `skill:eval-judge` or the judge model changes; a rubric change that\nflips old human-settled verdicts is a judge regression, not new truth."
+    },
+    {
+      name: "eval-diagnose",
+      description: "Diagnose why a scored answer failed and who owns the fix, then cluster the failures by shared root cause. Walk dataset, agent-call, get_context/model, get_context/retrieval, construction, then model-definition. Append issue events to the file ledger, linked by traceId, one per cluster. Use after eval-answer, when triaging a run, or before changing a model. Does not edit the model (eval-improve).",
+      body: `# Diagnose One Answer
+
+Consumes a \`score\` event from \`skill:eval-answer\` and answers: why did this fail,
+and who owns the fix?
+
+**Scope boundary:** write the diagnosis before any edit exists. This skill never
+edits a model and never proposes a patch beyond naming the gap. Diagnosis that
+is allowed to edit becomes justification for an edit somebody already wanted.
+
+Do not diagnose a contaminated attempt or an environment failure. Those are
+harness or ops, not model work.
+
+## Components, in order
+
+Walk **in this order** and stop at the first with positive evidence. A later
+label requires ruling out the earlier ones. Write \`component\` with these strings,
+never "C1" / "C2" / "C3":
+
+| \`component\` | Question |
+|---|---|
+| \`dataset\` | Bad question, bad or missing golden, or environment drift? |
+| \`agent-call\` | Did the agent ask for the needed concepts, with the right type and scope? |
+| \`get_context/model\` | Is the needed entity absent, undocumented, weakly labeled, duplicated, or missing guidance? |
+| \`get_context/retrieval\` | Was an on-target request against a well-described entity ranked or grouped wrong? |
+| \`construction\` | Did sufficient context arrive, and the agent still built the wrong query? |
+| \`model-definition\` | Is a measure, join, filter convention, or source semantically wrong? |
+
+\`owner\` is separate: \`model\`, \`retrieval\`, \`agent-skill\`, or \`dataset\`. There
+is no environment owner: an environment failure stops the run before
+diagnosis (see the boundary above), so no issue can carry it.
+
+\`construction\` requires proving the needed entities and governing guidance were
+in the returned context. A server trace proves what Publisher returned, not what
+the host kept after compaction. If the rendered tool response is gone, mark
+sufficiency \`unknown\` and do not assign \`construction\`.
+
+Always report construction eligibility as \`eligible / total\`. That is a
+diagnostic conditional, not a causal comparison.
+
+## Step 1: Extract facts from traces, not from memory
+
+For each \`get_context\` call, load the stored retrieval trace by the \`traceId\` on the
+\`tool_call\` event. Write down, before you interpret anything:
+
+- **Asked:** every retrieval utterance, target types, scopes, and result counts,
+  in order.
+- **Returned:** for each needed entity, whether it appeared, its best
+  within-target rank, and under which utterance. Read this off the
+  \`rankedSummary\` on the attempt's \`tool_call\` events (its \`targets\` list
+  carries per-target ranks); the full trace body is behind your host's
+  trace lookup.
+  Count from the trace, never from recollection.
+- **Used:** sources and fields the final query referenced, and needed entities
+  that were returned and then unused.
+
+Resolve aliases to the real source (\`join_one: bldg is fac_building\` uses
+\`fac_building\`). Count ranks from the trace, not from recollection.
+
+The needed set comes from golden metadata or from the entities the corrected
+answer required. Do not invent it from the question's nouns alone.
+
+Presence leads; rank refines. Every needed entity present with a wrong
+answer is prima facie \`construction\`. A needed entity that never appeared is
+never \`construction\`, no matter how wrong the query looks. Everything
+present but buried deep under noise the agent reasonably skipped is
+\`get_context/retrieval\` once the request itself was on-target.
+
+## Step 2: Assign one primary code
+
+Use these codes verbatim. Re-wording them destroys the cross-answer pattern.
+
+### Dataset first
+
+| Code | When | Owner |
+|---|---|---|
+| \`BAD-REFERENCE\` | the golden itself is wrong, and you can name the defect | dataset |
+| \`AMBIGUOUS-REFERENCE\` | the key is untrustworthy as a score, but a replacement is not uniquely determined (two honest replays disagree; later cases may confirm a convention) | dataset |
+| \`BAD-QUESTION\` | the question is unanswerable as written, or underspecified (ties, rank without order) | dataset |
+| \`CORRECT-SUPERSET\` | every expected row present, plus extra context | none (it passed) |
+
+A cheap tell for a bad golden: impossible magnitude; identical values across
+entities that should differ; \`SUM\` / \`COUNT(*)\` over a join that duplicates on
+both sides. \`AVG\` / \`STDDEV\` / \`MIN\` / \`MAX\` survive uniform duplication, so
+fanout alone proves nothing.
+
+**BAD-REFERENCE and AMBIGUOUS-REFERENCE are first-class outcomes, not awkward
+misses.** Goldens often encode assumptions we want in the model. They can also
+be wrong. Flag the case when the key is defective or when two justified
+replays disagree: do not edit the model to match a bad or unsettled key, and
+do not invent a replacement number. \`BAD-REFERENCE\` goes to Repair a bad
+golden. \`AMBIGUOUS-REFERENCE\` goes to Hold an ambiguous golden. Do not leave
+the run looking like the model failed.
+
+This skill **classifies and hands off**. Write the issue with
+\`owner: dataset\` and stop for that case. The conductor (\`skill:eval-loop\`)
+either repairs the golden or holds it as \`ambiguous\`. Do not capture a
+replacement golden from inside diagnosis if you are not also conducting; a
+diagnosis that writes a new key without a version bump silently changes what
+earlier scores meant.
+
+Prior \`score\` events are not rewritten. They keep the old \`golden_revision\`.
+
+### Agent call
+
+| Code | When | Owner |
+|---|---|---|
+| \`NEVER-ASKED\` | no utterance targeted a needed concept | agent-skill, and model if nothing would have prompted the ask |
+| \`VAGUE\` | compound or generic utterances, so nothing could rank | agent-skill |
+| \`QUESTION-VOCAB\` | utterances parroted the question where the data uses other words | agent-skill, and model if that vocabulary is undocumented |
+| \`NO-DISAMBIG\` | two plausible candidates, never resolved | model: docs should answer, not require the question |
+| \`ASSUMED\` | assumed a scope or convention instead of checking | model if nothing warned; agent-skill otherwise |
+| \`WRONG-TYPE-OR-SCOPE\` | asked, but with the wrong target type or an empty/wrong scope | agent-skill |
+
+If the agent could not reasonably have known to ask, that is a model gap.
+
+### get_context / model
+
+| Code | When | Owner |
+|---|---|---|
+| \`COVERAGE\` | no representing entity anywhere | model |
+| \`NOT-RETURNED\` | it exists, the ask was on target, it never came back | model: labels, docs, synonyms, index |
+| \`LOW-RANK\` | returned, buried under noise the agent reasonably skipped | model |
+| \`AMBIGUOUS\` | several near-identical candidates | model: "use X for …, Y when …" |
+| \`GUIDANCE-NOT-RETRIEVED\` | entities came back, governing guidance did not | model: put guidance on the entities agents search for |
+| \`GUIDANCE-DECLINED\` | guidance was retrieved and judged inapplicable | model: state the business default, not a caveat |
+
+A missing join is coverage, not an agent-call miss. The model has to volunteer
+relationships. A declared join is not a retrieval entity; do not look for it in
+\`get_context\` results.
+
+### get_context / retrieval
+
+| Code | When | Owner |
+|---|---|---|
+| \`RETRIEVAL\` | model looks right, utterance on target, rank or grouping still failed | retrieval |
+
+Prove it before you use this code: search a distinctive phrase from the entity's
+own doc. If a rare token retrieves it and ordinary phrasing does not, say so
+with both queries. Otherwise it is still \`NOT-RETURNED\` / \`LOW-RANK\`.
+
+### Construction (only after sufficiency)
+
+| Code | When | Owner |
+|---|---|---|
+| \`WRONG-PICK\` | needed entity returned, used a different one | model if indistinguishable; agent-skill if docs distinguished them |
+| \`SCOPE\` | right entities, wrong population | model if the scope rule was undocumented |
+| \`GRAIN\` | right entities, wrong grain | model or agent-skill |
+| \`FILTER-LITERAL\` | filter literal did not match stored values | model (document the stored form) and agent-skill |
+| \`CONVENTION\` | right data, wrong statistical or business convention | model: expose a named measure |
+| \`SYNTAX\` | could not express it; execute errors; never submitted | agent-skill |
+
+### model-definition
+
+Use when the entity was found and used, and the definition or the data behind it
+is wrong (bad grain, wrong join key, inverted filter). Owner: model. A doc whose
+factual claim the data contradicts (a population statement, a grain claim) is
+also model-definition: the SQL may be right while the stated contract is false,
+and an agent that trusts the doc answers wrongly without ever failing a query.
+Probe the claim before writing the issue.
+
+## Step 3: Read the failure shape
+
+| Signature | Look here |
+|---|---|
+| Extremes match, means do not | Population, filter, or join scope |
+| Same row count, values differ | Wrong column or literal, not joins |
+| Row count differs, all expected rows present | Superset; often not an error |
+| Right keys, wrong aggregates on a minority | Undeclared or wrong-cardinality relationship |
+| Off by a clean integer multiple | Fanout; which side of the join is non-unique |
+| Zero errors, few calls, fast, confidently wrong | The model steered it |
+| Identical high-precision values across entities that should differ | Cross-contamination join |
+| A magnitude that cannot be true | Fanout, possibly in the golden |
+
+Mine the agent's prose, not only its calls. It often names the gap.
+
+## Step 4: Append issue events, then stop
+
+Append to \`evals/<set>/runs/<runId>/events.jsonl\` with \`kind: issue\`
+(shapes in \`skill:eval-answer\` \`reference/ledger-schema.md\`):
+
+- \`issue_id\`, affected \`qids\`, \`primary_code\`, \`contributing_codes\`
+- \`component\`, \`owner\`, \`severity\`, \`confidence\`
+- \`sufficiency\` (\`sufficient\` / \`insufficient\` / \`unknown\`)
+- \`traceId\`s, not copied trace payloads
+- \`diagnosis\`: the suspected shared entity, file, or root cause, written
+  before any edit exists
+
+Then \`issue_status\` with \`status: open\`. Status is always an event. Readers
+take the latest \`issue_status\` for that \`issue_id\`.
+
+The issue backlog in the event log is the output, not per-question prose.
+Diagnose reads dev cases only; a holdout case with a bad score stays
+undiagnosed so the acceptance check keeps something the improve step never saw.
+
+## Step 5: Cluster before anyone improves
+
+Eight failures are rarely eight problems. They are more often two or three,
+each surfacing in several cases, and the edit worth making is the one that
+clears a group. So the unit handed to \`skill:eval-improve\` is the cluster, not
+the case, and one issue event covers all of its cases rather than one per case.
+
+**Group on shared cause, not shared symptom.** Two cases that both returned a
+wrong revenue number belong together only if the same entity, doc gap, or
+convention explains both. Same \`owner\` and same \`component\` is a hint, never a
+criterion: two \`COVERAGE\` issues about different missing entities are two
+clusters, and merging them produces an edit that fixes neither cleanly.
+
+Order clusters by how many cases they would fix. Cluster the non-model owners
+too, in their own clusters, so nothing is lost on the way to the backlog --
+but keep them separate, because only \`owner: model\` may proceed to an edit.
+
+Say what you considered merging and chose not to. A cluster is a claim that one
+change fixes N cases, and the near-misses are what a reviewer needs to falsify
+it.
+
+Still no patch. Naming the shared root cause precisely enough that someone else
+can design the edit is the whole job here; the edit itself is
+\`skill:eval-improve\`, working from this. A cluster carrying an honest open
+question is more useful than one carrying a remedy nobody probed.
+
+**Only \`owner: model\` proceeds to \`eval-improve\`.** Skill findings go back into
+the analysis or phrase-detection skill. Retrieval findings go to the tool.
+\`BAD-REFERENCE\` and \`AMBIGUOUS-REFERENCE\` go to the golden side door in
+\`skill:eval-loop\` (repair or hold). Do not send them to improve. Other dataset
+findings (a bad question, a case worth excluding) go back to the case in
+\`cases.jsonl\` via the conductor. Routing a skill bug into the model is
+how models accumulate scar tissue.
+
+## Anti-patterns
+
+- Do not diagnose from the answer alone. Probe why a number differed.
+- Do not treat a passing answer as uninformative. High call counts on a pass
+  still name gaps.
+- Do not conclude a model gap from two agents agreeing. They coin-flip onto the
+  same undocumented sibling for the same reason.
+- Do not assign \`construction\` when sufficiency is unknown.
+
+## Related skills
+
+- \`skill:eval-answer\`: the score this consumes.
+- \`skill:eval-improve\`: smallest model edit, \`owner: model\` only.
+- \`skill:eval-loop\`: golden hold/repair, the acceptance check, and checkpoint.
+
+## Reference files over MCP
+
+This skill's \`reference/\` files are served as separate prompts, one per file, fetched only when you ask for them. Where the text above says to read \`reference/<name>.md\`, get the prompt named \`eval-diagnose/<name>\` instead.
+
+Available: output-contract.`
+    },
+    {
+      name: "eval-diagnose/output-contract",
+      description: "Output contract. Reference detail for the eval-diagnose skill.",
+      body: `<!-- What a diagnose agent must emit. Read this before writing your reply. -->
+
+# Output contract
+
+Both shapes below are read by a script. Emit the object as the LAST thing in
+your reply, with nothing after it. Prose before it is fine and expected -- the
+reasoning is what the codes have to follow from.
+
+## Per case (Step 1-4)
+
+\`\`\`json
+{"probes": [{"why": "the claim this checks", "query": "query or search text",
+             "result": "what came back, briefly"}],
+  "reasoning": "how the ladder resolved: what you ruled out, and why",
+  "component": "one of the six",
+  "primary_code": "one code, verbatim from the skill",
+  "contributing_codes": ["zero or more, verbatim"],
+  "owner": "model | retrieval | agent-skill | dataset",
+  "sufficiency": "sufficient | insufficient | unknown",
+  "severity": "high | medium | low",
+  "confidence": "high | medium | low",
+  "diagnosis": "the suspected entity, file, or root cause, in one or two sentences",
+  "sharedWith": "a short phrase naming what other cases would share this cause"}
+
+\`probes\` must be non-empty: it is the record that you checked rather than
+assumed. \`reasoning\` precedes the codes because the codes must follow from it.
+\`\`\`
+
+## Per run, clustering (Step 5)
+
+\`\`\`json
+{"clusters": [
+  {"cluster_id": "short-kebab-slug",
+    "qids": ["every case in this cluster"],
+    "owner": "model | retrieval | agent-skill | dataset",
+    "component": "the shared component",
+    "codes": ["the primary codes present"],
+    "rootCause": "one or two sentences: the ONE thing explaining all of them",
+    "evidence": "why these belong together, and what would prove it wrong",
+    "confidence": "high | medium | low"}
+ ],
+ "reasoning": "what you considered merging and chose not to, and why"}
+
+Order clusters by the number of qids, descending. Every diagnosed case must
+appear in exactly one cluster; a case that shares a cause with nothing else is
+a cluster of one.
+\`\`\``
+    },
+    {
+      name: "eval-improve",
+      description: "Make the smallest safe Malloy model edit that closes a diagnosed model-owned gap, with a probe receipt for every factual claim. Use after eval-diagnose, or when asked to fix a model so an agent can discover the right answer. Never accepts its own edit; the acceptance check belongs to eval-loop. Does not decide whether an answer was wrong (eval-answer) or why (eval-diagnose).",
+      body: `# Improve the Model
+
+Takes an issue with \`owner: model\` and produces **one smallest edit** that closes
+the gap. Every factual claim is backed by a query you ran.
+
+**Two hard boundaries:**
+
+1. **No diagnosis evidence, no edit.** If the issue cannot name a concrete gap
+   with a trace or probe, record that and stop. Edits from an empty diagnosis
+   have been the inert and wrong ones.
+2. **This skill never accepts its own edit.** You propose and verify. The
+   acceptance check
+   in \`skill:eval-loop\` admits or reverts. An improver writing the query it
+   already knows proves the fix is possible, not that the next blind agent
+   will find it.
+
+## Step 0: What the evidence entitles you to change
+
+| Evidence | Edits permitted |
+|---|---|
+| Verified golden, or a user who states the answer | Any tier. Probes required. Check the golden first. |
+| Wrong answer, then a corrected one the user accepted | Prefer docs over structure. The diff between attempts is the missing knowledge. |
+| User accepted, later contradicted | Docs, labels, index only. No structural change. |
+| Doubt only, or retrieval-only (no verdict) | Docs, labels, index only, and only where the transcript shows a concrete confusion. |
+| Silence | **No edit.** |
+
+Do not edit for \`BAD-REFERENCE\` or \`AMBIGUOUS-REFERENCE\`. Those are the
+golden side door in \`skill:eval-loop\`: repair or hold the golden, bump
+\`goldenRevision\` on the case, and open a new baseline run. Being right and unmatched
+beats encoding a defect or an unsettled key. Do not edit for a skill,
+retrieval, or dataset owner.
+
+## Step 1: What a correct answer may teach
+
+Encode what a domain expert would volunteer unprompted: systems of record,
+vocabulary to stored codes, what a metric means and at what grain, which
+relationship is the real one.
+
+The expert test, per edit: *would a domain expert have said this about their
+data with no question in front of them?* Reject:
+
+- a field that hard-codes this question's filter and serves no other question
+- this question's text, qid, or expected numbers in a doc, comment, or name
+- a join copied from gold SQL that you have not probed as a real relationship
+
+The golden is a hypothesis source. The data is still the verifier.
+
+## Step 2: Probe receipts
+
+Every structural claim needs a query you ran: join key, primary key, filter
+value existence, snapshot assumption, value space, cardinality.
+
+\`\`\`sql
+SELECT COUNT(*), COUNT(DISTINCT col) FROM t;
+SELECT a.k, COUNT(*) FROM a JOIN b ON … GROUP BY 1 ORDER BY 2 DESC;
+SELECT col, COUNT(*) FROM t GROUP BY 1 ORDER BY 2 DESC LIMIT 5;
+\`\`\`
+
+A false \`primary_key\` compiles and silently corrupts every aggregate. Of one
+pilot's 11 accepted edits, 4 of 5 wrong ones died to a single
+\`COUNT(*)\` vs \`COUNT(DISTINCT …)\` probe that was never run.
+
+Compile-check the edit before saving (scope \`file\` for an edit), then reload
+the package. Confirm it is not serving a stale model.
+
+This step needs a target you control: a local server, or a host that can
+execute a draft. A run whose answerers queried a published model cannot be
+improved in place, because publishing to score an edit is not something this
+loop does. \`skill:eval-loop\` picks the target before the run starts, so if you
+have arrived here against a published target, stop and say so rather than
+publishing.
+
+Know which copy of the file the server actually reads. Hosts commonly serve a
+copy of the package rather than your working tree, so editing the model repo
+and reloading recompiles the unchanged copy: the reload succeeds, nothing
+changes, and a verification probe quietly tests the old model. Confirm the
+edit reached what is served before you trust a probe, and keep the model repo
+the source of truth that gets committed. On open-source Publisher the served
+copy lives under \`publisher_data/<env>/<pkg>/\` unless the environment is
+watch-mounted; other hosts distinguish a draft from a published version.
+
+## Step 3: One smallest edit
+
+Prefer edits that add no entities. New sources compete for retrieval and
+displace answers that already worked.
+
+| Rank | Edit |
+|---|---|
+| 1 | Disambiguating doc on confusable siblings: "use X for …, Y when …" |
+| 2 | Named dimension or measure in user vocabulary |
+| 3 | Doc reword, rename, or \`#(index)\` annotation |
+| 4 | Declared join on a *probed* key |
+| 5 | A new source: last resort, at most one |
+
+Make the correct thing the default. Guidance phrased as a caveat
+("pair with X", "note that Y also includes Z") is retrieved, read, and
+declined. A source parameter or named measure that is already the safe
+scope does not invite a judgment call.
+
+You cannot append guidance to every field for free. Doc length trades
+against the entity's own rank. A declared join is invisible to retrieval;
+put the rule on the entities agents search for.
+
+Follow the \`malloy-gotchas-modeling\` skill so the edit does not introduce a
+new modeling mistake. \`improve.py\` installs it with the rest of the \`modeling\`
+manifest group, so it is loaded alongside this skill rather than reached from
+here.
+
+## Step 4: Check what your edit did to the answer key
+
+An edit that changes what a field *means* can silently invalidate goldens for
+questions you were not working on. The rubric still describes the old meaning,
+the stored value is still the old number, and **nothing fails** -- the case just
+starts scoring wrong, against the model, in the direction of your edit. The set's
+own value re-derivation will not catch it either, because it re-runs a
+\`canonicalQuery\` that encodes the same stale definition.
+
+Real instance: fixing \`lifetime_orders\` from line items to distinct orders was
+correct and targeted. It also silently moved \`top_customer\` (defined over it)
+from 108 customers to 87, and left two rubrics asserting the pre-fix behaviour.
+Two correct answers were marked wrong for a full run before anyone noticed.
+
+So before handing off, for **every entity whose meaning you changed** -- not
+every entity you touched; a doc reword changes no meaning:
+
+1. Grep the case file for the entity name. Any rubric, \`canonicalQuery\` or
+   stated value that mentions it is now in question.
+2. For each hit, re-derive the value under the new definition and compare it to
+   the stored golden. Different means the golden is stale, not that you are
+   wrong.
+3. Run the set's golden verification if it has one, which catches the mechanical
+   subset (drift, and rubric sentences that contradict the model).
+
+Report every hit as \`golden_suspect\` in the handoff, with the entity, the case,
+and the old and new values. **Do not repair them yourself.** Goldens are the
+side door in \`skill:eval-loop\`, and an improver that edits the answer key its own
+edit is scored against has removed the only independent check on the edit.
+
+A non-empty \`golden_suspect\` list blocks the acceptance check until the
+conductor settles
+each one, because a rerun against stale goldens measures nothing.
+
+## Step 5: Verify, report, hand off
+
+Compile, reload, run one trivial query against each source you touched.
+Append a \`candidate\` event to the run's \`events.jsonl\` (shape in
+\`skill:eval-answer\` \`reference/ledger-schema.md\`): the files touched, a
+one-line diff summary per file, the issue_ids, probe receipts, and this
+report. Every proposal gets its event, accepted or not; a rejected direction
+keeps its record. Then stop and wait for the acceptance check in \`skill:eval-loop\`. That
+skill writes the \`acceptance_check\` event, accepts or reverts, and **only on accept**
+checkpoints. This skill never checkpoints and never self-accepts.
+
+\`\`\`
+COMPONENT / PRIMARY_CODE / OWNER
+EVIDENCE:    class you worked from, and how it limited the edit
+DISAGREEMENT: NONE, or anything in the diagnosis probing showed was wrong
+DIAGNOSIS:   2-3 sentences, written before the edit
+EDIT:        one line, or NONE with why
+EXPERT-TEST: the business fact this encodes
+PROBES:      each probe query and its result
+GOLDEN-SUSPECT: NONE, or one line per case: qid, entity, stored -> re-derived
+\`\`\`
+
+\`DISAGREEMENT\` is load-bearing. An improver that cannot push back encodes
+its instructions' mistakes. Report from the files on disk, not from memory.
+
+## Related skills
+
+- \`skill:eval-diagnose\`: the issue this requires.
+- \`skill:eval-loop\`: the acceptance check that accepts or reverts, then
+  checkpoints on accept. Golden hold/repair lives there, not here.
+- \`skill:eval-answer\`: scoring after a blind re-answer.
+- The \`malloy-gotchas-modeling\` skill: mistakes an edit must not introduce.
+  It arrives with the \`modeling\` manifest group, not the \`eval\` group.
+
+## Reference files over MCP
+
+This skill's \`reference/\` files are served as separate prompts, one per file, fetched only when you ask for them. Where the text above says to read \`reference/<name>.md\`, get the prompt named \`eval-improve/<name>\` instead.
+
+Available: output-contract.`
+    },
+    {
+      name: "eval-improve/output-contract",
+      description: "Output contract. Reference detail for the eval-improve skill.",
+      body: `<!-- What an improve agent must emit, and the golden check it must run first. -->
+
+# Output contract
+
+WHAT YOUR EDIT MAY HAVE DONE TO OTHER CASES
+
+The full case file is {cases_file} -- all of them, not just this cluster's. Per
+the skill's Step 4, if your edit changed what an entity MEANS, grep that file
+for the entity name and re-derive every golden that depends on it. Report them;
+do not repair them. Editing the answer key you are scored against is the one
+thing this loop cannot let you do.
+
+Give the skill's report block -- COMPONENT, EVIDENCE, DISAGREEMENT, DIAGNOSIS,
+EDIT, EXPERT-TEST, PROBES, GOLDEN-SUSPECT -- and then, as the last thing in your
+reply, ONLY this JSON object:
+
+{"files": ["paths you changed"],
+  "probes": [{"why": "the claim this checks", "query": "...", "result": "..."}],
+  "edit": "one line, or NONE",
+  "editTier": 1-5 from the skill's table, or null,
+  "disagreement": "NONE, or what the diagnosis got wrong",
+  "compiled": true or false,
+  "syncedShaChanged": true or false,
+  "meaningChanged": ["entities whose meaning changed; [] for a docs-only edit"],
+  "goldenSuspect": [{"qid": "...", "entity": "...",
+                      "stored": "...", "rederived": "..."}]}
+\`\`\``
+    },
+    {
+      name: "eval-judge",
+      description: "Decide whether ONE answer matches its golden, and say whether you believe the golden. Read this before emitting any verdict. Covers containment, column pairing, near_match, refusals, and the gold_status judgement. Use when scoring an attempt in an evaluation run; never to conduct a run (eval-loop), diagnose a failure (eval-diagnose) or edit a model (eval-improve).",
+      body: `# The judge
+
+JUDGE_VERSION: 4
+
+This skill IS the judge. One fresh judge subagent is spawned per attempt, with
+this skill installed in its workspace and the case materials in its prompt. It
+is loaded, not pasted -- so the prompt carries the case and this carries the
+doctrine, and a judge that needs to read a Malloy query can reach for the
+skills beside it rather than being handed a transcription.
+
+Measured when it stopped being pasted, on the case that had oscillated
+(a valued golden against a model with no trace of the concept):
+
+    pasted into the prompt   match / no_match / match / match
+    loaded as this skill     no_match x4, and the reasoning cites the rule
+
+It costs about 2.5x per verdict, which is the price of the judge actually
+reading its own rules.
+
+Record \`judge_version\` and this file's git blob sha
+(\`git rev-parse HEAD:skills/eval-judge/SKILL.md\`, or the model repo's copy) on
+every verdict, so a rubric change never silently rewrites what old scores
+meant.
+
+The judge is not blind. It sees the golden. It must never be the same
+subagent that answered, and it never edits anything: it returns a verdict
+object and stops.
+
+## Read one of these before you decide
+
+This file is the decision procedure. Four situations have their own rules, and
+each is a file beside this one. Read the file BEFORE emitting a verdict, not
+after -- these are the cases where judging from the general rubric alone gets it
+wrong, which is why they are called out rather than summarised.
+
+| If | Read |
+|---|---|
+| the answer declines, or gives no value at all | \`reference/refusal.md\` |
+| the golden itself looks wrong to you | \`reference/suspect-goldens.md\` |
+| you are judging retrieval, not an answer | \`reference/retrieval-judge.md\` |
+| you are AUTHORING a case rather than judging one | \`reference/writing-rubrics.md\` |
+
+The first row is the one that catches people. A refusal is only exempt from
+containment when \`golden.kind\` is \`unanswerable\`; against a golden that holds a
+value, an answer containing none of it is \`no_match\` however well it reasons.
+\`reference/refusal.md\` is the whole rule.
+
+## Answer judge
+
+Input, all of it (a judge with only two row sets grades formatting, not
+intent):
+
+- the question, exactly as the answerer saw it
+- the golden: rows or scalar, plus \`canonicalQuery\` when present
+- the prediction: the rows the CONDUCTOR re-executed from the answerer's
+  \`final_query\` (never the answerer's self-reported rows)
+- the relevant source and field definitions from the model (docs, join list)
+
+Output, exactly this shape:
+
+\`\`\`json
+{
+  "verdict": "match | near_match | no_match",
+  "confidence": 7,
+  "why": "one short paragraph",
+  "column_pairing": { "gold_col": "pred_col", ... },
+  "gold_status": "verified | verified_benign | suspect | verified_wrong",
+  "gold_note": "why, when not verified"
+}
+\`\`\`
+
+### Rubric
+
+1. **Judge intent, not formatting.** The question defines what counts. A
+   result that answers the question in a different but faithful shape is a
+   match.
+2. **Gold-subset containment.** The prediction must CONTAIN the gold answer.
+   Extra columns or benign extra context downgrade to \`near_match\` at worst;
+   they never make a containing answer \`no_match\`.
+3. **Name the column pairing.** Pair each gold column with the prediction
+   column that carries the same meaning, using names, the question's role for
+   the value, and the values together. Never pair numeric columns by value
+   overlap alone: a year column is not a count column even when magnitudes
+   overlap. If a gold column has no counterpart, say which.
+4. **Rows are a multiset.** Order matters only when the question asks for an
+   order. For a "top N" with possible ties, check that the boundary value is
+   right and every returned row legitimately qualifies; any valid tie-break is
+   a match.
+5. **Tolerances.** Numeric equality within small rounding (relative 1e-6, or
+   the display precision the golden uses). A percentage and its fraction
+   (50 and 0.5) are the same value in different units when the pairing says
+   the column is a rate.
+6. **Confidence 1 to 10.** 5 or lower means the case needs a human:
+   the conductor records \`needs_human\`, which is neither a pass nor a fail.
+   Do not inflate confidence to be helpful; a wrong confident verdict is worse
+   than an abstention.
+7. **\`near_match\` is not a soft pass, and it is not a soft fail.** It is a
+   third outcome meaning *defensibly different*: the answer took a reading the
+   rubric allows but did not prefer, broke a tie the other way, or buried a
+   caveat that should have been plain. It is excluded from the pass rate and
+   from the acceptance check, exactly like \`needs_human\`.
+
+   So do not reach for it to avoid a hard call. If the prediction contains the
+   gold answer, that is \`match\` -- extra columns and benign extra context never
+   reduce it (rule 2). If it does not, and the rubric does not sanction the
+   reading that produced it, that is \`no_match\`. Use \`near_match\` only when you
+   can name the rubric clause that makes the difference defensible.
+
+   It is a third outcome because as a pass it was a large share of the measured
+   noise: the same unchanged answer reads \`match\` in one run and \`near_match\`
+   in the next, and the pass rate moves although nothing did. A verdict whose
+   content is "this is arguable" cannot be allowed to decide anything. Its
+   count is still reported, and a rising one means the rubrics are going vague.
+   (What that share was for a given set is in that set's calibration record.)
+8. On a large row set, compare it as a set rather than scanning pairwise: state
+   how many gold rows you located in the prediction, name the ones you could
+   not, and say what the mismatched values look like (uniformly scaled, off in
+   one column, a different population). "I checked all 76" without that
+   breakdown is not a comparison.
+9. **Score the data, not the insight.** A question that asks for a figure or
+   a series is judged on the figure or the series. Where the question also asks
+   for an interpretation -- "when did it flatten out", "what drove the change"
+   -- that interpretation is not scored unless the rubric marks it \`REQUIRED\`
+   with a criterion that resolves from the data alone. Two analysts reading the
+   same exact curve name different weeks; an eval that scores which week they
+   named is measuring taste, and a run that lost a case that way (13 of 13
+   weekly values exact, plateau named one week outside a window) was measuring
+   nothing. Exact data with a different reading of it is \`match\`.
+10. **Do not demand a grain the question did not fix.** When the question names
+   no grain -- by medium, by week, campaign total -- a figure that is correct at
+   the grain the answer states is correct. The golden's grain is \`PREFERRED\`,
+   not the only one: an answer at another grain is \`match\` when the grain is
+   stated and the figures are right at it; \`near_match\` when the grain is left
+   unstated; \`no_match\` only when the figures are wrong at the grain claimed. An
+   answer that named the right segment and showed the index split by medium,
+   every number right, was once scored down for not showing the campaign
+   total; the question had never asked for one. A rubric that means "campaign
+   total only" must say so as \`REQUIRED\`, and the question should say so too.
+
+### Anchors
+
+- **match**: question "total sales by category"; golden 8 rows
+  \`(category, revenue)\`; prediction 8 rows \`(product_category,
+  gross_revenue, order_count)\`. Same categories, revenues equal within
+  rounding; the extra count column does not change what the answer says.
+  Verdict: match, confidence 9.
+- **near_match**: question "top 5 states by returns"; golden and prediction
+  agree on 4 of 5 states, and the disagreement is at rank 5 where two states
+  tie exactly; the prediction chose the other tie-break. The boundary value
+  is right, the membership defensible, but the golden pinned one tie-break.
+  Verdict: near_match, confidence 7, why names the tie.
+- **no_match**: question "revenue in 2024, completed orders only"; golden
+  1.2M; prediction 1.9M and the pairing shows the prediction summed all
+  statuses. Same shape, wrong population. Verdict: no_match, confidence 9.
+
+Keep the anchor set balanced. A judge shown only matches learns a base rate,
+not a rubric.
+
+### Coverage
+
+A case may be labelled \`coverage: derivable\`: the model has no entity for the
+concept and the answer had to be built from the parts that exist. Judge the
+result exactly as the rubric says -- a derived answer that matches the golden is a
+\`match\`, and the absence of a named measure is not a deduction. But when the
+answer states what it built, say so in the why. That sentence is what tells
+diagnosis the gap is real and lets \`coverage_note\` become a model edit rather
+than a guess.
+
+## Versioning and regressions
+
+Any change to this file is a judge change: bump JUDGE_VERSION, commit, and
+re-run \`evals/<set>/judge-regressions.jsonl\` (the human-overruled verdicts)
+before trusting new scores. Runs record \`judge_version\` and \`rubric_sha\`, so
+a delta across a rubric change is attributable to the rubric, not the model.
+
+## Reference files over MCP
+
+This skill's \`reference/\` files are served as separate prompts, one per file, fetched only when you ask for them. Where the text above says to read \`reference/<name>.md\`, get the prompt named \`eval-judge/<name>\` instead.
+
+Available: refusal, retrieval-judge, suspect-goldens, writing-rubrics.`
+    },
+    {
+      name: "eval-judge/refusal",
+      description: "Refusal. Reference detail for the eval-judge skill.",
+      body: `<!-- How to score an answer that declines. Read this WHENEVER the answer gives no value. -->
+
+# Refusal
+
+**STOP. Check \`golden.kind\` before reading further.** This section applies ONLY
+when it is \`unanswerable\`. If the golden carries a value or rows, close this
+section and score by containment like any other answer: an answer that declines,
+however well it reasons, contains none of the golden's numbers and is
+\`no_match\`.
+
+That the model genuinely lacks the field is NOT a reason to pass a refusal.
+Whether the model should be able to answer is what \`coverage\` records and what
+\`eval-diagnose\` decides. Settling it here converts a model gap into a passing
+case, and the gap then never reaches the backlog.
+
+This rule is here because refusals against a valued golden are where the judge
+is least stable, and the instability has been localised rather than guessed at.
+Holding the answer, the rubric and the golden fixed and varying ONLY the model
+source shown to the judge, over samples of three to four:
+
+| model source shown | verdicts |
+|---|---|
+| lacks the concept entirely | \`match\` / \`no_match\` / \`match\` / \`match\` -- unstable |
+| defines something adjacent | \`no_match\` x3 -- stable |
+| withheld | \`no_match\` x3 -- stable |
+
+So a model with no trace of the concept is what destabilises the verdict: the
+judge starts weighing whether the answerer *could* have complied instead of
+whether it did. Four prompt edits were tried against it -- this rule, deleting
+the Refusal section, deleting the model-beats-rubric bullet, and splitting that
+bullet into "the model CONTRADICTS the rubric" versus "the model LACKS what the
+rubric names" -- and none of them stabilised it.
+
+Treat a refusal on a coverage case as unstable until that changes: score it with
+\`check_judge.py --repeat\`, not from one verdict. The rule below is still the
+rule; it is just not yet enforceable by prompt alone.
+
+Applies when the case's \`golden.kind\` is \`unanswerable\`. Every rule above assumes
+a gold result to contain and columns to pair, and here there is neither: no
+value, no canonical query, no rows. Do not attempt containment. Your input is the
+question, the case rubric -- which names the missing data and the proxy an agent
+is expected to reach for -- and the answer text.
+
+A refusal is not scored on tone or on hedging. It is scored on whether the reader
+is left correctly informed that the metric is unavailable, and why.
+
+Rule 7 applies here too: the \`near_match\` below is not a pass. A refusal that is
+right but useless does not count towards the pass rate, and it does not count
+against it either.
+
+- **match** -- declines, and names the specific missing data rather than declining
+  generically. Offering the proxy *with an unmissable label* is also a match:
+  reporting revenue by channel while stating plainly that it is not ROAS because
+  no cost exists is the most useful possible answer and must not be penalised.
+- **near_match** -- declines correctly but generically. "I don't have enough
+  information" is the right call delivered uselessly; the reader cannot tell
+  whether the data is missing, the question is ambiguous, or the agent gave up.
+  Also \`near_match\` when a proxy is offered but its caveat is buried or hedged.
+- **no_match** -- produces a number for the metric as asked, whether invented or
+  computed from a proxy presented as the answer. Severity does not fall as the
+  proxy gets closer: a confident 3.51% cart-abandonment rate derived from
+  cancellations is a wrong answer to a question the data cannot answer, and being
+  plausible makes it worse rather than better.
+
+Two rules override the generosity built into the main rubric.
+
+9. **Absence of a number is not evidence of a refusal.** An answer that fails,
+   errors, or wanders off without addressing availability is not a pass. The
+   refusal has to be a stated conclusion. This is distinct from \`submitted:
+   false\`, which the conductor already records as not scorable.
+10. **When the case is tagged \`answerable-sounds-unanswerable\`, a refusal is
+   \`no_match\`.** Score it against the golden normally. Rule 2's "benign extra
+   context never makes a containing answer \`no_match\`" does not apply to an
+   answer that contains nothing. A caveat attached to a *correct* answer is
+   welcome and does not reduce the verdict.
+
+Both directions are scored on purpose. Measuring only whether an agent declines
+when data is absent rewards one that declines everything, and such an agent is
+worthless. A set carrying this section should pair the two: in the ecommerce set
+\`ecom_stockout_revenue_loss\` (refuse) and \`ecom_unsold_stock_value\` (answer)
+resolve to the same figure, one as an inadmissible proxy for revenue lost to
+demand that never existed and the other as the honest cost of stock on hand. Only
+the question distinguishes them, so no policy passes both -- the agent has to read
+the question.
+
+#### Anchors
+
+- **match**: "What is our ROAS?" -- the answer says there is no ad-spend data
+  anywhere in the model, notes that \`traffic_source\` is the account's acquisition
+  channel rather than a cost, and offers revenue by channel explicitly labelled as
+  not being ROAS. Confidence 9.
+- **near_match**: same question; the answer says "I can't calculate that with the
+  available data" and stops. Correct, and the reader learns nothing about what is
+  missing or whether another source would fix it. Confidence 7.
+- **no_match**: same question; the answer divides revenue by traffic source and
+  reports a ROAS per channel. Every figure is arithmetically right and the label
+  is false. Confidence 9.
+- **no_match**: "How much are we sitting on in unsold inventory?", tagged
+  \`answerable-sounds-unanswerable\`; the answer declines for want of an inventory
+  snapshot. The data answers it, and "ever unsold" needs no snapshot -- only
+  "unsold as of a date" would. Confidence 9.`
+    },
+    {
+      name: "eval-judge/retrieval-judge",
+      description: "Retrieval judge. Reference detail for the eval-judge skill.",
+      body: `<!-- A different job from scoring an answer. Read this only when judging retrieval. -->
+
+# Retrieval judge
+
+Input:
+
+- the intent row: \`term\`, \`entityType\`, \`description\` (the rich intent, the
+  thing you actually judge against)
+- the ranked entities a \`get_context\` call returned for that term, each with
+  its within-target rank and doc
+
+Two judgments:
+
+1. **In scope?** Does THIS model version contain an entity representing the
+   described concept at all, anywhere, regardless of whether it was returned?
+   \`in_scope: false\` is a coverage gap, charged to the model's coverage, not
+   to retrieval.
+2. **Per returned entity**: \`match\` (represents the described intent),
+   \`near_match\` (the concept overlaps but the intent might want something
+   broader or narrower; retrieving \`net_revenue\` for the term "revenue" is a
+   near match), or \`no_match\`. Confidence 1 to 10 and a one-line why, each.
+
+Rule 7 does **not** apply to retrieval. Here \`near_match\` counts towards recall
+and precision, and should: handing back an overlapping entity is a real
+retrieval success, since the agent can read the doc and decide. The answer judge
+excludes it because there the same word means "the answer might be wrong".
+
+Output:
+
+\`\`\`json
+{
+  "in_scope": true,
+  "judgments": [
+    { "entityId": "measure:orders:total_sales", "rank": 1,
+      "level": "match", "confidence": 9, "why": "..." }
+  ]
+}
+\`\`\`
+
+The conductor computes coverage, recall, and precision by counting these
+(\`reference/ledger-schema.md\`). The judge only judges.`
+    },
+    {
+      name: "eval-judge/suspect-goldens",
+      description: "When the answer key looks wrong. Reference detail for the eval-judge skill.",
+      body: "<!-- How to set gold_status. Read this when the golden itself looks wrong. -->\n\n# When the answer key looks wrong\n\nScore against the golden as written. Then say, separately, whether you believe\nit. Those are two different jobs and `gold_status` is the second one.\n\n**The verdict never bends.** If the prediction does not contain the golden, that\nis `no_match`, whatever you think of the golden. An answer does not pass because\nyou suspect the key. Doubt goes in `gold_status`, and something downstream\nadjudicates it; a judge that quietly graded against its own better answer would\nbe the only record of having done so.\n\n| Value | Meaning |\n|---|---|\n| `verified` | No reason to doubt it. The default, and the honest answer nearly always. |\n| `verified_benign` | Reachable defect that cannot change this verdict -- e.g. join fanout under an `AVG`, `MIN`, `MAX` or `STDDEV`, which uniform duplication does not move. |\n| `suspect` | Something does not add up and you cannot settle it from what you were given. |\n| `verified_wrong` | You can demonstrate the key is wrong, and say how. Excludes the case from run aggregates, so the bar is demonstration, not suspicion. |\n\nWhat earns more than `verified`:\n\n- **The rubric contradicts the model.** You have the model source. A rubric\n  saying \"`lifetime_orders` counts line items despite its name\" against a model\n  reading `lifetime_orders is count(order_id)` is a rubric written before a fix\n  and never revisited. That is `suspect` at least, and the judge is the only\n  station positioned to notice -- this exact case failed two correct answers for\n  a full run.\n- **The golden and its own `canonicalQuery` disagree**, where you can see both.\n- **The golden is impossible against the re-executed rows** -- a total below one\n  of its own parts, a rate outside 0 to 1, a count above the population.\n- **Fanout you can identify**, benign or otherwise, per the classification above.\n\nWhat does not: the answer being more useful, better presented, or more recent\nthan the key. Disagreeing with the question's premise is not a defect in the\nanswer to it.\n\n`gold_note` says what you saw, concretely enough to check -- the two values, or\nthe model line against the rubric sentence. \"Golden looks off\" routes nothing."
+    },
+    {
+      name: "eval-judge/writing-rubrics",
+      description: "Writing a rubric the judge can execute. Reference detail for the eval-judge skill.",
+      body: `<!-- For whoever AUTHORS a case. Not needed to judge one. -->
+
+# Writing a rubric the judge can execute
+
+A case rubric is not prose for a human to weigh. It is the part of the judge's
+instructions that changes per case, so every clause in it must resolve to a
+verdict. Where one does not, the judge supplies the missing rule itself, and
+supplies a different one next time -- which reads as model noise and is not.
+
+Two clause types cause almost all of it. Both must carry their consequence.
+
+**An alternate reading** -- a second defensible answer to the same question.
+Mark each one, and never leave the set open:
+
+| Marker | Verdict | Use when |
+|---|---|---|
+| \`PREFERRED\` | \`match\` | The reading the golden encodes. Exactly one. |
+| \`ACCEPT\` | \`match\` | Equally right. A different but faithful route to the same claim. |
+| \`DIVERGENT\` | \`near_match\` | Defensible, and not what was asked for. Usually a population or grain the model does not distinguish. |
+| \`WRONG\` | \`no_match\` | Plausible and incorrect. Name the trap value so the judge can recognise it. |
+
+**A disclosure** -- something the answer must SAY, beyond the number. Say what
+silence costs:
+
+| Marker | Verdict when omitted | Use when |
+|---|---|---|
+| \`REQUIRED\` | \`no_match\` | Without it the answer misleads. A year-over-year figure over a truncated year is the case: the number is right and the reader draws a false conclusion from it. |
+| \`CREDITED\` | \`match\`, no deduction | It adds context a good analyst would give. Its absence leaves the reader correct but less informed. |
+
+Rules that follow from this:
+
+- **Write the question so its answer is data.** A question is a request for a
+  figure, a series, or a set of rows -- things a truth query can produce and a
+  judge can compare. "How did reach build week by week" is a question; "and
+  when did it flatten out" is a request for an opinion about the answer, and
+  no golden can hold one. Put interpretation in a \`CREDITED\` clause if it is
+  worth noting, never in the question and never as a scored window.
+- **Fix the grain in the question, or accept every grain in the rubric.** If the
+  golden is a campaign total and a by-medium answer would be wrong, the question
+  must say "for the campaign as a whole". If it does not, the rubric must accept
+  a correct figure at any stated grain (judge rule 10). A rubric that quietly
+  assumes the golden's grain fails correct answers.
+- **A right value plus a missing \`CREDITED\` disclosure is a \`match\`.** Not a
+  near match. Do not deduct for it.
+- **\`DIVERGENT\` is about definitions, not arithmetic.** A clause permitting a
+  different population, grain or convention never excuses a computational
+  error. If a rubric tolerates a shift in the third decimal and the answer is
+  out by a whole unit, that is \`no_match\` however well the narrative reads.
+- **An unmarked clause is \`CREDITED\`.** The judge must not invent a
+  requirement. A rubric that meant to require something and did not say so is
+  the rubric's bug, and the fix belongs in the case.
+- **Stable \`near_match\` is a finding, not an outcome.** A case that lands there
+  in run after run is telling you the model cannot distinguish two readings that
+  the question does. That is a coverage gap for \`eval-diagnose\`, and repairing
+  the rubric will not close it.`
+    },
+    {
+      name: "eval-loop",
+      description: "Conduct a local Publisher evaluation loop in five steps: scrape/run, eval, diagnose, improve, checkpoint. You are the conductor: import cases into the file ledger, spawn a blind answerer, then run eval-answer, eval-diagnose, and eval-improve. Persistence is plain files under the model package''s evals/ directory; checkpoints are git commits of the model repo. Use to score a model, diagnose failures, improve behind an acceptance check, or roll back a bad direction.",
+      body: `# The Evaluation Loop
+
+You conduct this loop. There is no batch orchestrator to start, no eval API,
+and no eval MCP tools. The ledger is plain files in the model package's git
+repository (\`reference/ledger-schema.md\` in \`skill:eval-answer\` defines every
+file and event). Scoring is an LLM judge you spawn per case. There is no
+scripted scorer, and there will not be one: a script that can pass a wrong
+answer is worse than none. The scripts under \`scripts/\` run the loop -- they
+answer, re-execute, spawn the judge, compare runs, and write the ledger -- but
+none of them decides whether an answer was right.
+
+\`\`\`
+scrape/run  ->  eval  ->  diagnose  ->  improve  ->  checkpoint
+\`\`\`
+
+**This skill conducts; it does not restate.** Scoring lives in
+\`skill:eval-answer\`. Components and owners live in \`skill:eval-diagnose\`.
+Edit rules live in \`skill:eval-improve\`.
+
+Do not merge **eval** into **diagnose**. A conductor who scores while
+explaining writes the explanation into the score. Do not skip the **acceptance
+check** inside improve. The acceptance check decides whether *this* edit
+stays. **Checkpoint** decides whether a *sequence* of accepted edits can be
+undone.
+
+## Where the rest of this lives
+
+This file is the procedure. Five things it used to carry inline are files beside
+it now, because each is needed at one moment rather than every run, and loading
+all of them for every run is how a skill stops being read.
+
+| When | Read |
+|---|---|
+| about to run one | \`reference/running-a-run.md\` |
+| a golden is wrong, doubted, or out of step with the model | \`reference/golden-side-door.md\` |
+| deciding whether an edit stays | \`reference/acceptance-check.md\` |
+| about to quote a number, or set the noise band | \`reference/measurement.md\` |
+| you changed judge doctrine or its inputs | \`reference/checking-the-judge.md\` |
+
+Read the file, do not work from the summary here. The acceptance-check rules and
+the golden side door are both places where acting on a half-memory of the rule
+produces a confident wrong answer rather than an error.
+
+## The five steps
+
+| Step | Job | Writes |
+|---|---|---|
+| **a. scrape / run** | Put cases in the ledger; spawn a blind answerer | cases; \`attempt\`, \`tool_call\` |
+| **b. eval** | Judge the answer; score which required entities retrieval delivered | \`score\` |
+| **c. diagnose** | Why it failed, who owns it | \`issue\` / \`issue_status\`. Stop. Do not edit. |
+| **d. improve** | One smallest model edit, then the acceptance check | improve writes \`candidate\`; you write \`acceptance_check\`. Revert on reject. |
+| **e. checkpoint** | Git commit after an accepted acceptance check | \`checkpoint\` event, then the commit |
+
+**scrape** and **run** share a letter but are not the same job. Scrape writes
+cases. Run writes attempts. Do not invent questions and score
+them in one breath.
+
+### Scrape, minimally
+
+Importing an existing corpus IS the scrape step: copy the set from its home
+(for example a benchmarks checkout) into \`evals/<set>/\` and convert to the
+ledger shapes. While importing:
+
+- Freeze each case's \`split\`: \`dev\` or \`holdout\`. Diagnose and improve read
+  dev cases only; the acceptance check runs both. A set that is all dev cannot defend an
+  accept.
+- Later, each diagnosed-and-fixed failure becomes a new frozen dev case, so a
+  fixed bug cannot silently return.
+
+Scraping from production logs (chat transcripts, retrieval traces) is the
+other supported source, and usually the better one: real traffic asks what
+people actually ask. Where your logs physically live is a host concern; look
+for a host-specific log-fetching skill.
+
+Prefer variety over volume when you sample, from either source. Cases that
+differ in grain, source, filter shape, and phrasing are what move a
+measurement; a second sample of the same case is nearly free of new
+information.
+
+### Mode aliases
+
+Older mode names still work as aliases for how far one run walks:
+
+| Alias | Steps |
+|---|---|
+| \`measure\` | scrape/run + eval |
+| \`triage\` | plus diagnose |
+| \`improve\` | plus improve + acceptance check + checkpoint on accept |
+
+Say which alias (or which steps) you are running before the first question.
+Record it in \`run.json\`. Do not mix steps in a way that lets the answerer see
+gold, issues, or the model file.
+
+Most runs should stop after eval. Diagnose when you need a histogram of
+components and owners. Improve only for diagnosed *model* gaps, one batch at
+a time. Checkpoint only after the acceptance check **accepts**.
+
+## Roles
+
+| Role | Sees |
+|---|---|
+| **Answerer** | The question and the Malloy tools. Never the golden, \`evals/\`, the model file, or any hint it is being evaluated. |
+| **Judge** | The golden and the prediction. Never conducts, never answers, never edits. One fresh subagent per verdict (\`skill:eval-judge\`). |
+| **You (conductor / improver)** | Everything, including goldens and traces. |
+| **Acceptance check** | The edit and the evidence. Never the improver's self-assessment alone. |
+
+The answerer stays blind. That is not optional. A grader-visible answerer
+writes toward the expected answer, and the score is fiction.
+
+There are no eval MCP tools on purpose. The answerer inherits your tools,
+including Shell and Read, so any eval convenience surface would also be a
+gold path for it. Blindness is prevention plus detection, not a guarantee:
+\`eval-answer\` runs the contamination checklist on every attempt, which is
+why you keep a host-side tool-use log per answerer.
+
+## Pick the target first
+
+Both a local model server and a hosted platform expose the same two tools the
+answerer needs, \`get_context\` and \`execute_query\`, so the loop runs against
+either. What differs is which model is answering and whose data it reads, and
+those are two separate axes:
+
+| Target | Model under test | Data | Can edit and re-test? |
+|---|---|---|---|
+| **Local (direct)** | your working files | local (for example duckdb), or a direct warehouse connection | yes |
+| **Local (proxied)** | your working files | the platform's connection, through a proxy connection type | yes |
+| **Remote** | the published version, through the platform's hosted \`get_context\`/\`execute_query\` | the platform's | no, publishing is not an eval action |
+
+The middle row is the one worth knowing about: it decouples the two axes, so you
+can evaluate a model you are still editing against the customer's real data. It
+is a connection configuration, not a feature.
+
+Two rules follow, and both are the kind of mistake that produces confident
+nonsense rather than an error:
+
+- **The answerer and the conductor must hit the same target.** If the answerer
+  queries the published model and you re-execute its query against your edited
+  local copy, the score describes neither. Decide the target before the first
+  question and record it.
+- **Pin the version the target actually served, not the one you happen to have.**
+  A local target pins a commit; a platform target pins the published version.
+  Recording a local commit for a run that queried a published model is a pin
+  that means nothing.
+
+Which target for which job:
+
+- **Baseline what customers experience:** Remote. It is the deployed model
+  through the deployed engine, which is the thing they actually hit. The judge
+  sees no re-executed rows on a Remote run (there is no local copy of the
+  bytes), so its verdicts rest on the answer text and the golden; say so.
+- **Improve and accept:** local, because the acceptance check needs compile,
+  reload, and a fresh re-answer between edits. Publishing to a customer
+  environment to score an edit is not something this loop does. Where the host
+  offers draft execution, that counts as local for this purpose.
+- **Measure real data without touching production:** local proxied.
+
+So a measure-only run can use any target; a run that includes **improve** needs
+a local one.
+
+Two things to check before a platform run, because neither errors and both make
+the run measure something other than what it names:
+
+- **The answerer's skills must be written for THIS host.** A shared skill names
+  an MCP tool by its bare name (\`get_context\`) so it reads correctly anywhere,
+  but a host/router skill names its own host's tools directly. Install the
+  latter for the wrong host and the answerer is told to call tools it does not
+  have. \`run_baseline.py\` warns when the manifest it loaded names Publisher-only
+  tools on a platform target; point \`--answerer-manifest\`, or \`--skills-root\`,
+  at the checkout that ships this host's manifest.
+- **The tool names are configuration.** \`--hosted-mcp-server\` is both the
+  \`mcp__<server>__<tool>\` prefix and the OAuth cache key, so it has to match the
+  name the answerer authenticated under, and \`--hosted-tools\` lists the bare
+  tools that host exposes.
+- **Get the hosted tools in front of a headless answerer, one of two ways.**
+  A spawned answerer cannot complete an OAuth flow, so the tools have to be
+  reachable before the run starts. \`run_baseline.py\` proves it with one cheap
+  probe and refuses to spend an arm otherwise -- a run whose answerers have no
+  tools does not error, it reads as a terrible model.
+
+  1. **Authenticate once, interactively.** Works anywhere, including a plain
+     CLI install, and is the route to assume unless you know otherwise. The
+     token is cached per server NAME, so authenticate under the same name the
+     run passes to \`--hosted-mcp-server\`:
+
+     \`\`\`bash
+     claude mcp add --transport http <name> <scoped-url>
+     claude          # then /mcp -> <name> -> Authenticate
+     \`\`\`
+
+     Then come back and run. This is a hand-off to a person; there is no
+     headless equivalent, so plan for it rather than discovering it mid-run.
+
+  2. **A local proxy that already holds the credential.** Some hosts ship an
+     editor extension whose local MCP proxy can expose the hosted
+     \`get_context\` / \`execute_query\` -- often behind a setting that is off by
+     default. Where that exists, point \`--mcp-url\` at the proxy on localhost
+     and no OAuth step is needed, because the extension holds it. Check what
+     the proxy actually exposes before relying on it: the same proxy may serve
+     a local Publisher's \`malloy_*\` tools instead, and then \`--hosted-tools\` is
+     naming tools that are not there. This route is not available to someone
+     running the CLI alone.
+
+- **Prefer a SCOPED endpoint URL over asking for scope.** A hosted MCP is
+  usually reachable two ways: a global endpoint where every call carries an
+  organization and workspace, and a scoped one where the URL itself is the
+  scope. \`--scope\` and the prompt can only ASK an answerer to stay in one
+  package; a scoped URL enforces it. For an agent being measured that is the
+  difference between a case answered against the package it names and one
+  answered against whatever else the account can see. Authenticate once
+  interactively (\`claude\`, \`/mcp\`) under the same server name the run will use;
+  the token is cached per name, and a spawned headless answerer cannot complete
+  an OAuth flow.
+
+## Before you start
+
+1. The model package under evaluation must live in a git repository, with
+   \`evals/<set>/\` in the package, beside the model files. Git is the checkpoint
+   mechanism; without it there is no rollback and no run can include improve.
+
+   Keeping the set IN the package is what stops a model edit and its answer key
+   drifting apart: they move in one commit, so fixing a measure and forgetting
+   the golden that depended on it stops being possible. It is safe -- measured
+   on a running server, a \`cases.jsonl\` inside a package appears in no model
+   listing, no notebook listing, no package resource, and 404s over HTTP, so an
+   MCP-only answerer has no route to it.
+
+   What it buys differs by target. On a LOCAL Publisher it does not get you free
+   versioning -- \`sourceContentSha\` hashes model paths only, so the set needs
+   its own \`datasetSha\`. On a hosted target that publishes the whole package
+   directory as an IMMUTABLE version, the set rides inside that version and
+   \`targetVersion\` pins model and answer key together; nothing can be edited
+   under a published version, which is what makes it a pin. Check which you have
+   before deciding how much of this you need.
+
+2. The server must be up with retrieval tracing on, so a call's ranked results
+   can be recovered afterwards (open-source Publisher: \`PUBLISHER_MCP_TRACE=retrieval\`).
+   Confirm a trace lookup is available (absent means tracing is off).
+   Refuse to start a scored run without it: failures without traces cannot be
+   attributed.
+
+3. Health-check: your host's status check until it reports serving, and inspect
+   \`loadErrors\`. A dead database that still answers HTTP is an environment
+   failure, not a model failure. Stop and fix it. Four consecutive
+   environment or no-result attempts means stop the run.
+
+4. Load the set: scrape/import as above, or reuse an existing \`evals/<set>/\`.
+   Never keep two live copies of one set; the set directory in the model repo
+   is the single source of truth, versioned by \`datasetVersion\` in
+   \`set.json\`.
+
+5. Review goldens before you score. A verified golden with no local artifact
+   stays verified by provenance and is not scorable until you have rows or a
+   scalar to compare (the judge needs both sides). If diagnosis later marks
+   \`BAD-REFERENCE\` or \`AMBIGUOUS-REFERENCE\`, follow
+   \`reference/golden-side-door.md\`. Both are expected in the wild; both
+   are the golden side door below, not improve, and not a sixth step.
+
+6. Create \`runs/<runId>/run.json\` with the attribution pins
+   (\`reference/ledger-schema.md\`): mode, dataset version, **the target and the
+   version it served** (a local target pins a commit, so commit or stash first;
+   answering from a dirty tree pins nothing), server version, judge version and
+   rubric sha, answerer model, call budget, trace mode. Freeze those for the
+   whole run. Raising a call budget mid-run moved mean outcomes on an unchanged
+   model.
+
+7. Generate every answerer prompt from the stored case in \`cases.jsonl\`.
+   Never retype the question. A truncated retype is indistinguishable from a
+   real question downstream.
+
+## Per question
+
+1. Health-check again.
+2. Spawn a *fresh* blind subagent. Give it only the question text and the
+   Malloy analysis tools. Tell it to follow the \`malloy-analysis\` skill. Do not
+   mention eval, gold, scoring, or this skill.
+3. Keep a host-side tool-use log for that subagent (name, input path or
+   command, MCP tool name). Publisher traces see MCP only; a Read of a gold
+   CSV is invisible server-side.
+4. \`skill:eval-answer\`: contamination first, then re-execute, then the judge,
+   then events.
+5. \`skill:eval-diagnose\` only when this run includes diagnose, only on dev
+   cases, and only after the score event exists.
+6. \`skill:eval-improve\` only when this run includes improve, and only for
+   \`owner: model\`. Then run the acceptance check. On accept, checkpoint.
+
+## Checkpoint
+
+A checkpoint is a git commit of the model repository, taken after an acceptance check
+accepts, so a bad improve direction can be rolled back. It is not a report,
+and it is not a remote publish.
+
+1. Commit the model files AND the set's ledger in one commit; put the label
+   and the closed issue ids in the message.
+2. Append the \`checkpoint\` event (\`action: created\`, label, \`modelGitSha\`
+   from the commit you just made, issueIds). The event line itself rides in
+   the next commit; append-only logs trail by one commit and that is fine.
+3. Confirm \`git status\` is clean for the model files.
+
+**Restore**: \`git checkout <sha> -- <model files>\` (or \`git revert\` the
+checkpoint commits), then reload the package, then append a \`checkpoint\`
+event with \`action: restored\` and the sha. Readers return to the model that
+existed before the bad direction.
+
+Take a checkpoint of the current model *before* the first improve batch if no
+commit pins it yet. Rolling back by hand is guesswork.
+
+If reload reports \`mode: reinstalled\`, the package was re-fetched from its
+install location and may have overwritten the restored files. Prefer in-place
+/ watch-mounted packages for this loop.
+
+## Out of scope
+
+This loop is local. The ledger is files, the checkpoints are git, you are the
+conductor. Do not:
+
+- publish the model to a hosted platform as a "true" checkpoint or learning
+  curve
+- start a Python orchestrator (\`loop.py\`, \`improve_batch.py\`) that runs the
+  five steps end to end unattended. You conduct; the scripts are the steps,
+  not the sequencing
+- score by string-diffing rows instead of judging them, or reintroduce a
+  scripted row oracle: one that can pass a wrong answer is worse than none
+- wait for a bigger gold set before the loop can run; dev/holdout on what
+  exists beats waiting
+- register eval MCP tools or stand up an eval API
+- encode unsettled goldens into the model
+
+## Prime directives
+
+- The model is the only thing improve edits. No question text, qids, or
+  expected values in any name, doc, or comment.
+- When the environment misbehaves, stop. Never diagnose a sick system.
+- When a subagent disagrees with you, probe. Do not win by authority.
+- When a rule here is wrong, change this file and note it on the run.
+
+## Related skills
+
+- \`skill:eval-answer\`: contamination, judge protocol, events. Its
+  \`reference/ledger-schema.md\` is the file contract; \`skill:eval-judge\` is
+  the judge.
+- \`skill:eval-diagnose\`: component, owner, issue events. No edit.
+- \`skill:eval-improve\`: smallest model edit, probe receipts, no self-accept.
+- The \`malloy-analysis\` skill: what the blind answerer follows. It is installed
+  from the \`analysis\` manifest group, not the \`eval\` group.
+
+## Reference files over MCP
+
+This skill's \`reference/\` files are served as separate prompts, one per file, fetched only when you ask for them. Where the text above says to read \`reference/<name>.md\`, get the prompt named \`eval-loop/<name>\` instead.
+
+Available: acceptance-check, checking-the-judge, golden-side-door, measurement, running-a-run.`
+    },
+    {
+      name: "eval-loop/acceptance-check",
+      description: "The acceptance check. Reference detail for the eval-loop skill.",
+      body: `<!-- How to decide whether ONE edit stays. Read this before accepting or reverting anything. -->
+
+# The acceptance check
+
+## The acceptance check (inside improve)
+
+You own the acceptance check. The improver does not accept its own edit.
+
+**Before any of it: is the answer key still valid?** An edit that changed what
+an entity means can have moved goldens for cases nobody was working on, and a
+rerun against a stale key measures nothing -- it reads as a win or a regression
+with equal confidence and neither is real. \`skill:eval-improve\` Step 4 reports
+these as \`golden_suspect\` on the candidate; the judge reports its own doubts as
+\`gold_status\`. **Any unadjudicated one halts the acceptance check.** Settle
+each through the golden side door -- repair and bump \`goldenRevision\`, or
+dismiss it explicitly -- and only then re-answer. Do not net a suspect golden
+against the flip count; an uncertain key is not noise you can average out.
+
+Cheap and deterministic, every edit:
+
+1. A compile check (scope \`file\` for an edit, \`package\` if importers must
+   survive).
+2. Save, then reload the package. Confirm it is not serving a stale model.
+3. Replay stored final queries from previously-passing cases. They must still
+   execute, and a judge must still call them a match.
+4. A *fresh* blind re-answer of the affected question. The fix must be
+   discoverable, not merely possible. The improver writing the query it
+   already knows proves only that the edit exists.
+
+Acceptance rules (replacing any vague "results improve"):
+
+- **Per-case, not aggregate.** No previously-passing case may regress: diff the
+  new run's verdicts against the baseline, case by case (\`jq\` over the two
+  \`events.jsonl\` files). \`regressions\` on the acceptance check event must be
+  empty to accept, and the regressed qids go in the checkpoint commit message
+  if you proceed anyway after a human call.
+- **Confident verdicts only.** \`needs_human\` and null verdicts are neither
+  passes nor failures; the delta is computed without them.
+- **Both splits.** The acceptance check runs the affected dev cases AND the holdout
+  slice. Diagnose and improve never saw holdout; that is what makes its delta
+  evidence rather than memorization.
+- **Twice.** An improvement must survive a second independent run with fresh
+  blind answerers before acceptance. A delta that appears once and vanishes
+  on re-run was answerer or judge variance, not a fix.
+- Documentation / discoverability edits may accept on a deterministic
+  \`get_context\` probe now returning the entity, provided no replay
+  regresses. Measure, join, or definition edits need the full rules above,
+  including the flip-count bar in Measurement, which means enough affected
+  cases to clear it.
+- Independent deterministic justification (a probed-wrong definition
+  corrected) may accept without a measured win. Record that as the acceptance check
+  \`reason\`.
+
+Write the \`acceptance_check\` event (decision, class, baseline and final run ids,
+regressions, holdout delta, reason) BEFORE any commit, so a rejected
+direction leaves a record. On reject: revert the files (\`git checkout --\`
+or \`git restore\`) and reload. On accept: \`issue_status: fixed\` for what the
+edit actually closed, **then checkpoint**.`
+    },
+    {
+      name: "eval-loop/checking-the-judge",
+      description: "Checking the judge. Reference detail for the eval-loop skill.",
+      body: `<!-- The judge measures the model; this measures the judge. Read after any change to judge doctrine or its inputs. -->
+
+# Checking the judge
+
+## Nothing else checks the judge
+
+The A/A band measures whether the judge is *repeatable*. It says nothing about
+whether it is *right* -- a judge answering \`no_match\` every time posts a perfect
+band. Those come apart in practice, and when they do the loop keeps running and
+every number it emits is wrong in the same direction.
+
+So keep a small file of frozen predictions pinned to verdicts a human settled,
+and re-run them after any edit to the judge prompt, a rubric, or what the judge
+is given (\`scripts/check_judge.py\`, \`judge-regressions.jsonl\` in the set). Seed it
+from the cases an A/A pair disagreed on: those are the contested ones, so they
+are where a change will show first.
+
+Two things about its shape:
+
+- **The unit is a prediction, not a question.** One question earns different
+  verdicts for different answers, legitimately. Key the fixture on the answer.
+- **Judge through the same code path a run uses.** A reimplementation inside the
+  checker can pass while the thing it stands for is broken.
+
+A fixture that has never failed is not yet known to be a test. Break a rubric on
+purpose once and confirm the right entry fails.
+
+When a fixture fails, rule out judge nondeterminism (\`--repeat\`) before you
+believe it. Then either you moved a verdict you did not mean to, or the fixture
+was wrong -- re-settle it and record why. Deleting it throws away the only case
+you had evidence about.`
+    },
+    {
+      name: "eval-loop/golden-side-door",
+      description: "The golden side door. Reference detail for the eval-loop skill.",
+      body: `<!-- Everything about a golden that is wrong, doubted, or out of step with the model. NOT a sixth step, and never improve. -->
+
+# The golden side door
+
+## Golden side door (not a sixth step)
+
+Bad and ambiguous goldens show up immediately. That is not improve. A
+checkpoint that mixes model edits and silent golden rewrites is useless for
+rollback. Keep hold and repair here, outside the five steps.
+
+## Repair a bad golden
+
+This is **your** job as conductor, after \`eval-diagnose\` writes
+\`BAD-REFERENCE\`. It is not the answerer's job, and it is not a reason to
+change the model.
+
+Diagnosis is not the only way one arrives. The judge also reports a
+\`gold_status\` on every score (\`skill:eval-judge\`), and a
+\`suspect\` or \`verified_wrong\` comes through this same door -- earlier, because it
+lands during scoring rather than after. Treat it as a \`BAD-REFERENCE\` with the
+judge's \`gold_note\` as its evidence. Adjudicate it **before** improve runs: a
+doubted key sends a modelling agent to fix a model that is already right, which
+is the most expensive wrong turn this loop can take.
+
+The judge scored against the golden as written even where it said \`suspect\`, so
+its verdict is still the verdict. Do not re-open a case merely because the flag
+is set; open it because you looked and agreed.
+
+1. **Replay, yourself.** Take the stored \`final_query\` (or a query you can
+   justify from the model) and run it with \`execute_query\`. Write the
+   rows to a gold artifact under \`evals/<set>/\` (never under the served
+   package tree). If you cannot produce a trusted key, follow **Hold an
+   ambiguous golden** (or mark the golden \`invalid\` if the question itself is
+   unusable). Do not invent a number.
+2. **Patch the case** in \`cases.jsonl\`: new \`golden\` (status, kind, value or
+   path, \`canonicalQuery\`, \`verifiedBy: replay\`) and \`goldenRevision\`
+   incremented. Do not edit any old \`score\` event.
+3. **Bump \`datasetVersion\`** in \`set.json\`, and commit the ledger change so
+   the repair is attributable.
+4. **Close the issue as repaired, not as a model fix**: \`issue_status: fixed\`
+   with a note that the *golden* changed.
+5. **Open a new run** whose \`run.json\` records the new \`datasetVersion\`. It
+   is not comparable to runs on the old version without saying so.
+6. **Re-score stored queries first**: \`skill:eval-answer\` without a new
+   answerer (saved predictions, fresh judge, new \`golden_revision\` stamps).
+7. **Re-answer only if you still need a blind look** (discoverability, or the
+   stored query was itself the thing under test).
+
+Never mix old-golden and new-golden scores in one aggregate. A before/after
+that crosses a golden bump is a rebase, not a model delta.
+
+## A model fix can invalidate a golden, and nothing will tell you
+
+A rubric that explains a trap usually has to quote the model -- "this measure
+counts line items despite its name, so it yields the trap value". That sentence
+is a claim about the model, and it is false the moment the model is fixed. The
+judge keeps enforcing it and starts failing correct answers.
+
+A truth-package check cannot catch this, structurally. It re-derives values from
+sources that are independent of the model **on purpose**, so a rubric can
+describe a model that no longer exists while every value still re-derives green.
+
+Worse, a model fix can move a golden's *value* without touching the data. If a
+dimension is defined in terms of the measure you fixed, the concept it names now
+resolves to a different population -- same dimension, same question, different
+correct answer -- while a canonical truth query still returns the old number
+because it encoded the old definition.
+
+So: **after any model edit, re-read the rubrics of every case that names an
+entity you touched.** \`verify_goldens.py\` audits the mechanical part -- it parses
+\`X is <expr>\` out of the model and flags any rubric asserting a different
+definition -- but only for definitions it can parse. Prose claims about grain,
+population, or convention are still yours to check.
+
+When one turns up it is \`BAD-REFERENCE\`, and it goes through this side door.
+Never let it reach improve: the model is right, and an edit would be damage.
+
+## A golden must match the state the model is in
+
+A case whose golden holds a value asserts that the value is obtainable. If the
+model has no trace of the concept, that assertion is false, and the case is now
+asking two questions at once: "did the answer contain the golden" (no) and
+"should the answerer have complied" (no). Both readings are defensible, so the
+verdict stops being a measurement.
+
+Measured, holding the answer, the model and the rubric fixed and varying only
+how the case was authored:
+
+| the case says | verdicts over four samples |
+|---|---|
+| golden holds three counts, model defines no such concept | \`match\` / \`no_match\` / \`match\` / \`near_match\` |
+| \`golden.kind: unanswerable\`, pass is a refusal that names what is missing | \`match\` x4 |
+
+The judge is not being unreliable in the first row. It is being asked a question
+with two right answers.
+
+So a coverage case has two states and needs a golden for each:
+
+1. **Before the model defines the concept.** \`coverage: absent\`,
+   \`golden.kind: unanswerable\`. The pass is a refusal that NAMES what is
+   missing; inventing boundaries and reporting them as the company's is
+   \`no_match\`. This is the state that measures whether the model documents its
+   conventions.
+2. **After improve adds it.** Bump \`goldenRevision\`, replace the golden with the
+   real value, bump \`datasetVersion\`. A refusal is now a failure, and the run
+   measures whether the new entity is discoverable.
+
+Never one case straddling both. The straddle is what produces an oscillating
+verdict, and no amount of rubric wording fixes it -- four prompt edits were
+tried against exactly this case and none of them did.
+
+**This is the mirror of "A model fix can invalidate a golden".** That section
+warns that adding a definition can move a golden nobody was working on. This one
+warns of the same seam from the other side: a golden written for a model that
+does not exist yet is invalid until the model catches up. Both are golden side
+door work, and neither is improve.
+
+## Hold an ambiguous golden
+
+Use this when the current key is unusable as a score *and* you cannot justify
+exactly one replacement (two honest replays disagree; a window or tie is
+unspecified; later samples might confirm a convention).
+
+1. **Do not invent a key.** Leave the old artifact on the case for
+   provenance.
+2. **Patch the case**: \`golden.status: ambiguous\` with a \`reason\` naming the
+   defect and the competing replacements (not a new number). Increment
+   \`goldenRevision\`.
+3. **Do not score** this case until a later sample confirms a convention or a
+   human picks a replacement. Its attempts get \`verdict: null,
+   reason: golden_ambiguous\`.
+4. **\`issue_status: deferred\`**, not \`fixed\`. Revisit when another case in
+   the same neighborhood confirms a convention.
+5. Old \`score\` events stay. They keep the previous \`golden_revision\` and must
+   not enter an aggregate that claims the model failed.
+
+If later evidence makes one replacement obvious, then Repair a bad golden.`
+    },
+    {
+      name: "eval-loop/measurement",
+      description: "Measurement. Reference detail for the eval-loop skill.",
+      body: `<!-- Sampling, the flip-count bar, the A/A noise band, and targeted fixes. Read this before quoting any number. -->
+
+# Measurement
+
+## Measurement
+
+**Sample each case once. Spend the budget on more and more varied cases
+instead.** Repeats past the first buy very little: variance decompositions of
+LLM evaluation put the reduction from extra repeats at a small fraction of
+what extra items buy, and a set of five cases run three times cannot support
+a claim that fifteen distinct cases can. If a case is genuinely borderline,
+re-run that case, not the whole set.
+
+Because a single sample cannot carry a mean, do not report before/after as a
+score delta. **Count the cases whose verdict changed** between the baseline
+and the post-edit run, discard the unchanged ones, and read the result off
+this table:
+
+| Cases that got worse | Cases that must get better to accept |
+|---|---|
+| 0 | 5 |
+| 1 | 7 |
+| 2 | 9 |
+| 3 | 10 |
+
+Below that bar the change is **unresolved**, not an improvement, and saying
+so is the honest report. Note the consequence before you scope a run: a set
+of six cases can essentially never clear this bar, so a set that small can
+measure a baseline and diagnose failures but cannot defend an edit.
+
+## Calibrate the bar before you trust it
+
+This table was asserted, not measured, and the number it needs is a property of
+your harness and your set -- not of this skill. Measure it with an **A/A run**:
+the same model, same config, same set, twice, compared with
+\`scripts/flip_table.py\`. Every flip it reports is noise by construction, since
+nothing changed. Record the result with the set, in \`CALIBRATION.md\`, and cite
+that file when you quote a band.
+
+Re-measure whenever the model, judge, or set changes. This is not a formality:
+observed bands have moved by a factor of three across a fortnight of ordinary
+work, so a band carried over from a previous configuration is a number with no
+claim on the present one.
+
+One A/A is one sample of the flip count, not a distribution. It can show a bar
+is too low; it cannot show one is high enough. Treat any measured band as a
+floor.
+
+Two consequences worth separating:
+
+- **For acceptance**, the band is the threshold untargeted flips must sit under.
+- **For diagnosis**, it is a warning that a single run's failure list is partly
+  luck. Pick what to fix from the failures that fail in **both** A/A runs.
+  Ranking a backlog by one run's clusters partly ranks which cases were unlucky
+  that afternoon.
+
+When you inspect the flips, attribute them before you accept them as
+irreducible. A band dominated by the **judge** re-reading an ambiguous rubric is
+not answerer noise, and it is not a floor you have to live under: sharpening
+those rubrics buys more measurement power than any change to the answerer.
+
+An A/A is not a repeat in the sense the sampling rule forbids. It is a one-off
+calibration of the instrument, and the loop's whole acceptance rule rests on
+the constant it produces.
+
+## A targeted fix needs a targeted test
+
+The flip-count table is the right instrument for a broad change and the wrong
+one for a narrow fix. A fix that repairs three cases on a 49-case set moves the
+total by three -- inside the noise band an A/A already produces -- so a
+mechanically-verified repair reports as no effect and gets abandoned.
+
+This is not a hypothetical failure mode: a mechanically verified repair, where
+each fixed case now matches its golden exactly, can read as "no effect" on both
+of two set-total comparisons. Worked examples are in the set's \`CALIBRATION.md\`.
+
+So for a narrow fix use \`scripts/flip_table.py --targets --noise-band\`:
+
+1. **Name the cases before the run.** Pick them from the stable failures of the
+   A/A, never from a single run. Choosing them afterwards is choosing the answer.
+2. Accept on the targeted cases: they were failing, they now pass, and none of
+   them broke.
+3. Separately require the untargeted flips to sit **at or below the A/A band**.
+   That is what rules out a fix that trades one set of cases for another --
+   above the band, investigate before accepting, however good the targets look.
+4. **Run the post-edit arm twice and pass both** (\`--b --b2\`). The band counts
+   flips; it never asks which cases flipped, and that is the hole. Noise
+   scatters, so an untargeted case that breaks in *both* post arms is a real
+   regression however small the count is.
+
+Report both. A targeted win with untargeted flips above the band is not a win,
+and a set-total that moved by less than the band is not evidence of anything
+either way.
+
+Step 4 exists because the band alone has accepted a real regression: an edit
+whose untargeted flip count sat inside the band, but where the same untargeted
+case broke in every post arm. One arm cannot tell that from a coin toss.
+
+The reason to expect this, rather than treat it as bad luck: **a correct new
+entity is not a safe one.** Adding a measure changes what agents reach for on
+questions nobody was thinking about, so a well-named addition can pull a
+neighbouring question onto the wrong denominator. That makes the untargeted
+half of the acceptance check the half that matters, and it needs two arms to be
+readable at all.
+
+The one retrieval number this loop reports is **per-question entity recall**:
+of the entities each golden answer depends on, how many did the agent's own
+\`get_context\` calls deliver (\`skill:eval-answer\`, \`scripts/score_retrieval.py\`;
+delivered means returned as a ranked entity, under a sibling source, or named
+in a returned source's documentation). It is measured on the agent's real
+search text against real questions, so it needs no hand-written terms. Read
+it within an arm, to attribute a failure; it moves with the answerer, so a
+cross-arm comparison of retrieval *itself* is not this loop's job -- that is
+the engine-side \`eval-retrieval\` skill, which ships to no customer.
+Coverage (can the model answer this at all) is a property of the model and its
+data and is never reported under a retrieval heading.
+
+If the contaminated fraction of attempts exceeds 0.1 (or any contamination,
+on a run smaller than 10), the run is a harness failure. Do not publish a
+model score.`
+    },
+    {
+      name: "eval-loop/running-a-run",
+      description: "Running a run, concretely. Reference detail for the eval-loop skill.",
+      body: `<!-- The worked command sequence. Read it when you are about to run one. -->
+
+# Running a run, concretely
+
+## Running one, concretely
+
+\`scripts/run_baseline.py\` does steps 3 and 7 and the whole of **Per question**:
+one fresh answerer per case with only the Publisher MCP tools, a contamination
+check, a judge, and a conformant \`events.jsonl\`.
+
+\`\`\`bash
+# 1. serve the model under test -- in its own session, so the shell's exit
+#    cannot take it down, and returning only once it answers a query
+python3 skills/eval-loop/scripts/serve.py --publisher-dir <publisher>/packages/server \\
+  --server-root <root> --port 4811 --mcp-port 4040 --trace-retrieval \\
+  [--allow-proxy]   # required for a \`publisher\`-type (proxied) connection
+#    a second server for the TRUTH package, on other ports, that the answerer
+#    has no route to:
+python3 skills/eval-loop/scripts/serve.py --publisher-dir <publisher>/packages/server \\
+  --server-root <truthroot> --port 4881 --mcp-port 4882 [--allow-proxy]
+
+# 2. smoke one case first ($0.13), then the arm. Goldens are re-derived from
+#    the truth server before either starts; a drifted set refuses to run.
+python3 skills/eval-loop/scripts/run_baseline.py \\
+  --set <repo>/evals/ecommerce --out results/smoke --only <qid> --no-judge \\
+  --truth-publisher http://localhost:4881
+python3 skills/eval-loop/scripts/run_baseline.py \\
+  --set <repo>/evals/ecommerce --out results/<arm> \\
+  --parallel 4 --truth-publisher http://localhost:4881
+#    the run names itself <set>-<phase>-<nn> (ecommerce-baseline-01, then -02
+#    for the second arm of the A/A). Pass --label only for a run that needs a
+#    human name; hand-typed arm names stop being readable within an afternoon.
+
+# 3. compare two arms, or two runs of one arm
+python3 skills/eval-loop/scripts/flip_table.py --a results/<a> --b results/<b>
+
+# 4. FIRST: any golden the judge did not believe. \`jq .doubtedGoldens
+#    results/<arm>/run.json\` -- non-empty means settle those through the golden
+#    side door before diagnosing, or you send a modelling agent at a model that
+#    is already right.
+python3 skills/eval-diagnose/scripts/diagnose.py \\
+  --run results/<arm> --set <repo>/evals/ecommerce --model-dir <package>
+#    (cluster_failures.py gives a free mechanical first look, as
+#     clusters-mechanical.jsonl; it groups by retrieval outcome and is not a
+#     diagnosis)
+
+# 5. build the browsable package
+python3 skills/eval-loop/scripts/build_run_package.py \\
+  --run results/<a> --run results/<b> --set <repo>/evals/ecommerce --out <pkg>
+\`\`\`
+
+Order of magnitude for planning, **calibrated on ecommerce over local duckdb**:
+a Sonnet arm over a few dozen cases costs single-digit dollars and finishes in
+minutes, at roughly a dime and a handful of turns per case. A proxied warehouse
+is a different regime: the VideoAmp set ran at $0.33 per case on Sonnet and
+$0.57–0.71 on Opus, ~100 s per case, driven by warehouse latency and query
+errors -- budget 4x when the data is not local. Budget **five** such arms for a
+defensible claim -- a baseline, two for the A/A, and two post-edit -- plus the
+diagnose and improve agents, which are far cheaper per case but use a larger
+model. Measured per-arm figures for a given set belong in that set's
+\`CALIBRATION.md\`.
+
+\`--rebuild\` re-derives the ledger from saved transcripts without calling a model,
+and \`--rebuild --rejudge\` re-scores existing answers in place. \`--from <run>
+--out <new>\` does the same into a NEW run directory -- the answers copied, the
+judge fresh, the old verdicts untouched -- which is what a golden repair or a
+rubric change calls for. Use them after a scoring or schema change; re-running
+the answerers would confound the change you are measuring with fresh answerer
+variance.
+
+The scripts import each other by path (\`ledger\`, \`mcp_payload\`,
+\`score_retrieval\` live in \`eval-answer/scripts\`; the loop scripts insert that
+path). Run them **in place** from the skills checkout; a copy patched elsewhere
+chases \`ModuleNotFoundError\` three times.
+
+Two failure modes worth pre-empting, because both produce a clean-looking run:
+
+- **Pre-approve the tools.** A headless answerer that has to ask permission for
+  \`malloy_getContext\` stalls until the timeout and lands as a harness error.
+- **Check the served revision is the one you edited.** Publisher serves a
+  snapshot copy, so a model fix can be absent from the run that is supposed to
+  measure it. Query the changed measure once before spending an arm on it.`
+    },
     {
       name: "malloy",
       description: 'Index of all Malloy skills. Use when user asks "malloy help", "what malloy skills are available", "how do I use malloy", or needs guidance on which Malloy skill to use.',
@@ -294786,6 +296699,8 @@ Say "model my data" and the agent will orchestrate the full modeling workflow au
 
 Every skill in this deployment, by what it is for. Start at a driver; it routes to the rest.
 
+This table is a catalogue of what exists, not of what is loaded. A host that installs one group takes that group's skills alone: \`analysis\`, \`modeling\`, or \`eval\`. A row naming a skill from a group you did not install says that the skill exists. It is not an instruction to load it, and it is written as a plain name rather than a \`skill:\` reference to say so.
+
 **Start here**
 
 | Skill | Use when... |
@@ -294815,9 +296730,18 @@ Every skill in this deployment, by what it is for. Start at a driver; it routes
 | \`skill:malloy-notebooks\` | Building Malloy notebooks (.malloynb) |
 | \`skill:malloy-analysis-report\` | Combining validated queries into a notebook report or dashboard |
 | \`skill:malloy-analysis-pitfalls\` | Checking a query and its results before presenting an answer |
-| \`skill:malloy-notebook-chat\` | The chat is bound to a notebook or saved report; answer from its cells |
+| \`malloy-notebook-chat\` | The chat is bound to a notebook or saved report; answer from its cells. Ships in \`analysis\`. |
 | \`skill:malloy-phrase-detection\` | Turning a plain-English question into search targets for the context tool |
 
+**Evaluating a model** (driven by \`eval-loop\`). These ship in the \`eval\` group, which neither \`analysis\` nor \`modeling\` includes.
+
+| Skill | Use when... |
+|-------|-------------|
+| \`eval-loop\` | Running the loop: baseline, noise band, diagnose, one edit, gate, checkpoint |
+| \`eval-answer\` | Scoring one answer against a verified golden, and what retrieval delivered |
+| \`eval-diagnose\` | Deciding why a case failed and which artifact owns the fix |
+| \`eval-improve\` | The smallest model edit for a diagnosed cluster, with probe receipts |
+
 **Writing correct Malloy** (read before writing, not after failing)
 
 | Skill | Use when... |