@malloy-publisher/server 0.5.0 → 0.5.1

package/dist/server.mjs

@@ -232535,23 +232535,12 @@ var require_bcrypt_pbkdf = __commonJS((exports, module) => {
   };
 });
 
-// ../../node_modules/cpu-features/build/Release/cpufeatures.node
-var require_cpufeatures = __commonJS((exports, module) => {
-  module.exports = __require("./cpufeatures-1yrn0vtw.node");
-});
-
-// ../../node_modules/cpu-features/lib/index.js
-var require_lib8 = __commonJS((exports, module) => {
-  var binding = require_cpufeatures();
-  module.exports = binding.getCPUInfo;
-});
-
 // ../../node_modules/ssh2/lib/protocol/constants.js
 var require_constants42 = __commonJS((exports, module) => {
   var crypto3 = __require("crypto");
   var cpuInfo;
   try {
-    cpuInfo = require_lib8()();
+    cpuInfo = (()=>{throw new Error("Cannot require module "+"cpu-features");})()();
   } catch {}
   var { bindingAvailable, CIPHER_INFO, MAC_INFO } = require_crypto();
   var eddsaSupported = (() => {
@@ -247194,7 +247183,7 @@ var require_keygen = __commonJS((exports, module) => {
 });
 
 // ../../node_modules/ssh2/lib/index.js
-var require_lib9 = __commonJS((exports, module) => {
+var require_lib8 = __commonJS((exports, module) => {
   var {
     AgentProtocol,
     BaseAgent,
@@ -247377,7 +247366,7 @@ function openSshProxy(ssh, target) {
 var import_ssh2, ALLOW_UNVERIFIED_SSH_HOST_KEY_ENV = "PUBLISHER_ALLOW_UNVERIFIED_SSH_HOST_KEY", TRUTHY_ENV_VALUES, SSH_CONNECT_TIMEOUT_MS = 15000, SSH_KEEPALIVE_INTERVAL_MS = 15000;
 var init_proxy = __esm(() => {
   init_logger();
-  import_ssh2 = __toESM(require_lib9(), 1);
+  import_ssh2 = __toESM(require_lib8(), 1);
   TRUTHY_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
 });
 
@@ -267880,7 +267869,7 @@ function readGivenControlSpec(annotationTexts2) {
 }
 function malloyGivenToApi(given) {
   const type = given.type;
-  const renderedType = type.type === "filter expression" ? `filter<${type.filterType}>` : type.type;
+  const renderedType = type.type === "filter expression" ? `filter<${type.filterType}>` : type.type === "array" && type.elementTypeDef?.type ? `${type.elementTypeDef.type}[]` : type.type;
   const allNotes = given.annotations.forRoute(undefined);
   return {
     name: given.name,
@@ -270849,7 +270838,7 @@ var init_materialization_metrics = __esm(() => {
   storageTableRetainedCounter = lazyCounter3("publisher_storage_tables_retained_total", "Tables a FAILED run left in a storage= destination and deliberately did not " + "reclaim, because the source is refreshed incrementally and the name may be " + "the one it serves from. Label: destination. Not all of these are orphans — " + "a rebuild at a fresh generational name is, a seed on the live serving name " + "is not, and the manifest entry cannot separate them — so read this as an " + "upper bound on what is accumulating rather than a leak count. It is the " + "only accounting there is until reclaiming a destination exists, which is " + "why it is a counter and not just a log line: the question is a rate, not " + "whether it ever happened.");
   storageBuildFailureCounter = lazyCounter3("publisher_storage_build_failures_total", "storage= build failures (federation/passthrough/attach/CTAS), distinct from " + "in-warehouse build failures. Labels: destination (connection name), " + "reason ('build_failed'|'billed_read_not_captured'). The second is the " + "expensive one: the warehouse read ran and was charged, and the rows could " + "not be captured — so a re-drive pays for it again. Worth alerting on " + "separately from a failure that costs only a retry.");
   attributionSkippedCounter = lazyCounter3("publisher_storage_build_attribution_skipped_total", "storage= builds whose warehouse read went out UNATTRIBUTED while tagging was " + "on. Label: reason ('job_listing_unavailable'|'tag_failed'|" + "'read_row_not_found'|'read_row_ambiguous'|'cost_query_failed'). " + "The read still ran and the " + "build still succeeded — what was lost is the label in the customer's own " + "query history, and the cost on this side. Without this an operator who " + "turns tagging on and sees nothing has a single log line to go on.");
-  eligibilityRefusedCounter = lazyCounter3("publisher_materialization_eligibility_refused_total", "storage= materialization-eligibility refusals. Label: reason " + "('free_parameter'|'given'|'authorize'|'not_duckdb_portable'|" + "'public_surface_unknown').");
+  eligibilityRefusedCounter = lazyCounter3("publisher_materialization_eligibility_refused_total", "materialization-eligibility refusals, both tiers. Label: reason " + "('free_parameter'|'given_in_persisted_query'|'dynamic_projection'|" + "'dynamic_join'|'dynamic_joined_where'|'partition_without_storage'|" + "'partition_column_unknown'|'partition_column_not_public'|" + "'merge_key_scope_unresolved'|'preaggregate_over_dynamic_source'|" + "'authorize'|'partition'|'not_duckdb_portable'|" + "'public_surface_unknown'). 'given' is retained on the enum for records " + "that carry it and is no longer produced: where a given sits decides the " + "outcome, so one reason covering every placement cannot be raised.");
   serveShapeTierDropCounter = lazyCounter3("publisher_storage_serve_shape_tier_drop_total", "storage serve-shape compile escalations: a refinement tier failed to " + "compile and the riskiest category was dropped. Label: tier (the failed " + "tier index, 0=full).");
   serveShapeTypeFallbackCounter = lazyCounter3("publisher_storage_serve_shape_type_fallback_total", "Captured DuckDB column types mapped to json in the serve shape (type " + "fidelity loss). Label: kind ('array'|'unrecognized').");
   chainedStorageBuildCounter = lazyCounter3("publisher_storage_chained_build_total", "Chained storage= source builds (a source reading a storage-materialized " + "upstream). Label: outcome ('parent_reuse'|'inline_fallback'|" + "'strict_refused'). The parent_reuse share is the headline signal for how " + "far the stack-on-the-parent path gets us vs recompute-from-raw.");
@@ -271489,12 +271478,28 @@ source: ${sourceName} is ${binding.destinationName}.virtual('${binding.virtualHa
 `;
   return { modelText, shapeTypeName };
 }
+function serveShapeGivenLine(given) {
+  const suffix = given.defaultText !== undefined ? ` is ${given.defaultText}` : "";
+  return `  ${given.name} :: ${given.type}${suffix}`;
+}
 function serveShapeFragment(binding) {
   const shapeTypeName = `${binding.sourceName}__shape`;
   const fields = binding.schema.map((c) => `   ${emitFieldName(c.name)}::${duckdbTypeToMalloy(c.type)}`).join(`,
 `);
   let source = `source: ${binding.sourceName} is ` + `${binding.destinationName}.virtual('${binding.virtualHandle}')::${shapeTypeName}`;
-  const refinements = binding.refinements ?? [];
+  const lines = refinementLines(binding.refinements ?? []);
+  if (lines.length > 0) {
+    source += ` extend {
+${lines.join(`
+`)}
+}`;
+  }
+  return `type: ${shapeTypeName} is {
+${fields}
+}
+${source}`;
+}
+function refinementLines(refinements) {
   const lines = [];
   for (const r of refinements) {
     if (r.kind === "join")
@@ -271513,16 +271518,7 @@ function serveShapeFragment(binding) {
     if (r.kind === "view")
       lines.push(`   view: ${r.text}`);
   }
-  if (lines.length > 0) {
-    source += ` extend {
-${lines.join(`
-`)}
-}`;
-  }
-  return `type: ${shapeTypeName} is {
-${fields}
-}
-${source}`;
+  return lines;
 }
 function serveShapeDiagnostics(allBindings, freshBindings) {
   const shapeSources = freshBindings.map((b) => b.sourceName);
@@ -271532,20 +271528,104 @@ function serveShapeDiagnostics(allBindings, freshBindings) {
     staleSources: allBindings.map((b) => b.sourceName).filter((name) => !fresh.has(name))
   };
 }
-function buildServeShapeModelForBindings(bindings, rollupGroups = []) {
+function derivedSourceFragment(lift) {
+  const lines = refinementLines(lift.refinements);
+  const body = lines.length > 0 ? ` extend {
+${lines.join(`
+`)}
+}` : "";
+  return `source: ${lift.sourceName} is ${lift.base}${body}`;
+}
+function optsOutOfPersist(def) {
+  const notes = def.annotations?.blockNotes;
+  return (notes ?? []).some((note) => typeof note?.text === "string" && /^\s*#@\s*-persist\b/.test(note.text));
+}
+function liftDerivedSources(ctx) {
+  const lifts = [];
+  const available = new Set(ctx.shapeSourceNames);
+  const pending = Object.entries(ctx.contents).filter(([name, def]) => typeof def?.extends === "string" && !(ctx.boundSourceNames ?? ctx.shapeSourceNames).has(name) && def?.persistent !== true && !optsOutOfPersist(def));
+  let progressed = true;
+  while (progressed) {
+    progressed = false;
+    for (let i = 0;i < pending.length; i++) {
+      const entry = pending[i];
+      if (!entry)
+        continue;
+      const [name, def] = entry;
+      const base = ctx.sourceNameById.get(def.extends);
+      if (!base || !available.has(base))
+        continue;
+      const lift = liftOneDerivedSource(name, def, base, available, ctx);
+      pending[i] = undefined;
+      progressed = true;
+      if (!lift)
+        continue;
+      lifts.push(lift);
+      available.add(name);
+    }
+  }
+  return lifts;
+}
+function liftOneDerivedSource(sourceName, def, base, available, ctx) {
+  const baseDef = ctx.contents[base];
+  const baseFieldNames = new Set((baseDef?.fields ?? []).map((f) => fieldKey(f)).filter(Boolean));
+  const ownFields = (def.fields ?? []).filter((f) => !baseFieldNames.has(fieldKey(f)));
+  const declaredJoins = ownFields.filter((f) => typeof f.join === "string").length;
+  const joins = extractJoins(ownFields, {
+    sourceNameById: ctx.sourceNameById,
+    materializedSourceNames: available,
+    liftText: ctx.liftText
+  });
+  if (joins.length !== declaredJoins)
+    return;
+  return {
+    sourceName,
+    base,
+    refinements: [
+      ...joins,
+      ...extractRefinements(ownFields),
+      ...extractSourceFilters(ownFilterList(def.filterList, baseDef?.filterList)),
+      ...extractViews(ownFields, ctx.liftText)
+    ]
+  };
+}
+function fieldKey(field) {
+  const f = field;
+  if (typeof f?.as === "string")
+    return f.as;
+  return typeof f?.name === "string" ? f.name : "";
+}
+function ownFilterList(filterList, baseFilterList) {
+  const own = filterList ?? [];
+  const inherited = baseFilterList ?? [];
+  let shared = 0;
+  while (shared < inherited.length && shared < own.length && own[shared]?.code === inherited[shared]?.code) {
+    shared++;
+  }
+  return own.slice(shared);
+}
+function buildServeShapeModelForBindings(bindings, rollupGroups = [], givens = [], derived = []) {
   const fragments = orderBindingsByJoinDeps(bindings).map(serveShapeFragment).join(`
+`);
+  const lifted = derived.map(derivedSourceFragment).join(`
 `);
   const groups = rollupGroups.map(rollupServeShapeFragment).join(`
 `);
-  const flags = rollupGroups.length ? "##! experimental { virtual_source composite_sources }" : "##! experimental.virtual_source";
-  return {
-    modelText: groups ? `${flags}
-${fragments}
-${groups}
-` : `${flags}
-${fragments}
-`
-  };
+  const enabled = ["virtual_source"];
+  if (rollupGroups.length)
+    enabled.push("composite_sources");
+  if (givens.length)
+    enabled.push("givens");
+  const flags = enabled.length === 1 ? "##! experimental.virtual_source" : `##! experimental { ${enabled.join(" ")} }`;
+  const givenBlock = givens.length ? `given:
+${givens.map(serveShapeGivenLine).join(`
+`)}
+` : "";
+  const body = [fragments, lifted, groups].filter(Boolean).join(`
+`);
+  return { modelText: `${flags}
+${givenBlock}${body}
+` };
 }
 function buildServeShapeTiers(rollupGroups) {
   const always = [...NEVER_THINNED];
@@ -272514,6 +272594,209 @@ var init_gate_classification = __esm(() => {
   init_partition_annotation();
 });
 
+// src/service/persist_dynamic_terms.ts
+function givenNames(node) {
+  const names = new Set;
+  collectGivenRefs(node, names);
+  return [...names];
+}
+function readsAGiven(node, seen, depth = 0) {
+  if (depth > MAX_WALK_DEPTH) {
+    throw new Error("given walk exceeded max depth");
+  }
+  if (node === null || typeof node !== "object")
+    return false;
+  if (seen.has(node))
+    return false;
+  seen.add(node);
+  if (Array.isArray(node)) {
+    return node.some((item) => readsAGiven(item, seen, depth + 1));
+  }
+  const record = node;
+  if (record.node === "given" || record.node === "givenReference")
+    return true;
+  if (record.givenRef !== undefined)
+    return true;
+  const summary = record.refSummary;
+  if (summary?.givenUsage && summary.givenUsage.length > 0)
+    return true;
+  if (Array.isArray(record.givenUsage) && record.givenUsage.length > 0) {
+    return true;
+  }
+  return Object.values(record).some((v) => readsAGiven(v, seen, depth + 1));
+}
+function buildSubstitutesAGiven(persistSource) {
+  try {
+    const def = persistSource._sourceDef;
+    if (def === null || typeof def !== "object") {
+      throw new Error("compiled source definition is not readable");
+    }
+    const query = def.query;
+    if (query !== null && typeof query === "object") {
+      const usage = query.givenUsage;
+      if (Array.isArray(usage) && usage.length > 0)
+        return true;
+      if (argumentBindsAGiven(query))
+        return true;
+      if (Array.isArray(usage))
+        return false;
+    }
+    const {
+      filterList: _readTimeFilters,
+      fields: _readTimeFields,
+      ...relation
+    } = def;
+    return readsAGiven(relation, new WeakSet);
+  } catch {
+    return true;
+  }
+}
+function argumentBindsAGiven(node, depth = 0) {
+  if (depth > MAX_WALK_DEPTH) {
+    throw new Error("argument walk exceeded max depth");
+  }
+  if (node === null || typeof node !== "object")
+    return false;
+  if (Array.isArray(node)) {
+    return node.some((item) => argumentBindsAGiven(item, depth + 1));
+  }
+  for (const [key, value] of Object.entries(node)) {
+    if ((key === "arguments" || key === "sourceArguments") && readsAGiven(value, new WeakSet)) {
+      return true;
+    }
+    if (argumentBindsAGiven(value, depth + 1))
+      return true;
+  }
+  return false;
+}
+function classifyDynamicTerms(persistSource) {
+  if (buildSubstitutesAGiven(persistSource)) {
+    return {
+      ok: false,
+      reason: "given_in_persisted_query",
+      detail: `a given is read while the persisted relation is BUILT, so its ` + `value is substituted at build time — from the declaration's ` + `default, the only value available then — and every caller is ` + `served that one slice. Move the given out of the persisted query ` + `and into the source's extend block (\`where: …\`), where it is ` + `left out of the build and applied per caller when the artifact ` + `is read`
+    };
+  }
+  let def;
+  try {
+    def = persistSource._sourceDef;
+    if (def === null || typeof def !== "object") {
+      throw new Error("compiled source definition is not readable");
+    }
+  } catch (err) {
+    return {
+      ok: false,
+      reason: "given_in_persisted_query",
+      detail: `its given usage could not be determined ` + `(${err instanceof Error ? err.message : String(err)}), so the ` + `publisher cannot prove which givens the build would bake in`
+    };
+  }
+  try {
+    const fieldRefusal = classifyFields(def.fields);
+    if (fieldRefusal)
+      return fieldRefusal;
+    const terms = collectDynamicTerms(def.filterList);
+    const {
+      filterList: _accounted,
+      fields: _alsoAccounted,
+      query: _cleared,
+      ...rest
+    } = def;
+    if (readsAGiven(rest, new WeakSet)) {
+      return {
+        ok: false,
+        reason: "given_in_persisted_query",
+        detail: `a given sits in a part of the source this pass cannot place ` + `— it is neither an extend-block \`where:\` nor a field ` + `declared on the source — so the publisher cannot prove the ` + `build leaves it out`
+      };
+    }
+    return { ok: true, terms };
+  } catch (err) {
+    return {
+      ok: false,
+      reason: "given_in_persisted_query",
+      detail: `its given usage could not be determined ` + `(${err instanceof Error ? err.message : String(err)}), so the ` + `publisher cannot prove the build leaves every given out`
+    };
+  }
+}
+function classifyFields(fields) {
+  if (!Array.isArray(fields))
+    return;
+  for (const field of fields) {
+    if (field === null || typeof field !== "object")
+      continue;
+    const f = field;
+    const label = typeof f.as === "string" ? f.as : typeof f.name === "string" ? f.name : "(unnamed)";
+    if (f.join !== undefined) {
+      if (readsAGiven(f.onExpression, new WeakSet)) {
+        return {
+          ok: false,
+          reason: "dynamic_join",
+          detail: `the join '${label}' has a given in its \`on:\` condition, ` + `which decides which rows join per caller`
+        };
+      }
+      if (readsAGiven(f, new WeakSet)) {
+        return {
+          ok: false,
+          reason: "dynamic_joined_where",
+          detail: `the source joined as '${label}' is itself scoped by a ` + `given. Enter through a non-persisted extension that ` + `declares the join instead, so the given term is part of ` + `the query rather than of the artifact`
+        };
+      }
+      continue;
+    }
+    if (readsAGiven(f, new WeakSet)) {
+      return {
+        ok: false,
+        reason: "dynamic_projection",
+        detail: `the field '${label}' reads a given, so its value would ` + `differ per caller`
+      };
+    }
+  }
+  return;
+}
+function localFieldNames(entry) {
+  const usage = entry?.refSummary?.fieldUsage;
+  if (!Array.isArray(usage))
+    return [];
+  const names = new Set;
+  for (const use of usage) {
+    const path9 = use?.path;
+    if (!Array.isArray(path9) || path9.length !== 1 || typeof path9[0] !== "string") {
+      return [];
+    }
+    names.add(path9[0]);
+  }
+  return [...names];
+}
+function readTimeDynamicTerms(persistSource) {
+  try {
+    const def = persistSource._sourceDef;
+    return collectDynamicTerms(def?.filterList);
+  } catch {
+    return [{ code: "<unreadable>", givens: [], columns: [] }];
+  }
+}
+function collectDynamicTerms(filterList) {
+  if (!Array.isArray(filterList))
+    return [];
+  const terms = [];
+  for (const entry of filterList) {
+    if (entry === null || typeof entry !== "object")
+      continue;
+    if (!readsAGiven(entry, new WeakSet))
+      continue;
+    const code = entry.code;
+    terms.push({
+      code: typeof code === "string" ? code : "",
+      givens: givenNames(entry),
+      columns: localFieldNames(entry)
+    });
+  }
+  return terms;
+}
+var MAX_WALK_DEPTH = 200;
+var init_persist_dynamic_terms = __esm(() => {
+  init_given();
+});
+
 // src/service/incremental_declaration.ts
 function safeTag(source) {
   try {
@@ -272582,6 +272865,11 @@ function readKey(tag, key) {
 function resolveIncrementalDeclaration(source, annotationFields) {
   const tag = safeTag(source);
   const columns = outputColumnTypes(source);
+  const dynamicTerms = readTimeDynamicTerms(source);
+  const scopeColumns = [
+    ...new Set(dynamicTerms.flatMap((term) => term.columns))
+  ].filter((name) => columns.has(name));
+  const scopeIncomplete = dynamicTerms.some((term) => term.columns.length === 0) || dynamicTerms.some((term) => term.columns.some((name) => !columns.has(name)));
   const { aggregates, analytics } = queryDefinitionFieldKinds(source);
   const malformed = [];
   const rawRefresh = readKey(tag, "refresh");
@@ -272641,6 +272929,8 @@ function resolveIncrementalDeclaration(source, annotationFields) {
     watermark,
     watermarkOrderable,
     mergeKeys,
+    scopeColumns,
+    scopeIncomplete,
     watermarkInMergeKeys,
     strategy,
     malformed,
@@ -272652,6 +272942,7 @@ function resolveIncrementalDeclaration(source, annotationFields) {
 var REFRESH_MODES, RECOGNIZED_PERSIST_KEYS, ORDERABLE_TYPES;
 var init_incremental_declaration = __esm(() => {
   init_build_plan();
+  init_persist_dynamic_terms();
   REFRESH_MODES = ["full", "incremental"];
   RECOGNIZED_PERSIST_KEYS = new Set([
     "persist",
@@ -272663,6 +272954,7 @@ var init_incremental_declaration = __esm(() => {
     "merge_key",
     "freshness",
     "queryMetadata",
+    "partition",
     "sharing",
     "schedule"
   ]);
@@ -272674,8 +272966,70 @@ var init_incremental_declaration = __esm(() => {
   ]);
 });
 
+// src/service/persist_partition.ts
+function parsePartitionValue(raw) {
+  return raw.split(",").map((name) => name.trim()).filter((name) => name.length > 0);
+}
+function resolvePartitionColumns(persistSource, annotationFields) {
+  const raw = annotationFields.partition;
+  if (raw === undefined)
+    return { ok: true, columns: [] };
+  const columns = parsePartitionValue(raw);
+  if (columns.length === 0)
+    return { ok: true, columns: [] };
+  if (!annotationFields.storage) {
+    return {
+      ok: false,
+      reason: "partition_without_storage",
+      detail: `'partition=' is declared without 'storage='. A colocated ` + `'#@ persist' writes into the source's own warehouse, where the ` + `table layout is that warehouse's own DDL — the publisher does ` + `not lay it out. Add 'storage=' to build into a storage ` + `destination, or drop 'partition='`
+    };
+  }
+  const { publicNames, declaredNames } = columnSurface(persistSource);
+  for (const column of columns) {
+    if (publicNames.has(column))
+      continue;
+    if (declaredNames.has(column)) {
+      return {
+        ok: false,
+        reason: "partition_column_not_public",
+        detail: `'partition=' names '${column}', which the source hides. The ` + `stored table is narrowed to the source's public columns, so a ` + `hidden one is not there to partition by. Make it public, or ` + `partition by a column that is`
+      };
+    }
+    return {
+      ok: false,
+      reason: "partition_column_unknown",
+      detail: `'partition=' names '${column}', which is not a column of the ` + `source (a column removed with 'except:' is not one either). ` + `Partition by a column the source projects`
+    };
+  }
+  return { ok: true, columns };
+}
+function columnSurface(persistSource) {
+  const publicNames = new Set;
+  const declaredNames = new Set;
+  try {
+    const def = persistSource._sourceDef;
+    for (const field of def?.fields ?? []) {
+      if (field === null || typeof field !== "object")
+        continue;
+      const f = field;
+      if (f.join !== undefined)
+        continue;
+      const name = typeof f.as === "string" ? f.as : f.name;
+      if (typeof name !== "string" || name.length === 0)
+        continue;
+      declaredNames.add(name);
+      const access = f.accessModifier;
+      if (access == null || access === "public")
+        publicNames.add(name);
+    }
+  } catch {
+    return { publicNames: new Set, declaredNames: new Set };
+  }
+  return { publicNames, declaredNames };
+}
+
 // src/service/materialization_eligibility.ts
-function assertMaterializationEligible(persistSource) {
+function assertMaterializationEligible(persistSource, annotationFields) {
   const sourceName = persistSource.name;
   let unbound;
   try {
@@ -272694,13 +273048,15 @@ function assertMaterializationEligible(persistSource) {
       message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it has unbound parameter(s) ` + `${unbound.map((p) => `'${p}'`).join(", ")}. A source with a free ` + `parameter is a template instantiated per query, so there is no ` + `single relation to materialize. Bind the parameter to a constant, ` + `or drop 'storage=' to serve it live from the source warehouse.`
     });
   }
-  if (referencesGiven(persistSource)) {
-    recordEligibilityRefused("given");
+  const classified = classifyDynamicTerms(persistSource);
+  if (!classified.ok) {
+    recordEligibilityRefused(classified.reason);
     throw new MaterializationEligibilityError({
-      reason: "given",
-      message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it references a given. Givens bind per query and are ` + `used for row-level access control, so a materialized-once table ` + `served to everyone would leak filtered rows across tenants. This ` + `is refused for safety. Serve this source live (drop 'storage=').`
+      reason: classified.reason,
+      message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: ${classified.detail}.`
     });
   }
+  assertMergeKeyScopeResolvable(persistSource, readTimeDynamicTerms(persistSource), annotationFields, `Source '${sourceName}'`);
   if (referencesAuthorize(persistSource)) {
     recordEligibilityRefused("authorize");
     throw new MaterializationEligibilityError({
@@ -272715,8 +273071,39 @@ function assertMaterializationEligible(persistSource) {
       message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it declares a #(partition) marker. A partition ` + `filter binds a given at query time (the same mechanism as ` + `row-level access control), so a materialized-once table served ` + `frozen would leak every partition's rows to every caller. This ` + `is refused for safety. Serve this source live (drop 'storage=').`
     });
   }
+  const partition = resolvePartitionColumns(persistSource, annotationFields);
+  if (!partition.ok) {
+    recordEligibilityRefused(partition.reason);
+    throw new MaterializationEligibilityError({
+      reason: partition.reason,
+      message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: ${partition.detail}.`
+    });
+  }
+  return { terms: classified.terms, partitionColumns: partition.columns };
+}
+function assertMergeKeyScopeResolvable(persistSource, terms, annotationFields, what) {
+  if (!annotationFields.merge_key?.trim())
+    return;
+  const stored = new Set(deriveColumns(persistSource).map((column) => column.name).filter((name) => typeof name === "string"));
+  const unscopable = terms.filter((term) => term.columns.length === 0 || term.columns.some((name) => !stored.has(name)));
+  if (unscopable.length === 0)
+    return;
+  recordEligibilityRefused("merge_key_scope_unresolved");
+  throw new MaterializationEligibilityError({
+    reason: "merge_key_scope_unresolved",
+    message: `${what} cannot be materialized: it declares 'merge_key=' and is ` + `scoped by a term this pass cannot express as a predicate over the ` + `stored table's own columns (${unscopable.map((term) => `'${term.code}'`).join(", ")}). An incremental refresh matches rows by the merge ` + `key, and the stored table holds every caller's rows, so the match ` + `must also carry the scoping columns or it would update rows ` + `belonging to other callers. Scope this source with a term over its ` + `own columns, or drop 'merge_key=' to refresh by watermark range.`
+  });
 }
-function assertColocatedPersistNotAuthorizeGated(persistSource, sourceName = persistSource.name, origin2 = "persist", gateOutcome) {
+function assertColocatedPersistNotAuthorizeGated(persistSource, sourceName = persistSource.name, origin2 = "persist", gateOutcome, annotationFields = {}) {
+  const partition = resolvePartitionColumns(persistSource, annotationFields);
+  if (!partition.ok) {
+    recordEligibilityRefused(partition.reason);
+    const what2 = origin2 === "preaggregate" ? `Pre-aggregation rollup '${sourceName}'` : `Source '${sourceName}'`;
+    throw new MaterializationEligibilityError({
+      reason: partition.reason,
+      message: `${what2} cannot be materialized (colocated '#@ persist'): ` + `${partition.detail}.`
+    });
+  }
   if (referencesPartition(persistSource)) {
     recordEligibilityRefused("partition");
     const what2 = origin2 === "preaggregate" ? `Pre-aggregation rollup '${sourceName}'` : `Source '${sourceName}'`;
@@ -272727,13 +273114,21 @@ function assertColocatedPersistNotAuthorizeGated(persistSource, sourceName = per
     });
   }
   if (buildSubstitutesAGiven(persistSource)) {
+    if (origin2 === "preaggregate") {
+      recordEligibilityRefused("preaggregate_over_dynamic_source");
+      throw new MaterializationEligibilityError({
+        reason: "preaggregate_over_dynamic_source",
+        message: `Pre-aggregation rollup '${sourceName}' cannot be materialized: ` + `the source it rolls up is scoped by a given. Building the rollup ` + `reads that source, which applies its extend-block \`where:\` and ` + `substitutes the given's default — so the rollup would hold one ` + `caller's aggregate and serve it to everyone. Unlike a persisted ` + `source, a rollup has no read-time re-application to put the term ` + `back. Drop the '#@ preaggregate' from this measure, or roll up a ` + `source that is not caller-scoped.`
+      });
+    }
     recordEligibilityRefused("given_in_persisted_query");
-    const what2 = origin2 === "preaggregate" ? `Pre-aggregation rollup '${sourceName}'` : `Source '${sourceName}'`;
+    const what2 = `Source '${sourceName}'`;
     throw new MaterializationEligibilityError({
       reason: "given_in_persisted_query",
       message: `${what2} cannot be materialized (colocated '#@ persist'): its ` + `persisted query references a given. A given the persisted query ` + `is built with is substituted at BUILD time, so the table would ` + `hold the default's rows and serve them to every caller, ignoring ` + `the value each one supplies. Keep the given OUT of the persisted ` + `query and apply it when the source is read — a \`where:\` in the ` + `source's extend block, or a dimension, measure or join declared ` + `there — where it binds per caller over the materialized rows. ` + `Otherwise stop persisting this source.`
     });
   }
+  assertMergeKeyScopeResolvable(persistSource, readTimeDynamicTerms(persistSource), annotationFields, origin2 === "preaggregate" ? `Pre-aggregation rollup '${sourceName}'` : `Source '${sourceName}'`);
   if (!referencesAuthorize(persistSource))
     return;
   if (origin2 === "persist" && gateOutcome?.classification === "row_level" && gateOutcome.attributed) {
@@ -272768,91 +273163,6 @@ function unboundParameterNames(persistSource) {
   }
   return unbound;
 }
-function referencesGiven(persistSource) {
-  try {
-    return walkForGiven(persistSource._sourceDef, new WeakSet, 0);
-  } catch {
-    return true;
-  }
-}
-function argumentBindsAGiven(node, depth = 0) {
-  if (depth > MAX_GIVEN_WALK_DEPTH) {
-    throw new Error("argument walk exceeded max depth");
-  }
-  if (node === null || typeof node !== "object")
-    return false;
-  if (Array.isArray(node)) {
-    return node.some((item) => argumentBindsAGiven(item, depth + 1));
-  }
-  for (const [key, value] of Object.entries(node)) {
-    if ((key === "arguments" || key === "sourceArguments") && walkForGiven(value, new WeakSet, 0)) {
-      return true;
-    }
-    if (argumentBindsAGiven(value, depth + 1))
-      return true;
-  }
-  return false;
-}
-function buildSubstitutesAGiven(persistSource) {
-  try {
-    const def = persistSource._sourceDef;
-    if (def === null || typeof def !== "object") {
-      throw new Error("compiled source definition is not readable");
-    }
-    const query = def.query;
-    if (query !== null && typeof query === "object") {
-      const usage = query.givenUsage;
-      if (Array.isArray(usage) && usage.length > 0)
-        return true;
-      if (argumentBindsAGiven(query))
-        return true;
-      if (Array.isArray(usage))
-        return false;
-    }
-    const {
-      filterList: _sourceFilters,
-      fields: _queryTimeFields,
-      ...relation
-    } = def;
-    return walkForGiven(relation, new WeakSet, 0);
-  } catch {
-    return true;
-  }
-}
-function walkForGiven(node, seen, depth) {
-  if (depth > MAX_GIVEN_WALK_DEPTH) {
-    throw new Error("given-usage walk exceeded max depth");
-  }
-  if (node === null || typeof node !== "object")
-    return false;
-  if (seen.has(node))
-    return false;
-  seen.add(node);
-  if (Array.isArray(node)) {
-    for (const item of node) {
-      if (walkForGiven(item, seen, depth + 1))
-        return true;
-    }
-    return false;
-  }
-  const record = node;
-  const nodeKind = record.node;
-  if (nodeKind === "given" || nodeKind === "givenReference")
-    return true;
-  if (record.givenRef !== undefined)
-    return true;
-  const refSummary = record.refSummary;
-  if (refSummary?.givenUsage && refSummary.givenUsage.length > 0)
-    return true;
-  if (Array.isArray(record.givenUsage) && record.givenUsage.length > 0) {
-    return true;
-  }
-  for (const value of Object.values(record)) {
-    if (walkForGiven(value, seen, depth + 1))
-      return true;
-  }
-  return false;
-}
 function referencesPartition(persistSource) {
   try {
     return walkForPartition(persistSource._sourceDef, new WeakSet, 0);
@@ -272991,7 +273301,9 @@ var init_materialization_eligibility = __esm(() => {
   init_errors();
   init_materialization_metrics();
   init_authorize();
+  init_build_plan();
   init_partition_annotation();
+  init_persist_dynamic_terms();
 });
 
 // src/service/preaggregation_compile.ts
@@ -273055,6 +273367,21 @@ function deriveColumns(persistSource) {
     return [];
   }
 }
+function planDynamicFields(source, annotationFields) {
+  const out = {};
+  const partition = resolvePartitionColumns(source, annotationFields);
+  if (partition.ok && partition.columns.length > 0) {
+    out.partition = partition.columns;
+  }
+  const classified = classifyDynamicTerms(source);
+  if (classified.ok && classified.terms.length > 0) {
+    out.strippedTerms = classified.terms.map((term) => ({
+      code: term.code,
+      givens: term.givens
+    }));
+  }
+  return out;
+}
 function publicColumnNames(persistSource) {
   let names;
   try {
@@ -273208,7 +273535,19 @@ function* iterGraphSources(graph, sources) {
   }
 }
 function computeSourceEntityId(source, connectionDigests) {
-  return source.makeBuildId(connectionDigests[source.connectionName], source.getSQL());
+  return source.makeBuildId(connectionDigests[source.connectionName], addressPartitionLayout(source, source.getSQL()));
+}
+function addressPartitionLayout(source, sql) {
+  let columns = [];
+  try {
+    const resolved = resolvePartitionColumns(source, deriveAnnotationFields(source));
+    if (resolved.ok)
+      columns = resolved.columns;
+  } catch {}
+  if (columns.length === 0)
+    return sql;
+  return `${sql}
+-- partitioned by: ${columns.join(", ")}`;
 }
 async function resolvePackageConnections(pkg, names) {
   const map = new Map;
@@ -273401,9 +273740,9 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
       if (!rollup) {
         try {
           if (declaresStorage) {
-            assertMaterializationEligible(source);
+            assertMaterializationEligible(source, deriveAnnotationFields(source));
           } else {
-            assertColocatedPersistNotAuthorizeGated(source, source.name, "persist", options?.sourceGateOutcomes?.[sourceID]);
+            assertColocatedPersistNotAuthorizeGated(source, source.name, "persist", options?.sourceGateOutcomes?.[sourceID], annotationFields);
           }
         } catch (err) {
           if (!(err instanceof MaterializationEligibilityError))
@@ -273421,10 +273760,10 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
       } else {
         let storageRefused = false;
         try {
-          assertColocatedPersistNotAuthorizeGated(source, source.name, "preaggregate", options?.sourceGateOutcomes?.[sourceID]);
+          assertColocatedPersistNotAuthorizeGated(source, source.name, "preaggregate", options?.sourceGateOutcomes?.[sourceID], annotationFields);
           if (declaresStorage) {
             storageRefused = true;
-            assertMaterializationEligible(source);
+            assertMaterializationEligible(source, deriveAnnotationFields(source));
             storageRefused = false;
           }
         } catch (err) {
@@ -273459,6 +273798,7 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
         freshness: resolveFreshness(source, packageMaterialization),
         queryMetadata: resolveQueryMetadata(source, packageMaterialization),
         columns: deriveColumns(source),
+        ...planDynamicFields(source, annotationFields),
         annotationFields,
         modelPath: sourceModelPaths?.[sourceID]
       };
@@ -273507,7 +273847,7 @@ function collectSourceEligibility(sources) {
   const refused = {};
   for (const source of Object.values(sources)) {
     try {
-      assertMaterializationEligible(source);
+      assertMaterializationEligible(source, deriveAnnotationFields(source));
       eligible.push(source.name);
     } catch (err) {
       refused[source.name] = errMessage(err);
@@ -273521,7 +273861,7 @@ function collectColocatedSourceEligibility(compiled) {
   for (const [sourceID, source] of Object.entries(compiled.sources)) {
     const origin2 = compiled.preaggregatePlans?.[sourceID] ? "preaggregate" : "persist";
     try {
-      assertColocatedPersistNotAuthorizeGated(source, source.name, origin2, compiled.sourceGateOutcomes?.[sourceID]);
+      assertColocatedPersistNotAuthorizeGated(source, source.name, origin2, compiled.sourceGateOutcomes?.[sourceID], deriveAnnotationFields(source));
     } catch (err) {
       let sourceEntityId2;
       try {
@@ -273565,6 +273905,7 @@ var init_build_plan = __esm(() => {
   init_materialization_eligibility();
   init_model();
   init_preaggregation_compile();
+  init_persist_dynamic_terms();
   init_quoting();
   FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"];
 });
@@ -275574,7 +275915,7 @@ var init_model = __esm(() => {
         freshnessFallback: b.freshnessFallback
       }, at) === "serve_table");
     }
-    async loadServeShapeQuery(queryString) {
+    async loadServeShapeQuery(queryString, givens) {
       const freshBindings = this.freshServeBindings(Date.now());
       if (freshBindings.length === 0) {
         throw new Error("no fresh storage serve bindings for this query");
@@ -275600,7 +275941,7 @@ var init_model = __esm(() => {
         ...groups.flatMap((g) => g.members)
       ]);
       const runnable = this.serveShapeCache.materializer.loadRestrictedQuery(queryString);
-      const probeSQL = await runnable.getSQL({ virtualMap });
+      const probeSQL = await runnable.getSQL({ virtualMap, givens });
       const rollupPaths = [
         ...buildVirtualMap(groups.flatMap((g) => g.members))
       ].flatMap(([, byHandle]) => [...byHandle.values()]);
@@ -275617,7 +275958,7 @@ var init_model = __esm(() => {
           ...b,
           refinements: b.refinements.filter((r) => keep.has(r.kind))
         } : b);
-        const materializer = this.buildServeShapeMaterializer(shaped, groups);
+        const materializer = this.buildServeShapeMaterializer(shaped, groups, tier === 0 ? this.liftedDerivedSources(enriched) : []);
         if (tier === lastTier) {
           try {
             await materializer.getModel();
@@ -275678,8 +276019,23 @@ var init_model = __esm(() => {
       }
       return servable;
     }
-    buildServeShapeMaterializer(bindings, rollupGroups = []) {
-      const { modelText } = buildServeShapeModelForBindings(bindings, rollupGroups);
+    serveShapeGivens() {
+      const out = [];
+      for (const given of this.givens ?? []) {
+        if (typeof given.name !== "string" || given.name.length === 0)
+          continue;
+        if (typeof given.type !== "string" || given.type.length === 0)
+          continue;
+        out.push({
+          name: given.name,
+          type: given.type,
+          defaultText: typeof given.default === "string" ? given.default : undefined
+        });
+      }
+      return out;
+    }
+    buildServeShapeMaterializer(bindings, rollupGroups = [], derived = []) {
+      const { modelText } = buildServeShapeModelForBindings(bindings, rollupGroups, this.serveShapeGivens(), derived);
       const root = "file:///storage-serve-shape/";
       const url2 = `${root}shape.malloy`;
       const runtime = new Runtime2({
@@ -275693,15 +276049,14 @@ var init_model = __esm(() => {
         importBaseURL: new URL(root)
       });
     }
-    serveBindingsWithRefinements(bindings) {
-      const contents = this.modelDef?.contents;
+    authorModelLift() {
+      const contents = this.modelDef?.contents ?? {};
       const sourceNameById = new Map;
-      for (const [name, def] of Object.entries(contents ?? {})) {
+      for (const [name, def] of Object.entries(contents)) {
         if (typeof def?.sourceID === "string") {
           sourceNameById.set(def.sourceID, name);
         }
       }
-      const materializedSourceNames = new Set(bindings.map((b) => b.sourceName));
       const fileCache = new Map;
       const liftText = (location) => {
         if (!location?.url?.startsWith("file:"))
@@ -275716,6 +276071,21 @@ var init_model = __esm(() => {
         const text = fileCache.get(location.url);
         return text ? sliceSourceRange(text, location.range) : undefined;
       };
+      return { contents, sourceNameById, liftText };
+    }
+    liftedDerivedSources(bindings) {
+      const { contents, sourceNameById, liftText } = this.authorModelLift();
+      return liftDerivedSources({
+        contents,
+        sourceNameById,
+        shapeSourceNames: new Set(bindings.map((b) => b.sourceName)),
+        boundSourceNames: new Set(this.serveBindings.map((b) => b.sourceName)),
+        liftText
+      });
+    }
+    serveBindingsWithRefinements(bindings) {
+      const { contents, sourceNameById, liftText } = this.authorModelLift();
+      const materializedSourceNames = new Set(bindings.map((b) => b.sourceName));
       return bindings.map((b) => {
         const fields = contents?.[b.sourceName]?.fields;
         const schema = narrowSchemaToPublic(b.schema, fields);
@@ -275811,7 +276181,7 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
         }
         if (storageRoutingPossible && !routingBlockedByRowLevelGate) {
           try {
-            const shaped = await this.loadServeShapeQuery(queryString);
+            const shaped = await this.loadServeShapeQuery(queryString, querySurfaceGivens);
             runnable = shaped.runnable;
             serveVirtualMap = shaped.virtualMap;
             serveShapeBindings = shaped.bindings;
@@ -275889,7 +276259,7 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
       let executionTime = 0;
       let queryResults;
       let appliedQueryMetadata;
-      const effectiveGivens = serveVirtualMap ? undefined : querySurfaceGivens;
+      const effectiveGivens = querySurfaceGivens;
       try {
         const preparedResult = await runnable.getPreparedResult({
           givens: effectiveGivens,
@@ -308879,8 +309249,12 @@ function deltaStatements(params) {
   const q = (name) => quoteIdentifier(name, dialect);
   const columnList = columns.map(q).join(", ");
   if (mergeKeys.length > 0) {
+    const matchNames = [
+      ...mergeKeys,
+      ...(params.scopeColumns ?? []).filter((c) => !mergeKeys.includes(c))
+    ];
     const keys = new Set(mergeKeys);
-    const on = mergeKeys.map((k) => `(__t.${q(k)} = __s.${q(k)} OR (__t.${q(k)} IS NULL AND __s.${q(k)} IS NULL))`).join(" AND ");
+    const on = matchNames.map((k) => `(__t.${q(k)} = __s.${q(k)} OR (__t.${q(k)} IS NULL AND __s.${q(k)} IS NULL))`).join(" AND ");
     const updates = columns.filter((c) => !keys.has(c)).map((c) => `${q(c)} = __s.${q(c)}`);
     const clauses = [
       `MERGE INTO ${quotedTablePath} AS __t`,
@@ -309088,6 +309462,7 @@ async function planIncrementalStep(inputs) {
     deltaSQL: await target.deltaRows(start, end.bound),
     columns: inputs.columns,
     mergeKeys: lineage.mergeKeys,
+    scopeColumns: lineage.scopeColumns,
     watermarkName: lineage.watermarkName,
     start,
     end: end.bound
@@ -309827,6 +310202,7 @@ class Package {
       ...this.dashboardWarnings,
       ...this.storageWarnings(),
       ...this.droppedPersistWarnings(),
+      ...this.persistRefusalWarnings(),
       ...this.emptyDiscoveryWarnings(),
       ...this.persistenceCollisionWarnings().map((message) => ({ message })),
       ...this.preaggregateAccessWarnings(),
@@ -310013,6 +310389,20 @@ class Package {
       message: `is annotated '#@ persist' but was not recognized as a ` + `materializable source, so nothing is materialized and it is served ` + `live. Only query/aggregate sources build; a filtered pass-through ` + `does not. Persist a query source, or invoke a parameterized source ` + `with a bound argument.`
     }));
   }
+  persistRefusalWarnings() {
+    const refused = this.buildPlan?.refusedSources;
+    if (!refused)
+      return [];
+    return Object.entries(refused).map(([sourceID, source]) => {
+      const subject = source.tier === "preaggregate" ? this.buildPlan?.sources?.[sourceID]?.preaggregate?.baseSourceName : source.name;
+      const warning = { message: source.message };
+      if (source.modelPath)
+        warning.model = source.modelPath;
+      if (subject)
+        warning.subject = subject;
+      return warning;
+    });
+  }
   storageWarnings() {
     const mode = getPersistStorageMode();
     if (mode === "on" || !this.buildPlan?.sources)
@@ -317433,11 +317823,16 @@ a returned source's documentation -- text the answerer reads and acts on. Only
 \`missing\` is a retrieval miss; the route per entity is recorded so the strict
 count is still there.
 
-Recall 1.0 with a wrong answer exonerates retrieval: the failure is in the query.
+Recall 1.0 with a wrong answer exonerates retrieval: everything arrived. Whether
+the agent misused it or the docs never said how to use it is \`eval-diagnose\`'s
+call, sufficiency first, so the row reads \`delivered, wrong\` and names no owner.
 Recall below 1.0 and \`coverage: covered\` means the entity existed and search did
-not surface it; \`derivable\` or \`absent\` means there was nothing to surface. Those
-look identical in an answer score and have opposite owners, which is what makes
-this number worth having. It uses the search terms the answerer chose, so it
+not surface it -- a documentation finding, because the retrieval algorithm is
+fixed (semantic search over doc strings) and an entity that exists and does not
+come back is one whose docs do not say what people ask; \`eval-diagnose\` calls it
+\`NOT-RETURNED\`, owner model. \`derivable\` or \`absent\` means there was nothing to
+surface. Those look identical in an answer score and have different owners,
+which is what makes this number worth having. It uses the search terms the answerer chose, so it
 attributes a failure *within* an arm and does not compare retrieval across arms
 -- that is the engine-side \`eval-retrieval\` skill, which does not ship here.
 
@@ -317465,6 +317860,13 @@ How to invoke it. Either \`--model <file-or-dir>\` for a local package or
 python3 check_coverage.py --set evals/ecommerce --model model.malloy --version 0.0.58
 \`\`\`
 
+Then hand the report back to the run: \`run_baseline.py --coverage <report>\`.
+Its per-case verdict beats the case's authored \`coverage\` label in retrieval
+attribution, \`run.json\` records which report was read, and each retrieval row
+says whether \`measured\`, \`authored\` or \`none\` charged the failure. Without a
+report, a case with no label is attributed to nobody rather than to the model,
+which is what used to happen.
+
 Two flags change what the number means, so choose them rather than inheriting
 them. \`--repeat N\` samples each case N times and takes the majority; it defaults
 to **1 for a set**, because the score is a trend over many cases rather than a
@@ -317499,6 +317901,34 @@ stops being stable. Do not take the over-size message's advice to narrow
 \`--model\` to one file, which drops every imported source and manufactures
 \`COVERAGE\` verdicts.
 
+## Validate the definitions, not every answer
+
+A golden must not be derived through the definitions the question TESTS. It may
+reuse everything the question does not turn on -- joins, base sources, date
+handling -- because circularity only bites where the key and the answer share the
+step under measurement. That is what makes this affordable on a model too large
+to reimplement.
+
+\`scripts/verify_definitions.py\` checks each definition once against the layer
+directly beneath it, and every case depending on it inherits the result. A golden
+is trustworthy if it was derived independently, OR if every definition it tests
+has itself been validated.
+
+Two things it will not do, and both matter more than what it does. It never
+reports a definition reaching through a join as validated, because fanout
+inflates the measure and the control expression equally and the comparison stays
+green on a broken join. And building the ledger without a server exits 3, not 0:
+nothing was checked, and a caller must not read that as a pass.
+
+A raw check -- a definition that reaches through a join -- is an authored
+control: a person writes the population down as a query on the ledger record and
+the tool re-runs it every time. It is not found unaided, and the docs say so.
+\`verify_goldens.py --definitions <ledger>\` then applies the composition rule at
+the gate: a set with no truth package exits 0 when every value-bearing case's
+tested definitions are validated, and names the cases that are not.
+
+**Read \`reference/definition-ledger.md\` before building or quoting one.**
+
 ## Step 5: Distrust the golden
 
 A reference answer can be wrong (parent-column fanout, a join on a shared
@@ -317561,7 +317991,7 @@ original answerer to "try again" with the new key in context.
 
 This skill's \`reference/\` files are served as separate prompts, one per file, fetched only when you ask for them. Where the text above says to read \`reference/<name>.md\`, get the prompt named \`eval-answer/<name>\` instead.
 
-Available: coverage-limits, ledger-schema.`
+Available: coverage-limits, definition-ledger, ledger-schema.`
     },
     {
       name: "eval-answer/coverage-limits",
@@ -317781,11 +318211,320 @@ the candidate sources only, to settle \`AMBIGUOUS\`, \`NO-DISAMBIG\` and
 \`CONVENTION\`. Deliberately NOT retrieval ranking: folding retrieval into this
 number would collapse the distinction from recall that the metric exists to
 draw.`
+    },
+    {
+      name: "eval-answer/definition-ledger",
+      description: "The definition ledger. Reference detail for the eval-answer skill.",
+      body: `<!-- Validating a model's definitions once, so answer keys need not be re-derived per case. Read before building or reading a ledger. -->
+
+# The definition ledger
+
+## Why it exists
+
+The loop's standing rule is that a golden must never be derived through the model
+under test, because a model bug would otherwise certify its own golden. That rule
+is sound, and it does not scale: on a model with real transformation, deriving
+every key independently means reimplementing the model. You end up with two
+implementations and no reason to trust either, and the eval task becomes *rebuild
+the model to verify the model*.
+
+The ledger is the other half. **A definition is checked once, against the layer
+directly beneath it, and every case depending on it inherits the result.** Work
+scales with the number of definitions rather than the number of cases, and stops
+growing as cases are added.
+
+The rule this replaces:
+
+> A key must be independent of **the definitions the question tests**, not of the
+> whole model.
+
+And the rule that makes that sound:
+
+> A golden is trustworthy if it was derived independently, **or** if every
+> definition it tests has itself been validated.
+
+## Deep models are cheaper per case, not dearer
+
+The tractability argument is about definitions being **shared**, not about models
+being shallow. An earlier draft claimed every measure sits one or two hops from a
+raw column; measured across four models, that is false:
+
+| Model | Definitions | within 2 hops | deepest chain |
+|---|---|---|---|
+| ecommerce (sample) | 32 | 90.6% | 3 |
+| storefront (sample) | 19 | 94.7% | 3 |
+| a 492-definition benchmark | 492 | 100% | 2 |
+| a real 548-definition customer model | 548 | **42.5%** | **12** |
+
+On that customer model, the twenty deepest measures close over **812**
+definition-checks with no reuse and **116** with the ledger: an 86% saving.
+Reuse pays off *more* as chains deepen, because deep chains share their lower
+layers. Depth is what makes the ledger worth having.
+
+## What a record holds
+
+\`\`\`json
+{
+  "entityId": "measure:order_items:total_sales",
+  "kind": "measure",
+  "source": "order_items",
+  "name": "total_sales",
+  "expr": "sale_price.sum()",
+  "exprSha": "3f9a…",
+  "depends": ["measure:order_items:order_item_count"],
+  "check": {"kind": "within_model", "slice": "first 200 rows"},
+  "verdict": "agrees",
+  "needs": null
+}
+\`\`\`
+
+\`exprSha\` is a **Merkle chain**, not a hash of the one line: it covers the
+definition and everything beneath it. A measure whose own text never moved is
+still stale once something it builds on changes, and a flat hash would report it
+as current. \`packageSha\` exists already but is whole-package, so it invalidates
+everything on any edit and distinguishes nothing.
+
+## The two check kinds
+
+### \`within_model\` -- no raw access, no truth package
+
+Ask the model for the entity and for the expression it **claims** to be, in one
+query, and compare:
+
+\`\`\`malloy
+run: order_items -> {
+  aggregate:
+    stated is total_sales
+    control is sale_price.sum()
+}
+\`\`\`
+
+Both sides come from **one** query against one source, so they see identical
+rows: a difference is the definition, not a moving population.
+
+A dimension is scalar, and \`aggregate:\` rejects it outright ("Cannot use a scalar
+field in an aggregate"), so a dimension is compared row by row over a slice. The
+slice is recorded, because a slice is weaker evidence than a whole table and the
+report must not imply otherwise.
+
+This catches a definition that is not what it says: a filter nobody mentioned, a
+renamed column, the wrong aggregate, or a served model that has moved away from
+the text the ledger recorded.
+
+### \`raw\` -- needs a lens onto the base tables
+
+**A within-model check is blind to fanout.** A join that duplicates rows inflates
+the stated measure and the control expression equally, and the comparison stays
+green on a genuinely broken join. So an expression reaching through a join records
+\`unchecked\` with a reason, **never \`agrees\`** -- reporting it as validated would be
+the tool committing the exact error it exists to find.
+
+The exception is aggregates that survive uniform duplication: \`AVG\`, \`STDDEV\`,
+\`MIN\`, \`MAX\`. Those stay checkable across a join.
+
+Detection is by join **name**, not by spotting a dot. \`sale_price.sum()\` is a
+method call on a column in the same source and is perfectly checkable;
+\`inventory_items.cost.sum()\` traverses a join and is not. A first implementation
+matched any \`word.word\` and held back \`total_sales\` -- the measure twelve of the
+ecommerce set's cases depend on -- for a join it does not cross.
+
+## A raw check is a control a person wrote
+
+An earlier version of this page, and the plan behind it, said raw checks would
+"catch a definition wrong about the business, like the cogs case." **That is
+retracted.** Re-deriving \`total_gross_margin\`'s own expression over the base
+tables -- reconstructing the model's join from its declared key -- computes the
+same quantity, cancelled lines included, and agrees. Mechanically that catches
+only fanout from a mis-declared join, which Malloy's symmetric aggregates already
+mostly prevent. It cannot see that the population is wrong.
+
+What caught the cogs case was a person who read the \`status\` doc ("Cancelled
+lines never ship and are not sales") and wrote \`where: status != 'Cancelled'\`
+into the golden's query. That is an **authored assertion**. So a raw check is
+exactly that, made durable: the conductor writes a control query onto the ledger
+record, and \`verify_definitions.py\` re-runs and compares it on every rebuild.
+
+\`\`\`json
+{
+  "entityId": "measure:order_items:total_gross_margin",
+  "check": {
+    "kind": "raw",
+    "against": "model",
+    "query": "run: order_items -> { aggregate: control is sale_price.sum() { where: status != 'Cancelled' } - inventory_items.cost.sum() { where: status != 'Cancelled' } }",
+    "note": "excludes Cancelled, per status's own doc and units_sold"
+  },
+  "cause": "CONVENTION"
+}
+\`\`\`
+
+A control may be authored on **any** measure record, not only a \`raw\` one, and
+it supersedes the within-model check. That check can only say a measure is what
+its expression says; a person can say what its population should be. \`total_sales
+is sale_price.sum()\` agrees with itself while including the cancelled lines its
+own docs exclude -- the within-model check reads \`agrees\`, and it is the measure
+twelve of the sample set's cases depend on. A one-line control on it is the
+whole difference.
+
+The control returns one row with a \`control\` column. \`against: model\` runs it on
+the same server as the measure and is the default -- it reuses the model's joins
+and columns and asserts only the population. \`against: truth\` runs it on a truth
+package of raw tables (\`--truth-publisher\`, \`--truth-package\`) for a fully
+independent derivation, join included. The measure's own value is always fetched
+through the model, separately, because the control is *supposed* to differ in
+population and may run elsewhere.
+
+Authoring a control is judgement, and the same rules as authoring a golden apply:
+where two populations are both defensible from the docs, do not pick one --
+record the ambiguity in \`note\` and leave the record \`unchecked\`. \`cause\` is the
+\`eval-diagnose\` code the author expects a disagreement to mean (the cogs case is
+\`CONVENTION\`); it is carried onto the finding, not inferred.
+
+This is **validate what someone asserted**, not **find business-wrong definitions
+unaided**. It turns "I believe \`total_gross_margin\` should exclude cancelled
+lines" into a checked, re-runnable assertion with a sha, so the belief is tested
+on every rebuild instead of once in someone's head. The difference is the whole
+point, and the run report must not blur it.
+
+### Build once, then maintain
+
+\`--out\` writes a fresh ledger. \`--ledger <file>\` reads an existing one, keeps
+every field a person wrote (\`check.query\`, \`check.against\`, \`check.note\`,
+\`cause\`), re-parses the model, re-runs every check, and writes it back. Nothing
+else is carried: a verdict is never inherited from a definition that may have
+moved, and a record whose definition the model no longer declares is dropped,
+control and all -- a control for a measure that does not exist is not a
+validated measure.
+
+## What it will not claim
+
+Every one of these records \`unchecked\`, and a reader must treat \`unchecked\` as
+"nobody has established this", never as "fine":
+
+| Situation | Why not checked |
+|---|---|
+| reaches through a join, and no control has been authored | fanout moves both sides of a within-model check equally, and nobody has written down what the population should be |
+| spans more than one line | only the first line was read, so the recorded expression is a fragment |
+| the query failed | a check that could not run is not a check that passed |
+| no rows came back | nothing to compare |
+| every sampled row null on both sides | proves nothing about the definition |
+| no enclosing source | nothing to query it against |
+
+## Exit codes
+
+Matching \`verify_goldens.py\`, deliberately:
+
+| | |
+|---|---|
+| **0** | every check that could run ran, and agreed |
+| **1** | a definition disagrees with its own stated expression |
+| **3** | could not run: no model, no server, or a ledger built and never checked |
+
+**3 is load-bearing.** Building the ledger without \`--publisher\` returns 3, not 0:
+it validated nothing, and a caller must never read that as a pass.
+
+## Running it
+
+\`\`\`bash
+# build the ledger only (exits 3: nothing was checked)
+python3 verify_definitions.py --model ecommerce.malloy --out evals/definitions/ecommerce.jsonl
+
+# build and check
+python3 verify_definitions.py --model ecommerce.malloy \\
+  --publisher http://localhost:4811 --environment samples \\
+  --package ecommerce --model-path ecommerce.malloy \\
+  --out evals/definitions/ecommerce.jsonl
+\`\`\`
+
+Measured on the ecommerce sample: **28 definitions, 15 agree, 13 unchecked**, of
+which 4 need a raw check. The remainder are multi-line definitions and fields
+declared inside a nested block, which the line scanner attributes to the
+enclosing source and cannot address.
+
+## What a run reports
+
+\`run_baseline.py --definitions <ledger>\` adds an EVIDENCE block saying what the
+pass rate rests on, per case:
+
+\`\`\`
+EVIDENCE
+  basis         5 independent, 8 definitions, 36 unchecked
+                ! those cases rest on a definition nobody has validated. Not a
+                  failure, and not a pass either.
+\`\`\`
+
+| basis | meaning |
+|---|---|
+| \`independent\` | the key was established outside the model: two differently shaped derivations agree, or it holds no value at all (\`criteria\`, \`unanswerable\`) |
+| \`definitions\` | the key went through the model, and every definition it tests is validated |
+| \`unchecked\` | at least one tested definition is unvalidated, stale, or absent from the ledger |
+| \`disagrees\` | a definition the case depends on contradicts its own expression, so the model is wrong before the answer is |
+
+Independence is read from \`golden.verification\` (or \`gold/<qid>.json\`) through
+\`verify_goldens.has_second_derivation\` -- the structured record of "two
+differently shaped derivations agree". **Not from \`verifiedBy\`,** which is free
+text: a first implementation matched it by prefix and classified 34 goldens of
+the ecommerce set as \`unchecked\` when every one of them reads "authored and
+re-derived against ecommerce-truth". A well-founded set reading as unvalidated
+is the same over-claim as an unfounded one reading as validated, pointed the
+other way.
+
+Which definitions a case tests comes from \`expectedEntities\`, which already
+names entities in the \`kind:source:name\` form the ledger keys on. No new
+hand-maintained field: a wrong entity id is the failure mode that cost a real
+set two days, so this reuses a link the set already maintains and that
+\`verify_goldens\` check 5 audits against the model.
+
+**Without \`--definitions\` there is no EVIDENCE block at all.** Absent is a
+different fact from checked, and a reassuring empty block would be the
+over-claim this whole direction exists to remove.
+
+Staleness is a hash comparison against the run's own \`model.malloy\` snapshot --
+the bytes that actually answered -- so it costs nothing and runs every arm.
+
+## The composition rule at the gate
+
+\`verify_goldens.py\` used to exit 3 -- "could not run" -- on any set that named no
+truth package, because no golden value could be re-derived. That was right when
+re-derivation was the only evidence there was. With a ledger it is one of two:
+
+> A golden is trustworthy if it was derived independently, **or** if every
+> definition it tests has itself been validated.
+
+\`verify_goldens.py --definitions <ledger>\` applies the second half. For each
+value-bearing case it takes the entities the case names in \`expectedEntities\` --
+the same ids the ledger is keyed on -- and asks whether every one is \`agrees\` in
+the ledger and not stale against \`--model\`. If every case passes, the absent truth
+server is not a check that failed to happen, and the audit exits 0 with a line
+saying what validated it; \`--promote\` may then promote a \`provisional\` golden that
+also carries a second derivation, and \`verifiedBy\` names the ledger. If any case
+does not pass, the exit stays 3 and the message names the cases and the
+definitions that let them down. Nothing is inferred about a case that names no
+entities: it is unvalidated, and it says so.
+
+\`improve.py\` passes \`--definitions\` through when it has one, so the acceptance
+check can pass on a set that never had a truth package.
+
+## Where it goes
+
+\`evals/definitions/<package>.jsonl\` -- keyed by **package, not by set**, because
+definitions belong to the model and outlive any one set, and several sets against
+the same package share them. Never under the served package tree; the golden side
+door forbids eval artifacts there.
+
+## What this is not
+
+It is not a unit test. Publisher's roadmap plans fixture-based unit tests (a CSV
+per source plus an expected result) alongside data tests in \`malloy-pub lint\`.
+A fixture unit test proves an expression computes what you wrote -- so it would
+**not** catch a measure whose expression is correct and whose *population* is
+wrong, which is the defect class this exists for. The ledger is a data test, and
+it should hand off to \`lint\` when that governance work lands rather than becoming
+a permanent parallel mechanism.`
     },
     {
       name: "eval-answer/ledger-schema",
       description: "The eval ledger: files and events. Reference detail for the eval-answer skill.",
-      body: "# The eval ledger: files and events\n\nThe ledger is plain files in the model package's git repository. There is no\neval API and no eval database. The conductor (`skill:eval-loop`) reads and\nwrites these files directly; the stages share them as their contract.\n`eval-answer` writes `attempt`, `tool_call`, and `score`. An engine-side\n`eval-retrieval` skill, which does not ship here, writes `retrieval_score` and\n`probe`; both stay in this contract so one validator covers every run\ndirectory.\n`eval-diagnose` writes `issue` and `issue_status`. `eval-improve` writes\n`candidate`. `eval-loop` writes `acceptance_check`, further `issue_status`, and\n`checkpoint`.\n\n**The contract is code: `eval-answer/scripts/ledger.py`.** Every script that\nwrites `run.json` or `events.jsonl` builds its lines through that module, so a\nfield rename that reaches only one writer fails at write time. This document\nis the human-readable rendering; where the two disagree, the module is right\nand this file is the bug. Check any run directory with:\n\n```\npython3 skills/eval-answer/scripts/ledger.py validate <runDir>\n```\n\nErrors are broken identity (missing `run.json`, a missing required field, two\nscores for one attempt); warnings are missing comparison pins (a run without\n`skillsVersion` still measures, but cannot anchor an A/B on that pin) and\ngrandfathered unknown fields on old runs.\n\n## Layout\n\n```\nevals/<set>/\n  set.json                  # set metadata (below)\n  cases.jsonl               # one case per line\n  judge-regressions.jsonl   # judge verdicts a human overruled\n  runs/<runId>/\n    run.json                # run config, the attribution pins\n    events.jsonl            # append-only event lines\n    artifacts/              # prediction CSVs, judge outputs, transcripts\n```\n\nThe set directory lives in the SAME git repository as the model it evaluates,\nso a checkpoint (a git commit) pins the model and the ledger together. Never\nplace `evals/` inside the directory tree the answerer's package serves: gold\nin the served tree is a contamination path.\n\nRules that make the ledger trustworthy:\n\n- `events.jsonl` is append-only. Never edit a line. To void one, append a new\n  event whose payload marks the old one `voided`. One sanctioned exception: a\n  re-runnable stage (diagnose, improve) replaces ITS OWN prior events, keyed by\n  issue id, through `ledger.replace_events()` -- so a crash-and-rerun does not\n  duplicate issues. A stage never touches another stage's lines.\n- A stage never rewrites another stage's fields.\n- One artifact directory per attempt (`qid` plus `sample`). Never overwrite a\n  previous attempt's prediction CSV.\n- End-of-run numbers come from counting event lines (`jq` over\n  `events.jsonl`), never from arithmetic recalled in prose.\n\n## `set.json`\n\n| Field | Notes |\n|---|---|\n| `name` | Set name; also the directory name. |\n| `description` | |\n| `datasetVersion` | Integer. Bump on any golden repair or case change. Runs record the version they scored against. |\n| `targetModelPath` | Model path within the package. |\n| `truthPackage` | Name of the package holding the semantics-free sources every golden is re-derived from -- a package NAME, not an object. Without it `verify_goldens.py` re-derives nothing: it reports \"nothing to re-derive against\" and **exits 3**, which a caller must read as unverified rather than as a pass. The checks that need no truth server -- rubric figures, verification axis, stale rubric claims, set names -- still run and still report. `init_truth_package.py` scaffolds the package. |\n| `truthModel` | Model file inside that package. Default `truth.malloy`. |\n| `targetPackage` | Name of the package UNDER TEST. `verify_goldens.py` refuses a \"truth\" server that also serves it, because an answerer on that server can retrieve the raw truth sources beside the model and quietly change what is under test. `run_baseline.py` and `improve.py` pass the package they were given, so this field is the fallback for a direct call; without either the guard cannot fire. |\n| `truthTableRewrite` | Boolean, default false. Rewrites `duckdb.table('data/x.parquet')` refs to bare `x` in canonical queries, for a truth package whose tables are registered rather than read from files. |\n\n## `cases.jsonl`\n\nOne JSON object per line:\n\n| Field | Notes |\n|---|---|\n| `qid` | Stable case id. |\n| `question` | Exact text the answerer will see. |\n| `split` | `dev` or `holdout`. Frozen at import. Diagnose and improve read `dev` only; the acceptance check runs both. |\n| `tags` | list |\n| `state` | `candidate` / `selected` / `excluded`. |\n| `source` | Where the case came from. |\n| `golden` | `status` (`verified` / `provisional` / `invalid` / `ambiguous` / `verified_wrong`, the last being a key a person has established is wrong, which `run_baseline.py` drops from every aggregate), `kind`, `value` or `path` (artifact under the set directory), `canonicalQuery` (runs against the **truth** package, never the model under test), `verifiedBy`, and `verification` -- `{primaryAxis, variesAxis, note}` naming what the second derivation varied. **A golden is written only after two differently shaped derivations agree** (the truth query and a second one that varies a different axis -- group by something else, sum a different way, take a different route through the raw tables); `gold/<qid>.json` holds both (`truthRows`, `verifyRows`, `agreement`). Two derivations that vary the same axis share every other blind spot: that is how a golden that summed three overlapping time slices, 3x too high, passed its check. `verify_goldens.py` re-derives the value, reads the accepting clause of the rubric against the rows, and flags a second derivation that names no axis. **The question asks for data, never for an interpretation of it**, and either fixes the grain or the rubric accepts a correct figure at any stated grain (`skill:eval-judge` rules 10 and 11). |\n| `golden.kind` | What the key IS. A value (`scalar`, rows), `unanswerable`, or `criteria`: prose with no value, where the author's criteria are the whole key. `criteria` is the one kind that is legitimately `verified` on arrival, because there is no derivation to perform and nothing that can be numerically wrong; an imported key holding a VALUE is `provisional` until re-derived (`skill:eval-import`). **`verify_goldens.py --promote` is the only thing that writes `golden.status`**: it marks a `provisional` golden `verified` when its value re-derived cleanly from the truth package AND a second derivation exists. `--refresh` rewrites a drifted VALUE and never changes a status; `invalid` and `ambiguous` are settled by a person through the golden side door. |\n| `golden.rubric` | **On the golden, not on the case.** The prose the judge is shown as `CASE RUBRIC`, naming what counts as correct and what does not. `run_baseline.py` reads `golden.rubric`; a rubric written at the case's top level is silently not passed, the judge is told `CASE RUBRIC: none`, and it then scores from the golden value and the answer alone -- which reads as a judge that ignores its instructions. |\n| `golden.mustState` | **On the golden, not on the case.** What the answer has to say out loud. Read from `golden.mustState` by the run-package builder. |\n| `goldenRevision` | Integer. Bump on every golden change; `score` events stamp the revision they compared against. |\n| `expectedEntities` | `required`: entity ids (`kind:source:name`) the answer cannot be produced without. `requiredAnyOf`: a list of **groups**, each a list of ids of which any one suffices -- for a case the model can answer through more than one route. Naming only one route scores the other as a retrieval miss and steers diagnosis to \"retrieval ranking\" for a failure that was never retrieval's. `acceptable`: ids that are not noise if returned. An entity counts as delivered when returned under its exact id, under the same type and name on a sibling source, or by name in a returned source's documentation; `score_retrieval.py` records the route per entity as `delivery`. |\n\n## Term files (`eval-retrieval` only)\n\nA customer set has no term file. The engine-side `eval-retrieval` skill keeps\n`intents.jsonl` fixtures -- search term, entity type, a description of what the\nsearcher meant, a validity flag -- beside its own scripts, for evaluating the\nengine against fixed inputs. Their shape and the `retrieval_score` events they\nproduce are documented there and validated here.\n\n## `run.json`\n\nThe attribution pins. Two runs are comparable only when these match where it\nmatters. Two runs with different `target` values are comparable on answer\nverdicts only if the same model version reached both, which is rarely worth\nassuming:\n\nRequired fields are the run's **identity** -- every run ever written carries\nthem. The rest are optional in the schema, but the **comparison pins** are\nwarned about when absent, because a run missing one cannot take part in an A/B\non that axis.\n\n| Field | Notes |\n|---|---|\n| `runId` *(required)* | Directory name. |\n| `target` *(required)* | Which server answered, and how it reached the data: `local`, `local-proxied`, or `platform`. Decides what an improve step is even allowed to do (see `skill:eval-loop`). |\n| `answererModel` *(required)* | |\n| `phase` / `started` *(required)* | |\n| `judgeModel` / `judgeVersion` / `rubricSha` *(pins)* | The judge's model, and the version + content sha of `skill:eval-judge`. |\n| `datasetVersion` *(pin)* | From `set.json` at run time. |\n| `modelSha` *(pin)* | Content sha of the `model.malloy` snapshot in the run directory -- the bytes the answerer actually queried. A git sha is not enough: a snapshot host serves a copy, often of a dirty tree no commit names. |\n| `skillsVersion` *(pin)* | HEAD of the checkout the agents' skills were loaded from, dirty-marked (`ledger.skills_git_sha(root)`). The skills are the doctrine the agents load; a run that cannot name their version cannot anchor a skills A/B. |\n| `skillsRoot` / `harnessVersion` | Which checkout supplied the doctrine (`--skills-root`, e.g. a Publisher checkout for the open-source skills; default this one), and this checkout's own HEAD. The eval-* skills always come from the harness checkout, whatever `skillsRoot` says. Two runs whose `skillsRoot` differ are a skills A/B only if their manifests name the same skills. |\n| `diagnoserModel` / `improverModel` | Written by `diagnose.py` / `improve.py` when those stages run, so the run names every LLM that touched it. Absent on a run that was only answered and judged. |\n| `modelGitSha` | Commit of the model repo, from `--model-repo`; `-dirty` suffix when that path had uncommitted changes -- fine for a band measurement, not for an A/B pin. **Null unless `--model-repo` names the repo.** It cannot be inferred: Publisher serves a COPY under `publisher_data/`, so the tree around the served file is the server's storage, not the model's history, and the field named that until 2026-09-03. `modelSha` above is the pin that always works; this is the pointer to where those bytes are versioned, and absent beats wrong. |\n| `modelRepo` | The path `--model-repo` named, so a reader can tell an absent pin from an unrecorded one. |\n| `environment` / `package` / `modelPath` | What was served, and from where. On a platform target `environment` is the organization and `package` is the workspace the MCP URL is scoped to. |\n| `scope` | Platform target only: the `environment/package[@version]` the answerer was told to pass as an explicit `scopes` entry on every `get_context` / `execute_query` call. A workspace can serve many packages (a personal workspace serves every package the user can read), so without this the run also measures whether retrieval picks the right package -- a different measurement. Absent means unscoped. |\n| `targetVersion` *(pin)* | Platform target only: the published version the answers came from. Both hosted tools default an omitted version to the PINNED one, meaning whatever the workspace serves at the moment of the call, so this only describes the run when `--scope` carries it to every call. `--target-version` supplies it when the scope omits it; a scope naming a different version is refused rather than treated as an override, because a run cannot say which build answered if two pins disagree. |\n| `mcpUrl` / `publisher` | The endpoints the answerer and the re-execution used. |\n| `predictionsReExecuted` | Whether re-execution was POSSIBLE: the server was serving the bytes this run is pinned to. It is one bool for the whole run and says nothing about any given query, so it reads stronger than it is; `reExecution` below is what actually happened. |\n| `reExecution` | `{attempted, ok, failed, noQuery, notReExecuted, notJudged, missing}` over the run's predictions. `failed` means the query ran and the server returned an error, which is a different thing from an answer with no query at all. `notJudged` is a case the judge was never handed: a golden refused before judging (`provisional`, `invalid`, `ambiguous`, `verified_wrong`, missing), an attempt that submitted nothing, or every case under `--no-judge`. Those used to land in `missing` and read as corrupt artifacts on a freshly imported set. `ok`, `failed`, `noQuery`, `notReExecuted`, `notJudged` and `missing` partition the cases, so they sum to the case count; `attempted` is `ok + failed` and sits outside that sum. |\n| `retrievalMode` | Which retriever answered: `semantic`, `lexical`, `mixed` (it changed mid-run, so nothing can be averaged across it) or `unreported` (nothing ranked, which on a configured server means no embedding provider). Only a RANKING call names a retriever, so a call that enumerates a type or looks a name up is `unreported` on a fully semantic server and must not make the run read as `mixed`. Local retrieval degrades to lexical SILENTLY without an embedding key, and a pair compared across that reads as a model change, so `flip_table.py` refuses the pair unless both sides match. |\n| `retrievalCalls` | `{semantic, lexical, unreported}` call counts behind `retrievalMode`. `unreported` counts the calls that did not rank, which is a different thing from calls that fell back. |\n| `retrievalGate` | What the pre-arm retrieval gate found, or why it did not run. A run is held until `get_context` answers semantic twice in a row, because the first calls after a restart are lexical by design and an arm started immediately measures two retrievers. `not run (no answering phase)` under `--rebuild`/`--rejudge`, and `skipped by --no-retrieval-gate` when the operator opted out -- which is a different fact from a gate that passed, and the field is what tells them apart. |\n| `label` | `<set>-<phase>-<nn>`, assigned by `run_baseline.py` from the set name, the run's phase and the next free number beside it (`ecommerce-baseline-01`, `ecommerce-baseline-02`, `ecommerce-blind_gate-01`). The A/A pair is two runs of the same phase; the post-edit arms are two runs of `blind_gate`. Hand-typed names do not survive one afternoon of runs -- `base`, `rejudged2`, `r3`, `post1` sort wrongly, group not at all, and cannot be matched to an arm. `--label` overrides for a run that genuinely needs a human name. |\n| `effort` | |\n| `answererCostUsd` / `judgeCostUsd` | What the arm cost, split by role. The judge's half was discarded until 2026-09-02, so every \"cost per arm\" quoted before then was the answerer alone. |\n| `goldenCheck` | What `verify_goldens.py` said before the run started: `N ok, M drifted, K other finding(s)`, or why it did not run (no truth server on a platform target; `--skip-golden-check`; rebuild). A run that started on a drifted set says so here rather than pretending its verdicts mean something. `run_baseline` calls `verify()` without a model, so the set-name audit (check 5) does not run in-arm; the trailing `N stale entity name(s)` is the in-arm lint's own count, and `entity-name lint not run` says there was no model text to lint against. Without it the field read `0 other finding(s)` for a check that never happened. |\n| `staleEntityNames` | The `expectedEntities` names that appear nowhere in the served model, from the in-arm lint. Each one depresses the run's own recall on every case listing it, so a stale set reads as a retrieval failure until someone checks by hand. `null` when there was no model text to lint against (a platform target), which is a different fact from `[]`. |\n| `status` | `complete`, or `aborted` when four consecutive attempts errored or found the server dead and the harness stopped rather than spend the rest of the budget on attempts nobody will trust. |\n| `packageSha` / `servedRevision` *(pins)* | Taken from the server, not recomputed: `sourceContentSha` is a content hash over EVERY model path in the package, so an edit to an imported file moves it where a sha of the one `--model-path` does not; `servedRevision` is minted per load, so it identifies a load rather than content and is a poor pin alone. Measured: `publisher.json`'s `version` moves neither, and nothing in Publisher reads it -- it is not in the Package API schema and never returned, so it pins nothing. |\n| `datasetSha` *(pin)* | Content hash of `set.json` + `cases.jsonl`. Deliberately SEPARATE from the model's sha: a golden repair is not a model change, and one pin covering both would make every answer-key fix read as an edit to the model, which is the distinction an A/B rests on. Automatic, so nobody has to remember it; `datasetVersion` stays beside it as the human-readable sequence. **Local targets only.** A hosted target that publishes IMMUTABLE versions needs none of this: the set rides inside the version, and immutability -- not hashing -- is what makes a pin trustworthy. There, `targetVersion` alone identifies model and set together. |\n| `doubtedGoldens` | The cases whose golden is not believed: `qid`, `gold_status` (`suspect` or `verified_wrong`), `gold_note`, and `declaredBy` (`judge` or `set`). A status the SET declared is a key somebody already settled, not an opinion the judge formed this run, and attributing both to the judge claimed a judgement that never happened. **Read this before diagnose.** These are dataset issues, not model failures, and they go through the golden side door in `skill:eval-loop`. Empty list when no key is in doubt from either source. Written from the same scan that prints the end-of-run warning, because a warning that lives only in console text is one scrollback away from sending a modelling agent at a model that is already right. |\n| `mode` / `setName` / `serverVersion` / `traceMode` / `callBudget` | Defined and accepted, **not yet written by any harness** -- kept in the schema for the platform target and the conductor, which need them. (`targetVersion` and `status` were in this list and are both written now.) |\n\n## Events\n\nEvery line in `events.jsonl` is **flat**: `{ \"kind\": ..., <fields> }`, one\nJSON object per line, with an optional `at` ISO timestamp. Case-scoped kinds\n(`attempt`, `tool_call`, `score`) carry `qid`, `sample`, `phase` on the line;\nrun-level kinds do not. (An earlier draft of this document nested fields under\na `payload` with a `caseId`; no writer ever did that, and 23 run directories\nexist in the flat shape, so the flat shape is the contract.) `kind` is one of:\n`attempt`, `tool_call`, `score`, `retrieval_score`, `issue`, `issue_status`,\n`candidate`, `acceptance_check`, `checkpoint`.\n\n### `attempt`\n\n| Field | Type | Notes |\n|---|---|---|\n| `qid` | string | |\n| `sample` | int or null | Which repeat. Required even when null. |\n| `phase` | string | `baseline` / `loop` / `blind_gate` / `canary` / `final`. `phase` lives here, on the attempt, not in run config. |\n| `question_sha` | string | Hash of the exact text the answerer saw, so two runs can be compared on whether they asked the same thing. A case may carry `questionSha`, the seal stamped at conversion, and that wins when present; the ecommerce set writes a 16-char prefix of it, and a set without one left this null. Neither form proves the question was not edited, since both follow the case as it now reads: that comparison is check 6 of `verify_goldens.py`. |\n| `submitted` | bool | False when there was no final query. Not a wrong answer. |\n| `final_query` | string or null | Required to replay. A named view is recorded as the Malloy it stands for, `run: <source> -> <view>`, so every consumer sees one shape and the query re-executes. |\n| `final_query_source` | string or null | How `final_query` was chosen: `declared` (the answer printed it), `last_ok` (the last call the server answered) or `last`. `last` is a warning: a trailing sanity probe may be standing in for the answer's own query. |\n| `servedRevision` | string or null | From the package actually queried. |\n| `n_get_context` / `n_execute` / `n_execute_errors` | int | |\n| `host_tool_uses` | int | EVERY tool use the host logged, MCP calls included. It counted only the non-MCP ones until 2026-09-03, which made the under-report check below true of almost every clean attempt. |\n| `mcp_tool_uses` | int or null | The MCP subset: `get_context` plus `execute_query`. Null on a run written before the split. |\n| `reported_calls` | int | MCP calls the answerer claimed. `reported_calls > host_tool_uses` is the under-report floor from `skill:eval-answer`: the answerer cannot have made more calls than the host logged. `validate_run` warns on it, and only for runs that carry `mcp_tool_uses`, because the field meant something narrower before. |\n| `contaminated` | bool or `\"unknown\"` | `\"unknown\"` when no host log exists. Read it with `ledger.is_contaminated`, never for truthiness: runs written before 2026-09-03 carry the strings `\"true\"`/`\"false\"`, and `bool(\"false\")` is True. `ledger.event` now rejects those strings on write; `validate_run` grandfathers them on read as a warning. |\n| `contamination_reasons` | list | Empty when clean. |\n| `input_tokens` / `output_tokens` / `cache_read_tokens` | int or null | Answerer token usage. Null when the host does not report it. |\n| `cost_usd` | float or null | Answerer cost for this attempt. |\n| `num_turns` / `wall_seconds` | int, float or null | |\n| `answer_text` | string or null | The answer the judge scored. Kept so a verdict can be re-read without the transcript. |\n| `transcriptPath` | string | The answerer's transcript under `artifacts/`. |\n\nToken counts sit here rather than being derived later because the claim a\nsemantic model makes is about cost as well as correctness -- that a documented\nmodel reaches a good answer in fewer turns and tokens than working from raw\nschema. A ledger that counts calls but not tokens can state half of that.\n\n### `tool_call`\n\nOne event per MCP `get_context` or `execute_query` the attempt made.\n\n| Field | Type | Notes |\n|---|---|---|\n| `tool` | string | `get_context` or `execute_query`. |\n| `traceId` | string or null | `get_context` only; look up in your host's trace store. |\n| `targets` | string, list or null | **What the answerer asked for**: the search terms it sent to `get_context`. Null for `execute_query`. |\n| `rankedSummary` | object | Copied at capture from the trace so evidence survives trace eviction: `entityIds`, `ranks`, `resultCount`, and per-target `targets` with within-target ranks. |\n| `retrieval_mode` | string or null | `get_context` only: the `retrieval` field of the response that answered, `semantic` or `lexical`. Null when the server named none, which means no embedding provider. Recorded per call because the semantic path can fall over partway through a run. |\n| `query` | string or null | `execute_query` only: the Malloy the call sent, so the final query can be chosen by which call the server actually answered. |\n| `modelPath` | string or null | `execute_query` only: the model file the call named. A source does not resolve outside the file that declares it, so re-execution needs this. |\n| `error` | string or null | |\n\nNever persist `execute_query` result rows, givens, or credentials.\n\n`targets` records the request; `rankedSummary` records the response. Without\nboth, a low per-attempt recall has two readings that cannot be told apart: the\nanswerer searched for the wrong thing, or it searched well and retrieval ranked\nthe right entity too low. Those have opposite owners -- `agent-skill` and\n`retrieval` -- so a ledger holding only the response cannot attribute the\nfailure, and any recall computed from it is a blend of answerer behaviour and\nretrieval quality.\n\nThat blend is also why per-attempt recall is **not** comparable across arms: a\nstronger answerer searches better and scores higher retrieval recall without\nretrieval having changed. Use it for attribution within an arm. Comparing\nretrieval itself between engine versions is `eval-retrieval`'s job, with fixed\nterms, and is not something a customer run reports.\n\n### `score`\n\nThe answer judge's verdict for one attempt (protocol in\n`skill:eval-judge`). Every attempt in a scored run gets exactly one.\n\n| Field | Type | Notes |\n|---|---|---|\n| `verdict` | string or null | `match` / `near_match` / `no_match` / `needs_human`; null when the attempt is not scorable. Only `match` and `no_match` are decisions; see below. |\n| `reason` | string | Why, from the judge; for a null verdict, why not scorable: `not_submitted`, `contaminated`, or `golden_<status>` -- `golden_missing`, `golden_provisional`, `golden_invalid`, `golden_ambiguous`, `golden_verified_wrong`. The `golden_*` reasons are decided before the judge is called, so an unestablished key costs nothing to refuse. |\n| `confidence` | int or null | 1 to 10. Confidence of 5 or lower forces `needs_human`. |\n| `column_pairing` | object or null | The judge's named gold-to-prediction column correspondence. |\n| `judge_version` / `rubric_sha` | string | Pins which rubric produced this verdict. |\n| `golden_revision` | int | From the case at score time. |\n| `contaminated` | bool or `\"unknown\"` | Copied from the attempt; true or unknown means `verdict: null`. |\n| `must_not_use_hits` | list or null | Entries of `golden.mustNotUse` a script found in the final query. A hit forces `verdict: no_match`. Decided by `check_must_not_use.py`, never by the judge: it is a question about query text. Only a BARE name vetoes; an entry with a connective (`X as ...`, `X through ...`) objects to a use of X rather than to X, and goes to the judge with the prose entries and a path's bare leaf. |\n| `judge_verdict` | string or null | What the judge said when a `must_not_use_hits` veto overrode it. Null otherwise, so the judge's own agreement rate stays measurable across vetoes. |\n| `artifactPath` | string | The full judge output under `artifacts/`. |\n| `gold_status` | string | `verified` / `verified_benign` / `suspect` / `verified_wrong`, and never anything else: the case file's `golden.status` is a DIFFERENT vocabulary that shares only `verified` and `verified_wrong`, and only those two fall back into this field. A `provisional` / `invalid` / `ambiguous` key is reported through the verdict's `reason` (`golden_<status>`) instead, so nothing downstream has to match on two vocabularies in one field. **From the judge**, which scored against the golden as written and reports separately whether it believes it; falls back to the case's standing `golden.status` when the judge does not say, written onto the verdict itself so the run summary and the aggregates see it too. `verified_wrong` excludes the case from run aggregates -- from the judge OR from the case file; assigning the fallback into a copy of the verdict meant a key the set had marked wrong stayed in the aggregates unless the judge independently agreed. `suspect` and `verified_wrong` route to the golden side door as `dataset` issues, never to improve. |\n| `gold_note` | string or null | The judge's evidence for a non-`verified` status: the two values, or the model line against the rubric sentence. Null when `verified`. |\n\nA `golden_*` reason means the ANSWER KEY is not something a verdict can be\nissued against. A golden the set stamps `provisional`, `invalid` or `ambiguous`\nis refused, as is a case with no golden at all and one the set marks\n`verified_wrong`; a golden with NO `status` is unguarded rather than\nunscorable, because sets that predate the field would otherwise score zero of\nzero. A golden holding no value is not refused either: that is how a\n`kind: unanswerable` case and a `kind: criteria` golden are written. This\nmatters most for an imported set, because `skill:eval-import` stamps\n`provisional` on every golden it writes and enforces it -- so a set is\nunscorable by construction until its keys are re-derived, and `run_baseline`\nrefuses to spend at all when every case in a set is in that state.\n\n`not_submitted` means the attempt produced NOTHING to judge: no answer text and\nno query. An attempt that wrote prose without querying **is judged**, and\nagainst a golden that holds a value an answer containing none of it is\n`no_match` (`skill:eval-judge`, `reference/refusal.md` rule 10). Excusing those\nas unscorable dropped a confident refusal on an answerable case out of the pass\nrate, which is the one thing the answerable-sounds-unanswerable cases exist to\nmeasure. A refusal is exempt only where `golden.kind` is `unanswerable`, and\nthere naming the gap is the pass and a confident number is the fail.\n\nAggregates count decided verdicts only. `match` and `no_match` are the pass and\nthe fail; **`near_match`, `needs_human` and null are none of the above** and stay\nout of acceptance arithmetic. `near_match` is excluded because it means \"defensibly\ndifferent\", and an arguable verdict that moves a pass rate is a measurement\nartefact rather than a result (`skill:eval-judge` rule 7). Report its count:\nit rising is how a set tells you its rubrics are going vague.\n\nThis applies to `score`. On `retrieval_score` below, `near_match` **is** counted\ntowards recall and precision, because an overlapping entity is a genuine\nretrieval success. The two are different questions that share a word.\n\n### `retrieval_score` (written by `eval-retrieval`, not by a customer run)\n\nOne per term judged in a fixed-term replay (run-level; no `caseId`).\n\n| Field | Type | Notes |\n|---|---|---|\n| `intentId` / `term` / `entityType` | string | From the term file. |\n| `in_scope` | bool | Does THIS model version represent the concept? False is a coverage gap, not a retrieval failure. |\n| `judgments` | list | Per returned entity: `entityId`, `rank`, `level` (`match` / `near_match` / `no_match`), `confidence`, `why`. Empty when nothing returned. |\n| `judge_version` / `rubric_sha` | string | |\n| `traceId` | string or null | The `get_context` call judged. |\n\nRun-level metrics fall out by counting:\n\n- `coverage` = in-scope terms / valid terms -- a property of the model and its\n  data, reported on its own, never as a retrieval number.\n- `recall` (on in-scope terms) = fraction whose judgments contain a `match`.\n- `precision@N` = `match` judgments / judged, with N stated.\n\n### `issue`\n\n| Field | Type | Notes |\n|---|---|---|\n| `issue_id` | string | Stable across status events. |\n| `qids` | list | Affected cases. |\n| `primary_code` / `contributing_codes` | string / list | From `skill:eval-diagnose`, verbatim. |\n| `component` | string | `dataset` / `agent-call` / `get_context/model` / `get_context/retrieval` / `construction` / `model-definition`. |\n| `owner` | string | `model` / `retrieval` / `agent-skill` / `dataset`. Environment failures stop the run; they are never diagnosed, so there is no environment owner. |\n| `severity` / `confidence` | string | |\n| `sufficiency` | string | `sufficient` / `insufficient` / `unknown`. |\n| `traceIds` | list | |\n| `diagnosis` | string | Written before any edit exists. |\n\n### `issue_status`\n\n`issue_id` plus `status`: `open` / `batched` / `fixed` / `rejected` /\n`deferred`. Readers take the latest event for that `issue_id`. Do not invent\na status column.\n\n### `candidate`\n\nWritten by `eval-improve`, for every proposed edit, accepted or not. A\nrejected direction keeps its record.\n\n| Field | Notes |\n|---|---|\n| `issue_ids` | |\n| `files` | Paths the edit touched. |\n| `diffSummary` | One line per file. |\n| `probes` | Query and result for each factual claim. |\n| `meaningChanged` | Entities whose *meaning* the edit changed; `[]` for a docs-only edit. |\n| `goldenSuspect` | Each `{qid, entity, stored, rederived}`: a golden this edit may have invalidated. Reported by the improver, never repaired by it. **Non-empty halts the acceptance check** until adjudicated through the golden side door. |\n| `goldenAudit` | The set's `verify_goldens.py` run against the edited model: `{ran, clean, model, tail}`. Catches drift and rubric-vs-model contradictions only; a golden whose value silently moved is invisible to it, which is why `goldenSuspect` exists alongside. |\n\n### `acceptance_check`\n\nCalled `gate` before 2026-09-03. `ledger.py` reads the old kind as this one, so\nrun directories written earlier still validate and nothing rewrites them.\n\nWritten by `eval-loop`, one per acceptance check decision, BEFORE any checkpoint commit.\n\n| Field | Notes |\n|---|---|\n| `issue_ids` | |\n| `decision` | `accepted` / `rejected`. |\n| `class` | `docs` / `definition` / `retrieval` / `justified`. |\n| `baselineRunId` / `finalRunIds` | Plural: acceptance needs two independent runs. |\n| `regressions` | Case ids whose verdict got worse vs baseline. Must be empty to accept. |\n| `holdoutDelta` | Confident-verdict delta on the holdout slice. |\n| `reason` | Including independent deterministic justification when that is the basis. |\n\n### `checkpoint`\n\nWritten by `eval-loop` after an accepted acceptance check, or when a restore runs. The\nmodel bytes live in git, not in this payload.\n\n| Field | Notes |\n|---|---|\n| `action` | `created` / `restored`. |\n| `label` | |\n| `modelGitSha` | The commit this checkpoint names (create), or the commit restored to. |\n| `issueIds` | Issues the accepted edit closed. Empty on restore. |\n\n## `judge-regressions.jsonl`\n\nAppend a line whenever a human overrules a judge verdict: the case or intent,\nthe judge's verdict, the human's, and why. Re-run this file against the judge\nwhenever `skill:eval-judge` or the judge model changes; a rubric change that\nflips old human-settled verdicts is a judge regression, not new truth."
+      body: "# The eval ledger: files and events\n\nThe ledger is plain files in the model package's git repository. There is no\neval API and no eval database. The conductor (`skill:eval-loop`) reads and\nwrites these files directly; the stages share them as their contract.\n`eval-answer` writes `attempt`, `tool_call`, and `score`. An engine-side\n`eval-retrieval` skill, which does not ship here, writes `retrieval_score` and\n`probe`; both stay in this contract so one validator covers every run\ndirectory.\n`eval-diagnose` writes `issue` and `issue_status`. `eval-improve` writes\n`candidate`. `eval-loop` writes `acceptance_check`, further `issue_status`, and\n`checkpoint`.\n\n**The contract is code: `eval-answer/scripts/ledger.py`.** Every script that\nwrites `run.json` or `events.jsonl` builds its lines through that module, so a\nfield rename that reaches only one writer fails at write time. This document\nis the human-readable rendering; where the two disagree, the module is right\nand this file is the bug. Check any run directory with:\n\n```\npython3 skills/eval-answer/scripts/ledger.py validate <runDir>\n```\n\nErrors are broken identity (missing `run.json`, a missing required field, two\nscores for one attempt); warnings are missing comparison pins (a run without\n`skillsVersion` still measures, but cannot anchor an A/B on that pin) and\ngrandfathered unknown fields on old runs.\n\n## Layout\n\n```\nevals/<set>/\n  set.json                  # set metadata (below)\n  cases.jsonl               # one case per line\n  judge-regressions.jsonl   # judge verdicts a human overruled\n  runs/<runId>/\n    run.json                # run config, the attribution pins\n    events.jsonl            # append-only event lines\n    artifacts/              # prediction CSVs, judge outputs, transcripts\n```\n\nThe set directory lives in the SAME git repository as the model it evaluates,\nso a checkpoint (a git commit) pins the model and the ledger together. Never\nplace `evals/` inside the directory tree the answerer's package serves: gold\nin the served tree is a contamination path.\n\nRules that make the ledger trustworthy:\n\n- `events.jsonl` is append-only. Never edit a line. To void one, append a new\n  event whose payload marks the old one `voided`. One sanctioned exception: a\n  re-runnable stage (diagnose, improve) replaces ITS OWN prior events, keyed by\n  issue id, through `ledger.replace_events()` -- so a crash-and-rerun does not\n  duplicate issues. A stage never touches another stage's lines.\n- A stage never rewrites another stage's fields.\n- One artifact directory per attempt (`qid` plus `sample`). Never overwrite a\n  previous attempt's prediction CSV.\n- End-of-run numbers come from counting event lines (`jq` over\n  `events.jsonl`), never from arithmetic recalled in prose.\n\n## `set.json`\n\n| Field | Notes |\n|---|---|\n| `name` | Set name; also the directory name. |\n| `description` | |\n| `datasetVersion` | Integer. Bump on any golden repair or case change. Runs record the version they scored against. |\n| `targetModelPath` | Model path within the package. |\n| `truthPackage` | Name of the package holding the semantics-free sources every golden is re-derived from -- a package NAME, not an object. Without it `verify_goldens.py` re-derives nothing: it reports \"nothing to re-derive against\" and **exits 3**, which a caller must read as unverified rather than as a pass. The checks that need no truth server -- rubric figures, verification axis, stale rubric claims, set names -- still run and still report. `init_truth_package.py` scaffolds the package. |\n| `truthModel` | Model file inside that package. Default `truth.malloy`. |\n| `targetPackage` | Name of the package UNDER TEST. `verify_goldens.py` refuses a \"truth\" server that also serves it, because an answerer on that server can retrieve the raw truth sources beside the model and quietly change what is under test. `run_baseline.py` and `improve.py` pass the package they were given, so this field is the fallback for a direct call; without either the guard cannot fire. |\n| `truthTableRewrite` | Boolean, default false. Rewrites `duckdb.table('data/x.parquet')` refs to bare `x` in canonical queries, for a truth package whose tables are registered rather than read from files. |\n\n## `cases.jsonl`\n\nOne JSON object per line:\n\n| Field | Notes |\n|---|---|\n| `qid` | Stable case id. |\n| `question` | Exact text the answerer will see. |\n| `split` | `dev` or `holdout`. Frozen at import. Diagnose and improve read `dev` only; the acceptance check runs both. |\n| `tags` | list |\n| `state` | `candidate` / `selected` / `excluded`. |\n| `source` | Where the case came from. |\n| `golden` | `status` (`verified` / `provisional` / `invalid` / `ambiguous` / `verified_wrong`, the last being a key a person has established is wrong, which `run_baseline.py` drops from every aggregate), `kind`, `value` or `path` (artifact under the set directory), `canonicalQuery` (runs against the **truth** package, never the model under test), `verifiedBy`, and `verification` -- `{primaryAxis, variesAxis, note}` naming what the second derivation varied. **A golden is written only after two differently shaped derivations agree** (the truth query and a second one that varies a different axis -- group by something else, sum a different way, take a different route through the raw tables); `gold/<qid>.json` holds both (`truthRows`, `verifyRows`, `agreement`). Two derivations that vary the same axis share every other blind spot: that is how a golden that summed three overlapping time slices, 3x too high, passed its check. `verify_goldens.py` re-derives the value, reads the accepting clause of the rubric against the rows, and flags a second derivation that names no axis. **The question asks for data, never for an interpretation of it**, and either fixes the grain or the rubric accepts a correct figure at any stated grain (`skill:eval-judge` rules 10 and 11). |\n| `golden.kind` | What the key IS. A value (`scalar`, rows), `unanswerable`, or `criteria`: prose with no value, where the author's criteria are the whole key. `criteria` is the one kind that is legitimately `verified` on arrival, because there is no derivation to perform and nothing that can be numerically wrong; an imported key holding a VALUE is `provisional` until re-derived (`skill:eval-import`). **`verify_goldens.py --promote` is the only thing that writes `golden.status`**: it marks a `provisional` golden `verified` when its value re-derived cleanly from the truth package AND a second derivation exists. `--refresh` rewrites a drifted VALUE and never changes a status; `invalid` and `ambiguous` are settled by a person through the golden side door. |\n| `golden.rubric` | **On the golden, not on the case.** The prose the judge is shown as `CASE RUBRIC`, naming what counts as correct and what does not. `run_baseline.py` reads `golden.rubric`; a rubric written at the case's top level is silently not passed, the judge is told `CASE RUBRIC: none`, and it then scores from the golden value and the answer alone -- which reads as a judge that ignores its instructions. |\n| `golden.mustState` | **On the golden, not on the case.** What the answer has to say out loud. Read from `golden.mustState` by the run-package builder. |\n| `goldenRevision` | Integer. Bump on every golden change; `score` events stamp the revision they compared against. |\n| `expectedEntities` | `required`: entity ids (`kind:source:name`) the answer cannot be produced without. `requiredAnyOf`: a list of **groups**, each a list of ids of which any one suffices -- for a case the model can answer through more than one route. Naming only one route scores the other as a retrieval miss and steers the fix to the docs for an entity whose docs were never the problem. `acceptable`: ids that are not noise if returned. An entity counts as delivered when returned under its exact id, under the same type and name on a sibling source, or by name in a returned source's documentation; `score_retrieval.py` records the route per entity as `delivery`. |\n\n## Term files (`eval-retrieval` only)\n\nA customer set has no term file. The engine-side `eval-retrieval` skill keeps\n`intents.jsonl` fixtures -- search term, entity type, a description of what the\nsearcher meant, a validity flag -- beside its own scripts, for evaluating the\nengine against fixed inputs. Their shape and the `retrieval_score` events they\nproduce are documented there and validated here.\n\n## `run.json`\n\nThe attribution pins. Two runs are comparable only when these match where it\nmatters. Two runs with different `target` values are comparable on answer\nverdicts only if the same model version reached both, which is rarely worth\nassuming:\n\nRequired fields are the run's **identity** -- every run ever written carries\nthem. The rest are optional in the schema, but the **comparison pins** are\nwarned about when absent, because a run missing one cannot take part in an A/B\non that axis.\n\n| Field | Notes |\n|---|---|\n| `runId` *(required)* | Directory name. |\n| `target` *(required)* | Which server answered, and how it reached the data: `local`, `local-proxied`, or `platform`. Decides what an improve step is even allowed to do (see `skill:eval-loop`). |\n| `answererModel` *(required)* | |\n| `phase` / `started` *(required)* | |\n| `judgeModel` / `judgeVersion` / `rubricSha` *(pins)* | The judge's model, and the version + content sha of `skill:eval-judge`. |\n| `judgePromptSha` *(pin)* | Content sha of the prompt the judge is SHOWN, which is the harness's own template and not the skill file `rubricSha` covers. The template carries the golden rendering, the evidence blocks and the tie-breaking rules, so two runs whose prompts differed compared as the same judge -- the blind spot that let a golden-rendering defect run through three arms with every test passing, because the tests covered the code paths and not the text the model sees. Absent on runs written before it existed. |\n| `datasetVersion` *(pin)* | From `set.json` at run time. |\n| `modelSha` *(pin)* | Content sha of the `model.malloy` snapshot in the run directory -- the bytes the answerer actually queried. A git sha is not enough: a snapshot host serves a copy, often of a dirty tree no commit names. |\n| `skillsVersion` *(pin)* | HEAD of the checkout the agents' skills were loaded from, dirty-marked (`ledger.skills_git_sha(root)`). The skills are the doctrine the agents load; a run that cannot name their version cannot anchor a skills A/B. |\n| `skillsRoot` / `harnessVersion` | Which checkout supplied the doctrine (`--skills-root`, e.g. a Publisher checkout for the open-source skills; default this one), and this checkout's own HEAD. The eval-* skills always come from the harness checkout, whatever `skillsRoot` says. Two runs whose `skillsRoot` differ are a skills A/B only if their manifests name the same skills. |\n| `diagnoserModel` / `improverModel` | Written by `diagnose.py` / `improve.py` when those stages run, so the run names every LLM that touched it. Absent on a run that was only answered and judged. |\n| `modelGitSha` | Commit of the model repo, from `--model-repo`; `-dirty` suffix when that path had uncommitted changes -- fine for a band measurement, not for an A/B pin. **Null unless `--model-repo` names the repo.** It cannot be inferred: Publisher serves a COPY under `publisher_data/`, so the tree around the served file is the server's storage, not the model's history, and the field named that until 2026-09-03. `modelSha` above is the pin that always works; this is the pointer to where those bytes are versioned, and absent beats wrong. |\n| `modelRepo` | The path `--model-repo` named, so a reader can tell an absent pin from an unrecorded one. |\n| `environment` / `package` / `modelPath` | What was served, and from where. On a platform target `environment` is the organization and `package` is the workspace the MCP URL is scoped to. |\n| `scope` | Platform target only: the `environment/package[@version]` the answerer was told to pass as an explicit `scopes` entry on every `get_context` / `execute_query` call. A workspace can serve many packages (a personal workspace serves every package the user can read), so without this the run also measures whether retrieval picks the right package -- a different measurement. Absent means unscoped. |\n| `targetVersion` *(pin)* | Platform target only: the published version the answers came from. Both hosted tools default an omitted version to the PINNED one, meaning whatever the workspace serves at the moment of the call, so this only describes the run when `--scope` carries it to every call. `--target-version` supplies it when the scope omits it; a scope naming a different version is refused rather than treated as an override, because a run cannot say which build answered if two pins disagree. |\n| `mcpUrl` / `publisher` | The endpoints the answerer and the re-execution used. |\n| `predictionsReExecuted` | Whether re-execution was POSSIBLE: the server was serving the bytes this run is pinned to. It is one bool for the whole run and says nothing about any given query, so it reads stronger than it is; `reExecution` below is what actually happened. |\n| `reExecution` | `{attempted, ok, failed, noQuery, notReExecuted, notJudged, missing}` over the run's predictions. `failed` means the query ran and the server returned an error, which is a different thing from an answer with no query at all. `notJudged` is a case the judge was never handed: a golden refused before judging (`provisional`, `invalid`, `ambiguous`, `verified_wrong`, missing), an attempt that submitted nothing, or every case under `--no-judge`. Those used to land in `missing` and read as corrupt artifacts on a freshly imported set. `ok`, `failed`, `noQuery`, `notReExecuted`, `notJudged` and `missing` partition the cases, so they sum to the case count; `attempted` is `ok + failed` and sits outside that sum. |\n| `retrievalMode` | Which retriever answered: `semantic`, `lexical`, `mixed` (it changed mid-run, so nothing can be averaged across it) or `unreported` (nothing ranked, which on a configured server means no embedding provider). Only a RANKING call names a retriever, so a call that enumerates a type or looks a name up is `unreported` on a fully semantic server and must not make the run read as `mixed`. Local retrieval degrades to lexical SILENTLY without an embedding key, and a pair compared across that reads as a model change, so `flip_table.py` refuses the pair unless both sides match. |\n| `retrievalCalls` | `{semantic, lexical, unreported}` call counts behind `retrievalMode`. `unreported` counts the calls that did not rank, which is a different thing from calls that fell back. |\n| `retrievalGate` | What the pre-arm retrieval gate found, or why it did not run. A run is held until `get_context` answers semantic twice in a row, because the first calls after a restart are lexical by design and an arm started immediately measures two retrievers. `not run (no answering phase)` under `--rebuild`/`--rejudge`, and `skipped by --no-retrieval-gate` when the operator opted out -- which is a different fact from a gate that passed, and the field is what tells them apart. |\n| `coverageReport` | The `check_coverage.py --out` report this run read, when `--coverage` named one: its path, the model version it was stamped with, its `agentModel`, and `decided` of `cases`. Coverage is a read of the MODEL, so it is measured separately and costs a judge call per case; the run does not run it, it consumes it. Its per-case verdict then beats the authored `coverage` label in retrieval attribution, and each `score_retrieval` row records `coverage_source` (`measured`, `authored`, `none`) so nobody has to guess which one charged the failure. Absent when no report was given, which is a different fact from a report that found everything covered. |\n| `label` | `<set>-<phase>-<nn>`, assigned by `run_baseline.py` from the set name, the run's phase and the next free number beside it (`ecommerce-baseline-01`, `ecommerce-baseline-02`, `ecommerce-blind_gate-01`). The A/A pair is two runs of the same phase; the post-edit arms are two runs of `blind_gate`. Hand-typed names do not survive one afternoon of runs -- `base`, `rejudged2`, `r3`, `post1` sort wrongly, group not at all, and cannot be matched to an arm. `--label` overrides for a run that genuinely needs a human name. |\n| `effort` | |\n| `maxTurns` / `answererTimeout` *(pins)* | The answerer's turn cap and per-attempt timeout, as the arm ran them. `skill:eval-loop` step 6 lists the call budget among the pins to freeze for a whole run, and nothing wrote one, so no two runs could be compared on the setting that decides whether a case got to finish at all. Distinct from `callBudget`, which no harness has ever written. Size the cap from a pilot rather than taking the default: on a 29-case arm at the default 30, completed attempts had a median of 17 turns and a 90th percentile of 26, and the four cases that ran past it were recorded as wrong answers. |\n| `truncated` / `contaminated` | The qids excluded for a HARNESS-owned reason: the cap cut the answer off, or the attempt breached isolation. Both leave the denominator, both are named here rather than only in the console, and either one makes `status` `incomplete`. |\n| `answererCostUsd` / `judgeCostUsd` | What the arm cost, split by role. The judge's half was discarded until 2026-09-02, so every \"cost per arm\" quoted before then was the answerer alone. Per-case judge cost is on each `score` event. |\n| `answererCostCopiedFrom` | The run whose attempts `answererCostUsd` was COPIED from, when this run spawned no answerer -- a `--rejudge`, a `--rebuild`, a `--from`. Those copy the source's attempt events verbatim, `cost_usd` included, so the figure is right about the attempts and wrong as this run's spend: one re-judge reported $17.56 of answering that never happened. Null when this run paid, so a total over several run files can skip the copies. |\n| `goldenCheck` | What `verify_goldens.py` said before the run started: `N ok, M drifted, K other finding(s)`, or why it did not run (no truth server on a platform target; `--skip-golden-check`; rebuild). A run that started on a drifted set says so here rather than pretending its verdicts mean something. `run_baseline` calls `verify()` without a model, so the set-name audit (check 5) does not run in-arm; the trailing `N stale entity name(s)` is the in-arm lint's own count, and `entity-name lint not run` says there was no model text to lint against. Without it the field read `0 other finding(s)` for a check that never happened. |\n| `staleEntityNames` | The `expectedEntities` names that appear nowhere in the served model, from the in-arm lint. Each one depresses the run's own recall on every case listing it, so a stale set reads as a retrieval failure until someone checks by hand. `null` when there was no model text to lint against (a platform target), which is a different fact from `[]`. |\n| `status` | `complete`; `aborted` when four consecutive attempts errored or found the server dead and the harness stopped rather than spend the rest of the budget on attempts nobody will trust; or `incomplete` when the arm ran to the end and still cannot report a rate, because `truncated` or `contaminated` is non-empty. `incomplete` is a third state, not a soft `complete`: the arm did everything asked of it and the number is still unearned. It does NOT stop the run being compared: `flip_table.py` scores the cases both arms did, and names the ones this arm left unscored -- an excluded case carries no verdict, so it can never be counted as a flip. |\n| `packageSha` / `servedRevision` *(pins)* | Taken from the server, not recomputed: `sourceContentSha` is a content hash over EVERY model path in the package, so an edit to an imported file moves it where a sha of the one `--model-path` does not; `servedRevision` is minted per load, so it identifies a load rather than content and is a poor pin alone. Measured: `publisher.json`'s `version` moves neither, and nothing in Publisher reads it -- it is not in the Package API schema and never returned, so it pins nothing. |\n| `datasetSha` *(pin)* | Content hash of `set.json` + `cases.jsonl`. Deliberately SEPARATE from the model's sha: a golden repair is not a model change, and one pin covering both would make every answer-key fix read as an edit to the model, which is the distinction an A/B rests on. Automatic, so nobody has to remember it; `datasetVersion` stays beside it as the human-readable sequence. **Local targets only.** A hosted target that publishes IMMUTABLE versions needs none of this: the set rides inside the version, and immutability -- not hashing -- is what makes a pin trustworthy. There, `targetVersion` alone identifies model and set together. |\n| `doubtedGoldens` | The cases whose golden is not believed: `qid`, `gold_status` (`suspect` or `verified_wrong`), `gold_note`, and `declaredBy` (`judge` or `set`). A status the SET declared is a key somebody already settled, not an opinion the judge formed this run, and attributing both to the judge claimed a judgement that never happened. **Read this before diagnose.** These are dataset issues, not model failures, and they go through the golden side door in `skill:eval-loop`. Empty list when no key is in doubt from either source. Written from the same scan that prints the end-of-run warning, because a warning that lives only in console text is one scrollback away from sending a modelling agent at a model that is already right. |\n| `mode` / `setName` / `serverVersion` / `traceMode` / `callBudget` | Defined and accepted, **not yet written by any harness** -- kept in the schema for the platform target and the conductor, which need them. (`targetVersion` and `status` were in this list and are both written now.) |\n\n## Events\n\nEvery line in `events.jsonl` is **flat**: `{ \"kind\": ..., <fields> }`, one\nJSON object per line, with an optional `at` ISO timestamp. Case-scoped kinds\n(`attempt`, `tool_call`, `score`) carry `qid`, `sample`, `phase` on the line;\nrun-level kinds do not. (An earlier draft of this document nested fields under\na `payload` with a `caseId`; no writer ever did that, and 23 run directories\nexist in the flat shape, so the flat shape is the contract.) `kind` is one of:\n`attempt`, `tool_call`, `score`, `retrieval_score`, `issue`, `issue_status`,\n`candidate`, `acceptance_check`, `checkpoint`.\n\n### `attempt`\n\n| Field | Type | Notes |\n|---|---|---|\n| `qid` | string | |\n| `sample` | int or null | Which repeat. Required even when null. |\n| `phase` | string | `baseline` / `loop` / `blind_gate` / `canary` / `final`. `phase` lives here, on the attempt, not in run config. |\n| `question_sha` | string | Hash of the exact text the answerer saw, so two runs can be compared on whether they asked the same thing. A case may carry `questionSha`, the seal stamped at conversion, and that wins when present; the ecommerce set writes a 16-char prefix of it, and a set without one left this null. Neither form proves the question was not edited, since both follow the case as it now reads: that comparison is check 6 of `verify_goldens.py`. |\n| `submitted` | bool | False when there was no final query. Not a wrong answer. |\n| `final_query` | string or null | Required to replay. A named view is recorded as the Malloy it stands for, `run: <source> -> <view>`, so every consumer sees one shape and the query re-executes. |\n| `final_query_source` | string or null | How `final_query` was chosen: `declared` (the answer printed it), `last_ok` (the last call the server answered) or `last`. `last` is a warning: a trailing sanity probe may be standing in for the answer's own query. |\n| `servedRevision` | string or null | From the package actually queried. |\n| `n_get_context` / `n_execute` / `n_execute_errors` | int | |\n| `host_tool_uses` | int | EVERY tool use the host logged, MCP calls included. It counted only the non-MCP ones until 2026-09-03, which made the under-report check below true of almost every clean attempt. |\n| `mcp_tool_uses` | int or null | The MCP subset: `get_context` plus `execute_query`. Null on a run written before the split. |\n| `reported_calls` | int | MCP calls the answerer claimed. `reported_calls > host_tool_uses` is the under-report floor from `skill:eval-answer`: the answerer cannot have made more calls than the host logged. `validate_run` warns on it, and only for runs that carry `mcp_tool_uses`, because the field meant something narrower before. |\n| `contaminated` | bool or `\"unknown\"` | `\"unknown\"` when no host log exists. Read it with `ledger.is_contaminated`, never for truthiness: runs written before 2026-09-03 carry the strings `\"true\"`/`\"false\"`, and `bool(\"false\")` is True. `ledger.event` now rejects those strings on write; `validate_run` grandfathers them on read as a warning. |\n| `contamination_reasons` | list | Empty when clean. |\n| `input_tokens` / `output_tokens` / `cache_read_tokens` / `cache_write_tokens` | int or null | Answerer token usage, the four counts that reprice a run from its own ledger. Null when the host does not report it. `cost_usd` beside them comes from the CLI's `total_cost_usd`, which prices cache reads and writes at their rates, so it was always cache-inclusive; what was missing was the breakdown that reproduces it. `cache_write_tokens` (`cache_creation_input_tokens` upstream) was never captured before 2026-09-15, and on one analysed run cache writes were 44% of the agent's cost, the single largest line. **Read `input_tokens` with care:** it counts only the tokens after the last cache breakpoint. One run recorded 224 for 24 questions beside 2.4M cache reads; the 224 is not the context volume, and a token total that omits the cache columns is not a size. |\n| `cost_usd` | float or null | Answerer cost for this attempt. |\n| `num_turns` / `wall_seconds` | int, float or null | |\n| `answer_text` | string or null | The answer the judge scored. Kept so a verdict can be re-read without the transcript. |\n| `skills_invoked` | list of string | The skills the answerer invoked through the `Skill` tool, deduplicated and sorted, counted from its own transcript. Always a list on a run that recorded it, so an EMPTY list is the finding: the arm granted a manifest and the agent reached for none of it. Absent on runs written before 2026-09-15, which is why every reader takes `or []`. Only explicit `Skill` calls are counted, and only skills the arm granted -- one outside `--answerer-skills` is a contamination breach and lands in `contamination_reasons` instead. What it is for: a manifest names what an arm GRANTED, and comparing two arms' doctrine is meaningless if the doctrine never loaded. |\n| `transcriptPath` | string | The answerer's transcript under `artifacts/`. |\n\nToken counts sit here rather than being derived later because the claim a\nsemantic model makes is about cost as well as correctness -- that a documented\nmodel reaches a good answer in fewer turns and tokens than working from raw\nschema. A ledger that counts calls but not tokens can state half of that.\n\n### `tool_call`\n\nOne event per MCP `get_context` or `execute_query` the attempt made.\n\n| Field | Type | Notes |\n|---|---|---|\n| `tool` | string | `get_context` or `execute_query`. |\n| `traceId` | string or null | `get_context` only; look up in your host's trace store. |\n| `targets` | string, list or null | **What the answerer asked for**: the search terms it sent to `get_context`. Null for `execute_query`. |\n| `rankedSummary` | object | Copied at capture so evidence survives trace eviction: `entityIds` (identity, in response order), `hits` (one row per entity: `entity_id`, `source`, `relevance`, `matched_targets`, `position`, `position_in_source`), `resultCount`, `docTokens`. **There is no rank.** This field previously documented `ranks` and \"per-target `targets` with within-target ranks\"; no writer ever wrote either, and what `ranks` did hold was `range(1, n+1)` over a list already flattened source-card by source-card -- a position, not a rank, and not even a relevance order, since the response sorts globally and then buckets into cards. Ask relevance questions of `hits[].relevance`. |\n| `rankedSummary.hits[].matched_targets` | list or null | Which of the caller's search targets matched this entity, each with the server's own relevance. **`null` means no attribution was published**, which is not the same as `[]`: the server attaches it to entities only (a source card's attribution is discarded before an entity object exists) and only on the semantic path, because a lexical score is relative to its own query and comparing two of them means nothing. A lexical run therefore cannot say which target found what, and a reader must report that as unavailable rather than as \"never asked\". |\n| `retrieval_mode` | string or null | `get_context` only: the `retrieval` field of the response that answered, `semantic` or `lexical`. Null when the server named none, which means no embedding provider. Recorded per call because the semantic path can fall over partway through a run. |\n| `query` | string or null | `execute_query` only: the Malloy the call sent, so the final query can be chosen by which call the server actually answered. |\n| `modelPath` | string or null | `execute_query` only: the model file the call named. A source does not resolve outside the file that declares it, so re-execution needs this. |\n| `error` | string or null | |\n\nNever persist `execute_query` result rows, givens, or credentials.\n\n`targets` records the request; `rankedSummary` records the response. Without\nboth, a low per-attempt recall has two readings that cannot be told apart: the\nanswerer searched for the wrong thing, or it searched well and retrieval ranked\nthe right entity too low. Those have opposite owners -- `agent-skill` and\n`retrieval` -- so a ledger holding only the response cannot attribute the\nfailure, and any recall computed from it is a blend of answerer behaviour and\nretrieval quality.\n\nThat blend is also why per-attempt recall is **not** comparable across arms: a\nstronger answerer searches better and scores higher retrieval recall without\nretrieval having changed. Use it for attribution within an arm. Comparing\nretrieval itself between engine versions is `eval-retrieval`'s job, with fixed\nterms, and is not something a customer run reports.\n\n### `score`\n\nThe answer judge's verdict for one attempt (protocol in\n`skill:eval-judge`). Every attempt in a scored run gets exactly one.\n\n| Field | Type | Notes |\n|---|---|---|\n| `verdict` | string or null | `match` / `near_match` / `no_match` / `needs_human`; null when the attempt is not scorable. Only `match` and `no_match` are decisions; see below. |\n| `reason` | string | Why, from the judge; for a null verdict, why not scorable, and it is ALWAYS one of these: `not_submitted`, `answerer_truncated`, `contaminated`, `judge_unparseable`, `no_saved_verdict`, `judge_error: <text>`, or `golden_<status>` -- `golden_missing`, `golden_provisional`, `golden_invalid`, `golden_ambiguous`, `golden_verified_wrong`. A null verdict never arrives without one: a nulling that does not say which of these it was is indistinguishable from the field being dropped on the write path, and was read as exactly that from a real run. The `golden_*` reasons and `answerer_truncated` are decided before the judge is called, so neither an unestablished key nor a cut-off attempt costs a judge call. |\n| `confidence` | int or null | 1 to 10. Confidence of 5 or lower forces `needs_human`. |\n| `column_pairing` | object or null | The judge's named gold-to-prediction column correspondence. |\n| `judge_version` / `rubric_sha` | string | Pins which rubric produced this verdict. |\n| `golden_revision` | int | From the case at score time. |\n| `contaminated` | bool or `\"unknown\"` | Copied from the attempt; true or unknown means `verdict: null`. |\n| `must_not_use_hits` | list or null | Entries of `golden.mustNotUse` a script found in the final query. A hit forces `verdict: no_match`. Decided by `check_must_not_use.py`, never by the judge: it is a question about query text. Only a BARE name vetoes; an entry with a connective (`X as ...`, `X through ...`) objects to a use of X rather than to X, and goes to the judge with the prose entries and a path's bare leaf. |\n| `judge_verdict` | string or null | What the judge said when a MECHANICAL override replaced it: a `must_not_use_hits` veto, or contamination voiding the verdict. Null otherwise, so the judge's own agreement rate stays measurable across both. When a case is vetoed AND contaminated this still holds the judge's read, not the veto's `no_match`. |\n| `artifactPath` | string | The full judge output under `artifacts/`. |\n| `gold_status` | string | `verified` / `verified_benign` / `suspect` / `verified_wrong`, and never anything else: the case file's `golden.status` is a DIFFERENT vocabulary that shares only `verified` and `verified_wrong`, and only those two fall back into this field. A `provisional` / `invalid` / `ambiguous` key is reported through the verdict's `reason` (`golden_<status>`) instead, so nothing downstream has to match on two vocabularies in one field. **From the judge**, which scored against the golden as written and reports separately whether it believes it; falls back to the case's standing `golden.status` when the judge does not say, written onto the verdict itself so the run summary and the aggregates see it too. `verified_wrong` excludes the case from run aggregates -- from the judge OR from the case file; assigning the fallback into a copy of the verdict meant a key the set had marked wrong stayed in the aggregates unless the judge independently agreed. `suspect` and `verified_wrong` route to the golden side door as `dataset` issues, never to improve. |\n| `gold_note` | string or null | The judge's evidence for a non-`verified` status: the two values, or the model line against the rubric sentence. Null when `verified`. |\n\nA `golden_*` reason means the ANSWER KEY is not something a verdict can be\nissued against. A golden the set stamps `provisional`, `invalid` or `ambiguous`\nis refused, as is a case with no golden at all and one the set marks\n`verified_wrong`; a golden with NO `status` is unguarded rather than\nunscorable, because sets that predate the field would otherwise score zero of\nzero. A golden holding no value is not refused either: that is how a\n`kind: unanswerable` case and a `kind: criteria` golden are written. This\nmatters most for an imported set, because `skill:eval-import` stamps\n`provisional` on every golden it writes and enforces it -- so a set is\nunscorable by construction until its keys are re-derived, and `run_baseline`\nrefuses to spend at all when every case in a set is in that state.\n\n`not_submitted` means the attempt produced NOTHING to judge: no answer text and\nno query. An attempt that wrote prose without querying **is judged**, and\nagainst a golden that holds a value an answer containing none of it is\n`no_match` (`skill:eval-judge`, `reference/refusal.md` rule 10). Excusing those\nas unscorable dropped a confident refusal on an answerable case out of the pass\nrate, which is the one thing the answerable-sounds-unanswerable cases exist to\nmeasure. A refusal is exempt only where `golden.kind` is `unanswerable`, and\nthere naming the gap is the pass and a confident number is the fail.\n\n`answerer_truncated` means the answerer hit the run's `maxTurns` cap and the\nCLI returned whatever it had, which is usually a sentence stopping mid-thought.\nIt is refused BEFORE the golden gate and before a saved verdict is reused, so\n`--rebuild` over a run made without this re-derives the right ledger from its\ntranscripts. A cap is a harness setting, so a case cut off by one says nothing\nabout whether the model could answer it: measured on one 29-case arm, four\nattempts ended at exactly 31 turns and each was judged as a wrong answer, one\nof them on the text \"Let me build the correct monthly trend query.\"\n\n`contaminated` means the attempt breached isolation (`contamination_reasons`\nnames how) and the verdict is withheld rather than lost. The judge's own read\nmoves to `judge_verdict`. Before this reason was written, a voided verdict kept\nthe judge's prose in `reason` beside `verdict: null`, and a run report\nreasonably concluded the ledger was dropping the field -- it recovered nine\n\"lost\" verdicts by re-parsing the stored judge replies, every one of which had\nbeen voided on purpose.\n\nBoth are HARNESS-owned, not model-owned, and a run containing either reports\n`status: incomplete` in `run.json` and **prints no pass rate**. `unscorable`\n(no established golden) and `judge_unparseable` do not suppress the rate: the\nfirst is a dataset state no arm can fix, and the second already has a retry.\n\nAggregates count decided verdicts only. `match` and `no_match` are the pass and\nthe fail; **`near_match`, `needs_human` and null are none of the above** and stay\nout of acceptance arithmetic. `near_match` is excluded because it means \"defensibly\ndifferent\", and an arguable verdict that moves a pass rate is a measurement\nartefact rather than a result (`skill:eval-judge` rule 7). Report its count:\nit rising is how a set tells you its rubrics are going vague.\n\nThis applies to `score`. On `retrieval_score` below, `near_match` **is** counted\ntowards recall and precision, because an overlapping entity is a genuine\nretrieval success. The two are different questions that share a word.\n\n### `retrieval_score` (written by `eval-retrieval`, not by a customer run)\n\nOne per term judged in a fixed-term replay (run-level; no `caseId`).\n\n| Field | Type | Notes |\n|---|---|---|\n| `intentId` / `term` / `entityType` | string | From the term file. |\n| `in_scope` | bool | Does THIS model version represent the concept? False is a coverage gap, not a retrieval failure. |\n| `judgments` | list | Per returned entity: `entityId`, `rank`, `level` (`match` / `near_match` / `no_match`), `confidence`, `why`. Empty when nothing returned. |\n| `judge_version` / `rubric_sha` | string | |\n| `traceId` | string or null | The `get_context` call judged. |\n\nRun-level metrics fall out by counting:\n\n- `coverage` = in-scope terms / valid terms -- a property of the model and its\n  data, reported on its own, never as a retrieval number.\n- `recall` (on in-scope terms) = fraction whose judgments contain a `match`.\n- `precision@N` = `match` judgments / judged, with N stated.\n\n### `issue`\n\n| Field | Type | Notes |\n|---|---|---|\n| `issue_id` | string | Stable across status events. |\n| `qids` | list | Affected cases. |\n| `primary_code` / `contributing_codes` | string / list | From `skill:eval-diagnose`, verbatim. |\n| `component` | string | `dataset` / `agent-call` / `get_context/model` / `get_context/retrieval` / `construction` / `model-definition`. |\n| `owner` | string | `model` / `retrieval` / `agent-skill` / `dataset`. Environment failures stop the run; they are never diagnosed, so there is no environment owner. |\n| `severity` / `confidence` | string | |\n| `sufficiency` | string | `sufficient` / `insufficient` / `unknown`. |\n| `traceIds` | list | |\n| `diagnosis` | string | Written before any edit exists. |\n\n### `issue_status`\n\n`issue_id` plus `status`: `open` / `batched` / `fixed` / `rejected` /\n`deferred`. Readers take the latest event for that `issue_id`. Do not invent\na status column.\n\n### `candidate`\n\nWritten by `eval-improve`, for every proposed edit, accepted or not. A\nrejected direction keeps its record.\n\n| Field | Notes |\n|---|---|\n| `issue_ids` | |\n| `files` | Paths the edit touched. |\n| `diffSummary` | One line per file. |\n| `probes` | Query and result for each factual claim. |\n| `meaningChanged` | Entities whose *meaning* the edit changed; `[]` for a docs-only edit. |\n| `goldenSuspect` | Each `{qid, entity, stored, rederived}`: a golden this edit may have invalidated. Reported by the improver, never repaired by it. **Non-empty halts the acceptance check** until adjudicated through the golden side door. |\n| `goldenAudit` | The set's `verify_goldens.py` run against the edited model: `{ran, clean, model, tail}`. Catches drift and rubric-vs-model contradictions only; a golden whose value silently moved is invisible to it, which is why `goldenSuspect` exists alongside. |\n\n### `acceptance_check`\n\nCalled `gate` before 2026-09-03. `ledger.py` reads the old kind as this one, so\nrun directories written earlier still validate and nothing rewrites them.\n\nWritten by `eval-loop`, one per acceptance check decision, BEFORE any checkpoint commit.\n\n| Field | Notes |\n|---|---|\n| `issue_ids` | |\n| `decision` | `accepted` / `rejected`. |\n| `class` | `docs` / `definition` / `retrieval` / `justified`. |\n| `baselineRunId` / `finalRunIds` | Plural: acceptance needs two independent runs. |\n| `regressions` | Case ids whose verdict got worse vs baseline. Must be empty to accept. |\n| `holdoutDelta` | Confident-verdict delta on the holdout slice. |\n| `reason` | Including independent deterministic justification when that is the basis. |\n\n### `checkpoint`\n\nWritten by `eval-loop` after an accepted acceptance check, or when a restore runs. The\nmodel bytes live in git, not in this payload.\n\n| Field | Notes |\n|---|---|\n| `action` | `created` / `restored`. |\n| `label` | |\n| `modelGitSha` | The commit this checkpoint names (create), or the commit restored to. |\n| `issueIds` | Issues the accepted edit closed. Empty on restore. |\n\n## `judge-regressions.jsonl`\n\nAppend a line whenever a human overrules a judge verdict: the case or intent,\nthe judge's verdict, the human's, and why. Re-run this file against the judge\nwhenever `skill:eval-judge` or the judge model changes; a rubric change that\nflips old human-settled verdicts is a judge regression, not new truth."
     },
     {
       name: "eval-diagnose",
@@ -317810,6 +318549,35 @@ repair closes that. Take the stable list \`flip_table.py\` prints and pass
 \`--only <qids> --verdicts near_match\`. Never diagnose a one-armed \`near_match\`;
 that is noise, and it sends an agent to fix a model that is already right.
 
+**A correct answer can still carry a finding.** A case that answered right
+while a required entity never reached it is diagnosed too, for the retrieval
+miss alone, and \`diagnose.py\` selects it automatically. The answer was right by
+another route, and naming that route is the job: on the run this rule comes
+from it was always the same one, the agent rebuilding the model's own measure
+inline. That held while the measure was \`count()\` and failed the moment one
+carried a grain rule, producing the run's only wrong answer. Three of the four
+findings in that run sat on passing cases and, before this, produced nothing.
+
+**"It worked anyway" is not a reason to leave the model or the skills
+unfixed.** An answer that is right without the model's own entity is right for
+now, not right by design. Write the issue against the miss, record that the
+answer was correct so nobody reads it as a wrong number, and do not soften the
+finding because the number came out right. \`--no-retrieval-misses\` opts out for
+a run that only wants answer failures.
+
+Holdout is withheld so the acceptance check keeps something the improve step
+never saw. A **measure-only** run never reaches improve, so it is holding those
+cases back from nothing: pass \`--include-holdout\` there. The script refuses it
+on a run that already carries a \`candidate\`, because that run's holdout is the
+only thing left that can falsify the edit.
+
+**Say what the clusters do not cover.** \`diagnose.py\` prints a coverage account
+of every non-passing case and which bucket it fell in -- holdout, contaminated,
+a verdict outside \`--verdicts\`, unscored, or selected and not diagnosed. Quote
+it whenever you report clusters. One run's six clusters were read as covering
+its failures; they covered 8 of 18, and each individual exclusion had been
+correct and added up nowhere.
+
 ## Components, in order
 
 Walk **in this order** and stop at the first with positive evidence. A later
@@ -317821,7 +318589,7 @@ never "C1" / "C2" / "C3":
 | \`dataset\` | Bad question, bad or missing golden, or environment drift? |
 | \`agent-call\` | Did the agent ask for the needed concepts, with the right type and scope? |
 | \`get_context/model\` | Is the needed entity absent, undocumented, weakly labeled, duplicated, or missing guidance? |
-| \`get_context/retrieval\` | Was an on-target request against a well-described entity ranked or grouped wrong? |
+| \`get_context/retrieval\` | Was an on-target, in-scope request against a well-described entity ranked or grouped wrong? **Check the call's \`scopes\` first**: a call pinned to one source cannot return another source's entity, and that miss is \`agent-call\`. |
 | \`construction\` | Did sufficient context arrive, and the agent still built the wrong query? |
 | \`model-definition\` | Is a measure, join, filter convention, or source semantically wrong? |
 
@@ -318102,6 +318870,22 @@ Say what you considered merging and chose not to. A cluster is a claim that one
 change fixes N cases, and the near-misses are what a reviewer needs to falsify
 it.
 
+**Falsify a behavioural cluster against the passes.** This skill reads failures
+only, so any behaviour common to the whole run looks causal from inside it.
+\`diagnose.py\` hands the clustering step a \`CONTROLS\` block: the same
+measurements -- retrieval calls, targets carrying no \`search_text\`, queries,
+skills opened, turns -- taken on the cases that PASSED. Before claiming a
+behaviour explains a cluster, compare it there. If it occurs at a similar rate
+in the passes it does not separate the groups: mark the cluster \`contributing\`
+rather than \`primary\` and do not route it to an edit as the root cause.
+Measured, on the run this comes from: the largest cluster said the agent
+"substitutes broad enumeration for targeted retrieval", and bare targets were
+25% of all targets in the failures against 26% in the passes. What actually
+separated them was volume -- failures made about 50% more retrieval calls --
+and question difficulty explains that at least as well as call style does. With
+no controls at all, a behavioural cluster is unfalsified, which is a different
+claim from confirmed.
+
 Still no patch. Naming the shared root cause precisely enough that someone else
 can design the edit is the whole job here; the edit itself is
 \`skill:eval-improve\`, working from this. A cluster carrying an honest open
@@ -318300,6 +319084,134 @@ back until somebody derives keys. A set of bare questions already measures
 whether the model can express an answer at all, and the answers it produces
 are what the keys get derived from.
 
+### Deriving a key yourself, when nothing arrived with the question
+
+The table above is about what ARRIVED. Sometimes nothing did, nobody knows the
+right number, and the set still needs keys. You may derive one, and the rules
+are the ones already stated rather than new ones:
+
+- Use your full model access and query until you can defend the number. The
+  agent that gets tested later holds only \`get_context\` and \`execute_query\`;
+  you are not that agent, and the gap is the point.
+- Record the query as \`canonicalQuery\`. A number with no query is somebody's
+  guess, which is the row above it in the table.
+- Record the entities you used to get there as \`expectedEntities.required\`, in
+  the \`kind:source:name\` form: the measures, dimensions and views the answer
+  cannot be produced without. \`reference/case-format.md\` says to leave this
+  field empty at import because guessing it invents a retrieval expectation
+  nobody stated; you are not guessing, you just used them. That is the only
+  trustworthy producer this field has, and it is what makes retrieval
+  measurable on a set nobody sent keys for. Where the model offers two
+  legitimate routes, write a \`requiredAnyOf\` group rather than pick one.
+  \`verify_goldens.py\` check 5 audits every id against the model, so a wrong one
+  is a hard finding rather than a silent retrieval miss.
+- **Derive the list with the model in front of you, not from memory, and not
+  with grep.** The model knows what each field IS; an author does not reliably.
+  Ask the question of the model: which entities does an answer to this depend
+  on, and is each a measure or a dimension. Confirm every id with \`get_context\`
+  before writing it. A list written from recollection is the most common way a
+  set acquires a retrieval expectation nobody can satisfy.
+
+  Measured: an agent given the model and one question found alternatives a
+  hand-written key had missed on all three cases tried -- carrier name against
+  nickname against code, \`destination_count\` against its underlying
+  \`destination.airport_count\`. At roughly $0.13 a question. It is better than a
+  person working from notes at finding what the model actually offers.
+
+  It is worse at deciding what to REQUIRE, and the two rules below are why. On
+  the same three cases it proposed demoting two named measures on the grounds
+  that a raw expression answered identically. One of those equivalences was
+  false; the other was true today and is not a reason to demote anything.
+
+- **But VERIFY every "this is not strictly required" claim by running both
+  sides.** This is the rule that makes the step above safe, and it is not
+  optional.
+
+  The same probe got one badly wrong. Told that a named measure is not required
+  when a raw column plus a plain aggregate answers identically, it wrote that
+  \`average_plane_size\` was "equally answerable by
+  \`avg(aircraft.aircraft_models.seats)\`". Executed, those are 229 and 196.93.
+  The measure is \`aircraft.avg(...)\`, an average over distinct aircraft, and the
+  inline version averages over flights. The claim was false for exactly the
+  measure whose entire purpose is to carry that distinction -- and it is the
+  same mistake, on the same field, that produced the only wrong answer in the
+  run this set came from.
+
+  Had that proposal been written into the key, the case would have stopped
+  testing the thing it exists to test.
+
+  So: an equivalence is a claim about DATA, and reading a definition is not
+  evidence. Run both expressions and compare the rows.
+
+  **But agreeing today is not a reason to demote the model's entity, and this
+  is the part the probe got backwards.** \`airport_count is count()\` returns the
+  same 984 as a bare \`count()\` right now, and stops doing so the moment anyone
+  adds a null filter or a deduplication to it. The key would not notice: it
+  would go on passing while testing nothing. That applies to every measure that
+  is currently a thin wrapper, which is most of them until the day one is not.
+
+  **\`requiredAnyOf\` is for two routes the MODEL defines** -- a carrier's \`name\`
+  and its \`nickname\`, \`destination_count\` and the \`destination.airport_count\`
+  it wraps. Both are entities, both are retrievable, and an answer using either
+  has gone through the semantic layer. That is a real choice and the group
+  records it.
+
+  It is NOT for "the model's entity, or a raw expression that happens to
+  agree". Those are not two routes: one is the route, the other is a bypass
+  that currently works. Keeping the named entity in \`required\` is what makes
+  the eval measure what the semantic layer is for, and finding it IS the
+  retrieval behaviour under test -- an agent that searches for the pre-defined
+  entity inherits its definition and its rules, while one that rebuilds from
+  raw columns inherits whatever it thought of.
+
+  What the equivalence check is FOR is reading a run afterwards. When an answer
+  came out right without the entity, comparing the two says whether it got
+  lucky or got it wrong, and a set whose passes are merely lucky is a set of
+  near-misses.
+
+- **Then prove each one is retrievable, and record what finds it.**
+  \`check_findable.py --set <set> --mcp-url <mcp> --publisher <rest>
+  --environment <env> --package <pkg>\` runs two checks and needs no model call.
+  With \`--publisher\` it reads the COMPILED model and settles whether each field
+  exists and what KIND it is; then it searches for each entity to confirm the
+  index can actually deliver it.
+
+  This matters more than it looks, because \`required\` is what BOTH retrieval
+  numbers are computed against: whether the agent asked for the entity, and
+  whether it came back. An id retrieval cannot deliver makes both fiction, and
+  the case reports a retrieval miss on every run, which reads as a defect in
+  the model or the agent rather than in the key.
+
+  **Prefer the compiled model over a grep, and prefer it in both directions.**
+  Measured on one package: \`dimension:flights:flight_count\` names a field the
+  model really has, so grepping the text passes it, and the compiled model says
+  \`flight_count\` is a MEASURE, which no \`dimension\` request can return.
+  Conversely \`dimension:airports:own_type\` appears zero times in the \`.malloy\`
+  and the compiled model declares it, because the source exposes it implicitly
+  from the data; the grep calls it missing and acting on that deletes a good
+  entity. \`verify_goldens.py\` check 5 is the grep, it is free and needs no
+  server, and it is not the authority.
+
+  A pass is a floor, not a verdict on the docs: each entity is searched by its
+  OWN NAME, the easiest query that could find it. An entity that answers to its
+  identifier and not to the words a question uses still fails at run time, as
+  \`not retrieved\`.
+
+  Writing the list forces the question an author has to answer anyway: what
+  would a reasonable agent search for, and of what type, to find this? If you
+  cannot state that, the agent cannot be expected to guess it.
+- \`status: provisional\`, never \`verified\`, for the reason stated above: you
+  derived it THROUGH the model under test, so a model bug would certify its
+  own key. \`verify_goldens.py --promote\` is still the only way out.
+- **Where two readings are both defensible from the model, do not pick one.**
+  A model that documents two conventions for the same population produces two
+  honest numbers, and choosing quietly is how a confident wrong key gets
+  written. Go to Hold an ambiguous golden in \`skill:eval-loop\`'s
+  \`reference/golden-side-door.md\`: record the competing candidates rather than
+  a new number.
+- What no query can settle is not a failure to derive. It is step 4's second
+  kind, and those clauses score on day one.
+
 ## Step 3: run their query, if they gave one
 
 Do it at import, before any run. It is the cheapest finding in the whole loop.
@@ -318356,6 +319268,57 @@ Where you cannot decide, mark the case and report it rather than guessing. A
 criterion nobody could classify is a question for its author, and
 \`skill:eval-loop\`'s golden side door is where it waits.
 
+### Four rules for writing the rubric itself
+
+Each of these cost a scored case on a real set, and none of them is obvious
+while you are writing one.
+
+**The golden rows are the figures. The rubric's prose is a gloss on them, and
+where the two disagree the prose is what is stale.** On one set 26 of 29
+rubrics quoted a figure that appears nowhere in their own golden rows: the
+goldens were re-derived the next morning, the prose was not, and an agent that
+computed 747 and 370 -- the exact numbers in the golden JSON -- was failed
+against a rubric still saying 615 and 502. Say how to derive the figure, not
+what it equalled. \`verify_goldens.py\` check 2 reports figures in the accepting
+clause that are absent from the rows, but it reads only figures specific enough
+to be a quoted result -- a bare three-digit number is invisible to it, and 615
+is exactly that. So the check is a help, not a guarantee, and the rule above is
+yours to keep. The judge is told the same thing from the other side: where a
+rubric and a golden disagree about a figure, it scores against the golden.
+
+**Do not assert the model's current behaviour.** "\`contract_terms\` cannot be
+used here at all -- it returns ZERO rows" was true when written and false four
+hours later, when a commit unblocked that view. Nothing linked the two, and the
+judge then reasoned from the stale claim against an answer that was right.
+Co-locating the set with the model makes such drift visible in a diff; it does
+not detect it. A rubric that describes a bug is a rubric with an expiry date:
+write the requirement, not the defect.
+
+**When a question admits two honest populations, accept either and require the
+answer to name which.** "Products in a category" can mean listed-in or
+primary-category, and on one set the two readings differed by 4,070 against
+2,707. Six or more cases turned on it and one rubric had been written to accept
+both. That one was right. The rest were repaired afterwards, having failed
+correct answers in the meantime.
+
+**If the rubric accepts an alternative, the entity list must too.** The two
+halves of a key are read by different things: the rubric is prose for the judge,
+\`expectedEntities.required\` is ids for retrieval scoring. They can disagree
+without anything noticing. A rubric saying "either the full carrier name or the
+nickname is fine" beside a required list naming only \`dimension:carriers:name\`
+scores an answer that used the nickname CORRECT and docks it recall in the same
+run, and that lost recall then reads as a retrieval failure. Use a
+\`requiredAnyOf\` group, which is satisfied when any member is delivered.
+\`verify_goldens.py\` reports the mismatch as a review item, but the rule is
+yours: it is a heuristic over prose and cannot catch every phrasing.
+
+**A trap note is not a requirement.** Notes that arrive beside the questions
+describe what their author thought was hard, and they mention things the
+question never asked for. Turning one into a rubric clause invents a
+requirement the answerer was never given, and it happened: a term appearing
+nowhere in any question became a clause an answer was marked down for missing.
+Convert a note only where it constrains the answer to the question as asked.
+
 ## Step 5: never change a question, and seal it
 
 The question is the stimulus. It is what a human asked, and it is not yours to
@@ -318456,7 +319419,7 @@ Available: case-format.`
     {
       name: "eval-import/case-format",
       description: "Reading an arriving question list. Reference detail for the eval-import skill.",
-      body: "<!-- What an arriving item maps to, and how to read the four formats questions actually arrive in. The file shapes themselves are in eval-answer/reference/ledger-schema.md; this is only what is specific to an import. -->\n\n# Reading an arriving question list\n\n`reference/ledger-schema.md` in `skill:eval-answer` defines `set.json`,\n`cases.jsonl` and every golden field. This covers two things it does not: what\nan arriving item maps onto, and how the four formats break.\n\n## What an import fills in\n\nOn the case:\n\n| field | from the source | when it is missing |\n|---|---|---|\n| `qid` | their id, if they gave one, prefixed with the set name | derive from the question: a slug of its first words plus a counter. Stable, because scores are keyed on it |\n| `question` | the question text, byte for byte | it is not a case. Do not reconstruct a question from a criterion |\n| `questionSha` | never from the source | stamped at conversion by `import_cases.py --stamp` |\n| `split` | rarely present | you choose, and freeze it at import |\n| `source` | how it arrived: the filename, or the log query | say `unknown` rather than guessing a provenance |\n| `golden` | see the classification table in `SKILL.md` | absent. A case with no golden is legitimate |\n| `expectedEntities` | almost never present | leave it out. Guessing which fields an answer needs invents a retrieval expectation nobody stated, and it scores as a retrieval miss forever |\n\n`expectedEntities` is the one most worth leaving empty. A wrong `required` id\ncannot be delivered by any run, so it reads as a model failure on every case\nit touches: five copied ids cost a real set two days.\n\nOn `set.json`: `name`, `description`, `datasetVersion: 1`,\n`targetModelPath`, `targetPackage` (the package under test -- it arms the\nguard that refuses a \"truth\" server also serving it), and a `sourceNote` saying\nwhere the questions came from and what shape they arrived in. `truthPackage` is\nusually absent at import, and until it exists no golden can reach `verified`;\n`verify_goldens.py --promote` is what takes it there afterwards. `init_truth_package.py` in\n`skill:eval-answer` scaffolds one.\n\n## Their field names will not be your field names\n\nAn arriving JSONL is somebody else's schema. Map it explicitly rather than\ntrusting a name that looks familiar, and write down the mapping you used in\n`sourceNote`. Names seen in real sets:\n\n| theirs | usually means | trap |\n|---|---|---|\n| `prompt`, `query`, `input`, `nl_question` | the question | `query` is the trap: in half of these it is the SQL or Malloy they ran, not the question |\n| `answer`, `expected`, `output`, `ground_truth` | a golden value, prose, or both mixed in one string | a paragraph under `answer` is criteria, not a value. Classify it, do not store it as a number |\n| `sql`, `malloy`, `gold_query` | their derivation | run it (`SKILL.md` step 3). This is the field that upgrades a typed number into something checkable |\n| `criteria`, `rubric`, `assertions`, `must_include` | prose criteria | apply the query test in `SKILL.md` step 4 to EACH one, not to the field |\n| `id`, `case_id`, `uuid` | their id | keep it, prefixed. Two sets from one customer collide otherwise |\n| `difficulty`, `category`, `tags` | their own labels | keep as-is on the case for slicing. Never let one become `split` |\n\nA field you cannot map is a question for its author. Keep it verbatim under a\n`sourceFields` object on the case rather than discarding it or forcing it into\na field it does not mean.\n\n## The four formats\n\n**JSONL.** One object per line, and the easy one. The failures are all in the\nmapping above, plus two mechanical ones: a trailing incomplete line, and a\n`\\n` inside a question that a naive line split breaks. Parse it as JSON per\nline, and count the lines you produced against the lines they sent. Report the\ncount in the import summary: an import that silently dropped 3 of 50 is a\nmeasurement on 47 cases claiming to be one on 50.\n\n**CSV or a spreadsheet export.** Questions contain commas, quotes and newlines,\nso use a real CSV reader, never a split on `,`. Watch for a header row that is\nnot row one (an exported sheet often has a title and a blank line above it), for\nmerged cells arriving as empty strings that belong to the row above, and for a\nnumber that arrived as text with a thousands separator or a currency symbol. A\nnumber you cannot parse cleanly is a string; keep it as their text and say so,\nrather than reading `1,234` as `1.234` or as `1234` without checking.\n\n**A markdown doc or an email thread.** The questions are prose among other\nprose, so the hard part is deciding what IS a question. Rules that hold up:\na heading or a numbered list item is usually a case boundary; a sentence\nending in a question mark inside a paragraph about something else usually is\nnot. Criteria attached to a question tend to follow it as a bullet list or a\n\"should\" sentence. Quote a boundary you are unsure of and ask, rather than\nsplitting on your best guess: a question invented by a bad split is\nindistinguishable, later, from one a human asked. Nothing in this format is\nbyte-clean, which makes the verbatim copy in `as-received/` load-bearing.\n\n**A pull from production logs.** The best source, because real traffic asks\nwhat people actually ask, and the messiest. Three things to do:\nde-duplicate near-identical prompts before you import 40 of one question;\nstrip the parts that are not the question (a system preamble, a UI-injected\nscope line, a pasted table); and keep the trace or request id on the case as\n`source`, so a case can be taken back to the session it came from. Almost\nnone of these arrive with a golden, which is fine: bare questions are a\ncoverage measurement. Where the log also holds the query the agent ran, that\nis NOT a golden. It is what an unevaluated agent did, and importing it as a\nkey scores the model against its own past behavior.\n\n## A worked line\n\nTheirs:\n\n```json\n{\"case_id\": \"q-014\", \"prompt\": \"How did denim sell last quarter versus the one before?\",\n \"sql\": \"SELECT ...\", \"answer\": \"Up 12.4% to 1.83M. Should break out by channel.\",\n \"difficulty\": \"hard\"}\n```\n\nYours, after applying step 3 and step 4:\n\n```json\n{\"qid\": \"ecom-q-014\", \"question\": \"How did denim sell last quarter versus the one before?\",\n \"questionSha\": \"<stamped>\", \"split\": \"dev\", \"source\": \"questions.jsonl\",\n \"golden\": {\"status\": \"provisional\", \"kind\": \"scalar\", \"value\": 1830000,\n            \"canonicalQuery\": \"SELECT ...\", \"verifiedBy\": \"authored_query\",\n            \"rubric\": \"Reports the quarter-over-quarter change for denim, and breaks the figure out by channel.\"},\n \"sourceFields\": {\"difficulty\": \"hard\"}}\n```\n\nAnd one that arrived as criteria alone, with no number anywhere:\n\n```json\n{\"qid\": \"ecom-q-021\", \"question\": \"Which channels are underperforming?\",\n \"questionSha\": \"<stamped>\", \"split\": \"dev\", \"source\": \"questions.jsonl\",\n \"golden\": {\"status\": \"verified\", \"kind\": \"criteria\",\n            \"verifiedBy\": \"authored_criteria\",\n            \"rubric\": \"Names the channels and says what it measured underperformance against. Does not use list price.\",\n            \"mustNotUse\": [\"retail_price\"]}}\n```\n\nNo query settles \"underperforming\", so there is nothing to derive and nothing\nthat can be numerically wrong: the clauses are the whole key, and this case\nscores on the first run.\n\n`criteria` and `unanswerable` are the two kinds an import may mark `verified`,\nand they arrive looking alike. A criterion reading \"PASS: a refusal that names\nthe missing data\" belongs to the second: the key is that the data is absent,\nso `kind: unanswerable`, and filing it as `criteria` beside a value would make\na correct refusal fail.\n\nWhat happened to that one `answer` string: the number became a provisional\nvalue, checkable because they sent the query that produced it; \"should break\nout by channel\" is a shape nothing can settle with a query, so it became\nrubric prose and scores on day one; and \"Up 12.4%\" is a second figure the\nrubric now has to state consistently with the value, which\n`verify_goldens.py` reads the accepting clause for."
+      body: "<!-- What an arriving item maps to, and how to read the four formats questions actually arrive in. The file shapes themselves are in eval-answer/reference/ledger-schema.md; this is only what is specific to an import. -->\n\n# Reading an arriving question list\n\n`reference/ledger-schema.md` in `skill:eval-answer` defines `set.json`,\n`cases.jsonl` and every golden field. This covers two things it does not: what\nan arriving item maps onto, and how the four formats break.\n\n## What an import fills in\n\nOn the case:\n\n| field | from the source | when it is missing |\n|---|---|---|\n| `qid` | their id, if they gave one, prefixed with the set name | derive from the question: a slug of its first words plus a counter. Stable, because scores are keyed on it |\n| `question` | the question text, byte for byte | it is not a case. Do not reconstruct a question from a criterion |\n| `questionSha` | never from the source | stamped at conversion by `import_cases.py --stamp` |\n| `split` | rarely present | you choose, and freeze it at import |\n| `source` | how it arrived: the filename, or the log query | say `unknown` rather than guessing a provenance |\n| `golden` | see the classification table in `SKILL.md` | absent. A case with no golden is legitimate |\n| `expectedEntities` | almost never present | leave it out. Guessing which fields an answer needs invents a retrieval expectation nobody stated, and it scores as a retrieval miss forever. The one exception is a key you derived yourself (`SKILL.md`, Deriving a key yourself): then you are not guessing, you are recording what you used |\n\n`expectedEntities` is the one most worth leaving empty. A wrong `required` id\ncannot be delivered by any run, so it reads as a model failure on every case\nit touches: five copied ids cost a real set two days.\n\nOn `set.json`: `name`, `description`, `datasetVersion: 1`,\n`targetModelPath`, `targetPackage` (the package under test -- it arms the\nguard that refuses a \"truth\" server also serving it), and a `sourceNote` saying\nwhere the questions came from and what shape they arrived in. `truthPackage` is\nusually absent at import, and until it exists no golden can reach `verified`;\n`verify_goldens.py --promote` is what takes it there afterwards. `init_truth_package.py` in\n`skill:eval-answer` scaffolds one.\n\n## Their field names will not be your field names\n\nAn arriving JSONL is somebody else's schema. Map it explicitly rather than\ntrusting a name that looks familiar, and write down the mapping you used in\n`sourceNote`. Names seen in real sets:\n\n| theirs | usually means | trap |\n|---|---|---|\n| `prompt`, `query`, `input`, `nl_question` | the question | `query` is the trap: in half of these it is the SQL or Malloy they ran, not the question |\n| `answer`, `expected`, `output`, `ground_truth` | a golden value, prose, or both mixed in one string | a paragraph under `answer` is criteria, not a value. Classify it, do not store it as a number |\n| `sql`, `malloy`, `gold_query` | their derivation | run it (`SKILL.md` step 3). This is the field that upgrades a typed number into something checkable |\n| `criteria`, `rubric`, `assertions`, `must_include` | prose criteria | apply the query test in `SKILL.md` step 4 to EACH one, not to the field |\n| `id`, `case_id`, `uuid` | their id | keep it, prefixed. Two sets from one customer collide otherwise |\n| `difficulty`, `category`, `tags` | their own labels | keep as-is on the case for slicing. Never let one become `split` |\n\nA field you cannot map is a question for its author. Keep it verbatim under a\n`sourceFields` object on the case rather than discarding it or forcing it into\na field it does not mean.\n\n## The four formats\n\n**JSONL.** One object per line, and the easy one. The failures are all in the\nmapping above, plus two mechanical ones: a trailing incomplete line, and a\n`\\n` inside a question that a naive line split breaks. Parse it as JSON per\nline, and count the lines you produced against the lines they sent. Report the\ncount in the import summary: an import that silently dropped 3 of 50 is a\nmeasurement on 47 cases claiming to be one on 50.\n\n**CSV or a spreadsheet export.** Questions contain commas, quotes and newlines,\nso use a real CSV reader, never a split on `,`. Watch for a header row that is\nnot row one (an exported sheet often has a title and a blank line above it), for\nmerged cells arriving as empty strings that belong to the row above, and for a\nnumber that arrived as text with a thousands separator or a currency symbol. A\nnumber you cannot parse cleanly is a string; keep it as their text and say so,\nrather than reading `1,234` as `1.234` or as `1234` without checking.\n\n**A markdown doc or an email thread.** The questions are prose among other\nprose, so the hard part is deciding what IS a question. Rules that hold up:\na heading or a numbered list item is usually a case boundary; a sentence\nending in a question mark inside a paragraph about something else usually is\nnot. Criteria attached to a question tend to follow it as a bullet list or a\n\"should\" sentence. Quote a boundary you are unsure of and ask, rather than\nsplitting on your best guess: a question invented by a bad split is\nindistinguishable, later, from one a human asked. Nothing in this format is\nbyte-clean, which makes the verbatim copy in `as-received/` load-bearing.\n\n**A pull from production logs.** The best source, because real traffic asks\nwhat people actually ask, and the messiest. Three things to do:\nde-duplicate near-identical prompts before you import 40 of one question;\nstrip the parts that are not the question (a system preamble, a UI-injected\nscope line, a pasted table); and keep the trace or request id on the case as\n`source`, so a case can be taken back to the session it came from. Almost\nnone of these arrive with a golden, which is fine: bare questions are a\ncoverage measurement. Where the log also holds the query the agent ran, that\nis NOT a golden. It is what an unevaluated agent did, and importing it as a\nkey scores the model against its own past behavior.\n\n## A worked line\n\nTheirs:\n\n```json\n{\"case_id\": \"q-014\", \"prompt\": \"How did denim sell last quarter versus the one before?\",\n \"sql\": \"SELECT ...\", \"answer\": \"Up 12.4% to 1.83M. Should break out by channel.\",\n \"difficulty\": \"hard\"}\n```\n\nYours, after applying step 3 and step 4:\n\n```json\n{\"qid\": \"ecom-q-014\", \"question\": \"How did denim sell last quarter versus the one before?\",\n \"questionSha\": \"<stamped>\", \"split\": \"dev\", \"source\": \"questions.jsonl\",\n \"golden\": {\"status\": \"provisional\", \"kind\": \"scalar\", \"value\": 1830000,\n            \"canonicalQuery\": \"SELECT ...\", \"verifiedBy\": \"authored_query\",\n            \"rubric\": \"Reports the quarter-over-quarter change for denim, and breaks the figure out by channel.\"},\n \"sourceFields\": {\"difficulty\": \"hard\"}}\n```\n\nAnd one that arrived as criteria alone, with no number anywhere:\n\n```json\n{\"qid\": \"ecom-q-021\", \"question\": \"Which channels are underperforming?\",\n \"questionSha\": \"<stamped>\", \"split\": \"dev\", \"source\": \"questions.jsonl\",\n \"golden\": {\"status\": \"verified\", \"kind\": \"criteria\",\n            \"verifiedBy\": \"authored_criteria\",\n            \"rubric\": \"Names the channels and says what it measured underperformance against. Does not use list price.\",\n            \"mustNotUse\": [\"retail_price\"]}}\n```\n\nNo query settles \"underperforming\", so there is nothing to derive and nothing\nthat can be numerically wrong: the clauses are the whole key, and this case\nscores on the first run.\n\n`criteria` and `unanswerable` are the two kinds an import may mark `verified`,\nand they arrive looking alike. A criterion reading \"PASS: a refusal that names\nthe missing data\" belongs to the second: the key is that the data is absent,\nso `kind: unanswerable`, and filing it as `criteria` beside a value would make\na correct refusal fail.\n\nWhat happened to that one `answer` string: the number became a provisional\nvalue, checkable because they sent the query that produced it; \"should break\nout by channel\" is a shape nothing can settle with a query, so it became\nrubric prose and scores on day one; and \"Up 12.4%\" is a second figure the\nrubric now has to state consistently with the value, which\n`verify_goldens.py` reads the accepting clause for."
     },
     {
       name: "eval-improve",
@@ -319204,7 +320167,8 @@ all of them for every run is how a skill stops being read.
 | a golden is wrong, doubted, or out of step with the model | \`reference/golden-side-door.md\` |
 | auditing a key you doubt, or a set you did not author | \`reference/auditing-an-answer-key.md\` |
 | deciding whether an edit stays | \`reference/acceptance-check.md\` |
-| about to quote a number, or set the noise band | \`reference/measurement.md\` |
+| about to quote a number, set the band, or read a set's flips | \`reference/measurement.md\` |
+| the run finished and someone has to read it | \`skill:eval-report\` |
 | you changed judge doctrine or its inputs | \`reference/checking-the-judge.md\` |
 
 Read the file, do not work from the summary here. The acceptance-check rules and
@@ -319221,10 +320185,22 @@ produces a confident wrong answer rather than an error.
 | **d. improve** | One smallest model edit, then the acceptance check | improve writes \`candidate\`; you write \`acceptance_check\`. Revert on reject. |
 | **e. checkpoint** | Git commit after an accepted acceptance check | \`checkpoint\` event, then the commit |
 
+Every run then **reports** (below). A run whose result exists only as JSONL and
+a scrolled-away console summary has not been delivered.
+
 **scrape** and **run** share a letter but are not the same job. Scrape writes
 cases. Run writes attempts. Do not invent questions and score
 them in one breath.
 
+**Step d is \`skill:eval-improve\`, not a hand edit plus another arm.** The
+failure mode is specific and it has happened: the edits were made by hand,
+published, and validated by re-running the whole arm -- with the answer key
+repaired in the same window. No \`candidate\`, \`acceptance_check\` or \`checkpoint\`
+event exists in that run's ledger, and the resulting move cannot be attributed
+to the model, the key, or the answerer. Re-running a whole arm after changing
+two things is the one thing the acceptance check exists to replace: it scores
+the edit against dev AND holdout, and it is cheaper than the arm.
+
 ### Scrape, minimally
 
 Importing an existing corpus IS the scrape step: copy the set from its home
@@ -319455,6 +320431,29 @@ the run measure something other than what it names:
    under a published version, which is what makes it a pin. Check which you have
    before deciding how much of this you need.
 
+   **Look for a set and for prior runs before you author either.** A minute of
+   \`find . -name cases.jsonl\`, a glance at \`evals/*/runs/\` and at your host's
+   own transcripts for this repo. Two sessions fourteen minutes apart built the
+   same 29-case answer key from scratch, because the first had committed
+   nothing before it was deleted and the second had no way to know it existed.
+   Roughly a working day was spent twice, and three specific things were
+   rediscovered at cost: the MCP login flow, the retrieval gate's 401, and a
+   judge-rendering bug that had already cost $17 of arm once. The two keys,
+   independently authored from the same questions, differ by about ten points
+   on comparable answers -- which is the available measure of how much a key
+   depends on its author, and a reason to reuse one rather than rebuild it.
+
+   **Audit the set's entity ids before the first arm.** \`verify_goldens.py\`
+   checks that each id names something in the model; \`check_findable.py\` checks
+   that a search of its own kind actually returns it. Both are free of model
+   calls. An id that fails either scores a retrieval miss on every run, and the
+   miss reads as the model's fault.
+
+   **Commit the set before you spend money on an arm**, and keep durable
+   outputs in the repository. A findings document in \`~/Downloads\` is gone the
+   first time somebody tidies up; the set, the run directory and the write-up
+   belong in git beside the model.
+
 2. The server must be up with retrieval tracing on, so a call's ranked results
    can be recovered afterwards (open-source Publisher: \`PUBLISHER_MCP_TRACE=retrieval\`).
    Confirm a trace lookup is available (absent means tracing is off).
@@ -319495,6 +320494,44 @@ the run measure something other than what it names:
    whole run. Raising a call budget mid-run moved mean outcomes on an unchanged
    model.
 
+   The call budget is \`--max-turns\`, written to \`run.json\` as \`maxTurns\`.
+   **Size it from a pilot rather than taking the default of 30.** Run the three
+   cheapest cases uncapped (\`--max-turns 100\`) and set the cap at twice their
+   maximum. A set whose questions need two or three sources joined does not fit
+   a cap sized for single-source lookups: on one such set the completed
+   attempts had a median of 17 turns and a 90th percentile of 26 against a cap
+   of 30, and four cases died at it. A cap 15% above the 90th percentile of
+   completed work is not a safety margin. Note also that \`malloy-analysis\` tells
+   the answerer to persist -- retry a phrasing, let a small query settle whether
+   a field exists -- so a tight cap and that instruction are in direct conflict.
+
+   **Do not change the model and the measuring instrument in the same step.**
+   Those are two different axes and only one of them is cheap to separate.
+
+   Batching MODEL edits is fine and expected. Clustering exists so that one
+   edit closes several cases, and an arm per fix does not survive contact with
+   arithmetic: 20 fixes over 100 cases is 2,000 answers, and at the measured
+   $0.33 a case on a proxied warehouse that is $660 of answering to attribute
+   what the acceptance check attributes for the price of the affected cases
+   plus holdout. Budget five arms for a defensible claim (a baseline, two for
+   the A/A, two post-edit), not one per edit, and let \`skill:eval-improve\`
+   carry each cluster.
+
+   The instrument is the other axis: the answer key, the judge and its prompt,
+   the answerer model, the skills. Move one of those together with the model
+   and there is nothing left holding still, so the result measures neither. On
+   the run this comes from, nine model commits and a re-derived answer key
+   landed between two arms, and the move from 20% to 39% belongs to no one --
+   not because two model edits were batched, but because the ruler changed at
+   the same time as the thing being measured. A key repair mid-improve is not
+   forbidden; it ends that comparison, so re-baseline rather than quoting a
+   delta across it.
+
+   The answerer model is instrument too, and it is the one most often left
+   unstated. The same 23-case set read 12 match / 11 near / 0 no_match on one
+   model and 8 / 4 / 15 on a smaller one. No statement about "the agent's"
+   capability means anything until two arms name the same answerer.
+
 7. Generate every answerer prompt from the stored case in \`cases.jsonl\`.
    Never retype the question. A truncated retype is indistinguishable from a
    real question downstream.
@@ -319515,6 +320552,31 @@ the run measure something other than what it names:
 6. \`skill:eval-improve\` only when this run includes improve, and only for
    \`owner: model\`. Then run the acceptance check. On accept, checkpoint.
 
+## Report the run, or nobody can read it
+
+A run directory is JSONL. It is a record, not a result, and the console summary
+scrolls away. **Every run ends by producing something a person can open**, and
+that job is \`skill:eval-report\`: it builds the servable run package (the case
+matrix app and the aggregate notebook) and gives the template for the write-up.
+
+Read it at the end of every run, including a run that failed. The standing
+complaint about this loop is that "a bunch of stuff happens and it is hard to
+know the actual results", and a ledger nobody renders is why.
+
+Two rules from it are worth repeating here, because they are the ones a
+conductor skips:
+
+- **Separate a MODEL failure from an EVAL failure.** A wrong answer and a
+  broken measurement look identical in a pass rate and have nothing else in
+  common. An arm holding a truncated, contaminated or environment-failed
+  attempt has no rate to quote at all.
+- **Say when a step did not run, and why.** \`improve\` not running because every
+  cluster came back \`owner: agent-skill\` is a RESULT, and it reads identically
+  to having forgotten unless it is written down.
+
+Keep the write-up in the repository beside the set, not in a chat log and not
+in \`~/Downloads\`.
+
 ## Checkpoint
 
 A checkpoint is a git commit of the model repository, taken after an acceptance check
@@ -319564,7 +320626,16 @@ conductor. Do not:
 
 - The model is the only thing improve edits. No question text, qids, or
   expected values in any name, doc, or comment.
-- When the environment misbehaves, stop. Never diagnose a sick system.
+- When the environment misbehaves, stop. Never diagnose a sick system. The
+  harness is part of the environment: a known-broken measurement does not
+  become quotable by being finished. Measured against this directive, an arm
+  already paid for gets quoted anyway -- it happened three times in one run,
+  once with the confound stated in the same message that started the arm -- so
+  the harness now enforces the parts it can. A run with a truncated or
+  contaminated attempt prints no pass rate and records \`status: incomplete\`,
+  and \`flip_table.py\` names what one arm left unscored that the other did not.
+  Read \`run_error\` on the attempts
+  and the INCOMPLETE line before quoting any number.
 - When a subagent disagrees with you, probe. Do not win by authority.
 - When a rule here is wrong, change this file and note it on the run.
 
@@ -319575,6 +320646,7 @@ conductor. Do not:
   the judge.
 - \`skill:eval-diagnose\`: component, owner, issue events. No edit.
 - \`skill:eval-improve\`: smallest model edit, probe receipts, no self-accept.
+- \`skill:eval-report\`: the run package and the write-up a person reads.
 - The \`malloy-analysis\` skill: what the blind answerer follows. It is installed
   from the \`analysis\` manifest group, not the \`eval\` group.
 
@@ -320121,7 +321193,7 @@ If later evidence makes one replacement obvious, then Repair a bad golden.`
     {
       name: "eval-loop/measurement",
       description: "Measurement. Reference detail for the eval-loop skill.",
-      body: `<!-- Sampling, the flip-count bar, the A/A noise band, and targeted fixes. Read this before quoting any number. -->
+      body: `<!-- Sampling, the flip-count bar, the A/A band, reading the disagreement set, and targeted fixes. Read this before quoting any number. -->
 
 # Measurement
 
@@ -320156,9 +321228,49 @@ measure a baseline and diagnose failures but cannot defend an edit.
 This table was asserted, not measured, and the number it needs is a property of
 your harness and your set -- not of this skill. Measure it with an **A/A run**:
 the same model, same config, same set, twice, compared with
-\`scripts/flip_table.py\`. Every flip it reports is noise by construction, since
-nothing changed. Record the result with the set, in \`CALIBRATION.md\`, and cite
-that file when you quote a band.
+\`scripts/flip_table.py\`. Record the result with the set, in \`CALIBRATION.md\`,
+and cite that file when you quote a band.
+
+An A/A flip is unattributable to a CHANGE, because there was no change. That
+is not the same as saying it carries no information, and reading it as "noise
+by construction" throws away the most useful evidence in the run. See below.
+
+## Read the disagreement set before you average it
+
+**First move on a flip set: read the flipping cases. Second move: establish
+the band.** Both, in that order. The band alone is half the information.
+
+A case that flips is a case where the agent found two paths and the model did
+not make one of them obviously right. That is a MODEL-quality signal, and
+often a cheaper one than a stable failure, because two runs hand you a matched
+pair: same question, same model, one right answer and one wrong one, with the
+diff between them isolating the cause.
+
+Measured, on a real set. Two arms of one model over 28 cases disagreed on
+five. Read as a floor, that is "this set has 18% churn, plan around it". Read
+as evidence, it was a single named model defect:
+
+- all five were the same question -- does a blank field on a fee rule match
+  every value?
+- the two arms produced two discrete query idioms, not scattered answers
+- the wrong idiom returned 2 fee IDs where the answer has 443, and 1 where it
+  has 390
+- the two runs swapped: run 1 was right on one case and wrong on the other,
+  run 2 the reverse
+
+A model that documents a rule in prose while leaving an inviting wrong path
+reachable produces exactly this: a bimodal coin flip that averages to "noise".
+The variance WAS the finding. Averaging the five would have hidden it, and
+reading them took a day of manual transcript work that the table now does --
+\`flip_table.py\` prints the two final queries side by side for every flipped
+case, and \`diagnose.py --compare-run <other arm>\` hands the pair to the
+diagnosing agent.
+
+Some flips really are noise, and a set with no band cannot tell an improvement
+from a wobble, so keep the band. What changes is which you do first, and what
+you call the thing before you have looked at it. Prefer **disagreement set**
+or **unstable cases** to "churn": the first two name a question, the last
+names an answer you have not earned yet.
 
 Re-measure whenever the model, judge, or set changes. This is not a formality:
 observed bands have moved by a factor of three across a fortnight of ordinary
@@ -320321,7 +321433,11 @@ python3 skills/eval-loop/scripts/serve.py --publisher-dir <publisher>/packages/s
 #     every run. Pass --model on a platform target too, from a checkout of the
 #     served version, since the harness cannot see the model text there.
 #     Exits 3 when set.json names no truthPackage: the audits ran, no
-#     golden was re-derived, and 0 would have claimed otherwise.
+#     golden was re-derived, and 0 would have claimed otherwise. With
+#     --definitions <ledger> (verify_definitions.py), a set whose every
+#     value-bearing case rests on validated definitions exits 0 instead:
+#     the composition rule, values not re-derived but their definitions
+#     checked.
 python3 skills/eval-answer/scripts/verify_goldens.py \\
   --set <repo>/evals/ecommerce --publisher http://localhost:4881 \\
   --model <repo>/ecommerce --target-package ecommerce
@@ -320444,6 +321560,366 @@ Two failure modes worth pre-empting, because both produce a clean-looking run:
 - **Check the served revision is the one you edited.** Publisher serves a
   snapshot copy, so a model fix can be absent from the run that is supposed to
   measure it. Query the changed measure once before spending an arm on it.`
+    },
+    {
+      name: "eval-report",
+      description: "Write the summary a person reads after an evaluation run: what ran, what it scored, which failures were the MODEL and which were the EVAL itself (out of turns, contaminated, unestablished golden, unreadable judge), what was learned, and what to do next. Every claim links to the artifact behind it. Use at the end of any run, before quoting a number to anyone. Never scores an answer (eval-answer) or decides who owns a failure (eval-diagnose).",
+      body: `# Report a Run
+
+A run directory is JSONL and a console summary that scrolls away. Neither is a
+result. This skill turns one into something a person can open, and it is the
+last step of every run, including a run that failed.
+
+The complaint this exists to answer, in a reviewer's words: *"eval runs and a
+bunch of stuff happens and it is hard to know the actual results."*
+
+**Scope boundary:** this skill reports what the ledger already says. It does not
+score an answer (\`skill:eval-answer\`), decide who owns a failure
+(\`skill:eval-diagnose\`), or edit a model (\`skill:eval-improve\`). If a number is
+not in the ledger, do not put it in the report.
+
+**The report is about THIS RUN, not about the harness.** Defects you find in
+the eval tooling while running it are real and worth filing, and they do not
+belong here: the reader wants to know what their model scored and why, not what
+is wrong with the thing that measured it. File those against the harness. The
+one exception is anything that qualifies THIS run's number -- a truncated
+attempt, a contaminated one, an unestablished answer key -- which the "Eval
+failures" section exists for.
+
+## Step 1: build the artifacts, before writing a word
+
+\`\`\`bash
+python3 skills/eval-loop/scripts/build_run_package.py \\
+    --run <run-dir> --set <set-dir> --out /tmp/eval-<label>
+curl -sS -X POST http://<publisher>/api/v0/environments/<env>/packages \\
+    -H 'content-type: application/json' \\
+    -d '{"name":"eval-<label>","location":"/tmp/eval-<label>"}'
+\`\`\`
+
+That builds a Malloy package over the run's own CSVs and registers it with no
+restart. It gives you two things to link:
+
+| Artifact | What it is | URL |
+|---|---|---|
+| The case matrix app | Every question, its verdict, which needed entities retrieval delivered, and a drawer per case holding the reference answer, the judge's reasoning, the re-executed rows and every query the answerer ran | \`<publisher>/environments/<env>/packages/eval-<label>/\` |
+| \`eval_run.malloynb\` | The aggregate tables: pass rate, effort, cost, most-missed entities, the backlog | \`<publisher>/<env>/eval-<label>/eval_run.malloynb\` |
+
+**Those two URLs are in different path spaces, and guessing costs a 404.** The
+app is served by the in-package \`public/\` handler, which owns
+\`/environments/<env>/packages/<pkg>/<file>\`. The notebook is a MODEL, rendered
+by the Console, whose routes are \`/<env>/<package>/<model path>\` with no
+\`environments\` or \`packages\` segment at all. Putting the notebook under the
+app's prefix 404s, because the public-file handler answers there and the
+notebook is not in \`public/\`. Verified both, on a real package, by opening
+them.
+
+Pass \`--run\` more than once to put two arms side by side.
+
+## Step 2: write the report
+
+Put it in the repository beside the set, as \`RESULTS.md\`. Not in a chat log,
+not in \`~/Downloads\`, both of which have lost a findings document before.
+
+**Derive every number from \`events.jsonl\` and \`run.json\`, not from memory or
+from the console.** A figure retyped from scrollback is a figure nobody can
+check, and the console rounds.
+
+### The template
+
+\`\`\`markdown
+# <set> -- <label>
+
+<one sentence: what was being measured, against what, and why now>
+
+## What ran
+
+- Set \`<name>\` v\`<datasetVersion>\`, N cases (N dev, N holdout)
+- Target \`<env>/<package>\`, model \`<modelPath>\`, pinned \`<modelSha or targetVersion>\`
+- Answerer \`<model>\`, judge \`<model>\`, cap \`<maxTurns>\` turns
+- Steps run: scrape/run, eval<, diagnose><, improve>
+- Cost $X answerer + $Y judge
+
+## The result
+
+**N of M decided (P%).**   <-- or: **No pass rate.** See "Eval failures" below.
+
+| qid | question | verdict |
+|---|---|---|
+
+[case matrix](<url>) - [aggregate tables](<url>)
+
+## Retrieval and coverage
+
+Covered? -> Retrieved? -> Correct?, with the per-arm numbers under each.
+
+## Model failures
+
+One entry per wrong answer. What it got wrong in plain words, then the
+mechanism, then the link.
+
+## Eval failures
+
+What went wrong with the MEASUREMENT rather than the model. Empty is a real
+and good answer; say "none" rather than dropping the section.
+
+## Query errors
+
+Malloy the answerer wrote that would not run, and whether it recovered.
+
+## What this run taught us
+
+## What to do next
+\`\`\`
+
+### "What to do next" is a checklist, and most of it is derived
+
+Do not invent this section. Most of it falls out of what the run did NOT do,
+and a reader should be able to see that nothing was skipped silently. Walk
+these in order and put every one that fires into the list, with its command:
+
+| If the run shows | Then the next step is |
+|---|---|
+| \`coverage: unmeasured\` | run \`check_coverage.py --set <set> --model <pkg> --out coverage.json\`, then re-run with \`--coverage\` so it charges the failures |
+| \`goldenCheck: skipped\` or the set names no \`truthPackage\` | build one with \`init_truth_package.py\`; until then the goldens were derived through the model under test and certify themselves |
+| any golden still \`provisional\` | re-derive and \`verify_goldens.py --promote\` |
+| a stale entity name warning | fix \`expectedEntities\`; it scores as a retrieval miss on every run until you do |
+| a passing case with recall below 1.0 | check whether \`required\` over-specifies one path |
+| \`truncated\` non-empty | re-run those cases at a higher cap with \`--from\` |
+| diagnose did not run | run it, or say the failures have no owner yet |
+| a cluster with \`owner: model\` | \`skill:eval-improve\`, then the acceptance check |
+| a cluster with \`owner: agent-skill\` | **edit that skill.** This is NOT a dead end |
+| a cluster with \`owner: dataset\` | the golden side door in \`skill:eval-loop\` |
+| only one arm exists | note that no noise band has been measured for this set yet |
+
+**An \`agent-skill\` cluster is work, not an absence of work.** \`eval-improve\`
+may not touch it, and writing "nothing to do, the model is fine" there is how a
+real defect gets closed as a non-finding. Name the skill, name the rule to add
+or change, and say who owns it. The model being innocent is a statement about
+the model, never about the run.
+
+### Write it for someone who was not there
+
+The reader is a colleague who did not run this and does not know the loop's
+vocabulary. Two rules, both learned by handing a report to one:
+
+- **Never make a bare count carry the meaning.** "The only cluster is
+  \`owner: agent-skill\`" tells a reader nothing: what is a cluster, and what
+  follows from it? Write what happened and what to do: "The one wrong answer
+  came from the agent picking the wrong kind of field, so the fix is in the
+  analysis skill, not in the model."
+- **Do not open with a negation of something you just reported.** A section
+  that says "nothing to do" directly under a section reporting a wrong answer
+  reads as a contradiction, and the reader stops trusting both. Lead with the
+  wrong answer and what closes it; put anything genuinely needing no action
+  after that, and say why.
+
+## The two failure sections, and why they are separate
+
+This is the part most reports get wrong. A wrong answer and a broken
+measurement look identical in a pass rate and have nothing else in common: one
+is work for the model owner, the other is work for whoever runs the harness.
+Mixing them sends a modelling agent to fix a model that is fine.
+
+**Model failures** are cases that were fairly asked, fairly answered and got the
+wrong answer. One entry each: what the answer said, what was right, and the
+mechanism in one sentence. Do not write a cluster id here; write what it got
+wrong.
+
+**Eval failures** are everything that stopped the run measuring the model.
+Report every one that occurred, with its count and its qids, and say plainly
+that these are NOT evidence about the model:
+
+| What happened | How it reads in the ledger | Who fixes it |
+|---|---|---|
+| Answerer ran out of turns | \`reason: answerer_truncated\`, and \`run.json\` \`truncated\` | raise \`--max-turns\`, re-run those cases with \`--from\` |
+| Isolation breached | \`reason: contaminated\`, \`contamination_reasons\` on the attempt | harness config; the answerer held a tool it should not |
+| Harness or server failed | \`reason: environment_failure: <what>\` | fix the environment, re-run the arm |
+| Answer key not established | \`reason: golden_provisional\` / \`_invalid\` / \`_ambiguous\` / \`_missing\` | derive the key (\`skill:eval-import\`), or the golden side door |
+| Judge reply unreadable | \`reason: judge_unparseable\` | it already retried; read \`artifacts/<qid>/judge.md\` |
+| Judge doubted the key | \`gold_status\` \`suspect\` / \`verified_wrong\`, \`run.json\` \`doubtedGoldens\` | the golden side door in \`skill:eval-loop\`, NOT improve |
+| Rubric quoted a stale figure | \`verify_goldens.py\` check 2 review items | repair the rubric against its own golden rows |
+| Arm stopped early | \`run.json\` \`status: aborted\` | four consecutive dead attempts; fix and re-run |
+
+**If any of the first three occurred, there is no pass rate to quote.** The
+harness prints \`INCOMPLETE\` and withholds the percentage; the report does the
+same. Give the counts and the re-run command instead of a number with a caveat,
+because the number is what gets repeated and the caveat is what gets dropped.
+
+## Retrieval and coverage belong in every report
+
+A pass rate says an answer was wrong. It does not say WHERE, and the three
+metrics that do are already in the run summary and the notebook. Report them,
+because a report that omits them makes every failure look like the model's:
+
+**Copy the cascade the run printed. Do not re-derive it.** \`cascade_lines()\`
+in \`run_baseline.py\` already renders every rung, and it carries one field a
+hand-made table keeps losing: how many cases **answered correctly anyway**
+after stopping on that rung. Paste its block into the report.
+
+**A finer measurement must never revise the headline result.** This is the rule
+the re-derivation breaks. Written as a funnel with a shrinking denominator --
+\`5 of 10 covered\`, then \`3 of the 5\`, then \`3 of the 3\` -- the last rung reads
+as "only 3 of 10 succeeded" on a run where **all ten answers were correct**.
+That report went to a reviewer, and the objection was the right one: adding
+detail about retrieval cannot turn a 10 of 10 into a 3.
+
+Two things stop it:
+
+- **Report every rung over all N cases**, never over the survivors of the rung
+  above. The rungs are three measurements of the same cases, not a narrowing of
+  them.
+- **\`delivered, right\` is NOT the pass rate.** The passing cases are
+  \`delivered, right\` plus \`passed_not_covered\` plus \`passed_not_retrieved\`. A
+  report that omits those last two has dropped the only numbers that reconcile
+  the cascade with the score.
+
+**Label a rung for what it measures, not for what a reader will assume.**
+"Can the model express an answer?" is wrong for most of what lands on the \`no\`
+side of the first rung. \`CONVENTION\` means the data is present and no named
+measure encodes the statistical or business convention; \`NO-DISAMBIG\` means two
+plausible candidates and no doc saying which the question means. The model can
+express an answer in both -- it does not say WHICH answer is meant. So the rung
+asks whether the model NAMES what the question needs.
+
+**Give the per-entity number too, and label both.**
+
+- per CASE, the cascade: did this case get everything it needed
+- per ENTITY, recall: of the N entities the answers depended on, how many were
+  delivered
+
+They answer different questions, and the entity number is the more natural read
+of "how did search do". One run reported \`3 of 5\` cases and buried \`16 required
+entities, 80% delivered\`, and the case number was the only thing a reader saw.
+
+- **Covered?** Can the model express a correct answer at all? Not computed by
+  the run: \`check_coverage.py\` reads the MODEL rather than the answers, and the
+  run consumes its report through \`--coverage\`. **If it was not run, say
+  \`unmeasured\` rather than leaving the row out** -- an unmeasured coverage
+  label is not evidence that the model covers the question.
+- **Retrieved?** Of the entities the golden answer depends on, how many did
+  \`get_context\` hand back. Quote \`required_count\`, \`delivered_count\` and how
+  many were ranked rather than merely mentioned in a returned source's docs.
+- **Correct?** The pass rate, which is the rung the other two qualify.
+
+Two numbers here are routinely misread, so qualify them in the report or leave
+them out:
+
+- **Entity precision is a PAYLOAD number, not a retrieval-quality one.** Report
+  it as what it measures: how much context the agent was handed against how
+  much it needed. 52 entities returned per attempt for the 2 an answer used is
+  a real cost, in tokens and in attention, and it is worth tracking. What it
+  cannot tell you is whether retrieval worked, because the denominator is
+  everything returned and it ignores rank: an entity that came back at rank 3
+  of 51 scores identically to one at rank 51.
+- **For quality, report rank.** "Required entities came back at median rank 3,
+  9 of 16 in the top 5" is a statement about retrieval. Precision alone reads
+  as an indictment of it and is mostly a statement about how many fields the
+  package has.
+- **Report the misses split by cause, not as one recall figure.** The run
+  attributes each to \`never asked\` (no target of a type that could return it --
+  deterministic, because \`target_type\` is a hard filter on the server, so no
+  documentation could have delivered it) or \`not retrieved\` (a compatible target
+  was issued and it still did not come back, which is the docs or the wording,
+  and diagnose separates them). Those have different owners and one recall
+  number hides which you have.
+- **A miss on a case that PASSED is still a miss, and the report says so.** The
+  answer was right by another route; name the route. Measured on one run, 3 of
+  4 findings were on passing cases and the route was always the same: the agent
+  rebuilding the model's own measure inline, which held for \`count()\` and failed
+  on the one measure carrying a grain rule.
+- **Read the entities that did NOT come back, and what was asked for.** That is
+  where the retrieval signal actually is, and the misses are rarely independent.
+  Measured on one run: 5 required entities never came back as ranked results,
+  and 4 of the 5 had the same cause -- the agent sent only \`source\` and
+  \`dimension\` targets, and \`target_type\` is a hard filter, so no measure could
+  be returned however well the model documents it. One defect, five symptoms,
+  and the same one that produced the run's only wrong answer. A per-case list of
+  misses beside what was asked for would have shown it in a glance.
+- **A PASSING case with recall below 1.0 is usually an expectation defect**, not
+  a retrieval miss: the set named one path to an answer the agent reached by
+  another. Report the count and read it as a prompt to check \`required\`.
+
+## Query errors are worth a section of their own
+
+Malloy the answerer wrote that would not compile or run, whether or not the
+case passed. These sit on \`tool_call\` events as \`error\`, and nothing else
+surfaces them: a case that errored twice and then recovered scores exactly like
+one that got it right first time, so the effort disappears.
+
+They are the sharpest available evidence on whether the skills and the docs are
+leading agents astray, because each one names a specific thing the agent
+believed and the language does not support. Report the count, the distinct
+error kinds, and for each whether an existing skill already covers it. An error
+whose fix IS documented, in a skill the answerer did not open, is a finding
+about the skills rather than about the agent.
+
+\`eval-diagnose\` does not see these today: it reads failing cases only, so an
+error inside a passing case is invisible to it. Say so rather than implying
+they were triaged.
+
+## Say what did not run, and why
+
+A step that did not run is a result, and it is indistinguishable from having
+forgotten unless it is written down. The common one: \`improve\` does not run when
+every cluster came back \`owner: agent-skill\` or \`owner: dataset\`, because
+\`skill:eval-improve\` may only touch \`owner: model\`. That means the model is not
+at fault, which is worth a sentence rather than a silence.
+
+Same for diagnose. If it did not run, no failure in this report has an owner
+yet, and the report says so rather than implying the failures are the model's.
+
+## Findings and improvement ideas
+
+Two different things, kept apart.
+
+- **Findings** are what this run established, each with the evidence beside it.
+  A finding with no artifact behind it is an opinion.
+- **Improvement ideas** are what somebody might do about them, tagged with who
+  owns each: the model, the answer key, the analysis skills, or the harness.
+  An idea is a proposal, not a decision, and the report does not pretend a
+  cluster has been triaged when it has not.
+
+Rank by how many cases each would move, and say when the answer is "nothing":
+a 90% run whose one failure is a known fan-out trap may need no action at all.
+
+**Report facts, do not instruct the reader.** "This set has had one arm, so no
+noise band exists for it" is a fact they can act on. "Do not quote 90% until X"
+is a lecture about their own number, and they did not ask for one. State what
+was and was not measured; what to do with it is theirs.
+
+## Make the links clickable
+
+A report whose evidence cannot be opened is a report nobody checks.
+
+- **Absolute paths**, never relative ones. A reader is not in your working
+  directory, and a terminal only makes an absolute path clickable.
+- **The served URL** for the app and the notebook, with the real host and port
+  the run used, not a placeholder.
+- Per case, link the FILES, not the directory: \`artifacts/<qid>/answer.md\`,
+  \`artifacts/<qid>/judge.md\`, \`artifacts/<qid>/answerer.jsonl\`. A \`file://\`
+  link to a directory opens nothing in most editors, which is a dead link that
+  looks live.
+- **Move the run somewhere durable before you link it.** A run written under a
+  session scratch directory (\`/tmp/...\`, a host's per-session sandbox) is
+  deleted when the session ends, and many editors will not linkify a path there
+  even while it exists. Copy the run directory, artifacts included, next to the
+  report under the set, then link that. A report whose evidence is in a temp
+  directory is a report with no evidence by next week. This was caught twice by
+  a reader clicking and getting nothing.
+- Write a local path bare, as \`/abs/path/to/file\`, not as \`[text](file:///...)\`.
+  Terminals and editors linkify a bare absolute path; a \`file://\` markdown link
+  is frequently inert, and an inert link is worse than the path in plain text
+  because the reader has nothing to copy.
+- Never link a run LABEL or a qid as though it were a path. \`faa-v1-baseline-01\`
+  is a label; the path is the run directory. A backticked label that looks like
+  a link and resolves to nothing is worse than plain text, and it has already
+  wasted a reader's time.
+
+## Related skills
+
+- \`skill:eval-loop\`: conducts the run this reports on.
+- \`skill:eval-answer\`: the verdicts and the ledger schema this reads.
+- \`skill:eval-diagnose\`: owners and clusters, when it ran.`
     },
     {
       name: "malloy",
@@ -320502,6 +321978,7 @@ This table is a catalogue of what exists, not of what is loaded. A host that ins
 | \`eval-improve\` | The smallest model edit for a diagnosed cluster, with probe receipts |
 | \`eval-judge\` | Scoring one attempt against its golden, and saying whether the golden itself is believed |
 | \`eval-import\` | Turning questions that arrived from outside into a set, before the first run against it |
+| \`eval-report\` | Turning a finished run into something a person can open, and saying what it did and did not measure |
 
 **Writing correct Malloy** (read before writing, not after failing)
 
@@ -320654,7 +322131,7 @@ End with a short **Next steps**: one or two specific deeper analyses the data co
     {
       name: "malloy-analysis-pitfalls",
       description: "Checks to run on a query and its numbers before a result is presented - wrong grain, fan-out, invented field names, mismatched filters, count vs distinct. The common data-analysis pitfalls.",
-      body: "# Data Analysis Pitfalls\n\nWatch for these common mistakes throughout the analysis workflow. When you encounter one, fix it before presenting results.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Query Construction\n\n### Wrong grain / fan-out\nUsing dimensions or measures from a joined source that has a finer grain than the base source can silently multiply rows, inflating aggregates. For example, aggregating revenue while grouping by a line-item field may double- or triple-count totals. If your query touches fields from a joined source, compare `count(key_field)` to `count()`: if the row count is significantly higher than the distinct key count, you likely have fan-out.\n\n### Invented entity names\nNever guess field names. Use only the exact field paths defined in the model (find them with `get_context`). A plausible-sounding name that does not exist in the model will produce an error, or worse, silently reference the wrong field.\n\n### Mismatched filter values\nDimensional values are case-sensitive and format-specific. Common mismatches include case differences (\"Nike\" vs \"NIKE\" vs \"nike, inc.\"), partial matches (\"New York\" when the data has \"New York City\"), and aliased values (\"USA\" vs \"United States\"). A filter on a value that doesn't exist in the data silently returns zero rows without erroring. Always use the exact dimensional values from retrieval results, and if in doubt, run a distinct-values query on the dimension to confirm.\n\n### Filtering on the wrong field\nIf the user asks to filter by \"brand\", confirm which dimension corresponds to \"brand\" in the model. There may be multiple fields with similar names at different levels of the hierarchy.\n\n### Missing filters\nIf the user asks about \"last quarter\" but you don't apply a time filter, you are returning all-time data. Always check whether the question implies filters you have not yet applied.\n\n### Misinterpreted entities\nA field can exist in the model and still be the wrong choice. For example, using `revenue` (which may be gross) when the question asks about profit, or treating a count measure as if it were a sum. Cross-reference the field definitions and `#(doc)` descriptions in the model to confirm a field means what you think it means.\n\n### Semantic ambiguity\nField names or descriptions sometimes suggest one interpretation while the actual values tell a different story, for example, a field labeled \"annual ridership\" containing values that look like average weekday traffic, or a field named `revenue` that appears to represent net revenue in practice. When values don't match expectations, note whether the ambiguity affects the answer and call it out.\n\n### Fragile ad-hoc definitions\nWhen defining a new measure or dimension inline, common mistakes include assuming a field is numeric when it's actually a string, ignoring nulls in arithmetic (e.g., `a - b` yields null if either is null), and building logic around values that only cover a subset of the data.\n\nPay special attention to value coverage when defining a dimension that categorizes or subsets data. It's easy to capture too few values (missing categories that should be included) or too many (grouping in values that don't belong). For example, a `pick` expression that maps tier names to \"Premium\" and \"Standard\" might miss a tier that should be Premium, or a filter meant to isolate one product line might inadvertently include related but distinct products. Before relying on such a definition, run a distinct-values query on the underlying field to see the full set of values and confirm your logic handles them all correctly. This applies equally whether you define the logic as a new dimension or apply it directly as a `where` clause: the risk of incomplete value coverage is the same either way.\n\n### Hidden filters in views\nViews can have built-in `where` clauses that pre-filter the data. A view named `recent_orders` might only include the last 90 days, or `active_customers` might exclude certain statuses. If the view's definition is available, read it to understand what filters are baked in; they may conflict with what the user is asking for. When in doubt, query the base source directly and apply filters explicitly.\n\n## Result Interpretation\n\n### Implausible magnitudes\nIf a \"total\" is suspiciously small or large, question it. Common causes: missing filters (too large), over-filtering (too small), wrong unit (dollars vs. cents), or fan-out from joins (inflated).\n\n### Nulls distorting aggregations\nNull values are silently excluded from `avg()` and can make `sum()` results lower than expected. If a significant portion of a field's data is null, aggregations over that field may be misleading. When results seem off, compare total row count to a count of non-null values for the key field to gauge how much data is missing.\n\n### Confusing count vs. count distinct\n`count()` counts rows while measures defined with `count(field)` count distinct values of that field. Using a row count when you need distinct values (or vice versa) is a frequent source of inflated or deflated numbers, especially when the query touches joined sources.\n\n### Percentage of what?\nWhen computing percentages or shares, be explicit about the denominator. \"30% of revenue\" means nothing if you don't confirm what the total revenue is and whether it's filtered the same way.\n\n### Time period mismatches\nComparing metrics across different time periods without normalizing (e.g., comparing a full year to a partial quarter) produces misleading conclusions.\n\n### Extremes of a rate on a tiny denominator\n\"Which one is worst\" ranked on a ratio hands the top spot to whichever group has the fewest rows: one failure out of three is a 33% failure rate, and it will outrank a group with thousands. Before reporting an extreme of any rate, look at what is underneath it. Apply the model's documented minimum volume if it has one; if it has none, pick a floor, say which floor you picked, and print the denominator next to the rate rather than the rate alone.\n\n## Verification Signals\n\n### Parts don't sum to the whole\nIf you break down a total by category, the categories should sum to the total (or close to it, accounting for nulls). If they don't, something is wrong with the grain or filters.\n\n### A total the model tells you not to take\nThe check above assumes the parts are additive. Sometimes they are not, and the model says so: a doc noting that one event can be logged on two layers, or that a measure is non-additive, is telling you the sum across that dimension double-counts. Do not print a total row across such a dimension, and least of all one the user did not ask for. If a total is genuinely wanted, say why the model does not support it and give the per-category figures.\n\n### Row count surprises\nBefore interpreting results, check whether the row count makes sense. An unexpectedly high row count often indicates fan-out from a join. An unexpectedly low count may mean an overly restrictive filter.\n\n### Zero or empty results\nIf a query returns no rows, don't report \"there is no data.\" First verify that your filters are correct and the field names are right. The absence of results is usually a query problem, not a data problem."
+      body: "# Data Analysis Pitfalls\n\nWatch for these common mistakes throughout the analysis workflow. When you encounter one, fix it before presenting results.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Query Construction\n\n### Wrong grain / fan-out\nUsing dimensions or measures from a joined source that has a finer grain than the base source can silently multiply rows, inflating aggregates. For example, aggregating revenue while grouping by a line-item field may double- or triple-count totals. If your query touches fields from a joined source, compare `count(key_field)` to `count()`: if the row count is significantly higher than the distinct key count, you likely have fan-out.\n\nThat check reads in ONE direction only. `count(joined.field)` is a distinct count evaluated at the joined source's grain, so for an ordinary many-to-one join it is much SMALLER than `count()`, and that is the expected, correct result -- not missing data. Ten thousand flights joined to four hundred aircraft give `count(aircraft.tail_num)` of 400, and nothing is wrong. Do not read a low value as a coverage problem and do not abandon the answer over it. **Coverage is measured at the base grain**, by counting base rows whose joined value is null:\n\n```malloy\nrun: flights -> {\n  aggregate:\n    all_rows is count()\n    missing is count() { where: aircraft.aircraft_models.seats = null }\n}\n```\n\nAn agent once refused a correct answer it had already computed because `count(aircraft.tail_num)` returned 36 against 17,875 flights; measured at the base grain, zero flights were missing a seat value.\n\n### Invented entity names\nNever guess field names. Use only the exact field paths defined in the model (find them with `get_context`). A plausible-sounding name that does not exist in the model will produce an error, or worse, silently reference the wrong field.\n\n### Mismatched filter values\nDimensional values are case-sensitive and format-specific. Common mismatches include case differences (\"Nike\" vs \"NIKE\" vs \"nike, inc.\"), partial matches (\"New York\" when the data has \"New York City\"), and aliased values (\"USA\" vs \"United States\"). A filter on a value that doesn't exist in the data silently returns zero rows without erroring. Always use the exact dimensional values from retrieval results, and if in doubt, run a distinct-values query on the dimension to confirm.\n\n### Filtering on the wrong field\nIf the user asks to filter by \"brand\", confirm which dimension corresponds to \"brand\" in the model. There may be multiple fields with similar names at different levels of the hierarchy.\n\n### Missing filters\nIf the user asks about \"last quarter\" but you don't apply a time filter, you are returning all-time data. Always check whether the question implies filters you have not yet applied.\n\n### Misinterpreted entities\nA field can exist in the model and still be the wrong choice. For example, using `revenue` (which may be gross) when the question asks about profit, or treating a count measure as if it were a sum. Cross-reference the field definitions and `#(doc)` descriptions in the model to confirm a field means what you think it means.\n\n### Semantic ambiguity\nField names or descriptions sometimes suggest one interpretation while the actual values tell a different story, for example, a field labeled \"annual ridership\" containing values that look like average weekday traffic, or a field named `revenue` that appears to represent net revenue in practice. When values don't match expectations, note whether the ambiguity affects the answer and call it out.\n\n### Fragile ad-hoc definitions\nWhen defining a new measure or dimension inline, common mistakes include assuming a field is numeric when it's actually a string, ignoring nulls in arithmetic (e.g., `a - b` yields null if either is null), and building logic around values that only cover a subset of the data.\n\nPay special attention to value coverage when defining a dimension that categorizes or subsets data. It's easy to capture too few values (missing categories that should be included) or too many (grouping in values that don't belong). For example, a `pick` expression that maps tier names to \"Premium\" and \"Standard\" might miss a tier that should be Premium, or a filter meant to isolate one product line might inadvertently include related but distinct products. Before relying on such a definition, run a distinct-values query on the underlying field to see the full set of values and confirm your logic handles them all correctly. This applies equally whether you define the logic as a new dimension or apply it directly as a `where` clause: the risk of incomplete value coverage is the same either way.\n\n### Hidden filters in views\nViews can have built-in `where` clauses that pre-filter the data. A view named `recent_orders` might only include the last 90 days, or `active_customers` might exclude certain statuses. If the view's definition is available, read it to understand what filters are baked in; they may conflict with what the user is asking for. When in doubt, query the base source directly and apply filters explicitly.\n\n## Result Interpretation\n\n### Implausible magnitudes\nIf a \"total\" is suspiciously small or large, question it. Common causes: missing filters (too large), over-filtering (too small), wrong unit (dollars vs. cents), or fan-out from joins (inflated).\n\n### Nulls distorting aggregations\nNull values are silently excluded from `avg()` and can make `sum()` results lower than expected. If a significant portion of a field's data is null, aggregations over that field may be misleading. When results seem off, compare total row count to a count of non-null values for the key field to gauge how much data is missing.\n\n### Confusing count vs. count distinct\n`count()` counts rows while measures defined with `count(field)` count distinct values of that field. Using a row count when you need distinct values (or vice versa) is a frequent source of inflated or deflated numbers, especially when the query touches joined sources.\n\n### Percentage of what?\nWhen computing percentages or shares, be explicit about the denominator. \"30% of revenue\" means nothing if you don't confirm what the total revenue is and whether it's filtered the same way.\n\n### Time period mismatches\nComparing metrics across different time periods without normalizing (e.g., comparing a full year to a partial quarter) produces misleading conclusions.\n\n### Extremes of a rate on a tiny denominator\n\"Which one is worst\" ranked on a ratio hands the top spot to whichever group has the fewest rows: one failure out of three is a 33% failure rate, and it will outrank a group with thousands. Before reporting an extreme of any rate, look at what is underneath it. Apply the model's documented minimum volume if it has one; if it has none, pick a floor, say which floor you picked, and print the denominator next to the rate rather than the rate alone.\n\n## Verification Signals\n\n### Parts don't sum to the whole\nIf you break down a total by category, the categories should sum to the total (or close to it, accounting for nulls). If they don't, something is wrong with the grain or filters.\n\n### A total the model tells you not to take\nThe check above assumes the parts are additive. Sometimes they are not, and the model says so: a doc noting that one event can be logged on two layers, or that a measure is non-additive, is telling you the sum across that dimension double-counts. Do not print a total row across such a dimension, and least of all one the user did not ask for. If a total is genuinely wanted, say why the model does not support it and give the per-category figures.\n\n### Row count surprises\nBefore interpreting results, check whether the row count makes sense. An unexpectedly high row count often indicates fan-out from a join. An unexpectedly low count may mean an overly restrictive filter.\n\n### Zero or empty results\nIf a query returns no rows, don't report \"there is no data.\" First verify that your filters are correct and the field names are right. The absence of results is usually a query problem, not a data problem."
     },
     {
       name: "malloy-analysis-report",
@@ -323749,7 +325226,7 @@ Steps to follow when the user asks a question:
     {
       name: "malloy-phrase-detection",
       description: "How to phrase search_text on a get_context call so retrieval returns the fields you need instead of a truncated catalog. Covers target-type classification and decomposition patterns.",
-      body: "# Search Target Construction for `get_context`\n\nThe `get_context` tool description defines each field and what a call returns. This skill focuses on the parts you won't get right by default: classifying concepts into target types and splitting ambiguous phrases.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n**Scope of this skill:** the patterns below build `dimension` / `measure` / `view` targets. Phrasing for `source` targets is covered at the end.\n\n**A note on matching:** `get_context` searches over the model (sources, fields, views, and their descriptions), not the distinct categorical *values* stored in the data. To find which literal values a categorical dimension holds, target the dimension, then query its distinct values with `execute_query` (see the patterns below).\n\n## Always send `search_text`\n\n**Do not enumerate.** Omitting `search_text` lists a catalog rather than searching it. Knowing the package narrows *where* to look; it does not substitute for saying *what* you need: if you know the package, that is a reason to scope, not a reason to skip `search_text`. Enumerated listings are capped per source and per entity type, and with no relevance signal the cap drops the fields your question is about while keeping join-path noise.\n\nA bare listing has two legitimate uses. The first is answering \"what data is here?\" when the user has named no subject at all. The second is reading a specific entity you already have the exact name of: scope to its source, set `entity_name`, and pass `search_text: null`, which returns that entity's docstring and Malloy code without spending a search. Every other call carries `search_text` on every target.\n\n## Authoring `search_text` for entity targets\n\nWrite `search_text` as a brief semantic **description** of what you're looking for, not an echo of the user's word. This applies even when you already know the entity name from a prior result: still describe it, don't just repeat the name. That is a rule about how to *phrase* a search, and it does not conflict with the exact-entity lookup above: if you want that one entity's code and docstring rather than a ranked set, pin it with `entity_name` and skip the search entirely. Sending its name back as `search_text` is the move this rule forbids, because it searches for a name instead of either describing a concept or asking for the entity.\n\nOne target per concept is enough: the tool handles phrasing variants internally. Don't pile up dimension targets that point at the same field. Use multiple targets only when they describe genuinely distinct concepts (see \"Non-obvious decomposition patterns\" below).\n\n## Target-type decision guide\n\n- **`dimension`**: categorical attribute to group, filter, or join on. Also used for time and numeric fields.\n  - \"region\" becomes `\"the geographic region\"`\n- **`measure`**: aggregation metric (count, sum, average, rate).\n  - \"total revenue\" becomes `\"the total revenue or sales amount\"`\n- **`view`**: pre-built analysis. Include one whenever the question sounds like a canned report (summary, breakdown, top-N, trend).\n  - \"sales summary\" becomes `\"a summary of sales metrics\"`\n- **`source`**: data domain, for a question that names a subject area rather than fields (phrasing below).\n\n**Resolving categorical values (no value-search target in v1).** When the user names a literal value like \"premium\" or \"New York City\", target the *dimension* it lives on (`\"the subscription tier\"`, `\"the city where the subscriber lives\"`). Then confirm the exact stored string by querying that dimension's distinct values with `execute_query` before you filter on it. The data may store `\"Premium\"`, `\"PREMIUM\"`, `\"NYC\"`, or `\"New York\"`, and only the data tells you which.\n\n## Non-obvious decomposition patterns\n\nThese are the rules you won't apply correctly by default:\n\n1. **Adjective + noun, split.** \"active users\" becomes two dimension targets: one for the attribute (`\"the status of the user account\"`) and one for the noun (`\"the user or account holder\"`). Resolve the modifier (\"active\") to the exact stored value by querying the status dimension's distinct values with `execute_query`.\n2. **Ambiguous concept, cover both types.** \"rating\", \"duration\", and the like could be either a dimension or a measure: create one target of each type.\n3. **Time references are dimensions.** \"last year\" becomes a dimension target for the relevant date field (`\"the date the event occurred\"`).\n4. **Numeric ranges are dimensions.** \"aged 50\", \"revenue over $1M\" become dimension targets; the comparison is applied in the query, not matched as text.\n5. **Categorical strings that look numeric are still dimensions.** \"18-30\", \"<5 days\", \"tier 2\" are stored as literal strings on a dimension. Target that dimension, then confirm the exact string with `execute_query`.\n6. **\"Top N\" without a named measure, add a ranking measure.** \"top 6 products\" becomes a measure for the ranking concept (`\"the performance metric for a product\"`) plus a dimension for the entity. If the measure is explicit (\"top products by total sales\"), use it directly and skip the generic ranking measure.\n7. **Multiple values for one concept, one dimension target.** Several values (\"premium and basic\") still map to a single dimension target for the parent field; enumerate the exact stored values with `execute_query`.\n8. **A population qualifier is a target, and so is the one the question omits.** Words like \"real\", \"actual\", \"genuine\", \"live\" or \"production\" are not filler: they name rows the model marks for exclusion. Target the flag itself (`\"the flag marking synthetic, test or monitoring traffic\"`), not just the noun they modify. Add one such target even when the question carries no qualifier at all, because a table of events, requests, sessions or logs usually holds test, internal or cancelled rows and nothing in the wording will say so. Resolve it to the model's own flag rather than inventing a filter on an id or a name; a hand-rolled exclusion and the documented one rarely select the same rows.\n\n## Worked example\n\n**User:** \"Customer churn in NYC over the last year for premium and basic subscribers\"\n\nThe targets for this question:\n\n| target_type | search_text |\n|---|---|\n| `measure` | `\"the rate at which customers leave the service\"` |\n| `dimension` | `\"the city where the subscriber lives\"` |\n| `dimension` | `\"the date the subscription was canceled\"` |\n| `dimension` | `\"the tier of the subscription\"` |\n| `view` | `\"subscriber churn or retention analysis\"` |\n\nKey moves: time (\"last year\") becomes a dimension on the cancellation date; \"NYC\" and \"premium/basic subscribers\" do not get their own value targets, they resolve to the city and tier dimensions. One `view` target is included to surface any canned churn analysis.\n\nThe response returns the source these fields live on (here `subscriptions`) with the matched fields on its card. Then run `execute_query` to read the city and tier dimensions' distinct values and confirm the exact strings to filter on (\"New York City\" vs \"NYC\", \"premium\" vs \"Premium\").\n\n## Authoring `search_text` for `source` targets\n\nAim for **3-8 words that name the entity and its business process**. Don't include filter values, time ranges, or aggregations: those belong in entity targets.\n\n| Too vague | Over-specific (entity-shaped) | Good |\n|---|---|---|\n| `\"orders\"` | `\"total order revenue by customer last year\"` | `\"customer order history and line items\"` |\n| `\"customer data\"` | `\"premium subscribers who churned in NYC\"` | `\"subscriber accounts and churn\"` |\n| `\"metrics\"` | `\"monthly revenue variance by account\"` | `\"sales pipeline and revenue forecasts\"` |\n\nHeuristics:\n\n1. **Translate, don't echo.** \"How did sales go last month?\" becomes `\"sales order revenue\"`, not `\"sales last month\"`.\n2. **Differentiate by data shape or business process, not by the user's industry, brand, or product category.** Prefer `\"order fulfillment and shipping\"` over `\"ecommerce data\"` when multiple commerce-ish packages exist. Do NOT add words like `\"eyewear\"`, `\"subscription box\"`, `\"Acme Corp\"`, or the user's specific vertical/brand. Source summaries describe data structure, not the customer's vertical, so those words add noise and can hurt matching.\n3. **Retry with alternative phrasings** if the right source isn't in the results before concluding it's missing."
+      body: "# Search Target Construction for `get_context`\n\nThe `get_context` tool description defines each field and what a call returns. This skill focuses on the parts you won't get right by default: classifying concepts into target types and splitting ambiguous phrases.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n**Scope of this skill:** the patterns below build `dimension` / `measure` / `view` targets. Phrasing for `source` targets is covered at the end.\n\n**A note on matching:** `get_context` searches over the model (sources, fields, views, and their descriptions), not the distinct categorical *values* stored in the data. To find which literal values a categorical dimension holds, target the dimension, then query its distinct values with `execute_query` (see the patterns below).\n\n## Always send `search_text`\n\n**Do not enumerate.** Omitting `search_text` lists a catalog rather than searching it. Knowing the package narrows *where* to look; it does not substitute for saying *what* you need: if you know the package, that is a reason to scope, not a reason to skip `search_text`. Enumerated listings are capped per source and per entity type, and with no relevance signal the cap drops the fields your question is about while keeping join-path noise.\n\nA bare listing has two legitimate uses. The first is answering \"what data is here?\" when the user has named no subject at all. The second is reading a specific entity you already have the exact name of: scope to its source, set `entity_name`, and pass `search_text: null`, which returns that entity's docstring and Malloy code without spending a search. Every other call carries `search_text` on every target.\n\n## Authoring `search_text` for entity targets\n\nWrite `search_text` as a brief semantic **description** of what you're looking for, not an echo of the user's word. This applies even when you already know the entity name from a prior result: still describe it, don't just repeat the name. That is a rule about how to *phrase* a search, and it does not conflict with the exact-entity lookup above: if you want that one entity's code and docstring rather than a ranked set, pin it with `entity_name` and skip the search entirely. Sending its name back as `search_text` is the move this rule forbids, because it searches for a name instead of either describing a concept or asking for the entity.\n\nOne target per concept is enough: the tool handles phrasing variants internally. Don't pile up dimension targets that point at the same field. Use multiple targets only when they describe genuinely distinct concepts (see \"Non-obvious decomposition patterns\" below).\n\n## Target-type decision guide\n\n- **`dimension`**: categorical attribute to group, filter, or join on. Also used for time and numeric fields.\n  - \"region\" becomes `\"the geographic region\"`\n- **`measure`**: aggregation metric (count, sum, average, rate).\n  - \"total revenue\" becomes `\"the total revenue or sales amount\"`\n- **`view`**: pre-built analysis. Include one whenever the question sounds like a canned report (summary, breakdown, top-N, trend).\n  - \"sales summary\" becomes `\"a summary of sales metrics\"`\n- **`source`**: data domain, for a question that names a subject area rather than fields (phrasing below).\n\n**Resolving categorical values (no value-search target in v1).** When the user names a literal value like \"premium\" or \"New York City\", target the *dimension* it lives on (`\"the subscription tier\"`, `\"the city where the subscriber lives\"`). Then confirm the exact stored string by querying that dimension's distinct values with `execute_query` before you filter on it. The data may store `\"Premium\"`, `\"PREMIUM\"`, `\"NYC\"`, or `\"New York\"`, and only the data tells you which.\n\n## Non-obvious decomposition patterns\n\nThese are the rules you won't apply correctly by default:\n\n1. **Adjective + noun, split.** \"active users\" becomes two dimension targets: one for the attribute (`\"the status of the user account\"`) and one for the noun (`\"the user or account holder\"`). Resolve the modifier (\"active\") to the exact stored value by querying the status dimension's distinct values with `execute_query`.\n2. **Ambiguous concept, cover both types.** \"rating\", \"duration\", and the like could be either a dimension or a measure: create one target of each type.\n3. **Time references are dimensions.** \"last year\" becomes a dimension target for the relevant date field (`\"the date the event occurred\"`).\n4. **Numeric ranges are dimensions.** \"aged 50\", \"revenue over $1M\" become dimension targets; the comparison is applied in the query, not matched as text.\n5. **Categorical strings that look numeric are still dimensions.** \"18-30\", \"<5 days\", \"tier 2\" are stored as literal strings on a dimension. Target that dimension, then confirm the exact string with `execute_query`.\n6. **\"Top N\" without a named measure, add a ranking measure.** \"top 6 products\" becomes a measure for the ranking concept (`\"the performance metric for a product\"`) plus a dimension for the entity. If the measure is explicit (\"top products by total sales\"), use it directly and skip the generic ranking measure.\n7. **Multiple values for one concept, one dimension target.** Several values (\"premium and basic\") still map to a single dimension target for the parent field; enumerate the exact stored values with `execute_query`.\n8. **The quantity asked for is a measure, even when nothing names it.** \"How many titles were released in 2019?\" is asking for a count, but no noun in it is the count: the visible phrases are the subject (\"titles\") and the filter (\"2019\"), and mapping only those yields a `source` and a `dimension` target with no way to answer. Always add a measure target for the quantity itself (`\"the number of titles\"`). This holds for every interrogative that IS the aggregation -- \"how many\", \"how much\", \"how often\" -- and it is the pattern most easily lost when a filter or grouping is the loud part of the sentence. Measured: an answerer got this right on \"How many titles are in the dataset?\" and dropped it on \"How many titles were released in 2019?\", where the year took the attention.\n9. **A population qualifier is a target, and so is the one the question omits.** Words like \"real\", \"actual\", \"genuine\", \"live\" or \"production\" are not filler: they name rows the model marks for exclusion. Target the flag itself (`\"the flag marking synthetic, test or monitoring traffic\"`), not just the noun they modify. Add one such target even when the question carries no qualifier at all, because a table of events, requests, sessions or logs usually holds test, internal or cancelled rows and nothing in the wording will say so. Resolve it to the model's own flag rather than inventing a filter on an id or a name; a hand-rolled exclusion and the documented one rarely select the same rows.\n\n## Worked example\n\n**User:** \"Customer churn in NYC over the last year for premium and basic subscribers\"\n\nThe targets for this question:\n\n| target_type | search_text |\n|---|---|\n| `measure` | `\"the rate at which customers leave the service\"` |\n| `dimension` | `\"the city where the subscriber lives\"` |\n| `dimension` | `\"the date the subscription was canceled\"` |\n| `dimension` | `\"the tier of the subscription\"` |\n| `view` | `\"subscriber churn or retention analysis\"` |\n\nKey moves: time (\"last year\") becomes a dimension on the cancellation date; \"NYC\" and \"premium/basic subscribers\" do not get their own value targets, they resolve to the city and tier dimensions. One `view` target is included to surface any canned churn analysis.\n\nThe response returns the source these fields live on (here `subscriptions`) with the matched fields on its card. Then run `execute_query` to read the city and tier dimensions' distinct values and confirm the exact strings to filter on (\"New York City\" vs \"NYC\", \"premium\" vs \"Premium\").\n\n## Authoring `search_text` for `source` targets\n\nAim for **3-8 words that name the entity and its business process**. Don't include filter values, time ranges, or aggregations: those belong in entity targets.\n\n| Too vague | Over-specific (entity-shaped) | Good |\n|---|---|---|\n| `\"orders\"` | `\"total order revenue by customer last year\"` | `\"customer order history and line items\"` |\n| `\"customer data\"` | `\"premium subscribers who churned in NYC\"` | `\"subscriber accounts and churn\"` |\n| `\"metrics\"` | `\"monthly revenue variance by account\"` | `\"sales pipeline and revenue forecasts\"` |\n\nHeuristics:\n\n1. **Translate, don't echo.** \"How did sales go last month?\" becomes `\"sales order revenue\"`, not `\"sales last month\"`.\n2. **Differentiate by data shape or business process, not by the user's industry, brand, or product category.** Prefer `\"order fulfillment and shipping\"` over `\"ecommerce data\"` when multiple commerce-ish packages exist. Do NOT add words like `\"eyewear\"`, `\"subscription box\"`, `\"Acme Corp\"`, or the user's specific vertical/brand. Source summaries describe data structure, not the customer's vertical, so those words add noise and can hurt matching.\n3. **Retry with alternative phrasings** if the right source isn't in the results before concluding it's missing."
     },
     {
       name: "malloy-publish",
@@ -325827,6 +327304,8 @@ function incrementalLineage(params) {
   const mergeKeys = d.mergeKeys.filter((k) => k.kind === "dimension").map((k) => k.name);
   if (mergeKeys.length !== d.mergeKeys.length)
     return;
+  if (mergeKeys.length > 0 && d.scopeIncomplete)
+    return;
   return {
     physicalTableName: params.physicalTableName,
     connectionName: params.connectionName,
@@ -325835,6 +327314,7 @@ function incrementalLineage(params) {
     watermarkName: watermark.name,
     watermarkType: watermark.malloyType,
     mergeKeys,
+    scopeColumns: d.scopeColumns,
     strategy: d.strategy
   };
 }
@@ -326416,7 +327896,7 @@ async function buildSourceIntoStorage(params) {
       }
     }
     const passthrough = await read(session, sourceType, federated.handle, buildSQL, queryMetadata);
-    const schema = await createTableAndDescribe(session, target, passthrough.selectSQL);
+    const schema = await createTableAndDescribe(session, target, passthrough.selectSQL, params.partitionColumns ?? []);
     const seededThrough = await params.incremental?.afterSeed({
       session,
       quotedTablePath: target
@@ -326483,7 +327963,7 @@ async function buildDownstreamIntoStorage(params) {
     }
     const sql = projectToPublicColumns(downstream, downstream.getSQL({ virtualMap }));
     const target = quoteManifestTablePath(`${destinationName}.${physicalTableName}`, STORAGE_TARGET_DIALECT);
-    const schema = await createTableAndDescribe(session, target, sql);
+    const schema = await createTableAndDescribe(session, target, sql, params.partitionColumns ?? []);
     return {
       storageDestinationName: destinationName,
       schema,
@@ -326555,22 +328035,53 @@ function sourceFederationConfig(sourceConnection) {
     proxy: sourceConnection.proxy
   };
 }
-async function createTableAndDescribe(session, quotedTablePath, selectSQL) {
-  await session.runSQL(`CREATE OR REPLACE TABLE ${quotedTablePath} AS (${selectSQL})`);
+async function createTableAndDescribe(session, quotedTablePath, selectSQL, partitionColumns = []) {
+  if (partitionColumns.length === 0) {
+    await session.runSQL(`CREATE OR REPLACE TABLE ${quotedTablePath} AS (${selectSQL})`);
+    return await describeOrDrop(session, quotedTablePath);
+  }
+  const columns = partitionColumns.map((name) => quoteIdentifier(name, STORAGE_TARGET_DIALECT)).join(", ");
+  let schema;
+  await session.runSQL("BEGIN TRANSACTION");
   try {
-    return await describeTable(session, quotedTablePath);
-  } catch (describeErr) {
+    await session.runSQL(`CREATE OR REPLACE TABLE ${quotedTablePath} AS (${selectSQL}) WITH NO DATA`);
+    await session.runSQL(`ALTER TABLE ${quotedTablePath} SET PARTITIONED BY (${columns})`);
+    await session.runSQL(`INSERT INTO ${quotedTablePath} (${selectSQL})`);
+    schema = await describeTable(session, quotedTablePath);
+    await session.runSQL("COMMIT");
+  } catch (buildErr) {
     try {
-      await session.runSQL(`DROP TABLE IF EXISTS ${quotedTablePath}`);
-    } catch (dropErr) {
-      logger.warn("Failed to drop a storage table stranded by a schema read-back " + "failure (physical leak)", {
+      await session.runSQL("ROLLBACK");
+    } catch (rollbackErr) {
+      logger.warn("Failed to roll back a partitioned storage build; the served " + "generation may have been replaced", {
         table: quotedTablePath,
-        error: errMessage(dropErr)
+        error: errMessage(rollbackErr),
+        cause: errMessage(buildErr)
       });
     }
+    throw buildErr;
+  }
+  return schema;
+}
+async function describeOrDrop(session, quotedTablePath) {
+  try {
+    return await describeTable(session, quotedTablePath);
+  } catch (describeErr) {
+    await dropStranded(session, quotedTablePath, describeErr);
     throw describeErr;
   }
 }
+async function dropStranded(session, quotedTablePath, cause) {
+  try {
+    await session.runSQL(`DROP TABLE IF EXISTS ${quotedTablePath}`);
+  } catch (dropErr) {
+    logger.warn("Failed to drop a storage table stranded by a failed build " + "(physical leak)", {
+      table: quotedTablePath,
+      error: errMessage(dropErr),
+      cause: errMessage(cause)
+    });
+  }
+}
 async function describeTable(session, quotedTablePath) {
   const result = await session.runSQL(`DESCRIBE ${quotedTablePath}`);
   const rows = Array.isArray(result) ? result : result.rows ?? [];
@@ -326719,6 +328230,16 @@ function manifestExcludingStorage(manifest, builtEntries) {
 function declaredStorage(persistSource) {
   return deriveAnnotationFields(persistSource).storage?.trim() || undefined;
 }
+function partitionColumnsForBuild(persistSource) {
+  const resolved = resolvePartitionColumns(persistSource, deriveAnnotationFields(persistSource));
+  if (!resolved.ok) {
+    throw new MaterializationEligibilityError({
+      reason: resolved.reason,
+      message: `Source '${persistSource.name}' reached a build with an ` + `unusable 'partition=': ${resolved.detail}. The eligibility gate ` + `should have refused this before any warehouse work ran.`
+    });
+  }
+  return resolved.columns;
+}
 function resolveStorageDestination(persistSource) {
   if (getPersistStorageMode() === "off")
     return;
@@ -327005,7 +328526,7 @@ class MaterializationService {
         if (destination) {
           if (compiled.preaggregatePlans?.[persistSource.sourceID]) {
             try {
-              assertMaterializationEligible(persistSource);
+              assertMaterializationEligible(persistSource, deriveAnnotationFields(persistSource));
             } catch (err) {
               if (!(err instanceof MaterializationEligibilityError))
                 throw err;
@@ -327016,10 +328537,10 @@ class MaterializationService {
               continue;
             }
           } else {
-            assertMaterializationEligible(persistSource);
+            assertMaterializationEligible(persistSource, deriveAnnotationFields(persistSource));
           }
         } else {
-          assertColocatedPersistNotAuthorizeGated(persistSource, persistSource.name, compiled.preaggregatePlans?.[persistSource.sourceID] ? "preaggregate" : "persist", compiled.sourceGateOutcomes?.[persistSource.sourceID]);
+          assertColocatedPersistNotAuthorizeGated(persistSource, persistSource.name, compiled.preaggregatePlans?.[persistSource.sourceID] ? "preaggregate" : "persist", compiled.sourceGateOutcomes?.[persistSource.sourceID], deriveAnnotationFields(persistSource));
         }
         const sourceEntityId = computeSourceEntityId(persistSource, compiled.connectionDigests);
         if (seen.has(sourceEntityId))
@@ -327357,13 +328878,13 @@ class MaterializationService {
           if (!instruction)
             continue;
           if (orchestratedInstruction?.destination && getPersistStorageMode() !== "off") {
-            assertMaterializationEligible(persistSource);
+            assertMaterializationEligible(persistSource, deriveAnnotationFields(persistSource));
           } else {
-            assertColocatedPersistNotAuthorizeGated(persistSource, persistSource.name, compiled.preaggregatePlans?.[persistSource.sourceID] ? "preaggregate" : "persist", compiled.sourceGateOutcomes?.[persistSource.sourceID]);
+            assertColocatedPersistNotAuthorizeGated(persistSource, persistSource.name, compiled.preaggregatePlans?.[persistSource.sourceID] ? "preaggregate" : "persist", compiled.sourceGateOutcomes?.[persistSource.sourceID], deriveAnnotationFields(persistSource));
           }
           sourceEntityId ??= computeSourceEntityId(persistSource, connectionDigests);
           if (!orchestratedInstruction && instruction.destination && getPersistStorageMode() !== "off") {
-            assertMaterializationEligible(persistSource);
+            assertMaterializationEligible(persistSource, deriveAnnotationFields(persistSource));
           }
           const target = physicalTargetKey(instruction, graph.connectionName);
           const written = writtenTargets.get(target);
@@ -327782,6 +329303,7 @@ class MaterializationService {
           sourceConnection,
           buildSQL: params.publicBuildSQL,
           physicalTableName,
+          partitionColumns: partitionColumnsForBuild(persistSource),
           environmentPath: environment.getEnvironmentPath(),
           queryMetadata,
           incremental: refresh
@@ -327935,6 +329457,7 @@ class MaterializationService {
       downstreamName: persistSource.name,
       virtualMap: buildVirtualMap(upstreams),
       physicalTableName,
+      partitionColumns: partitionColumnsForBuild(persistSource),
       environmentPath: environment.getEnvironmentPath()
     });
   }