incremnt 0.7.1 → 0.8.0

package/src/summary-evals.js

@@ -3,13 +3,16 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import {
   askContext,
-  askRoutedContext,
+  canonicalExerciseName,
   checkpointContext,
   cycleSummaryContext,
+  executeCoachReadTool,
   normalizeExerciseName,
   workoutSummaryContext,
   vitalsSummaryContext
 } from './queries.js';
+import { askRoutedContext, buildAskStructuredResponse } from './ask-coach.js';
+import { formatIncrementScorePrelude, isScoreQuestion } from './score-prelude.js';
 import {
   AI_PROMPT_VERSIONS,
   generateAskAnswer,
@@ -19,6 +22,8 @@ import {
   generateWorkoutCoachingSummary
 } from './openrouter.js';
 import { computeScoreBand } from './score-context.js';
+import { stripXMLTagBlocks } from './prompt-security.js';
+import { extractAskProgramDraft, hasProgramDraftBlock } from './program-draft.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -29,6 +34,14 @@ export function defaultCaseSetName() {
   return process.env.SUMMARY_EVAL_CASE_SET || 'synthetic';
 }
 
+function envFlag(name, env = process.env) {
+  return ['1', 'true', 'yes'].includes(String(env[name] ?? '').toLowerCase());
+}
+
+export function summaryEvalsLiveGenerationEnabled(env = process.env) {
+  return envFlag('SUMMARY_EVALS_LIVE', env) || envFlag('PROMPTFOO_LIVE', env);
+}
+
 function stableSortByDateDesc(items, selector) {
   return [...items].sort((lhs, rhs) => String(selector(rhs)).localeCompare(String(selector(lhs))));
 }
@@ -86,12 +99,22 @@ export function buildSummaryEvalContext(snapshot, testCase) {
       return vitalsSummaryContext(snapshot, { exclude: new Set(testCase.exclude ?? []) });
     case 'ask': {
       const question = testCase.context?.question ?? testCase.question ?? '';
+      const today = testCase.context?.today ?? testCase.today ?? null;
+      const history = Array.isArray(testCase.context?.history) ? testCase.context.history : [];
       const routed = question
-        ? askRoutedContext(snapshot, question, { exclude: new Set(testCase.exclude ?? []) })
+        ? askRoutedContext(snapshot, question, { exclude: new Set(testCase.exclude ?? []), history, today: today ?? new Date() })
         : null;
+      // Mirror production: the live /cli/ask path prepends the Increment Score
+      // prelude to the routed context. Including it here means a live eval feeds
+      // the model the same dump-prone material, so evaluateAskScoreVoice actually
+      // guards the prompt, not just the checker.
+      const prelude = formatIncrementScorePrelude(scoreHistoryFromSnapshot(snapshot), { question });
+      const routedContext = routed?.context ?? null;
+      const trainingData = testCase.context?.trainingData
+        ?? (prelude && routedContext ? `${prelude}\n\n${routedContext}` : (routedContext ?? prelude));
       return {
         ...(testCase.context ?? {}),
-        trainingData: testCase.context?.trainingData ?? routed?.context ?? null,
+        trainingData,
         routedMetadata: routed?.metadata ?? null
       };
     }
@@ -115,11 +138,38 @@ function summaryEvalGenerationMetadata(result) {
   );
 }
 
+function buildAskEvalStructuredMetadata(testCase, context, output) {
+  if (testCase.surface !== 'ask') return {};
+  const parsedAsk = extractAskProgramDraft(output, {
+    canonicalizeExerciseName: canonicalExerciseName
+  });
+  const answer = stripXMLTagBlocks(parsedAsk.answerText);
+  const question = context?.question ?? testCase.context?.question ?? testCase.question ?? '';
+  const routingMetadata = context?.routedMetadata ?? null;
+  return {
+    routingMetadata,
+    structured: buildAskStructuredResponse(answer, routingMetadata ?? {}, {
+      programDraft: askStructuredProgramDraft(parsedAsk, routingMetadata),
+      question
+    })
+  };
+}
+
+function summaryEvalProviderMetadata(testCase, context, output, result = null) {
+  return {
+    ...summaryEvalGenerationMetadata(result),
+    ...buildAskEvalStructuredMetadata(testCase, context, output)
+  };
+}
+
 export async function generateSummaryEvalOutputWithMetadata(testCase, context, snapshot = null) {
-  const liveGenerationEnabled = process.env.SUMMARY_EVALS_LIVE === '1';
+  const liveGenerationEnabled = summaryEvalsLiveGenerationEnabled();
   const apiKey = process.env.OPENROUTER_API_KEY;
   if (!liveGenerationEnabled || !apiKey || testCase.shouldPass === false) {
-    return { output: testCase.output, metadata: {} };
+    return {
+      output: testCase.output,
+      metadata: summaryEvalProviderMetadata(testCase, context, testCase.output)
+    };
   }
 
   let result;
@@ -149,7 +199,8 @@ export async function generateSummaryEvalOutputWithMetadata(testCase, context, s
         apiKey,
         history: context.history ?? [],
         tone: context.tone,
-        model: context.model
+        model: context.model,
+        routingMetadata: context.routedMetadata ?? undefined
       });
       break;
     }
@@ -159,7 +210,7 @@ export async function generateSummaryEvalOutputWithMetadata(testCase, context, s
 
   return {
     output: result.text,
-    metadata: summaryEvalGenerationMetadata(result)
+    metadata: summaryEvalProviderMetadata(testCase, context, result.text, result)
   };
 }
 
@@ -258,8 +309,44 @@ function isSingleParagraph(text) {
   return !normalizeText(text).includes('\n\n');
 }
 
-function lowerIncludes(text, snippet) {
-  return normalizeText(text).toLowerCase().includes(String(snippet).toLowerCase());
+// Canonicalizes free-form coach text and required-mention snippets to the same
+// surface form before substring matching. The goal is to keep grounding checks
+// (does the answer cite this real number?) while tolerating the formatting an
+// LLM legitimately varies: unicode × vs ASCII x, set-token unit placement,
+// signed deltas, and rep-sequence separators (8/8/7 vs "8, 8, and 7").
+// It only adds equivalences; it never strips the digits a check is grounded on,
+// so a genuinely absent number still fails.
+function normalizeForMention(value) {
+  let s = normalizeText(value).toLowerCase();
+  // Unicode multiplication / bullet / asterisk between digits -> ASCII x.
+  s = s.replace(/(\d)\s*[×✕╳·∗*]\s*(\d)/g, '$1x$2');
+  // Drop weight units that sit inside set tokens: "80kg x 7" / "40 kg" -> "80 x 7" / "40".
+  s = s.replace(/(\d(?:\.\d+)?)\s*(?:kgs?|lbs?|pounds)\b/g, '$1');
+  // Collapse spaces around an x that joins two numbers: "80 x 7" -> "80x7".
+  s = s.replace(/(\d)\s*x\s*(?=\d)/g, '$1x');
+  // Unify rep-sequence separators: "8/8/7" and "8, 8, and 7" -> "8,8,7".
+  // Lookahead keeps the trailing digit so chained separators all collapse.
+  s = s.replace(/(\d)\s*\/\s*(?=\d)/g, '$1,');
+  s = s.replace(/(\d)\s*,\s*and\s+(?=\d)/g, '$1,');
+  s = s.replace(/(\d)\s*,\s*(?=\d)/g, '$1,');
+  return s.replace(/\s+/g, ' ').trim();
+}
+
+// A required/any-of mention may be a string or an array of acceptable
+// alternatives (matches if any alternative is present). Arrays express
+// AND-of-ORs at the fixture level: every top-level entry must match, and an
+// array entry matches when any of its phrasings appears.
+function mentionMatches(output, mention) {
+  const normalizedOutput = normalizeForMention(output);
+  const alternatives = Array.isArray(mention) ? mention : [mention];
+  return alternatives
+    .map((alternative) => normalizeForMention(alternative))
+    .filter(Boolean)
+    .some((alternative) => normalizedOutput.includes(alternative));
+}
+
+function describeMention(mention) {
+  return Array.isArray(mention) ? `one of [${mention.join(' | ')}]` : String(mention);
 }
 
 function phraseIncludes(text, snippet) {
@@ -305,7 +392,6 @@ const historicalExerciseModifiers = new Set([
   'leg',
   'weighted',
   'romanian',
-  'hack',
   'full',
   'grip'
 ]);
@@ -358,6 +444,18 @@ function collectAllowedExerciseNames(surface, context) {
     for (const sc of context.planComparison?.setsComparison ?? []) {
       if (sc.exercise) names.add(sc.exercise);
     }
+    // Planned-but-skipped and unplanned-but-added exercises are legitimate
+    // planned-vs-actual subjects: the model is handed "skipped X, added Y" in
+    // its context, so a correct "you skipped X" note must not be flagged as an
+    // unauthorized mention. `setsComparison` only carries *performed* planned
+    // exercises (queries.js builds it with a performedNames filter), so skipped
+    // lifts would otherwise never be authorized.
+    for (const exerciseName of context.planComparison?.skipped ?? []) {
+      if (exerciseName) names.add(exerciseName);
+    }
+    for (const exerciseName of context.planComparison?.added ?? []) {
+      if (exerciseName) names.add(exerciseName);
+    }
   }
 
   if (surface === 'cycle' && context && typeof context === 'object') {
@@ -399,6 +497,49 @@ function collectAllowedExerciseNames(surface, context) {
   return [...names];
 }
 
+// Project the allow-set from the actual context object the model was handed.
+// Any known exercise name (from the snapshot's vocabulary) that appears anywhere
+// in the serialized context — structured fields, plan comparison, prior-session
+// comparisons, nearby cardio, or free-text session/exercise notes — is something
+// the model could legitimately reference. Deriving authorization this way means
+// the allow-set can never drift behind a newly added context field: the failure
+// mode that flagged a correct "you skipped Hack Squat" note and a note-echoed
+// lift. A genuine invention is a known exercise present in the output but absent
+// from the context entirely.
+function collectContextExerciseNames(context, knownNames) {
+  if (!context || typeof context !== 'object') return [];
+  let serialized;
+  try {
+    serialized = JSON.stringify(context);
+  } catch {
+    return [];
+  }
+  const contextText = normalizeExerciseName(serialized);
+  if (!contextText) return [];
+  const matches = [];
+  for (const name of knownNames) {
+    const normalized = normalizeExerciseName(name);
+    if (!normalized) continue;
+    const pattern = new RegExp(`(?<!\\S)${escapeRegex(normalized)}(?!\\S)`, 'g');
+    for (const match of contextText.matchAll(pattern)) {
+      matches.push({
+        name,
+        normalized,
+        start: match.index,
+        end: (match.index ?? 0) + normalized.length
+      });
+    }
+  }
+  return uniqueStrings(matches
+    .filter((match) => !matches.some((candidate) =>
+      candidate !== match &&
+      candidate.normalized.length > match.normalized.length &&
+      candidate.start <= match.start &&
+      candidate.end >= match.end
+    ))
+    .map((match) => match.name));
+}
+
 function historicalExerciseVariants(name) {
   const normalized = normalizeExerciseName(name);
   if (!normalized) return [];
@@ -431,9 +572,15 @@ function evaluateExerciseMentions(output, snapshot, context, surface, testCase)
 
   const outputText = surface === 'scoreCommentary' ? scoreCommentaryText(output) : output;
   const isStored = testCase.source === 'stored';
+  const allNames = collectAllExerciseNames(snapshot);
+
+  // Union the hand-built field list with a projection of the actual context, so
+  // this can only ever *reduce* false positives. testCase.allowedExerciseMentions
+  // stays as an explicit override for cases that need it.
   const allowed = new Set();
   for (const name of [
     ...collectAllowedExerciseNames(surface, context),
+    ...collectContextExerciseNames(context, allNames),
     ...(testCase.allowedExerciseMentions ?? [])
   ]) {
     const variants = isStored ? historicalExerciseVariants(name) : [normalizeExerciseName(name)];
@@ -441,7 +588,6 @@ function evaluateExerciseMentions(output, snapshot, context, surface, testCase)
       allowed.add(variant);
     }
   }
-  const allNames = collectAllExerciseNames(snapshot);
   const normalizedOutput = normalizeExerciseName(outputText);
   const mentions = [];
 
@@ -463,8 +609,14 @@ function evaluateExerciseMentions(output, snapshot, context, surface, testCase)
 
   const unauthorized = mentions
     .filter((mention) => !mention.allowed)
+    // Collapse a shorter mention into any longer mention that spans the same
+    // text — regardless of whether the covering mention is itself allowed.
+    // "Squat" matches inside "Hack Squat" (whitespace word boundary), so without
+    // this an unauthorized "Hack Squat" was double-counted as both "Hack Squat"
+    // and "Squat". The covering mention carries the real verdict; the substring
+    // is never a distinct mention.
     .filter((mention) => !mentions.some((candidate) =>
-      candidate.allowed &&
+      candidate !== mention &&
       candidate.normalizedName.length > mention.normalizedName.length &&
       candidate.start <= mention.start &&
       candidate.end >= mention.end
@@ -481,11 +633,14 @@ function evaluateExerciseMentions(output, snapshot, context, surface, testCase)
 }
 
 function evaluateRequiredMentions(output, testCase) {
-  const missing = uniqueStrings(testCase.requiredMentions).filter((mention) => !lowerIncludes(output, mention));
+  const mentions = Array.isArray(testCase.requiredMentions) ? testCase.requiredMentions : [];
+  const missing = mentions.filter((mention) => !mentionMatches(output, mention));
   return {
     key: 'required_mentions',
     passed: missing.length === 0,
-    reason: missing.length === 0 ? 'All required mentions present.' : `Missing required mention(s): ${missing.join(', ')}`
+    reason: missing.length === 0
+      ? 'All required mentions present.'
+      : `Missing required mention(s): ${missing.map(describeMention).join(', ')}`
   };
 }
 
@@ -499,7 +654,7 @@ function evaluateAnyOfMentions(output, testCase) {
     };
   }
 
-  const matched = candidates.some((mention) => lowerIncludes(output, mention));
+  const matched = candidates.some((mention) => mentionMatches(output, mention));
   return {
     key: 'required_any_of_mentions',
     passed: matched,
@@ -1095,19 +1250,48 @@ function hasAskFatigueSupport(snapshot, lookbackDays = 7) {
   return false;
 }
 
+function parseWeightNumber(raw) {
+  return Number(String(raw).replace(/,/g, ''));
+}
+
 function extractAskWeightClaims(text) {
   const claims = [];
-  const pattern = /\b(\d+(?:\.\d+)?)\s*(?:kg|kilograms?)\b/gi;
+  // Accept comma-grouped thousands ("40,500 kg") as a single number so volume
+  // figures are not shredded into bogus "500 kg" / "000 kg" claims. Volume/total
+  // figures are excluded by isVolumeWeightClaim at the call sites, not by a
+  // magnitude cap — heavy machine work (leg press, sled) legitimately exceeds
+  // 1000 kg, and a fabricated heavy load must still be graded.
+  const pattern = /\b(\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?)\s*(?:kg|kilograms?)\b/gi;
   for (const match of text.matchAll(pattern)) {
     claims.push({
       text: match[0],
-      value: Number(match[1]),
+      value: parseWeightNumber(match[1]),
       index: match.index ?? -1
     });
   }
   return claims;
 }
 
+function extractAskWeightedSetClaims(text) {
+  const claims = [];
+  // A weight×reps pair is only unambiguous with "x"/"×" (e.g. "70 kg x 5"), or
+  // an explicit "for N rep(s)". Bare "X kg for N" is NOT a rep claim — N is
+  // almost always a SET count ("70 kg for 4 working sets") or a duration, and
+  // treating it as reps flags real data as a fabricated pair. So match only the
+  // unambiguous forms; the plain-weight loop still grounds the weight itself.
+  const pattern = /\b(\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?)\s*(?:kg|kilograms?)\s*(?:(?:x|×)\s*(\d+)|for\s+(\d+)\s*reps?)\b/gi;
+  for (const match of text.matchAll(pattern)) {
+    claims.push({
+      text: match[0],
+      weight: parseWeightNumber(match[1]),
+      reps: Number(match[2] ?? match[3]),
+      index: match.index ?? -1,
+      end: (match.index ?? -1) + match[0].length
+    });
+  }
+  return claims;
+}
+
 function allowedWeightsForExercise(snapshot, normalizedExerciseName) {
   const weights = [];
   for (const session of snapshot?.sessions ?? []) {
@@ -1143,10 +1327,359 @@ function isEstimatedOneRepMaxWeightClaim(text, claim) {
 }
 
 function isVolumeWeightClaim(text, claim) {
-  const start = Math.max(0, claim.index - 30);
-  const end = Math.min(text.length, claim.index + claim.text.length + 30);
-  const window = text.slice(start, end);
-  return /\bvolume\b/i.test(window);
+  // A kg figure in a clause about volume/tonnage/total load is a workload total
+  // (e.g. "weekly strength volume fell from 44,000 kg to 40,500 kg"), not an
+  // exercise load. Scope to the claim's clause so a fabricated exercise load
+  // earlier in the same sentence is still graded.
+  return /\b(?:volume|tonnage|total\s+(?:load|work|volume|tonnage))\b/i.test(claimClause(text, claim));
+}
+
+function claimClause(text, claim) {
+  const boundaries = [
+    '\n',
+    '. ',
+    ';',
+    ', while',
+    ', whereas',
+    ', but',
+    ' while ',
+    ' whereas ',
+    ' but '
+  ];
+  let start = 0;
+  for (const boundary of boundaries) {
+    const index = text.lastIndexOf(boundary, claim.index);
+    if (index >= 0) start = Math.max(start, index + boundary.length);
+  }
+
+  let end = text.length;
+  for (const boundary of boundaries) {
+    const index = text.indexOf(boundary, claim.index + claim.text.length);
+    if (index >= 0) end = Math.min(end, index);
+  }
+  return text.slice(start, end);
+}
+
+// Returns the sentence containing the claim, so context guards can look at the
+// whole clause rather than a fixed-width window (body-weight phrasing can put
+// the "body weight" anchor well before the kg figure).
+function claimSentence(text, claim) {
+  const before = text.slice(0, claim.index);
+  const startBreak = Math.max(before.lastIndexOf('. '), before.lastIndexOf('\n'));
+  const start = startBreak >= 0 ? startBreak + 1 : 0;
+  const after = text.slice(claim.index);
+  const breaks = [after.indexOf('. '), after.indexOf('\n')].filter((i) => i >= 0);
+  const end = breaks.length ? claim.index + Math.min(...breaks) : text.length;
+  return text.slice(start, end);
+}
+
+// Body-weight figures ("body weight is up 0.6 kg", "80.0 kg latest") are not
+// exercise-load claims. findNearestMentionedExercise would otherwise attribute
+// them to the previously named lift and flag a correct answer as a
+// hallucination, so skip any kg figure stated in a body-weight clause.
+function isBodyWeightClaim(text, claim) {
+  return /\bbody\s*weight\b|\bbodyweight\b|\bweight\s+trend\b/i.test(claimSentence(text, claim));
+}
+
+function askWorkingTopSetRows(snapshot) {
+  const rows = [];
+  for (const session of stableSortByDateDesc(snapshot?.sessions ?? [], (session) => session.completedAt ?? session.date)) {
+    const completedAt = session.completedAt ?? session.date;
+    for (const exercise of session.exercises ?? []) {
+      const workingSets = (exercise.sets ?? [])
+        .filter((set) => set?.isComplete && !set?.isWarmup)
+        .map((set) => ({
+          weight: Number(set.weight) || 0,
+          reps: Number(set.reps) || 0
+        }));
+      if (workingSets.length === 0) continue;
+      const topSet = workingSets.sort((a, b) => b.weight - a.weight || b.reps - a.reps)[0];
+      rows.push({
+        sessionId: session.id ?? null,
+        date: String(completedAt ?? '').slice(0, 10),
+        exerciseName: exercise.name,
+        normalizedName: normalizeExerciseName(exercise.name),
+        ...topSet
+      });
+    }
+  }
+  return rows;
+}
+
+function daysAgoForEval(date, testCase) {
+  const today = testCase.context?.today ?? testCase.today;
+  if (!today) return null;
+  const dateMs = Date.parse(`${String(date ?? '').slice(0, 10)}T00:00:00.000Z`);
+  const todayMs = Date.parse(`${String(today).slice(0, 10)}T00:00:00.000Z`);
+  if (!Number.isFinite(dateMs) || !Number.isFinite(todayMs)) return null;
+  return Math.max(0, Math.round((todayMs - dateMs) / (24 * 60 * 60 * 1000)));
+}
+
+function hasUnqualifiedDeclineLanguage(window) {
+  const text = normalizeText(window);
+  const decline = /\b(drop(?:ped|ping|s)?(?: off)?|drop-off|declin(?:e|ed|ing)|regress(?:ed|ion|ing)?|fell|fall(?:ing)?|decreas(?:e|ed|ing)|lower|worse|slid|slipped)\b/i;
+  if (!decline.test(text)) return false;
+  if (/\b(?:no|not|isn'?t|wasn'?t|without|rather than)\b.{0,45}\b(drop(?:ped|ping|s)?(?: off)?|drop-off|declin(?:e|ed|ing)?|decreas(?:e|ed|ing)?|regress(?:ed|ion|ing)?|fall(?:ing)?|fell|lower|worse|slid|slipped)\b/i.test(text)) return false;
+  if (/\b(?:rep|reps)\b.{0,20}\b(drop(?:ped|ping|s)?(?: off)?|drop-off|slip(?:ped|ping)?|fell|fall(?:ing)?|lower|declin(?:e|ed|ing)?|decreas(?:e|ed|ing)?|worse)\b/i.test(text)) return false;
+  if (/\b(drop(?:ped|ping|s)?(?: off)?|drop-off|slip(?:ped|ping)?|fell|fall(?:ing)?|lower|declin(?:e|ed|ing)?|decreas(?:e|ed|ing)?|worse)\b.{0,20}\b(?:rep|reps)\b/i.test(text)) return false;
+  return true;
+}
+
+function hasUnqualifiedImprovementLanguage(window) {
+  const text = normalizeText(window);
+  const improvement = /\b(improv(?:e|ed|ing|ement)|progress(?:ed|ing)?|stronger|increas(?:e|ed|ing)|moving up|went up|up from|load jump|jumped)\b/i;
+  if (!improvement.test(text)) return false;
+  if (/\b(?:no|not|isn'?t|wasn'?t|without|rather than)\b.{0,35}\b(improv(?:e|ed|ing|ement)?|progress(?:ed|ing)?|stronger|increas(?:e|ed|ing)?|moving up|went up|up from|load jump|jump(?:ed|ing)?)\b/i.test(text)) return false;
+  if (/\b(?:rep|reps)\b.{0,20}\b(improv(?:e|ed|ing|ement)?|increas(?:e|ed|ing)?|better|moving up|went up|up from|load jump|jump(?:ed|ing)?)\b/i.test(text)) return false;
+  if (/\b(improv(?:e|ed|ing|ement)?|increas(?:e|ed|ing)?|better|moving up|went up|up from|load jump|jump(?:ed|ing)?)\b.{0,20}\b(?:rep|reps)\b/i.test(text)) return false;
+  return true;
+}
+
+function isReferentialDirectionContinuation(sentence) {
+  return /^(?:that|this|it|there|still|the\s+(?:latest|top)|top\s+set|same\s+load)\b/i.test(sentence);
+}
+
+function directionEvaluationWindows(outputText, exerciseName, exerciseNames = []) {
+  const normalizedExercise = normalizeExerciseName(exerciseName);
+  const otherExercises = [...new Set(exerciseNames.map(normalizeExerciseName))]
+    .filter((name) => name && name !== normalizedExercise);
+  const sentences = outputText
+    .split(/(?<=[.!?])\s+/)
+    .map((sentence) => sentence.trim())
+    .filter(Boolean);
+  if (!normalizedExercise) return sentences;
+  const windows = [];
+  for (let index = 0; index < sentences.length; index++) {
+    if (!normalizeExerciseName(sentences[index]).includes(normalizedExercise)) continue;
+    windows.push(sentences[index]);
+    for (let nextIndex = index + 1; nextIndex < sentences.length; nextIndex++) {
+      const normalizedNext = normalizeExerciseName(sentences[nextIndex]);
+      if (otherExercises.some((name) => normalizedNext.includes(name))) break;
+      if (!isReferentialDirectionContinuation(sentences[nextIndex])) break;
+      windows.push(sentences[nextIndex]);
+    }
+  }
+  return windows.length > 0 ? [...new Set(windows)] : [outputText];
+}
+
+function evaluateAskDirectionalConsistency(output, snapshot, testCase) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_directional_consistency', passed: true, reason: 'Not an ask answer.' };
+  }
+
+  const required = Array.isArray(testCase.directionalConsistency)
+    ? testCase.directionalConsistency
+    : [];
+  if (required.length === 0) {
+    return { key: 'ask_directional_consistency', passed: true, reason: 'No directional assertions configured.' };
+  }
+
+  const rows = askWorkingTopSetRows(snapshot);
+  const outputText = normalizeText(output);
+  const failures = [];
+
+  for (const expectation of required) {
+    const normalizedName = normalizeExerciseName(expectation.exercise ?? expectation.exerciseName);
+    const history = rows.filter((row) => row.normalizedName === normalizedName);
+    if (history.length < 2) continue;
+    const latest = history[0];
+    const previous = history[1];
+    const loadDelta = latest.weight - previous.weight;
+    const actualDirection = loadDelta > 0 ? 'up' : loadDelta < 0 ? 'down' : 'flat';
+    const expectedDirection = expectation.loadDirection ?? actualDirection;
+    if (expectedDirection !== actualDirection) {
+      failures.push(`Configured expected direction for ${latest.exerciseName} is ${expectedDirection}, but snapshot top-load direction is ${actualDirection}.`);
+      continue;
+    }
+
+    const windows = directionEvaluationWindows(
+      outputText,
+      expectation.exercise ?? expectation.exerciseName,
+      rows.map((row) => row.exerciseName)
+    );
+    if (actualDirection === 'up' && windows.some(hasUnqualifiedDeclineLanguage)) {
+      failures.push(`Ask answer frames ${latest.exerciseName} as declining/drop-off even though top load increased from ${previous.weight} kg to ${latest.weight} kg.`);
+    }
+    if (actualDirection === 'down' && windows.some(hasUnqualifiedImprovementLanguage)) {
+      failures.push(`Ask answer frames ${latest.exerciseName} as improving even though top load decreased from ${previous.weight} kg to ${latest.weight} kg.`);
+    }
+    if (actualDirection === 'flat' && windows.some((window) => hasUnqualifiedDeclineLanguage(window) || hasUnqualifiedImprovementLanguage(window))) {
+      failures.push(`Ask answer invents a load direction for ${latest.exerciseName}, but top load was flat at ${latest.weight} kg.`);
+    }
+  }
+
+  return {
+    key: 'ask_directional_consistency',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Ask answer does not invert configured load directions.'
+      : failures.join(' ')
+  };
+}
+
+// Increment Score component names. Recited with a number, these are the raw
+// sub-scores the coach-observation-voice spec marks Tier 1 — never surface.
+const SCORE_COMPONENT_NAMES = ['coverage', 'stimulus', 'execution', 'progression', 'recovery'];
+
+// A score-like magnitude: 1-3 digits, optional one decimal place.
+const SCORE_NUMBER = '\\d{1,3}(?:\\.\\d+)?';
+
+// Contexts that mean the number is real-world data — reps, load, time, counts,
+// ratios, percentages, the /100 headline — not a raw component sub-score. A
+// number directly followed by one of these is left alone.
+const NON_SCORE_UNIT =
+  '(?:kg|kilo|lbs?|pounds?|reps?|sets?|%|percent|pct|x\\b|for\\s+\\d|sessions?|days?|nights?|weeks?|months?|' +
+  'years?|yrs?|hrs?|hours?|mins?|minutes?|secs?|seconds?|rpe|rir|am|pm|out\\s+of|of\\b|/\\s*\\d)';
+
+// Heuristic, not a parser. Flags a component name followed — within a short,
+// period/newline-free gap (one clause) — by a score-like number that is not a
+// real-world unit. The bounded gap (excludes digits, so it can't skip a unit'd
+// number) catches the natural phrasings an LLM actually emits — "recovery 35",
+// "recovery is 35", "recovery is sitting at 35", "recovery came in at 35",
+// "recovery (35)", "recovery is much lower at 42.8", "coverage 100" — while the
+// unit lookahead keeps clean prose ("recovery over the last 3 sessions",
+// "recovery after 3 hours of sleep", "execution at 9/10 RPE") from tripping.
+const SCORE_COMPONENT_DUMP_PATTERN = new RegExp(
+  // `(?!\\.\\d)` rejects a number that is really the integer part of a decimal —
+  // without it, backtracking matches "2" in "progression of 2.5 kg" (the unit
+  // guard only sees the ".5 kg" tail) and false-flags real load/time data.
+  `\\b(${SCORE_COMPONENT_NAMES.join('|')})\\b[^.\\d\\n]{0,25}?(${SCORE_NUMBER})\\b(?!\\.\\d)(?!\\s*${NON_SCORE_UNIT})`,
+  'gi'
+);
+
+// The other dump the prelude used to emit and the model parroted: an explicit
+// day-over-day delta number ("-13 day-over-day delta", "down 11 points day over
+// day"). A bare "down day-over-day" with no number is fine.
+const SCORE_DELTA_DUMP_PATTERN =
+  /[+-]\d+(?:\.\d+)?[^.\n]{0,16}?day[- ]over[- ]day|(?:\d+(?:\.\d+)?\s*points?)[^.\n]{0,16}?day[- ]over[- ]day|day[- ]over[- ]day[^.\n]{0,16}?(?:[+-]\d+(?:\.\d+)?|\d+(?:\.\d+)?\s*points?)/i;
+
+export function evaluateAskScoreVoice(output, testCase) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_score_voice', passed: true, reason: 'Not an ask answer.' };
+  }
+  // Escape hatch for cases that legitimately need raw component values
+  // (e.g. an ask case paired with the numbers-only tone).
+  if (testCase.allowScoreComponents === true) {
+    return { key: 'ask_score_voice', passed: true, reason: 'Score-component voice check opted out for this case.' };
+  }
+
+  const text = normalizeText(output);
+  const hits = new Set();
+  for (const match of text.matchAll(SCORE_COMPONENT_DUMP_PATTERN)) {
+    hits.add(`${match[1]} ${match[2]}`);
+  }
+  if (SCORE_DELTA_DUMP_PATTERN.test(text)) {
+    hits.add('day-over-day delta number');
+  }
+
+  return {
+    key: 'ask_score_voice',
+    passed: hits.size === 0,
+    reason: hits.size === 0
+      ? 'Ask answer does not recite raw Increment Score component sub-scores.'
+      : `Ask answer recites raw score internals: ${[...hits].join(', ')}. Speak in training reality, not raw sub-scores.`
+  };
+}
+
+function relevantSessionsForStaleness(snapshot, testCase) {
+  const configuredExercise = testCase.staleness?.exercise ?? testCase.staleness?.exerciseName
+    ?? testCase.directionalConsistency?.[0]?.exercise
+    ?? testCase.directionalConsistency?.[0]?.exerciseName
+    ?? null;
+  if (!configuredExercise) return snapshot?.sessions ?? [];
+  const normalized = normalizeExerciseName(configuredExercise);
+  return (snapshot?.sessions ?? []).filter((session) => (
+    (session.exercises ?? []).some((exercise) => normalizeExerciseName(exercise.name) === normalized)
+  ));
+}
+
+// The coach IS the coach — it must speak in the first person and never refer to
+// itself or its own outputs as a third party ("the coach observation says…",
+// "the system shows…"). Own the observation instead ("I flagged…").
+const ASK_SELF_REFERENCE_PATTERNS = [
+  /\bthe coach observations?\b/i,
+  /\bthe coach\b/i,
+  /\bthe ai coach\b/i,
+  /\byour coach\b/i,
+  /\bthis coach\b/i,
+  /\bthe system\b/i,
+  /\bthe assistant\b/i
+];
+
+function evaluateAskSelfReference(output, testCase) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_self_reference', passed: true, reason: 'Not an ask answer.' };
+  }
+  const text = normalizeText(output);
+  if (text === 'NO_INSIGHT' || !text) {
+    return { key: 'ask_self_reference', passed: true, reason: 'No answer text.' };
+  }
+  const hits = [];
+  for (const pattern of ASK_SELF_REFERENCE_PATTERNS) {
+    const match = text.match(pattern);
+    if (match) hits.push(match[0]);
+  }
+  const unique = uniqueStrings(hits);
+  return {
+    key: 'ask_self_reference',
+    passed: unique.length === 0,
+    reason: unique.length === 0
+      ? 'Ask answer speaks in the first person.'
+      : `Ask answer refers to itself in the third person: ${unique.join(', ')}. You ARE the coach — own it ("I flagged…", "your data shows…").`
+  };
+}
+
+// On a question that is not about the Increment Score, the coach must not
+// volunteer the bare overall score number (e.g. "your score is 92/100"). The
+// prelude withholds the number for non-score questions; this guards the answer.
+function evaluateAskVolunteeredScore(output, testCase) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_volunteered_score', passed: true, reason: 'Not an ask answer.' };
+  }
+  const question = testCase.context?.question ?? testCase.question ?? '';
+  if (isScoreQuestion(question)) {
+    return { key: 'ask_volunteered_score', passed: true, reason: 'Question is about the score; naming it is allowed.' };
+  }
+  const text = normalizeText(output);
+  const volunteered = /\b\d{2,3}\s*\/\s*100\b/.test(text)
+    || /\b(?:increment\s+)?score\s+(?:is|of|at|sits at|currently|was)\b[^.\n]*\b\d{2,3}\b/i.test(text);
+  return {
+    key: 'ask_volunteered_score',
+    passed: !volunteered,
+    reason: volunteered
+      ? 'Ask answer volunteers the overall Increment Score number on a question that was not about the score. Translate it to the limiter instead.'
+      : 'Ask answer does not volunteer the score number unprompted.'
+  };
+}
+
+function evaluateAskStaleness(output, snapshot, testCase) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_staleness', passed: true, reason: 'Not an ask answer.' };
+  }
+  const maxRecentDays = testCase.staleness?.maxRecentDays;
+  if (!Number.isFinite(Number(maxRecentDays))) {
+    return { key: 'ask_staleness', passed: true, reason: 'No staleness assertion configured.' };
+  }
+
+  const latestSession = stableSortByDateDesc(relevantSessionsForStaleness(snapshot, testCase), (session) => session.completedAt ?? session.date)[0] ?? null;
+  const daysAgo = daysAgoForEval(latestSession?.completedAt ?? latestSession?.date, testCase);
+  if (daysAgo == null || daysAgo <= Number(maxRecentDays)) {
+    return { key: 'ask_staleness', passed: true, reason: 'Latest session is inside the configured recency window.' };
+  }
+
+  const normalized = normalizeText(output);
+  const claimsRecent = /\brecent(?:ly)?\b/i.test(normalized);
+  const explicitlyNotRecent = /\b(?:not|isn'?t|wasn'?t|no longer)\s+(?:a\s+)?recent\b/i.test(normalized)
+    || /\brecent\b.{0,20}\b(?:not|isn'?t|wasn'?t)\b/i.test(normalized);
+  const includesAge = new RegExp(`\\b${daysAgo}\\s+days?\\s+ago\\b`, 'i').test(normalized);
+  const passed = !claimsRecent || explicitlyNotRecent || includesAge;
+  return {
+    key: 'ask_staleness',
+    passed,
+    reason: passed
+      ? 'Ask answer does not present stale sessions as simply recent.'
+      : `Ask answer calls a ${daysAgo}-day-old session recent without the days-ago label.`
+  };
 }
 
 function evaluateAskClaims(output, snapshot, testCase) {
@@ -1212,6 +1745,7 @@ function evaluateAskClaims(output, snapshot, testCase) {
   for (const claim of extractAskWeightClaims(normalized)) {
     if (isEstimatedOneRepMaxWeightClaim(normalized, claim)) continue;
     if (isVolumeWeightClaim(normalized, claim)) continue;
+    if (isBodyWeightClaim(normalized, claim)) continue;
     const referencedExercise = findNearestMentionedExercise(mentionedExercises, claim.index);
     if (!referencedExercise) continue;
     const allowedWeights = allowedWeightsForExercise(snapshot, referencedExercise.normalizedName);
@@ -1234,14 +1768,195 @@ function evaluateAskClaims(output, snapshot, testCase) {
   };
 }
 
-function evaluateAskToolProvenance(output, context, testCase) {
+function routedToolResultsForEval(snapshot, context) {
+  const routedMetadata = context?.routedMetadata ?? {};
+  const toolParams = routedMetadata.toolParams ?? {};
+  const toolResults = [];
+  const replayFailures = [];
+  for (const toolName of uniqueStrings(routedMetadata.toolsUsed ?? [])) {
+    try {
+      toolResults.push(executeCoachReadTool(snapshot, toolName, toolParams[toolName] ?? {}));
+    } catch (error) {
+      replayFailures.push(`Could not replay routed tool ${toolName}: ${error?.message ?? String(error)}`);
+    }
+  }
+  return { toolResults, replayFailures };
+}
+
+function addAskToolEvidenceRow(rows, toolName, row, inherited = {}) {
+  const exerciseName = row?.exerciseName ?? row?.name ?? inherited.exerciseName ?? null;
+  const normalizedName = normalizeExerciseName(exerciseName);
+  if (!normalizedName) return;
+  rows.push({
+    toolName,
+    exerciseName,
+    normalizedName,
+    date: row?.date ?? inherited.date ?? null,
+    daysAgo: row?.daysAgo ?? inherited.daysAgo ?? null,
+    recencyLabel: row?.recencyLabel ?? inherited.recencyLabel ?? null,
+    isStale: row?.isStale ?? inherited.isStale ?? false,
+    recencyCutoffDays: row?.recencyCutoffDays ?? inherited.recencyCutoffDays ?? null,
+    warmupSetCount: row?.warmupSetCount ?? 0,
+    workingSetCount: row?.workingSetCount ?? null,
+    topSet: row?.topSet ?? null,
+    comparedToPreviousSession: row?.comparedToPreviousSession ?? null,
+    sets: Array.isArray(row?.sets) ? row.sets : []
+  });
+}
+
+function askToolEvidenceRows(toolResults = []) {
+  const rows = [];
+  for (const toolResult of toolResults) {
+    for (const row of toolResult?.rows ?? []) {
+      if (Array.isArray(row?.exercises)) {
+        for (const exercise of row.exercises) {
+          addAskToolEvidenceRow(rows, toolResult.toolName, exercise, {
+            date: row.date,
+            daysAgo: row.daysAgo,
+            recencyLabel: row.recencyLabel,
+            isStale: row.isStale,
+            recencyCutoffDays: row.recencyCutoffDays
+          });
+        }
+      } else {
+        addAskToolEvidenceRow(rows, toolResult.toolName, row);
+      }
+    }
+  }
+  return rows;
+}
+
+function askToolEvidenceWeights(rows = []) {
+  const weights = [];
+  for (const row of rows) {
+    for (const set of row.sets ?? []) {
+      const weight = Number(set.weight);
+      if (Number.isFinite(weight)) weights.push(weight);
+    }
+    const topWeight = Number(row.topSet?.weight);
+    if (Number.isFinite(topWeight)) weights.push(topWeight);
+    const previousTopWeight = Number(row.comparedToPreviousSession?.previousTopSet?.weight);
+    if (Number.isFinite(previousTopWeight)) weights.push(previousTopWeight);
+  }
+  return weights;
+}
+
+function askToolEvidenceSetPairs(rows = []) {
+  const pairs = [];
+  for (const row of rows) {
+    for (const set of row.sets ?? []) {
+      const weight = Number(set.weight);
+      const reps = Number(set.reps);
+      if (Number.isFinite(weight) && Number.isFinite(reps)) pairs.push({ weight, reps });
+    }
+    const topWeight = Number(row.topSet?.weight);
+    const topReps = Number(row.topSet?.reps);
+    if (Number.isFinite(topWeight) && Number.isFinite(topReps)) pairs.push({ weight: topWeight, reps: topReps });
+    const previousTopWeight = Number(row.comparedToPreviousSession?.previousTopSet?.weight);
+    const previousTopReps = Number(row.comparedToPreviousSession?.previousTopSet?.reps);
+    if (Number.isFinite(previousTopWeight) && Number.isFinite(previousTopReps)) {
+      pairs.push({ weight: previousTopWeight, reps: previousTopReps });
+    }
+  }
+  return pairs;
+}
+
+function toolEvidenceSupportsWeightClaim(claim, rows) {
+  if (weightClaimSupported(claim, askToolEvidenceWeights(rows))) return true;
+  return false;
+}
+
+function toolEvidenceSupportsWeightedSetClaim(claim, rows) {
+  if (askToolEvidenceSetPairs(rows).some((pair) => (
+    Math.abs(pair.weight - claim.weight) < 0.01 && pair.reps === claim.reps
+  ))) {
+    return true;
+  }
+  return false;
+}
+
+function compareToolEvidenceRecency(lhs, rhs) {
+  const lhsDaysAgo = Number(lhs?.daysAgo);
+  const rhsDaysAgo = Number(rhs?.daysAgo);
+  if (Number.isFinite(lhsDaysAgo) && Number.isFinite(rhsDaysAgo)) return lhsDaysAgo - rhsDaysAgo;
+  if (Number.isFinite(lhsDaysAgo)) return -1;
+  if (Number.isFinite(rhsDaysAgo)) return 1;
+  return String(rhs?.date ?? '').localeCompare(String(lhs?.date ?? ''));
+}
+
+function newestToolEvidenceRow(rows = [], predicate = () => true) {
+  return rows
+    .filter(predicate)
+    .sort(compareToolEvidenceRecency)[0] ?? null;
+}
+
+function latestComparableToolRow(rows = []) {
+  return newestToolEvidenceRow(rows, (row) => row.comparedToPreviousSession?.loadDirection) ?? null;
+}
+
+function isWithinWeightedSetClaim(claim, weightedSetClaims) {
+  return weightedSetClaims.some((setClaim) => claim.index >= setClaim.index && claim.index < setClaim.end);
+}
+
+function rowIsStaleForEval(row, testCase) {
+  const daysAgo = Number(row?.daysAgo);
+  const cutoff = Number(testCase.staleness?.maxRecentDays ?? row?.recencyCutoffDays);
+  if (!Number.isFinite(daysAgo) || !Number.isFinite(cutoff)) return Boolean(row?.isStale);
+  return daysAgo > cutoff;
+}
+
+function outputCallsStaleEvidenceRecent(outputText, row) {
+  const normalized = normalizeText(outputText);
+  const claimsRecent = /\brecent(?:ly)?\b/i.test(normalized);
+  if (!claimsRecent) return false;
+  const explicitlyNotRecent = /\b(?:not|isn'?t|wasn'?t|no longer)\s+(?:a\s+)?recent\b/i.test(normalized)
+    || /\brecent\b.{0,20}\b(?:not|isn'?t|wasn'?t)\b/i.test(normalized);
+  if (explicitlyNotRecent) return false;
+  const daysAgo = Number(row?.daysAgo);
+  return !Number.isFinite(daysAgo) || !new RegExp(`\\b${daysAgo}\\s+days?\\s+ago\\b`, 'i').test(normalized);
+}
+
+function recencyEvaluationWindows(outputText, exerciseName, exerciseNames = []) {
+  const normalizedExercise = normalizeExerciseName(exerciseName);
+  const otherExercises = [...new Set(exerciseNames.map(normalizeExerciseName))]
+    .filter((name) => name && name !== normalizedExercise);
+  const windows = directionEvaluationWindows(outputText, exerciseName, exerciseNames);
+  if (!normalizedExercise) return windows;
+  const scoped = [];
+  for (const window of windows) {
+    const clauses = window
+      .split(/\s*(?:[.;:]|,\s+|\b(?:while|whereas|but|and)\b)\s*/i)
+      .map((clause) => clause.trim())
+      .filter(Boolean);
+    let matched = false;
+    for (let index = 0; index < clauses.length; index++) {
+      if (!normalizeExerciseName(clauses[index]).includes(normalizedExercise)) continue;
+      matched = true;
+      let scopedWindow = clauses[index];
+      for (let nextIndex = index + 1; nextIndex < clauses.length; nextIndex++) {
+        const normalizedNext = normalizeExerciseName(clauses[nextIndex]);
+        if (otherExercises.some((name) => normalizedNext.includes(name))) break;
+        scopedWindow += ` ${clauses[nextIndex]}`;
+      }
+      scoped.push(scopedWindow);
+    }
+    if (!matched) scoped.push(window);
+  }
+  return scoped.length > 0 ? [...new Set(scoped)] : windows;
+}
+
+function evaluateAskToolProvenance(output, context, testCase, snapshot) {
   if (testCase.surface !== 'ask') {
     return { key: 'ask_tool_provenance', passed: true, reason: 'Not an ask answer.' };
   }
 
   const routedMetadata = context?.routedMetadata ?? {};
   const toolsUsed = new Set(routedMetadata.toolsUsed ?? []);
-  const failures = [];
+  const { toolResults, replayFailures } = routedToolResultsForEval(snapshot, context);
+  const evidenceRows = askToolEvidenceRows(toolResults);
+  const mentionedExercises = findMentionedExercises(output, snapshot);
+  const unroutedMentionNames = new Set();
+  const failures = [...replayFailures];
   for (const toolName of uniqueStrings(testCase.requiredTools)) {
     if (!toolsUsed.has(toolName)) {
       failures.push(`Expected routed Ask Coach context to use ${toolName}.`);
@@ -1252,6 +1967,75 @@ function evaluateAskToolProvenance(output, context, testCase) {
     failures.push('Ask answer mentions e1RM/1RM, but routed context did not use get_records.');
   }
 
+  const weightedSetClaims = extractAskWeightedSetClaims(output);
+  for (const claim of weightedSetClaims) {
+    if (isEstimatedOneRepMaxWeightClaim(output, claim)) continue;
+    const referencedExercise = findNearestMentionedExercise(mentionedExercises, claim.index);
+    if (!referencedExercise) continue;
+    const rows = evidenceRows.filter((row) => row.normalizedName === referencedExercise.normalizedName);
+    if (rows.length === 0) {
+      unroutedMentionNames.add(referencedExercise.normalizedName);
+      failures.push(`Ask answer asserts ${claim.text} for ${referencedExercise.name}, but ${referencedExercise.name} was not present in routed tool outputs.`);
+      continue;
+    }
+    if (!toolEvidenceSupportsWeightedSetClaim(claim, rows)) {
+      failures.push(`Ask answer asserts ${claim.text} for ${referencedExercise.name}, but routed tool outputs for ${referencedExercise.name} did not include that weight/reps pair.`);
+    }
+  }
+
+  for (const claim of extractAskWeightClaims(output)) {
+    if (isWithinWeightedSetClaim(claim, weightedSetClaims)) continue;
+    if (isEstimatedOneRepMaxWeightClaim(output, claim)) continue;
+    if (isVolumeWeightClaim(output, claim)) continue;
+    if (isBodyWeightClaim(output, claim)) continue;
+    const referencedExercise = findNearestMentionedExercise(mentionedExercises, claim.index);
+    if (!referencedExercise) continue;
+    const rows = evidenceRows.filter((row) => row.normalizedName === referencedExercise.normalizedName);
+    if (rows.length === 0) {
+      unroutedMentionNames.add(referencedExercise.normalizedName);
+      failures.push(`Ask answer asserts ${claim.text} for ${referencedExercise.name}, but ${referencedExercise.name} was not present in routed tool outputs.`);
+      continue;
+    }
+    if (!toolEvidenceSupportsWeightClaim(claim, rows)) {
+      failures.push(`Ask answer asserts ${claim.text} for ${referencedExercise.name}, but routed tool outputs for ${referencedExercise.name} did not include that weight.`);
+    }
+  }
+
+  const exerciseNames = evidenceRows.map((row) => row.exerciseName);
+  for (const mention of mentionedExercises) {
+    const rows = evidenceRows.filter((row) => row.normalizedName === mention.normalizedName);
+    if (rows.length === 0) {
+      if (toolResults.length > 0 && !unroutedMentionNames.has(mention.normalizedName)) {
+        unroutedMentionNames.add(mention.normalizedName);
+        failures.push(`Ask answer mentions ${mention.name}, but ${mention.name} was not present in routed tool outputs.`);
+      }
+      continue;
+    }
+    const comparable = latestComparableToolRow(rows);
+    if (comparable) {
+      const direction = comparable.comparedToPreviousSession.loadDirection;
+      const previous = comparable.comparedToPreviousSession.previousTopSet;
+      const windows = directionEvaluationWindows(output, mention.name, exerciseNames);
+      if (direction === 'up' && windows.some(hasUnqualifiedDeclineLanguage)) {
+        failures.push(`Ask answer frames ${mention.name} as declining/drop-off, but routed ${comparable.toolName} evidence says top load increased from ${previous.weight} kg to ${comparable.topSet.weight} kg.`);
+      }
+      if (direction === 'down' && windows.some(hasUnqualifiedImprovementLanguage)) {
+        failures.push(`Ask answer frames ${mention.name} as improving, but routed ${comparable.toolName} evidence says top load decreased from ${previous.weight} kg to ${comparable.topSet.weight} kg.`);
+      }
+      if (direction === 'flat' && windows.some((window) => hasUnqualifiedDeclineLanguage(window) || hasUnqualifiedImprovementLanguage(window))) {
+        failures.push(`Ask answer invents a load direction for ${mention.name}, but routed ${comparable.toolName} evidence says top load was flat at ${comparable.topSet.weight} kg.`);
+      }
+    }
+
+    const latestDatedRow = newestToolEvidenceRow(rows, (row) => row.daysAgo != null);
+    if (latestDatedRow && rowIsStaleForEval(latestDatedRow, testCase)) {
+      const windows = recencyEvaluationWindows(output, mention.name, exerciseNames);
+      if (windows.some((window) => outputCallsStaleEvidenceRecent(window, latestDatedRow))) {
+        failures.push(`Ask answer calls ${mention.name} recent, but routed tool evidence says the latest relevant session was ${latestDatedRow.daysAgo} days ago.`);
+      }
+    }
+  }
+
   return {
     key: 'ask_tool_provenance',
     passed: failures.length === 0,
@@ -1261,6 +2045,309 @@ function evaluateAskToolProvenance(output, context, testCase) {
   };
 }
 
+function scoreFormulaEntries(snapshot) {
+  const seen = new Set();
+  return scoreHistoryFromSnapshot(snapshot).filter((entry) => {
+    if (!entry) return false;
+    const key = entry.id ?? entry.snapshotAt;
+    if (key == null) return true;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+function evaluateFormulaVersion(_output, snapshot, testCase) {
+  const expected = testCase.expectedFormulaVersion ?? testCase.formulaVersion ?? null;
+  if (!expected) {
+    return { key: 'formula_version', passed: true, reason: 'No formula version pin configured.' };
+  }
+
+  const entries = scoreFormulaEntries(snapshot);
+  const missingCount = entries.filter((entry) => !entry?.formulaVersion).length;
+  const versions = uniqueStrings(entries.map((entry) => entry?.formulaVersion));
+  const passed = entries.length > 0 && missingCount === 0 && versions.every((version) => version === expected);
+  return {
+    key: 'formula_version',
+    passed,
+    reason: passed
+      ? `Formula version is pinned to ${expected}.`
+      : missingCount > 0
+        ? `Expected formula version ${expected}, but ${missingCount} score snapshot(s) have no formula version.`
+        : versions.length > 0
+        ? `Expected formula version ${expected}, got ${versions.join(', ')}.`
+        : `Expected formula version ${expected}, but snapshot has no increment score formula version.`
+  };
+}
+
+function arrayContainsAll(actual = [], expected = []) {
+  const actualSet = new Set(actual ?? []);
+  return (expected ?? []).every((item) => actualSet.has(item));
+}
+
+function arrayEquals(actual = [], expected = []) {
+  if (!Array.isArray(actual) || !Array.isArray(expected) || actual.length !== expected.length) {
+    return false;
+  }
+  const sortedActual = [...actual].sort();
+  const sortedExpected = [...expected].sort();
+  return sortedActual.every((item, index) => item === sortedExpected[index]);
+}
+
+function askObservationCheckMatches(actualCheck, expectedCheck) {
+  return Object.entries(expectedCheck ?? {}).every(([key, value]) => {
+    if (Array.isArray(value)) return arrayContainsAll(actualCheck?.[key], value);
+    return actualCheck?.[key] === value;
+  });
+}
+
+function evaluateAskEvidencePlan(_output, context, testCase) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_evidence_plan', passed: true, reason: 'Not an ask answer.' };
+  }
+
+  const expected = testCase.expectedEvidencePlan ?? null;
+  if (!expected) {
+    return { key: 'ask_evidence_plan', passed: true, reason: 'No evidence plan assertion configured.' };
+  }
+
+  const plan = context?.routedMetadata?.evidencePlan ?? null;
+  const failures = [];
+  if (!plan) {
+    failures.push('Routed Ask context did not expose metadata.evidencePlan.');
+  } else {
+    for (const key of ['route', 'effectiveRoute', 'fallbackRoute']) {
+      if (key in expected && plan[key] !== expected[key]) {
+        failures.push(`Expected evidencePlan.${key}=${expected[key] ?? 'null'}, got ${plan[key] ?? 'null'}.`);
+      }
+    }
+
+    for (const key of ['requiredTools', 'optionalTools', 'executedTools', 'evidenceGaps']) {
+      if (Array.isArray(expected[key]) && !arrayEquals(plan[key], expected[key])) {
+        failures.push(`Expected evidencePlan.${key} to equal ${expected[key].join(', ')}; got ${(plan[key] ?? []).join(', ')}.`);
+      }
+    }
+
+    if (Array.isArray(expected.excludedExecutedTools)) {
+      const executed = new Set(plan.executedTools ?? []);
+      const hits = expected.excludedExecutedTools.filter((toolName) => executed.has(toolName));
+      if (hits.length > 0) {
+        failures.push(`Expected evidencePlan.executedTools to exclude ${hits.join(', ')}.`);
+      }
+    }
+
+    for (const expectedCheck of expected.observationChecks ?? []) {
+      const matched = (plan.observationChecks ?? []).some((actualCheck) => askObservationCheckMatches(actualCheck, expectedCheck));
+      if (!matched) {
+        failures.push(`Expected observation check ${JSON.stringify(expectedCheck)}; got ${JSON.stringify(plan.observationChecks ?? [])}.`);
+      }
+    }
+    if (Array.isArray(expected.observationChecks) && (plan.observationChecks ?? []).length !== expected.observationChecks.length) {
+      failures.push(`Expected ${expected.observationChecks.length} observation check(s), got ${(plan.observationChecks ?? []).length}.`);
+    }
+  }
+
+  return {
+    key: 'ask_evidence_plan',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Ask evidence plan matches configured assertions.'
+      : failures.join(' ')
+  };
+}
+
+function askMetadataObservationReferences(metadata) {
+  const references = new Set([
+    ...(metadata?.includedCoachObservationIds ?? []),
+    ...(metadata?.coachObservationIds ?? [])
+  ]);
+  for (const comparison of metadata?.sessionObservationComparisons ?? []) {
+    if (comparison?.observationId) references.add(comparison.observationId);
+  }
+  for (const item of metadata?.provenance ?? []) {
+    for (const sourceId of item?.sourceIds ?? []) {
+      references.add(sourceId);
+    }
+  }
+  return references;
+}
+
+function evaluateAskMetadata(output, context, testCase) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_metadata', passed: true, reason: 'Not an ask answer.' };
+  }
+
+  const expected = testCase.expectedMetadata ?? null;
+  if (!expected) {
+    return { key: 'ask_metadata', passed: true, reason: 'No Ask metadata assertion configured.' };
+  }
+
+  const metadata = context?.routedMetadata ?? {};
+  const failures = [];
+
+  if (Array.isArray(expected.includedCoachObservationIds)) {
+    const included = new Set(metadata.includedCoachObservationIds ?? []);
+    const missing = expected.includedCoachObservationIds.filter((id) => !included.has(id));
+    if (missing.length > 0) {
+      failures.push(`Expected included coach observation id(s): ${missing.join(', ')}.`);
+    }
+  }
+
+  if (Array.isArray(expected.excludedCoachObservationIds)) {
+    const references = askMetadataObservationReferences(metadata);
+    const hits = expected.excludedCoachObservationIds.filter((id) => references.has(id));
+    if (hits.length > 0) {
+      failures.push(`Expected coach observation id(s) to be excluded from rendered metadata: ${hits.join(', ')}.`);
+    }
+  }
+
+  if (Array.isArray(expected.forbiddenObservationPhrases)) {
+    const hits = uniqueStrings(expected.forbiddenObservationPhrases).filter((phrase) => phraseIncludes(output, phrase));
+    if (hits.length > 0) {
+      failures.push(`Dismissed or excluded observation phrase(s) leaked into Ask answer: ${hits.join(', ')}.`);
+    }
+  }
+
+  return {
+    key: 'ask_metadata',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Ask metadata matches configured assertions.'
+      : failures.join(' ')
+  };
+}
+
+function normalizedStructuredText(value) {
+  return String(value ?? '')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, ' ')
+    .replace(/\b(my|the|a|an)\b/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function structuredStringArray(value) {
+  return Array.isArray(value)
+    ? value.map((item) => String(item ?? '').trim()).filter(Boolean)
+    : [];
+}
+
+function structuredObjectStringArray(items, key) {
+  return Array.isArray(items)
+    ? items.map((item) => String(item?.[key] ?? '').trim()).filter(Boolean)
+    : [];
+}
+
+function requireStructuredStrings(actual, expected, label, failures) {
+  if (!Array.isArray(expected)) return;
+  const actualSet = new Set(structuredStringArray(actual));
+  const missing = expected.filter((item) => !actualSet.has(item));
+  if (missing.length > 0) {
+    failures.push(`Expected structured ${label}: ${missing.join(', ')}.`);
+  }
+}
+
+function forbidStructuredSuggestions(actual, forbidden, failures) {
+  if (!Array.isArray(forbidden)) return;
+  const normalizedActual = new Set(structuredStringArray(actual).map(normalizedStructuredText).filter(Boolean));
+  const hits = forbidden.filter((item) => normalizedActual.has(normalizedStructuredText(item)));
+  if (hits.length > 0) {
+    failures.push(`Forbidden follow-up suggestion(s) present: ${hits.join(', ')}.`);
+  }
+}
+
+function evaluateAskStructuredResponse(_output, context, testCase, structured) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_structured_response', passed: true, reason: 'Not an ask answer.' };
+  }
+
+  const expected = testCase.expectedStructuredResponse ?? null;
+  if (!expected) {
+    return { key: 'ask_structured_response', passed: true, reason: 'No structured response assertion configured.' };
+  }
+
+  const failures = [];
+  if (!structured || typeof structured !== 'object' || Array.isArray(structured)) {
+    failures.push('Ask structured response was not generated.');
+  } else {
+    if (expected.confidence && structured.confidence !== expected.confidence) {
+      failures.push(`Expected structured confidence ${expected.confidence}, got ${structured.confidence ?? 'null'}.`);
+    }
+
+    requireStructuredStrings(
+      structuredObjectStringArray(structured.evidenceUsed, 'toolName'),
+      expected.requiredEvidenceTools,
+      'evidence tool(s)',
+      failures
+    );
+    requireStructuredStrings(
+      structuredObjectStringArray(structured.evidenceUsed, 'label'),
+      expected.requiredEvidenceLabels,
+      'evidence label(s)',
+      failures
+    );
+    requireStructuredStrings(
+      structuredObjectStringArray(structured.recommendedActions, 'label'),
+      expected.requiredRecommendedActionLabels,
+      'recommended action label(s)',
+      failures
+    );
+    requireStructuredStrings(
+      structured.followUpSuggestions,
+      expected.requiredFollowUpSuggestions,
+      'follow-up suggestion(s)',
+      failures
+    );
+    requireStructuredStrings(
+      structured.limitations,
+      expected.requiredLimitations,
+      'limitation(s)',
+      failures
+    );
+    forbidStructuredSuggestions(structured.followUpSuggestions, expected.forbiddenFollowUpSuggestions, failures);
+
+    const followUps = structuredStringArray(structured.followUpSuggestions);
+    const normalizedFollowUps = followUps.map(normalizedStructuredText).filter(Boolean);
+    const duplicateCount = normalizedFollowUps.length - new Set(normalizedFollowUps).size;
+    if (duplicateCount > 0) {
+      failures.push('Structured follow-up suggestions must be unique.');
+    }
+
+    const normalizedQuestion = normalizedStructuredText(context?.question ?? testCase.context?.question ?? testCase.question ?? '');
+    if (normalizedQuestion && normalizedFollowUps.includes(normalizedQuestion)) {
+      failures.push('Structured follow-up suggestions must not repeat the current user question.');
+    }
+
+    if (Number.isFinite(expected.maxFollowUpSuggestions) && followUps.length > expected.maxFollowUpSuggestions) {
+      failures.push(`Expected at most ${expected.maxFollowUpSuggestions} follow-up suggestion(s), got ${followUps.length}.`);
+    }
+    if (Number.isFinite(expected.minFollowUpSuggestions) && followUps.length < expected.minFollowUpSuggestions) {
+      failures.push(`Expected at least ${expected.minFollowUpSuggestions} follow-up suggestion(s), got ${followUps.length}.`);
+    }
+
+    if (typeof expected.programDraftPresent === 'boolean') {
+      const hasProgramDraft = structured.programDraft != null;
+      if (hasProgramDraft !== expected.programDraftPresent) {
+        failures.push(`Expected programDraft present=${expected.programDraftPresent}, got ${hasProgramDraft}.`);
+      }
+    }
+  }
+
+  return {
+    key: 'ask_structured_response',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Ask structured response matches configured assertions.'
+      : failures.join(' ')
+  };
+}
+
+function askStructuredProgramDraft(parsedAsk, routingMetadata) {
+  const shouldSuppressDraft = routingMetadata?.requestedCoachObservationIntent === 'successor_plan'
+    && routingMetadata?.coachObservationFollowUpMissing === true;
+  return shouldSuppressDraft ? undefined : parsedAsk?.programDraft;
+}
+
 function firstAction(payload) {
   const actions = Array.isArray(payload?.recommendedNextActions) ? payload.recommendedNextActions : [];
   return actions.find((action) => typeof action?.action === 'string' && action.action.trim());
@@ -1489,17 +2576,20 @@ function evaluatePersonaMotivation(output, context, testCase) {
     failures.push('Feedback acknowledges a PR/positive result but frames later-set dropoff in a demotivating way.');
   }
 
+  const dataLimitationCaveat = /\bnot enough\s+(?:\w+\s+){0,5}?(?:data|details?|context|evidence|information|info|signals?|metrics?)\b/i.test(normalized)
+    || /\bnot enough\s+(?:\w+\s+){0,8}?to\s+(?:separate|infer|tie|connect|attribute|blame|claim|say|show|prove|know|call)\b/i.test(normalized);
+
   const discouragingPatterns = [
     /\bdisappointing\b/i,
     /\bunderwhelming\b/i,
     /\bunderperformed\b/i,
     /\bpoor\b/i,
-    /\bnot enough\b/i,
+    /\bnot enough\s+(?:effort|work|volume|intensity|reps?|sets?|weight|load|progress|consistency)\b/i,
     /\bfailed to\b/i,
     /\bstruggled\b/i
   ];
 
-  if (discouragingPatterns.some((pattern) => pattern.test(normalized))) {
+  if (!dataLimitationCaveat && discouragingPatterns.some((pattern) => pattern.test(normalized))) {
     failures.push('Feedback uses discouraging language that is likely to reduce motivation.');
   }
 
@@ -1517,6 +2607,33 @@ export async function runSummaryEvalCase(testCase) {
   return runSummaryEvalCaseFromSnapshot(testCase, snapshot);
 }
 
+// When an ask answer emits a <program_draft> block, it must be valid JSON in the
+// exact Program shape (enums, limits, no forbidden keys) — validated by the same
+// normalizer the runtime uses to accept/drop drafts. Catches malformed drafts in
+// CI instead of silently dropping them in prod. No block = nothing to check.
+function evaluateProgramDraft(output, testCase, parsedAsk = null) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'program_draft', passed: true, reason: 'Not an ask answer.' };
+  }
+  if (!hasProgramDraftBlock(output)) {
+    return { key: 'program_draft', passed: true, reason: 'No program draft block.' };
+  }
+  // Validate against the EXACT runtime rules — the runtime passes
+  // canonicalExerciseName, which strips non-alphanumerics; without it the eval
+  // would green-light drafts (e.g. punctuation-only names) that prod silently drops.
+  const { programDraft } = parsedAsk ?? extractAskProgramDraft(output, {
+    canonicalizeExerciseName: canonicalExerciseName,
+    strict: true
+  });
+  return {
+    key: 'program_draft',
+    passed: programDraft != null,
+    reason: programDraft != null
+      ? 'Program draft is valid JSON matching the required shape.'
+      : 'Program draft block is malformed (invalid JSON, or fails shape/enum/limit validation).'
+  };
+}
+
 export function evaluateSummaryOutputFromSnapshot(testCase, snapshot, output) {
   const context = buildSummaryEvalContext(snapshot, testCase);
   if (context == null) {
@@ -1527,24 +2644,53 @@ export function evaluateSummaryOutputFromSnapshot(testCase, snapshot, output) {
     throw new Error(`Eval case ${testCase.id} produced an empty output`);
   }
 
+  // strict: eval rejects a draft with any malformed nested item (the runtime
+  // salvages it, but partial malformation is a regression signal). The parsed
+  // result also feeds <program_draft> stripping for the other checks.
+  const parsedAsk = testCase.surface === 'ask'
+    ? extractAskProgramDraft(output, { canonicalizeExerciseName: canonicalExerciseName, strict: true })
+    : null;
+  const structuredParsedAsk = testCase.surface === 'ask'
+    ? extractAskProgramDraft(output, { canonicalizeExerciseName: canonicalExerciseName })
+    : null;
+  const visibleOutput = parsedAsk
+    ? stripXMLTagBlocks(parsedAsk.answerText)
+    : output;
+  const structuredAsk = testCase.surface === 'ask'
+    ? buildAskStructuredResponse(visibleOutput, context.routedMetadata ?? {}, {
+        programDraft: askStructuredProgramDraft(structuredParsedAsk, context.routedMetadata),
+        question: context.question ?? testCase.context?.question ?? testCase.question ?? ''
+      })
+    : null;
+
   const checks = [
-    evaluateNoInsight(output, testCase),
-    evaluateShape(output, testCase),
-    evaluateRequiredMentions(output, testCase),
-    evaluateAnyOfMentions(output, testCase),
-    evaluateForbiddenPhrases(output, testCase),
-    evaluateForbiddenMentions(output, testCase),
-    evaluateExerciseMentions(output, snapshot, context, testCase.surface, testCase),
-    evaluateWorkoutClaims(output, context, testCase),
-    evaluateAskClaims(output, snapshot, testCase),
-    evaluateAskToolProvenance(output, context, testCase),
-    evaluateScoreCommentaryAction(output, context, testCase),
-    evaluateScoreCommentarySynthesis(output, context, testCase),
-    evaluateScoreCommentaryExerciseInvention(output, snapshot, context, testCase),
-    evaluateScoreCommentaryBand(output, context, testCase),
-    evaluateScoreCommentaryTone(output, testCase),
-    evaluateScoreCommentaryLength(output, testCase),
-    evaluatePersonaMotivation(output, context, testCase)
+    evaluateNoInsight(visibleOutput, testCase),
+    evaluateShape(visibleOutput, testCase),
+    evaluateRequiredMentions(visibleOutput, testCase),
+    evaluateAnyOfMentions(visibleOutput, testCase),
+    evaluateForbiddenPhrases(visibleOutput, testCase),
+    evaluateForbiddenMentions(visibleOutput, testCase),
+    evaluateExerciseMentions(visibleOutput, snapshot, context, testCase.surface, testCase),
+    evaluateWorkoutClaims(visibleOutput, context, testCase),
+    evaluateAskClaims(visibleOutput, snapshot, testCase),
+    evaluateAskDirectionalConsistency(visibleOutput, snapshot, testCase),
+    evaluateAskScoreVoice(visibleOutput, testCase),
+    evaluateAskSelfReference(visibleOutput, testCase),
+    evaluateAskVolunteeredScore(visibleOutput, testCase),
+    evaluateAskStaleness(visibleOutput, snapshot, testCase),
+    evaluateAskToolProvenance(visibleOutput, context, testCase, snapshot),
+    evaluateFormulaVersion(visibleOutput, snapshot, testCase),
+    evaluateAskEvidencePlan(visibleOutput, context, testCase),
+    evaluateAskMetadata(visibleOutput, context, testCase),
+    evaluateAskStructuredResponse(visibleOutput, context, testCase, structuredAsk),
+    evaluateScoreCommentaryAction(visibleOutput, context, testCase),
+    evaluateScoreCommentarySynthesis(visibleOutput, context, testCase),
+    evaluateScoreCommentaryExerciseInvention(visibleOutput, snapshot, context, testCase),
+    evaluateScoreCommentaryBand(visibleOutput, context, testCase),
+    evaluateScoreCommentaryTone(visibleOutput, testCase),
+    evaluateScoreCommentaryLength(visibleOutput, testCase),
+    evaluatePersonaMotivation(visibleOutput, context, testCase),
+    evaluateProgramDraft(output, testCase, parsedAsk)
   ];
 
   return {
@@ -1567,12 +2713,14 @@ export async function runSummaryEvalCaseFromSnapshot(testCase, snapshot) {
   return evaluateSummaryOutputFromSnapshot(testCase, snapshot, output);
 }
 
-function genericForbiddenPhrasesForSurface(surface) {
+export function genericForbiddenPhrasesForSurface(surface) {
   switch (surface) {
     case 'workout':
       return ['solid progress', 'trust the process', 'keep it up', 'quality work', 'in a great place', 'continue progressive overload', 'as fatigue accumulates'];
     case 'cycle':
-      return ['solid progress', 'trust the process', 'in a great place', 'continue progressive overload', 'as fatigue accumulates', 'solid session', 'quality work'];
+      // 'solid first week' enforces the FIRST_WEEK_CYCLE_PROMPT's "do not say
+      // solid first week" rule, which was previously prompt-only (unguarded).
+      return ['solid progress', 'trust the process', 'in a great place', 'continue progressive overload', 'as fatigue accumulates', 'solid session', 'quality work', 'solid first week'];
     case 'checkpoint':
       return ['solid progress', 'quality work', 'trust the process', 'in a great place'];
     case 'vitals':