incremnt 0.4.0 → 0.6.0

package/src/summary-evals.js

@@ -2,6 +2,8 @@ import { readFile, readdir } from 'node:fs/promises';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import {
+  askContext,
+  askRoutedContext,
   checkpointContext,
   cycleSummaryContext,
   normalizeExerciseName,
@@ -9,11 +11,14 @@ import {
   vitalsSummaryContext
 } from './queries.js';
 import {
+  AI_PROMPT_VERSIONS,
+  generateAskAnswer,
   generateCheckpointSummary,
   generateCoachingSummary,
   generateVitalsSummary,
   generateWorkoutCoachingSummary
 } from './openrouter.js';
+import { computeScoreBand } from './score-context.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -79,36 +84,164 @@ export function buildSummaryEvalContext(snapshot, testCase) {
     }
     case 'vitals':
       return vitalsSummaryContext(snapshot, { exclude: new Set(testCase.exclude ?? []) });
+    case 'ask': {
+      const question = testCase.context?.question ?? testCase.question ?? '';
+      const routed = question
+        ? askRoutedContext(snapshot, question, { exclude: new Set(testCase.exclude ?? []) })
+        : null;
+      return {
+        ...(testCase.context ?? {}),
+        trainingData: testCase.context?.trainingData ?? routed?.context ?? null,
+        routedMetadata: routed?.metadata ?? null
+      };
+    }
+    case 'scoreCommentary':
+      return buildScoreCommentaryContext(snapshot, testCase);
     default:
       throw new Error(`Unsupported summary eval surface: ${testCase.surface}`);
   }
 }
 
-export async function generateSummaryEvalOutput(testCase, context) {
+function summaryEvalGenerationMetadata(result) {
+  if (!result || typeof result !== 'object') return {};
+  return Object.fromEntries(
+    Object.entries({
+      model: result.model,
+      durationMs: result.durationMs,
+      fallback: result.fallback,
+      langfuseTraceId: result.langfuseTraceId,
+      langfuseObservationId: result.langfuseObservationId
+    }).filter(([, value]) => value !== undefined && value !== null)
+  );
+}
+
+export async function generateSummaryEvalOutputWithMetadata(testCase, context, snapshot = null) {
   const liveGenerationEnabled = process.env.SUMMARY_EVALS_LIVE === '1';
   const apiKey = process.env.OPENROUTER_API_KEY;
   if (!liveGenerationEnabled || !apiKey || testCase.shouldPass === false) {
-    return testCase.output;
+    return { output: testCase.output, metadata: {} };
   }
 
+  let result;
   switch (testCase.surface) {
+    case 'scoreCommentary':
+      return { output: testCase.output, metadata: {} };
     case 'workout':
-      return (await generateWorkoutCoachingSummary(context, { apiKey })).text;
+      result = await generateWorkoutCoachingSummary(context, { apiKey });
+      break;
     case 'cycle':
-      return (await generateCoachingSummary(context, { apiKey })).text;
+      result = await generateCoachingSummary(context, { apiKey });
+      break;
     case 'checkpoint':
-      return (await generateCheckpointSummary(context, { apiKey })).text;
+      result = await generateCheckpointSummary(context, { apiKey });
+      break;
     case 'vitals':
-      return (await generateVitalsSummary(context, { apiKey })).text;
+      result = await generateVitalsSummary(context, { apiKey });
+      break;
+    case 'ask': {
+      if (!snapshot) return { output: testCase.output, metadata: {} };
+      const question = context.question ?? testCase.question ?? '';
+      if (!question) return { output: testCase.output, metadata: {} };
+      const trainingContext = context.trainingData
+        ?? askRoutedContext(snapshot, question, { exclude: new Set(testCase.exclude ?? []) }).context
+        ?? askContext(snapshot, { exclude: new Set(testCase.exclude ?? []) });
+      result = await generateAskAnswer(trainingContext, question, {
+        apiKey,
+        history: context.history ?? [],
+        tone: context.tone,
+        model: context.model
+      });
+      break;
+    }
     default:
       throw new Error(`Unsupported summary eval surface: ${testCase.surface}`);
   }
+
+  return {
+    output: result.text,
+    metadata: summaryEvalGenerationMetadata(result)
+  };
+}
+
+export async function generateSummaryEvalOutput(testCase, context, snapshot = null) {
+  return (await generateSummaryEvalOutputWithMetadata(testCase, context, snapshot)).output;
 }
 
 function normalizeText(value) {
   return String(value ?? '').trim();
 }
 
+function parseJsonOutput(output) {
+  const normalized = normalizeText(output);
+  if (!normalized) return null;
+  try {
+    return JSON.parse(normalized);
+  } catch {
+    return null;
+  }
+}
+
+function scoreCommentaryPayload(output) {
+  const parsed = parseJsonOutput(output);
+  if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+    return parsed;
+  }
+  return { summaryText: normalizeText(output) };
+}
+
+function scoreCommentaryText(output) {
+  const payload = scoreCommentaryPayload(output);
+  const pieces = [
+    payload.headline,
+    payload.subhead,
+    payload.summaryText,
+    payload.scoreBand
+  ];
+  const actions = Array.isArray(payload.recommendedNextActions) ? payload.recommendedNextActions : [];
+  for (const action of actions) {
+    pieces.push(action?.driverMessage, action?.action);
+  }
+  const deltaDrivers = Array.isArray(payload.deltaDrivers) ? payload.deltaDrivers : [];
+  for (const driver of deltaDrivers) {
+    pieces.push(driver?.component);
+  }
+  return pieces.filter(Boolean).join(' ');
+}
+
+function scoreHistoryFromSnapshot(snapshot) {
+  const raw = snapshot?.incrementScore;
+  if (Array.isArray(raw)) return raw;
+
+  const history = Array.isArray(raw?.history) ? raw.history : [];
+  const latest = raw?.latest;
+  if (!latest) return history;
+
+  return [
+    latest,
+    ...history.filter((entry) => {
+      if (!entry) return false;
+      if (latest?.id != null && entry?.id != null) return entry.id !== latest.id;
+      if (latest?.snapshotAt != null && entry?.snapshotAt != null) {
+        return entry.snapshotAt !== latest.snapshotAt;
+      }
+      return entry !== latest;
+    })
+  ];
+}
+
+function buildScoreCommentaryContext(snapshot, testCase) {
+  const history = scoreHistoryFromSnapshot(snapshot);
+  const selector = testCase.selector ?? {};
+  if (selector.snapshotId) {
+    return history.find((entry) => entry.id === selector.snapshotId) ?? null;
+  }
+  if (selector.snapshotAt) {
+    return history.find((entry) => entry.snapshotAt === selector.snapshotAt) ?? null;
+  }
+  const index = Number.isInteger(selector.index) ? selector.index : 0;
+  return history[index] ?? null;
+}
+
 function paragraphCount(text) {
   return normalizeText(text)
     .split(/\n\s*\n/)
@@ -129,6 +262,21 @@ function lowerIncludes(text, snippet) {
   return normalizeText(text).toLowerCase().includes(String(snippet).toLowerCase());
 }
 
+function phraseIncludes(text, snippet) {
+  const normalizedSnippet = normalizeText(snippet).trim();
+  if (!normalizedSnippet) return false;
+  const escapedSnippet = escapeRegExp(normalizedSnippet).replace(/\s+/g, '\\s+');
+  const startsWithWord = /^\w/.test(normalizedSnippet);
+  const endsWithWord = /\w$/.test(normalizedSnippet);
+  const pluralSuffix = /^[A-Za-z]{1,3}$/.test(normalizedSnippet) ? 's?' : '';
+  const pattern = `${startsWithWord ? '\\b' : ''}${escapedSnippet}${pluralSuffix}${endsWithWord ? '\\b' : ''}`;
+  return new RegExp(pattern, 'i').test(normalizeText(text));
+}
+
+function escapeRegExp(value) {
+  return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
 function uniqueStrings(values) {
   return [...new Set((values ?? []).filter(Boolean))];
 }
@@ -207,6 +355,9 @@ function collectAllowedExerciseNames(surface, context) {
     for (const pr of context.repPrs ?? []) {
       if (pr.exerciseName) names.add(pr.exerciseName);
     }
+    for (const sc of context.planComparison?.setsComparison ?? []) {
+      if (sc.exercise) names.add(sc.exercise);
+    }
   }
 
   if (surface === 'cycle' && context && typeof context === 'object') {
@@ -235,6 +386,16 @@ function collectAllowedExerciseNames(surface, context) {
     }
   }
 
+  if (surface === 'scoreCommentary' && context && typeof context === 'object') {
+    for (const driver of [
+      ...(context.topPositiveDrivers ?? []),
+      ...(context.topNegativeDrivers ?? [])
+    ]) {
+      if (driver?.relatedExerciseName) names.add(driver.relatedExerciseName);
+      if (driver?.exerciseName) names.add(driver.exerciseName);
+    }
+  }
+
   return [...names];
 }
 
@@ -260,6 +421,15 @@ function historicalExerciseVariants(name) {
 }
 
 function evaluateExerciseMentions(output, snapshot, context, surface, testCase) {
+  if (surface === 'ask') {
+    return {
+      key: 'exercise_mentions',
+      passed: true,
+      reason: 'Ask-coach answers may mention any exercise from the training history.'
+    };
+  }
+
+  const outputText = surface === 'scoreCommentary' ? scoreCommentaryText(output) : output;
   const isStored = testCase.source === 'stored';
   const allowed = new Set();
   for (const name of [
@@ -272,7 +442,7 @@ function evaluateExerciseMentions(output, snapshot, context, surface, testCase)
     }
   }
   const allNames = collectAllExerciseNames(snapshot);
-  const normalizedOutput = normalizeExerciseName(output);
+  const normalizedOutput = normalizeExerciseName(outputText);
   const mentions = [];
 
   for (const exerciseName of allNames) {
@@ -340,7 +510,7 @@ function evaluateAnyOfMentions(output, testCase) {
 }
 
 function evaluateForbiddenPhrases(output, testCase) {
-  const hits = uniqueStrings(testCase.forbiddenPhrases).filter((phrase) => lowerIncludes(output, phrase));
+  const hits = uniqueStrings(testCase.forbiddenPhrases).filter((phrase) => phraseIncludes(output, phrase));
   return {
     key: 'forbidden_phrases',
     passed: hits.length === 0,
@@ -349,7 +519,7 @@ function evaluateForbiddenPhrases(output, testCase) {
 }
 
 function evaluateForbiddenMentions(output, testCase) {
-  const hits = uniqueStrings(testCase.forbiddenMentions).filter((phrase) => lowerIncludes(output, phrase));
+  const hits = uniqueStrings(testCase.forbiddenMentions).filter((phrase) => phraseIncludes(output, phrase));
   return {
     key: 'forbidden_mentions',
     passed: hits.length === 0,
@@ -373,7 +543,9 @@ function evaluateNoInsight(output, testCase) {
 }
 
 function evaluateShape(output, testCase) {
-  const normalized = normalizeText(output);
+  const normalized = testCase.surface === 'scoreCommentary'
+    ? scoreCommentaryText(output)
+    : normalizeText(output);
   if (normalized === 'NO_INSIGHT') {
     return {
       key: 'shape',
@@ -382,8 +554,8 @@ function evaluateShape(output, testCase) {
     };
   }
 
-  const sentences = sentenceCount(output);
-  const paragraphs = paragraphCount(output);
+  const sentences = sentenceCount(normalized);
+  const paragraphs = paragraphCount(normalized);
   const isStored = testCase.source === 'stored';
   let passed = true;
   const reasons = [];
@@ -422,9 +594,17 @@ function evaluateShape(output, testCase) {
       }
       break;
     case 'cycle':
-      if (paragraphs < 1 || paragraphs > 4) {
+      if ((!isStored && (sentences < 2 || sentences > 8)) || (isStored && (sentences < 1 || sentences > 10))) {
         passed = false;
-        reasons.push(`Cycle summaries must be 1-4 paragraphs, got ${paragraphs}.`);
+        reasons.push(isStored
+          ? `Stored cycle summaries must be 1-10 sentences, got ${sentences}.`
+          : `Cycle summaries must be 2-8 sentences, got ${sentences}.`);
+      }
+      if ((!isStored && (paragraphs < 1 || paragraphs > 2)) || (isStored && (paragraphs < 1 || paragraphs > 3))) {
+        passed = false;
+        reasons.push(isStored
+          ? `Stored cycle summaries must be 1-3 paragraphs, got ${paragraphs}.`
+          : `Cycle summaries must be 1-2 paragraphs, got ${paragraphs}.`);
       }
       break;
     case 'checkpoint':
@@ -433,6 +613,18 @@ function evaluateShape(output, testCase) {
         reasons.push(`Checkpoint summaries must be 2-3 paragraphs, got ${paragraphs}.`);
       }
       break;
+    case 'ask':
+      if (sentences < 1 || sentences > 12) {
+        passed = false;
+        reasons.push(`Ask-coach answers must be 1-12 sentences, got ${sentences}.`);
+      }
+      break;
+    case 'scoreCommentary':
+      if (sentences < 1 || sentences > 8) {
+        passed = false;
+        reasons.push(`Score commentary must be 1-8 sentences, got ${sentences}.`);
+      }
+      break;
     default:
       break;
   }
@@ -700,18 +892,598 @@ function evaluateWorkoutClaims(output, context, testCase) {
   };
 }
 
+function extractAskTargetHitClaims(text) {
+  const claims = [];
+  const patterns = [
+    /\b(?:you\s+)?hit(?:ting)?\s+all\s+(?:your\s+)?target(?:ed)?\s+reps?\b/gi,
+    /\b(?:you\s+)?hit\s+all\s+(?:the\s+)?targets?\b/gi,
+    /\b(?:you\s+)?hit\s+(?:the|your)\s+target\b(?!\s+(?:of|for|on))/gi
+  ];
+  for (const pattern of patterns) {
+    for (const match of text.matchAll(pattern)) {
+      claims.push({ text: match[0] });
+    }
+  }
+  return claims;
+}
+
+function extractAskCleanConsistencyClaims(text) {
+  const claims = [];
+  const patterns = [
+    /\bclean,\s+consistent\b/gi,
+    /\bclean\s+and\s+consistent\b/gi,
+    /\bconsistent\s+set\s+of\s+work\b/gi,
+    /\bacross\s+the\s+board\b/gi
+  ];
+  for (const pattern of patterns) {
+    for (const match of text.matchAll(pattern)) {
+      claims.push({ text: match[0] });
+    }
+  }
+  return claims;
+}
+
+function extractAskPlannedListClaims(text) {
+  const claims = [];
+  const pattern = /\((\s*\d+(?:\s*,\s*\d+){2,})\s+planned\s*\)/gi;
+  for (const match of text.matchAll(pattern)) {
+    const reps = match[1].split(',').map((value) => Number(value.trim())).filter(Number.isFinite);
+    claims.push({ text: match[0], reps, index: match.index ?? -1 });
+  }
+  return claims;
+}
+
+function sessionPlannedReps(session) {
+  const prescription = session?.prescriptionSnapshot?.exercises ?? [];
+  const values = [];
+  for (const exercise of prescription) {
+    const target = Number(exercise.targetReps);
+    if (Number.isFinite(target) && target > 0) values.push(target);
+  }
+  return values;
+}
+
+function findMentionedExercises(text, snapshot) {
+  const exercisesByName = new Map();
+  for (const session of snapshot?.sessions ?? []) {
+    for (const exercise of session.exercises ?? []) {
+      if (!exercise?.name) continue;
+      const normalizedName = normalizeExerciseName(exercise.name);
+      if (!normalizedName || exercisesByName.has(normalizedName)) continue;
+      exercisesByName.set(normalizedName, exercise.name);
+    }
+    for (const exercise of session.prescriptionSnapshot?.exercises ?? []) {
+      if (!exercise?.exerciseName) continue;
+      const normalizedName = normalizeExerciseName(exercise.exerciseName);
+      if (!normalizedName || exercisesByName.has(normalizedName)) continue;
+      exercisesByName.set(normalizedName, exercise.exerciseName);
+    }
+  }
+
+  const mentions = [];
+  for (const [normalizedName, displayName] of exercisesByName) {
+    const pattern = new RegExp(`\\b${escapeRegExp(displayName)}\\b`, 'gi');
+    for (const match of text.matchAll(pattern)) {
+      mentions.push({
+        index: match.index ?? -1,
+        end: (match.index ?? -1) + match[0].length,
+        name: displayName,
+        normalizedName
+      });
+    }
+  }
+  return mentions
+    .filter((mention, index, allMentions) => !allMentions.some((candidate, candidateIndex) =>
+      candidateIndex !== index &&
+      candidate.index <= mention.index &&
+      candidate.end >= mention.end &&
+      candidate.normalizedName.length > mention.normalizedName.length
+    ))
+    .sort((lhs, rhs) => lhs.index - rhs.index);
+}
+
+function findRecentSessionMisses(snapshot, { lookbackDays = 7, exerciseNames = null } = {}) {
+  const sessions = snapshot?.sessions ?? [];
+  const cutoff = Date.now() - lookbackDays * 24 * 60 * 60 * 1000;
+  const scopedExerciseNames = exerciseNames && exerciseNames.length > 0 ? new Set(exerciseNames) : null;
+  const misses = [];
+  for (const session of sessions) {
+    const completedAt = session.completedAt || session.date;
+    const completedTime = Date.parse(completedAt);
+    if (!Number.isFinite(completedTime) || completedTime < cutoff) continue;
+    const targetByExercise = new Map();
+    for (const planned of session.prescriptionSnapshot?.exercises ?? []) {
+      const target = Number(planned.targetReps);
+      if (Number.isFinite(target) && target > 0) {
+        targetByExercise.set(normalizeExerciseName(planned.exerciseName), target);
+      }
+    }
+    for (const exercise of session.exercises ?? []) {
+      const normalizedExerciseName = normalizeExerciseName(exercise.name);
+      if (scopedExerciseNames && !scopedExerciseNames.has(normalizedExerciseName)) continue;
+      const target = targetByExercise.get(normalizedExerciseName);
+      if (!Number.isFinite(target)) continue;
+      for (const set of exercise.sets ?? []) {
+        const reps = Number(set.reps);
+        if (set.isComplete && Number.isFinite(reps) && reps < target) {
+          misses.push({ sessionId: session.id, exerciseName: exercise.name, reps, target });
+        }
+      }
+    }
+  }
+  return misses;
+}
+
+function findNearestMentionedExercise(mentions, index) {
+  let candidate = null;
+  for (const mention of mentions) {
+    if (mention.index > index) break;
+    candidate = mention;
+  }
+  return candidate;
+}
+
+function hasAskFatigueSupport(snapshot, lookbackDays = 7) {
+  const cutoff = Date.now() - lookbackDays * 24 * 60 * 60 * 1000;
+  const withinCutoff = (dateValue) => {
+    const ms = Date.parse(dateValue);
+    return Number.isFinite(ms) && ms >= cutoff;
+  };
+
+  const vitalsSummaries = snapshot?.vitalsSummaries ?? [];
+  if (vitalsSummaries.some((entry) => withinCutoff(entry.date))) return true;
+
+  const metrics = snapshot?.healthMetrics ?? {};
+  for (const key of ['restingHR', 'hrv', 'sleep']) {
+    const readings = Array.isArray(metrics[key]) ? metrics[key] : [];
+    if (readings.some((reading) => withinCutoff(reading.date))) return true;
+  }
+
+  for (const session of snapshot?.sessions ?? []) {
+    const completedAt = session.completedAt || session.date;
+    if (!withinCutoff(completedAt)) continue;
+    for (const exercise of session.exercises ?? []) {
+      const reps = (exercise.sets ?? [])
+        .map((set) => Number(set.reps))
+        .filter((value) => Number.isFinite(value) && value > 0);
+      if (reps.length < 2) continue;
+      const first = reps[0];
+      const last = reps[reps.length - 1];
+      if (first > 0 && (first - last) / first >= 0.3) return true;
+    }
+  }
+
+  return false;
+}
+
+function extractAskWeightClaims(text) {
+  const claims = [];
+  const pattern = /\b(\d+(?:\.\d+)?)\s*(?:kg|kilograms?)\b/gi;
+  for (const match of text.matchAll(pattern)) {
+    claims.push({
+      text: match[0],
+      value: Number(match[1]),
+      index: match.index ?? -1
+    });
+  }
+  return claims;
+}
+
+function allowedWeightsForExercise(snapshot, normalizedExerciseName) {
+  const weights = [];
+  for (const session of snapshot?.sessions ?? []) {
+    for (const exercise of session.exercises ?? []) {
+      if (normalizeExerciseName(exercise.name) !== normalizedExerciseName) continue;
+      for (const set of exercise.sets ?? []) {
+        const weight = Number(set.weight);
+        if (Number.isFinite(weight)) weights.push(weight);
+      }
+    }
+    for (const exercise of session.prescriptionSnapshot?.exercises ?? []) {
+      if (normalizeExerciseName(exercise.exerciseName) !== normalizedExerciseName) continue;
+      const targetWeight = Number(exercise.targetWeight);
+      if (Number.isFinite(targetWeight)) weights.push(targetWeight);
+      for (const targetSet of exercise.targetSets ?? []) {
+        const weight = Number(targetSet.weight ?? targetSet.targetWeight);
+        if (Number.isFinite(weight)) weights.push(weight);
+      }
+    }
+  }
+  return weights;
+}
+
+function weightClaimSupported(claim, allowedWeights) {
+  return allowedWeights.some((weight) => Math.abs(weight - claim.value) < 0.01);
+}
+
+function isEstimatedOneRepMaxWeightClaim(text, claim) {
+  const start = Math.max(0, claim.index - 40);
+  const end = Math.min(text.length, claim.index + claim.text.length + 40);
+  const window = text.slice(start, end);
+  return /\b(?:estimated\s+)?(?:1rm|one[-\s]?rep\s+max)\b/i.test(window);
+}
+
+function isVolumeWeightClaim(text, claim) {
+  const start = Math.max(0, claim.index - 30);
+  const end = Math.min(text.length, claim.index + claim.text.length + 30);
+  const window = text.slice(start, end);
+  return /\bvolume\b/i.test(window);
+}
+
+function evaluateAskClaims(output, snapshot, testCase) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_claims', passed: true, reason: 'Not an ask answer.' };
+  }
+
+  const normalized = normalizeText(output);
+  if (normalized === 'NO_INSIGHT' || !normalized) {
+    return { key: 'ask_claims', passed: true, reason: 'NO_INSIGHT bypasses ask claim validation.' };
+  }
+
+  const failures = [];
+  const mentionedExercises = findMentionedExercises(output, snapshot);
+  const scopedExerciseNames = uniqueStrings(mentionedExercises.map((mention) => mention.normalizedName));
+
+  const targetHitClaims = extractAskTargetHitClaims(normalized);
+  if (targetHitClaims.length > 0) {
+    const misses = findRecentSessionMisses(snapshot, { exerciseNames: scopedExerciseNames });
+    if (misses.length > 0) {
+      const sample = misses[0];
+      failures.push(`Ask answer claims targets hit ("${targetHitClaims[0].text}") but recent session ${sample.sessionId} has ${sample.exerciseName} at ${sample.reps} reps below target ${sample.target}.`);
+    }
+  }
+
+  const cleanConsistencyClaims = extractAskCleanConsistencyClaims(normalized);
+  if (cleanConsistencyClaims.length > 0) {
+    const misses = findRecentSessionMisses(snapshot, { exerciseNames: scopedExerciseNames });
+    if (misses.length > 0) {
+      const sample = misses[0];
+      failures.push(`Ask answer frames missed target reps as "${cleanConsistencyClaims[0].text}", but recent session ${sample.sessionId} has ${sample.exerciseName} at ${sample.reps} reps below target ${sample.target}.`);
+    }
+  }
+
+  for (const claim of extractAskPlannedListClaims(normalized)) {
+    const uniquePlanned = new Set(claim.reps);
+    if (uniquePlanned.size > 1) {
+      failures.push(`Ask answer asserts non-uniform plan "${claim.text}". Plans are uniform per exercise; this list likely restates actual reps as if they were planned.`);
+      continue;
+    }
+    const plannedValue = claim.reps[0];
+    const referencedExercise = findNearestMentionedExercise(mentionedExercises, claim.index);
+    const anySessionHasThisPlan = (snapshot?.sessions ?? []).some((session) => {
+      if (!referencedExercise) {
+        return sessionPlannedReps(session).some((value) => value === plannedValue);
+      }
+      const plannedExercises = session?.prescriptionSnapshot?.exercises ?? [];
+      return plannedExercises.some((exercise) => {
+        const exerciseName = normalizeExerciseName(exercise.exerciseName);
+        const target = Number(exercise.targetReps);
+        return exerciseName === referencedExercise.normalizedName && target === plannedValue;
+      });
+    });
+    if (!anySessionHasThisPlan) {
+      if (referencedExercise) {
+        failures.push(`Ask answer asserts plan "${claim.text}" for ${referencedExercise.name} but no session in the snapshot has that target rep count for that exercise.`);
+      } else {
+        failures.push(`Ask answer asserts plan "${claim.text}" but no session in the snapshot has that target rep count.`);
+      }
+    }
+  }
+
+  for (const claim of extractAskWeightClaims(normalized)) {
+    if (isEstimatedOneRepMaxWeightClaim(normalized, claim)) continue;
+    if (isVolumeWeightClaim(normalized, claim)) continue;
+    const referencedExercise = findNearestMentionedExercise(mentionedExercises, claim.index);
+    if (!referencedExercise) continue;
+    const allowedWeights = allowedWeightsForExercise(snapshot, referencedExercise.normalizedName);
+    if (allowedWeights.length === 0) continue;
+    if (!weightClaimSupported(claim, allowedWeights)) {
+      failures.push(`Ask answer asserts ${claim.text} for ${referencedExercise.name}, but that weight is not present in recorded or planned sets for that exercise.`);
+    }
+  }
+
+  if (hasFatigueLanguage(normalized) && !hasAskFatigueSupport(snapshot)) {
+    failures.push('Ask answer uses fatigue/recovery language but the snapshot has no recent vitals, sleep, or rep-dropoff signals to support it.');
+  }
+
+  return {
+    key: 'ask_claims',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Ask claims are supported by the snapshot.'
+      : failures.join(' ')
+  };
+}
+
+function evaluateAskToolProvenance(output, context, testCase) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_tool_provenance', passed: true, reason: 'Not an ask answer.' };
+  }
+
+  const routedMetadata = context?.routedMetadata ?? {};
+  const toolsUsed = new Set(routedMetadata.toolsUsed ?? []);
+  const failures = [];
+  for (const toolName of uniqueStrings(testCase.requiredTools)) {
+    if (!toolsUsed.has(toolName)) {
+      failures.push(`Expected routed Ask Coach context to use ${toolName}.`);
+    }
+  }
+
+  if (/\b(?:estimated\s+)?(?:e1rm|1rm|one[- ]rep max)\b/i.test(output) && !toolsUsed.has('get_records')) {
+    failures.push('Ask answer mentions e1RM/1RM, but routed context did not use get_records.');
+  }
+
+  return {
+    key: 'ask_tool_provenance',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Ask tool provenance matches requirements.'
+      : failures.join(' ')
+  };
+}
+
+function firstAction(payload) {
+  const actions = Array.isArray(payload?.recommendedNextActions) ? payload.recommendedNextActions : [];
+  return actions.find((action) => typeof action?.action === 'string' && action.action.trim());
+}
+
+function evaluateScoreCommentaryAction(output, context, testCase) {
+  if (testCase.surface !== 'scoreCommentary') {
+    return { key: 'score_commentary_action', passed: true, reason: 'Not score commentary.' };
+  }
+
+  const payload = scoreCommentaryPayload(output);
+  const hasNegativeDrivers = Array.isArray(context?.topNegativeDrivers) && context.topNegativeDrivers.length > 0;
+  const hasAction = Boolean(firstAction(payload));
+  const noTrainingSignal = context?.dataTier === 'noTrainingSignal';
+  const failures = [];
+
+  if (noTrainingSignal && hasAction) {
+    failures.push('No-training-signal score commentary must not include a recommended action.');
+  }
+  if (!noTrainingSignal && hasNegativeDrivers && !hasAction) {
+    failures.push('Score commentary with negative drivers must include a recommended action.');
+  }
+
+  return {
+    key: 'score_commentary_action',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Score commentary action presence matches score state.'
+      : failures.join(' ')
+  };
+}
+
+function driverKeyword(driver) {
+  const text = normalizeText(driver?.message ?? driver?.label ?? driver?.driverMessage ?? driver?.component ?? '');
+  const tokens = text
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((token) => token.length >= 4 && !['score', 'sets', 'work', 'this', 'that', 'with', 'from', 'your'].includes(token));
+  return tokens[0] ?? null;
+}
+
+function evaluateScoreCommentarySynthesis(output, context, testCase) {
+  if (testCase.surface !== 'scoreCommentary') {
+    return { key: 'score_commentary_synthesis', passed: true, reason: 'Not score commentary.' };
+  }
+
+  const positives = context?.topPositiveDrivers ?? [];
+  const negatives = context?.topNegativeDrivers ?? [];
+  if (positives.length === 0 || negatives.length === 0) {
+    return {
+      key: 'score_commentary_synthesis',
+      passed: true,
+      reason: 'Mixed-driver synthesis is not required for one-sided commentary.'
+    };
+  }
+
+  const text = scoreCommentaryText(output).toLowerCase();
+  const positiveKeyword = driverKeyword(positives[0]);
+  const negativeKeyword = driverKeyword(negatives[0]);
+  const hasPositive = positiveKeyword ? text.includes(positiveKeyword) : true;
+  const hasNegative = negativeKeyword ? text.includes(negativeKeyword) : true;
+  const hasContrast = /\bbut\b|\bwhile\b|\balthough\b|\bhowever\b/.test(text);
+
+  return {
+    key: 'score_commentary_synthesis',
+    passed: hasPositive && hasNegative && hasContrast,
+    reason: hasPositive && hasNegative && hasContrast
+      ? 'Mixed positive and negative drivers are synthesized.'
+      : 'Mixed-driver commentary must reference both directions with contrast language.'
+  };
+}
+
+function evaluateScoreCommentaryExerciseInvention(output, snapshot, context, testCase) {
+  if (testCase.surface !== 'scoreCommentary') {
+    return { key: 'score_commentary_no_exercise_invention', passed: true, reason: 'Not score commentary.' };
+  }
+
+  const allowedText = [
+    ...(context?.topPositiveDrivers ?? []),
+    ...(context?.topNegativeDrivers ?? [])
+  ].map((driver) => [
+    driver?.message,
+    driver?.label,
+    driver?.relatedExerciseName,
+    driver?.relatedExerciseSlug,
+    driver?.exerciseName
+  ].filter(Boolean).join(' ')).join(' ');
+  const normalizedAllowed = normalizeExerciseName(allowedText);
+  const normalizedOutput = normalizeExerciseName(scoreCommentaryText(output));
+  const invented = [];
+
+  for (const exerciseName of collectAllExerciseNames(snapshot)) {
+    const normalizedName = normalizeExerciseName(exerciseName);
+    if (!normalizedName) continue;
+    const pattern = new RegExp(`(?<!\\S)${escapeRegex(normalizedName)}(?!\\S)`);
+    if (pattern.test(normalizedOutput) && !pattern.test(normalizedAllowed)) {
+      invented.push(exerciseName);
+    }
+  }
+
+  return {
+    key: 'score_commentary_no_exercise_invention',
+    passed: invented.length === 0,
+    reason: invented.length === 0
+      ? 'Score commentary only names exercises present in score drivers.'
+      : `Invented exercise mention(s): ${uniqueStrings(invented).join(', ')}`
+  };
+}
+
+function evaluateScoreCommentaryBand(output, context, testCase) {
+  if (testCase.surface !== 'scoreCommentary') {
+    return { key: 'score_commentary_score_band', passed: true, reason: 'Not score commentary.' };
+  }
+
+  const payload = scoreCommentaryPayload(output);
+  const expected = computeScoreBand(context?.score);
+  const actual = payload.scoreBand ?? null;
+  return {
+    key: 'score_commentary_score_band',
+    passed: actual === expected,
+    reason: actual === expected
+      ? 'Score band matches canonical thresholds.'
+      : `Expected scoreBand ${expected ?? 'null'}, got ${actual ?? 'null'}.`
+  };
+}
+
+function evaluateScoreCommentaryTone(output, testCase) {
+  if (testCase.surface !== 'scoreCommentary') {
+    return { key: 'score_commentary_tone', passed: true, reason: 'Not score commentary.' };
+  }
+
+  const text = scoreCommentaryText(output);
+  const failures = [];
+  if (text.includes('!')) failures.push('exclamation mark');
+  if (/\b(?:great job|awesome|crushing it|amazing work)\b/i.test(text)) failures.push('cheerleading phrase');
+  if (/[\u{1F300}-\u{1FAFF}]/u.test(text)) failures.push('emoji');
+
+  return {
+    key: 'score_commentary_tone',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Score commentary tone is calm and non-cheerleading.'
+      : `Tone violation(s): ${failures.join(', ')}.`
+  };
+}
+
+function evaluateScoreCommentaryLength(output, testCase) {
+  if (testCase.surface !== 'scoreCommentary') {
+    return { key: 'score_commentary_length', passed: true, reason: 'Not score commentary.' };
+  }
+
+  const payload = scoreCommentaryPayload(output);
+  const summary = normalizeText(payload.subhead ?? payload.summaryText);
+  const action = normalizeText(firstAction(payload)?.action);
+  const failures = [];
+  if (summary.length > 140) failures.push(`summaryText/subhead is ${summary.length} chars, expected <= 140.`);
+  if (action && action.length > 140) failures.push(`action is ${action.length} chars, expected <= 140.`);
+
+  return {
+    key: 'score_commentary_length',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Score commentary text fits length bounds.'
+      : failures.join(' ')
+  };
+}
+
+function personaEvalConfig(testCase) {
+  if (testCase.personaEval === false) return null;
+  if (testCase.personaEval && typeof testCase.personaEval === 'object') {
+    return testCase.personaEval;
+  }
+  if (testCase.surface === 'workout' || testCase.surface === 'ask') {
+    return {
+      profile: 'motivated-athlete'
+    };
+  }
+  return null;
+}
+
+function hasPrSignal(output, context, testCase) {
+  const normalized = normalizeText(output);
+  if (/\b(?:new\s+)?(?:rep\s+)?prs?\b/i.test(normalized)) return true;
+  if (/\bstrength prs?\b/i.test(normalized)) return true;
+  if (/\bpersonal record\b/i.test(normalized)) return true;
+  if (testCase.surface === 'workout') {
+    const totalPrCount = (context.prs?.length ?? 0) + (context.bwPrs?.length ?? 0) + (context.repPrs?.length ?? 0);
+    return totalPrCount > 0;
+  }
+  return false;
+}
+
+function evaluatePersonaMotivation(output, context, testCase) {
+  const config = personaEvalConfig(testCase);
+  if (!config) {
+    return {
+      key: 'persona_motivation',
+      passed: true,
+      reason: 'Persona evaluation disabled.'
+    };
+  }
+
+  const normalized = normalizeText(output);
+  if (normalized === 'NO_INSIGHT') {
+    return {
+      key: 'persona_motivation',
+      passed: true,
+      reason: 'NO_INSIGHT bypasses persona evaluation.'
+    };
+  }
+
+  const failures = [];
+  const prSignal = hasPrSignal(normalized, context, testCase);
+
+  const minimizingPatterns = [
+    /\bcould only\b/i,
+    /\bonly\s+\d+\s+reps?\b/i,
+    /\bjust\s+\d+\s+reps?\b/i,
+    /\bbefore tapering to\b/i,
+    /\bbefore dropping to\b/i,
+    /\bbut dropped to\b/i,
+    /\bbut fell to\b/i
+  ];
+
+  if (prSignal && minimizingPatterns.some((pattern) => pattern.test(normalized))) {
+    failures.push('Feedback acknowledges a PR/positive result but frames later-set dropoff in a demotivating way.');
+  }
+
+  const discouragingPatterns = [
+    /\bdisappointing\b/i,
+    /\bunderwhelming\b/i,
+    /\bunderperformed\b/i,
+    /\bpoor\b/i,
+    /\bnot enough\b/i,
+    /\bfailed to\b/i,
+    /\bstruggled\b/i
+  ];
+
+  if (discouragingPatterns.some((pattern) => pattern.test(normalized))) {
+    failures.push('Feedback uses discouraging language that is likely to reduce motivation.');
+  }
+
+  return {
+    key: 'persona_motivation',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? `Output stays aligned with the ${config.profile ?? 'persona'} perspective.`
+      : failures.join(' ')
+  };
+}
+
 export async function runSummaryEvalCase(testCase) {
   const snapshot = await loadSummaryEvalSnapshot(testCase);
   return runSummaryEvalCaseFromSnapshot(testCase, snapshot);
 }
 
-export async function runSummaryEvalCaseFromSnapshot(testCase, snapshot) {
+export function evaluateSummaryOutputFromSnapshot(testCase, snapshot, output) {
   const context = buildSummaryEvalContext(snapshot, testCase);
   if (context == null) {
     throw new Error(`Eval case ${testCase.id} produced no context`);
   }
 
-  const output = await generateSummaryEvalOutput(testCase, context);
   if (!normalizeText(output)) {
     throw new Error(`Eval case ${testCase.id} produced an empty output`);
   }
@@ -724,7 +1496,16 @@ export async function runSummaryEvalCaseFromSnapshot(testCase, snapshot) {
     evaluateForbiddenPhrases(output, testCase),
     evaluateForbiddenMentions(output, testCase),
     evaluateExerciseMentions(output, snapshot, context, testCase.surface, testCase),
-    evaluateWorkoutClaims(output, context, testCase)
+    evaluateWorkoutClaims(output, context, testCase),
+    evaluateAskClaims(output, snapshot, testCase),
+    evaluateAskToolProvenance(output, context, testCase),
+    evaluateScoreCommentaryAction(output, context, testCase),
+    evaluateScoreCommentarySynthesis(output, context, testCase),
+    evaluateScoreCommentaryExerciseInvention(output, snapshot, context, testCase),
+    evaluateScoreCommentaryBand(output, context, testCase),
+    evaluateScoreCommentaryTone(output, testCase),
+    evaluateScoreCommentaryLength(output, testCase),
+    evaluatePersonaMotivation(output, context, testCase)
   ];
 
   return {
@@ -732,11 +1513,21 @@ export async function runSummaryEvalCaseFromSnapshot(testCase, snapshot) {
     surface: testCase.surface,
     name: testCase.name,
     output,
+    metadata: testCase.metadata ?? null,
     passed: checks.every((check) => check.passed),
     checks
   };
 }
 
+export async function runSummaryEvalCaseFromSnapshot(testCase, snapshot) {
+  const context = buildSummaryEvalContext(snapshot, testCase);
+  if (context == null) {
+    throw new Error(`Eval case ${testCase.id} produced no context`);
+  }
+  const output = await generateSummaryEvalOutput(testCase, context, snapshot);
+  return evaluateSummaryOutputFromSnapshot(testCase, snapshot, output);
+}
+
 function genericForbiddenPhrasesForSurface(surface) {
   switch (surface) {
     case 'workout':
@@ -747,6 +1538,10 @@ function genericForbiddenPhrasesForSurface(surface) {
       return ['solid progress', 'quality work', 'trust the process', 'in a great place'];
     case 'vitals':
       return [];
+    case 'ask':
+      return [];
+    case 'scoreCommentary':
+      return ['great job', 'awesome', 'crushing it', 'amazing work'];
     default:
       return [];
   }
@@ -764,7 +1559,51 @@ function latestCycleSummariesWithAI(snapshot) {
   return [...latestByProgram.values()];
 }
 
-export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapshot') {
+function normalizeStoredAIMetadata(surface, metadata, {
+  model = null,
+  generatedAt = null,
+  promptVersion = null,
+  gitSha = null
+} = {}) {
+  if (!metadata && !model && !generatedAt && !promptVersion && !gitSha) {
+    return null;
+  }
+
+  return {
+    surface,
+    generatedAt: metadata?.generatedAt ?? generatedAt ?? null,
+    model: metadata?.model ?? model ?? null,
+    promptVersion: metadata?.promptVersion ?? promptVersion ?? null,
+    gitSha: metadata?.gitSha ?? gitSha ?? null,
+    langfuseTraceId: metadata?.langfuseTraceId ?? metadata?.traceId ?? null,
+    langfuseObservationId: metadata?.langfuseObservationId ?? metadata?.observationId ?? null
+  };
+}
+
+export function currentPromptVersions() {
+  return new Set(Object.values(AI_PROMPT_VERSIONS));
+}
+
+function metadataPassesFilter(metadata, { onlyPromptVersions, since }) {
+  if (onlyPromptVersions) {
+    const version = metadata?.promptVersion;
+    if (!version || !onlyPromptVersions.has(version)) return false;
+  }
+  if (since) {
+    const generatedAt = metadata?.generatedAt;
+    if (!generatedAt || generatedAt < since) return false;
+  }
+  return true;
+}
+
+export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapshot', options = {}) {
+  const filter = {
+    onlyPromptVersions: options.onlyPromptVersions ?? null,
+    since: options.since ?? null
+  };
+  const pushIfIncluded = (bucket, item) => {
+    if (metadataPassesFilter(item.metadata, filter)) bucket.push(item);
+  };
   const harvested = [];
 
   for (const session of snapshot.sessions ?? []) {
@@ -774,7 +1613,7 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
       ...(session.exercises ?? []).map((exercise) => exercise.name),
       session.dayName
     ].filter(Boolean);
-    harvested.push({
+    pushIfIncluded(harvested, {
       id: `stored-workout-${session.id}`,
       name: `Stored workout summary ${session.id}`,
       surface: 'workout',
@@ -782,10 +1621,14 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
       snapshotLabel,
       selector: { sessionId: session.id },
       output: text,
+      metadata: normalizeStoredAIMetadata('workout', session.summary?.aiCoachMetadata, {
+        model: session.summary?.aiCoachModel ?? null
+      }),
       requiredMentions: [],
       requiredAnyOfMentions: anyOf,
       forbiddenPhrases: genericForbiddenPhrasesForSurface('workout'),
       forbiddenMentions: [],
+      personaEval: { profile: 'motivated-athlete' },
       expectNoInsight: false,
       shouldPass: true
     });
@@ -799,7 +1642,7 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
       'hrv',
       'resting hr'
     ].filter(Boolean);
-    harvested.push({
+    pushIfIncluded(harvested, {
       id: `stored-cycle-${summary.id}`,
       name: `Stored cycle summary ${summary.id}`,
       surface: 'cycle',
@@ -807,10 +1650,14 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
       snapshotLabel,
       selector: { programId: summary.programId },
       output: summary.aiSummary,
+      metadata: normalizeStoredAIMetadata('cycle', summary.aiSummaryMetadata, {
+        model: summary.aiSummaryModel ?? null
+      }),
       requiredMentions: [],
       requiredAnyOfMentions: anyOf,
       forbiddenPhrases: genericForbiddenPhrasesForSurface('cycle'),
       forbiddenMentions: [],
+      personaEval: false,
       expectNoInsight: false,
       shouldPass: true
     });
@@ -818,7 +1665,7 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
 
   const latestVitals = stableSortByDateDesc(snapshot.vitalsSummaries ?? [], (entry) => entry.date)[0];
   if (latestVitals?.summary) {
-    harvested.push({
+    pushIfIncluded(harvested, {
       id: `stored-vitals-${latestVitals.id ?? latestVitals.date}`,
       name: `Stored vitals summary ${latestVitals.date}`,
       surface: 'vitals',
@@ -826,10 +1673,42 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
       snapshotLabel,
       selector: {},
       output: latestVitals.summary,
+      metadata: normalizeStoredAIMetadata('vitals', latestVitals.metadata, {
+        model: latestVitals.model ?? null,
+        generatedAt: latestVitals.date ?? null
+      }),
       requiredMentions: [],
       requiredAnyOfMentions: ['recovery', 'readiness', 'train', 'session', 'today'],
       forbiddenPhrases: genericForbiddenPhrasesForSurface('vitals'),
       forbiddenMentions: [],
+      personaEval: false,
+      expectNoInsight: false,
+      shouldPass: true
+    });
+  }
+
+  for (const conversation of stableSortByDateDesc(snapshot.askConversations ?? [], (entry) => entry.createdAt ?? '')) {
+    const answer = conversation?.messages?.filter((message) => message?.role === 'assistant').at(-1)?.content ?? null;
+    const question = conversation?.messages?.find((message) => message?.role === 'user')?.content ?? '';
+    if (!answer) continue;
+
+    pushIfIncluded(harvested, {
+      id: `stored-ask-${conversation.id}`,
+      name: `Stored ask conversation ${conversation.id}`,
+      surface: 'ask',
+      source: 'stored',
+      snapshotLabel,
+      context: { question, conversationId: conversation.id },
+      output: answer,
+      metadata: normalizeStoredAIMetadata('ask', conversation.metadata, {
+        model: conversation.model ?? null,
+        generatedAt: conversation.createdAt ?? null
+      }),
+      requiredMentions: [],
+      requiredAnyOfMentions: [],
+      forbiddenPhrases: genericForbiddenPhrasesForSurface('ask'),
+      forbiddenMentions: [],
+      personaEval: { profile: 'motivated-athlete' },
       expectNoInsight: false,
       shouldPass: true
     });