incremnt 0.3.0 → 0.4.0

package/src/summary-evals.js

@@ -0,0 +1,839 @@
+import { readFile, readdir } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  checkpointContext,
+  cycleSummaryContext,
+  normalizeExerciseName,
+  workoutSummaryContext,
+  vitalsSummaryContext
+} from './queries.js';
+import {
+  generateCheckpointSummary,
+  generateCoachingSummary,
+  generateVitalsSummary,
+  generateWorkoutCoachingSummary
+} from './openrouter.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+export const summaryEvalFixturesRoot = path.resolve(__dirname, '../test/fixtures/summary-evals');
+
+export function defaultCaseSetName() {
+  return process.env.SUMMARY_EVAL_CASE_SET || 'synthetic';
+}
+
+function stableSortByDateDesc(items, selector) {
+  return [...items].sort((lhs, rhs) => String(selector(rhs)).localeCompare(String(selector(lhs))));
+}
+
+export async function loadSummaryEvalCases(caseSet = defaultCaseSetName()) {
+  const casesDir = path.join(summaryEvalFixturesRoot, caseSet, 'cases');
+  let caseFiles;
+  try {
+    caseFiles = (await readdir(casesDir))
+      .filter((file) => file.endsWith('.json'))
+      .sort();
+  } catch (error) {
+    if (error?.code === 'ENOENT') {
+      return [];
+    }
+    throw error;
+  }
+
+  const loaded = [];
+  for (const file of caseFiles) {
+    const data = JSON.parse(await readFile(path.join(casesDir, file), 'utf8'));
+    loaded.push({
+      ...data,
+      caseSet,
+      fixtureFile: file
+    });
+  }
+  return loaded;
+}
+
+export async function loadSummaryEvalSnapshot(testCase) {
+  if (!testCase.snapshotFile) {
+    throw new Error(`Eval case ${testCase.id} is missing snapshotFile`);
+  }
+  const snapshotPath = path.join(summaryEvalFixturesRoot, testCase.caseSet, 'snapshots', testCase.snapshotFile);
+  return JSON.parse(await readFile(snapshotPath, 'utf8'));
+}
+
+export function buildSummaryEvalContext(snapshot, testCase) {
+  switch (testCase.surface) {
+    case 'workout': {
+      const sessionId = testCase.selector?.sessionId;
+      return workoutSummaryContext(snapshot, sessionId, { exclude: new Set(testCase.exclude ?? []) });
+    }
+    case 'cycle': {
+      const programId = testCase.selector?.programId ?? null;
+      return cycleSummaryContext(snapshot, programId, { exclude: new Set(testCase.exclude ?? []) });
+    }
+    case 'checkpoint': {
+      const programId = testCase.selector?.programId ?? null;
+      const checkpointWeek = testCase.selector?.checkpointWeek;
+      return checkpointContext(snapshot, programId, checkpointWeek, { exclude: new Set(testCase.exclude ?? []) });
+    }
+    case 'vitals':
+      return vitalsSummaryContext(snapshot, { exclude: new Set(testCase.exclude ?? []) });
+    default:
+      throw new Error(`Unsupported summary eval surface: ${testCase.surface}`);
+  }
+}
+
+export async function generateSummaryEvalOutput(testCase, context) {
+  const liveGenerationEnabled = process.env.SUMMARY_EVALS_LIVE === '1';
+  const apiKey = process.env.OPENROUTER_API_KEY;
+  if (!liveGenerationEnabled || !apiKey || testCase.shouldPass === false) {
+    return testCase.output;
+  }
+
+  switch (testCase.surface) {
+    case 'workout':
+      return (await generateWorkoutCoachingSummary(context, { apiKey })).text;
+    case 'cycle':
+      return (await generateCoachingSummary(context, { apiKey })).text;
+    case 'checkpoint':
+      return (await generateCheckpointSummary(context, { apiKey })).text;
+    case 'vitals':
+      return (await generateVitalsSummary(context, { apiKey })).text;
+    default:
+      throw new Error(`Unsupported summary eval surface: ${testCase.surface}`);
+  }
+}
+
+function normalizeText(value) {
+  return String(value ?? '').trim();
+}
+
+function paragraphCount(text) {
+  return normalizeText(text)
+    .split(/\n\s*\n/)
+    .filter(Boolean)
+    .length;
+}
+
+function sentenceCount(text) {
+  const matches = normalizeText(text).match(/[^.!?]+[.!?]+/g);
+  return matches ? matches.length : 0;
+}
+
+function isSingleParagraph(text) {
+  return !normalizeText(text).includes('\n\n');
+}
+
+function lowerIncludes(text, snippet) {
+  return normalizeText(text).toLowerCase().includes(String(snippet).toLowerCase());
+}
+
+function uniqueStrings(values) {
+  return [...new Set((values ?? []).filter(Boolean))];
+}
+
+function escapeRegex(value) {
+  return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+const historicalExerciseModifiers = new Set([
+  'barbell',
+  'dumbbell',
+  'cable',
+  'machine',
+  'seated',
+  'standing',
+  'smith',
+  'wide',
+  'narrow',
+  'close',
+  'reverse',
+  'incline',
+  'decline',
+  'single',
+  'one',
+  'arm',
+  'leg',
+  'weighted',
+  'romanian',
+  'hack',
+  'full',
+  'grip'
+]);
+
+function collectAllExerciseNames(snapshot) {
+  const names = new Set();
+
+  for (const session of snapshot.sessions ?? []) {
+    for (const exercise of session.exercises ?? []) {
+      if (exercise.name) names.add(exercise.name);
+    }
+  }
+
+  for (const program of snapshot.programs ?? []) {
+    for (const day of program.days ?? []) {
+      for (const exercise of day.exercises ?? []) {
+        if (exercise.name) names.add(exercise.name);
+      }
+    }
+  }
+
+  for (const plan of snapshot.strengthPlans ?? []) {
+    for (const goal of plan.liftGoals ?? []) {
+      if (goal.exerciseDisplayName) names.add(goal.exerciseDisplayName);
+    }
+  }
+
+  return [...names];
+}
+
+function collectAllowedExerciseNames(surface, context) {
+  const names = new Set();
+
+  if (surface === 'workout' && context && typeof context === 'object') {
+    for (const exercise of context.exercises ?? []) {
+      if (exercise.exerciseName) names.add(exercise.exerciseName);
+    }
+    for (const exerciseName of context.nextSession?.exerciseNames ?? []) {
+      names.add(exerciseName);
+    }
+    for (const pr of context.prs ?? []) {
+      if (pr.exerciseName) names.add(pr.exerciseName);
+    }
+    for (const pr of context.bwPrs ?? []) {
+      if (pr.exerciseName) names.add(pr.exerciseName);
+    }
+    for (const pr of context.repPrs ?? []) {
+      if (pr.exerciseName) names.add(pr.exerciseName);
+    }
+  }
+
+  if (surface === 'cycle' && context && typeof context === 'object') {
+    for (const session of context.sessions ?? []) {
+      for (const exercise of session.exercises ?? []) {
+        if (exercise.exerciseName) names.add(exercise.exerciseName);
+      }
+    }
+    for (const progress of context.goalProgress ?? []) {
+      if (progress.exerciseName) names.add(progress.exerciseName);
+    }
+    for (const decision of context.progressionDecisions ?? []) {
+      if (decision.exerciseName) names.add(decision.exerciseName);
+    }
+    for (const pr of context.prsThisCycle ?? []) {
+      if (pr.exerciseName) names.add(pr.exerciseName);
+    }
+    for (const pr of context.bwPrsThisCycle ?? []) {
+      if (pr.exerciseName) names.add(pr.exerciseName);
+    }
+  }
+
+  if (surface === 'checkpoint' && context && typeof context === 'object') {
+    for (const exercise of context.exercises ?? []) {
+      if (exercise.name) names.add(exercise.name);
+    }
+  }
+
+  return [...names];
+}
+
+function historicalExerciseVariants(name) {
+  const normalized = normalizeExerciseName(name);
+  if (!normalized) return [];
+
+  const variants = new Set([normalized]);
+  const tokens = normalized.split(' ').filter(Boolean);
+
+  let start = 0;
+  while (start < tokens.length - 1 && historicalExerciseModifiers.has(tokens[start])) {
+    start += 1;
+    const variant = tokens.slice(start).join(' ');
+    if (variant) variants.add(variant);
+  }
+
+  if (tokens.length >= 2) {
+    variants.add(tokens.slice(-2).join(' '));
+  }
+
+  return [...variants].filter(Boolean);
+}
+
+function evaluateExerciseMentions(output, snapshot, context, surface, testCase) {
+  const isStored = testCase.source === 'stored';
+  const allowed = new Set();
+  for (const name of [
+    ...collectAllowedExerciseNames(surface, context),
+    ...(testCase.allowedExerciseMentions ?? [])
+  ]) {
+    const variants = isStored ? historicalExerciseVariants(name) : [normalizeExerciseName(name)];
+    for (const variant of variants) {
+      allowed.add(variant);
+    }
+  }
+  const allNames = collectAllExerciseNames(snapshot);
+  const normalizedOutput = normalizeExerciseName(output);
+  const mentions = [];
+
+  for (const exerciseName of allNames) {
+    const normalizedName = normalizeExerciseName(exerciseName);
+    if (!normalizedName) continue;
+
+    const pattern = new RegExp(`(?<!\\S)${escapeRegex(normalizedName)}(?!\\S)`, 'g');
+    for (const match of normalizedOutput.matchAll(pattern)) {
+      mentions.push({
+        exerciseName,
+        normalizedName,
+        start: match.index,
+        end: (match.index ?? 0) + normalizedName.length,
+        allowed: allowed.has(normalizedName)
+      });
+    }
+  }
+
+  const unauthorized = mentions
+    .filter((mention) => !mention.allowed)
+    .filter((mention) => !mentions.some((candidate) =>
+      candidate.allowed &&
+      candidate.normalizedName.length > mention.normalizedName.length &&
+      candidate.start <= mention.start &&
+      candidate.end >= mention.end
+    ))
+    .map((mention) => mention.exerciseName);
+
+  return {
+    key: 'exercise_mentions',
+    passed: unauthorized.length === 0,
+    reason: unauthorized.length === 0
+      ? 'No unauthorized exercise mentions.'
+      : `Unauthorized exercise mention(s): ${uniqueStrings(unauthorized).join(', ')}`
+  };
+}
+
+function evaluateRequiredMentions(output, testCase) {
+  const missing = uniqueStrings(testCase.requiredMentions).filter((mention) => !lowerIncludes(output, mention));
+  return {
+    key: 'required_mentions',
+    passed: missing.length === 0,
+    reason: missing.length === 0 ? 'All required mentions present.' : `Missing required mention(s): ${missing.join(', ')}`
+  };
+}
+
+function evaluateAnyOfMentions(output, testCase) {
+  const candidates = uniqueStrings(testCase.requiredAnyOfMentions);
+  if (candidates.length === 0) {
+    return {
+      key: 'required_any_of_mentions',
+      passed: true,
+      reason: 'No any-of mention requirement.'
+    };
+  }
+
+  const matched = candidates.some((mention) => lowerIncludes(output, mention));
+  return {
+    key: 'required_any_of_mentions',
+    passed: matched,
+    reason: matched
+      ? 'Matched at least one required any-of mention.'
+      : `Missing all any-of mentions: ${candidates.join(', ')}`
+  };
+}
+
+function evaluateForbiddenPhrases(output, testCase) {
+  const hits = uniqueStrings(testCase.forbiddenPhrases).filter((phrase) => lowerIncludes(output, phrase));
+  return {
+    key: 'forbidden_phrases',
+    passed: hits.length === 0,
+    reason: hits.length === 0 ? 'No forbidden phrases detected.' : `Forbidden phrase(s): ${hits.join(', ')}`
+  };
+}
+
+function evaluateForbiddenMentions(output, testCase) {
+  const hits = uniqueStrings(testCase.forbiddenMentions).filter((phrase) => lowerIncludes(output, phrase));
+  return {
+    key: 'forbidden_mentions',
+    passed: hits.length === 0,
+    reason: hits.length === 0 ? 'No forbidden mentions detected.' : `Forbidden mention(s): ${hits.join(', ')}`
+  };
+}
+
+function evaluateNoInsight(output, testCase) {
+  const normalized = normalizeText(output);
+  const expectedNoInsight = Boolean(testCase.expectNoInsight);
+  const passed = expectedNoInsight
+    ? normalized === 'NO_INSIGHT'
+    : normalized !== 'NO_INSIGHT';
+  return {
+    key: 'no_insight',
+    passed,
+    reason: expectedNoInsight
+      ? passed ? 'Correctly returned NO_INSIGHT.' : 'Expected exact NO_INSIGHT output.'
+      : passed ? 'Output is not NO_INSIGHT.' : 'Unexpected NO_INSIGHT output.'
+  };
+}
+
+function evaluateShape(output, testCase) {
+  const normalized = normalizeText(output);
+  if (normalized === 'NO_INSIGHT') {
+    return {
+      key: 'shape',
+      passed: true,
+      reason: 'NO_INSIGHT bypasses shape rules.'
+    };
+  }
+
+  const sentences = sentenceCount(output);
+  const paragraphs = paragraphCount(output);
+  const isStored = testCase.source === 'stored';
+  let passed = true;
+  const reasons = [];
+
+  switch (testCase.surface) {
+    case 'workout':
+      if (!isStored && !isSingleParagraph(output)) {
+        passed = false;
+        reasons.push('Workout summaries must be a single paragraph.');
+      }
+      if ((!isStored && (sentences < 2 || sentences > 4)) || (isStored && (sentences < 1 || sentences > 12))) {
+        passed = false;
+        reasons.push(isStored
+          ? `Stored workout summaries must be 1-12 sentences, got ${sentences}.`
+          : `Workout summaries must be 2-4 sentences, got ${sentences}.`);
+      }
+      if (isStored && (paragraphs < 1 || paragraphs > 3)) {
+        passed = false;
+        reasons.push(`Stored workout summaries must be 1-3 paragraphs, got ${paragraphs}.`);
+      }
+      break;
+    case 'vitals':
+      if (!isStored && !isSingleParagraph(output)) {
+        passed = false;
+        reasons.push('Vitals summaries must be a single paragraph.');
+      }
+      if ((!isStored && (sentences < 2 || sentences > 3)) || (isStored && (sentences < 1 || sentences > 5))) {
+        passed = false;
+        reasons.push(isStored
+          ? `Stored vitals summaries must be 1-5 sentences, got ${sentences}.`
+          : `Vitals summaries must be 2-3 sentences, got ${sentences}.`);
+      }
+      if (isStored && (paragraphs < 1 || paragraphs > 2)) {
+        passed = false;
+        reasons.push(`Stored vitals summaries must be 1-2 paragraphs, got ${paragraphs}.`);
+      }
+      break;
+    case 'cycle':
+      if (paragraphs < 1 || paragraphs > 4) {
+        passed = false;
+        reasons.push(`Cycle summaries must be 1-4 paragraphs, got ${paragraphs}.`);
+      }
+      break;
+    case 'checkpoint':
+      if (paragraphs < 2 || paragraphs > 3) {
+        passed = false;
+        reasons.push(`Checkpoint summaries must be 2-3 paragraphs, got ${paragraphs}.`);
+      }
+      break;
+    default:
+      break;
+  }
+
+  return {
+    key: 'shape',
+    passed,
+    reason: passed ? 'Output shape matches surface rules.' : reasons.join(' ')
+  };
+}
+
+function parseClaimCount(rawValue) {
+  const normalized = String(rawValue).toLowerCase();
+  if (/^\d+$/.test(normalized)) return Number(normalized);
+  return {
+    one: 1,
+    two: 2,
+    three: 3,
+    four: 4,
+    five: 5
+  }[normalized] ?? null;
+}
+
+function extractWorkoutVolumeClaims(output) {
+  const text = normalizeText(output);
+  const claims = [];
+  const patterns = [
+    /volume (?:is )?(?:down|dropped|drop|lower)\s+(\d{1,3})%/gi,
+    /(\d{1,3})%\s+volume (?:drop|decrease|lower)/gi,
+    /volume (?:is )?(?:up|increased|increase|jumped|jump)\s+(\d{1,3})%/gi,
+    /(\d{1,3})%\s+volume (?:jump|increase|rise|up)/gi
+  ];
+
+  for (const pattern of patterns) {
+    for (const match of text.matchAll(pattern)) {
+      const direction = /down|drop|lower|decrease/.test(match[0].toLowerCase()) ? 'down' : 'up';
+      claims.push({
+        direction,
+        percent: Number(match[1]),
+        text: match[0]
+      });
+    }
+  }
+
+  return claims;
+}
+
+function extractWorkoutPrCountClaims(output) {
+  const text = normalizeText(output);
+  const claims = [];
+  const pattern = /\b(one|two|three|four|five|\d+)\s+(?:new\s+)?(?:rep\s+)?prs?\b/gi;
+  for (const match of text.matchAll(pattern)) {
+    const count = parseClaimCount(match[1]);
+    if (count != null) {
+      claims.push({ count, text: match[0] });
+    }
+  }
+  return claims;
+}
+
+function extractSkippedExerciseClaims(output) {
+  const text = normalizeText(output);
+  const claims = [];
+  const pattern = /\bskipped\s+([A-Za-z][A-Za-z0-9\- ]{1,40}?)(?:\s+(?:entirely|today|this session|altogether))?(?=[,.!;]| but | and |$)/gi;
+  for (const match of text.matchAll(pattern)) {
+    const exerciseName = match[1].trim().replace(/\s+/g, ' ');
+    if (exerciseName) {
+      claims.push({ exerciseName, text: match[0] });
+    }
+  }
+  return claims;
+}
+
+function extractExerciseSpecificPrClaims(output) {
+  const text = normalizeText(output);
+  const claims = [];
+  const patterns = [
+    /\b([A-Za-z][A-Za-z0-9\- ]{1,50}?)\s+(?:hit|hits|landed|lands|got|gets|notched|posted)\s+(?:a\s+)?(?:new\s+)?(?:rep\s+)?pr\b/gi,
+    /\b(?:new\s+)?(?:rep\s+)?pr\s+(?:on|for)\s+([A-Za-z][A-Za-z0-9\- ]{1,50}?)(?=[,.!;]| and | but |$)/gi
+  ];
+  for (const pattern of patterns) {
+    for (const match of text.matchAll(pattern)) {
+      claims.push({ exerciseName: match[1].trim(), text: match[0] });
+    }
+  }
+  return claims;
+}
+
+function extractStallClaims(output) {
+  const text = normalizeText(output);
+  const claims = [];
+  const patterns = [
+    /\b([A-Za-z][A-Za-z0-9\- ]{1,50}?)\s+(?:has\s+)?(?:stalled|plateaued)\b/gi,
+    /\bstall(?:ed)?\s+on\s+([A-Za-z][A-Za-z0-9\- ]{1,50}?)(?=[,.!;]| and | but |$)/gi,
+    /\bplateau(?:ed)?\s+on\s+([A-Za-z][A-Za-z0-9\- ]{1,50}?)(?=[,.!;]| and | but |$)/gi
+  ];
+  for (const pattern of patterns) {
+    for (const match of text.matchAll(pattern)) {
+      claims.push({ exerciseName: match[1].trim(), text: match[0] });
+    }
+  }
+  return claims;
+}
+
+function hasFatigueLanguage(output) {
+  return /\b(fatigue|fatigued|underrecovered|recovery debt|fatigue ceiling|limited by recovery|limited by fatigue|accumulated fatigue)\b/i.test(output);
+}
+
+function matchesHistoricalFamilyName(claimName, actualName) {
+  const claimVariants = new Set(historicalExerciseVariants(claimName));
+  const actualVariants = new Set(historicalExerciseVariants(actualName));
+  for (const variant of claimVariants) {
+    if (actualVariants.has(variant)) return true;
+  }
+  return false;
+}
+
+function supportedSpecificExerciseClaim(claimName, actualName) {
+  const normalizedClaim = normalizeExerciseName(claimName);
+  if (!normalizedClaim) return false;
+  return historicalExerciseVariants(actualName).includes(normalizedClaim);
+}
+
+function findExerciseContext(exercises, claimName) {
+  return (exercises ?? []).find((exercise) => matchesHistoricalFamilyName(claimName, exercise.exerciseName));
+}
+
+function isStallSupported(exerciseContext) {
+  if (!exerciseContext?.topSet || !Array.isArray(exerciseContext.recentWeights)) return false;
+  const currentTopWeight = Number(exerciseContext.topSet.weight);
+  if (!Number.isFinite(currentTopWeight) || currentTopWeight <= 0) return false;
+  const recent = exerciseContext.recentWeights
+    .map((entry) => Number(entry.topWeight))
+    .filter((value) => Number.isFinite(value) && value > 0);
+  if (recent.length < 2) return false;
+  const window = [currentTopWeight, ...recent.slice(0, 2)];
+  return window.length === 3 && window.every((weight) => weight === currentTopWeight);
+}
+
+function fatigueSupportSignals(context) {
+  let count = 0;
+
+  if (context.readiness?.dominantSignal || context.readiness?.adaptationApplied) {
+    count += 1;
+  }
+
+  if (Number.isFinite(context.hrvOnDay) && Number.isFinite(context.hrvBaseline) && context.hrvOnDay <= context.hrvBaseline * 0.9) {
+    count += 1;
+  }
+
+  if (Number.isFinite(context.restingHROnDay) && Number.isFinite(context.restingHRBaseline) && context.restingHROnDay >= context.restingHRBaseline + 4) {
+    count += 1;
+  }
+
+  if (Number.isFinite(context.sleepNight?.durationMins) && context.sleepNight.durationMins <= 360) {
+    count += 1;
+  }
+
+  const cardioMinutes = (context.nearbyCardio ?? []).reduce((sum, workout) => sum + ((Number(workout.durationSecs) || 0) / 60), 0);
+  if (cardioMinutes >= 180) {
+    count += 1;
+  }
+
+  const recentComparisons = context.recentComparisons ?? [];
+  const currentVolume = Number(context.totalVolume);
+  if (Number.isFinite(currentVolume) && recentComparisons.some((comparison) => {
+    const baseline = Number(comparison.totalVolume);
+    return Number.isFinite(baseline) && baseline > 0 && currentVolume <= baseline * 0.85;
+  })) {
+    count += 1;
+  }
+
+  return count;
+}
+
+function evaluateWorkoutClaims(output, context, testCase) {
+  if (testCase.surface !== 'workout') {
+    return {
+      key: 'workout_claims',
+      passed: true,
+      reason: 'Not a workout summary.'
+    };
+  }
+
+  const normalized = normalizeText(output);
+  if (normalized === 'NO_INSIGHT') {
+    return {
+      key: 'workout_claims',
+      passed: true,
+      reason: 'NO_INSIGHT bypasses workout claim validation.'
+    };
+  }
+
+  const failures = [];
+  const totalPrCount = (context.prs?.length ?? 0) + (context.bwPrs?.length ?? 0) + (context.repPrs?.length ?? 0);
+
+  if (/\b(?:new\s+)?(?:rep\s+)?prs?\b/i.test(normalized) && totalPrCount === 0) {
+    failures.push('Summary claims a PR but the context contains no PRs.');
+  }
+
+  for (const claim of extractWorkoutPrCountClaims(normalized)) {
+    if (totalPrCount !== claim.count) {
+      failures.push(`PR count claim "${claim.text}" does not match actual PR count ${totalPrCount}.`);
+    }
+  }
+
+  const prExerciseNames = [
+    ...(context.prs ?? []).map((pr) => pr.exerciseName),
+    ...(context.bwPrs ?? []).map((pr) => pr.exerciseName),
+    ...(context.repPrs ?? []).map((pr) => pr.exerciseName)
+  ];
+  for (const claim of extractExerciseSpecificPrClaims(normalized)) {
+    const matched = prExerciseNames.some((exerciseName) => supportedSpecificExerciseClaim(claim.exerciseName, exerciseName));
+    if (!matched) {
+      failures.push(`Exercise-specific PR claim "${claim.text}" is not supported by the recorded PR exercises.`);
+    }
+  }
+
+  const comparisons = context.recentComparisons ?? [];
+  for (const claim of extractWorkoutVolumeClaims(normalized)) {
+    if (comparisons.length === 0) {
+      failures.push(`Volume claim "${claim.text}" cannot be verified because no comparison sessions are available.`);
+      continue;
+    }
+
+    const matched = comparisons.some((comparison) => {
+      const baseline = Number(comparison.totalVolume);
+      const current = Number(context.totalVolume);
+      if (!Number.isFinite(baseline) || baseline <= 0 || !Number.isFinite(current)) return false;
+      const delta = ((current - baseline) / baseline) * 100;
+      const directionMatches = claim.direction === 'down' ? delta < 0 : delta > 0;
+      return directionMatches && Math.abs(Math.abs(delta) - claim.percent) <= 5;
+    });
+
+    if (!matched) {
+      failures.push(`Volume claim "${claim.text}" is not supported by recent comparison volumes.`);
+    }
+  }
+
+  const skippedExercises = context.planComparison?.skipped ?? [];
+  for (const claim of extractSkippedExerciseClaims(normalized)) {
+    const matched = skippedExercises.some((exerciseName) => matchesHistoricalFamilyName(claim.exerciseName, exerciseName));
+    if (!matched) {
+      failures.push(`Skipped-exercise claim "${claim.text}" is not supported by the plan comparison.`);
+    }
+  }
+
+  for (const claim of extractStallClaims(normalized)) {
+    const exerciseContext = findExerciseContext(context.exercises, claim.exerciseName);
+    if (!isStallSupported(exerciseContext)) {
+      failures.push(`Stall claim "${claim.text}" is not supported by repeated top weights in recent sessions.`);
+    }
+  }
+
+  if (hasFatigueLanguage(normalized) && fatigueSupportSignals(context) < 2) {
+    failures.push('Fatigue/recovery claim is not supported by enough recovery or load signals.');
+  }
+
+  return {
+    key: 'workout_claims',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Workout claims are supported by the context.'
+      : failures.join(' ')
+  };
+}
+
+export async function runSummaryEvalCase(testCase) {
+  const snapshot = await loadSummaryEvalSnapshot(testCase);
+  return runSummaryEvalCaseFromSnapshot(testCase, snapshot);
+}
+
+export async function runSummaryEvalCaseFromSnapshot(testCase, snapshot) {
+  const context = buildSummaryEvalContext(snapshot, testCase);
+  if (context == null) {
+    throw new Error(`Eval case ${testCase.id} produced no context`);
+  }
+
+  const output = await generateSummaryEvalOutput(testCase, context);
+  if (!normalizeText(output)) {
+    throw new Error(`Eval case ${testCase.id} produced an empty output`);
+  }
+
+  const checks = [
+    evaluateNoInsight(output, testCase),
+    evaluateShape(output, testCase),
+    evaluateRequiredMentions(output, testCase),
+    evaluateAnyOfMentions(output, testCase),
+    evaluateForbiddenPhrases(output, testCase),
+    evaluateForbiddenMentions(output, testCase),
+    evaluateExerciseMentions(output, snapshot, context, testCase.surface, testCase),
+    evaluateWorkoutClaims(output, context, testCase)
+  ];
+
+  return {
+    id: testCase.id,
+    surface: testCase.surface,
+    name: testCase.name,
+    output,
+    passed: checks.every((check) => check.passed),
+    checks
+  };
+}
+
+function genericForbiddenPhrasesForSurface(surface) {
+  switch (surface) {
+    case 'workout':
+      return ['solid progress', 'trust the process', 'keep it up', 'quality work', 'in a great place', 'continue progressive overload', 'as fatigue accumulates'];
+    case 'cycle':
+      return ['solid progress', 'trust the process', 'in a great place', 'continue progressive overload', 'as fatigue accumulates', 'solid session', 'quality work'];
+    case 'checkpoint':
+      return ['solid progress', 'quality work', 'trust the process', 'in a great place'];
+    case 'vitals':
+      return [];
+    default:
+      return [];
+  }
+}
+
+function latestCycleSummariesWithAI(snapshot) {
+  const summaries = snapshot.cycleSummaries ?? [];
+  const latestByProgram = new Map();
+
+  for (const summary of stableSortByDateDesc(summaries, (item) => item.completedDate)) {
+    if (!summary.aiSummary || latestByProgram.has(summary.programId)) continue;
+    latestByProgram.set(summary.programId, summary);
+  }
+
+  return [...latestByProgram.values()];
+}
+
+export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapshot') {
+  const harvested = [];
+
+  for (const session of snapshot.sessions ?? []) {
+    const text = session.summary?.aiCoachNotes;
+    if (!text) continue;
+    const anyOf = [
+      ...(session.exercises ?? []).map((exercise) => exercise.name),
+      session.dayName
+    ].filter(Boolean);
+    harvested.push({
+      id: `stored-workout-${session.id}`,
+      name: `Stored workout summary ${session.id}`,
+      surface: 'workout',
+      source: 'stored',
+      snapshotLabel,
+      selector: { sessionId: session.id },
+      output: text,
+      requiredMentions: [],
+      requiredAnyOfMentions: anyOf,
+      forbiddenPhrases: genericForbiddenPhrasesForSurface('workout'),
+      forbiddenMentions: [],
+      expectNoInsight: false,
+      shouldPass: true
+    });
+  }
+
+  for (const summary of latestCycleSummariesWithAI(snapshot)) {
+    const anyOf = [
+      ...(summary.progressionUpdates ?? []).map((update) => update.exerciseName),
+      ...(summary.prs ?? []).map((pr) => pr.exerciseName),
+      'sleep',
+      'hrv',
+      'resting hr'
+    ].filter(Boolean);
+    harvested.push({
+      id: `stored-cycle-${summary.id}`,
+      name: `Stored cycle summary ${summary.id}`,
+      surface: 'cycle',
+      source: 'stored',
+      snapshotLabel,
+      selector: { programId: summary.programId },
+      output: summary.aiSummary,
+      requiredMentions: [],
+      requiredAnyOfMentions: anyOf,
+      forbiddenPhrases: genericForbiddenPhrasesForSurface('cycle'),
+      forbiddenMentions: [],
+      expectNoInsight: false,
+      shouldPass: true
+    });
+  }
+
+  const latestVitals = stableSortByDateDesc(snapshot.vitalsSummaries ?? [], (entry) => entry.date)[0];
+  if (latestVitals?.summary) {
+    harvested.push({
+      id: `stored-vitals-${latestVitals.id ?? latestVitals.date}`,
+      name: `Stored vitals summary ${latestVitals.date}`,
+      surface: 'vitals',
+      source: 'stored',
+      snapshotLabel,
+      selector: {},
+      output: latestVitals.summary,
+      requiredMentions: [],
+      requiredAnyOfMentions: ['recovery', 'readiness', 'train', 'session', 'today'],
+      forbiddenPhrases: genericForbiddenPhrasesForSurface('vitals'),
+      forbiddenMentions: [],
+      expectNoInsight: false,
+      shouldPass: true
+    });
+  }
+
+  return harvested;
+}