incremnt 0.4.0 → 0.5.0

package/src/summary-evals.js

@@ -2,6 +2,8 @@ import { readFile, readdir } from 'node:fs/promises';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import {
+  askContext,
+  askRoutedContext,
   checkpointContext,
   cycleSummaryContext,
   normalizeExerciseName,
@@ -9,6 +11,8 @@ import {
   vitalsSummaryContext
 } from './queries.js';
 import {
+  AI_PROMPT_VERSIONS,
+  generateAskAnswer,
   generateCheckpointSummary,
   generateCoachingSummary,
   generateVitalsSummary,
@@ -79,30 +83,83 @@ export function buildSummaryEvalContext(snapshot, testCase) {
     }
     case 'vitals':
       return vitalsSummaryContext(snapshot, { exclude: new Set(testCase.exclude ?? []) });
+    case 'ask': {
+      const question = testCase.context?.question ?? testCase.question ?? '';
+      const routed = question
+        ? askRoutedContext(snapshot, question, { exclude: new Set(testCase.exclude ?? []) })
+        : null;
+      return {
+        ...(testCase.context ?? {}),
+        trainingData: testCase.context?.trainingData ?? routed?.context ?? null,
+        routedMetadata: routed?.metadata ?? null
+      };
+    }
     default:
       throw new Error(`Unsupported summary eval surface: ${testCase.surface}`);
   }
 }
 
-export async function generateSummaryEvalOutput(testCase, context) {
+function summaryEvalGenerationMetadata(result) {
+  if (!result || typeof result !== 'object') return {};
+  return Object.fromEntries(
+    Object.entries({
+      model: result.model,
+      durationMs: result.durationMs,
+      fallback: result.fallback,
+      langfuseTraceId: result.langfuseTraceId,
+      langfuseObservationId: result.langfuseObservationId
+    }).filter(([, value]) => value !== undefined && value !== null)
+  );
+}
+
+export async function generateSummaryEvalOutputWithMetadata(testCase, context, snapshot = null) {
   const liveGenerationEnabled = process.env.SUMMARY_EVALS_LIVE === '1';
   const apiKey = process.env.OPENROUTER_API_KEY;
   if (!liveGenerationEnabled || !apiKey || testCase.shouldPass === false) {
-    return testCase.output;
+    return { output: testCase.output, metadata: {} };
   }
 
+  let result;
   switch (testCase.surface) {
     case 'workout':
-      return (await generateWorkoutCoachingSummary(context, { apiKey })).text;
+      result = await generateWorkoutCoachingSummary(context, { apiKey });
+      break;
     case 'cycle':
-      return (await generateCoachingSummary(context, { apiKey })).text;
+      result = await generateCoachingSummary(context, { apiKey });
+      break;
     case 'checkpoint':
-      return (await generateCheckpointSummary(context, { apiKey })).text;
+      result = await generateCheckpointSummary(context, { apiKey });
+      break;
     case 'vitals':
-      return (await generateVitalsSummary(context, { apiKey })).text;
+      result = await generateVitalsSummary(context, { apiKey });
+      break;
+    case 'ask': {
+      if (!snapshot) return { output: testCase.output, metadata: {} };
+      const question = context.question ?? testCase.question ?? '';
+      if (!question) return { output: testCase.output, metadata: {} };
+      const trainingContext = context.trainingData
+        ?? askRoutedContext(snapshot, question, { exclude: new Set(testCase.exclude ?? []) }).context
+        ?? askContext(snapshot, { exclude: new Set(testCase.exclude ?? []) });
+      result = await generateAskAnswer(trainingContext, question, {
+        apiKey,
+        history: context.history ?? [],
+        tone: context.tone,
+        model: context.model
+      });
+      break;
+    }
     default:
       throw new Error(`Unsupported summary eval surface: ${testCase.surface}`);
   }
+
+  return {
+    output: result.text,
+    metadata: summaryEvalGenerationMetadata(result)
+  };
+}
+
+export async function generateSummaryEvalOutput(testCase, context, snapshot = null) {
+  return (await generateSummaryEvalOutputWithMetadata(testCase, context, snapshot)).output;
 }
 
 function normalizeText(value) {
@@ -129,6 +186,21 @@ function lowerIncludes(text, snippet) {
   return normalizeText(text).toLowerCase().includes(String(snippet).toLowerCase());
 }
 
+function phraseIncludes(text, snippet) {
+  const normalizedSnippet = normalizeText(snippet).trim();
+  if (!normalizedSnippet) return false;
+  const escapedSnippet = escapeRegExp(normalizedSnippet).replace(/\s+/g, '\\s+');
+  const startsWithWord = /^\w/.test(normalizedSnippet);
+  const endsWithWord = /\w$/.test(normalizedSnippet);
+  const pluralSuffix = /^[A-Za-z]{1,3}$/.test(normalizedSnippet) ? 's?' : '';
+  const pattern = `${startsWithWord ? '\\b' : ''}${escapedSnippet}${pluralSuffix}${endsWithWord ? '\\b' : ''}`;
+  return new RegExp(pattern, 'i').test(normalizeText(text));
+}
+
+function escapeRegExp(value) {
+  return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
 function uniqueStrings(values) {
   return [...new Set((values ?? []).filter(Boolean))];
 }
@@ -260,6 +332,14 @@ function historicalExerciseVariants(name) {
 }
 
 function evaluateExerciseMentions(output, snapshot, context, surface, testCase) {
+  if (surface === 'ask') {
+    return {
+      key: 'exercise_mentions',
+      passed: true,
+      reason: 'Ask-coach answers may mention any exercise from the training history.'
+    };
+  }
+
   const isStored = testCase.source === 'stored';
   const allowed = new Set();
   for (const name of [
@@ -340,7 +420,7 @@ function evaluateAnyOfMentions(output, testCase) {
 }
 
 function evaluateForbiddenPhrases(output, testCase) {
-  const hits = uniqueStrings(testCase.forbiddenPhrases).filter((phrase) => lowerIncludes(output, phrase));
+  const hits = uniqueStrings(testCase.forbiddenPhrases).filter((phrase) => phraseIncludes(output, phrase));
   return {
     key: 'forbidden_phrases',
     passed: hits.length === 0,
@@ -349,7 +429,7 @@ function evaluateForbiddenPhrases(output, testCase) {
 }
 
 function evaluateForbiddenMentions(output, testCase) {
-  const hits = uniqueStrings(testCase.forbiddenMentions).filter((phrase) => lowerIncludes(output, phrase));
+  const hits = uniqueStrings(testCase.forbiddenMentions).filter((phrase) => phraseIncludes(output, phrase));
   return {
     key: 'forbidden_mentions',
     passed: hits.length === 0,
@@ -422,9 +502,17 @@ function evaluateShape(output, testCase) {
       }
       break;
     case 'cycle':
-      if (paragraphs < 1 || paragraphs > 4) {
+      if ((!isStored && (sentences < 2 || sentences > 8)) || (isStored && (sentences < 1 || sentences > 10))) {
         passed = false;
-        reasons.push(`Cycle summaries must be 1-4 paragraphs, got ${paragraphs}.`);
+        reasons.push(isStored
+          ? `Stored cycle summaries must be 1-10 sentences, got ${sentences}.`
+          : `Cycle summaries must be 2-8 sentences, got ${sentences}.`);
+      }
+      if ((!isStored && (paragraphs < 1 || paragraphs > 2)) || (isStored && (paragraphs < 1 || paragraphs > 3))) {
+        passed = false;
+        reasons.push(isStored
+          ? `Stored cycle summaries must be 1-3 paragraphs, got ${paragraphs}.`
+          : `Cycle summaries must be 1-2 paragraphs, got ${paragraphs}.`);
       }
       break;
     case 'checkpoint':
@@ -433,6 +521,12 @@ function evaluateShape(output, testCase) {
         reasons.push(`Checkpoint summaries must be 2-3 paragraphs, got ${paragraphs}.`);
       }
       break;
+    case 'ask':
+      if (sentences < 1 || sentences > 12) {
+        passed = false;
+        reasons.push(`Ask-coach answers must be 1-12 sentences, got ${sentences}.`);
+      }
+      break;
     default:
       break;
   }
@@ -700,18 +794,431 @@ function evaluateWorkoutClaims(output, context, testCase) {
   };
 }
 
+function extractAskTargetHitClaims(text) {
+  const claims = [];
+  const patterns = [
+    /\b(?:you\s+)?hit(?:ting)?\s+all\s+(?:your\s+)?target(?:ed)?\s+reps?\b/gi,
+    /\b(?:you\s+)?hit\s+all\s+(?:the\s+)?targets?\b/gi,
+    /\b(?:you\s+)?hit\s+(?:the|your)\s+target\b(?!\s+(?:of|for|on))/gi
+  ];
+  for (const pattern of patterns) {
+    for (const match of text.matchAll(pattern)) {
+      claims.push({ text: match[0] });
+    }
+  }
+  return claims;
+}
+
+function extractAskCleanConsistencyClaims(text) {
+  const claims = [];
+  const patterns = [
+    /\bclean,\s+consistent\b/gi,
+    /\bclean\s+and\s+consistent\b/gi,
+    /\bconsistent\s+set\s+of\s+work\b/gi,
+    /\bacross\s+the\s+board\b/gi
+  ];
+  for (const pattern of patterns) {
+    for (const match of text.matchAll(pattern)) {
+      claims.push({ text: match[0] });
+    }
+  }
+  return claims;
+}
+
+function extractAskPlannedListClaims(text) {
+  const claims = [];
+  const pattern = /\((\s*\d+(?:\s*,\s*\d+){2,})\s+planned\s*\)/gi;
+  for (const match of text.matchAll(pattern)) {
+    const reps = match[1].split(',').map((value) => Number(value.trim())).filter(Number.isFinite);
+    claims.push({ text: match[0], reps, index: match.index ?? -1 });
+  }
+  return claims;
+}
+
+function sessionPlannedReps(session) {
+  const prescription = session?.prescriptionSnapshot?.exercises ?? [];
+  const values = [];
+  for (const exercise of prescription) {
+    const target = Number(exercise.targetReps);
+    if (Number.isFinite(target) && target > 0) values.push(target);
+  }
+  return values;
+}
+
+function findMentionedExercises(text, snapshot) {
+  const exercisesByName = new Map();
+  for (const session of snapshot?.sessions ?? []) {
+    for (const exercise of session.exercises ?? []) {
+      if (!exercise?.name) continue;
+      const normalizedName = normalizeExerciseName(exercise.name);
+      if (!normalizedName || exercisesByName.has(normalizedName)) continue;
+      exercisesByName.set(normalizedName, exercise.name);
+    }
+    for (const exercise of session.prescriptionSnapshot?.exercises ?? []) {
+      if (!exercise?.exerciseName) continue;
+      const normalizedName = normalizeExerciseName(exercise.exerciseName);
+      if (!normalizedName || exercisesByName.has(normalizedName)) continue;
+      exercisesByName.set(normalizedName, exercise.exerciseName);
+    }
+  }
+
+  const mentions = [];
+  for (const [normalizedName, displayName] of exercisesByName) {
+    const pattern = new RegExp(`\\b${escapeRegExp(displayName)}\\b`, 'gi');
+    for (const match of text.matchAll(pattern)) {
+      mentions.push({
+        index: match.index ?? -1,
+        end: (match.index ?? -1) + match[0].length,
+        name: displayName,
+        normalizedName
+      });
+    }
+  }
+  return mentions
+    .filter((mention, index, allMentions) => !allMentions.some((candidate, candidateIndex) =>
+      candidateIndex !== index &&
+      candidate.index <= mention.index &&
+      candidate.end >= mention.end &&
+      candidate.normalizedName.length > mention.normalizedName.length
+    ))
+    .sort((lhs, rhs) => lhs.index - rhs.index);
+}
+
+function findRecentSessionMisses(snapshot, { lookbackDays = 7, exerciseNames = null } = {}) {
+  const sessions = snapshot?.sessions ?? [];
+  const cutoff = Date.now() - lookbackDays * 24 * 60 * 60 * 1000;
+  const scopedExerciseNames = exerciseNames && exerciseNames.length > 0 ? new Set(exerciseNames) : null;
+  const misses = [];
+  for (const session of sessions) {
+    const completedAt = session.completedAt || session.date;
+    const completedTime = Date.parse(completedAt);
+    if (!Number.isFinite(completedTime) || completedTime < cutoff) continue;
+    const targetByExercise = new Map();
+    for (const planned of session.prescriptionSnapshot?.exercises ?? []) {
+      const target = Number(planned.targetReps);
+      if (Number.isFinite(target) && target > 0) {
+        targetByExercise.set(normalizeExerciseName(planned.exerciseName), target);
+      }
+    }
+    for (const exercise of session.exercises ?? []) {
+      const normalizedExerciseName = normalizeExerciseName(exercise.name);
+      if (scopedExerciseNames && !scopedExerciseNames.has(normalizedExerciseName)) continue;
+      const target = targetByExercise.get(normalizedExerciseName);
+      if (!Number.isFinite(target)) continue;
+      for (const set of exercise.sets ?? []) {
+        const reps = Number(set.reps);
+        if (set.isComplete && Number.isFinite(reps) && reps < target) {
+          misses.push({ sessionId: session.id, exerciseName: exercise.name, reps, target });
+        }
+      }
+    }
+  }
+  return misses;
+}
+
+function findNearestMentionedExercise(mentions, index) {
+  let candidate = null;
+  for (const mention of mentions) {
+    if (mention.index > index) break;
+    candidate = mention;
+  }
+  return candidate;
+}
+
+function hasAskFatigueSupport(snapshot, lookbackDays = 7) {
+  const cutoff = Date.now() - lookbackDays * 24 * 60 * 60 * 1000;
+  const withinCutoff = (dateValue) => {
+    const ms = Date.parse(dateValue);
+    return Number.isFinite(ms) && ms >= cutoff;
+  };
+
+  const vitalsSummaries = snapshot?.vitalsSummaries ?? [];
+  if (vitalsSummaries.some((entry) => withinCutoff(entry.date))) return true;
+
+  const metrics = snapshot?.healthMetrics ?? {};
+  for (const key of ['restingHR', 'hrv', 'sleep']) {
+    const readings = Array.isArray(metrics[key]) ? metrics[key] : [];
+    if (readings.some((reading) => withinCutoff(reading.date))) return true;
+  }
+
+  for (const session of snapshot?.sessions ?? []) {
+    const completedAt = session.completedAt || session.date;
+    if (!withinCutoff(completedAt)) continue;
+    for (const exercise of session.exercises ?? []) {
+      const reps = (exercise.sets ?? [])
+        .map((set) => Number(set.reps))
+        .filter((value) => Number.isFinite(value) && value > 0);
+      if (reps.length < 2) continue;
+      const first = reps[0];
+      const last = reps[reps.length - 1];
+      if (first > 0 && (first - last) / first >= 0.3) return true;
+    }
+  }
+
+  return false;
+}
+
+function extractAskWeightClaims(text) {
+  const claims = [];
+  const pattern = /\b(\d+(?:\.\d+)?)\s*(?:kg|kilograms?)\b/gi;
+  for (const match of text.matchAll(pattern)) {
+    claims.push({
+      text: match[0],
+      value: Number(match[1]),
+      index: match.index ?? -1
+    });
+  }
+  return claims;
+}
+
+function allowedWeightsForExercise(snapshot, normalizedExerciseName) {
+  const weights = [];
+  for (const session of snapshot?.sessions ?? []) {
+    for (const exercise of session.exercises ?? []) {
+      if (normalizeExerciseName(exercise.name) !== normalizedExerciseName) continue;
+      for (const set of exercise.sets ?? []) {
+        const weight = Number(set.weight);
+        if (Number.isFinite(weight)) weights.push(weight);
+      }
+    }
+    for (const exercise of session.prescriptionSnapshot?.exercises ?? []) {
+      if (normalizeExerciseName(exercise.exerciseName) !== normalizedExerciseName) continue;
+      const targetWeight = Number(exercise.targetWeight);
+      if (Number.isFinite(targetWeight)) weights.push(targetWeight);
+      for (const targetSet of exercise.targetSets ?? []) {
+        const weight = Number(targetSet.weight ?? targetSet.targetWeight);
+        if (Number.isFinite(weight)) weights.push(weight);
+      }
+    }
+  }
+  return weights;
+}
+
+function weightClaimSupported(claim, allowedWeights) {
+  return allowedWeights.some((weight) => Math.abs(weight - claim.value) < 0.01);
+}
+
+function isEstimatedOneRepMaxWeightClaim(text, claim) {
+  const start = Math.max(0, claim.index - 40);
+  const end = Math.min(text.length, claim.index + claim.text.length + 40);
+  const window = text.slice(start, end);
+  return /\b(?:estimated\s+)?(?:1rm|one[-\s]?rep\s+max)\b/i.test(window);
+}
+
+function isVolumeWeightClaim(text, claim) {
+  const start = Math.max(0, claim.index - 30);
+  const end = Math.min(text.length, claim.index + claim.text.length + 30);
+  const window = text.slice(start, end);
+  return /\bvolume\b/i.test(window);
+}
+
+function evaluateAskClaims(output, snapshot, testCase) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_claims', passed: true, reason: 'Not an ask answer.' };
+  }
+
+  const normalized = normalizeText(output);
+  if (normalized === 'NO_INSIGHT' || !normalized) {
+    return { key: 'ask_claims', passed: true, reason: 'NO_INSIGHT bypasses ask claim validation.' };
+  }
+
+  const failures = [];
+  const mentionedExercises = findMentionedExercises(output, snapshot);
+  const scopedExerciseNames = uniqueStrings(mentionedExercises.map((mention) => mention.normalizedName));
+
+  const targetHitClaims = extractAskTargetHitClaims(normalized);
+  if (targetHitClaims.length > 0) {
+    const misses = findRecentSessionMisses(snapshot, { exerciseNames: scopedExerciseNames });
+    if (misses.length > 0) {
+      const sample = misses[0];
+      failures.push(`Ask answer claims targets hit ("${targetHitClaims[0].text}") but recent session ${sample.sessionId} has ${sample.exerciseName} at ${sample.reps} reps below target ${sample.target}.`);
+    }
+  }
+
+  const cleanConsistencyClaims = extractAskCleanConsistencyClaims(normalized);
+  if (cleanConsistencyClaims.length > 0) {
+    const misses = findRecentSessionMisses(snapshot, { exerciseNames: scopedExerciseNames });
+    if (misses.length > 0) {
+      const sample = misses[0];
+      failures.push(`Ask answer frames missed target reps as "${cleanConsistencyClaims[0].text}", but recent session ${sample.sessionId} has ${sample.exerciseName} at ${sample.reps} reps below target ${sample.target}.`);
+    }
+  }
+
+  for (const claim of extractAskPlannedListClaims(normalized)) {
+    const uniquePlanned = new Set(claim.reps);
+    if (uniquePlanned.size > 1) {
+      failures.push(`Ask answer asserts non-uniform plan "${claim.text}". Plans are uniform per exercise; this list likely restates actual reps as if they were planned.`);
+      continue;
+    }
+    const plannedValue = claim.reps[0];
+    const referencedExercise = findNearestMentionedExercise(mentionedExercises, claim.index);
+    const anySessionHasThisPlan = (snapshot?.sessions ?? []).some((session) => {
+      if (!referencedExercise) {
+        return sessionPlannedReps(session).some((value) => value === plannedValue);
+      }
+      const plannedExercises = session?.prescriptionSnapshot?.exercises ?? [];
+      return plannedExercises.some((exercise) => {
+        const exerciseName = normalizeExerciseName(exercise.exerciseName);
+        const target = Number(exercise.targetReps);
+        return exerciseName === referencedExercise.normalizedName && target === plannedValue;
+      });
+    });
+    if (!anySessionHasThisPlan) {
+      if (referencedExercise) {
+        failures.push(`Ask answer asserts plan "${claim.text}" for ${referencedExercise.name} but no session in the snapshot has that target rep count for that exercise.`);
+      } else {
+        failures.push(`Ask answer asserts plan "${claim.text}" but no session in the snapshot has that target rep count.`);
+      }
+    }
+  }
+
+  for (const claim of extractAskWeightClaims(normalized)) {
+    if (isEstimatedOneRepMaxWeightClaim(normalized, claim)) continue;
+    if (isVolumeWeightClaim(normalized, claim)) continue;
+    const referencedExercise = findNearestMentionedExercise(mentionedExercises, claim.index);
+    if (!referencedExercise) continue;
+    const allowedWeights = allowedWeightsForExercise(snapshot, referencedExercise.normalizedName);
+    if (allowedWeights.length === 0) continue;
+    if (!weightClaimSupported(claim, allowedWeights)) {
+      failures.push(`Ask answer asserts ${claim.text} for ${referencedExercise.name}, but that weight is not present in recorded or planned sets for that exercise.`);
+    }
+  }
+
+  if (hasFatigueLanguage(normalized) && !hasAskFatigueSupport(snapshot)) {
+    failures.push('Ask answer uses fatigue/recovery language but the snapshot has no recent vitals, sleep, or rep-dropoff signals to support it.');
+  }
+
+  return {
+    key: 'ask_claims',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Ask claims are supported by the snapshot.'
+      : failures.join(' ')
+  };
+}
+
+function evaluateAskToolProvenance(output, context, testCase) {
+  if (testCase.surface !== 'ask') {
+    return { key: 'ask_tool_provenance', passed: true, reason: 'Not an ask answer.' };
+  }
+
+  const routedMetadata = context?.routedMetadata ?? {};
+  const toolsUsed = new Set(routedMetadata.toolsUsed ?? []);
+  const failures = [];
+  for (const toolName of uniqueStrings(testCase.requiredTools)) {
+    if (!toolsUsed.has(toolName)) {
+      failures.push(`Expected routed Ask Coach context to use ${toolName}.`);
+    }
+  }
+
+  if (/\b(?:estimated\s+)?(?:e1rm|1rm|one[- ]rep max)\b/i.test(output) && !toolsUsed.has('get_records')) {
+    failures.push('Ask answer mentions e1RM/1RM, but routed context did not use get_records.');
+  }
+
+  return {
+    key: 'ask_tool_provenance',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? 'Ask tool provenance matches requirements.'
+      : failures.join(' ')
+  };
+}
+
+function personaEvalConfig(testCase) {
+  if (testCase.personaEval === false) return null;
+  if (testCase.personaEval && typeof testCase.personaEval === 'object') {
+    return testCase.personaEval;
+  }
+  if (testCase.surface === 'workout' || testCase.surface === 'ask') {
+    return {
+      profile: 'motivated-athlete'
+    };
+  }
+  return null;
+}
+
+function hasPrSignal(output, context, testCase) {
+  const normalized = normalizeText(output);
+  if (/\b(?:new\s+)?(?:rep\s+)?prs?\b/i.test(normalized)) return true;
+  if (/\bstrength prs?\b/i.test(normalized)) return true;
+  if (/\bpersonal record\b/i.test(normalized)) return true;
+  if (testCase.surface === 'workout') {
+    const totalPrCount = (context.prs?.length ?? 0) + (context.bwPrs?.length ?? 0) + (context.repPrs?.length ?? 0);
+    return totalPrCount > 0;
+  }
+  return false;
+}
+
+function evaluatePersonaMotivation(output, context, testCase) {
+  const config = personaEvalConfig(testCase);
+  if (!config) {
+    return {
+      key: 'persona_motivation',
+      passed: true,
+      reason: 'Persona evaluation disabled.'
+    };
+  }
+
+  const normalized = normalizeText(output);
+  if (normalized === 'NO_INSIGHT') {
+    return {
+      key: 'persona_motivation',
+      passed: true,
+      reason: 'NO_INSIGHT bypasses persona evaluation.'
+    };
+  }
+
+  const failures = [];
+  const prSignal = hasPrSignal(normalized, context, testCase);
+
+  const minimizingPatterns = [
+    /\bcould only\b/i,
+    /\bonly\s+\d+\s+reps?\b/i,
+    /\bjust\s+\d+\s+reps?\b/i,
+    /\bbefore tapering to\b/i,
+    /\bbefore dropping to\b/i,
+    /\bbut dropped to\b/i,
+    /\bbut fell to\b/i
+  ];
+
+  if (prSignal && minimizingPatterns.some((pattern) => pattern.test(normalized))) {
+    failures.push('Feedback acknowledges a PR/positive result but frames later-set dropoff in a demotivating way.');
+  }
+
+  const discouragingPatterns = [
+    /\bdisappointing\b/i,
+    /\bunderwhelming\b/i,
+    /\bunderperformed\b/i,
+    /\bpoor\b/i,
+    /\bnot enough\b/i,
+    /\bfailed to\b/i,
+    /\bstruggled\b/i
+  ];
+
+  if (discouragingPatterns.some((pattern) => pattern.test(normalized))) {
+    failures.push('Feedback uses discouraging language that is likely to reduce motivation.');
+  }
+
+  return {
+    key: 'persona_motivation',
+    passed: failures.length === 0,
+    reason: failures.length === 0
+      ? `Output stays aligned with the ${config.profile ?? 'persona'} perspective.`
+      : failures.join(' ')
+  };
+}
+
 export async function runSummaryEvalCase(testCase) {
   const snapshot = await loadSummaryEvalSnapshot(testCase);
   return runSummaryEvalCaseFromSnapshot(testCase, snapshot);
 }
 
-export async function runSummaryEvalCaseFromSnapshot(testCase, snapshot) {
+export function evaluateSummaryOutputFromSnapshot(testCase, snapshot, output) {
   const context = buildSummaryEvalContext(snapshot, testCase);
   if (context == null) {
     throw new Error(`Eval case ${testCase.id} produced no context`);
   }
 
-  const output = await generateSummaryEvalOutput(testCase, context);
   if (!normalizeText(output)) {
     throw new Error(`Eval case ${testCase.id} produced an empty output`);
   }
@@ -724,7 +1231,10 @@ export async function runSummaryEvalCaseFromSnapshot(testCase, snapshot) {
     evaluateForbiddenPhrases(output, testCase),
     evaluateForbiddenMentions(output, testCase),
     evaluateExerciseMentions(output, snapshot, context, testCase.surface, testCase),
-    evaluateWorkoutClaims(output, context, testCase)
+    evaluateWorkoutClaims(output, context, testCase),
+    evaluateAskClaims(output, snapshot, testCase),
+    evaluateAskToolProvenance(output, context, testCase),
+    evaluatePersonaMotivation(output, context, testCase)
   ];
 
   return {
@@ -732,11 +1242,21 @@ export async function runSummaryEvalCaseFromSnapshot(testCase, snapshot) {
     surface: testCase.surface,
     name: testCase.name,
     output,
+    metadata: testCase.metadata ?? null,
     passed: checks.every((check) => check.passed),
     checks
   };
 }
 
+export async function runSummaryEvalCaseFromSnapshot(testCase, snapshot) {
+  const context = buildSummaryEvalContext(snapshot, testCase);
+  if (context == null) {
+    throw new Error(`Eval case ${testCase.id} produced no context`);
+  }
+  const output = await generateSummaryEvalOutput(testCase, context, snapshot);
+  return evaluateSummaryOutputFromSnapshot(testCase, snapshot, output);
+}
+
 function genericForbiddenPhrasesForSurface(surface) {
   switch (surface) {
     case 'workout':
@@ -747,6 +1267,8 @@ function genericForbiddenPhrasesForSurface(surface) {
       return ['solid progress', 'quality work', 'trust the process', 'in a great place'];
     case 'vitals':
       return [];
+    case 'ask':
+      return [];
     default:
       return [];
   }
@@ -764,7 +1286,51 @@ function latestCycleSummariesWithAI(snapshot) {
   return [...latestByProgram.values()];
 }
 
-export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapshot') {
+function normalizeStoredAIMetadata(surface, metadata, {
+  model = null,
+  generatedAt = null,
+  promptVersion = null,
+  gitSha = null
+} = {}) {
+  if (!metadata && !model && !generatedAt && !promptVersion && !gitSha) {
+    return null;
+  }
+
+  return {
+    surface,
+    generatedAt: metadata?.generatedAt ?? generatedAt ?? null,
+    model: metadata?.model ?? model ?? null,
+    promptVersion: metadata?.promptVersion ?? promptVersion ?? null,
+    gitSha: metadata?.gitSha ?? gitSha ?? null,
+    langfuseTraceId: metadata?.langfuseTraceId ?? metadata?.traceId ?? null,
+    langfuseObservationId: metadata?.langfuseObservationId ?? metadata?.observationId ?? null
+  };
+}
+
+export function currentPromptVersions() {
+  return new Set(Object.values(AI_PROMPT_VERSIONS));
+}
+
+function metadataPassesFilter(metadata, { onlyPromptVersions, since }) {
+  if (onlyPromptVersions) {
+    const version = metadata?.promptVersion;
+    if (!version || !onlyPromptVersions.has(version)) return false;
+  }
+  if (since) {
+    const generatedAt = metadata?.generatedAt;
+    if (!generatedAt || generatedAt < since) return false;
+  }
+  return true;
+}
+
+export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapshot', options = {}) {
+  const filter = {
+    onlyPromptVersions: options.onlyPromptVersions ?? null,
+    since: options.since ?? null
+  };
+  const pushIfIncluded = (bucket, item) => {
+    if (metadataPassesFilter(item.metadata, filter)) bucket.push(item);
+  };
   const harvested = [];
 
   for (const session of snapshot.sessions ?? []) {
@@ -774,7 +1340,7 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
       ...(session.exercises ?? []).map((exercise) => exercise.name),
       session.dayName
     ].filter(Boolean);
-    harvested.push({
+    pushIfIncluded(harvested, {
       id: `stored-workout-${session.id}`,
       name: `Stored workout summary ${session.id}`,
       surface: 'workout',
@@ -782,10 +1348,14 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
       snapshotLabel,
       selector: { sessionId: session.id },
       output: text,
+      metadata: normalizeStoredAIMetadata('workout', session.summary?.aiCoachMetadata, {
+        model: session.summary?.aiCoachModel ?? null
+      }),
       requiredMentions: [],
       requiredAnyOfMentions: anyOf,
       forbiddenPhrases: genericForbiddenPhrasesForSurface('workout'),
       forbiddenMentions: [],
+      personaEval: { profile: 'motivated-athlete' },
       expectNoInsight: false,
       shouldPass: true
     });
@@ -799,7 +1369,7 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
       'hrv',
       'resting hr'
     ].filter(Boolean);
-    harvested.push({
+    pushIfIncluded(harvested, {
       id: `stored-cycle-${summary.id}`,
       name: `Stored cycle summary ${summary.id}`,
       surface: 'cycle',
@@ -807,10 +1377,14 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
       snapshotLabel,
       selector: { programId: summary.programId },
       output: summary.aiSummary,
+      metadata: normalizeStoredAIMetadata('cycle', summary.aiSummaryMetadata, {
+        model: summary.aiSummaryModel ?? null
+      }),
       requiredMentions: [],
       requiredAnyOfMentions: anyOf,
       forbiddenPhrases: genericForbiddenPhrasesForSurface('cycle'),
       forbiddenMentions: [],
+      personaEval: false,
       expectNoInsight: false,
       shouldPass: true
     });
@@ -818,7 +1392,7 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
 
   const latestVitals = stableSortByDateDesc(snapshot.vitalsSummaries ?? [], (entry) => entry.date)[0];
   if (latestVitals?.summary) {
-    harvested.push({
+    pushIfIncluded(harvested, {
       id: `stored-vitals-${latestVitals.id ?? latestVitals.date}`,
       name: `Stored vitals summary ${latestVitals.date}`,
       surface: 'vitals',
@@ -826,10 +1400,42 @@ export function harvestStoredSummaryEvalCases(snapshot, snapshotLabel = 'snapsho
       snapshotLabel,
       selector: {},
       output: latestVitals.summary,
+      metadata: normalizeStoredAIMetadata('vitals', latestVitals.metadata, {
+        model: latestVitals.model ?? null,
+        generatedAt: latestVitals.date ?? null
+      }),
       requiredMentions: [],
       requiredAnyOfMentions: ['recovery', 'readiness', 'train', 'session', 'today'],
       forbiddenPhrases: genericForbiddenPhrasesForSurface('vitals'),
       forbiddenMentions: [],
+      personaEval: false,
+      expectNoInsight: false,
+      shouldPass: true
+    });
+  }
+
+  for (const conversation of stableSortByDateDesc(snapshot.askConversations ?? [], (entry) => entry.createdAt ?? '')) {
+    const answer = conversation?.messages?.filter((message) => message?.role === 'assistant').at(-1)?.content ?? null;
+    const question = conversation?.messages?.find((message) => message?.role === 'user')?.content ?? '';
+    if (!answer) continue;
+
+    pushIfIncluded(harvested, {
+      id: `stored-ask-${conversation.id}`,
+      name: `Stored ask conversation ${conversation.id}`,
+      surface: 'ask',
+      source: 'stored',
+      snapshotLabel,
+      context: { question, conversationId: conversation.id },
+      output: answer,
+      metadata: normalizeStoredAIMetadata('ask', conversation.metadata, {
+        model: conversation.model ?? null,
+        generatedAt: conversation.createdAt ?? null
+      }),
+      requiredMentions: [],
+      requiredAnyOfMentions: [],
+      forbiddenPhrases: genericForbiddenPhrasesForSurface('ask'),
+      forbiddenMentions: [],
+      personaEval: { profile: 'motivated-athlete' },
       expectNoInsight: false,
       shouldPass: true
     });