incremnt 0.7.2 → 0.8.0

package/src/program-draft.js

@@ -0,0 +1,230 @@
+// Single source of truth for the AI coach's <program_draft> block: extraction,
+// JSON-shape validation, and normalization. Lives here (not in sync-service.js)
+// so both the runtime (askCoach drops invalid drafts) and the eval harness
+// (summary-evals.js catches malformed drafts in CI, before they ship and get
+// silently dropped in prod) validate against the exact same rules. Moved verbatim
+// from sync-service.js — behaviour-preserving.
+
+export const PROGRAM_DRAFT_VERSION = 1;
+export const VALID_PROGRAM_DRAFT_EQUIPMENT_TIERS = new Set(['fullGym', 'benchDumbbells', 'dumbbellsOnly', 'bodyweightOnly']);
+export const VALID_PROGRAM_DRAFT_VOLUME_LEVELS = new Set(['minimum', 'moderate', 'high']);
+
+export const PROGRAM_DRAFT_LIMITS = {
+  nameMaxLen: 120,
+  muscleGroupMaxLen: 60,
+  dayLabelMaxLen: 60,
+  dayTitleMaxLen: 120,
+  daySubtitleMaxLen: 120,
+  noteMaxLen: 1000,
+  minWeight: 0,
+  maxWeight: 600,
+  minReps: 1,
+  maxReps: 30,
+  minRir: 0,
+  maxRir: 5,
+  minSetsPerExercise: 1,
+  maxSetsPerExercise: 12,
+  minExercisesPerDay: 1,
+  maxExercisesPerDay: 24,
+  minDaysPerWeek: 1,
+  maxDaysPerWeek: 7,
+  minDays: 1,
+  maxDays: 14
+};
+
+function collapseBlankLines(text) {
+  return String(text ?? '')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+function titleCaseExerciseName(name) {
+  return String(name ?? '')
+    .split(' ')
+    .filter(Boolean)
+    .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+    .join(' ');
+}
+
+function normalizedExerciseDisplayName(name, canonicalizeExerciseName) {
+  const trimmed = String(name ?? '').trim();
+  if (!trimmed) return '';
+  const canonical = canonicalizeExerciseName ? canonicalizeExerciseName(trimmed) : trimmed.toLowerCase();
+  return titleCaseExerciseName(canonical);
+}
+
+function hasOnlyAllowedKeys(value, allowedKeys) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
+  return Object.keys(value).every((key) => allowedKeys.has(key));
+}
+
+function normalizeProgramDraftSet(set) {
+  if (!hasOnlyAllowedKeys(set, new Set(['weight', 'reps', 'isWarmup']))) return null;
+
+  const weight = Number(set?.weight);
+  const reps = Number(set?.reps);
+  if (!Number.isFinite(weight) || !Number.isInteger(reps)) return null;
+  if (
+    weight < PROGRAM_DRAFT_LIMITS.minWeight ||
+    weight > PROGRAM_DRAFT_LIMITS.maxWeight ||
+    reps < PROGRAM_DRAFT_LIMITS.minReps ||
+    reps > PROGRAM_DRAFT_LIMITS.maxReps
+  ) return null;
+  return {
+    weight,
+    reps,
+    isComplete: false,
+    isWarmup: set?.isWarmup === true
+  };
+}
+
+function normalizeProgramDraftExercise(exercise, canonicalizeExerciseName, strict = false) {
+  if (!hasOnlyAllowedKeys(exercise, new Set(['name', 'muscleGroup', 'sets', 'rir', 'note']))) return null;
+
+  const name = normalizedExerciseDisplayName(exercise?.name, canonicalizeExerciseName);
+  const muscleGroup = String(exercise?.muscleGroup ?? '').trim();
+  // strict (eval): any invalid set rejects the whole draft — catches partial
+  // malformation as a regression signal. lenient (runtime, default): drop the
+  // bad set and salvage a usable program for the user.
+  const mappedSets = Array.isArray(exercise?.sets) ? exercise.sets.map(normalizeProgramDraftSet) : [];
+  if (strict && mappedSets.some((set) => !set)) return null;
+  const sets = mappedSets.filter(Boolean);
+
+  if (!name || name.length > PROGRAM_DRAFT_LIMITS.nameMaxLen) return null;
+  if (!muscleGroup || muscleGroup.length > PROGRAM_DRAFT_LIMITS.muscleGroupMaxLen) return null;
+  if (
+    sets.length < PROGRAM_DRAFT_LIMITS.minSetsPerExercise ||
+    sets.length > PROGRAM_DRAFT_LIMITS.maxSetsPerExercise
+  ) return null;
+
+  const rir = exercise?.rir == null ? null : Number(exercise.rir);
+  if (rir != null && (
+    !Number.isInteger(rir) ||
+    rir < PROGRAM_DRAFT_LIMITS.minRir ||
+    rir > PROGRAM_DRAFT_LIMITS.maxRir
+  )) return null;
+
+  const note = exercise?.note == null ? null : String(exercise.note);
+  if (note && note.length > PROGRAM_DRAFT_LIMITS.noteMaxLen) return null;
+
+  return {
+    name,
+    muscleGroup,
+    lastSuggestion: '',
+    nextSuggestion: '',
+    sets,
+    ...(note ? { note } : {}),
+    ...(rir != null ? { rir } : {})
+  };
+}
+
+function normalizeProgramDraftDay(day, canonicalizeExerciseName, strict = false) {
+  if (!hasOnlyAllowedKeys(day, new Set(['dayLabel', 'title', 'subtitle', 'exercises']))) return null;
+
+  const dayLabel = String(day?.dayLabel ?? '').trim();
+  const title = String(day?.title ?? '').trim();
+  const subtitle = String(day?.subtitle ?? '').trim();
+  const mappedExercises = Array.isArray(day?.exercises)
+    ? day.exercises.map((exercise) => normalizeProgramDraftExercise(exercise, canonicalizeExerciseName, strict))
+    : [];
+  if (strict && mappedExercises.some((exercise) => !exercise)) return null;
+  const exercises = mappedExercises.filter(Boolean);
+
+  if (!dayLabel || dayLabel.length > PROGRAM_DRAFT_LIMITS.dayLabelMaxLen) return null;
+  if (!title || title.length > PROGRAM_DRAFT_LIMITS.dayTitleMaxLen) return null;
+  if (subtitle.length > PROGRAM_DRAFT_LIMITS.daySubtitleMaxLen) return null;
+  if (
+    exercises.length < PROGRAM_DRAFT_LIMITS.minExercisesPerDay ||
+    exercises.length > PROGRAM_DRAFT_LIMITS.maxExercisesPerDay
+  ) return null;
+
+  return { dayLabel, title, subtitle, exercises };
+}
+
+export function normalizeProgramDraft(rawProgram, { canonicalizeExerciseName, strict = false } = {}) {
+  if (!rawProgram || typeof rawProgram !== 'object' || Array.isArray(rawProgram)) return null;
+  if (!hasOnlyAllowedKeys(rawProgram, new Set([
+    'name',
+    'daysPerWeek',
+    'equipmentTier',
+    'volumeLevel',
+    'currentDayIndex',
+    'days'
+  ]))) return null;
+
+  const name = String(rawProgram.name ?? '').trim();
+  const mappedDays = Array.isArray(rawProgram.days)
+    ? rawProgram.days.map((day) => normalizeProgramDraftDay(day, canonicalizeExerciseName, strict))
+    : [];
+  if (strict && mappedDays.some((day) => !day)) return null;
+  const days = mappedDays.filter(Boolean);
+  const daysPerWeek = Number(rawProgram.daysPerWeek);
+  const currentDayIndex = rawProgram.currentDayIndex == null ? 0 : Number(rawProgram.currentDayIndex);
+  const equipmentTier = String(rawProgram.equipmentTier ?? 'fullGym').trim();
+  const volumeLevel = String(rawProgram.volumeLevel ?? 'moderate').trim();
+
+  if (!name || name.length > PROGRAM_DRAFT_LIMITS.nameMaxLen) return null;
+  if (days.length < PROGRAM_DRAFT_LIMITS.minDays || days.length > PROGRAM_DRAFT_LIMITS.maxDays) return null;
+  if (
+    !Number.isInteger(daysPerWeek) ||
+    daysPerWeek < PROGRAM_DRAFT_LIMITS.minDaysPerWeek ||
+    daysPerWeek > PROGRAM_DRAFT_LIMITS.maxDaysPerWeek
+  ) return null;
+  if (!Number.isInteger(currentDayIndex) || currentDayIndex < 0 || currentDayIndex >= days.length) return null;
+  if (!VALID_PROGRAM_DRAFT_EQUIPMENT_TIERS.has(equipmentTier) || !VALID_PROGRAM_DRAFT_VOLUME_LEVELS.has(volumeLevel)) return null;
+
+  return {
+    name,
+    daysPerWeek,
+    equipmentTier,
+    volumeLevel,
+    source: 'guided',
+    days,
+    currentDayIndex
+  };
+}
+
+export function extractAskProgramDraft(rawText, { canonicalizeExerciseName, strict = false } = {}) {
+  const text = String(rawText ?? '');
+  const match = text.match(/<program_draft>\s*([\s\S]*?)\s*<\/program_draft>/i);
+  if (!match) {
+    return { answerText: text.trim(), programDraft: null };
+  }
+
+  const answerText = collapseBlankLines(text.replace(match[0], ''));
+  let parsed;
+  try {
+    parsed = JSON.parse(match[1]);
+  } catch (err) {
+    console.warn('askCoach: <program_draft> JSON parse failed — dropping draft:', err.message);
+    return { answerText, programDraft: null };
+  }
+
+  const program = normalizeProgramDraft(parsed, { canonicalizeExerciseName, strict });
+  if (!program) {
+    console.warn('askCoach: <program_draft> payload failed validation — dropping draft');
+    return { answerText, programDraft: null };
+  }
+
+  return {
+    answerText,
+    programDraft: {
+      program,
+      provenance: {
+        source: 'ai-coach',
+        type: 'program',
+        version: PROGRAM_DRAFT_VERSION,
+        createdAt: new Date().toISOString(),
+        tokenHint: null
+      }
+    }
+  };
+}
+
+/**
+ * Whether `rawText` contains a <program_draft> tag at all (valid or not).
+ * Lets the eval distinguish "no draft" from "malformed draft".
+ */
+export function hasProgramDraftBlock(rawText) {
+  return /<\s*\/?\s*program_draft\b[^>]*>/i.test(String(rawText ?? ''));
+}

package/src/prompt-changelog.js

@@ -22,6 +22,69 @@ export const PROMPT_CHANGELOG_TYPES = Object.freeze([
 ]);
 
 export const PROMPT_CHANGELOG = Object.freeze([
+  {
+    version: 'ask_agentic_v2026_06_02_1',
+    surface: 'askAgentic',
+    date: '2026-06-02',
+    type: 'feature',
+    summary:
+      'Broad progress/bodyweight/on-track answers use coach-operator shape: verdict, signal, evidence, caveat, and the next decision. Progress reviews may ask one goal-defining question when body-composition tradeoffs depend on missing goal context, and now synthesize bodyweight/readiness evidence when routed context provides it.',
+    eval: 'ask_progress_review_golden'
+  },
+  {
+    version: 'ask_v2026_06_02_1',
+    surface: 'ask',
+    date: '2026-06-02',
+    type: 'feature',
+    summary:
+      'Broad progress/bodyweight/on-track answers use coach-operator shape: verdict, signal, evidence, caveat, and the next decision. Progress reviews may ask one goal-defining question when body-composition tradeoffs depend on missing goal context, and now synthesize bodyweight/readiness evidence when routed context provides it.',
+    eval: 'ask_progress_review_golden'
+  },
+  {
+    version: 'ask_agentic_v2026_06_01_1',
+    surface: 'askAgentic',
+    date: '2026-06-01',
+    type: 'safety',
+    summary:
+      'Hoist a high-salience "Hard limits" block to the top of ASK_RULES restating the most-violated nevers (no 1RM/PR/records unless asked, except the routed broad-review PR count; no fatigue/recovery/readiness language without an explicit signal; no warmup/backoff loads as working sets; no raw Increment Score sub-scores). Also: speak in the first person (never "the coach"/"the coach observation"/"the system") and never volunteer the overall score number unless asked — paired with a question-gated score prelude that withholds the numeric headline on non-score questions. Reinforcement of buried rules plus the self-reference and volunteered-score fixes the live history showed.',
+    eval: 'ask_why_failed_no_vitals'
+  },
+  {
+    version: 'ask_v2026_06_01_1',
+    surface: 'ask',
+    date: '2026-06-01',
+    type: 'safety',
+    summary:
+      'Hoist a high-salience "Hard limits" block to the top of ASK_RULES restating the most-violated nevers (no 1RM/PR/records unless asked, except the routed broad-review PR count; no fatigue/recovery/readiness language without an explicit signal; no warmup/backoff loads as working sets; no raw Increment Score sub-scores; speak in the first person, never "the coach"/"the system"; never volunteer the overall score number unless asked). Reinforcement of buried rules plus self-reference and volunteered-score fixes.',
+    eval: 'ask_why_failed_no_vitals'
+  },
+  {
+    version: 'ask_agentic_v2026_05_30_3',
+    surface: 'askAgentic',
+    date: '2026-05-30',
+    type: 'fix',
+    summary:
+      'Broad progress reviews must include the observed training frequency/session count alongside volume, body-weight, and recent PR-count evidence, so live Ask replays do not skip the basic activity denominator.',
+    eval: 'ask_progress_review_golden'
+  },
+  {
+    version: 'ask_agentic_v2026_05_30_2',
+    surface: 'askAgentic',
+    date: '2026-05-30',
+    type: 'fix',
+    summary:
+      'For broad progress-review questions, carry the base Ask rule that recent all-time estimated 1RM PR counts must be mentioned when the routed context provides them; preserves the bounded read-only tool loop from ask_agentic_v2026_05_30_1.',
+    eval: 'ask_progress_review_golden'
+  },
+  {
+    version: 'ask_agentic_v2026_05_30_1',
+    surface: 'askAgentic',
+    date: '2026-05-30',
+    type: 'feature',
+    summary:
+      'Agentic Ask generation: the model receives the routed context as a warm start plus a read-only tool menu (records, body weight, weekly volume, readiness, etc.) and fetches missing evidence over a bounded, deduped loop instead of answering one-shot from a fixed route. Server-side privacy exclusions are forced into every tool call; all fetched tools are folded into provenance. Inherits the ask_v2026_05_30_1 rules via an appended tool-use addendum.',
+    eval: 'ask_tool_provenance'
+  },
   {
     version: 'workout_v2026_05_23_1',
     surface: 'workout',
@@ -31,6 +94,33 @@ export const PROMPT_CHANGELOG = Object.freeze([
       'Keep skipped-exercise mentions generic unless plan comparison supports naming the lift; anchor the note to completed-session work.',
     eval: 'exercise_mentions'
   },
+  {
+    version: 'ask_v2026_05_30_3',
+    surface: 'ask',
+    date: '2026-05-30',
+    type: 'fix',
+    summary:
+      'Broad progress reviews must include the observed training frequency/session count alongside volume, body-weight, and recent PR-count evidence, so the answer keeps the activity denominator visible.',
+    eval: 'ask_progress_review_golden'
+  },
+  {
+    version: 'ask_v2026_05_30_2',
+    surface: 'ask',
+    date: '2026-05-30',
+    type: 'fix',
+    summary:
+      'Broad progress reviews must explicitly mention the recent all-time estimated 1RM PR count when the context includes it, preventing recent PR density from being softened into vague "several lifts moved" language.',
+    eval: 'ask_progress_review_golden'
+  },
+  {
+    version: 'ask_v2026_05_30_1',
+    surface: 'ask',
+    date: '2026-05-30',
+    type: 'safety',
+    summary:
+      'Enforce score-voice: name the Increment Score and its overall value/direction, but never recite raw component sub-scores, decimals, or daily score lists; translate the score into training reality. Answer training questions first (no score-dump lead), do not re-recite the breakdown on follow-ups, and answer retrospectives at the multi-week altitude asked. Paired with a voice-safe formatIncrementScorePrelude.',
+    eval: 'ask_score_voice'
+  },
   {
     version: 'ask_v2026_05_23_1',
     surface: 'ask',

package/src/promptfoo-evals.js

@@ -4,7 +4,8 @@ import {
   loadSummaryEvalSnapshot,
   summaryEvalFixturesRoot,
   buildSummaryEvalContext,
-  generateSummaryEvalOutputWithMetadata
+  generateSummaryEvalOutputWithMetadata,
+  summaryEvalsLiveGenerationEnabled
 } from './summary-evals.js';
 import { publishPromptfooLangfuseScore } from './promptfoo-langfuse-scores.js';
 
@@ -130,15 +131,20 @@ export async function assertPromptfooDomain(output, context = {}) {
 
 export async function callPromptfooProvider(prompt, context = {}) {
   const { testCase, snapshot } = await resolvePromptfooEval(context.vars ?? {});
-  const liveGenerationEnabled = envFlag('SUMMARY_EVALS_LIVE') || envFlag('PROMPTFOO_LIVE');
+  const liveGenerationEnabled = summaryEvalsLiveGenerationEnabled();
 
   if (!liveGenerationEnabled) {
+    const evalContext = buildSummaryEvalContext(snapshot, testCase);
+    const generation = await generateSummaryEvalOutputWithMetadata(testCase, evalContext, snapshot);
+    promptfooProviderMetadata.set(promptfooMetadataKey(context.vars ?? {}), generation.metadata);
+
     return {
-      output: testCase.output,
+      output: generation.output,
       metadata: {
         caseId: testCase.id,
         surface: testCase.surface,
-        mode: 'stored'
+        mode: 'stored',
+        ...generation.metadata
       }
     };
   }

package/src/promptfoo-langfuse-scores.js

@@ -100,6 +100,23 @@ function failedChecks(result) {
   return (result.checks ?? []).filter((check) => !check.passed);
 }
 
+function nonEmptyStrings(values, { max = 12 } = {}) {
+  const unique = [];
+  const seen = new Set();
+  for (const value of Array.isArray(values) ? values : []) {
+    const text = typeof value === 'string' ? value.trim() : '';
+    if (!text || seen.has(text)) continue;
+    seen.add(text);
+    unique.push(text);
+    if (unique.length >= max) break;
+  }
+  return unique.length > 0 ? unique : undefined;
+}
+
+function countArray(values) {
+  return Array.isArray(values) ? values.length : undefined;
+}
+
 function langfuseScoreTarget({ testCase = {}, context = {}, sessionId }) {
   const providerMetadata = context.providerResponse?.metadata ?? {};
   const caseMetadata = testCase.metadata ?? {};
@@ -138,6 +155,21 @@ export function buildPromptfooLangfuseScorePayload({
   const runId = scoreRunId(context, env, now);
   const mode = promptfooEvalMode(env);
   const provider = providerLabel(context);
+  const providerMetadata = context.providerResponse?.metadata ?? {};
+  const routingMetadata = providerMetadata.routingMetadata && typeof providerMetadata.routingMetadata === 'object'
+    ? providerMetadata.routingMetadata
+    : {};
+  const evidencePlan = routingMetadata.evidencePlan && typeof routingMetadata.evidencePlan === 'object'
+    ? routingMetadata.evidencePlan
+    : providerMetadata.evidencePlan && typeof providerMetadata.evidencePlan === 'object'
+      ? providerMetadata.evidencePlan
+      : {};
+  const contextBundle = routingMetadata.contextBundle && typeof routingMetadata.contextBundle === 'object'
+    ? routingMetadata.contextBundle
+    : {};
+  const structured = providerMetadata.structured && typeof providerMetadata.structured === 'object'
+    ? providerMetadata.structured
+    : {};
   const failed = failedChecks(result);
   const promptVersion = vars.promptVersion
     ?? testCase.metadata?.promptVersion
@@ -179,6 +211,29 @@ export function buildPromptfooLangfuseScorePayload({
       assertionCount: result.checks?.length ?? 0,
       failedAssertionKeys: failed.map((check) => check.key),
       failedAssertionReasons: failed.map((check) => check.reason).slice(0, 10),
+      route: firstString(routingMetadata.route, evidencePlan.route),
+      effectiveRoute: firstString(routingMetadata.effectiveRoute, evidencePlan.effectiveRoute),
+      requestedAction: firstString(routingMetadata.intent?.requestedAction),
+      intentConfidence: typeof routingMetadata.intent?.confidence === 'number' ? routingMetadata.intent.confidence : undefined,
+      contextCharCount: typeof routingMetadata.contextCharCount === 'number' ? routingMetadata.contextCharCount : undefined,
+      historyTurnCount: typeof routingMetadata.historyTurnCount === 'number' ? routingMetadata.historyTurnCount : undefined,
+      requiredTools: nonEmptyStrings(evidencePlan.requiredTools),
+      executedTools: nonEmptyStrings(evidencePlan.executedTools ?? routingMetadata.toolsUsed ?? contextBundle.executedTools),
+      evidenceGaps: nonEmptyStrings(evidencePlan.evidenceGaps),
+      missingDataFlags: nonEmptyStrings([
+        ...(routingMetadata.missingDataFlags ?? []),
+        ...(contextBundle.missingDataFlags ?? []),
+        ...(evidencePlan.evidenceGaps ?? [])
+      ]),
+      structuredResponsePresent: Object.keys(structured).length > 0 ? true : undefined,
+      structuredConfidence: firstString(structured.confidence),
+      followUpSuggestionCount: countArray(structured.followUpSuggestions),
+      limitationCount: countArray(structured.limitations),
+      evidenceUsedLabels: nonEmptyStrings((structured.evidenceUsed ?? []).map((item) => item?.label)),
+      evidenceUsedTools: nonEmptyStrings((structured.evidenceUsed ?? []).map((item) => item?.toolName)),
+      recommendedActionIds: nonEmptyStrings((structured.recommendedActions ?? []).map((item) => item?.id)),
+      recommendedActionLabels: nonEmptyStrings((structured.recommendedActions ?? []).map((item) => item?.label)),
+      hasProgramDraft: structured.programDraft != null ? true : undefined,
       generatedAt: now.toISOString()
     }),
     environment: env.LANGFUSE_ENVIRONMENT ?? env.NODE_ENV ?? 'development'