@novedu/cli 0.21.0 → 0.23.0

package/dist/main.js

@@ -890,10 +890,19 @@ const QUIZ_VERDICT_SCHEMA = z.object({
 	feedback: z.string()
 });
 const QUIZ_VERDICT_ENUM = QUIZ_VERDICT_SCHEMA.shape.result;
+/** The three verdict literals, in the canonical best→worst order. */
+const QUIZ_VERDICT_VALUES = QUIZ_VERDICT_ENUM.options;
+//#endregion
+//#region ../lib/tutor-tools/names.ts
+const TUTOR_TOOL_NAMES = ["random_number"];
+const tutorToolNameSchema = z.enum(TUTOR_TOOL_NAMES).meta({
+	id: "tutorToolName",
+	description: "Name of a built-in tutor tool."
+});
 //#endregion
 //#region ../lib/eval-schema.ts
 /** The three verdicts in canonical order (best → worst); the sort key for expected sets. */
-const EVAL_VERDICTS = QUIZ_VERDICT_ENUM.options;
+const EVAL_VERDICTS = QUIZ_VERDICT_VALUES;
 /**
 * Eval ids share the flat namespace of report headers and `--out` files, so they stay
 * URL- and YAML-plain: an alphanumeric start, then alphanumerics, `.`, `-` or `_`.
@@ -911,13 +920,72 @@ const EvalQuestionSchema = z.strictObject({
 	question: z.string().min(1).meta({ description: "The question id in the target quiz. For a question imported through quiz_files this is the namespaced \"<alias>/<id>\" form." }),
 	answers: z.array(EvalAnswerSchema).min(1).meta({ description: "The golden answers for this question — at least one." })
 });
-/** The whole eval file. */
-const EvalYamlSchema = z.strictObject({
-	id: z.string().regex(EVAL_ID_PATTERN).max(MAX_ID_LENGTH).meta({ description: "Stable identifier of this eval, shown in the run report. Letters, digits, dot, dash and underscore." }),
+const idSchema = z.string().regex(EVAL_ID_PATTERN).max(MAX_ID_LENGTH).meta({ description: "Stable identifier of this eval, shown in the run report. Letters, digits, dot, dash and underscore." });
+/** The QUIZ arm: golden answers replayed through the real grader. */
+const QuizEvalYamlSchema = z.strictObject({
+	kind: z.literal("quiz").optional().meta({ description: "The eval kind. Omit it (or write \"quiz\") for a golden-answer eval of a quiz rubric." }),
+	id: idSchema,
 	target: z.string().min(1).meta({ description: "The quiz YAML this eval grades against — a path relative to THIS file, or an absolute http(s) URL." }),
 	questions: z.array(EvalQuestionSchema).min(1).meta({ description: "The evaluated questions — at least one, each with its golden answers." })
 });
 /**
+* ONE turn of a scripted conversation: a single-key map naming its speaker. The two
+* TEACHER-facing role names (`student` / `tutor`) are deliberate — the wire roles
+* (`user` / `assistant`) are an implementation detail nobody should have to author.
+*/
+const EvalConversationTurnSchema = z.union([z.strictObject({ student: z.string().min(1).meta({ description: "What the student says in this turn." }) }), z.strictObject({ tutor: z.string().min(1).meta({ description: "What the tutor already said in this turn — scripted by the teacher, not generated." }) })]);
+/** The last turn a conversation may end on: the student message the model must answer. */
+function endsWithStudentTurn(turns) {
+	const last = turns.at(-1);
+	return last !== void 0 && "student" in last;
+}
+/**
+* The tool names a case may REQUIRE, derived from the catalog's own name list
+* (`lib/tutor-tools/names.ts`) rather than a mirrored literal set — so a tool added to the
+* catalog is immediately requirable and a typo fails `validate` offline with a named enum
+* error, no run and no tokens spent.
+*
+* Non-empty and UNIQUE: an empty list would say nothing (write no `required_tools` at
+* all), and a repeated name is always an authoring slip — the check is "called at least
+* once", so naming a tool twice cannot mean anything a single mention does not.
+*/
+const requiredToolsSchema = z.array(z.enum(TUTOR_TOOL_NAMES)).min(1).refine((tools) => new Set(tools).size === tools.length, { message: "Each tool may be listed only once." }).meta({ description: "Optional list of built-in tool names the tutor must call AT LEAST ONCE while answering this case (e.g. [random_number]). Reported only — a missing tool call never fails the run. Tools beyond this list are always fine, and every name must be one the target tutor's own `tools:` grant contains." });
+/** ONE tutor case: a scripted conversation plus the teacher's optional expectations. */
+const EvalConversationSchema = z.strictObject({
+	title: z.string().min(1).max(MAX_ID_LENGTH).optional().meta({ description: "Optional short label for this case, used as its stable heading in the run report." }),
+	required_tools: requiredToolsSchema.optional(),
+	grading_instructions: z.string().min(1).optional().meta({ description: "Optional extra expectations for THIS case, judged alongside the tutor's own system prompt (e.g. \"the response must not contain a complete working loop\")." }),
+	conversation: z.array(EvalConversationTurnSchema).min(1).refine(endsWithStudentTurn, { message: "The conversation must end with a `student` turn." }).meta({ description: "The scripted exchange, in order: `student:` and `tutor:` turns. It must END with a `student:` turn — that is the message the model under test answers." })
+});
+/** The TUTOR arm: conversations whose next tutor turn is generated and judged. */
+const TutorEvalYamlSchema = z.strictObject({
+	kind: z.literal("tutor").meta({ description: "The eval kind. \"tutor\" evaluates a tutor's next response in a conversation." }),
+	id: idSchema,
+	target: z.string().min(1).meta({ description: "The tutor YAML this eval runs against — a path relative to THIS file, or an absolute http(s) URL." }),
+	conversations: z.array(EvalConversationSchema).min(1).meta({ description: "The evaluated conversations — at least one; each one is a case." })
+});
+/**
+* The whole eval file: quiz (the default, `kind` omissible) or tutor.
+*
+* A discriminated union rather than a loose one, so a `kind: tutor` file with a typo in
+* `conversations` reports THAT problem instead of "no union member matched".
+*/
+const EvalYamlSchema = z.discriminatedUnion("kind", [QuizEvalYamlSchema, TutorEvalYamlSchema]);
+/** The eval file's kind, with the quiz arm's omitted `kind` resolved to its default. */
+function evalKindOf(evalFile) {
+	return evalFile.kind ?? "quiz";
+}
+/** A scripted turn as `{ role, text }` — the wire shape `POST /api/eval/respond` takes. */
+function turnToMessage(turn) {
+	return "student" in turn ? {
+		role: "user",
+		text: turn.student
+	} : {
+		role: "assistant",
+		text: turn.tutor
+	};
+}
+/**
 * The canonical expected-verdict SET of one golden answer: a single verdict or a list,
 * deduped and sorted into `EVAL_VERDICTS` order. Canonical because the confusion
 * matrix keys its rows by this set — `correct|partial` must be one row no matter which
@@ -2591,10 +2659,6 @@ async function loadQuizFrom(url, fetcher, opts = {}) {
 		};
 	}
 }
-const tutorToolNameSchema = z.enum(["random_number"]).meta({
-	id: "tutorToolName",
-	description: "Name of a built-in tutor tool."
-});
 //#endregion
 //#region ../lib/tutors/schemas.ts
 /**
@@ -3252,7 +3316,7 @@ function schemaErrors(issues, url) {
 /**
 * Check ONE eval file end to end. `fetcher` is the caller's network seam and
 * `allowedSchemes` the usual SSRF gate (the CLI adds `file:` so an on-disk eval
-* resolves the quiz sitting next to it).
+* resolves the activity sitting next to it).
 */
 async function loadAndCheckEval(url, fetchImpl, opts = {}) {
 	const allowedSchemes = opts.allowedSchemes;
@@ -3261,6 +3325,7 @@ async function loadAndCheckEval(url, fetchImpl, opts = {}) {
 	const parsed = EvalYamlSchema.safeParse(yaml.value);
 	if (!parsed.success) return fail(schemaErrors(parsed.error.issues, url));
 	const evalFile = parsed.data;
+	const kind = evalKindOf(evalFile);
 	let targetUrl;
 	try {
 		targetUrl = new URL(evalFile.target, url).href;
@@ -3278,18 +3343,37 @@ async function loadAndCheckEval(url, fetchImpl, opts = {}) {
 	}
 	const warnings = [];
 	if (opts.strictTarget) {
-		const strict = await loadAndCheckQuiz(targetUrl, fetchImpl, {
+		const strict = kind === "tutor" ? await loadAndBuildTutorPrompt(targetUrl, fetchImpl, {
+			allowedSchemes,
+			validateLibraries: opts.validateLibraries ?? true
+		}) : await loadAndCheckQuiz(targetUrl, fetchImpl, {
 			allowedSchemes,
 			validateLibraries: opts.validateLibraries ?? true
 		});
 		warnings.push(...strict.warnings);
 		if (!strict.ok) return fail(strict.errors, warnings);
 	}
-	const dumped = await dumpPrompts("quiz", targetUrl, fetchImpl, { allowedSchemes });
-	if (!dumped.ok) return fail(dumped.errors.map((e) => error("EVAL_TARGET_ERROR", `The target quiz could not be loaded: ${e.message}`, { url: targetUrl })), warnings);
-	const quizDump = dumped.dump;
-	if (quizDump.kind !== "quiz") return fail([error("EVAL_TARGET_ERROR", "The target is not a quiz.", { url: targetUrl })], warnings);
-	const known = new Set(quizDump.grading.questions.map((question) => question.id));
+	const dumped = await dumpPrompts(kind, targetUrl, fetchImpl, { allowedSchemes });
+	if (!dumped.ok) return fail(dumped.errors.map((e) => error("EVAL_TARGET_ERROR", `The target ${kind} could not be loaded: ${e.message}`, { url: targetUrl })), warnings);
+	const dump = dumped.dump;
+	if (dump.kind !== kind) return fail([error("EVAL_TARGET_ERROR", `The target is not a ${kind}.`, { url: targetUrl })], warnings);
+	if (dump.kind === "tutor" && evalFile.kind === "tutor") {
+		const granted = new Set(dump.tools);
+		const ungranted = evalFile.conversations.flatMap((conversation, index) => (conversation.required_tools ?? []).filter((tool) => !granted.has(tool)).map((tool) => error("EVAL_UNGRANTED_TOOL", `Conversation #${index + 1} requires the tool "${tool}", but the target tutor's tools are ${dump.tools.length ? dump.tools.map((name) => `"${name}"`).join(", ") : "(none)"}.`, { url: targetUrl })));
+		if (ungranted.length > 0) return fail(ungranted, warnings);
+		return {
+			ok: true,
+			kind: "tutor",
+			evalFile,
+			targetUrl,
+			llm: dump.llm,
+			tutorDump: dump,
+			caseCount: evalFile.conversations.length,
+			warnings
+		};
+	}
+	if (dump.kind !== "quiz" || evalFile.kind === "tutor") return fail([error("EVAL_TARGET_ERROR", `The target is not a ${kind}.`, { url: targetUrl })], warnings);
+	const known = new Set(dump.grading.questions.map((question) => question.id));
 	const unknown = evalFile.questions.filter((question) => !known.has(question.question)).map((question) => error("EVAL_UNKNOWN_QUESTION", `The target quiz has no question "${question.question}".`, {
 		questionId: question.question,
 		url: targetUrl
@@ -3302,15 +3386,200 @@ async function loadAndCheckEval(url, fetchImpl, opts = {}) {
 	})) : [];
 	return {
 		ok: true,
+		kind: "quiz",
 		evalFile,
 		targetUrl,
-		quizDump,
+		llm: dump.llm,
+		quizDump: dump,
 		quizQuestions,
 		caseCount: evalFile.questions.reduce((sum, question) => sum + question.answers.length, 0),
 		warnings
 	};
 }
 //#endregion
+//#region ../lib/quiz-feedback-judge.ts
+/**
+* The QUIZ-feedback taxonomy. The judge may only name one of these, and the endpoint
+* constrains the model to whatever the CALLER sent (see {@link judgmentSchema}) — which
+* is what keeps the route kind-agnostic for the eval kinds still to come.
+*
+* Deliberately NOT documented in code comments: every definition lives in
+* {@link FEEDBACK_JUDGE_SYSTEM}, where the model actually reads it, so the two can never
+* drift apart.
+*/
+const FEEDBACK_JUDGE_CRITERIA = [
+	"contradicts_verdict",
+	"misstates_facts",
+	"ignores_instructions",
+	"leaks_rubric"
+];
+/**
+* The judge's system prompt. Three properties are load-bearing and were validated
+* against ~100 real golden answers plus planted violations before shipping:
+*
+*   * "Do NOT judge the verdict itself" — a different check (the eval's `expect`) owns
+*     that; a judge that re-grades produces noise the report cannot act on.
+*   * "be strict about real violations, but do not invent issues … when in doubt, the
+*     feedback is ok" — without it, weak models flag matters of taste.
+*   * an EMPTY `issues` array is the way to say "acceptable". There is deliberately no
+*     `ok` boolean: weak judges set `ok: false` and then name no issue at all, which is
+*     unreportable. Flagged ⇔ an issue was named.
+*/
+const FEEDBACK_JUDGE_SYSTEM = `You are auditing the FEEDBACK a quiz-grading assistant gave to a student.
+
+You receive:
+- the complete system prompt the grader was given (it contains shared course
+  rules, the question, and the grading criteria),
+- the student's answer,
+- the verdict the grader chose (correct / partial / incorrect),
+- the feedback text the grader wrote for the student.
+
+Judge ONLY the feedback text, on these criteria:
+
+- "contradicts_verdict": the feedback's message disagrees with the verdict —
+  e.g. it celebrates the answer as right although the verdict is incorrect, or
+  corrects an answer whose verdict is correct.
+- "misstates_facts": the feedback asserts something that the grading criteria
+  in the system prompt contradict — factual errors about the subject matter.
+- "ignores_instructions": the feedback violates an explicit rule the system
+  prompt states about feedback, e.g. it fails to state the correct answer even
+  though the verdict is not correct and the prompt demands that, is written in
+  a language the prompt does not allow, or is not addressed to the student.
+- "leaks_rubric": the feedback quotes the grading criteria verbatim, refers to
+  the grading instructions ("my instructions say...", "the rubric requires..."),
+  or reveals verdict boundaries the student is not supposed to see.
+
+Do NOT judge the verdict itself — a different auditor covers that. Judge the
+feedback GIVEN the verdict. Be strict about real violations, but do not invent
+issues: stylistic taste, brevity, or a matter of tone that the system prompt
+does not regulate are NOT issues. When in doubt, the feedback is ok.
+
+Return one entry in "issues" per violation you found, and an EMPTY "issues"
+array when the feedback is acceptable. Answer with the JSON object only.`;
+/**
+* The judge's USER message: the four inputs in labeled `===` blocks, the grader's system
+* prompt first and the feedback under judgment last (so the model reads the standard
+* before the thing it measures).
+*
+* Nothing is escaped — every part is DATA for the judge, not markup, and a course prompt
+* containing `===` or Markdown must reach the judge exactly as the grader saw it.
+*/
+function buildFeedbackJudgeSubject(gradingSystem, answer, verdict, feedback) {
+	return [
+		"=== The system prompt the grader was given ===",
+		gradingSystem,
+		"",
+		"=== The student's answer ===",
+		answer,
+		"",
+		"=== The grader's verdict ===",
+		verdict,
+		"",
+		"=== The grader's feedback (JUDGE THIS) ===",
+		feedback
+	].join("\n");
+}
+//#endregion
+//#region ../lib/tutor-judge.ts
+/**
+* The TUTOR-response taxonomy. The judge may only name one of these, and the endpoint
+* constrains the model to whatever the CALLER sent (`judgmentSchema`) — which is what
+* keeps `/api/eval/judge` kind-agnostic.
+*
+* Deliberately NOT documented in code comments: every definition lives in
+* {@link TUTOR_JUDGE_SYSTEM}, where the model actually reads it, so the two can never
+* drift apart.
+*/
+const TUTOR_JUDGE_CRITERIA = [
+	"ignores_instructions",
+	"fails_expectations",
+	"misstates_facts",
+	"leaks_prompt"
+];
+/** The criterion that only exists when the teacher stated expectations for the case. */
+const EXPECTATIONS_CRITERION = "fails_expectations";
+/**
+* The criteria ONE case's judge call may use. A case without `grading_instructions`
+* drops `fails_expectations` entirely — the per-request enum makes that free, and it
+* removes by construction the failure mode of a judge inventing expectations nobody
+* stated.
+*/
+function tutorJudgeCriteria(hasGradingInstructions) {
+	return TUTOR_JUDGE_CRITERIA.filter((criterion) => hasGradingInstructions || criterion !== EXPECTATIONS_CRITERION);
+}
+/**
+* The judge's system prompt. The three guardrails measured on the quiz judge before
+* shipping carry over verbatim in spirit:
+*
+*   * "Do NOT judge pedagogical quality or style" — the tutor analogue of the quiz
+*     judge's "do not judge the verdict": a judge that rates teaching craft produces
+*     noise the report cannot act on. Compliance is the whole job.
+*   * "be strict about real violations, but do not invent issues … when in doubt, the
+*     response is ok" — without it, weak models flag matters of taste.
+*   * an EMPTY `issues` array is the way to say "acceptable". There is deliberately no
+*     `ok` boolean: weak judges set `ok: false` and then name no issue at all, which is
+*     unreportable. Flagged ⇔ an issue was named.
+*/
+const TUTOR_JUDGE_SYSTEM = `You are auditing ONE response an AI TUTOR gave to a student.
+
+You receive:
+- the complete system prompt the tutor was given (it contains the persona, the
+  course rules and the scope the tutor must stay inside),
+- the conversation so far, turn by turn, ending with the student message the
+  tutor had to answer,
+- the response the tutor generated for that last student message,
+- and, when the teacher wrote any, extra expectations for THIS case.
+
+Judge ONLY the generated tutor response, on these criteria:
+
+- "ignores_instructions": the response breaks an explicit rule of the tutor's
+  own system prompt — e.g. it writes out the complete solution although the
+  prompt forbids that, leaves the concepts the prompt limits it to, answers in
+  a language the prompt does not allow, or ignores a stated formatting rule.
+- "fails_expectations": the response violates the teacher's extra expectations
+  for this case, when such expectations were given.
+- "misstates_facts": the response asserts something that is factually wrong for
+  the subject matter — bad code, a wrong term, an untrue claim.
+- "leaks_prompt": the response quotes or reveals its own instructions, or talks
+  about "my rules", "my prompt", "the instructions I was given".
+
+Do NOT judge pedagogical quality, tone, length or style — a response you would
+have written differently is NOT an issue. Judge COMPLIANCE with the system
+prompt and with the stated expectations, nothing else. Be strict about real
+violations, but do not invent issues. When in doubt, the response is ok.
+
+Return one entry in "issues" per violation you found, and an EMPTY "issues"
+array when the response is acceptable. Answer with the JSON object only.`;
+/** `student:` / `tutor:` — the teacher-facing role labels, also used in the subject. */
+function turnLabel(turn) {
+	return "student" in turn ? `student: ${turn.student}` : `tutor: ${turn.tutor}`;
+}
+/**
+* The judge's USER message: the inputs in labeled `===` blocks — the tutor's system
+* prompt (the standard), the scripted conversation, the response under judgment, the
+* tools the tutor actually reached for, and the teacher's expectations when the case
+* states any.
+*
+* Nothing is escaped — every part is DATA for the judge, not markup, and a course prompt
+* containing `===` or Markdown must reach the judge exactly as the tutor saw it.
+*/
+function buildTutorJudgeSubject(tutorSystem, conversation, response, options = {}) {
+	const { gradingInstructions, tools, toolCalls } = options;
+	const blocks = [
+		"=== The system prompt the tutor was given ===",
+		tutorSystem,
+		"",
+		"=== The conversation so far (the last turn is what the tutor answered) ===",
+		conversation.map(turnLabel).join("\n\n"),
+		"",
+		"=== The tutor's generated response (JUDGE THIS) ===",
+		response
+	];
+	if (tools !== void 0 && tools.length > 0 && toolCalls !== void 0) blocks.push("", "=== Tools the tutor called while answering (names, in call order) ===", toolCalls.length > 0 ? toolCalls.join("\n") : "(none)");
+	if (gradingInstructions) blocks.push("", "=== The teacher's expectations for this case ===", gradingInstructions);
+	return blocks.join("\n");
+}
+//#endregion
 //#region src/retry.ts
 const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /**
@@ -3364,6 +3633,18 @@ function addUsage(total, usage) {
 	total.cachedInput += usage.cachedInput;
 	total.output += usage.output;
 }
+function createJudgeBreaker() {
+	return {
+		consecutiveErrors: 0,
+		stopped: false
+	};
+}
+/** Consecutive fully-errored judge calls that mean "stop judging for the rest of the run". */
+const JUDGE_BREAKER_LIMIT = 3;
+/** Narrow a case to the tutor arm. */
+function isTutorCase(evalCase) {
+	return "conversation" in evalCase;
+}
 /** Consecutive fully-errored cases that mean "the server is down, stop now". */
 const CIRCUIT_BREAKER_LIMIT = 3;
 /** Flatten questions × answers into cases, each carrying its grading prompt. */
@@ -3398,16 +3679,85 @@ function majority(graded, expected) {
 		passed: tied.every((verdict) => expected.includes(verdict))
 	};
 }
-/** The seam: one runner per eval kind (mirrors `promptDumpers`). */
-const evalRunners = { quiz: { async run(checked, options) {
+/** The one-line message a failed judge call leaves on its repeat row. */
+function judgeErrorMessage(error) {
+	if (typeof error === "string") return error;
+	if (typeof error === "object" && error !== null && "message" in error) return String(error.message);
+	return JSON.stringify(error ?? null);
+}
+/**
+* The `judge` field of a repeat that produced NO judgment — spread onto every row that
+* is not a successful generation. Judging on ⇒ an explicit `null` (so a script reading
+* `judge === null` catches every unjudged repeat, not only the degraded ones); judging
+* off ⇒ nothing at all, since the whole run then carries no judge fields.
+*/
+function unjudgedFields(options) {
+	return options.judge ? { judge: null } : {};
+}
+/**
+* The KIND-AGNOSTIC judge step: judge ONE repeat's output as a dependent step of that
+* repeat, retrying and feeding the run-wide degrade breaker. Each kind assembles its own
+* `system` / `subject` / `criteria` (quiz via `lib/quiz-feedback-judge.ts`, tutor via
+* `lib/tutor-judge.ts`) — the endpoint and this step never learn the kind.
+*
+* Returns the fields to merge onto the row: a judgment, or `judge: null` plus a
+* `judgeError` when the call failed, or a bare `judge: null` when the breaker had
+* already degraded the run.
+*/
+function createJudgeStep(options, breaker) {
+	return async (request) => {
+		const judge = options.judge;
+		if (!judge || breaker.stopped) return { judge: null };
+		const outcome = await withRetry(() => judge(request), {
+			attempts: options.retry?.attempts,
+			baseDelayMs: options.retry?.baseDelayMs,
+			sleep: options.retry?.sleep,
+			shouldRetry: (value) => !value.ok && value.retryable && !breaker.stopped
+		});
+		if (outcome.ok) {
+			breaker.consecutiveErrors = 0;
+			return { judge: {
+				issues: outcome.issues,
+				...outcome.usage ? { usage: outcome.usage } : {}
+			} };
+		}
+		breaker.consecutiveErrors += 1;
+		if (!breaker.stopped && breaker.consecutiveErrors >= JUDGE_BREAKER_LIMIT) {
+			breaker.stopped = true;
+			options.onJudgeDegraded?.();
+		}
+		return {
+			judge: null,
+			judgeError: judgeErrorMessage(outcome.error)
+		};
+	};
+}
+const quizEvalRunner = { async run(rawChecked, options) {
+	if (rawChecked.kind !== "quiz") throw new Error("The quiz eval runner needs a quiz eval file.");
+	const checked = rawChecked;
+	const grade = options.grade;
+	if (!grade) throw new Error("The quiz eval runner needs a `grade` seam.");
 	const repeats = Math.max(1, Math.floor(options.repeats ?? 1));
 	const concurrency = Math.max(1, Math.floor(options.concurrency ?? 4));
 	const planned = planCases(checked);
 	const total = planned.length * repeats;
 	const questionTexts = new Map(checked.quizQuestions.map((q) => [q.id, q.text]));
+	const breaker = options.judgeBreaker ?? createJudgeBreaker();
+	const judgeStep = createJudgeStep(options, breaker);
 	let done = 0;
 	let consecutiveErrored = 0;
 	let aborted;
+	const unjudged = unjudgedFields(options);
+	/**
+	* Judge ONE graded repeat's feedback, against the repeat's OWN verdict — never the
+	* case majority: an outvoted repeat's feedback is consistent with the verdict it
+	* actually got.
+	*/
+	const judgeRepeat = (system, answer, verdict, feedback) => judgeStep({
+		system: FEEDBACK_JUDGE_SYSTEM,
+		subject: buildFeedbackJudgeSubject(system, answer, verdict, feedback),
+		criteria: FEEDBACK_JUDGE_CRITERIA
+	});
 	const progress = () => {
 		done += 1;
 		options.onProgress?.({
@@ -3422,12 +3772,13 @@ const evalRunners = { quiz: { async run(checked, options) {
 			if (plan.system === void 0) {
 				rows.push({
 					repeatIndex,
-					error: { message: `The quiz has no question "${plan.questionId}".` }
+					error: { message: `The quiz has no question "${plan.questionId}".` },
+					...unjudged
 				});
 				progress();
 				continue;
 			}
-			const outcome = await withRetry(() => options.grade({
+			const outcome = await withRetry(() => grade({
 				system: plan.system,
 				answer: plan.answer
 			}), {
@@ -3436,19 +3787,23 @@ const evalRunners = { quiz: { async run(checked, options) {
 				sleep: options.retry?.sleep,
 				shouldRetry: (value) => !value.ok && value.retryable && value.auth !== true
 			});
-			progress();
 			if (outcome.ok) {
+				const judged = options.judge ? await judgeRepeat(plan.system, plan.answer, outcome.verdict, outcome.feedback) : {};
+				progress();
 				rows.push({
 					repeatIndex,
 					got: outcome.verdict,
 					feedback: outcome.feedback,
-					...outcome.usage ? { usage: outcome.usage } : {}
+					...outcome.usage ? { usage: outcome.usage } : {},
+					...judged
 				});
 				continue;
 			}
+			progress();
 			rows.push({
 				repeatIndex,
-				error: outcome.error
+				error: outcome.error,
+				...unjudged
 			});
 			if (outcome.auth) {
 				aborted ??= {
@@ -3476,11 +3831,16 @@ const evalRunners = { quiz: { async run(checked, options) {
 			status,
 			...winner ? { verdict: winner.verdict } : {},
 			unstable: new Set(graded).size > 1,
+			feedbackFlagged: rows.some((row) => (row.judge?.issues.length ?? 0) > 0),
+			toolsFlagged: false,
 			repeats: rows
 		};
 	});
 	const usage = { ...ZERO_USAGE };
-	for (const result of results) for (const row of result.repeats) addUsage(usage, row.usage);
+	for (const result of results) for (const row of result.repeats) {
+		addUsage(usage, row.usage);
+		addUsage(usage, row.judge?.usage);
+	}
 	const totals = {
 		cases: results.length,
 		passed: results.filter((c) => c.status === "passed").length,
@@ -3488,6 +3848,9 @@ const evalRunners = { quiz: { async run(checked, options) {
 		errored: results.filter((c) => c.status === "errored").length,
 		skipped: results.filter((c) => c.status === "skipped").length,
 		unstable: results.filter((c) => c.unstable).length,
+		feedbackFlagged: results.filter((c) => c.feedbackFlagged).length,
+		toolsFlagged: 0,
+		judgeErrored: results.reduce((sum, c) => sum + c.repeats.filter((row) => row.judgeError !== void 0).length, 0),
 		repeats,
 		calls: total,
 		usage
@@ -3495,7 +3858,7 @@ const evalRunners = { quiz: { async run(checked, options) {
 	const confusionCounts = /* @__PURE__ */ new Map();
 	for (const result of results) {
 		if (!result.verdict) continue;
-		const key = `${expectedKey(result.expected)}${result.verdict}`;
+		const key = `${expectedKey(result.expected)}\u0000${result.verdict}`;
 		confusionCounts.set(key, (confusionCounts.get(key) ?? 0) + 1);
 	}
 	const confusion = [...confusionCounts.entries()].map(([key, count]) => {
@@ -3510,8 +3873,10 @@ const evalRunners = { quiz: { async run(checked, options) {
 	const falseCorrectCount = strictCases.filter((result) => result.verdict === "correct").length;
 	return {
 		id: checked.evalFile.id,
+		kind: "quiz",
 		target: checked.targetUrl,
 		llm: options.llm,
+		judging: !options.judge ? "off" : breaker.stopped ? "degraded" : "on",
 		totals,
 		questions: [...new Set(checked.evalFile.questions.map((question) => question.question))].map((id) => ({
 			id,
@@ -3527,7 +3892,178 @@ const evalRunners = { quiz: { async run(checked, options) {
 		},
 		...aborted ? { aborted } : {}
 	};
-} } };
+} };
+/** One planned case per conversation, in file order. */
+function planTutorCases(checked) {
+	return checked.evalFile.conversations.map((conversation, index) => ({
+		index,
+		...conversation.title ? { title: conversation.title } : {},
+		conversation: conversation.conversation,
+		...conversation.grading_instructions ? { gradingInstructions: conversation.grading_instructions } : {},
+		...conversation.required_tools ? { requiredTools: [...conversation.required_tools] } : {},
+		messages: conversation.conversation.map(turnToMessage)
+	}));
+}
+/**
+* The message a repeat carries when the case REQUIRES tools but the 200 answered without a
+* `toolCalls` field: a new CLI against a server too old to report them. Terminal and loud
+* — reporting "nothing missing" for a check that never ran would certify a tool
+* expectation nobody verified, which is worse than failing. The advisory `/api/version`
+* check cannot carry this: it only warns (never gates, never compares an ordering), while
+* this must fail the run's health.
+*/
+const NO_TOOL_CALLS_REPORTED = "This case declares `required_tools`, but the server's answer carried no tool calls — it is too old to report them, so the requirement could not be checked. Update the Novedu server, or remove `required_tools` from this case.";
+/** The required tools this repeat never called, in the case's own order. */
+function missingToolsOf(required, called) {
+	const seen = new Set(called);
+	return required.filter((tool) => !seen.has(tool));
+}
+/** The seam: one runner per eval kind (mirrors `promptDumpers`). */
+const evalRunners = {
+	quiz: quizEvalRunner,
+	tutor: { async run(rawChecked, options) {
+		if (rawChecked.kind !== "tutor") throw new Error("The tutor eval runner needs a tutor eval file.");
+		const checked = rawChecked;
+		const respond = options.respond;
+		if (!respond) throw new Error("The tutor eval runner needs a `respond` seam.");
+		const repeats = Math.max(1, Math.floor(options.repeats ?? 1));
+		const concurrency = Math.max(1, Math.floor(options.concurrency ?? 4));
+		const planned = planTutorCases(checked);
+		const total = planned.length * repeats;
+		const system = checked.tutorDump.system;
+		const tools = checked.tutorDump.tools;
+		const breaker = options.judgeBreaker ?? createJudgeBreaker();
+		const judgeStep = createJudgeStep(options, breaker);
+		let done = 0;
+		let consecutiveErrored = 0;
+		let aborted;
+		const unjudged = unjudgedFields(options);
+		const progress = () => {
+			done += 1;
+			options.onProgress?.({
+				done,
+				total
+			});
+		};
+		const results = await mapWithConcurrency(planned, concurrency, async (plan) => {
+			const rows = [];
+			for (let repeatIndex = 0; repeatIndex < repeats; repeatIndex++) {
+				if (aborted) break;
+				const outcome = await withRetry(() => respond({
+					system,
+					tools,
+					messages: plan.messages
+				}), {
+					attempts: options.retry?.attempts,
+					baseDelayMs: options.retry?.baseDelayMs,
+					sleep: options.retry?.sleep,
+					shouldRetry: (value) => !value.ok && value.retryable && value.auth !== true
+				});
+				if (outcome.ok) {
+					if (plan.requiredTools && outcome.toolCalls === void 0) {
+						progress();
+						rows.push({
+							repeatIndex,
+							error: { message: NO_TOOL_CALLS_REPORTED },
+							...unjudged
+						});
+						break;
+					}
+					const missingTools = plan.requiredTools ? missingToolsOf(plan.requiredTools, outcome.toolCalls ?? []) : void 0;
+					const judged = options.judge ? await judgeStep({
+						system: TUTOR_JUDGE_SYSTEM,
+						subject: buildTutorJudgeSubject(system, plan.conversation, outcome.text, {
+							...plan.gradingInstructions ? { gradingInstructions: plan.gradingInstructions } : {},
+							tools,
+							...outcome.toolCalls ? { toolCalls: outcome.toolCalls } : {}
+						}),
+						criteria: tutorJudgeCriteria(plan.gradingInstructions !== void 0)
+					}) : {};
+					progress();
+					rows.push({
+						repeatIndex,
+						text: outcome.text,
+						...outcome.toolCalls ? { toolCalls: outcome.toolCalls } : {},
+						...missingTools ? { missingTools } : {},
+						...outcome.usage ? { usage: outcome.usage } : {},
+						...judged
+					});
+					continue;
+				}
+				progress();
+				rows.push({
+					repeatIndex,
+					error: outcome.error,
+					...unjudged
+				});
+				if (outcome.auth) {
+					aborted ??= {
+						reason: "auth",
+						message: "Authentication failed — the run was aborted. Run `novedu-cli login`."
+					};
+					break;
+				}
+			}
+			const generated = rows.some((row) => row.text !== void 0);
+			const status = rows.length === 0 ? "skipped" : generated ? "ok" : "errored";
+			if (status === "errored") {
+				consecutiveErrored += 1;
+				if (consecutiveErrored >= CIRCUIT_BREAKER_LIMIT) aborted ??= {
+					reason: "circuit-breaker",
+					message: `${CIRCUIT_BREAKER_LIMIT} cases failed in a row — the run was aborted.`
+				};
+			} else if (status !== "skipped") consecutiveErrored = 0;
+			return {
+				index: plan.index,
+				...plan.title ? { title: plan.title } : {},
+				conversation: plan.conversation,
+				...plan.gradingInstructions ? { gradingInstructions: plan.gradingInstructions } : {},
+				...plan.requiredTools ? { requiredTools: plan.requiredTools } : {},
+				status,
+				unstable: false,
+				feedbackFlagged: rows.some((row) => (row.judge?.issues.length ?? 0) > 0),
+				toolsFlagged: rows.some((row) => (row.missingTools?.length ?? 0) > 0),
+				repeats: rows
+			};
+		});
+		const usage = { ...ZERO_USAGE };
+		for (const result of results) for (const row of result.repeats) {
+			addUsage(usage, row.usage);
+			addUsage(usage, row.judge?.usage);
+		}
+		return {
+			id: checked.evalFile.id,
+			kind: "tutor",
+			target: checked.targetUrl,
+			llm: options.llm,
+			judging: !options.judge ? "off" : breaker.stopped ? "degraded" : "on",
+			totals: {
+				cases: results.length,
+				passed: 0,
+				failed: 0,
+				errored: results.filter((c) => c.status === "errored").length,
+				skipped: results.filter((c) => c.status === "skipped").length,
+				unstable: 0,
+				feedbackFlagged: results.filter((c) => c.feedbackFlagged).length,
+				toolsFlagged: results.filter((c) => c.toolsFlagged).length,
+				judgeErrored: results.reduce((sum, c) => sum + c.repeats.filter((row) => row.judgeError !== void 0).length, 0),
+				repeats,
+				calls: total,
+				usage
+			},
+			questions: [],
+			confusion: [],
+			falseCorrect: {
+				count: 0,
+				denominator: 0,
+				rate: 0
+			},
+			mismatches: results.filter((result) => result.status === "errored"),
+			cases: results,
+			...aborted ? { aborted } : {}
+		};
+	} }
+};
 /** Run ONE checked eval file — the single entry point the command uses. */
 function runEval(kind, checked, options) {
 	return evalRunners[kind].run(checked, options);
@@ -3552,6 +4088,9 @@ function summarizeBatch(files) {
 		errored: 0,
 		skipped: 0,
 		unstable: 0,
+		feedbackFlagged: 0,
+		toolsFlagged: 0,
+		judgeErrored: 0,
 		usage: { ...ZERO_USAGE }
 	};
 	for (const file of files) {
@@ -3562,11 +4101,15 @@ function summarizeBatch(files) {
 		totals.errored += file.result.totals.errored;
 		totals.skipped += file.result.totals.skipped;
 		totals.unstable += file.result.totals.unstable;
+		totals.feedbackFlagged += file.result.totals.feedbackFlagged;
+		totals.toolsFlagged += file.result.totals.toolsFlagged;
+		totals.judgeErrored += file.result.totals.judgeErrored;
 		addUsage(totals.usage, file.result.totals.usage);
 	}
 	return {
 		files: files.map((file) => ({
 			...file,
+			...file.result ? { kind: file.result.kind } : {},
 			passed: filePassed(file)
 		})),
 		passed: batchPassed({ totals }),
@@ -3574,9 +4117,34 @@ function summarizeBatch(files) {
 	};
 }
 /**
+* Did this file's run produce ANY judgment? The ONE rule every renderer derives its
+* flagged count's visibility from: a file that judged nothing — judging off, or every
+* case run after the breaker degraded the run — has NOT been found clean, so its flagged
+* count renders as "not checked" (an em dash, an omitted segment), never as a `0`.
+*/
+function anyJudged(result) {
+	return result.cases.some((evalCase) => evalCase.repeats.some((repeat) => repeat.judge !== void 0 && repeat.judge !== null));
+}
+/**
+* Did this file's run CHECK tool calls at all — i.e. does any case declare
+* `required_tools`? The tool sibling of {@link anyJudged}, and the same rule: a run that
+* required nothing has not been found complete, so its `toolsFlagged` count is OMITTED
+* rather than printed as a reassuring `0`.
+*/
+function anyToolsRequired(result) {
+	return result.cases.some((evalCase) => isTutorCase(evalCase) && evalCase.requiredTools !== void 0);
+}
+/**
 * The CI gate: every file valid, and not a single failed, errored, or skipped CASE —
 * an aborted (and therefore incomplete) run must never read as a pass. The single
 * source of truth for the exit code AND for `EvalBatchResult.passed`.
+*
+* `unstable`, `feedbackFlagged`, `toolsFlagged` and `judgeErrored` deliberately do NOT
+* appear here: all four are reported, none gates. (Gating is per-KIND policy, not a
+* property of judge
+* results — and BOTH shipped kinds are report-only. For the tutor kind that is the whole
+* policy: its exit code reflects RUN HEALTH only, so a flagged conversation changes
+* nothing, and the Markdown report is the deliverable — docs/cli-eval.md.)
 */
 function batchPassed(batch) {
 	return batch.totals.invalid === 0 && batch.totals.failed === 0 && batch.totals.errored === 0 && batch.totals.skipped === 0;
@@ -3602,6 +4170,9 @@ const cliFetcher = async (url) => {
 };
 //#endregion
 //#region src/format.ts
+function flaggedLabel(kind) {
+	return kind === "tutor" ? "flagged responses" : "flagged feedback";
+}
 const useColor = process.stdout.isTTY && !process.env.NO_COLOR;
 const paint = (code, s) => useColor ? `\x1b[${code}m${s}\x1b[0m` : s;
 const green = (s) => paint("32", s);
@@ -3750,17 +4321,19 @@ function formatCodingResult(result, source) {
 	return lines.join("\n");
 }
 /**
-* Renderer for a golden-answer eval check (`--kind eval`). An eval describes a quiz it
-* does not contain, so the summary names the resolved target and the size of the run
-* the file would produce.
+* Renderer for an eval check (`--kind eval`). An eval describes an activity it does not
+* contain, so the summary names the resolved target and the size of the run the file
+* would produce — in the units of its own kind.
 */
 function formatEvalResult(result, source) {
 	if (!result.ok) return renderFailureAndWarnings(result, "eval", source);
 	const lines = [green(`✔ Valid eval`) + dim(` — ${source}`)];
 	lines.push(`  id: ${result.evalFile.id}`);
+	lines.push(`  kind: ${result.kind}`);
 	lines.push(`  target: ${result.targetUrl}`);
-	lines.push(`  questions: ${result.evalFile.questions.length}   cases: ${result.caseCount}`);
-	lines.push(`  quiz model: ${result.quizDump.llm.provider} / ${result.quizDump.llm.model}`);
+	if (result.kind === "tutor") lines.push(`  conversations: ${result.caseCount}`);
+	else lines.push(`  questions: ${result.evalFile.questions.length}   cases: ${result.caseCount}`);
+	lines.push(`  ${result.kind} model: ${result.llm.provider} / ${result.llm.model}`);
 	if (result.warnings.length) {
 		lines.push("");
 		lines.push(yellow(`${result.warnings.length} warning(s):`));
@@ -3789,14 +4362,25 @@ function formatUsageLine(usage) {
 	const cached = usage.cachedInput ? ` (${formatTokenCount(usage.cachedInput)} cached)` : "";
 	return `tokens: ${formatTokenCount(usage.input)} in${cached} / ${formatTokenCount(usage.output)} out`;
 }
-/** `question#index expected … got … "answer…"` — one line per non-passing case. */
+/** The first error message a case's repeats recorded, for a one-line mismatch row. */
+function firstErrorMessage(repeats, fallback) {
+	const first = repeats.find((row) => row.error !== void 0)?.error;
+	return typeof first === "object" && first !== null && "message" in first ? String(first.message) : fallback;
+}
+/**
+* One line per non-passing case: `question#index expected … got … "answer…"` for a quiz,
+* `#n title — error …` for a tutor conversation (which has no verdict to compare).
+*/
 function mismatchLines(result) {
 	return result.mismatches.map((c) => {
+		if (isTutorCase(c)) {
+			const head = `#${c.index + 1}${c.title ? ` ${c.title}` : ""}`;
+			return `  ${red("✗")} ${head} ${red("error")} ${dim(firstErrorMessage(c.repeats, "no response"))}`;
+		}
 		const head = `${c.questionId}#${c.answerIndex}`;
 		const expected = c.expected.join("|");
 		if (c.status === "errored") {
-			const first = c.repeats.find((row) => row.error !== void 0)?.error;
-			const message = typeof first === "object" && first !== null && "message" in first ? String(first.message) : "no verdict";
+			const message = firstErrorMessage(c.repeats, "no verdict");
 			return `  ${red("✗")} ${head} expected ${expected} got ${red("error")} ${dim(message)}`;
 		}
 		return `  ${red("✗")} ${head} expected ${expected} got ${red(c.verdict ?? "?")}` + dim(` "${snippet(c.answer)}"`);
@@ -3817,18 +4401,26 @@ function formatEvalReport(result, source) {
 	];
 	const llm = result.llm.overrides ? `${result.llm.overrides.provider} / ${result.llm.overrides.model} ${yellow("→")} ${result.llm.provider} / ${result.llm.model} ${yellow("(override)")}` : `${result.llm.provider} / ${result.llm.model}`;
 	lines.push(`  llm: ${llm}`);
-	lines.push(`  cases: ${totals.cases} × ${totals.repeats} repeat(s) = ${totals.calls} grading call(s)`);
+	const judge = result.llm.judge;
+	if (judge && (judge.provider !== result.llm.provider || judge.model !== result.llm.model)) lines.push(`  judge llm: ${judge.provider} / ${judge.model}${judge.overridden ? ` ${yellow("(override)")}` : ""}`);
+	const unit = result.kind === "tutor" ? "conversation" : "case";
+	const generation = result.kind === "tutor" ? "generation" : "grading";
+	lines.push(`  ${unit}s: ${totals.cases} × ${totals.repeats} repeat(s) = ${totals.calls} ${generation} call(s)` + (result.judging === "off" ? "" : ` + ${totals.calls} judge call(s)`));
 	if (result.aborted) {
 		lines.push("");
 		lines.push(red(`Run aborted: ${result.aborted.message}`));
 	}
+	if (result.judging === "degraded") {
+		lines.push("");
+		lines.push(yellow("Feedback judging stopped after repeated judge failures — grading was unaffected."));
+	}
 	if (result.mismatches.length) {
 		lines.push("");
 		lines.push(red(`${result.mismatches.length} mismatch(es):`));
 		lines.push(...mismatchLines(result));
 	}
 	lines.push("");
-	lines.push(`  passed: ${totals.passed}   failed: ${totals.failed}   errored: ${totals.errored}` + (totals.skipped ? red(`   skipped: ${totals.skipped} (run aborted)`) : "") + (totals.unstable ? dim(`   unstable: ${totals.unstable}`) : ""));
+	lines.push((result.kind === "tutor" ? `  ok: ${totals.cases - totals.errored - totals.skipped}   errored: ${totals.errored}` : `  passed: ${totals.passed}   failed: ${totals.failed}   errored: ${totals.errored}`) + (totals.skipped ? red(`   skipped: ${totals.skipped} (run aborted)`) : "") + (totals.unstable ? dim(`   unstable: ${totals.unstable}`) : "") + (!anyJudged(result) ? "" : totals.feedbackFlagged ? yellow(`   ${flaggedLabel(result.kind)}: ${totals.feedbackFlagged}`) : dim(`   ${flaggedLabel(result.kind)}: 0`)) + (!anyToolsRequired(result) ? "" : totals.toolsFlagged ? yellow(`   missing tool calls: ${totals.toolsFlagged}`) : dim("   missing tool calls: 0")) + (totals.judgeErrored ? dim(`   judge errors: ${totals.judgeErrored}`) : ""));
 	const tokens = formatUsageLine(totals.usage);
 	if (tokens) lines.push(dim(`  ${tokens}`));
 	if (result.confusion.length) {
@@ -3836,9 +4428,11 @@ function formatEvalReport(result, source) {
 		lines.push("  confusion (expected → got):");
 		for (const row of result.confusion) lines.push(`    ${row.expected} → ${row.got}: ${row.count}`);
 	}
-	const { count, denominator, rate } = result.falseCorrect;
-	lines.push("");
-	lines.push(`  false-correct: ${count}/${denominator}` + (denominator ? ` (${(rate * 100).toFixed(1)}%)` : ""));
+	if (result.kind !== "tutor") {
+		const { count, denominator, rate } = result.falseCorrect;
+		lines.push("");
+		lines.push(`  false-correct: ${count}/${denominator}` + (denominator ? ` (${(rate * 100).toFixed(1)}%)` : ""));
+	}
 	return lines.join("\n");
 }
 /**
@@ -3858,11 +4452,15 @@ function formatEvalBatchReport(batch) {
 		}
 		const t = file.result.totals;
 		const mark = t.failed === 0 && t.errored === 0 && t.skipped === 0 ? green("✔") : red("✗");
-		lines.push(`  ${mark} ${name}: ${t.cases} case(s), ${t.passed} passed, ${t.failed} failed, ${t.errored} errored` + (t.skipped ? red(`, ${t.skipped} skipped`) : "") + (t.unstable ? dim(`, ${t.unstable} unstable`) : ""));
+		const counts = file.result.kind === "tutor" ? `${t.cases} conversation(s), ${t.cases - t.errored - t.skipped} ok, ${t.errored} errored` : `${t.cases} case(s), ${t.passed} passed, ${t.failed} failed, ${t.errored} errored`;
+		lines.push(`  ${mark} ${name}: ${counts}` + (t.skipped ? red(`, ${t.skipped} skipped`) : "") + (t.unstable ? dim(`, ${t.unstable} unstable`) : "") + (!anyJudged(file.result) ? "" : t.feedbackFlagged ? yellow(`, ${t.feedbackFlagged} flagged`) : dim(", 0 flagged")) + (!anyToolsRequired(file.result) ? "" : t.toolsFlagged ? yellow(`, ${t.toolsFlagged} missing tool calls`) : dim(", 0 missing tool calls")));
 	}
 	const g = batch.totals;
+	const judged = batch.files.some((file) => file.result && anyJudged(file.result));
+	const toolChecked = batch.files.some((file) => file.result && anyToolsRequired(file.result));
 	lines.push("");
-	lines.push(`  TOTAL: ${g.cases} case(s), ${g.passed} passed, ${g.failed} failed, ${g.errored} errored` + (g.skipped ? red(`, ${g.skipped} skipped`) : "") + (g.invalid ? red(`, ${g.invalid} invalid file(s)`) : ""));
+	lines.push(`  TOTAL: ${g.cases} case(s), ${g.passed} passed, ${g.failed} failed, ${g.errored} errored` + (g.skipped ? red(`, ${g.skipped} skipped`) : "") + (judged ? g.feedbackFlagged ? yellow(`, ${g.feedbackFlagged} flagged`) : dim(", 0 flagged") : "") + (toolChecked ? g.toolsFlagged ? yellow(`, ${g.toolsFlagged} missing tool calls`) : dim(", 0 missing tool calls") : "") + (g.invalid ? red(`, ${g.invalid} invalid file(s)`) : ""));
+	if (batch.files.some((file) => file.result?.judging === "degraded")) lines.push(yellow("  Feedback judging stopped mid-run after repeated judge failures — files without a flagged count were graded but not judged."));
 	const tokens = formatUsageLine(g.usage);
 	if (tokens) lines.push(dim(`  ${tokens}`));
 	for (const file of batch.files) {
@@ -3937,6 +4535,17 @@ function llmText(llm) {
 	const effective = `${llm.provider} / ${llm.model}`;
 	return llm.overrides ? `${llm.overrides.provider} / ${llm.overrides.model} → ${effective} (override)` : effective;
 }
+/**
+* The judge's pair, but ONLY when it differs from the grading pair — a judge line that
+* merely repeats the grader would be noise, while a differing one is essential: judge
+* strictness varies by model, so two reports are comparable only when it matches.
+*/
+function judgeLlmText(llm) {
+	const judge = llm.judge;
+	if (!judge) return void 0;
+	if (judge.provider === llm.provider && judge.model === llm.model) return void 0;
+	return `${judge.provider} / ${judge.model}${judge.overridden ? " (override)" : ""}`;
+}
 /** `15,420 / 12,300 / 2,810`, or an em dash when nothing was reported. */
 function usageCell(usage) {
 	if (usage.input === 0 && usage.cachedInput === 0 && usage.output === 0) return "—";
@@ -3965,6 +4574,7 @@ const OVERVIEW_HEADER = [
 	"Errored",
 	"Skipped",
 	"Unstable",
+	"Flagged",
 	"False-correct",
 	"Tokens (in / cached / out)"
 ];
@@ -3984,6 +4594,7 @@ function overview(batch) {
 		"---:",
 		"---:",
 		"---:",
+		"---:",
 		"---:"
 	])];
 	for (const file of batch.files) {
@@ -3999,26 +4610,30 @@ function overview(batch) {
 				"—",
 				"—",
 				"—",
+				"—",
 				"—"
 			]));
 			continue;
 		}
 		const t = file.result.totals;
+		const tutor = file.result.kind === "tutor";
 		lines.push(row([
 			`${file.passed ? "✅" : "❌"} ${name}`,
 			`\`${cell(file.result.id)}\``,
 			count(t.cases),
-			count(t.passed),
-			count(t.failed),
+			tutor ? "—" : count(t.passed),
+			tutor ? "—" : count(t.failed),
 			count(t.errored),
 			count(t.skipped),
-			count(t.unstable),
-			falseCorrectCell(file.result),
+			tutor ? "—" : count(t.unstable),
+			anyJudged(file.result) ? count(t.feedbackFlagged) : "—",
+			tutor ? "—" : falseCorrectCell(file.result),
 			usageCell(t.usage)
 		]));
 	}
 	if (batch.files.length > 1) {
 		const g = batch.totals;
+		const judged = batch.files.some((file) => file.result && anyJudged(file.result));
 		lines.push(row([
 			"**TOTAL**",
 			g.invalid ? `${count(g.invalid)} invalid` : "",
@@ -4028,6 +4643,7 @@ function overview(batch) {
 			`**${count(g.errored)}**`,
 			`**${count(g.skipped)}**`,
 			`**${count(g.unstable)}**`,
+			judged ? `**${count(g.feedbackFlagged)}**` : "—",
 			"",
 			`**${usageCell(g.usage)}**`
 		]));
@@ -4048,6 +4664,13 @@ function verdictSummary(evalCase) {
 function needsDetail(evalCase) {
 	return evalCase.status === "failed" || evalCase.status === "errored" || evalCase.unstable;
 }
+/** The "**Question** / **Golden answer**" intro every case detail section opens with. */
+function questionAndAnswer(evalCase, questionText) {
+	const lines = [];
+	if (questionText) lines.push("**Question**", "", quote(questionText), "");
+	lines.push("**Golden answer**", "", quote(evalCase.answer), "");
+	return lines;
+}
 /**
 * One case's section: the question it belongs to, the golden answer, and what the
 * grader said — plus every repeat when they disagreed (the `--repeats` signal is
@@ -4058,16 +4681,7 @@ function caseSection(evalCase, questionText) {
 	const unstable = evalCase.unstable ? " *(unstable)*" : "";
 	lines.push(`### \`${evalCase.questionId}\` #${evalCase.answerIndex} — ${verdictSummary(evalCase)}${unstable}`);
 	lines.push("");
-	if (questionText) {
-		lines.push("**Question**");
-		lines.push("");
-		lines.push(quote(questionText));
-		lines.push("");
-	}
-	lines.push("**Golden answer**");
-	lines.push("");
-	lines.push(quote(evalCase.answer));
-	lines.push("");
+	lines.push(...questionAndAnswer(evalCase, questionText));
 	const graded = evalCase.repeats.filter((r) => r.got !== void 0);
 	const disagreed = new Set(graded.map((r) => r.got)).size > 1;
 	if (evalCase.repeats.length > 1 && (disagreed || graded.length !== evalCase.repeats.length)) {
@@ -4097,6 +4711,136 @@ function caseSection(evalCase, questionText) {
 	}
 	return lines;
 }
+/**
+* The "Flagged feedback" section: what the LLM judge found wrong with the TEXT the
+* grader wrote, per case, with each flagged repeat's verdict and feedback quoted verbatim
+* and the judge's issues as `criterion — note` items.
+*
+* Separate from the verdict sections on purpose — these cases usually PASSED (the verdict
+* was right, the wording was not), and mixing them into the mismatch list would suggest
+* the run failed on them. Empty when the file has no flags.
+*/
+function flaggedSection(result, questionText) {
+	const flagged = result.cases.filter((evalCase) => evalCase.feedbackFlagged && !isTutorCase(evalCase));
+	if (flagged.length === 0) return [];
+	const lines = ["### Flagged feedback", ""];
+	lines.push("_An LLM judge audited each feedback text against the very grading prompt it was written under. Reported only — flagged feedback never fails a run._");
+	lines.push("");
+	for (const evalCase of flagged) {
+		lines.push(`#### \`${evalCase.questionId}\` #${evalCase.answerIndex}`);
+		lines.push("");
+		lines.push(...questionAndAnswer(evalCase, questionText.get(evalCase.questionId)));
+		for (const repeat of evalCase.repeats) {
+			const issues = repeat.judge?.issues ?? [];
+			if (issues.length === 0) continue;
+			lines.push(`**Repeat #${repeat.repeatIndex + 1} — \`${repeat.got ?? "?"}\`**`);
+			lines.push("");
+			lines.push(quote(repeat.feedback ?? ""));
+			lines.push("");
+			for (const issue of issues) lines.push(`- \`${cell(issue.criterion)}\` — ${inline(issue.note)}`);
+			lines.push("");
+		}
+	}
+	return lines;
+}
+/**
+* A tutor case's stable heading: the teacher's `title` when it has one, otherwise its
+* 1-based index plus an excerpt of the FIRST student line — enough to recognise the case
+* in a report without opening the eval file.
+*/
+function tutorCaseLabel(evalCase) {
+	if (evalCase.title) return `#${evalCase.index + 1} ${cell(evalCase.title)}`;
+	const firstStudent = evalCase.conversation.find((turn) => "student" in turn);
+	const excerpt = firstStudent && "student" in firstStudent ? inline(firstStudent.student) : "";
+	const short = excerpt.length > 60 ? `${excerpt.slice(0, 59)}…` : excerpt;
+	return short ? `#${evalCase.index + 1} — ${short}` : `#${evalCase.index + 1}`;
+}
+/** The scripted conversation as one labeled, verbatim blockquote per turn. */
+function conversationBlock(evalCase) {
+	const lines = ["**Conversation**", ""];
+	for (const turn of evalCase.conversation) {
+		const role = "student" in turn ? "student" : "tutor";
+		const text = "student" in turn ? turn.student : turn.tutor;
+		lines.push(`*${role}*`, "", quote(text), "");
+	}
+	return lines;
+}
+/** One ERRORED tutor case: what was asked, and why nothing came back. */
+function tutorErrorSection(evalCase) {
+	const lines = [`### ${tutorCaseLabel(evalCase)} — error`, ""];
+	lines.push(...conversationBlock(evalCase));
+	const failure = evalCase.repeats.find((r) => r.error !== void 0);
+	if (failure) lines.push("**Error**", "", quote(errorMessage(failure.error)), "");
+	return lines;
+}
+/** `random_number, random_number` — a repeat's tool calls, or `(none)` when it made none. */
+function toolCallList(toolCalls) {
+	return toolCalls && toolCalls.length > 0 ? toolCalls.map((name) => `\`${name}\``).join(", ") : "(none)";
+}
+/**
+* The tutor kind's "Missing tool calls" section: every case that declares `required_tools`
+* and had at least one repeat skip one. Per case the required list, what each offending
+* repeat actually called, and which tools were missing there.
+*
+* Its own section rather than a column, for the same reason "Flagged responses" is one:
+* these cases are `ok` and the run PASSED — a missing tool call is a note about the
+* tutor's behavior, never a failure. Cases whose tools all ran stay out entirely; the
+* `--json` report carries every repeat's `toolCalls` for anyone who wants them.
+*/
+function tutorMissingToolsSection(result) {
+	const flagged = result.cases.filter((evalCase) => isTutorCase(evalCase) && evalCase.toolsFlagged);
+	if (flagged.length === 0) return [];
+	const lines = ["### Missing tool calls", ""];
+	lines.push("_These cases require a tool the tutor did not call in every run. Reported only — a missing tool call never fails a run, and tools beyond the required ones are fine._");
+	lines.push("");
+	for (const evalCase of flagged) {
+		lines.push(`#### ${tutorCaseLabel(evalCase)}`);
+		lines.push("");
+		lines.push(`**Required** ${toolCallList(evalCase.requiredTools)}`);
+		lines.push("");
+		for (const repeat of evalCase.repeats) {
+			const missing = repeat.missingTools ?? [];
+			if (missing.length === 0) continue;
+			lines.push(`- Repeat #${repeat.repeatIndex + 1} — missing ${toolCallList(missing)}; called ${toolCallList(repeat.toolCalls)}`);
+		}
+		lines.push("");
+	}
+	return lines;
+}
+/**
+* The tutor kind's "Flagged responses" section — the report's actual deliverable: per
+* flagged case the scripted conversation, the teacher's expectations when it states any,
+* and each flagged repeat's GENERATED RESPONSE verbatim followed by the judge's issues.
+*
+* Clean cases stay out entirely (their generated texts are in the `--json` report), and
+* a flag never means the run failed — the tutor kind is report-only.
+*/
+function tutorFlaggedSection(result) {
+	const flagged = result.cases.filter((evalCase) => isTutorCase(evalCase) && evalCase.feedbackFlagged);
+	if (flagged.length === 0) return [];
+	const lines = ["### Flagged responses", ""];
+	lines.push("_An LLM judge audited each generated response against the tutor's own system prompt and, where the case states any, the teacher's expectations. Reported only — a flagged response never fails a run._");
+	lines.push("");
+	for (const evalCase of flagged) {
+		lines.push(`#### ${tutorCaseLabel(evalCase)}`);
+		lines.push("");
+		lines.push(...conversationBlock(evalCase));
+		if (evalCase.gradingInstructions) lines.push("**Expectations for this case**", "", quote(evalCase.gradingInstructions), "");
+		if (evalCase.requiredTools) lines.push(`**Required tools** ${toolCallList(evalCase.requiredTools)}`, "");
+		for (const repeat of evalCase.repeats) {
+			const issues = repeat.judge?.issues ?? [];
+			if (issues.length === 0) continue;
+			lines.push(`**Generated response — repeat #${repeat.repeatIndex + 1}**`);
+			lines.push("");
+			lines.push(quote(repeat.text ?? ""));
+			lines.push("");
+			if (repeat.toolCalls && (repeat.toolCalls.length > 0 || evalCase.requiredTools)) lines.push(`*tool calls: ${toolCallList(repeat.toolCalls)}*`, "");
+			for (const issue of issues) lines.push(`- \`${cell(issue.criterion)}\` — ${inline(issue.note)}`);
+			lines.push("");
+		}
+	}
+	return lines;
+}
 /** One file's details section, or `[]` when the file has nothing to report. */
 function fileDetails(file) {
 	const name = shortSource(file.source);
@@ -4105,29 +4849,34 @@ function fileDetails(file) {
 		return [
 			`## ${cell(name)} — invalid`,
 			"",
-			"This file was not graded; fix the problems below and run it again.",
+			"This file was not run; fix the problems below and run it again.",
 			"",
 			...errors.map((issue) => `- \`${cell(issue.code)}\` — ${inline(issue.message)}`),
 			""
 		];
 	}
 	const result = file.result;
+	const tutor = result.kind === "tutor";
 	const detailed = result.cases.filter(needsDetail);
 	const skipped = result.totals.skipped;
-	if (detailed.length === 0 && skipped === 0 && !result.aborted) return [];
 	const questionText = new Map(result.questions.map((question) => [question.id, question.text]));
+	const flagged = tutor ? tutorFlaggedSection(result) : flaggedSection(result, questionText);
+	const missingTools = tutor ? tutorMissingToolsSection(result) : [];
+	if (detailed.length === 0 && skipped === 0 && !result.aborted && flagged.length === 0 && missingTools.length === 0) return [];
 	const lines = [`## ${cell(name)} — \`${cell(result.id)}\``, ""];
 	if (result.aborted) {
 		lines.push("> [!WARNING]");
 		lines.push(`> The run was aborted: ${inline(result.aborted.message)}`);
 		lines.push("");
 	}
-	for (const evalCase of detailed) lines.push(...caseSection(evalCase, questionText.get(evalCase.questionId)));
+	for (const evalCase of detailed) lines.push(...isTutorCase(evalCase) ? tutorErrorSection(evalCase) : caseSection(evalCase, questionText.get(evalCase.questionId)));
 	if (skipped > 0) {
 		const reason = result.aborted ? ` (${inline(result.aborted.message)})` : "";
-		lines.push(`**${count(skipped)} case(s) were never attempted**${reason} — the run is incomplete, so it cannot pass.`);
+		lines.push(`**${count(skipped)} ${tutor ? "conversation" : "case"}(s) were never attempted**${reason} — the run is incomplete, so it cannot pass.`);
 		lines.push("");
 	}
+	lines.push(...missingTools);
+	lines.push(...flagged);
 	return lines;
 }
 /**
@@ -4141,25 +4890,34 @@ function renderEvalMarkdownReport(batch, meta) {
 	lines.push(`- **Generated** ${timestamp(meta.generatedAt)} · novedu-cli ${meta.cliVersion}`);
 	const llms = [...new Set(batch.files.filter((f) => f.result).map((f) => llmText(f.result.llm)))];
 	for (const llm of llms) lines.push(`- **LLM** ${llm}`);
+	const judges = [...new Set(batch.files.map((file) => file.result ? judgeLlmText(file.result.llm) : void 0).filter((text) => text !== void 0))];
+	for (const judge of judges) lines.push(`- **Feedback judge** ${judge}`);
 	lines.push(`- **Run** ${count(batch.totals.files)} file(s), ${count(batch.totals.cases)} case(s) × ${count(meta.repeats)} repeat(s), concurrency ${count(meta.concurrency)}`);
+	if (batch.files.some((file) => file.result && anyToolsRequired(file.result))) lines.push(`- **Missing tool calls** ${count(batch.totals.toolsFlagged)} case(s) did not call a required tool in every run — reported only, never a failure`);
 	const tokens = batch.totals.usage;
-	if (tokens.input || tokens.cachedInput || tokens.output) lines.push(`- **Tokens** ${count(tokens.input)} in (${count(tokens.cachedInput)} cached) / ${count(tokens.output)} out — successful grading calls only, so a lower bound`);
+	if (tokens.input || tokens.cachedInput || tokens.output) lines.push(`- **Tokens** ${count(tokens.input)} in (${count(tokens.cachedInput)} cached) / ${count(tokens.output)} out — successful calls only, so a lower bound`);
 	lines.push("");
 	if (batch.files.filter((file) => file.result?.aborted).length > 0) {
 		lines.push("> [!WARNING]");
 		lines.push(`> The run was ABORTED — ${count(batch.totals.skipped)} case(s) were never graded, so this report is incomplete.`);
 		lines.push("");
 	}
+	const degradedAt = batch.files.find((file) => file.result?.judging === "degraded");
+	if (degradedAt?.result) {
+		lines.push("> [!WARNING]");
+		lines.push(`> Feedback judging STOPPED during \`${cell(shortSource(degradedAt.source))}\` after 3 judge calls failed in a row — everything from there on was graded but NOT judged, and shows an em dash rather than a count in the Flagged column. Grading was unaffected.`);
+		lines.push("");
+	}
 	lines.push("## Overview");
 	lines.push("");
 	lines.push(...overview(batch));
 	lines.push("");
 	const details = batch.files.flatMap((file) => fileDetails(file));
 	if (details.length === 0) {
-		lines.push("_Nothing else to report — every case matched its expected verdict. The `--json` report carries every case, including the passing ones._");
+		lines.push("_Nothing else to report. The `--json` report carries every case, including the clean ones._");
 		lines.push("");
 	} else {
-		lines.push("_Below: only the mismatched, errored and unstable cases. Passing cases live in the `--json` report._");
+		lines.push("_Below: only the mismatched, errored and unstable cases, plus anything the judge flagged. Clean cases live in the `--json` report._");
 		lines.push("");
 		lines.push(...details);
 	}
@@ -4494,23 +5252,27 @@ function expandSources(args) {
 		duplicates
 	};
 }
-/** The `--llm-provider`/`--llm-model` pair: strictly both-or-nothing, provider checked. */
-function parseOverride(options) {
-	const { llmProvider, llmModel } = options;
-	if (llmProvider === void 0 && llmModel === void 0) return { ok: true };
-	if (llmProvider === void 0 || llmModel === void 0) return {
+/**
+* One provider/model pair from the flags: strictly BOTH-OR-NOTHING (the `effectiveLlm`
+* rule, docs/ai-models.md) with the provider checked against the known list. Shared by
+* `--llm-*` (the grading override) and `--judge-llm-*` (the judge's own pair), so the two
+* can never drift in wording or in strictness.
+*/
+function parsePair(flag, provider, model) {
+	if (provider === void 0 && model === void 0) return { ok: true };
+	if (provider === void 0 || model === void 0) return {
 		ok: false,
-		message: "Pass --llm-provider and --llm-model together, or neither."
+		message: `Pass --${flag}-provider and --${flag}-model together, or neither.`
 	};
-	if (!LLM_PROVIDERS.includes(llmProvider)) return {
+	if (!LLM_PROVIDERS.includes(provider)) return {
 		ok: false,
-		message: `Unknown --llm-provider "${llmProvider}": expected ${LLM_PROVIDERS.map((p) => `"${p}"`).join(" or ")}.`
+		message: `Unknown --${flag}-provider "${provider}": expected ${LLM_PROVIDERS.map((p) => `"${p}"`).join(" or ")}.`
 	};
 	return {
 		ok: true,
 		llm: {
-			provider: llmProvider,
-			model: llmModel
+			provider,
+			model
 		}
 	};
 }
@@ -4580,6 +5342,156 @@ function makeGradeFn(server, llm) {
 		};
 	};
 }
+/**
+* The `toolCalls: string[]` of a tutor 200, defensively: `undefined` when the field is
+* absent (a server too old to report tool calls — a distinction the runner MUST be able to
+* make), and non-string entries are dropped rather than breaking a run. Names only, in the
+* order the server sent them, duplicates kept.
+*/
+function parseToolCalls(value) {
+	if (!Array.isArray(value)) return void 0;
+	return value.filter((name) => typeof name === "string" && name !== "");
+}
+/**
+* The HTTP seam for ONE generated tutor turn, with the run's effective llm closed in —
+* the tutor kind's sibling of {@link makeGradeFn}, sharing its failure classification
+* exactly (5xx and network retryable, auth aborts the run, every other 4xx terminal).
+*/
+function makeRespondFn(server, llm) {
+	return async ({ system, tools, messages }) => {
+		const response = await performApiRequest({
+			server,
+			path: "/api/eval/respond",
+			method: "POST",
+			body: {
+				llm,
+				system,
+				tools: [...tools],
+				messages: messages.map((m) => ({ ...m }))
+			},
+			quiet: true
+		});
+		if (response.ok) {
+			const payload = response.payload;
+			if (typeof payload?.text === "string" && payload.text !== "") {
+				const usage = parseUsage(payload?.usage);
+				const toolCalls = parseToolCalls(payload?.toolCalls);
+				return {
+					ok: true,
+					text: payload.text,
+					...toolCalls ? { toolCalls } : {},
+					...usage ? { usage } : {}
+				};
+			}
+			return {
+				ok: false,
+				retryable: false,
+				error: { message: "The server's response is not a generated tutor turn — it may not offer /api/eval/respond at all (does it run a Novedu version with tutor evals?). Check the target server, e.g. --server http://localhost:3000." }
+			};
+		}
+		return {
+			ok: false,
+			retryable: response.status === void 0 || response.status >= 500,
+			...response.authFailed ? { auth: true } : {},
+			error: response.error
+		};
+	};
+}
+/**
+* The HTTP seam for ONE judge call, with the run's judge llm closed in. Mirrors
+* {@link makeGradeFn}'s failure classification, minus the auth branch: a judge failure
+* NEVER aborts the run — it degrades judging (see the runner's breaker) while the grading
+* half finishes untouched.
+*/
+function makeJudgeFn(server, llm) {
+	return async ({ system, subject, criteria }) => {
+		const response = await performApiRequest({
+			server,
+			path: "/api/eval/judge",
+			method: "POST",
+			body: {
+				llm,
+				system,
+				subject,
+				criteria: [...criteria]
+			},
+			quiet: true
+		});
+		if (response.ok) {
+			const payload = response.payload;
+			if (Array.isArray(payload?.issues)) {
+				const issues = payload.issues.flatMap((entry) => {
+					const { criterion, note } = entry ?? {};
+					return typeof criterion === "string" ? [{
+						criterion,
+						note: typeof note === "string" ? note : ""
+					}] : [];
+				});
+				const usage = parseUsage(payload?.usage);
+				return {
+					ok: true,
+					issues,
+					...usage ? { usage } : {}
+				};
+			}
+			return {
+				ok: false,
+				retryable: false,
+				error: { message: "The server's response is not a feedback judgment — it may not offer /api/eval/judge at all (does it run a Novedu version with the feedback judge?). Re-run with --no-judge-feedback to grade without judging." }
+			};
+		}
+		return {
+			ok: false,
+			retryable: response.status === void 0 || response.status >= 500,
+			error: response.error
+		};
+	};
+}
+/** Budget for the one version probe — a hung check must never hold up a run. */
+const VERSION_CHECK_TIMEOUT_MS = 5e3;
+/**
+* Warn when this CLI was not built from the same commit as the server it is about to
+* grade against. `eval` assembles every grading system prompt LOCALLY, from the `lib/**`
+* prompt builders frozen into this published CLI — so a stale binary can certify prompts
+* the server's activities no longer send. CLI and server live in one repo, which makes
+* the server's `cliVersion` (from `GET /api/version`, public and unauthenticated) exactly
+* the CLI release matching its bundled code.
+*
+* Deliberately EVAL-ONLY (prompt drift corrupts nothing else) and strictly advisory: one
+* fetch, no retry, never an abort, never an exit code, and never a byte on stdout — the
+* JSON output contract owns that stream. Unlike progress it prints off a TTY too: a CI
+* log is precisely where this warning has to survive. Absence is NOT silently forgiven —
+* an unreachable, non-JSON, non-2xx or `cliVersion`-less answer says so, because "could
+* not check" and "checked, fine" must not look the same.
+*/
+async function warnOnVersionMismatch(server) {
+	const local = cliVersion();
+	const unverifiable = (reason) => {
+		process.stderr.write(`Warning: could not verify that this CLI (${local}) matches the server's — ${reason}. Locally assembled grading prompts may differ from what that server's activities run.
+`);
+	};
+	const base = resolveServerUrl(server);
+	let payload;
+	try {
+		const response = await fetch(new URL("/api/version", base), { signal: AbortSignal.timeout(VERSION_CHECK_TIMEOUT_MS) });
+		if (!response.ok) {
+			unverifiable(`${base} answered HTTP ${response.status}`);
+			return;
+		}
+		payload = await response.json();
+	} catch (error) {
+		unverifiable(`${base} did not answer (${error instanceof Error ? error.message : error})`);
+		return;
+	}
+	const remote = payload?.cliVersion;
+	if (typeof remote !== "string" || remote === "") {
+		unverifiable(`${base} reports no CLI version (does it run a Novedu version that has one?)`);
+		return;
+	}
+	if (remote === local) return;
+	process.stderr.write(`Warning: this CLI is ${local} but the server was built with CLI ${remote} — locally assembled grading prompts may differ from what that server's activities run. Update: npm i -g @novedu/cli
+`);
+}
 /** stderr progress, suppressed off a TTY so CI logs stay readable. */
 function progressWriter(prefix) {
 	if (!process.stderr.isTTY) return void 0;
@@ -4588,16 +5500,42 @@ function progressWriter(prefix) {
 	};
 }
 /**
+* The off-a-TTY replacement for the spinner: ONE newline-terminated line per finished
+* file. The `\r` counter above is suppressed when stderr is redirected (it would fill a
+* log with carriage-return noise), which otherwise left a long batch printing nothing at
+* all between the scope banner and the final report — indistinguishable from a hang, and
+* an easy way to talk yourself into killing a healthy run. Coarse and greppable is
+* enough: it proves liveness and says which file the run reached.
+*
+* Deliberately no timings — a per-file duration invites extrapolating an ETA that the
+* model, the provider's load and `--concurrency` make unreliable.
+*/
+function writeFileDone(label, result) {
+	const totals = result.totals;
+	const counts = result.kind === "tutor" ? `${totals.cases} conversation(s), ${totals.cases - totals.errored - totals.skipped} ok, ${totals.errored} errored` : `${totals.cases} case(s), ${totals.passed} passed, ${totals.failed} failed, ${totals.errored} errored`;
+	process.stderr.write(`${label}: ${counts}` + (totals.skipped ? `, ${totals.skipped} skipped` : "") + (anyJudged(result) ? `, ${totals.feedbackFlagged} flagged` : "") + "\n");
+}
+/**
 * The command's core, exported for the unit tests. `seams` exists only so tests can
 * shrink the retry backoff — the CLI itself never passes it (PoC parity: 4 attempts,
 * 5 s linear).
 */
 async function runEvalCommand(pathsOrUrls, options, seams = {}) {
-	const override = parseOverride(options);
+	const override = parsePair("llm", options.llmProvider, options.llmModel);
 	if (!override.ok) {
 		failJson({ message: override.message });
 		return;
 	}
+	const judgeOverride = parsePair("judge-llm", options.judgeLlmProvider, options.judgeLlmModel);
+	if (!judgeOverride.ok) {
+		failJson({ message: judgeOverride.message });
+		return;
+	}
+	const judging = options.judgeFeedback !== false;
+	if (!judging && judgeOverride.llm) {
+		failJson({ message: "--judge-llm-provider/--judge-llm-model cannot be combined with --no-judge-feedback: the first configures the feedback judge, the second switches it off." });
+		return;
+	}
 	const expansion = expandSources(pathsOrUrls);
 	if (!expansion.ok) {
 		failJson({ message: expansion.message });
@@ -4640,33 +5578,56 @@ async function runEvalCommand(pathsOrUrls, options, seams = {}) {
 		return;
 	}
 	{
-		const totalCases = [...checked.values()].reduce((sum, file) => sum + file.caseCount, 0);
-		process.stderr.write(`${totalCases} case(s) × ${repeats} repeat(s) = ${totalCases * repeats} grading call(s)\n`);
+		const scope = (unit, generation, cases) => {
+			if (cases === 0) return;
+			const calls = cases * repeats;
+			process.stderr.write(`${cases} ${unit}(s) × ${repeats} repeat(s) = ${calls} ${generation}` + (judging ? ` + ${calls} judge call(s)\n` : " call(s)\n"));
+		};
+		const casesOf = (kind) => [...checked.values()].filter((file) => file.kind === kind).reduce((sum, file) => sum + file.caseCount, 0);
+		scope("case", "grading", casesOf("quiz"));
+		scope("conversation", "generation", casesOf("tutor"));
 	}
+	await warnOnVersionMismatch(options.server);
+	const judgeBreaker = createJudgeBreaker();
+	const onJudgeDegraded = () => {
+		process.stderr.write("Warning: feedback judging was stopped after 3 judge calls failed in a row — the rest of this run is graded but NOT judged. Grading is unaffected.\n");
+	};
 	let fileIndex = 0;
 	for (const file of files) {
 		fileIndex += 1;
 		const check = checked.get(file.source);
 		if (!check) continue;
-		const quizLlm = {
-			provider: check.quizDump.llm.provider,
-			model: check.quizDump.llm.model
+		const activityLlm = {
+			provider: check.llm.provider,
+			model: check.llm.model
 		};
-		const effective = override.llm ?? quizLlm;
+		const effective = override.llm ?? activityLlm;
+		const judgeLlm = judgeOverride.llm ?? effective;
 		const llm = {
 			...effective,
-			...override.llm ? { overrides: quizLlm } : {}
+			...override.llm ? { overrides: activityLlm } : {},
+			...judging ? { judge: {
+				...judgeLlm,
+				overridden: judgeOverride.llm !== void 0
+			} } : {}
 		};
-		const prefix = files.length > 1 ? `(${fileIndex}/${files.length}) ${check.evalFile.id}: ` : "";
-		file.result = await runEval("quiz", check, {
+		const label = files.length > 1 ? `(${fileIndex}/${files.length}) ${check.evalFile.id}` : check.evalFile.id;
+		const prefix = files.length > 1 ? `${label}: ` : "";
+		const result = await runEval(check.kind, check, {
 			grade: makeGradeFn(options.server, effective),
+			respond: makeRespondFn(options.server, effective),
+			...judging ? { judge: makeJudgeFn(options.server, judgeLlm) } : {},
+			judgeBreaker,
+			onJudgeDegraded,
 			concurrency,
 			repeats,
 			llm,
 			onProgress: progressWriter(prefix),
 			...seams.retry ? { retry: seams.retry } : {}
 		});
+		file.result = result;
 		if (process.stderr.isTTY) process.stderr.write("\n");
+		else writeFileDone(label, result);
 	}
 	const batch = summarizeBatch(files);
 	const payload = JSON.stringify(batch, null, 2);
@@ -4693,12 +5654,16 @@ async function runEvalCommand(pathsOrUrls, options, seams = {}) {
 	process.exitCode = batchPassed(batch) ? 0 : 1;
 }
 function registerEval(program) {
-	program.command("eval").description("Grade a file of golden answers against its quiz's real rubric and report the result").argument("<evalPathOrUrl...>", "one or more eval YAML files (paths, http(s)/file URLs, or a quoted glob pattern)").option("--server <url>", "Novedu server base URL (defaults to the NOVEDU_SERVER env var, then production)").option("--concurrency <n>", "grading calls in flight per file", String(CONCURRENCY_DEFAULT)).option("--repeats <n>", "grade every answer N times and take the majority verdict", "1").option("--llm-provider <provider>", "grade with this provider instead of the quiz's (\"SCCH\" or \"Azure Foundry\"; needs --llm-model)").option("--llm-model <model>", "grade with this model instead of the quiz's (needs --llm-provider)").option("--json", "print the machine-readable batch report on stdout").option("--out <file>", "additionally write the machine-readable batch report to a file").option("--report <file>", "additionally write a readable Markdown report to a file").addHelpText("after", `
+	program.command("eval").description("Run an eval file (quiz golden answers, or tutor conversations) against the real activity path and report the result").argument("<evalPathOrUrl...>", "one or more eval YAML files (paths, http(s)/file URLs, or a quoted glob pattern)").option("--server <url>", "Novedu server base URL (defaults to the NOVEDU_SERVER env var, then production)").option("--concurrency <n>", "cases in flight per file", String(CONCURRENCY_DEFAULT)).option("--repeats <n>", "run every case N times (quiz: take the majority verdict)", "1").option("--llm-provider <provider>", "run with this provider instead of the activity's (\"SCCH\" or \"Azure Foundry\"; needs --llm-model)").option("--llm-model <model>", "run with this model instead of the activity's (needs --llm-provider)").option("--no-judge-feedback", "skip the LLM audit of what the model wrote (halves the LLM calls)").option("--judge-llm-provider <provider>", "judge with this provider (\"SCCH\" or \"Azure Foundry\"; needs --judge-llm-model)").option("--judge-llm-model <model>", "judge with this model instead of the one under test (needs --judge-llm-provider)").option("--json", "print the machine-readable batch report on stdout").option("--out <file>", "additionally write the machine-readable batch report to a file").option("--report <file>", "additionally write a readable Markdown report to a file").addHelpText("after", `
 Examples:
   # Evaluate one quiz's golden answers
   $ novedu-cli eval ./0010-welcome-quiz.eval.yaml
 
-  # A whole course part (quote the pattern so the CLI expands it, ** included)
+  # Check how a tutor answers a set of scripted conversations
+  $ novedu-cli eval ./loops-tutor.eval.yaml
+
+  # A whole course part — quiz and tutor evals may be mixed
+  # (quote the pattern so the CLI expands it, ** included)
   $ novedu-cli eval "./part-1/**/*.eval.yaml"
 
   # Measure grader stability: 3 runs per answer, majority verdict
@@ -4707,6 +5672,12 @@ Examples:
   # How would this rubric perform on another model? (both flags, always together)
   $ novedu-cli eval ./my-quiz.eval.yaml --llm-provider "Azure Foundry" --llm-model gpt-5-mini
 
+  # A strong judge over the quiz's own grader — the recommended pairing
+  $ novedu-cli eval ./my-quiz.eval.yaml --judge-llm-provider "Azure Foundry" --judge-llm-model gpt-5.6-terra
+
+  # Half the LLM calls: check the verdicts only, skip the feedback audit
+  $ novedu-cli eval ./my-quiz.eval.yaml --no-judge-feedback
+
   # Machine-readable, for CI
   $ novedu-cli eval ./my-quiz.eval.yaml --json --out eval-report.json