@iris-eval/mcp-server 0.8.1 → 0.9.0

package/dist/eval/response-schema.js

@@ -0,0 +1,130 @@
+/*
+ * The one zod object that IS the evaluation response.
+ *
+ * Three things read it: the response-shape drift-lock test (which runs the
+ * real tool handlers and validates what they return), the tools'
+ * `outputSchema` once the structured-content release lands, and — at 1.0 —
+ * the published response-schema-v1.json. Keeping them one object is what
+ * makes "the response shape is settled" checkable: a field added here is
+ * additive by construction (every object is loose — unknown keys pass), and a field
+ * removed or re-meant fails the drift-lock before it can ship.
+ *
+ * Optional everywhere a field is optional on the TypeScript type; the test
+ * asserts presence where the release promises it (every built-in result
+ * carries kind, role, saw, uncertainty), so this schema states the shape
+ * and the test states the promise.
+ */
+import { z } from 'zod';
+export const intervalSchema = z.looseObject({ point: z.number(), lo: z.number(), hi: z.number() });
+const priorSchema = z.looseObject({ pi: z.number(), source: z.enum(['default', 'config', 'estimated']) });
+const corpusSchema = z.looseObject({
+    n: z.number().int(),
+    tp: z.number().int(),
+    fp: z.number().int(),
+    fn: z.number().int(),
+    tn: z.number().int(),
+    version: z.string(),
+    release: z.string(),
+    labelling: z.enum(['same-model', 'human-verified']),
+});
+export const uncertaintySchema = z.discriminatedUnion('basis', [
+    // One member per basis (zod's discriminator must be unique); the fired /
+    // not-fired split is enforced by the refinement: a fire carries ppv, a
+    // quiet result carries missRate.
+    z
+        .looseObject({ basis: z.literal('published_accuracy'), fired: z.boolean(), ppv: intervalSchema.optional(), missRate: intervalSchema.optional(), prior: priorSchema, corpus: corpusSchema })
+        .refine((v) => (v.fired ? v.ppv !== undefined : v.missRate !== undefined), { message: 'a fired result carries ppv; a quiet one carries missRate' }),
+    z.looseObject({ basis: z.literal('definition'), conformance: z.looseObject({ n: z.number().int(), matched: z.number().int() }) }),
+    z.looseObject({ basis: z.literal('self_consistency'), samples: z.number().int(), voteFraction: z.number(), scoreSd: z.number() }),
+    z.looseObject({ basis: z.literal('local_labels'), precision: intervalSchema, n: z.number().int() }),
+    z.looseObject({ basis: z.literal('policy') }),
+    z.looseObject({ basis: z.literal('unmeasured'), why: z.string() }),
+]);
+export const evidenceSchema = z.discriminatedUnion('type', [
+    z.looseObject({ type: z.literal('span'), source: z.string(), start: z.number().int().nonnegative(), end: z.number().int().nonnegative(), label: z.string() }),
+    z.looseObject({ type: z.literal('pattern'), name: z.string(), count: z.number().int().nonnegative() }),
+    z.looseObject({ type: z.literal('toolCall'), index: z.number().int().nonnegative(), toolName: z.string(), label: z.string() }),
+    z.looseObject({ type: z.literal('citation'), url: z.string(), status: z.enum(['resolved', 'dead', 'unverifiable', 'supported', 'unsupported']) }),
+    z.looseObject({ type: z.literal('count'), stat: z.string(), unit: z.string(), value: z.number(), threshold: z.number().optional(), thresholdSource: z.enum(['default', 'config', 'call', 'rule']).optional() }),
+]);
+export const measuredValueSchema = z.looseObject({ stat: z.string(), unit: z.string(), value: z.number() });
+export const claimKindSchema = z.enum(['measurement', 'detection', 'inference', 'judgment', 'policy', 'verification']);
+export const roleSchema = z.enum(['gate', 'veto', 'risk', 'advisory', 'term']);
+export const skipClassSchema = z.enum(['not_applicable', 'defeated', 'config_invalid']);
+export const needSchema = z.enum(['output', 'input', 'expected', 'tool_calls', 'tool_outputs', 'tools_catalogue', 'cost', 'tokens', 'citations']);
+export const questionIdSchema = z.enum(['safe_output', 'grounded', 'complete', 'relevant', 'task_completed', 'tool_use_correct', 'within_budget']);
+export const evalRuleResultSchema = z.looseObject({
+    ruleName: z.string(),
+    ruleId: z.string().optional(),
+    category: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']).optional(),
+    critical: z.boolean().optional(),
+    criticalSource: z.enum(['default', 'config']).optional(),
+    passed: z.boolean(),
+    score: z.number(),
+    message: z.string(),
+    skipped: z.boolean().optional(),
+    skipReason: z.string().optional(),
+    configInvalid: z.boolean().optional(),
+    budgetExceeded: z.boolean().optional(),
+    kind: claimKindSchema.optional(),
+    role: roleSchema.optional(),
+    question: questionIdSchema.optional(),
+    classes: z.array(z.string()).optional(),
+    ruleVersion: z.number().int().optional(),
+    saw: z.array(needSchema).optional(),
+    skipClass: skipClassSchema.optional(),
+    uncertainty: uncertaintySchema.optional(),
+    evidence: z.array(evidenceSchema).optional(),
+    value: measuredValueSchema.optional(),
+});
+export const evalCategoryResultSchema = z.looseObject({
+    score: z.number().nullable(),
+    passed: z.boolean().nullable(),
+    rules_evaluated: z.number().int(),
+    rules_skipped: z.number().int(),
+    insufficient_data: z.boolean(),
+    critical_failures: z.array(z.string()).optional(),
+    critical_skipped: z.array(z.string()).optional(),
+});
+export const coverageSchema = z.looseObject({
+    inputs: z.record(z.string(), z.boolean()),
+    questions: z.array(z.looseObject({ id: questionIdSchema, status: z.enum(['judged', 'unjudged', 'not_applicable']), why: z.string().optional() })),
+    dormant: z.array(z.looseObject({ ruleId: z.string(), name: z.string(), reason: z.string() })).optional(),
+});
+export const verdictSchema = z.looseObject({
+    state: z.enum(['pass', 'fail', 'unknown']),
+    passed: z.boolean(),
+    basis: z.enum(['policy_gate', 'detector_veto', 'critical_unknown', 'required_evidence_missing', 'risk_over_loss', 'score_below_threshold', 'clean', 'no_rules']),
+    by: z.array(z.string()),
+    risk: z.looseObject({ pBad: z.number(), lo: z.number(), hi: z.number() }).nullable(),
+    confidence: z.enum(['decisive', 'marginal']).optional(),
+});
+export const provenanceSchema = z.looseObject({
+    irisVersion: z.string(),
+    rulesetHash: z.string(),
+    configHash: z.string(),
+    thresholds: z.looseObject({ default: z.number(), perRule: z.record(z.string(), z.unknown()).optional() }),
+    corpusVersion: z.string(),
+    judgedAt: z.string(),
+});
+/** The `evaluate_output` response — the same object the engine returns plus the tool's own fields. */
+export const evaluateOutputResponseSchema = z.looseObject({
+    id: z.string().describe('the evaluation id, readable at iris://evaluations/{id}'),
+    trace_id: z.string().optional().describe('the linked trace, when named'),
+    verdict: verdictSchema.optional().describe('state, passed, basis (which layer decided), by (the rules), risk'),
+    coverage: coverageSchema.optional().describe('per question: judged, unjudged and why, or not_applicable; plus the inputs carried'),
+    provenance: provenanceSchema.optional().describe('Iris version, ruleset and config hashes, thresholds, corpus version, time'),
+    erased_at: z.string().optional().describe('set once the linked trace was deleted'),
+    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom', 'all']).describe('the bundle that ran'),
+    score: z.number().describe('0..1 weighted quality over the rules that ran'),
+    passed: z.boolean().describe('the ship verdict; false when nothing was judged'),
+    rule_results: z.array(evalRuleResultSchema).describe('per rule: verdict, message, kind, role, question, saw, evidence, uncertainty'),
+    suggestions: z.array(z.string()).describe('what to change'),
+    rules_evaluated: z.number().int().describe('rules that judged'),
+    rules_skipped: z.number().int().describe('rules that skipped'),
+    insufficient_data: z.boolean().describe('true when no rule could judge'),
+    critical_failures: z.array(z.string()).optional().describe('critical rules that failed and vetoed passed'),
+    critical_skipped: z.array(z.string()).optional().describe('critical rules that could not judge; treat as unknown'),
+    categories: z.record(z.string(), evalCategoryResultSchema).optional().describe('per-bundle verdicts for eval_type all'),
+    note: z.string().optional().describe('present when eval_type was omitted'),
+});

package/dist/eval/response.d.ts

@@ -0,0 +1,12 @@
+import type { EvalResult } from '../types/eval.js';
+import type { DormantRule } from './dormant.js';
+export interface EvaluationResponseOptions {
+    /** The trace the evaluation was linked to, echoed so a caller can join the two without a second read. */
+    traceId?: string;
+    /** Present only when `eval_type` was omitted and the default ran. */
+    note?: string;
+    /** Quarantined gating rules on this server, carried as coverage.dormant (a gate reads the verdict, never list_rules). */
+    dormant?: DormantRule[];
+}
+/** The response body for an evaluation — what the tool returns as text and, later, as structured content. */
+export declare function toEvaluationResponse(result: EvalResult, options?: EvaluationResponseOptions): Record<string, unknown>;

package/dist/eval/response.js

@@ -0,0 +1,30 @@
+/** The response body for an evaluation — what the tool returns as text and, later, as structured content. */
+export function toEvaluationResponse(result, options = {}) {
+    const traceId = options.traceId ?? result.trace_id;
+    return {
+        id: result.id,
+        ...(traceId ? { trace_id: traceId } : {}),
+        // Echo which bundle actually ran. Without this, a caller who omitted
+        // eval_type could not tell a "safety pass" from a completeness eval
+        // that never ran a single safety rule.
+        eval_type: result.eval_type,
+        score: result.score,
+        passed: result.passed,
+        ...(result.verdict ? { verdict: result.verdict } : {}),
+        ...(result.critical_failures?.length ? { critical_failures: result.critical_failures } : {}),
+        // The other half of the veto contract: every critical rule that SKIPPED
+        // is named so a fail-closed gate can treat the evaluation as unknown.
+        ...(result.critical_skipped?.length ? { critical_skipped: result.critical_skipped } : {}),
+        rule_results: result.rule_results,
+        suggestions: result.suggestions,
+        rules_evaluated: result.rules_evaluated,
+        rules_skipped: result.rules_skipped,
+        insufficient_data: result.insufficient_data,
+        ...(result.coverage ? { coverage: options.dormant?.length ? { ...result.coverage, dormant: options.dormant } : result.coverage } : {}),
+        ...(result.erased_at ? { erased_at: result.erased_at } : {}),
+        ...(result.provenance ? { provenance: result.provenance } : {}),
+        // Per-bundle breakdown — eval_type="all" only.
+        ...(result.categories ? { categories: result.categories } : {}),
+        ...(options.note ? { note: options.note } : {}),
+    };
+}

package/dist/eval/rules/completeness.js

@@ -3,6 +3,12 @@ export const minOutputLength = {
     description: 'Output must meet a minimum character length',
     evalType: 'completeness',
     weight: 1,
+    kind: 'measurement',
+    mechanism: 'formula',
+    needs: ['output'],
+    question: 'complete',
+    classes: ['format'],
+    version: 1,
     evaluate(context) {
         const minLen = context.customConfig?.min_output_length
             ?? context.customConfig?.min_length
@@ -13,6 +19,8 @@ export const minOutputLength = {
             ruleName: 'min_output_length',
             passed,
             score: passed ? 1 : Math.min(len / minLen, 0.99),
+            value: { stat: 'length', unit: 'chars', value: len },
+            evidence: [{ type: 'count', stat: 'length', unit: 'chars', value: len, threshold: minLen, thresholdSource: minLen === 50 ? 'default' : 'config' }],
             message: passed ? `Output length (${len}) meets minimum (${minLen})` : `Output length (${len}) below minimum (${minLen})`,
         };
     },
@@ -22,12 +30,19 @@ export const nonEmptyOutput = {
     description: 'Output must not be empty or whitespace-only',
     evalType: 'completeness',
     weight: 2,
+    kind: 'policy',
+    mechanism: 'formula',
+    needs: ['output'],
+    question: 'complete',
+    classes: ['format'],
+    version: 1,
     evaluate(context) {
         const passed = context.output.trim().length > 0;
         return {
             ruleName: 'non_empty_output',
             passed,
             score: passed ? 1 : 0,
+            value: { stat: 'length', unit: 'chars', value: context.output.trim().length },
             message: passed ? 'Output is non-empty' : 'Output is empty or whitespace-only',
         };
     },
@@ -37,6 +52,12 @@ export const sentenceCount = {
     description: 'Output must contain a minimum number of sentences',
     evalType: 'completeness',
     weight: 0.5,
+    kind: 'measurement',
+    mechanism: 'formula',
+    needs: ['output'],
+    question: 'complete',
+    classes: ['format'],
+    version: 1,
     evaluate(context) {
         const minSentences = context.customConfig?.min_sentences ?? 2;
         const sentences = context.output.split(/[.!?]+/).filter((s) => s.trim().length > 0).length;
@@ -45,6 +66,8 @@ export const sentenceCount = {
             ruleName: 'sentence_count',
             passed,
             score: passed ? 1 : Math.min(sentences / minSentences, 0.99),
+            value: { stat: 'sentences', unit: 'sentences', value: sentences },
+            evidence: [{ type: 'count', stat: 'sentences', unit: 'sentences', value: sentences, threshold: minSentences, thresholdSource: minSentences === 2 ? 'default' : 'config' }],
             message: passed ? `Sentence count (${sentences}) meets minimum (${minSentences})` : `Sentence count (${sentences}) below minimum (${minSentences})`,
         };
     },
@@ -54,6 +77,12 @@ export const expectedCoverage = {
     description: 'Output must cover key terms from expected output',
     evalType: 'completeness',
     weight: 1.5,
+    kind: 'measurement',
+    mechanism: 'formula',
+    needs: ['output', 'expected'],
+    question: 'complete',
+    classes: ['incomplete_ask'],
+    version: 1,
     evaluate(context) {
         if (!context.expected) {
             return { ruleName: 'expected_coverage', passed: false, score: 0, message: 'No expected output provided', skipped: true, skipReason: 'context.expected not provided' };
@@ -74,6 +103,8 @@ export const expectedCoverage = {
             ruleName: 'expected_coverage',
             passed,
             score: ratio,
+            value: { stat: 'expected_terms_covered', unit: 'ratio', value: ratio },
+            evidence: [{ type: 'count', stat: 'expected_terms_covered', unit: 'ratio', value: ratio, threshold: 0.5, thresholdSource: 'rule' }],
             message: `Covered ${covered}/${expectedWords.size} expected terms (${(ratio * 100).toFixed(0)}%)`,
         };
     },

package/dist/eval/rules/cost.d.ts

@@ -1,4 +1,4 @@
-import type { EvalRule } from '../../types/eval.js';
+import { type EvalRule } from '../../types/eval.js';
 export declare const costUnderThreshold: EvalRule;
 export declare const tokenEfficiency: EvalRule;
 /** Default for config key `max_tool_repeats`: how many identical calls are tolerated. */

package/dist/eval/rules/cost.js

@@ -1,9 +1,16 @@
+import { MAX_EVIDENCE_ITEMS } from '../../types/eval.js';
 import { callKey, describeInput, skipWithoutTrajectory } from './trajectory.js';
 export const costUnderThreshold = {
     name: 'cost_under_threshold',
     description: 'Total cost must be under a configurable USD threshold',
     evalType: 'cost',
     weight: 1,
+    kind: 'policy',
+    mechanism: 'formula',
+    needs: ['cost'],
+    question: 'within_budget',
+    classes: ['over_budget'],
+    version: 1,
     evaluate(context) {
         if (context.costUsd === undefined || context.costUsd === null) {
             return { ruleName: 'cost_under_threshold', passed: false, score: 0, message: 'Cost data not provided', skipped: true, skipReason: 'context.costUsd not provided' };
@@ -15,6 +22,8 @@ export const costUnderThreshold = {
             ruleName: 'cost_under_threshold',
             passed,
             score: passed ? 1 : Math.max(0, 1 - (cost - threshold) / threshold),
+            value: { stat: 'cost', unit: 'usd', value: cost },
+            evidence: [{ type: 'count', stat: 'cost', unit: 'usd', value: cost, threshold, thresholdSource: threshold === 0.10 ? 'default' : 'config' }],
             message: passed
                 ? `Cost ($${cost.toFixed(4)}) is under threshold ($${threshold.toFixed(4)})`
                 : `Cost ($${cost.toFixed(4)}) exceeds threshold ($${threshold.toFixed(4)})`,
@@ -26,6 +35,12 @@ export const tokenEfficiency = {
     description: 'Checks output-to-input token ratio for efficiency',
     evalType: 'cost',
     weight: 0.5,
+    kind: 'measurement',
+    mechanism: 'formula',
+    needs: ['tokens'],
+    question: 'within_budget',
+    classes: ['over_budget'],
+    version: 1,
     evaluate(context) {
         const prompt = context.tokenUsage?.prompt_tokens;
         const completion = context.tokenUsage?.completion_tokens;
@@ -39,6 +54,8 @@ export const tokenEfficiency = {
             ruleName: 'token_efficiency',
             passed,
             score: passed ? 1 : Math.max(0, 1 - (ratio - maxRatio) / maxRatio),
+            value: { stat: 'completion_to_prompt_ratio', unit: 'ratio', value: ratio },
+            evidence: [{ type: 'count', stat: 'completion_to_prompt_ratio', unit: 'ratio', value: ratio, threshold: maxRatio, thresholdSource: maxRatio === 5 ? 'default' : 'config' }],
             message: passed
                 ? `Token ratio (${ratio.toFixed(2)}) is within limits (max ${maxRatio})`
                 : `Token ratio (${ratio.toFixed(2)}) exceeds max (${maxRatio})`,
@@ -91,6 +108,12 @@ export const noToolLoop = {
     description: 'The agent must not repeat itself. Fails when one tool is called with an identical input (object keys sorted, whitespace collapsed) more than max_tool_repeats times — default 3, config key `max_tool_repeats` — or when two calls alternate for more than two complete A,B,A,B cycles. Skips when no tool calls are provided, so an evaluation with no trajectory reports "not judged" rather than clean. Catches the wasted spend a cost threshold cannot see: five identical calls can still bill under a per-evaluation cost limit',
     evalType: 'cost',
     weight: 1,
+    kind: 'detection',
+    mechanism: 'formula',
+    needs: ['tool_calls'],
+    question: 'tool_use_correct',
+    classes: ['tool_loop'],
+    version: 1,
     evaluate(context) {
         const skip = skipWithoutTrajectory('no_tool_loop', context);
         if (skip)
@@ -112,12 +135,22 @@ export const noToolLoop = {
                 worstCount = count;
             }
         }
+        const repeatsEvidence = [
+            { type: 'count', stat: 'max_repeats_of_one_call', unit: 'calls', value: worstCount, threshold: maxRepeats, thresholdSource: maxRepeats === DEFAULT_MAX_TOOL_REPEATS ? 'default' : 'config' },
+        ];
         if (worstCount > maxRepeats) {
             const call = calls[keys.indexOf(worstKey)];
+            const evidence = [...repeatsEvidence];
+            keys.forEach((key, index) => {
+                if (key === worstKey && evidence.length < MAX_EVIDENCE_ITEMS)
+                    evidence.push({ type: 'toolCall', index, toolName: calls[index].tool_name, label: 'repeated call' });
+            });
             return {
                 ruleName: 'no_tool_loop',
                 passed: false,
                 score: Math.max(0, 1 - (worstCount - maxRepeats) * 0.25),
+                value: { stat: 'max_repeats_of_one_call', unit: 'calls', value: worstCount },
+                evidence,
                 message: `Tool loop: ${call.tool_name} called ${worstCount} times with the same input — ${describeInput(call.input)} — over ${calls.length} call${calls.length === 1 ? '' : 's'} (max ${maxRepeats})`,
             };
         }
@@ -125,10 +158,19 @@ export const noToolLoop = {
         if (cycle !== null && cycle.cycles > MAX_TWO_CALL_CYCLES) {
             const a = calls[keys.indexOf(cycle.a)];
             const b = calls[keys.indexOf(cycle.b)];
+            const evidence = [
+                { type: 'count', stat: 'two_call_cycles', unit: 'cycles', value: cycle.cycles, threshold: MAX_TWO_CALL_CYCLES, thresholdSource: 'rule' },
+            ];
+            keys.forEach((key, index) => {
+                if ((key === cycle.a || key === cycle.b) && evidence.length < MAX_EVIDENCE_ITEMS)
+                    evidence.push({ type: 'toolCall', index, toolName: calls[index].tool_name, label: 'alternating call' });
+            });
             return {
                 ruleName: 'no_tool_loop',
                 passed: false,
                 score: Math.max(0, 1 - (cycle.cycles - MAX_TWO_CALL_CYCLES) * 0.25),
+                value: { stat: 'two_call_cycles', unit: 'cycles', value: cycle.cycles },
+                evidence,
                 message: `Tool loop: ${a.tool_name} (${describeInput(a.input)}) and ${b.tool_name} (${describeInput(b.input)}) alternate for ${cycle.cycles} cycles (max ${MAX_TWO_CALL_CYCLES})`,
             };
         }
@@ -137,6 +179,8 @@ export const noToolLoop = {
             passed: true,
             score: 1,
             message: `No repeated tool call (${calls.length} call${calls.length === 1 ? '' : 's'}; most repeated ran ${worstCount}×, max ${maxRepeats})`,
+            value: { stat: 'max_repeats_of_one_call', unit: 'calls', value: worstCount },
+            evidence: repeatsEvidence,
         };
     },
 };

package/dist/eval/rules/custom.d.ts

@@ -15,16 +15,4 @@ export declare function normalizeRegexSource(patternStr: string, flags: string):
     pattern: string;
     flags: string;
 };
-/**
- * Builds a runnable EvalRule from a persisted/inline definition.
- *
- * `severity` comes from the DEPLOYED rule's metadata (deploy_rule / the
- * dashboard composer). high/critical severities make the rule CRITICAL:
- * a failing evaluation forces the overall eval to passed=false regardless
- * of the weighted score. Before this, a rule-author could deploy a
- * severity="critical" policy rule, watch it FAIL on a violating output,
- * and still get passed:true (score 0.895) — severity affected nothing but
- * dashboard sorting. Inline custom_rules (evaluate_output's per-call
- * definitions) carry no severity and stay weight-only.
- */
 export declare function createCustomRule(definition: CustomRuleDefinition, severity?: RuleSeverity): EvalRule;

package/dist/eval/rules/custom.js

@@ -220,13 +220,34 @@ function runSandboxed(definition, pattern, flags, context) {
  * dashboard sorting. Inline custom_rules (evaluate_output's per-call
  * definitions) carry no severity and stay weight-only.
  */
+/**
+ * A custom rule is the author's own constraint, so its kind is `policy`
+ * whatever its mechanism; the mechanism and the inputs it reads follow the
+ * type. The question it answers is the author's and is not guessed here.
+ */
+const CUSTOM_TYPE_META = {
+    regex_match: { mechanism: 'pattern', needs: ['output'] },
+    regex_no_match: { mechanism: 'pattern', needs: ['output'] },
+    min_length: { mechanism: 'formula', needs: ['output'] },
+    max_length: { mechanism: 'formula', needs: ['output'] },
+    contains_keywords: { mechanism: 'pattern', needs: ['output'] },
+    excludes_keywords: { mechanism: 'pattern', needs: ['output'] },
+    json_schema: { mechanism: 'formula', needs: ['output'] },
+    cost_threshold: { mechanism: 'formula', needs: ['cost'] },
+};
 export function createCustomRule(definition, severity) {
+    const meta = CUSTOM_TYPE_META[definition.type];
     return {
         name: definition.name,
         description: `Custom rule: ${definition.name}`,
         evalType: 'custom',
         weight: definition.weight ?? 1,
         critical: severity === 'high' || severity === 'critical',
+        kind: 'policy',
+        mechanism: meta?.mechanism ?? 'formula',
+        needs: meta?.needs ?? ['output'],
+        classes: [],
+        version: 1,
         evaluate(context) {
             switch (definition.type) {
                 case 'regex_match': {

package/dist/eval/rules/relevance.js

@@ -127,6 +127,12 @@ export const keywordOverlap = {
     description: 'Recall of the input\'s content terms in the output: stopwords and request verbs are not terms, code identifiers and paths are split into their words, inflections are folded (purge/purged/purging). Passes when at least 35% of the input\'s terms appear in the output (configurable: keyword_overlap)',
     evalType: 'relevance',
     weight: 1,
+    kind: 'measurement',
+    mechanism: 'formula',
+    needs: ['output', 'input'],
+    question: 'relevant',
+    classes: ['off_task'],
+    version: 1,
     evaluate(context) {
         if (!context.input) {
             return { ruleName: 'keyword_overlap', passed: false, score: 0, message: 'No input provided', skipped: true, skipReason: 'context.input not provided' };
@@ -145,6 +151,8 @@ export const keywordOverlap = {
         const threshold = context.customConfig?.keyword_overlap ?? 0.35;
         const passed = ratio >= threshold;
         return {
+            value: { stat: 'input_terms_in_output', unit: 'ratio', value: ratio },
+            evidence: [{ type: 'count', stat: 'input_terms_in_output', unit: 'ratio', value: ratio, threshold, thresholdSource: threshold === 0.35 ? 'default' : 'config' }],
             ruleName: 'keyword_overlap',
             passed,
             score: Math.min(ratio * 2, 1),
@@ -177,6 +185,12 @@ export const topicConsistency = {
     description: 'Continuity with the input: the share of the output\'s content-bearing sentences that connect to the input\'s topic — a sentence connects when it shares a content term with the input or with an earlier connected sentence (list items are read under the sentence that introduces them). Passes when at least a third connect (configurable: topic_consistency); a third, not half, so a short honest answer that elaborates in fresh words is not read as drift. Replaces the output-word-ratio measure that failed every grounded technical answer. Skipped when the output is too brief for meaningful comparison',
     evalType: 'relevance',
     weight: 1,
+    kind: 'measurement',
+    mechanism: 'formula',
+    needs: ['output', 'input'],
+    question: 'relevant',
+    classes: ['off_task'],
+    version: 1,
     evaluate(context) {
         if (!context.input) {
             return { ruleName: 'topic_consistency', passed: false, score: 0, message: 'No input provided', skipped: true, skipReason: 'context.input not provided' };
@@ -240,6 +254,8 @@ export const topicConsistency = {
             passed,
             // Full marks at two thirds connected; proportional below.
             score: Math.min(ratio * 1.5, 1),
+            value: { stat: 'connected_sentences', unit: 'ratio', value: ratio },
+            evidence: [{ type: 'count', stat: 'connected_sentences', unit: 'ratio', value: ratio, threshold, thresholdSource: threshold === DEFAULT_TOPIC_THRESHOLD ? 'default' : 'config' }],
             message: `Topic consistency: ${connected}/${sentences} content sentences connect to the input's topic (${(ratio * 100).toFixed(0)}%)`,
         };
     },