@iris-eval/mcp-server 0.8.2 → 0.10.0

package/dist/eval/response-schema.js

@@ -0,0 +1,130 @@
+/*
+ * The one zod object that IS the evaluation response.
+ *
+ * Three things read it: the response-shape drift-lock test (which runs the
+ * real tool handlers and validates what they return), the tools'
+ * `outputSchema` once the structured-content release lands, and — at 1.0 —
+ * the published response-schema-v1.json. Keeping them one object is what
+ * makes "the response shape is settled" checkable: a field added here is
+ * additive by construction (every object is loose — unknown keys pass), and a field
+ * removed or re-meant fails the drift-lock before it can ship.
+ *
+ * Optional everywhere a field is optional on the TypeScript type; the test
+ * asserts presence where the release promises it (every built-in result
+ * carries kind, role, saw, uncertainty), so this schema states the shape
+ * and the test states the promise.
+ */
+import { z } from 'zod';
+export const intervalSchema = z.looseObject({ point: z.number(), lo: z.number(), hi: z.number() });
+const priorSchema = z.looseObject({ pi: z.number(), source: z.enum(['default', 'config', 'estimated']) });
+const corpusSchema = z.looseObject({
+    n: z.number().int(),
+    tp: z.number().int(),
+    fp: z.number().int(),
+    fn: z.number().int(),
+    tn: z.number().int(),
+    version: z.string(),
+    release: z.string(),
+    labelling: z.enum(['same-model', 'human-verified']),
+});
+export const uncertaintySchema = z.discriminatedUnion('basis', [
+    // One member per basis (zod's discriminator must be unique); the fired /
+    // not-fired split is enforced by the refinement: a fire carries ppv, a
+    // quiet result carries missRate.
+    z
+        .looseObject({ basis: z.literal('published_accuracy'), fired: z.boolean(), ppv: intervalSchema.optional(), missRate: intervalSchema.optional(), prior: priorSchema, corpus: corpusSchema })
+        .refine((v) => (v.fired ? v.ppv !== undefined : v.missRate !== undefined), { message: 'a fired result carries ppv; a quiet one carries missRate' }),
+    z.looseObject({ basis: z.literal('definition'), conformance: z.looseObject({ n: z.number().int(), matched: z.number().int() }) }),
+    z.looseObject({ basis: z.literal('self_consistency'), samples: z.number().int(), voteFraction: z.number(), scoreSd: z.number() }),
+    z.looseObject({ basis: z.literal('local_labels'), precision: intervalSchema, n: z.number().int() }),
+    z.looseObject({ basis: z.literal('policy') }),
+    z.looseObject({ basis: z.literal('unmeasured'), why: z.string() }),
+]);
+export const evidenceSchema = z.discriminatedUnion('type', [
+    z.looseObject({ type: z.literal('span'), source: z.string(), start: z.number().int().nonnegative(), end: z.number().int().nonnegative(), label: z.string() }),
+    z.looseObject({ type: z.literal('pattern'), name: z.string(), count: z.number().int().nonnegative() }),
+    z.looseObject({ type: z.literal('toolCall'), index: z.number().int().nonnegative(), toolName: z.string(), label: z.string() }),
+    z.looseObject({ type: z.literal('citation'), url: z.string(), status: z.enum(['resolved', 'dead', 'unverifiable', 'supported', 'unsupported']) }),
+    z.looseObject({ type: z.literal('count'), stat: z.string(), unit: z.string(), value: z.number(), threshold: z.number().optional(), thresholdSource: z.enum(['default', 'config', 'call', 'rule']).optional() }),
+]);
+export const measuredValueSchema = z.looseObject({ stat: z.string(), unit: z.string(), value: z.number() });
+export const claimKindSchema = z.enum(['measurement', 'detection', 'inference', 'judgment', 'policy', 'verification']);
+export const roleSchema = z.enum(['gate', 'veto', 'risk', 'advisory', 'term']);
+export const skipClassSchema = z.enum(['not_applicable', 'defeated', 'config_invalid']);
+export const needSchema = z.enum(['output', 'input', 'expected', 'tool_calls', 'tool_outputs', 'tools_catalogue', 'cost', 'tokens', 'citations']);
+export const questionIdSchema = z.enum(['safe_output', 'grounded', 'complete', 'relevant', 'task_completed', 'tool_use_correct', 'within_budget']);
+export const evalRuleResultSchema = z.looseObject({
+    ruleName: z.string(),
+    ruleId: z.string().optional(),
+    category: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']).optional(),
+    critical: z.boolean().optional(),
+    criticalSource: z.enum(['default', 'config']).optional(),
+    passed: z.boolean(),
+    score: z.number(),
+    message: z.string(),
+    skipped: z.boolean().optional(),
+    skipReason: z.string().optional(),
+    configInvalid: z.boolean().optional(),
+    budgetExceeded: z.boolean().optional(),
+    kind: claimKindSchema.optional(),
+    role: roleSchema.optional(),
+    question: questionIdSchema.optional(),
+    classes: z.array(z.string()).optional(),
+    ruleVersion: z.number().int().optional(),
+    saw: z.array(needSchema).optional(),
+    skipClass: skipClassSchema.optional(),
+    uncertainty: uncertaintySchema.optional(),
+    evidence: z.array(evidenceSchema).optional(),
+    value: measuredValueSchema.optional(),
+});
+export const evalCategoryResultSchema = z.looseObject({
+    score: z.number().nullable(),
+    passed: z.boolean().nullable(),
+    rules_evaluated: z.number().int(),
+    rules_skipped: z.number().int(),
+    insufficient_data: z.boolean(),
+    critical_failures: z.array(z.string()).optional(),
+    critical_skipped: z.array(z.string()).optional(),
+});
+export const coverageSchema = z.looseObject({
+    inputs: z.record(z.string(), z.boolean()),
+    questions: z.array(z.looseObject({ id: questionIdSchema, status: z.enum(['judged', 'unjudged', 'not_applicable']), why: z.string().optional() })),
+    dormant: z.array(z.looseObject({ ruleId: z.string(), name: z.string(), reason: z.string() })).optional(),
+});
+export const verdictSchema = z.looseObject({
+    state: z.enum(['pass', 'fail', 'unknown']),
+    passed: z.boolean(),
+    basis: z.enum(['policy_gate', 'detector_veto', 'critical_unknown', 'required_evidence_missing', 'risk_over_loss', 'score_below_threshold', 'clean', 'no_rules']),
+    by: z.array(z.string()),
+    risk: z.looseObject({ pBad: z.number(), lo: z.number(), hi: z.number() }).nullable(),
+    confidence: z.enum(['decisive', 'marginal']).optional(),
+});
+export const provenanceSchema = z.looseObject({
+    irisVersion: z.string(),
+    rulesetHash: z.string(),
+    configHash: z.string(),
+    thresholds: z.looseObject({ default: z.number(), perRule: z.record(z.string(), z.unknown()).optional() }),
+    corpusVersion: z.string(),
+    judgedAt: z.string(),
+});
+/** The `evaluate_output` response — the same object the engine returns plus the tool's own fields. */
+export const evaluateOutputResponseSchema = z.looseObject({
+    id: z.string().describe('the evaluation id, readable at iris://evaluations/{id}'),
+    trace_id: z.string().optional().describe('the linked trace, when named'),
+    verdict: verdictSchema.optional().describe('state, passed, basis (which layer decided), by (the rules), risk'),
+    coverage: coverageSchema.optional().describe('per question: judged, unjudged and why, or not_applicable; plus the inputs carried'),
+    provenance: provenanceSchema.optional().describe('Iris version, ruleset and config hashes, thresholds, corpus version, time'),
+    erased_at: z.string().optional().describe('set once the linked trace was deleted'),
+    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom', 'all']).describe('the bundle that ran'),
+    score: z.number().describe('0..1 weighted quality over the rules that ran'),
+    passed: z.boolean().describe('the ship verdict; false when nothing was judged'),
+    rule_results: z.array(evalRuleResultSchema).describe('per rule: verdict, message, kind, role, question, saw, evidence, uncertainty'),
+    suggestions: z.array(z.string()).describe('what to change'),
+    rules_evaluated: z.number().int().describe('rules that judged'),
+    rules_skipped: z.number().int().describe('rules that skipped'),
+    insufficient_data: z.boolean().describe('true when no rule could judge'),
+    critical_failures: z.array(z.string()).optional().describe('critical rules that failed and vetoed passed'),
+    critical_skipped: z.array(z.string()).optional().describe('critical rules that could not judge; treat as unknown'),
+    categories: z.record(z.string(), evalCategoryResultSchema).optional().describe('per-bundle verdicts for eval_type all'),
+    note: z.string().optional().describe('present when eval_type was omitted'),
+});

package/dist/eval/response.d.ts

@@ -0,0 +1,12 @@
+import type { EvalResult } from '../types/eval.js';
+import type { DormantRule } from './dormant.js';
+export interface EvaluationResponseOptions {
+    /** The trace the evaluation was linked to, echoed so a caller can join the two without a second read. */
+    traceId?: string;
+    /** Present only when `eval_type` was omitted and the default ran. */
+    note?: string;
+    /** Quarantined gating rules on this server, carried as coverage.dormant (a gate reads the verdict, never list_rules). */
+    dormant?: DormantRule[];
+}
+/** The response body for an evaluation — what the tool returns as text and, later, as structured content. */
+export declare function toEvaluationResponse(result: EvalResult, options?: EvaluationResponseOptions): Record<string, unknown>;

package/dist/eval/response.js

@@ -0,0 +1,30 @@
+/** The response body for an evaluation — what the tool returns as text and, later, as structured content. */
+export function toEvaluationResponse(result, options = {}) {
+    const traceId = options.traceId ?? result.trace_id;
+    return {
+        id: result.id,
+        ...(traceId ? { trace_id: traceId } : {}),
+        // Echo which bundle actually ran. Without this, a caller who omitted
+        // eval_type could not tell a "safety pass" from a completeness eval
+        // that never ran a single safety rule.
+        eval_type: result.eval_type,
+        score: result.score,
+        passed: result.passed,
+        ...(result.verdict ? { verdict: result.verdict } : {}),
+        ...(result.critical_failures?.length ? { critical_failures: result.critical_failures } : {}),
+        // The other half of the veto contract: every critical rule that SKIPPED
+        // is named so a fail-closed gate can treat the evaluation as unknown.
+        ...(result.critical_skipped?.length ? { critical_skipped: result.critical_skipped } : {}),
+        rule_results: result.rule_results,
+        suggestions: result.suggestions,
+        rules_evaluated: result.rules_evaluated,
+        rules_skipped: result.rules_skipped,
+        insufficient_data: result.insufficient_data,
+        ...(result.coverage ? { coverage: options.dormant?.length ? { ...result.coverage, dormant: options.dormant } : result.coverage } : {}),
+        ...(result.erased_at ? { erased_at: result.erased_at } : {}),
+        ...(result.provenance ? { provenance: result.provenance } : {}),
+        // Per-bundle breakdown — eval_type="all" only.
+        ...(result.categories ? { categories: result.categories } : {}),
+        ...(options.note ? { note: options.note } : {}),
+    };
+}

package/dist/eval/risk.d.ts

@@ -0,0 +1,60 @@
+import type { EvalResult, FailureClass } from '../types/eval.js';
+export declare const RISK_DRAWS = 2000;
+export declare const DEFAULT_PRIOR = 0.5;
+/**
+ * What the prior means (arc 2 finding, for arc 3's deliberation):
+ *   'per-class'  — plan §4.3 as written: π is the prior that EACH examined
+ *                  class is present. With K classes examined the prior that
+ *                  nothing is wrong is (1 − π)^K — at π = 0.5 and K = 10 it
+ *                  is 0.001, so the noisy-OR blocks nearly everything.
+ *   'per-output' — π is the prior that the OUTPUT is bad (any class
+ *                  present); it is spread over the K examined classes as
+ *                  π_c = 1 − (1 − π)^(1/K), so that Π(1 − π_c) = 1 − π.
+ * Both are measured on the composite corpus so the choice is made on a
+ * number, not a preference.
+ */
+export type PriorMode = 'per-class' | 'per-output';
+export declare const DEFAULT_PRIOR_MODE: PriorMode;
+/** τ = 1 / (1 + c) with c = 1: a false pass costs the same as a false block. */
+export declare const DEFAULT_FALSE_PASS_COST = 1;
+export declare const DEFAULT_TAU: number;
+export interface RiskEstimate {
+    pBad: number;
+    lo: number;
+    hi: number;
+    perClass: Record<string, number | null>;
+    assumptions: string[];
+}
+export type RiskBasis = 'policy_gate' | 'detector_veto' | 'risk_over_loss' | 'clean' | 'no_rules';
+export interface RiskVerdict {
+    state: 'pass' | 'fail' | 'unknown';
+    basis: RiskBasis;
+    by: string[];
+    risk: RiskEstimate | null;
+    confidence: 'decisive' | 'marginal' | null;
+}
+interface Detector {
+    name: string;
+    classes: FailureClass[];
+    fired: boolean;
+    counts: {
+        tp: number;
+        fp: number;
+        fn: number;
+        tn: number;
+    };
+}
+/** The evaluated detections and inferences with a published family, one entry per rule. */
+export declare function detectorsOf(result: EvalResult): Detector[];
+/** The per-class prior under a mode, given how many classes the detectors examine. */
+export declare function classPrior(prior: number, mode: PriorMode, examinedClasses: number): number;
+/** p_bad with a 95% credible interval from the Beta posteriors of every detector's sensitivity and specificity. */
+export declare function riskEstimate(result: EvalResult, prior?: number, mode?: PriorMode): RiskEstimate | null;
+/**
+ * Compose by kind, as arc 3 will: gates (a failing policy that is effectively
+ * critical here), then vetoes (a failing effectively-critical detection),
+ * then the risk against τ. `unknown` when a critical rule was asked and could
+ * not answer (defeated or config_invalid) — the fail-closed seam.
+ */
+export declare function riskVerdict(result: EvalResult, tau?: number, prior?: number, mode?: PriorMode): RiskVerdict;
+export {};

package/dist/eval/risk.js

@@ -0,0 +1,187 @@
+import { publishedAccuracyFor } from './accuracy.js';
+import { PUBLISHED_ACCURACY_CORPUS_VERSION } from './published-accuracy.js';
+import { FAILURE_CLASS_IDS } from './failure-classes.js';
+import { fnv1a, mulberry32 } from './seeded-random.js';
+/** Jeffreys prior: half a count on each cell, so a family that made no mistakes does not claim certainty. */
+const HALF = 0.5;
+const sensOf = (d) => (d.counts.tp + HALF) / (d.counts.tp + d.counts.fn + 2 * HALF);
+const specOf = (d) => (d.counts.tn + HALF) / (d.counts.tn + d.counts.fp + 2 * HALF);
+export const RISK_DRAWS = 2000;
+export const DEFAULT_PRIOR = 0.5;
+export const DEFAULT_PRIOR_MODE = 'per-output';
+/** τ = 1 / (1 + c) with c = 1: a false pass costs the same as a false block. */
+export const DEFAULT_FALSE_PASS_COST = 1;
+export const DEFAULT_TAU = 1 / (1 + DEFAULT_FALSE_PASS_COST);
+function beta(a, b, rng) {
+    // Marsaglia–Tsang gamma with the shape < 1 boost; beta = ga / (ga + gb).
+    const gamma = (shape) => {
+        if (shape < 1)
+            return gamma(shape + 1) * Math.pow(rng(), 1 / shape);
+        const d = shape - 1 / 3;
+        const c = 1 / Math.sqrt(9 * d);
+        for (;;) {
+            let x;
+            let v;
+            do {
+                const u1 = rng();
+                const u2 = rng();
+                x = Math.sqrt(-2 * Math.log(u1 || 1e-12)) * Math.cos(2 * Math.PI * u2);
+                v = 1 + c * x;
+            } while (v <= 0);
+            v = v * v * v;
+            const u = rng();
+            if (u < 1 - 0.0331 * x * x * x * x)
+                return d * v;
+            if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v)))
+                return d * v;
+        }
+    };
+    const ga = gamma(a);
+    const gb = gamma(b);
+    return ga / (ga + gb);
+}
+/** The evaluated detections and inferences with a published family, one entry per rule. */
+export function detectorsOf(result) {
+    const out = [];
+    for (const r of result.rule_results) {
+        if (r.skipped)
+            continue;
+        if (r.kind !== 'detection' && r.kind !== 'inference')
+            continue;
+        const acc = publishedAccuracyFor(r.ruleName);
+        if (!acc)
+            continue;
+        out.push({
+            name: r.ruleName,
+            classes: (r.classes ?? []),
+            fired: r.passed === false,
+            counts: { tp: acc.tp, fp: acc.fp, fn: acc.fn, tn: acc.tn },
+        });
+    }
+    return out;
+}
+/** The per-class prior under a mode, given how many classes the detectors examine. */
+export function classPrior(prior, mode, examinedClasses) {
+    if (mode === 'per-class' || examinedClasses <= 1)
+        return prior;
+    return 1 - Math.pow(1 - prior, 1 / examinedClasses);
+}
+function pBadFrom(detectors, prior, mode, sensOf, specOf) {
+    const perClass = {};
+    let survive = 1;
+    const examinedClasses = FAILURE_CLASS_IDS.filter((cls) => detectors.some((d) => d.classes.includes(cls))).length;
+    const priorC = classPrior(prior, mode, examinedClasses);
+    for (const cls of FAILURE_CLASS_IDS) {
+        const examined = detectors.filter((d) => d.classes.includes(cls));
+        if (examined.length === 0) {
+            perClass[cls] = null;
+            continue;
+        }
+        const fired = examined.filter((d) => d.fired);
+        let q;
+        if (fired.length > 0) {
+            q = Math.max(...fired.map((d) => {
+                const s = sensOf(d);
+                const p = specOf(d);
+                const den = s * priorC + (1 - p) * (1 - priorC);
+                return den === 0 ? 0 : (s * priorC) / den;
+            }));
+        }
+        else {
+            let missAll = 1;
+            let specAll = 1;
+            for (const d of examined) {
+                missAll *= 1 - sensOf(d);
+                specAll *= specOf(d);
+            }
+            const den = priorC * missAll + (1 - priorC) * specAll;
+            q = den === 0 ? 0 : (priorC * missAll) / den;
+        }
+        perClass[cls] = q;
+        survive *= 1 - q;
+    }
+    return { pBad: 1 - survive, perClass };
+}
+const round4 = (x) => Math.round(x * 10_000) / 10_000;
+/** p_bad with a 95% credible interval from the Beta posteriors of every detector's sensitivity and specificity. */
+export function riskEstimate(result, prior = DEFAULT_PRIOR, mode = DEFAULT_PRIOR_MODE) {
+    const detectors = detectorsOf(result);
+    if (detectors.length === 0)
+        return null;
+    /*
+     * Jeffreys half-counts in the POINT estimate, not only in the draws.
+     *
+     * Twelve of the fifteen published families recorded zero false positives.
+     * With specificity exactly 1 the positive predictive value of a fire is
+     * exactly 1 at every prior, so p_bad read exactly 1.000 on 31 of the 111
+     * composite cases — the same overconfidence the credible interval was
+     * added to cure, reintroduced one layer down. A half-count on each cell
+     * says what thirty cases can actually support: no_silent_tool_failure's
+     * single-fire contribution moves from 1.000 to about 0.69, still well
+     * over the shipped threshold, and now with an interval that means
+     * something. Found by arc 2 while writing up the composer, not by
+     * reading it.
+     */
+    const point = pBadFrom(detectors, prior, mode, sensOf, specOf);
+    const rng = mulberry32(fnv1a(`risk:${PUBLISHED_ACCURACY_CORPUS_VERSION}:${mode}:${prior.toFixed(3)}:${detectors.map((d) => `${d.name}${d.fired ? '!' : ''}`).join(',')}`));
+    const draws = [];
+    for (let i = 0; i < RISK_DRAWS; i++) {
+        const sens = new Map();
+        const spec = new Map();
+        for (const d of detectors) {
+            sens.set(d.name, beta(d.counts.tp + 0.5, d.counts.fn + 0.5, rng));
+            spec.set(d.name, beta(d.counts.tn + 0.5, d.counts.fp + 0.5, rng));
+        }
+        draws.push(pBadFrom(detectors, prior, mode, (d) => sens.get(d.name), (d) => spec.get(d.name)).pBad);
+    }
+    draws.sort((a, b) => a - b);
+    const at = (q) => draws[Math.min(draws.length - 1, Math.max(0, Math.ceil(q * draws.length) - 1))];
+    // The point uses the observed rates; a rate at exactly 1 (no false positives
+    // in the family) puts the point above every posterior draw, so the interval
+    // is widened to contain it — an interval that excludes its own point is a
+    // presentation error, not a finding.
+    return {
+        pBad: round4(point.pBad),
+        lo: round4(Math.min(at(0.025), point.pBad)),
+        hi: round4(Math.max(at(0.975), point.pBad)),
+        perClass: Object.fromEntries(Object.entries(point.perClass).map(([k, v]) => [k, v === null ? null : round4(v)])),
+        assumptions: [
+            'detectors independent across classes',
+            'published accuracy is in-sample, same-model labelled',
+            'sensitivity and specificity carry a half-count prior, so a family with no observed errors does not read as certain',
+            `prior ${prior}, spread ${mode}`,
+        ],
+    };
+}
+const isEffectivelyCritical = (r) => r.critical === true;
+/**
+ * Compose by kind, as arc 3 will: gates (a failing policy that is effectively
+ * critical here), then vetoes (a failing effectively-critical detection),
+ * then the risk against τ. `unknown` when a critical rule was asked and could
+ * not answer (defeated or config_invalid) — the fail-closed seam.
+ */
+export function riskVerdict(result, tau = DEFAULT_TAU, prior = DEFAULT_PRIOR, mode = DEFAULT_PRIOR_MODE) {
+    const rows = result.rule_results;
+    const gates = rows.filter((r) => r.kind === 'policy' && !r.skipped && r.passed === false && isEffectivelyCritical(r));
+    if (gates.length > 0)
+        return { state: 'fail', basis: 'policy_gate', by: gates.map((r) => r.ruleName), risk: riskEstimate(result, prior, mode), confidence: null };
+    const vetoes = rows.filter((r) => (r.kind === 'detection' || r.kind === 'inference') && !r.skipped && r.passed === false && isEffectivelyCritical(r));
+    if (vetoes.length > 0)
+        return { state: 'fail', basis: 'detector_veto', by: vetoes.map((r) => r.ruleName), risk: riskEstimate(result, prior, mode), confidence: null };
+    const unknown = rows.filter((r) => isEffectivelyCritical(r) && r.skipped && r.skipClass && r.skipClass !== 'not_applicable');
+    if (unknown.length > 0)
+        return { state: 'unknown', basis: 'clean', by: unknown.map((r) => r.ruleName), risk: null, confidence: null };
+    const risk = riskEstimate(result, prior, mode);
+    if (!risk) {
+        const judged = rows.some((r) => !r.skipped);
+        return { state: judged ? 'pass' : 'unknown', basis: judged ? 'clean' : 'no_rules', by: [], risk: null, confidence: null };
+    }
+    const confidence = risk.lo <= tau && tau <= risk.hi ? 'marginal' : 'decisive';
+    if (risk.pBad > tau) {
+        const by = Object.entries(risk.perClass)
+            .filter(([, q]) => q !== null && q > 0.5)
+            .map(([cls]) => cls);
+        return { state: 'fail', basis: 'risk_over_loss', by, risk, confidence };
+    }
+    return { state: 'pass', basis: 'clean', by: [], risk, confidence };
+}

package/dist/eval/rules/completeness.js

@@ -1,8 +1,15 @@
+import { countSentences } from '../text/sentences.js';
 export const minOutputLength = {
     name: 'min_output_length',
     description: 'Output must meet a minimum character length',
     evalType: 'completeness',
     weight: 1,
+    kind: 'measurement',
+    mechanism: 'formula',
+    needs: ['output'],
+    question: 'complete',
+    classes: ['format'],
+    version: 1,
     evaluate(context) {
         const minLen = context.customConfig?.min_output_length
             ?? context.customConfig?.min_length
@@ -13,6 +20,8 @@ export const minOutputLength = {
             ruleName: 'min_output_length',
             passed,
             score: passed ? 1 : Math.min(len / minLen, 0.99),
+            value: { stat: 'length', unit: 'chars', value: len },
+            evidence: [{ type: 'count', stat: 'length', unit: 'chars', value: len, threshold: minLen, thresholdSource: minLen === 50 ? 'default' : 'config' }],
             message: passed ? `Output length (${len}) meets minimum (${minLen})` : `Output length (${len}) below minimum (${minLen})`,
         };
     },
@@ -22,12 +31,19 @@ export const nonEmptyOutput = {
     description: 'Output must not be empty or whitespace-only',
     evalType: 'completeness',
     weight: 2,
+    kind: 'policy',
+    mechanism: 'formula',
+    needs: ['output'],
+    question: 'complete',
+    classes: ['format'],
+    version: 1,
     evaluate(context) {
         const passed = context.output.trim().length > 0;
         return {
             ruleName: 'non_empty_output',
             passed,
             score: passed ? 1 : 0,
+            value: { stat: 'length', unit: 'chars', value: context.output.trim().length },
             message: passed ? 'Output is non-empty' : 'Output is empty or whitespace-only',
         };
     },
@@ -37,14 +53,25 @@ export const sentenceCount = {
     description: 'Output must contain a minimum number of sentences',
     evalType: 'completeness',
     weight: 0.5,
+    kind: 'measurement',
+    mechanism: 'formula',
+    needs: ['output'],
+    question: 'complete',
+    classes: ['format'],
+    version: 1,
     evaluate(context) {
         const minSentences = context.customConfig?.min_sentences ?? 2;
-        const sentences = context.output.split(/[.!?]+/).filter((s) => s.trim().length > 0).length;
+        // One splitter, shared with topic_consistency (src/eval/text/sentences.ts).
+        // Splitting on /[.!?]+/ counted "The latency is 3.5 seconds." as two
+        // sentences and "Dr. Chen approved it." as two more.
+        const sentences = countSentences(context.output);
         const passed = sentences >= minSentences;
         return {
             ruleName: 'sentence_count',
             passed,
             score: passed ? 1 : Math.min(sentences / minSentences, 0.99),
+            value: { stat: 'sentences', unit: 'sentences', value: sentences },
+            evidence: [{ type: 'count', stat: 'sentences', unit: 'sentences', value: sentences, threshold: minSentences, thresholdSource: minSentences === 2 ? 'default' : 'config' }],
             message: passed ? `Sentence count (${sentences}) meets minimum (${minSentences})` : `Sentence count (${sentences}) below minimum (${minSentences})`,
         };
     },
@@ -54,6 +81,12 @@ export const expectedCoverage = {
     description: 'Output must cover key terms from expected output',
     evalType: 'completeness',
     weight: 1.5,
+    kind: 'measurement',
+    mechanism: 'formula',
+    needs: ['output', 'expected'],
+    question: 'complete',
+    classes: ['incomplete_ask'],
+    version: 1,
     evaluate(context) {
         if (!context.expected) {
             return { ruleName: 'expected_coverage', passed: false, score: 0, message: 'No expected output provided', skipped: true, skipReason: 'context.expected not provided' };
@@ -74,6 +107,8 @@ export const expectedCoverage = {
             ruleName: 'expected_coverage',
             passed,
             score: ratio,
+            value: { stat: 'expected_terms_covered', unit: 'ratio', value: ratio },
+            evidence: [{ type: 'count', stat: 'expected_terms_covered', unit: 'ratio', value: ratio, threshold: 0.5, thresholdSource: 'rule' }],
             message: `Covered ${covered}/${expectedWords.size} expected terms (${(ratio * 100).toFixed(0)}%)`,
         };
     },

package/dist/eval/rules/cost.d.ts

@@ -1,6 +1,6 @@
-import type { EvalRule } from '../../types/eval.js';
+import { type EvalRule } from '../../types/eval.js';
 export declare const costUnderThreshold: EvalRule;
-export declare const tokenEfficiency: EvalRule;
+export declare const verbosityRatio: EvalRule;
 /** Default for config key `max_tool_repeats`: how many identical calls are tolerated. */
 export declare const DEFAULT_MAX_TOOL_REPEATS = 3;
 /**

package/dist/eval/rules/cost.js

@@ -1,9 +1,16 @@
+import { MAX_EVIDENCE_ITEMS } from '../../types/eval.js';
 import { callKey, describeInput, skipWithoutTrajectory } from './trajectory.js';
 export const costUnderThreshold = {
     name: 'cost_under_threshold',
     description: 'Total cost must be under a configurable USD threshold',
     evalType: 'cost',
     weight: 1,
+    kind: 'policy',
+    mechanism: 'formula',
+    needs: ['cost'],
+    question: 'within_budget',
+    classes: ['over_budget'],
+    version: 1,
     evaluate(context) {
         if (context.costUsd === undefined || context.costUsd === null) {
             return { ruleName: 'cost_under_threshold', passed: false, score: 0, message: 'Cost data not provided', skipped: true, skipReason: 'context.costUsd not provided' };
@@ -15,30 +22,40 @@ export const costUnderThreshold = {
             ruleName: 'cost_under_threshold',
             passed,
             score: passed ? 1 : Math.max(0, 1 - (cost - threshold) / threshold),
+            value: { stat: 'cost', unit: 'usd', value: cost },
+            evidence: [{ type: 'count', stat: 'cost', unit: 'usd', value: cost, threshold, thresholdSource: threshold === 0.10 ? 'default' : 'config' }],
             message: passed
                 ? `Cost ($${cost.toFixed(4)}) is under threshold ($${threshold.toFixed(4)})`
                 : `Cost ($${cost.toFixed(4)}) exceeds threshold ($${threshold.toFixed(4)})`,
         };
     },
 };
-export const tokenEfficiency = {
-    name: 'token_efficiency',
-    description: 'Checks output-to-input token ratio for efficiency',
+export const verbosityRatio = {
+    name: 'verbosity_ratio',
+    description: 'The completion-to-prompt token ratio against a ceiling: completion_tokens / prompt_tokens must not exceed max_token_ratio (default 5). This measures output VERBOSITY relative to prompt size, not efficiency — a long answer to a long prompt passes and a long answer to a short prompt fails, and neither says whether the tokens were well spent. Skipped when token usage is not supplied. Renamed from token_efficiency in 0.10.0, because the old name named something the rule does not measure',
     evalType: 'cost',
     weight: 0.5,
+    kind: 'measurement',
+    mechanism: 'formula',
+    needs: ['tokens'],
+    question: 'within_budget',
+    classes: ['over_budget'],
+    version: 1,
     evaluate(context) {
         const prompt = context.tokenUsage?.prompt_tokens;
         const completion = context.tokenUsage?.completion_tokens;
         if (prompt === undefined || completion === undefined || prompt === 0) {
-            return { ruleName: 'token_efficiency', passed: false, score: 0, message: 'Token usage not provided', skipped: true, skipReason: 'context.tokenUsage not provided' };
+            return { ruleName: 'verbosity_ratio', passed: false, score: 0, message: 'Token usage not provided', skipped: true, skipReason: 'context.tokenUsage not provided' };
         }
         const ratio = completion / prompt;
         const maxRatio = context.customConfig?.max_token_ratio ?? 5;
         const passed = ratio <= maxRatio;
         return {
-            ruleName: 'token_efficiency',
+            ruleName: 'verbosity_ratio',
             passed,
             score: passed ? 1 : Math.max(0, 1 - (ratio - maxRatio) / maxRatio),
+            value: { stat: 'completion_to_prompt_ratio', unit: 'ratio', value: ratio },
+            evidence: [{ type: 'count', stat: 'completion_to_prompt_ratio', unit: 'ratio', value: ratio, threshold: maxRatio, thresholdSource: maxRatio === 5 ? 'default' : 'config' }],
             message: passed
                 ? `Token ratio (${ratio.toFixed(2)}) is within limits (max ${maxRatio})`
                 : `Token ratio (${ratio.toFixed(2)}) exceeds max (${maxRatio})`,
@@ -91,6 +108,12 @@ export const noToolLoop = {
     description: 'The agent must not repeat itself. Fails when one tool is called with an identical input (object keys sorted, whitespace collapsed) more than max_tool_repeats times — default 3, config key `max_tool_repeats` — or when two calls alternate for more than two complete A,B,A,B cycles. Skips when no tool calls are provided, so an evaluation with no trajectory reports "not judged" rather than clean. Catches the wasted spend a cost threshold cannot see: five identical calls can still bill under a per-evaluation cost limit',
     evalType: 'cost',
     weight: 1,
+    kind: 'detection',
+    mechanism: 'formula',
+    needs: ['tool_calls'],
+    question: 'tool_use_correct',
+    classes: ['tool_loop'],
+    version: 1,
     evaluate(context) {
         const skip = skipWithoutTrajectory('no_tool_loop', context);
         if (skip)
@@ -112,12 +135,22 @@ export const noToolLoop = {
                 worstCount = count;
             }
         }
+        const repeatsEvidence = [
+            { type: 'count', stat: 'max_repeats_of_one_call', unit: 'calls', value: worstCount, threshold: maxRepeats, thresholdSource: maxRepeats === DEFAULT_MAX_TOOL_REPEATS ? 'default' : 'config' },
+        ];
         if (worstCount > maxRepeats) {
             const call = calls[keys.indexOf(worstKey)];
+            const evidence = [...repeatsEvidence];
+            keys.forEach((key, index) => {
+                if (key === worstKey && evidence.length < MAX_EVIDENCE_ITEMS)
+                    evidence.push({ type: 'toolCall', index, toolName: calls[index].tool_name, label: 'repeated call' });
+            });
             return {
                 ruleName: 'no_tool_loop',
                 passed: false,
                 score: Math.max(0, 1 - (worstCount - maxRepeats) * 0.25),
+                value: { stat: 'max_repeats_of_one_call', unit: 'calls', value: worstCount },
+                evidence,
                 message: `Tool loop: ${call.tool_name} called ${worstCount} times with the same input — ${describeInput(call.input)} — over ${calls.length} call${calls.length === 1 ? '' : 's'} (max ${maxRepeats})`,
             };
         }
@@ -125,10 +158,19 @@ export const noToolLoop = {
         if (cycle !== null && cycle.cycles > MAX_TWO_CALL_CYCLES) {
             const a = calls[keys.indexOf(cycle.a)];
             const b = calls[keys.indexOf(cycle.b)];
+            const evidence = [
+                { type: 'count', stat: 'two_call_cycles', unit: 'cycles', value: cycle.cycles, threshold: MAX_TWO_CALL_CYCLES, thresholdSource: 'rule' },
+            ];
+            keys.forEach((key, index) => {
+                if ((key === cycle.a || key === cycle.b) && evidence.length < MAX_EVIDENCE_ITEMS)
+                    evidence.push({ type: 'toolCall', index, toolName: calls[index].tool_name, label: 'alternating call' });
+            });
             return {
                 ruleName: 'no_tool_loop',
                 passed: false,
                 score: Math.max(0, 1 - (cycle.cycles - MAX_TWO_CALL_CYCLES) * 0.25),
+                value: { stat: 'two_call_cycles', unit: 'cycles', value: cycle.cycles },
+                evidence,
                 message: `Tool loop: ${a.tool_name} (${describeInput(a.input)}) and ${b.tool_name} (${describeInput(b.input)}) alternate for ${cycle.cycles} cycles (max ${MAX_TWO_CALL_CYCLES})`,
             };
         }
@@ -137,7 +179,9 @@ export const noToolLoop = {
             passed: true,
             score: 1,
             message: `No repeated tool call (${calls.length} call${calls.length === 1 ? '' : 's'}; most repeated ran ${worstCount}×, max ${maxRepeats})`,
+            value: { stat: 'max_repeats_of_one_call', unit: 'calls', value: worstCount },
+            evidence: repeatsEvidence,
         };
     },
 };
-export const costRules = [costUnderThreshold, tokenEfficiency, noToolLoop];
+export const costRules = [costUnderThreshold, verbosityRatio, noToolLoop];