@iris-eval/mcp-server 0.6.0 → 0.8.0

package/dist/eval/rules/cost.js

@@ -1,3 +1,4 @@
+import { callKey, describeInput, skipWithoutTrajectory } from './trajectory.js';
 export const costUnderThreshold = {
     name: 'cost_under_threshold',
     description: 'Total cost must be under a configurable USD threshold',
@@ -44,4 +45,99 @@ export const tokenEfficiency = {
         };
     },
 };
-export const costRules = [costUnderThreshold, tokenEfficiency];
+/** Default for config key `max_tool_repeats`: how many identical calls are tolerated. */
+export const DEFAULT_MAX_TOOL_REPEATS = 3;
+/**
+ * How many complete A,B,A,B cycles are tolerated before the alternation is
+ * a loop. Three cycles is six consecutive calls that between them made two
+ * distinct requests.
+ */
+export const MAX_TWO_CALL_CYCLES = 2;
+/** The longest run of alternating A,B calls, and how many complete cycles it holds. */
+function longestTwoCallCycle(keys) {
+    let best = null;
+    for (let start = 0; start + 3 < keys.length; start++) {
+        const a = keys[start];
+        const b = keys[start + 1];
+        if (a === b)
+            continue;
+        let len = 2;
+        while (start + len < keys.length && keys[start + len] === (len % 2 === 0 ? a : b))
+            len++;
+        const cycles = Math.floor(len / 2);
+        if (cycles >= 2 && (best === null || cycles > best.cycles))
+            best = { a, b, cycles };
+    }
+    return best;
+}
+/*
+ * The other half of what the trajectory shows: not a wrong answer, a wasted
+ * one.
+ *
+ * Transcript t-16 answers the question correctly, and gets there by running
+ * the identical `ls src/tools` five times with five identical results. Four
+ * of those turns bought nothing, and each one resent the whole context —
+ * 18,918 prompt tokens for a directory listing. cost_under_threshold cannot
+ * see it: the bill still comes to $0.0621, well under the $0.10 default. The
+ * loop is only visible in the sequence of calls.
+ *
+ * Cost, not completeness or safety, because that is where the harm lands —
+ * spend and latency, on an answer that was already available. Non-critical
+ * for the same reason: a repetitive agent is wasteful, not unsafe, and
+ * `passed` should not be vetoed by a behavioural signal.
+ */
+export const noToolLoop = {
+    name: 'no_tool_loop',
+    description: 'The agent must not repeat itself. Fails when one tool is called with an identical input (object keys sorted, whitespace collapsed) more than max_tool_repeats times — default 3, config key `max_tool_repeats` — or when two calls alternate for more than two complete A,B,A,B cycles. Skips when no tool calls are provided, so an evaluation with no trajectory reports "not judged" rather than clean. Catches the wasted spend a cost threshold cannot see: five identical calls can still bill under a per-evaluation cost limit',
+    evalType: 'cost',
+    weight: 1,
+    evaluate(context) {
+        const skip = skipWithoutTrajectory('no_tool_loop', context);
+        if (skip)
+            return skip;
+        const calls = context.toolCalls ?? [];
+        const configured = context.customConfig?.max_tool_repeats;
+        const maxRepeats = typeof configured === 'number' && Number.isFinite(configured) && configured >= 1
+            ? Math.floor(configured)
+            : DEFAULT_MAX_TOOL_REPEATS;
+        const keys = calls.map(callKey);
+        const counts = new Map();
+        for (const key of keys)
+            counts.set(key, (counts.get(key) ?? 0) + 1);
+        let worstKey = '';
+        let worstCount = 0;
+        for (const [key, count] of counts) {
+            if (count > worstCount) {
+                worstKey = key;
+                worstCount = count;
+            }
+        }
+        if (worstCount > maxRepeats) {
+            const call = calls[keys.indexOf(worstKey)];
+            return {
+                ruleName: 'no_tool_loop',
+                passed: false,
+                score: Math.max(0, 1 - (worstCount - maxRepeats) * 0.25),
+                message: `Tool loop: ${call.tool_name} called ${worstCount} times with the same input — ${describeInput(call.input)} — over ${calls.length} call${calls.length === 1 ? '' : 's'} (max ${maxRepeats})`,
+            };
+        }
+        const cycle = longestTwoCallCycle(keys);
+        if (cycle !== null && cycle.cycles > MAX_TWO_CALL_CYCLES) {
+            const a = calls[keys.indexOf(cycle.a)];
+            const b = calls[keys.indexOf(cycle.b)];
+            return {
+                ruleName: 'no_tool_loop',
+                passed: false,
+                score: Math.max(0, 1 - (cycle.cycles - MAX_TWO_CALL_CYCLES) * 0.25),
+                message: `Tool loop: ${a.tool_name} (${describeInput(a.input)}) and ${b.tool_name} (${describeInput(b.input)}) alternate for ${cycle.cycles} cycles (max ${MAX_TWO_CALL_CYCLES})`,
+            };
+        }
+        return {
+            ruleName: 'no_tool_loop',
+            passed: true,
+            score: 1,
+            message: `No repeated tool call (${calls.length} call${calls.length === 1 ? '' : 's'}; most repeated ran ${worstCount}×, max ${maxRepeats})`,
+        };
+    },
+};
+export const costRules = [costUnderThreshold, tokenEfficiency, noToolLoop];

package/dist/eval/rules/relevance.d.ts

@@ -1,4 +1,17 @@
 import type { EvalRule } from '../../types/eval.js';
+/**
+ * Light stemmer: plurals, -ing/-ed/-ly, -ation/-ator/-ate/-ion, a trailing
+ * e, and a doubled final consonant. Crude on purpose (see the header):
+ * both sides are stemmed identically.
+ */
+export declare function stemTerm(word: string): string;
+/**
+ * Content terms of a text: fenced code removed, camelCase split, everything
+ * that is not a run of three or more letters treated as a separator (so
+ * paths, flags, snake_case and dotted identifiers fall apart into their
+ * words and numbers vanish), stopwords dropped, the rest stemmed.
+ */
+export declare function contentTerms(text: string): string[];
 export declare const keywordOverlap: EvalRule;
 export declare const topicConsistency: EvalRule;
 export declare const relevanceRules: EvalRule[];

package/dist/eval/rules/relevance.js

@@ -1,30 +1,154 @@
+/*
+ * Relevance rules — one tokenizer, two DISTINCT signals.
+ *
+ * Redesigned after the arc-one acceptance pass ran twenty-four transcripts
+ * produced by an agent genuinely working against this repository
+ * (tests/fixtures/real-transcripts/). Three grounded, correct technical
+ * answers — what `--purge` does, what `eval_type: "all"` returns, a
+ * one-paragraph product description — failed topic_consistency at
+ * 6.7% / 3.6% / 2.0%. The old measure was the fraction of OUTPUT words that
+ * also appear in the INPUT, which punishes precisely what a good technical
+ * answer does: bring the source's vocabulary (identifiers, file names,
+ * exact values, mechanism) to a short question that did not contain it.
+ * The failure was structural — no threshold rescues a measure that reads
+ * new, correct vocabulary as drift — so the measure changed, not the number.
+ *
+ *   keyword_overlap   RECALL. What share of the ask's content terms does the
+ *                     output engage at all? Fails a refusal, a different
+ *                     product, filler, an answer that never touches the
+ *                     subject.
+ *   topic_consistency CONTINUITY. What share of the output's content-
+ *                     bearing sentences connect to the ask — directly, or
+ *                     through an earlier connected sentence? Fails an answer
+ *                     that opens on topic and wanders, and everything
+ *                     keyword_overlap fails. Grounded answers chain their
+ *                     vocabulary back to the ask; a ramble does not.
+ *
+ * Both rules share the tokenizer below, so they agree on what a "term" is
+ * and no longer double-count one measurement:
+ *   - stopwords (articles, pronouns, auxiliaries, question words, the
+ *     request verbs — "explain", "summarise", "tell me" — and the form of
+ *     the deliverable — "paragraph", "bullets", "summary") are not terms;
+ *   - code identifiers, paths and flags are SPLIT into their words
+ *     (`EvalEngine.evaluateAll()` → eval, engine, evaluate; `src/index.ts`
+ *     → src, index) rather than dropped: the words inside an identifier
+ *     ARE topic vocabulary, and dropping them was half of the old failure;
+ *   - numbers and fenced code blocks are neutral (neither for nor against);
+ *   - a light stemmer folds inflections (purge/purged/purging, rule/rules,
+ *     evaluate/evaluation/evaluator) so the same word in a different form
+ *     still counts. It is deliberately crude — both sides get the same
+ *     treatment, so an imperfect stem only lowers sensitivity, never
+ *     invents a match.
+ *
+ * Honest limits (lexical, no model): an answer that paraphrases the ask
+ * with none of its words reads as off topic; a coherent essay on the wrong
+ * subject that happens to reuse one of the ask's words reads as on topic.
+ * Semantic relevance is the LLM judge's job (evaluate_with_llm_judge,
+ * `relevance` template).
+ */
+const STOPWORDS = new Set(('a an the and or nor but if then else than that this these those there here it its is are was were be been being ' +
+    'am do does did done doing have has had having will would shall should can could may might must not no yes of in on ' +
+    'at to for from by with without into onto over under about above below between among through during before after ' +
+    'again further once out off up down as so such very really just only also too either neither both each every all any ' +
+    'some few more most less least other another same own new old first second third next last one two three four five ' +
+    'ten i me my mine we us our ours you your yours he him his she her hers they them their theirs who whom whose which ' +
+    'what when where why how because while until unless since although though even ever never always often sometimes ' +
+    'usually still yet already now anywhere everywhere something anything nothing everything someone anyone everyone ' +
+    'nobody thing things way ways kind kinds sort sorts lot lots much many get gets got getting give gives gave given ' +
+    'giving take takes took taken taking make makes made making use uses used using see sees saw seen seeing know knows ' +
+    'knew known knowing think thinks thought thinking want wants wanted wanting need needs needed needing let lets tell ' +
+    'tells told telling say says said saying ask asks asked asking read reads reading look looks looked looking find ' +
+    'finds found finding show shows showed shown showing explain explains explained explaining describe describes ' +
+    'described describing summarise summarize summarises summarizes summarised summarized answer answers answered ' +
+    'answering question questions please help helps helped helping like likes liked well good bad better best right ' +
+    'wrong true false able keep keeps kept put puts go goes went gone going come comes came coming back also etc via per ' +
+    // The FORM of the deliverable, not its subject — "a one-paragraph
+    // description", "a few bullets", "a short summary", "in detail".
+    'paragraph paragraphs sentence sentences bullet bullets summary overview description brief briefly detail details ' +
+    'detailed word words line lines short long quick quickly ' +
+    // URL and domain furniture — "iris-eval.com" splits into iris, eval, com.
+    'com org net www http https').split(' '));
+/**
+ * Light stemmer: plurals, -ing/-ed/-ly, -ation/-ator/-ate/-ion, a trailing
+ * e, and a doubled final consonant. Crude on purpose (see the header):
+ * both sides are stemmed identically.
+ */
+export function stemTerm(word) {
+    let w = word;
+    if (w.length <= 3)
+        return w;
+    if (w.endsWith('ies'))
+        w = w.slice(0, -3) + 'i';
+    else if (w.endsWith('sses'))
+        w = w.slice(0, -2);
+    else if (w.endsWith('s') && !/(?:ss|us|is)$/.test(w))
+        w = w.slice(0, -1);
+    if (w.length > 5 && w.endsWith('ing'))
+        w = w.slice(0, -3);
+    else if (w.length > 4 && w.endsWith('ed'))
+        w = w.slice(0, -2);
+    else if (w.length > 4 && w.endsWith('ly'))
+        w = w.slice(0, -2);
+    else if (w.length > 6 && w.endsWith('ation'))
+        w = w.slice(0, -5);
+    else if (w.length > 5 && w.endsWith('ator'))
+        w = w.slice(0, -4);
+    else if (w.length > 5 && w.endsWith('ate'))
+        w = w.slice(0, -3);
+    else if (w.length > 5 && w.endsWith('ion'))
+        w = w.slice(0, -3);
+    if (w.length > 3 && w.endsWith('e'))
+        w = w.slice(0, -1);
+    if (w.length > 3 && /([^aeiou])\1$/.test(w) && !/[lsz]$/.test(w))
+        w = w.slice(0, -1);
+    return w;
+}
+const FENCED_CODE = /```[\s\S]*?```/g;
+const CAMEL_BOUNDARY = /([a-z])([A-Z])/g;
+const WORD = /[a-z]{3,}/g;
+/**
+ * Content terms of a text: fenced code removed, camelCase split, everything
+ * that is not a run of three or more letters treated as a separator (so
+ * paths, flags, snake_case and dotted identifiers fall apart into their
+ * words and numbers vanish), stopwords dropped, the rest stemmed.
+ */
+export function contentTerms(text) {
+    const terms = [];
+    const lowered = text.replace(FENCED_CODE, '\n').replace(CAMEL_BOUNDARY, '$1 $2').toLowerCase();
+    for (const match of lowered.matchAll(WORD)) {
+        if (STOPWORDS.has(match[0]))
+            continue;
+        terms.push(stemTerm(match[0]));
+    }
+    return terms;
+}
 export const keywordOverlap = {
     name: 'keyword_overlap',
-    description: 'Measures word overlap between input and output',
+    description: 'Recall of the input\'s content terms in the output: stopwords and request verbs are not terms, code identifiers and paths are split into their words, inflections are folded (purge/purged/purging). Passes when at least 35% of the input\'s terms appear in the output (configurable: keyword_overlap)',
     evalType: 'relevance',
     weight: 1,
     evaluate(context) {
         if (!context.input) {
             return { ruleName: 'keyword_overlap', passed: false, score: 0, message: 'No input provided', skipped: true, skipReason: 'context.input not provided' };
         }
-        const inputWords = new Set(context.input.toLowerCase().split(/\W+/).filter((w) => w.length > 2));
-        const outputWords = new Set(context.output.toLowerCase().split(/\W+/).filter((w) => w.length > 2));
-        if (inputWords.size === 0) {
+        const inputTerms = new Set(contentTerms(context.input));
+        if (inputTerms.size === 0) {
             return { ruleName: 'keyword_overlap', passed: true, score: 1, message: 'No meaningful words in input' };
         }
+        const outputTerms = new Set(contentTerms(context.output));
         let overlap = 0;
-        for (const word of inputWords) {
-            if (outputWords.has(word))
+        for (const term of inputTerms) {
+            if (outputTerms.has(term))
                 overlap++;
         }
-        const ratio = overlap / inputWords.size;
+        const ratio = overlap / inputTerms.size;
         const threshold = context.customConfig?.keyword_overlap ?? 0.35;
         const passed = ratio >= threshold;
         return {
             ruleName: 'keyword_overlap',
             passed,
             score: Math.min(ratio * 2, 1),
-            message: `${overlap}/${inputWords.size} input keywords found in output (${(ratio * 100).toFixed(0)}%)`,
+            message: `${overlap}/${inputTerms.size} input keywords found in output (${(ratio * 100).toFixed(0)}%)`,
         };
     },
 };
@@ -35,9 +159,22 @@ export const keywordOverlap = {
  * the dashboard's safety-violations panel, and the storage adapter's
  * violation counts have always placed it.
  */
+/**
+ * A third of the content-bearing sentences must connect. Why a third and
+ * not half: the measure is a floor against drift, not a target — grounded
+ * answers in the real-transcript set connect 67–100% of their sentences —
+ * and the false positive that matters is the SHORT honest answer whose
+ * second and third sentences elaborate in fresh words ("It is sunny today.
+ * Expect a high of 75°F. Bring sunglasses."). At a half that answer fails;
+ * at a third it passes while "one on-topic sentence, then three about
+ * something else" (25%) still fails.
+ */
+const DEFAULT_TOPIC_THRESHOLD = 1 / 3;
+const LIST_ITEM = /^\s*(?:[-*+•]|\d{1,3}[.)])\s+/;
+const SENTENCE_BREAK = /(?<=[.!?])\s+/;
 export const topicConsistency = {
     name: 'topic_consistency',
-    description: 'Output stays on topic relative to input (skipped when output too brief for meaningful comparison)',
+    description: 'Continuity with the input: the share of the output\'s content-bearing sentences that connect to the input\'s topic — a sentence connects when it shares a content term with the input or with an earlier connected sentence (list items are read under the sentence that introduces them). Passes when at least a third connect (configurable: topic_consistency); a third, not half, so a short honest answer that elaborates in fresh words is not read as drift. Replaces the output-word-ratio measure that failed every grounded technical answer. Skipped when the output is too brief for meaningful comparison',
     evalType: 'relevance',
     weight: 1,
     evaluate(context) {
@@ -49,9 +186,8 @@ export const topicConsistency = {
         if (inputWords.length === 0 || outputWords.length === 0) {
             return { ruleName: 'topic_consistency', passed: false, score: 0, message: 'Insufficient text for topic analysis', skipped: true, skipReason: 'input or output has no words > 3 chars' };
         }
-        // v0.3.1 fix: skip when output is too brief — short outputs (1-5 words >3 chars)
-        // produce noisy ratios where the threshold can't meaningfully discriminate.
-        // The previous version over-triggered as a false-positive on brief but valid responses.
+        // v0.3.1: skip when the output is too brief — a handful of words cannot
+        // be judged on topic or off it, and the rule used to cry wolf there.
         const minOutputWords = context.customConfig?.topic_consistency_min_words ?? 6;
         if (outputWords.length < minOutputWords) {
             return {
@@ -63,20 +199,48 @@ export const topicConsistency = {
                 skipReason: `output has < ${minOutputWords} words ≥ 4 chars`,
             };
         }
-        const inputSet = new Set(inputWords);
-        let relevant = 0;
-        for (const word of outputWords) {
-            if (inputSet.has(word))
-                relevant++;
+        const topic = new Set(contentTerms(context.input));
+        if (topic.size === 0) {
+            return { ruleName: 'topic_consistency', passed: false, score: 0, message: 'Insufficient text for topic analysis', skipped: true, skipReason: 'input has no content terms' };
+        }
+        // Walk the output line by line so list items can be read under their
+        // lead-in, and sentence by sentence within a line. `seen` is the topic
+        // so far: the input's terms plus every connected sentence's terms.
+        const seen = new Set(topic);
+        let sentences = 0;
+        let connected = 0;
+        let leadInConnected = false;
+        for (const line of context.output.replace(FENCED_CODE, '\n').split('\n')) {
+            const isItem = LIST_ITEM.test(line);
+            let lineConnected = false;
+            for (const sentence of line.split(SENTENCE_BREAK)) {
+                const terms = contentTerms(sentence);
+                if (terms.length === 0)
+                    continue;
+                sentences++;
+                const hit = terms.some((t) => seen.has(t)) || (isItem && leadInConnected);
+                if (hit) {
+                    connected++;
+                    lineConnected = true;
+                    for (const t of terms)
+                        seen.add(t);
+                }
+            }
+            if (!isItem && line.trim().length > 0)
+                leadInConnected = lineConnected;
+        }
+        if (sentences === 0) {
+            return { ruleName: 'topic_consistency', passed: false, score: 0, message: 'Insufficient text for topic analysis', skipped: true, skipReason: 'output has no content terms' };
         }
-        const ratio = relevant / outputWords.length;
-        const threshold = context.customConfig?.topic_consistency ?? 0.10;
+        const ratio = connected / sentences;
+        const threshold = context.customConfig?.topic_consistency ?? DEFAULT_TOPIC_THRESHOLD;
         const passed = ratio >= threshold;
         return {
             ruleName: 'topic_consistency',
             passed,
-            score: Math.min(ratio * 5, 1),
-            message: `Topic consistency: ${(ratio * 100).toFixed(1)}% of output words relate to input`,
+            // Full marks at two thirds connected; proportional below.
+            score: Math.min(ratio * 1.5, 1),
+            message: `Topic consistency: ${connected}/${sentences} content sentences connect to the input's topic (${(ratio * 100).toFixed(0)}%)`,
         };
     },
 };

package/dist/eval/rules/safety.d.ts

@@ -9,12 +9,23 @@ export declare const PII_PATTERNS: Array<{
  * with the count and the pattern names (#370): a builder smoke-testing with
  * `bob@example.com` or a 555 number used to read a bare "No PII detected"
  * and conclude detection was broken, when the rule had recognised the
- * value as documentation on purpose.
+ * value as documentation on purpose. Reserved IP addresses get their own
+ * clause in the same style — they are not documentation values, they are
+ * addresses that cannot identify anyone.
  */
 export declare function describeSuppressedPlaceholders(suppressed: Map<string, number>): string;
 export declare const noPii: EvalRule;
 export declare const noBlocklistWords: EvalRule;
 export declare const INJECTION_PATTERNS: RegExp[];
+/**
+ * The one true sentence about what the injection rule looks at. It is
+ * shared verbatim by the rule description, the evaluate_output tool
+ * description and the docs (a drift-lock test pins every copy), because
+ * the tool used to sell unscoped "prompt injection" while the rule reads
+ * `context.output` and nothing else — a builder could reasonably take it
+ * for an input firewall, which it is not.
+ */
+export declare const INJECTION_SCOPE_SENTENCE = "no_injection_patterns inspects the agent's OUTPUT text for injection-shaped content \u2014 attack phrasing and structural directives the output echoes or complies with \u2014 and never reads the input, so it is not an input firewall.";
 export declare const noInjectionPatterns: EvalRule;
 export declare const noStubOutput: EvalRule;
 export interface HallucinationSignal {
@@ -26,4 +37,5 @@ export interface HallucinationSignal {
 }
 export declare const HALLUCINATION_MARKERS: ReadonlyArray<HallucinationSignal>;
 export declare const noHallucinationMarkers: EvalRule;
+export declare const noSilentToolFailure: EvalRule;
 export declare const safetyRules: EvalRule[];