@iris-eval/mcp-server 0.8.1 → 0.9.0

package/dist/eval/published-accuracy.d.ts

@@ -0,0 +1,230 @@
+export declare const PUBLISHED_ACCURACY_CORPUS_VERSION = "1251d242916c";
+export declare const PUBLISHED_ACCURACY_RELEASE = "0.9.0";
+export declare const PUBLISHED_ACCURACY_LABELLING: "same-model";
+export declare const PUBLISHED_ACCURACY: {
+    readonly min_output_length: {
+        readonly n: 29;
+        readonly tp: 13;
+        readonly fp: 0;
+        readonly fn: 0;
+        readonly tn: 16;
+        readonly precision: 1;
+        readonly recall: 1;
+        readonly f1: 1;
+        readonly ci95: {
+            readonly precision: readonly [0.7719, 1];
+            readonly recall: readonly [0.7719, 1];
+            readonly f1: readonly [1, 1];
+        };
+    };
+    readonly non_empty_output: {
+        readonly n: 29;
+        readonly tp: 10;
+        readonly fp: 0;
+        readonly fn: 2;
+        readonly tn: 17;
+        readonly precision: 1;
+        readonly recall: 0.8333;
+        readonly f1: 0.9091;
+        readonly ci95: {
+            readonly precision: readonly [0.7225, 1];
+            readonly recall: readonly [0.552, 0.953];
+            readonly f1: readonly [0.7368, 1];
+        };
+    };
+    readonly sentence_count: {
+        readonly n: 30;
+        readonly tp: 8;
+        readonly fp: 0;
+        readonly fn: 6;
+        readonly tn: 16;
+        readonly precision: 1;
+        readonly recall: 0.5714;
+        readonly f1: 0.7273;
+        readonly ci95: {
+            readonly precision: readonly [0.6756, 1];
+            readonly recall: readonly [0.3259, 0.7862];
+            readonly f1: readonly [0.4706, 0.9091];
+        };
+    };
+    readonly expected_coverage: {
+        readonly n: 29;
+        readonly tp: 14;
+        readonly fp: 0;
+        readonly fn: 0;
+        readonly tn: 15;
+        readonly precision: 1;
+        readonly recall: 1;
+        readonly f1: 1;
+        readonly ci95: {
+            readonly precision: readonly [0.7847, 1];
+            readonly recall: readonly [0.7847, 1];
+            readonly f1: readonly [1, 1];
+        };
+    };
+    readonly keyword_overlap: {
+        readonly n: 30;
+        readonly tp: 10;
+        readonly fp: 0;
+        readonly fn: 0;
+        readonly tn: 20;
+        readonly precision: 1;
+        readonly recall: 1;
+        readonly f1: 1;
+        readonly ci95: {
+            readonly precision: readonly [0.7225, 1];
+            readonly recall: readonly [0.7225, 1];
+            readonly f1: readonly [1, 1];
+        };
+    };
+    readonly topic_consistency: {
+        readonly n: 31;
+        readonly tp: 11;
+        readonly fp: 0;
+        readonly fn: 1;
+        readonly tn: 19;
+        readonly precision: 1;
+        readonly recall: 0.9167;
+        readonly f1: 0.9565;
+        readonly ci95: {
+            readonly precision: readonly [0.7412, 1];
+            readonly recall: readonly [0.6461, 0.9851];
+            readonly f1: readonly [0.8333, 1];
+        };
+    };
+    readonly no_pii: {
+        readonly n: 90;
+        readonly tp: 34;
+        readonly fp: 5;
+        readonly fn: 11;
+        readonly tn: 40;
+        readonly precision: 0.8718;
+        readonly recall: 0.7556;
+        readonly f1: 0.8095;
+        readonly ci95: {
+            readonly precision: readonly [0.7329, 0.944];
+            readonly recall: readonly [0.6133, 0.8576];
+            readonly f1: readonly [0.7105, 0.8936];
+        };
+    };
+    readonly no_blocklist_words: {
+        readonly n: 32;
+        readonly tp: 11;
+        readonly fp: 1;
+        readonly fn: 4;
+        readonly tn: 16;
+        readonly precision: 0.9167;
+        readonly recall: 0.7333;
+        readonly f1: 0.8148;
+        readonly ci95: {
+            readonly precision: readonly [0.6461, 0.9851];
+            readonly recall: readonly [0.4805, 0.891];
+            readonly f1: readonly [0.6087, 0.9524];
+        };
+    };
+    readonly no_injection_patterns: {
+        readonly n: 90;
+        readonly tp: 41;
+        readonly fp: 0;
+        readonly fn: 1;
+        readonly tn: 48;
+        readonly precision: 1;
+        readonly recall: 0.9762;
+        readonly f1: 0.988;
+        readonly ci95: {
+            readonly precision: readonly [0.9143, 1];
+            readonly recall: readonly [0.8768, 0.9958];
+            readonly f1: readonly [0.96, 1];
+        };
+    };
+    readonly no_stub_output: {
+        readonly n: 89;
+        readonly tp: 30;
+        readonly fp: 5;
+        readonly fn: 12;
+        readonly tn: 42;
+        readonly precision: 0.8571;
+        readonly recall: 0.7143;
+        readonly f1: 0.7792;
+        readonly ci95: {
+            readonly precision: readonly [0.7062, 0.9374];
+            readonly recall: readonly [0.5643, 0.8283];
+            readonly f1: readonly [0.6667, 0.8736];
+        };
+    };
+    readonly no_hallucination_markers: {
+        readonly n: 90;
+        readonly tp: 34;
+        readonly fp: 0;
+        readonly fn: 12;
+        readonly tn: 44;
+        readonly precision: 1;
+        readonly recall: 0.7391;
+        readonly f1: 0.85;
+        readonly ci95: {
+            readonly precision: readonly [0.8985, 1];
+            readonly recall: readonly [0.5974, 0.844];
+            readonly f1: readonly [0.7576, 0.9213];
+        };
+    };
+    readonly no_silent_tool_failure: {
+        readonly n: 30;
+        readonly tp: 13;
+        readonly fp: 0;
+        readonly fn: 1;
+        readonly tn: 16;
+        readonly precision: 1;
+        readonly recall: 0.9286;
+        readonly f1: 0.963;
+        readonly ci95: {
+            readonly precision: readonly [0.7719, 1];
+            readonly recall: readonly [0.6853, 0.9873];
+            readonly f1: readonly [0.8667, 1];
+        };
+    };
+    readonly cost_under_threshold: {
+        readonly n: 26;
+        readonly tp: 10;
+        readonly fp: 0;
+        readonly fn: 0;
+        readonly tn: 16;
+        readonly precision: 1;
+        readonly recall: 1;
+        readonly f1: 1;
+        readonly ci95: {
+            readonly precision: readonly [0.7225, 1];
+            readonly recall: readonly [0.7225, 1];
+            readonly f1: readonly [1, 1];
+        };
+    };
+    readonly token_efficiency: {
+        readonly n: 25;
+        readonly tp: 9;
+        readonly fp: 0;
+        readonly fn: 0;
+        readonly tn: 16;
+        readonly precision: 1;
+        readonly recall: 1;
+        readonly f1: 1;
+        readonly ci95: {
+            readonly precision: readonly [0.7009, 1];
+            readonly recall: readonly [0.7009, 1];
+            readonly f1: readonly [1, 1];
+        };
+    };
+    readonly no_tool_loop: {
+        readonly n: 28;
+        readonly tp: 12;
+        readonly fp: 0;
+        readonly fn: 0;
+        readonly tn: 16;
+        readonly precision: 1;
+        readonly recall: 1;
+        readonly f1: 1;
+        readonly ci95: {
+            readonly precision: readonly [0.7575, 1];
+            readonly recall: readonly [0.7575, 1];
+            readonly f1: readonly [1, 1];
+        };
+    };
+};

package/dist/eval/published-accuracy.js

@@ -0,0 +1,86 @@
+/*
+ * GENERATED by `npm run proof` from proof/results.json — do not edit by hand.
+ * `npm run proof -- --check` fails CI when this file differs from what the
+ * runner produces. Read by src/eval/accuracy.ts; the numbers a verdict carries
+ * are the numbers on https://iris-eval.com/proof, for the release named below.
+ */
+export const PUBLISHED_ACCURACY_CORPUS_VERSION = '1251d242916c';
+export const PUBLISHED_ACCURACY_RELEASE = '0.9.0';
+export const PUBLISHED_ACCURACY_LABELLING = 'same-model';
+export const PUBLISHED_ACCURACY = {
+    min_output_length: {
+        n: 29, tp: 13, fp: 0, fn: 0, tn: 16,
+        precision: 1, recall: 1, f1: 1,
+        ci95: { precision: [0.7719, 1], recall: [0.7719, 1], f1: [1, 1] },
+    },
+    non_empty_output: {
+        n: 29, tp: 10, fp: 0, fn: 2, tn: 17,
+        precision: 1, recall: 0.8333, f1: 0.9091,
+        ci95: { precision: [0.7225, 1], recall: [0.552, 0.953], f1: [0.7368, 1] },
+    },
+    sentence_count: {
+        n: 30, tp: 8, fp: 0, fn: 6, tn: 16,
+        precision: 1, recall: 0.5714, f1: 0.7273,
+        ci95: { precision: [0.6756, 1], recall: [0.3259, 0.7862], f1: [0.4706, 0.9091] },
+    },
+    expected_coverage: {
+        n: 29, tp: 14, fp: 0, fn: 0, tn: 15,
+        precision: 1, recall: 1, f1: 1,
+        ci95: { precision: [0.7847, 1], recall: [0.7847, 1], f1: [1, 1] },
+    },
+    keyword_overlap: {
+        n: 30, tp: 10, fp: 0, fn: 0, tn: 20,
+        precision: 1, recall: 1, f1: 1,
+        ci95: { precision: [0.7225, 1], recall: [0.7225, 1], f1: [1, 1] },
+    },
+    topic_consistency: {
+        n: 31, tp: 11, fp: 0, fn: 1, tn: 19,
+        precision: 1, recall: 0.9167, f1: 0.9565,
+        ci95: { precision: [0.7412, 1], recall: [0.6461, 0.9851], f1: [0.8333, 1] },
+    },
+    no_pii: {
+        n: 90, tp: 34, fp: 5, fn: 11, tn: 40,
+        precision: 0.8718, recall: 0.7556, f1: 0.8095,
+        ci95: { precision: [0.7329, 0.944], recall: [0.6133, 0.8576], f1: [0.7105, 0.8936] },
+    },
+    no_blocklist_words: {
+        n: 32, tp: 11, fp: 1, fn: 4, tn: 16,
+        precision: 0.9167, recall: 0.7333, f1: 0.8148,
+        ci95: { precision: [0.6461, 0.9851], recall: [0.4805, 0.891], f1: [0.6087, 0.9524] },
+    },
+    no_injection_patterns: {
+        n: 90, tp: 41, fp: 0, fn: 1, tn: 48,
+        precision: 1, recall: 0.9762, f1: 0.988,
+        ci95: { precision: [0.9143, 1], recall: [0.8768, 0.9958], f1: [0.96, 1] },
+    },
+    no_stub_output: {
+        n: 89, tp: 30, fp: 5, fn: 12, tn: 42,
+        precision: 0.8571, recall: 0.7143, f1: 0.7792,
+        ci95: { precision: [0.7062, 0.9374], recall: [0.5643, 0.8283], f1: [0.6667, 0.8736] },
+    },
+    no_hallucination_markers: {
+        n: 90, tp: 34, fp: 0, fn: 12, tn: 44,
+        precision: 1, recall: 0.7391, f1: 0.85,
+        ci95: { precision: [0.8985, 1], recall: [0.5974, 0.844], f1: [0.7576, 0.9213] },
+    },
+    no_silent_tool_failure: {
+        n: 30, tp: 13, fp: 0, fn: 1, tn: 16,
+        precision: 1, recall: 0.9286, f1: 0.963,
+        ci95: { precision: [0.7719, 1], recall: [0.6853, 0.9873], f1: [0.8667, 1] },
+    },
+    cost_under_threshold: {
+        n: 26, tp: 10, fp: 0, fn: 0, tn: 16,
+        precision: 1, recall: 1, f1: 1,
+        ci95: { precision: [0.7225, 1], recall: [0.7225, 1], f1: [1, 1] },
+    },
+    token_efficiency: {
+        n: 25, tp: 9, fp: 0, fn: 0, tn: 16,
+        precision: 1, recall: 1, f1: 1,
+        ci95: { precision: [0.7009, 1], recall: [0.7009, 1], f1: [1, 1] },
+    },
+    no_tool_loop: {
+        n: 28, tp: 12, fp: 0, fn: 0, tn: 16,
+        precision: 1, recall: 1, f1: 1,
+        ci95: { precision: [0.7575, 1], recall: [0.7575, 1], f1: [1, 1] },
+    },
+};

package/dist/eval/questions.d.ts

@@ -0,0 +1,12 @@
+import type { QuestionId } from '../types/eval.js';
+export type CapabilityQuestionId = QuestionId | 'better_or_worse' | 'where_and_why' | 'trusted';
+export interface EvaluationQuestion {
+    id: CapabilityQuestionId;
+    /** The question in a reader's words. */
+    text: string;
+    /** What answers it: a rule (stamped on results), a tool (arc 5's run comparison) or a surface (dashboard, /proof). */
+    answeredBy: 'rule' | 'tool' | 'surface';
+}
+export declare const QUESTIONS: readonly EvaluationQuestion[];
+/** The questions a rule may declare. */
+export declare const RULE_QUESTION_IDS: readonly QuestionId[];

package/dist/eval/questions.js

@@ -0,0 +1,14 @@
+export const QUESTIONS = [
+    { id: 'safe_output', text: 'Is the output safe to show — no leaked personal data or credentials, no injection, nothing the deployment prohibits?', answeredBy: 'rule' },
+    { id: 'grounded', text: 'Is the output grounded in what the agent was given or read, rather than invented?', answeredBy: 'rule' },
+    { id: 'complete', text: 'Did the agent produce a full answer rather than a stub, a fragment or a promise?', answeredBy: 'rule' },
+    { id: 'relevant', text: 'Is the output on task — does it address what was asked?', answeredBy: 'rule' },
+    { id: 'task_completed', text: 'Did the task actually complete, as opposed to reading as if it had?', answeredBy: 'rule' },
+    { id: 'tool_use_correct', text: 'Did the agent act well — the right tools, valid arguments, no loops, failures acknowledged?', answeredBy: 'rule' },
+    { id: 'within_budget', text: 'Did the run cost what the deployment allows, in money and tokens?', answeredBy: 'rule' },
+    { id: 'better_or_worse', text: 'Is this agent better or worse than before, with an interval?', answeredBy: 'tool' },
+    { id: 'where_and_why', text: 'Where does it fail, and why?', answeredBy: 'surface' },
+    { id: 'trusted', text: 'Can this verdict be trusted — how often is each evaluator wrong?', answeredBy: 'surface' },
+];
+/** The questions a rule may declare. */
+export const RULE_QUESTION_IDS = QUESTIONS.filter((q) => q.answeredBy === 'rule').map((q) => q.id);