@iris-eval/mcp-server 0.8.2 → 0.10.0

package/dist/types/config.d.ts

@@ -2,6 +2,13 @@ export interface IrisConfig {
     storage: {
         type: 'sqlite';
         path: string;
+        /**
+         * `critical_spans` stores each evaluation's output text with the spans a
+         * critical detector flagged replaced by `[REDACTED:<pattern>]`, so a tool
+         * that detects leaks need not keep the leak it found. The evidence
+         * offsets still index the original text the caller saw. Default `none`.
+         */
+        redact?: 'none' | 'critical_spans';
     };
     server: {
         name: string;
@@ -47,12 +54,49 @@ export interface IrisConfig {
          * name in both lists is a config error: it does not say what you want.
          */
         nonCriticalRules?: string[];
+        /** `risk` composes by kind (gates, vetoes, unknown, then the risk); `legacy` runs the pre-0.10.0 weighted mean. */
+        composer?: 'risk' | 'legacy';
+        /**
+         * How many wrongly blocked builds one shipped failure is worth. The risk
+         * threshold is 1 / (1 + this), so 1 means a false pass and a false block
+         * cost the same; a continuous-integration gate that hates flakiness sets
+         * it low, a compliance gate sets it high.
+         */
+        falsePassCost?: number;
+        /**
+         * What a critical rule that was ASKED and could not answer does to the
+         * verdict — defeated by the output, or configured invalidly. Not the
+         * same as never asked, which is coverage. Today's behaviour is `pass`,
+         * which is the fail-open seam; the default is `unknown`.
+         */
+        onCriticalSkipped?: 'unknown' | 'fail' | 'pass';
+        /** Inputs every evaluation must carry; an absent one makes the verdict unknown rather than clean. */
+        requiredEvidence?: string[];
+        /**
+         * Whether a threshold IRIS ships decides the verdict, or only advises.
+         * A default is our guess about a deployment we have never seen; a
+         * threshold you set is your decision. A policy with no number in it —
+         * "the output is empty" — gates either way.
+         */
+        defaultsGate?: boolean;
+        /** The prior that an output is bad before any rule speaks. 0.5 matches the proof corpus, not your traffic. */
+        prior?: number;
+        /**
+         * How that prior is spread over the failure classes the detectors
+         * examine. `per-output` keeps it at the stated value for the output as a
+         * whole; `per-class` applies it to each class independently, which makes
+         * installing another detector raise the prior before that detector has
+         * looked at anything.
+         */
+        priorMode?: 'per-output' | 'per-class';
     };
     logging: {
         level: 'debug' | 'info' | 'warn' | 'error';
     };
     retention: {
         days: number;
+        /** How often the sweep re-runs after boot, in hours; 0 disables the timer (the boot sweep still runs). Default 24. */
+        sweepIntervalHours: number;
     };
     security: {
         apiKey?: string;

package/dist/types/eval.d.ts

@@ -8,6 +8,23 @@ export type EvalType = 'completeness' | 'relevance' | 'safety' | 'cost' | 'custo
  * under a real bundle, never under 'all'.
  */
 export type EvalResultType = EvalType | 'all';
+/**
+ * What KIND of claim a rule makes — the mandate's distinction between a
+ * measurement (a statistic against a threshold), a detection (a pattern is
+ * present, with a measured error rate), an inference (a signal standing in
+ * for an unobservable property), a judgment (a model's reasoning), a policy
+ * (the deployment's own constraint) and an external verification. Kind is
+ * the claim; `mechanism` is how the claim is measured. The composer decides
+ * by kind and never averages kinds together.
+ */
+export type ClaimKind = 'measurement' | 'detection' | 'inference' | 'judgment' | 'policy' | 'verification';
+export type Mechanism = 'formula' | 'pattern' | 'heuristic' | 'model' | 'external';
+/** An input a rule reads. A rule skips — never passes — when a declared need is absent. */
+export type Need = 'output' | 'input' | 'expected' | 'tool_calls' | 'tool_outputs' | 'tools_catalogue' | 'cost' | 'tokens' | 'citations';
+/** The evaluation question a rule answers; the registry is src/eval/questions.ts. */
+export type QuestionId = 'safe_output' | 'grounded' | 'complete' | 'relevant' | 'task_completed' | 'tool_use_correct' | 'within_budget';
+/** What went wrong, in the reader's words, independent of which rule caught it; the registry is src/eval/failure-classes.ts. */
+export type FailureClass = 'pii_leak' | 'credential_leak' | 'injection' | 'injection_compliance' | 'silent_tool_failure' | 'tool_loop' | 'stub' | 'fabrication' | 'ungrounded' | 'incomplete_ask' | 'off_task' | 'over_budget' | 'format' | 'invalid_tool_call';
 export interface EvalRule {
     name: string;
     description: string;
@@ -25,6 +42,31 @@ export interface EvalRule {
      * is the verdict, and a critical violation must never be averaged away.
      */
     critical?: boolean;
+    /**
+     * The rule's metadata — what kind of claim it makes, how it measures it,
+     * what it reads, which question it answers, which failure classes a
+     * failing result belongs to, and the version of its definition. Every
+     * built-in declares all six (tests/unit/eval/rule-metadata.test.ts);
+     * custom types declare kind, mechanism, needs and version and leave the
+     * question to their author. Optional on the interface so a rule built
+     * elsewhere still compiles; a result from a rule without them carries no
+     * `kind`, which reads as unknown — never as a measurement.
+     */
+    kind?: ClaimKind;
+    mechanism?: Mechanism;
+    needs?: readonly Need[];
+    question?: QuestionId;
+    classes?: readonly FailureClass[];
+    /** Bumped when the rule's meaning changes, so a stored result names the definition that produced it. */
+    version?: number;
+    /**
+     * Who wrote this rule. `custom` marks anything `createCustomRule`
+     * produced — a deployed rule or one passed inline in the call. The
+     * composer needs it: for OUR rule a shipped threshold is a guess and only
+     * advises, while for THEIRS the severity they deployed it at is their own
+     * statement of how much it matters. Absent means built-in.
+     */
+    origin?: 'built-in' | 'custom';
     evaluate(context: EvalContext): EvalRuleResult;
 }
 export interface EvalContext {
@@ -63,9 +105,263 @@ export interface EvalContext {
     regexBudget?: {
         breaches: number;
     };
+    /**
+  
+     * Whether this evaluation may call a paid provider. Set ONLY by the tools
+  
+     * whose whole purpose is to do so — the LLM judge and the citation
+  
+     * verifier. The engine refuses to run a judgment rule without it, which
+  
+     * is what makes "evaluate_output never spends" a property of the engine
+  
+     * rather than a promise in a tool description.
+  
+     */
+    allowPaid?: boolean;
+}
+/**
+ * What the composer DID with a result under this deployment's configuration
+ * — distinct from `kind`, which is what the rule claims. Today's composer
+ * (a weighted mean plus the critical veto) knows two roles: `veto` for an
+ * effectively critical rule and `term` for one that feeds the score. The
+ * compose-by-kind release adds `gate` (a configured policy that decides),
+ * `risk` (a detection or inference feeding the risk estimate) and
+ * `advisory` (reported, deciding nothing).
+ */
+export type Role = 'gate' | 'veto' | 'risk' | 'advisory' | 'term';
+/**
+ * Why a rule skipped. `not_applicable`: the evidence it needs was not
+ * supplied (never asked — coverage). `defeated`: asked and could not answer,
+ * because this output stalled its pattern past the sandbox budget.
+ * `config_invalid`: asked and could not answer, because its definition is
+ * broken. A gate that fails closed treats the last two as unknown; the first
+ * is a coverage fact, not a verdict.
+ */
+export type SkipClass = 'not_applicable' | 'defeated' | 'config_invalid';
+export interface Interval {
+    point: number;
+    lo: number;
+    hi: number;
+}
+/**
+ * What a rule saw — typed, locatable, never an excerpt. A detection reports
+ * the OFFSETS of what it matched (into the raw text, so a leak detector can
+ * redact the span it found without ever repeating it); a trajectory rule the
+ * index of the call it judged; a measurement its statistic with a unit and
+ * the threshold it was held to; a signal that yields no offset yet reports
+ * its name and count. The reader can locate every claim; the stored row
+ * can be redacted; nothing here restates the offending text.
+ */
+export type Evidence = {
+    type: 'span';
+    source: 'output' | 'input' | `tool_outputs[${number}]`;
+    start: number;
+    end: number;
+    label: string;
+} | {
+    type: 'pattern';
+    name: string;
+    count: number;
+} | {
+    type: 'toolCall';
+    index: number;
+    toolName: string;
+    label: string;
+} | {
+    type: 'citation';
+    url: string;
+    status: 'resolved' | 'dead' | 'unverifiable' | 'supported' | 'unsupported';
+} | {
+    type: 'count';
+    stat: string;
+    unit: string;
+    value: number;
+    threshold?: number;
+    thresholdSource?: 'default' | 'config' | 'call' | 'rule';
+} | {
+    type: 'sample';
+    score: number;
+    selfReportedPass?: boolean;
+    rationaleHash: string;
+};
+/** A measurement's statistic — the number the rule computed, with its unit, before any score transform. */
+export interface MeasuredValue {
+    stat: string;
+    unit: string;
+    value: number;
+}
+/** Evidence lists are capped so a pathological output cannot balloon a stored row. */
+export declare const MAX_EVIDENCE_ITEMS = 25;
+/**
+ * Which evaluation questions this evaluation judged, which it did not and
+ * why — coverage by question, not by rule count. `inputs` says what the
+ * call carried; a question is `judged` when at least one rule that answers
+ * it ran, `unjudged` when every such rule skipped (the reason names the
+ * missing input, or that the rule was defeated or broken), and
+ * `not_applicable` when no rule for it was in the selected bundles.
+ */
+export interface Coverage {
+    inputs: Record<Need, boolean>;
+    questions: Array<{
+        id: QuestionId;
+        status: 'judged' | 'unjudged' | 'not_applicable';
+        why?: string;
+    }>;
+    /** Quarantined critical rules that did not run (surfaced by the rule-store release). */
+    dormant?: Array<{
+        ruleId: string;
+        name: string;
+        reason: string;
+    }>;
+}
+/**
+ * The verdict with its basis. `passed` is `state === 'pass'` and equals the
+ * top-level `passed`; `basis` says which layer decided — a configured policy,
+ * a detector's veto, nothing judged, or the score against the threshold.
+ * `risk` is null until the compose-by-kind release computes it.
+ */
+/**
+ * A sentence a reader needs that the verdict alone does not carry, with who
+ * it is for and what to change. The one that must exist: when a rule
+ * visibly FIRED and the verdict still passed, say why and name the setting
+ * that would change it — "cost_under_threshold failed" beside
+ * "passed: true" reads as a bug to anyone who has not read the composer.
+ *
+ * `suggestions` remains for now and is rendered from these; it is deprecated
+ * from 0.13.0 and removed at 1.0, per VERSIONING.md's two-minor rule.
+ */
+export interface Interpretation {
+    severity: 'block' | 'warn' | 'note';
+    addressee: 'agent' | 'operator' | 'author';
+    /** The rule this is about, when it is about one. */
+    rule?: string;
+    text: string;
+    /** The configuration key that changes this behaviour, when there is one. */
+    configKey?: string;
+}
+/** Placed on EvalResult by the engine; see Interpretation above. */
+export interface Verdict {
+    state: 'pass' | 'fail' | 'unknown';
+    passed: boolean;
+    basis: 'policy_gate' | 'detector_veto' | 'critical_unknown' | 'required_evidence_missing' | 'risk_over_loss' | 'score_below_threshold' | 'clean' | 'no_rules';
+    by: string[];
+    risk: {
+        pBad: number;
+        lo: number;
+        hi: number;
+        perClass: Partial<Record<FailureClass, number | null>>;
+        assumptions: string[];
+    } | null;
+    confidence?: 'decisive' | 'marginal';
+}
+/** What produced this verdict, so it can be replayed or compared: the release, the ruleset, the configuration, the thresholds, the proof corpus, the time. */
+export interface Provenance {
+    irisVersion: string;
+    rulesetHash: string;
+    configHash: string;
+    thresholds: {
+        default: number;
+        perRule?: Record<string, unknown>;
+    };
+    corpusVersion: string;
+    judgedAt: string;
 }
+/**
+ * How wrong this result tends to be, and on what basis. `published_accuracy`
+ * carries the rule's measured numbers from the shipped proof (src/eval/
+ * published-accuracy.ts): for a fired detection or inference the positive
+ * predictive value at the stated prior, for one that did not fire the
+ * residual miss rate, each with a 95% credible interval. `definition` is a
+ * measurement's conformance to its formula (n cases, matched). `policy` is
+ * the deployment's own constraint — no error rate applies. `self_consistency`
+ * and `local_labels` arrive with the judge-through-the-engine and the
+ * own-traffic labels releases. `unmeasured` says why nothing can be stated.
+ */
+export type Uncertainty = {
+    basis: 'published_accuracy';
+    fired: true;
+    ppv: Interval;
+    prior: {
+        pi: number;
+        source: 'default' | 'config' | 'estimated';
+    };
+    corpus: {
+        n: number;
+        tp: number;
+        fp: number;
+        fn: number;
+        tn: number;
+        version: string;
+        release: string;
+        labelling: 'same-model' | 'human-verified';
+    };
+} | {
+    basis: 'published_accuracy';
+    fired: false;
+    missRate: Interval;
+    prior: {
+        pi: number;
+        source: 'default' | 'config' | 'estimated';
+    };
+    corpus: {
+        n: number;
+        tp: number;
+        fp: number;
+        fn: number;
+        tn: number;
+        version: string;
+        release: string;
+        labelling: 'same-model' | 'human-verified';
+    };
+} | {
+    basis: 'definition';
+    conformance: {
+        n: number;
+        matched: number;
+    };
+} | {
+    basis: 'self_consistency';
+    samples: number;
+    voteFraction: number;
+    scoreSd: number;
+} | {
+    basis: 'local_labels';
+    precision: Interval;
+    n: number;
+} | {
+    basis: 'policy';
+} | {
+    basis: 'unmeasured';
+    why: string;
+};
 export interface EvalRuleResult {
     ruleName: string;
+    /**
+     * What kind of claim this result makes, what the composer did with it,
+     * which question it answers and which failure classes a failure belongs
+     * to — stamped by the engine from the rule's declaration (0.9.0). Absent
+     * on results written before that release and on rules that declare no
+     * metadata; never fabricated on read.
+     */
+    kind?: ClaimKind;
+    role?: Role;
+    question?: QuestionId;
+    classes?: FailureClass[];
+    /** The version of the rule definition that produced this result. */
+    ruleVersion?: number;
+    /** Who wrote the rule: `custom` for anything createCustomRule produced. See EvalRule.origin. */
+    origin?: 'built-in' | 'custom';
+    /** Which of the rule's declared needs the call actually carried — what the rule SAW. */
+    saw?: Need[];
+    /** Present only when `skipped`; says whether the rule was never asked or was asked and could not answer. */
+    skipClass?: SkipClass;
+    /** How wrong this result tends to be, and on what basis. Present on every result that made a claim (not on skips). */
+    uncertainty?: Uncertainty;
+    /** What the rule saw: spans (offsets, never text), tool-call indices, pattern names, counts. Present on every fired detection or inference, and on measurements. */
+    evidence?: Evidence[];
+    /** A measurement's statistic and unit — the number before the score transform. */
+    value?: MeasuredValue;
     /**
      * Deployed rule id (rule-<hex>) when the rule came from the custom-rule
      * store. Absent for built-in rules and for inline custom_rules. Names are
@@ -169,6 +465,19 @@ export interface EvalResult {
      * rule_results carry a `category` per rule so a reader can regroup.
      */
     categories?: Partial<Record<EvalType, EvalCategoryResult>>;
+    /** The verdict with its basis (0.9.0) — computed by the engine, derived on read for stored rows that carry provenance. */
+    verdict?: Verdict;
+    /** Sentences a reader needs that the verdict alone does not carry (0.10.0). */
+    interpretations?: Interpretation[];
+    /** Coverage by evaluation question (0.9.0) — computed by the engine, derived on read from the stamped rule results. */
+    coverage?: Coverage;
+    /** What produced this verdict (0.9.0) — persisted; absent on rows written before it, never fabricated. */
+    provenance?: Provenance;
+    /** What the evaluation itself cost (the judge's spend); undefined for the free rules. */
+    eval_cost_usd?: number;
+    eval_tokens?: number;
+    /** Set when the linked trace was deleted (delete_trace or the retention sweep) and this row's text was erased. */
+    erased_at?: string;
 }
 export type CustomRuleType = 'regex_match' | 'regex_no_match' | 'min_length' | 'max_length' | 'contains_keywords' | 'excludes_keywords' | 'json_schema' | 'cost_threshold';
 export interface CustomRuleDefinition {

package/dist/types/eval.js

@@ -1 +1,2 @@
-export {};
+/** Evidence lists are capped so a pathological output cannot balloon a stored row. */
+export const MAX_EVIDENCE_ITEMS = 25;

package/dist/types/query.d.ts

@@ -82,6 +82,8 @@ export interface IStorageAdapter {
     getSpansByTraceId(tenantId: TenantId, traceId: string): Promise<Span[]>;
     insertEvalResult(tenantId: TenantId, result: EvalResult): Promise<void>;
     getEvalsByTraceId(tenantId: TenantId, traceId: string): Promise<EvalResult[]>;
+    /** One stored evaluation by id, in the same derived-on-read shape as every other reader; null when absent. */
+    getEvalById(tenantId: TenantId, id: string): Promise<EvalResult | null>;
     queryEvalResults(tenantId: TenantId, options: {
         eval_type?: string;
         passed?: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.8.2",
+  "version": "0.10.0",
   "description": "Stop shipping agents on vibes. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.8.2",
+  "version": "0.10.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.8.2",
+      "version": "0.10.0",
       "transport": {
         "type": "stdio"
       },

package/dist/resources/dashboard-summary.d.ts

@@ -1,3 +0,0 @@
-import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-import type { IStorageAdapter } from '../types/query.js';
-export declare function registerDashboardSummaryResource(server: McpServer, storage: IStorageAdapter): void;

package/dist/resources/dashboard-summary.js

@@ -1,16 +0,0 @@
-import { LOCAL_TENANT } from '../types/tenant.js';
-export function registerDashboardSummaryResource(server, storage) {
-    server.resource('dashboard-summary', 'iris://dashboard/summary', { description: 'Dashboard summary with key metrics and trends' }, async () => {
-        // OSS single-tenant: summary scopes to the local user.
-        const summary = await storage.getDashboardSummary(LOCAL_TENANT);
-        return {
-            contents: [
-                {
-                    uri: 'iris://dashboard/summary',
-                    mimeType: 'application/json',
-                    text: JSON.stringify(summary, null, 2),
-                },
-            ],
-        };
-    });
-}

package/dist/resources/trace-detail.d.ts

@@ -1,3 +0,0 @@
-import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-import type { IStorageAdapter } from '../types/query.js';
-export declare function registerTraceDetailResource(server: McpServer, storage: IStorageAdapter): void;

package/dist/resources/trace-detail.js

@@ -1,30 +0,0 @@
-import { LOCAL_TENANT } from '../types/tenant.js';
-export function registerTraceDetailResource(server, storage) {
-    server.resource('trace-detail', 'iris://traces/{trace_id}', { description: 'Full trace detail with spans and evaluation results' }, async (uri) => {
-        const traceId = uri.pathname.split('/').pop();
-        // OSS single-tenant: MCP caller is the local user.
-        const trace = await storage.getTrace(LOCAL_TENANT, traceId);
-        if (!trace) {
-            return {
-                contents: [
-                    {
-                        uri: uri.href,
-                        mimeType: 'application/json',
-                        text: JSON.stringify({ error: 'Trace not found' }),
-                    },
-                ],
-            };
-        }
-        const spans = await storage.getSpansByTraceId(LOCAL_TENANT, traceId);
-        const evals = await storage.getEvalsByTraceId(LOCAL_TENANT, traceId);
-        return {
-            contents: [
-                {
-                    uri: uri.href,
-                    mimeType: 'application/json',
-                    text: JSON.stringify({ trace, spans, evals }, null, 2),
-                },
-            ],
-        };
-    });
-}