@iris-eval/mcp-server 0.7.0 → 0.8.0

package/dist/tools/list-rules.js

@@ -11,6 +11,7 @@
  * needs to manage the rule set programmatically.
  */
 import { z } from 'zod';
+import { builtInRuleRoster } from '../eval/criticality.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { strictInput } from './strict-input.js';
 const inputSchema = {
@@ -23,7 +24,7 @@ const inputSchema = {
         .default(false)
         .describe('Return only enabled rules (excludes disabled ones)'),
 };
-export function registerListRulesTool(server, customRuleStore) {
+export function registerListRulesTool(server, customRuleStore, evalEngine) {
     server.registerTool('list_rules', {
         title: 'List Custom Rules',
         description: [
@@ -33,11 +34,13 @@ export function registerListRulesTool(server, customRuleStore) {
             '',
             'Behavior. Pure read of ~/.iris/custom-rules.json (in-memory cached; no disk read per call after server boot). No mutation, no external network. Tenant-scoped in Cloud tier; OSS returns all rules for the single local tenant. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Returns in <5ms.',
             '',
-            'Output shape. Returns JSON: `{ "rules": [{ "id": "rule-XXXX", "name", "description", "evalType", "severity", "definition": { name, type, config, weight? }, "enabled": boolean, "createdAt": ISO timestamp, "updatedAt": ISO timestamp, "version": number, "sourceMomentId?": string }], "total": number, "enabled_count": number }`. Empty array + total=0 when no rules deployed. A deployed rule fires only on evaluate_output calls whose eval_type equals its evalType (or eval_type="all", which runs every bundle).',
+            'Output shape. Returns JSON: `{ "rules": [{ "id": "rule-XXXX", "name", "description", "evalType", "severity", "definition": { name, type, config, weight? }, "enabled": boolean, "createdAt": ISO timestamp, "updatedAt": ISO timestamp, "version": number, "sourceMomentId?": string }], "total": number, "enabled_count": number, "built_in": [{ "name", "category", "weight", "critical": boolean, "criticalSource": "default" | "config" }] }`. Empty `rules` array + total=0 when no custom rules are deployed. A deployed rule fires only on evaluate_output calls whose eval_type equals its evalType (or eval_type="all", which runs every bundle). `built_in` is the shipped rule set, always present and NOT narrowed by the filters; `total` and `enabled_count` count custom rules only.',
+            '',
+            'Why `built_in` carries criticality. A critical rule vetoes `passed` regardless of the weighted score, and which built-in rules are critical is configurable (`eval.criticalRules` / `eval.nonCriticalRules`). `critical` is the EFFECTIVE value this server applies and `criticalSource` says who decided it: `default` is the declaration on the rule itself, `config` means one of those lists named it. Read it before trusting a `passed: true` — it is how you tell "nothing was violated" from "the rule that would have vetoed is demoted on this server".',
             '',
             'Use when you need to know what custom rules are currently live (before calling evaluate_output, before deploying a similar rule to avoid duplicates, or when building a dashboard view). Filter with `eval_type` to scope to a specific category, or `enabled_only: true` to exclude disabled rules. Use get_traces to see trace data; use evaluate_output to run scoring; use list_rules only when you need the RULE INVENTORY.',
             '',
-            "Don't use to count traces or evals (that's get_traces). Don't use to inspect built-in (non-custom) rules — those ship with the iris binary and are listed in docs/api-reference.md, not in the rule store. Don't use to deploy a rule (use deploy_rule); don't use to remove one (use delete_rule).",
+            "Don't use to count traces or evals (that's get_traces). Don't use to deploy a rule (use deploy_rule); don't use to remove one (use delete_rule). Built-in rules are not in the store and cannot be deployed, deleted or disabled — they appear under `built_in` for reference, carrying the criticality this server applies.",
             '',
             'Parameters. eval_type filter is exact-match against each rule\'s evalType field (no wildcards). enabled_only excludes rules that are deployed-but-disabled — a rule is disabled without deleting it via delete_rule with `enabled: false` (and re-enabled with `enabled: true`), or from the dashboard; disabled rules stay in the store with their history but do not fire. Both filters are AND-combined when both are set. Both are optional; with no filter, all rules return. Defaults: eval_type=undefined (no filter), enabled_only=false (returns all rules including disabled).',
             '',
@@ -63,11 +66,26 @@ export function registerListRulesTool(server, customRuleStore) {
         }
         const total = rules.length;
         const enabled_count = rules.filter((r) => r.enabled).length;
+        /*
+         * The built-in roster, with the criticality THIS engine applies.
+         * Until eval.criticalRules existed, "which rules veto" was a constant
+         * a reader could look up in the docs; it is now per-deployment, and a
+         * caller deciding whether to trust `passed` has no other way to see
+         * that a veto was promoted or demoted underneath them. Unfiltered on
+         * purpose — the filters describe the custom-rule store.
+         */
+        const built_in = builtInRuleRoster((rule) => evalEngine.effectiveCriticality(rule)).map((r) => ({
+            name: r.name,
+            category: r.category,
+            weight: r.weight,
+            critical: r.critical,
+            criticalSource: r.criticalSource,
+        }));
         return {
             content: [
                 {
                     type: 'text',
-                    text: JSON.stringify({ rules, total, enabled_count }),
+                    text: JSON.stringify({ rules, total, enabled_count, built_in }),
                 },
             ],
         };

package/dist/tools/log-trace.d.ts

@@ -1,6 +1,13 @@
 import { z } from 'zod';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
+export declare const toolCallSchema: z.ZodObject<{
+    tool_name: z.ZodString;
+    input: z.ZodOptional<z.ZodUnknown>;
+    output: z.ZodOptional<z.ZodUnknown>;
+    latency_ms: z.ZodOptional<z.ZodNumber>;
+    error: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
 export declare const logTraceInputShape: {
     agent_name: z.ZodString;
     framework: z.ZodOptional<z.ZodString>;
@@ -12,7 +19,7 @@ export declare const logTraceInputShape: {
         output: z.ZodOptional<z.ZodUnknown>;
         latency_ms: z.ZodOptional<z.ZodNumber>;
         error: z.ZodOptional<z.ZodString>;
-    }, z.core.$strip>>>;
+    }, z.core.$strict>>>;
     latency_ms: z.ZodOptional<z.ZodNumber>;
     token_usage: z.ZodOptional<z.ZodObject<{
         prompt_tokens: z.ZodOptional<z.ZodNumber>;

package/dist/tools/log-trace.js

@@ -2,14 +2,29 @@ import { z } from 'zod';
 import { generateTraceId, generateSpanId } from '../utils/ids.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { bestEffortExport } from '../otel/lazy.js';
-import { strictInput } from './strict-input.js';
-const ToolCallSchema = z.object({
+import { strictInput, strictNested } from './strict-input.js';
+/*
+ * The tool-call record — one entry of `tool_calls[]`.
+ *
+ * Exported because it is now read on THREE paths, not one: log_trace and
+ * the HTTP ingest capture it, and evaluate_output accepts it directly so
+ * the trajectory rules (no_silent_tool_failure, no_tool_loop) can judge
+ * what the agent DID. All three must agree on the field names, so they all
+ * derive from this one schema rather than restating it.
+ *
+ * Strict for the same reason custom_rules entries are (#376): a dropped
+ * key here is silent AND load-bearing. `{ tool_name, output, err: "..." }`
+ * used to parse with `err` discarded, and a trajectory rule reading
+ * `error` would then score a failed call as a clean one — the exact
+ * failure mode the rules exist to catch, reintroduced by a typo.
+ */
+export const toolCallSchema = strictNested({
     tool_name: z.string(),
     input: z.unknown().optional(),
     output: z.unknown().optional(),
     latency_ms: z.number().optional(),
     error: z.string().optional(),
-});
+}, 'a tool_calls entry');
 const SpanSchema = z.object({
     span_id: z.string().optional(),
     parent_span_id: z.string().optional(),
@@ -43,7 +58,7 @@ export const logTraceInputShape = {
     framework: z.string().optional().describe('Agent framework identifier (e.g., langchain, autogen, custom)'),
     input: z.string().optional().describe('Agent input text — the user prompt or upstream input that produced this output'),
     output: z.string().optional().describe('Agent output text — what the agent produced (pass to evaluate_output for scoring)'),
-    tool_calls: z.array(ToolCallSchema).optional().describe('Tool calls made during execution (per-call latency, errors, input/output)'),
+    tool_calls: z.array(toolCallSchema).optional().describe('Tool calls made during execution (per-call latency, errors, input/output)'),
     latency_ms: z.number().optional().describe('Total execution time in milliseconds (end-to-end agent latency)'),
     token_usage: TokenUsageSchema.optional().describe('Token usage breakdown (prompt/completion/total — used for cost analysis)'),
     cost_usd: z.number().optional().describe('Total cost in USD — overrides per-span aggregation when provided (treated as authoritative)'),

package/dist/tools/trace-link.d.ts

@@ -1,7 +1,17 @@
 import type { IStorageAdapter } from '../types/query.js';
 import type { EvalResult } from '../types/eval.js';
+import type { Trace } from '../types/trace.js';
 import type { TenantId } from '../types/tenant.js';
 export declare function unknownTraceMessage(traceId: string): string;
+/**
+ * The same refuse-before-any-work check, returning the row it already read.
+ *
+ * evaluate_output needs the trace itself (its `tool_calls`, so a caller who
+ * has already logged the trajectory does not have to resend it), and the
+ * existence check had to load the row anyway. Fetching it twice would be
+ * two reads for one fact — and two chances for them to disagree.
+ */
+export declare function getTraceOrThrow(storage: IStorageAdapter, tenantId: TenantId, traceId: string): Promise<Trace>;
 export declare function assertTraceExists(storage: IStorageAdapter, tenantId: TenantId, traceId: string): Promise<void>;
 /** insertEvalResult with the foreign-key race translated into the same clear message. */
 export declare function insertLinkedEvalResult(storage: IStorageAdapter, tenantId: TenantId, result: EvalResult): Promise<void>;

package/dist/tools/trace-link.js

@@ -18,10 +18,22 @@ export function unknownTraceMessage(traceId) {
         'Nothing was evaluated or written. Pass the trace_id returned by log_trace (or listed by get_traces), ' +
         'or omit trace_id to store an unlinked evaluation.');
 }
-export async function assertTraceExists(storage, tenantId, traceId) {
+/**
+ * The same refuse-before-any-work check, returning the row it already read.
+ *
+ * evaluate_output needs the trace itself (its `tool_calls`, so a caller who
+ * has already logged the trajectory does not have to resend it), and the
+ * existence check had to load the row anyway. Fetching it twice would be
+ * two reads for one fact — and two chances for them to disagree.
+ */
+export async function getTraceOrThrow(storage, tenantId, traceId) {
     const trace = await storage.getTrace(tenantId, traceId);
     if (!trace)
         throw new Error(unknownTraceMessage(traceId));
+    return trace;
+}
+export async function assertTraceExists(storage, tenantId, traceId) {
+    await getTraceOrThrow(storage, tenantId, traceId);
 }
 /** insertEvalResult with the foreign-key race translated into the same clear message. */
 export async function insertLinkedEvalResult(storage, tenantId, result) {

package/dist/types/config.d.ts

@@ -32,7 +32,21 @@ export interface IrisConfig {
             topic_consistency?: number;
             cost_threshold?: number;
             max_token_ratio?: number;
+            max_tool_repeats?: number;
         };
+        /**
+         * Built-in rule names promoted to CRITICAL — a failure vetoes `passed`
+         * regardless of the weighted score. Validated against the rule registry
+         * when the config loads; an unknown name is a startup error naming the
+         * valid list, never a silent no-op.
+         */
+        criticalRules?: string[];
+        /**
+         * Built-in rule names demoted from critical — they still score and still
+         * report a failure, but they stop vetoing `passed`. Same validation. A
+         * name in both lists is a config error: it does not say what you want.
+         */
+        nonCriticalRules?: string[];
     };
     logging: {
         level: 'debug' | 'info' | 'warn' | 'error';

package/dist/types/eval.d.ts

@@ -1,3 +1,4 @@
+import type { ToolCallRecord } from './trace.js';
 export type EvalType = 'completeness' | 'relevance' | 'safety' | 'cost' | 'custom';
 /**
  * What an EvalResult can be tagged as: a single bundle (EvalType), or
@@ -30,11 +31,19 @@ export interface EvalContext {
     output: string;
     expected?: string;
     input?: string;
-    toolCalls?: Array<{
-        tool_name: string;
-        input?: unknown;
-        output?: unknown;
-    }>;
+    /**
+     * The agent's trajectory — what it actually DID, in call order.
+     *
+     * Deliberately the SAME record the capture path stores (ToolCallRecord =
+     * log_trace's `tool_calls[]`), not a narrower local shape. It used to be
+     * a three-field inline type without `error`, so a rule could see that a
+     * tool was called but never that it FAILED: the acceptance pass found
+     * three real transcripts that answered confidently after a grep exited 1,
+     * an ls hit a missing directory and a node -e threw, and no rule could
+     * reach the fact. Re-declaring a subset here would reintroduce exactly
+     * that gap the next time a field is added to the capture shape.
+     */
+    toolCalls?: ToolCallRecord[];
     tokenUsage?: {
         prompt_tokens?: number;
         completion_tokens?: number;
@@ -71,6 +80,22 @@ export interface EvalRuleResult {
      * regroup them.
      */
     category?: EvalType;
+    /**
+     * Whether this rule VETOES the verdict — its EFFECTIVE criticality, after
+     * `eval.criticalRules` / `eval.nonCriticalRules` are applied, not the
+     * value on the rule's definition. A reader holding a failed evaluation
+     * could otherwise not tell a hard violation from a low score without
+     * knowing the rule library by heart.
+     */
+    critical?: boolean;
+    /**
+     * Who decided that: 'default' is the rule's own declaration (for a
+     * deployed custom rule, the severity it was deployed with); 'config' means
+     * one of the two override lists named it. The distinction is the point of
+     * making criticality configurable — an operator reading a verdict must be
+     * able to see that their own promotion caused it.
+     */
+    criticalSource?: 'default' | 'config';
     passed: boolean;
     score: number;
     message: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Stop shipping agents on vibes. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.7.0",
+  "version": "0.8.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.7.0",
+      "version": "0.8.0",
       "transport": {
         "type": "stdio"
       },