@iris-eval/mcp-server 0.4.5 → 0.5.0

package/dist/tools/deploy-rule.js

@@ -11,7 +11,9 @@
  * eval_type.
  */
 import { z } from 'zod';
+import { createCustomRule } from '../eval/rules/custom.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
+import { strictInput } from './strict-input.js';
 const CustomRuleDefinitionSchema = z.object({
     name: z.string(),
     type: z.enum([
@@ -28,7 +30,11 @@ const CustomRuleDefinitionSchema = z.object({
     weight: z.number().optional(),
 });
 const inputSchema = {
-    name: z.string().min(1).max(120).describe('Human-readable rule name (used in eval results)'),
+    // 80 mirrors the persisted store's cap (custom-rule-store.ts). The tool
+    // used to allow 120, so a 100-char name passed the tool schema and then
+    // surfaced the store's ZodError as a raw 500 (#332). One limit, enforced
+    // at the boundary, fails cleanly as a 400.
+    name: z.string().min(1).max(80).describe('Human-readable rule name (1-80 chars; used in eval results)'),
     description: z
         .string()
         .max(500)
@@ -40,14 +46,14 @@ const inputSchema = {
     severity: z
         .enum(['low', 'medium', 'high', 'critical'])
         .default('medium')
-        .describe('Severity used for dashboard sort + audit alerts'),
+        .describe('What a FAILURE of this rule means. low/medium: informational — contributes to the weighted score only (plus dashboard sort + audit alerts). high/critical: hard-fail — a failing evaluation of this rule forces the overall passed=false regardless of the weighted score'),
     definition: CustomRuleDefinitionSchema.describe('Check definition (regex, length, keyword, cost, or schema)'),
     sourceMomentId: z
         .string()
         .optional()
         .describe('Optional Decision Moment ID the rule was derived from (preserves workflow-inversion provenance)'),
 };
-export function registerDeployRuleTool(server, customRuleStore) {
+export function registerDeployRuleTool(server, customRuleStore, evalEngine) {
     server.registerTool('deploy_rule', {
         title: 'Deploy Custom Rule',
         description: [
@@ -63,11 +69,11 @@ export function registerDeployRuleTool(server, customRuleStore) {
             '',
             "Don't use to VALIDATE a rule before committing — deploy writes immediately. Use the dashboard's preview endpoint (POST /api/v1/rules/custom/preview) for dry-run validation against sample output. Don't use to EDIT an existing rule — this call only creates; edits require a dedicated flow (coming in v0.5). To update a rule today: delete_rule then deploy_rule with the new definition.",
             '',
-            'Parameters. name is 1-120 chars (Zod-enforced min/max); appears in eval_result rule_results so make it human-readable. description is optional, max 500 chars (used in dashboard tooltips). evalType determines WHEN the rule fires (must match the eval_type your evaluate_output calls use; e.g., a "completeness" rule fires on every evaluate_output where eval_type="completeness" OR eval_type="custom"). severity affects dashboard sort + audit log signal but does NOT affect scoring (scoring uses the rule\'s weight). definition.type and definition.config must match (e.g., regex_match needs config.pattern; cost_threshold needs config.max_cost; min_length needs config.min_length; max_length needs config.max_length; contains_keywords/excludes_keywords need config.keywords). Invalid configs are now REJECTED at deploy time with the offending field named, instead of deploying and then failing every evaluation. sourceMomentId is optional but recommended (preserves workflow-inversion provenance from Make-This-A-Rule composer). Defaults: severity="medium".',
+            'Parameters. name is 1-80 chars (Zod-enforced min/max — the same cap the persisted store applies); appears in eval_result rule_results so make it human-readable. description is optional, max 500 chars (used in dashboard tooltips). evalType determines WHEN the rule fires (must match the eval_type your evaluate_output calls use; e.g., a "completeness" rule fires on every evaluate_output where eval_type="completeness" OR eval_type="custom"). severity decides what a FAILURE of the rule does: low/medium failures only lower the weighted score (and drive dashboard sort + audit alerts); high/critical failures HARD-FAIL the evaluation — the overall `passed` is forced to false regardless of the weighted score, and the rule is listed in the response\'s `critical_failures`. Severity never changes the numeric score itself (that uses the rule\'s weight). definition.type and definition.config must match (e.g., regex_match needs config.pattern; cost_threshold needs config.max_cost; min_length needs config.min_length; max_length needs config.max_length; contains_keywords/excludes_keywords need config.keywords). Invalid configs are now REJECTED at deploy time with the offending field named, instead of deploying and then failing every evaluation. sourceMomentId is optional but recommended (preserves workflow-inversion provenance from Make-This-A-Rule composer). Defaults: severity="medium".',
             '',
-            "Error modes. Throws 400 on invalid definition (Zod rejects — e.g., regex that fails safe-regex2 ReDoS check, or length > 1000 chars). Throws 400 on empty `name`. Throws 400 if the eval category mismatches the definition type. Returns 429 when HTTP rate limit exceeded. File-write failures (disk full, read-only fs) propagate as 500; the audit log is best-effort and does not block deploy.",
+            "Error modes. Throws 400 on invalid definition (Zod rejects — e.g., regex that fails safe-regex2 ReDoS check, or length > 1000 chars). Throws 400 on empty `name` or `name` over 80 chars. Any evalType/definition.type combination is valid (a regex_match rule can enforce a safety policy; a max_length rule can express completeness) — there is no category/type mismatch error. Returns 429 when HTTP rate limit exceeded. File-write failures (disk full, read-only fs) propagate as 500; the audit log is best-effort and does not block deploy.",
         ].join('\n'),
-        inputSchema,
+        inputSchema: strictInput(inputSchema),
         annotations: {
             readOnlyHint: false,
             destructiveHint: false,
@@ -75,16 +81,32 @@ export function registerDeployRuleTool(server, customRuleStore) {
             openWorldHint: false,
         },
     }, async (args) => {
+        // Server overrides the inner definition's `name` so it always matches
+        // the user-facing rule name — same normalization the dashboard's
+        // deploy route applies. Also keeps the tool's 80-char cap authoritative
+        // (an unchecked definition.name used to reach the store and surface its
+        // ZodError as a raw 500).
+        const definition = {
+            ...args.definition,
+            name: args.name,
+        };
         // OSS: MCP tools operate under LOCAL_TENANT. See list-rules.ts for context.
         const rule = customRuleStore.deploy(LOCAL_TENANT, {
             name: args.name,
             description: args.description,
             evalType: args.evalType,
             severity: args.severity,
-            definition: args.definition,
+            definition,
             sourceMomentId: args.sourceMomentId,
             user: 'mcp',
         });
+        // Register with the live engine so the rule fires on the very next
+        // evaluate_output call — the "activates immediately for the running
+        // process" this description promises. Previously only the dashboard's
+        // deploy route did this; MCP deploys silently waited for a restart.
+        // Registered under its rule id so delete_rule can hot-remove it.
+        // Severity rides along: high/critical makes the rule hard-failing.
+        evalEngine.registerRule(rule.evalType, createCustomRule(rule.definition, rule.severity), rule.id);
         return {
             content: [
                 {

package/dist/tools/evaluate-output.js

@@ -1,5 +1,6 @@
 import { z } from 'zod';
 import { LOCAL_TENANT } from '../types/tenant.js';
+import { strictInput } from './strict-input.js';
 const CustomRuleSchema = z.object({
     name: z.string(),
     type: z.enum([
@@ -11,11 +12,20 @@ const CustomRuleSchema = z.object({
 });
 const inputSchema = {
     output: z.string().describe('The output text to evaluate (the agent\'s response that gets scored against rules)'),
-    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']).default('completeness').describe('Rule bundle to apply: completeness | relevance | safety | cost | custom — picks which built-in rules fire'),
+    // .optional() rather than .default('completeness') so the handler can tell
+    // "caller chose completeness" apart from "caller never chose" — the second
+    // case gets a note in the response saying safety rules did not run. The
+    // effective default is still completeness.
+    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']).optional().describe('Rule bundle to apply: completeness | relevance | safety | cost | custom — picks which built-in rules fire. Defaults to "completeness" when omitted (the response then carries a note that safety rules did not run)'),
     expected: z.string().optional().describe('Expected output for comparison — REQUIRED when eval_type="relevance" (used as keyword-overlap target)'),
-    input: z.string().optional().describe('Original input for context — improves relevance scoring (keyword overlap vs input)'),
+    input: z.string().optional().describe('Original input for context (the ask + any source material the agent was given) — improves relevance scoring and grounds the safety bundle\'s hallucination signals'),
     trace_id: z.string().optional().describe('Link evaluation to a trace — surfaces this eval in the dashboard\'s trace drill-through'),
-    custom_rules: z.array(CustomRuleSchema).optional().describe('Custom evaluation rules — fires REGARDLESS of eval_type; pass eval_type="custom" if you want ONLY these'),
+    // .max(10): inline rules skip the deploy-time probe, and the engine runs
+    // rules synchronously — without a cap, one request carrying N sandbox-
+    // defeating regex rules stalls the server linearly in N (measured 9.3s at
+    // N=50). Ten is ample for per-call rules; persistent sets belong in
+    // deploy_rule, where deploy-time validation probes each pattern.
+    custom_rules: z.array(CustomRuleSchema).max(10).optional().describe('Custom evaluation rules, max 10 per call (deploy persistent rule sets via deploy_rule instead) — fires REGARDLESS of eval_type; pass eval_type="custom" if you want ONLY these'),
     cost_usd: z.number().optional().describe('Cost in USD — only consulted when eval_type="cost" (compared against cost_threshold rules)'),
     token_usage: z.object({
         prompt_tokens: z.number().optional(),
@@ -33,17 +43,19 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
             '',
             'Behavior. Deterministic, in-process scoring — same inputs always produce the same result. Writes one eval_result row to Iris storage (linked to trace_id if provided; unlinked otherwise). No external network calls in heuristic mode (v0.4 adds an llm_as_judge eval_type that DOES call LLM APIs; see the separate evaluate_with_llm_judge tool for that). Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Runs in ~5-50ms for rule-based evaluation.',
             '',
-            'Output shape. Returns JSON: `{ "id": "<uuid>", "score": 0..1, "passed": boolean, "rule_results": [{ "ruleName", "passed", "score", "message", "skipped?" }], "suggestions": string[], "rules_evaluated": number, "rules_skipped": number, "insufficient_data": boolean }`. `insufficient_data=true` means no applicable rules fired (e.g., safety eval with only cost data).',
+            'Output shape. Returns JSON: `{ "id": "<uuid>", "eval_type": "<bundle that ran>", "score": 0..1, "passed": boolean, "critical_failures?": string[], "rule_results": [{ "ruleName", "passed", "score", "message", "skipped?" }], "suggestions": string[], "rules_evaluated": number, "rules_skipped": number, "insufficient_data": boolean, "note?": string }`. `insufficient_data=true` means no applicable rules fired (e.g., safety eval with only cost data). `note` appears only when eval_type was omitted, naming the defaulted bundle and that safety rules did not run.',
             '',
-            'Use when you want a quality score on a specific output — typically after log_trace records the execution. Pass `eval_type` to route to the right rule bundle: `completeness` (length, sentence count, relevance to input), `relevance` (keyword overlap, topic consistency), `safety` (PII leak, prompt injection, hallucination markers, stub-output detection), `cost` (budget threshold), or `custom` (bring your own rules via `custom_rules`).',
+            'What `passed` means. `score` and `passed` answer different questions. `score` is the weighted average across the rules that ran — a 0..1 quality gradient. `passed` is the ship/no-ship verdict: true only when the score clears the pass threshold (default 0.7, configurable via config `eval.defaultThreshold`) AND no critical rule failed. Critical rules HARD-FAIL: if one fails, `passed` is false regardless of the weighted score, and the culprits are listed in `critical_failures`. The critical rules are the genuine safety violations — `no_pii`, `no_injection_patterns`, `no_blocklist_words` — plus any deployed custom rule with severity high/critical. A leaked SSN can never be averaged away by other rules passing.',
+            '',
+            'Use when you want a quality score on a specific output — typically after log_trace records the execution. Pass `eval_type` to route to the right rule bundle: `completeness` (length, sentence count, relevance to input), `relevance` (keyword overlap, topic consistency), `safety` (PII leak, prompt injection, hallucination markers, stub-output detection — pass `input` so the hallucination signals can cross-check the output against the material the agent was given), `cost` (budget threshold), or `custom` (bring your own rules via `custom_rules`).',
             '',
             'Don\'t use when the output is empty or has no applicable rules — the eval_type decides which rules apply, and invalid combinations return score=0 + insufficient_data=true (not an error, but not actionable). Don\'t use to VALIDATE JSON schemas directly (use your language\'s JSON Schema validator — Iris\'s `json_schema` custom rule type is for output-shape assertions, not arbitrary validation).',
             '',
-            'Parameters. expected is REQUIRED when eval_type="relevance" (used as the comparison target for keyword overlap + topic consistency); ignored for other eval_types. cost_usd + token_usage are ONLY consulted when eval_type="cost" (ignored otherwise). custom_rules ALWAYS fires regardless of eval_type — pass eval_type="custom" if you want ONLY your rules to run (otherwise both your rules AND the eval_type bundle run together). trace_id is optional but recommended (linking the eval to its trace surfaces it in the dashboard\'s drill-through). input adds context to keyword-overlap relevance checks; ignored otherwise. Defaults: eval_type="completeness".',
+            'Parameters. expected is REQUIRED when eval_type="relevance" (used as the comparison target for keyword overlap + topic consistency); ignored for other eval_types. cost_usd + token_usage are ONLY consulted when eval_type="cost" (ignored otherwise). custom_rules ALWAYS fires regardless of eval_type — pass eval_type="custom" if you want ONLY your rules to run (otherwise both your rules AND the eval_type bundle run together). trace_id is optional but recommended (linking the eval to its trace surfaces it in the dashboard\'s drill-through). input adds context to keyword-overlap relevance checks AND grounds the safety bundle\'s hallucination signals (without it those signals stay silent rather than guess); ignored otherwise. Defaults: eval_type="completeness" — and when you rely on that default, the response carries a `note` reminding you that the safety bundle did not run.',
             '',
-            'Error modes. Throws on malformed custom_rules (Zod rejects). Returns 400 on regex patterns that fail safe-regex2 ReDoS check or exceed 1000-char limit. Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. The eval itself never throws — failing rules report `passed: false` with a message, they don\'t bubble exceptions.',
+            'Error modes. Throws on unknown argument names (strict schema — a misspelled argument is rejected with the valid argument list, never silently dropped). Throws on malformed custom_rules (Zod rejects) and on more than 10 custom_rules in one call (use deploy_rule for persistent rule sets). Returns 400 on regex patterns that fail safe-regex2 ReDoS check or exceed 1000-char limit. Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. The eval itself never throws — failing rules report `passed: false` with a message, they don\'t bubble exceptions. A regex that exceeds the 100ms sandbox matching budget on a given output reports skipped with budgetExceeded=true instead of hanging the server (fail-open per rule — gate on that flag if you must fail closed).',
         ].join('\n'),
-        inputSchema,
+        inputSchema: strictInput(inputSchema),
         annotations: {
             readOnlyHint: false, // Writes an eval_result row
             destructiveHint: false, // Creates new data; doesn't overwrite or delete
@@ -51,7 +63,12 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
             openWorldHint: false, // No external network in heuristic mode; LLM-as-judge has its own tool with openWorldHint:true
         },
     }, async (args) => {
-        const evalType = args.eval_type;
+        // Track omission explicitly: a caller who never chose a bundle gets
+        // the completeness default AND a note saying so — six of seven UAT
+        // personas read passed:true on PII-laden text with no hint that the
+        // safety bundle never ran.
+        const evalTypeOmitted = args.eval_type === undefined;
+        const evalType = (args.eval_type ?? 'completeness');
         const result = evalEngine.evaluate(evalType, {
             output: args.output,
             expected: args.expected,
@@ -71,13 +88,23 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
                     type: 'text',
                     text: JSON.stringify({
                         id: result.id,
+                        // Echo which bundle actually ran. Without this, a caller who
+                        // omitted eval_type could not tell a "safety pass" from a
+                        // completeness eval that never ran a single safety rule.
+                        eval_type: result.eval_type,
                         score: result.score,
                         passed: result.passed,
+                        ...(result.critical_failures ? { critical_failures: result.critical_failures } : {}),
                         rule_results: result.rule_results,
                         suggestions: result.suggestions,
                         rules_evaluated: result.rules_evaluated,
                         rules_skipped: result.rules_skipped,
                         insufficient_data: result.insufficient_data,
+                        ...(evalTypeOmitted
+                            ? {
+                                note: 'eval_type was omitted, so the default "completeness" bundle ran. Safety rules (PII, injection, blocklist, stub, hallucination) were NOT part of this evaluation — pass eval_type="safety" to run them.',
+                            }
+                            : {}),
                     }),
                 },
             ],

package/dist/tools/evaluate-with-llm-judge.js

@@ -3,6 +3,7 @@ import { LOCAL_TENANT } from '../types/tenant.js';
 import { evaluateWithLLMJudge } from '../eval/llm-judge/evaluator.js';
 import { findPricing } from '../eval/llm-judge/pricing.js';
 import { generateEvalId } from '../utils/ids.js';
+import { strictInput } from './strict-input.js';
 const inputSchema = {
     output: z.string().min(1).describe('The agent output text to evaluate'),
     template: z
@@ -73,7 +74,7 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
             '',
             'Error modes. Throws when the required API key env var is missing. Throws when the estimated worst-case cost exceeds max_cost_usd (raise the cap or trim prompts). Throws LLMJudgeError on provider errors — kind=`auth` on 401/403, `rate_limit` on 429 (auto-retried once), `server_error` on 5xx, `timeout` on abort, `malformed_response` when the judge fails to emit valid JSON on both attempts. Throws "Unknown model" for unsupported model IDs — update src/eval/llm-judge/pricing.ts first.',
         ].join('\n'),
-        inputSchema,
+        inputSchema: strictInput(inputSchema),
         annotations: {
             readOnlyHint: false, // Writes eval_result; also spends money (external API cost)
             destructiveHint: false, // Creates data; doesn't overwrite or delete

package/dist/tools/get-traces.js

@@ -1,5 +1,6 @@
 import { z } from 'zod';
 import { LOCAL_TENANT } from '../types/tenant.js';
+import { strictInput } from './strict-input.js';
 const inputSchema = {
     agent_name: z.string().optional().describe('Filter by agent name — exact match (no wildcards in v0.4)'),
     framework: z.string().optional().describe('Filter by agent framework — exact match (e.g., langchain, autogen)'),
@@ -7,7 +8,10 @@ const inputSchema = {
     until: z.string().optional().describe('ISO timestamp upper bound — return traces with timestamp < this'),
     min_score: z.number().optional().describe('Minimum eval score filter (0..1) — applied to LATEST eval per trace, not all evals'),
     max_score: z.number().optional().describe('Maximum eval score filter (0..1) — applied to LATEST eval per trace'),
-    limit: z.number().default(50).describe('Results per page (default 50, max 1000 — values >1000 return 400)'),
+    // Mirrors traceQuerySchema in dashboard/validation.ts — both capture paths
+    // (MCP tool, HTTP query) enforce the same 1..1000 bound. Unclamped, limit:-1
+    // meant "LIMIT -1" in SQLite, i.e. every row (#332).
+    limit: z.number().int().min(1).max(1000).default(50).describe('Results per page (default 50, max 1000 — values >1000 return 400)'),
     offset: z.number().default(0).describe('Zero-based pagination offset — skip first N results'),
     sort_by: z.enum(['timestamp', 'latency_ms', 'cost_usd']).default('timestamp').describe('Sort by timestamp | latency_ms | cost_usd (default timestamp)'),
     sort_order: z.enum(['asc', 'desc']).default('desc').describe('Sort order: asc | desc (default desc — most recent / highest first)'),
@@ -33,7 +37,7 @@ export function registerGetTracesTool(server, storage) {
             '',
             'Error modes. Returns 400 on invalid sort_by / sort_order (Zod enum). Returns 400 if limit > 1000. Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. Empty result with `total: 0` on no matches (not an error).',
         ].join('\n'),
-        inputSchema,
+        inputSchema: strictInput(inputSchema),
         annotations: {
             readOnlyHint: true, // Pure query: never writes, never deletes
             destructiveHint: false, // Inverse of readOnly — trivially false

package/dist/tools/index.js

@@ -12,8 +12,8 @@ export function registerAllTools(server, storage, evalEngine, customRuleStore) {
     registerEvaluateOutputTool(server, storage, evalEngine);
     registerGetTracesTool(server, storage);
     registerListRulesTool(server, customRuleStore);
-    registerDeployRuleTool(server, customRuleStore);
-    registerDeleteRuleTool(server, customRuleStore);
+    registerDeployRuleTool(server, customRuleStore, evalEngine);
+    registerDeleteRuleTool(server, customRuleStore, evalEngine);
     registerDeleteTraceTool(server, storage);
     registerEvaluateWithLLMJudgeTool(server, storage);
     registerVerifyCitationsTool(server, storage);

package/dist/tools/list-rules.js

@@ -12,6 +12,7 @@
  */
 import { z } from 'zod';
 import { LOCAL_TENANT } from '../types/tenant.js';
+import { strictInput } from './strict-input.js';
 const inputSchema = {
     eval_type: z
         .enum(['completeness', 'relevance', 'safety', 'cost', 'custom'])
@@ -42,7 +43,7 @@ export function registerListRulesTool(server, customRuleStore) {
             '',
             "Error modes. Returns empty list if the rule store file doesn't exist (first run). Returns 429 if HTTP rate limit exceeded. Never throws on valid input.",
         ].join('\n'),
-        inputSchema,
+        inputSchema: strictInput(inputSchema),
         annotations: {
             readOnlyHint: true,
             destructiveHint: false,

package/dist/tools/log-trace.d.ts

@@ -1,3 +1,54 @@
+import { z } from 'zod';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
+export declare const logTraceInputShape: {
+    agent_name: z.ZodString;
+    framework: z.ZodOptional<z.ZodString>;
+    input: z.ZodOptional<z.ZodString>;
+    output: z.ZodOptional<z.ZodString>;
+    tool_calls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        tool_name: z.ZodString;
+        input: z.ZodOptional<z.ZodUnknown>;
+        output: z.ZodOptional<z.ZodUnknown>;
+        latency_ms: z.ZodOptional<z.ZodNumber>;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>>;
+    latency_ms: z.ZodOptional<z.ZodNumber>;
+    token_usage: z.ZodOptional<z.ZodObject<{
+        prompt_tokens: z.ZodOptional<z.ZodNumber>;
+        completion_tokens: z.ZodOptional<z.ZodNumber>;
+        total_tokens: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+    cost_usd: z.ZodOptional<z.ZodNumber>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    spans: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        span_id: z.ZodOptional<z.ZodString>;
+        parent_span_id: z.ZodOptional<z.ZodString>;
+        name: z.ZodString;
+        kind: z.ZodDefault<z.ZodEnum<{
+            INTERNAL: "INTERNAL";
+            SERVER: "SERVER";
+            CLIENT: "CLIENT";
+            PRODUCER: "PRODUCER";
+            CONSUMER: "CONSUMER";
+            LLM: "LLM";
+            TOOL: "TOOL";
+        }>>;
+        status_code: z.ZodDefault<z.ZodEnum<{
+            UNSET: "UNSET";
+            OK: "OK";
+            ERROR: "ERROR";
+        }>>;
+        status_message: z.ZodOptional<z.ZodString>;
+        start_time: z.ZodString;
+        end_time: z.ZodOptional<z.ZodString>;
+        attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        events: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            timestamp: z.ZodString;
+            attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strip>>>;
+    }, z.core.$strip>>>;
+    timestamp: z.ZodOptional<z.ZodString>;
+};
 export declare function registerLogTraceTool(server: McpServer, storage: IStorageAdapter): void;

package/dist/tools/log-trace.js

@@ -2,6 +2,7 @@ import { z } from 'zod';
 import { generateTraceId, generateSpanId } from '../utils/ids.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { bestEffortExport } from '../otel/lazy.js';
+import { strictInput } from './strict-input.js';
 const ToolCallSchema = z.object({
     tool_name: z.string(),
     input: z.unknown().optional(),
@@ -30,7 +31,14 @@ const TokenUsageSchema = z.object({
     completion_tokens: z.number().optional(),
     total_tokens: z.number().optional(),
 });
-const inputSchema = {
+/*
+ * The log_trace input contract. Exported because POST /api/v1/traces
+ * (src/dashboard/routes/traces.ts) accepts the SAME body — one schema,
+ * two capture paths. Duplicating it there would let the tool and the
+ * HTTP endpoint drift apart silently; importing it means a field added
+ * here is accepted (and validated identically) on both.
+ */
+export const logTraceInputShape = {
     agent_name: z.string().describe('Agent name — used for filtering in get_traces (e.g., "customer-support-bot")'),
     framework: z.string().optional().describe('Agent framework identifier (e.g., langchain, autogen, custom)'),
     input: z.string().optional().describe('Agent input text — the user prompt or upstream input that produced this output'),
@@ -63,7 +71,11 @@ export function registerLogTraceTool(server, storage) {
             '',
             'Error modes. Throws on missing agent_name. Throws on malformed span or tool_call objects (Zod rejects). Returns 500 on storage failure (disk full, DB locked). Never blocks on the agent — returns within ~50ms for typical payloads.',
         ].join('\n'),
-        inputSchema,
+        // Strict at the MCP boundary (unknown args rejected, not stripped).
+        // The dashboard's HTTP ingest builds its own schema FROM this shape
+        // (dashboard/validation.ts) and keeps default stripping there on
+        // purpose — it relies on it to discard a client-supplied trace_id.
+        inputSchema: strictInput(logTraceInputShape),
         annotations: {
             readOnlyHint: false, // Writes a row to storage
             destructiveHint: false, // Creates new data; doesn't overwrite or delete

package/dist/tools/strict-input.d.ts

@@ -0,0 +1,2 @@
+import { z } from 'zod';
+export declare function strictInput<T extends z.ZodRawShape>(shape: T): z.ZodObject<{ -readonly [P in keyof T]: T[P]; }, z.core.$strict>;

package/dist/tools/strict-input.js

@@ -0,0 +1,35 @@
+import { z } from 'zod';
+/*
+ * Wraps a tool's input shape in a STRICT object schema so unknown argument
+ * names are REJECTED with an error that names the offending key(s) and
+ * lists the valid ones.
+ *
+ * Why this exists: a bare shape (or z.object()) silently STRIPS unknown
+ * keys. At the MCP tool boundary that is dangerous, not lenient — an LLM
+ * guessing an argument name is the normal case, not an edge case. Before
+ * this wrapper, `evaluate_output({ criteria: ["safety"], ... })` (a
+ * plausible guess) and `eval_typ: "safety"` (a one-character typo) both
+ * "succeeded": the arguments were dropped, the DEFAULT completeness bundle
+ * ran instead of the safety rules, and the response said passed:true on
+ * text containing real PII — with nothing indicating the arguments were
+ * ignored. Meanwhile a missing REQUIRED field produced a precise Zod
+ * error, so the failure mode was inconsistent as well as unsafe.
+ *
+ * The MCP SDK accepts a schema object (not just a raw shape) for
+ * inputSchema and validates tool calls through it, so the custom
+ * unrecognized-keys message below is exactly what the caller sees.
+ * Strictness also reaches tools/list: the generated JSON Schema carries
+ * additionalProperties:false, telling well-behaved clients up front.
+ */
+export function strictInput(shape) {
+    const validKeys = Object.keys(shape).join(', ');
+    return z.strictObject(shape, {
+        error: (issue) => issue.code === 'unrecognized_keys'
+            ? `Unknown argument(s): ${issue.keys.map((k) => `"${k}"`).join(', ')}. ` +
+                `Valid arguments: ${validKeys}. ` +
+                'Unknown arguments are rejected rather than silently ignored, so a misspelled ' +
+                'argument name cannot change what gets evaluated — check the spelling against ' +
+                "the tool's input schema and retry."
+            : undefined,
+    });
+}

package/dist/tools/verify-citations.js

@@ -3,6 +3,7 @@ import { LOCAL_TENANT } from '../types/tenant.js';
 import { verifyCitations } from '../eval/citation-verify/verifier.js';
 import { findPricing } from '../eval/llm-judge/pricing.js';
 import { generateEvalId } from '../utils/ids.js';
+import { strictInput } from './strict-input.js';
 const inputSchema = {
     output: z.string().min(1).describe('The agent output containing citations to verify'),
     model: z
@@ -57,7 +58,7 @@ export function registerVerifyCitationsTool(server, storage) {
             '',
             'Behavior. Three-phase pipeline: (1) regex extraction of [N] numbered refs, (Author, Year) parentheticals, bare URLs, and DOIs (in-process, no network); (2) SSRF-guarded fetch of URL + DOI citations, with scheme allowlist, private/link-local/cloud-metadata IP blocking, optional domain allowlist (IRIS_CITATION_DOMAINS), 10s timeout, 5MB body cap, manual redirect chase (max 3, re-checked), in-process LRU cache; (3) per-citation LLM judge call asking "does this source support this claim?" with a 256-token verdict. Opt-in via allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — Iris refuses outbound HTTP by default. Cost-capped across the entire call by max_cost_usd_total (default $1.00) — the pipeline stops when the cap would be exceeded. Rate-limited to 20 req/min on HTTP MCP. Writes one eval_result row tagged with per-citation provenance.',
             '',
-            'Output shape. Returns JSON: `{ "id": "<uuid>", "overall_score": 0..1|null, "passed": boolean, "total_citations_found": number, "total_resolved": number, "total_supported": number, "total_cost_usd": number, "citations": [{ "citation": { "raw", "kind", "identifier", "offset_start", "offset_end" }, "resolve_status": "ok"|"skipped"|"error", "resolve_error"?, "source"?: { "url", "status", "content_type", "bytes_fetched", "truncated" }, "judge"?: { "supported", "confidence", "rationale", "cost_usd", "latency_ms", "input_tokens", "output_tokens" } }] }`. `overall_score = supported / resolved`; `null` when nothing resolvable was found.',
+            'Output shape. Returns JSON: `{ "id": "<uuid>", "overall_score": 0..1|null, "passed": boolean, "total_citations_found": number, "total_resolved": number, "total_judged": number, "total_supported": number, "total_cost_usd": number, "citations": [{ "citation": { "raw", "kind", "identifier", "offset_start", "offset_end" }, "resolve_status": "ok"|"skipped"|"error", "resolve_error"?, "source"?: { "url", "status", "content_type", "bytes_fetched", "truncated" }, "judge"?: { "supported", "confidence", "rationale", "cost_usd", "latency_ms", "input_tokens", "output_tokens" } }] }`. `overall_score = supported / judged`; `null` when nothing was judged (no resolvable citations, or every judge call failed). Infrastructure failures (cost cap, judge timeout/error, malformed verdict) leave a citation resolved-but-unjudged — reported per-citation via resolve_error, never scored as unsupported.',
             '',
             'Use when the output makes factual claims backed by [1]-style references, DOIs, or URLs and you want to separate "cited correctly" from "cited and wrong" from "cited but unresolvable". Particularly useful for research/legal/medical agents where fabricated citations are the dominant failure mode.',
             "",
@@ -67,7 +68,7 @@ export function registerVerifyCitationsTool(server, storage) {
             '',
             'Error modes. Throws when the API key env var is missing. Throws "Unknown model" on unsupported model IDs. Per-citation errors are collected (resolve_error.kind = bad_scheme / ssrf / not_allowed_domain / timeout / too_large / bad_status / redirect_loop / not_text / fetch_disabled / malformed_judge_response / cost_cap_reached / unresolvable_kind) and returned in the response rather than thrown. An empty output or output with zero extractable citations returns overall_score=null + passed=true (nothing to fail).',
         ].join('\n'),
-        inputSchema,
+        inputSchema: strictInput(inputSchema),
         annotations: {
             readOnlyHint: false, // Writes eval_result + spends money
             destructiveHint: false, // Creates data; doesn't overwrite/delete
@@ -109,11 +110,11 @@ export function registerVerifyCitationsTool(server, storage) {
                     passed: result.passed,
                     score,
                     message: result.overallScore === null
-                        ? `No resolvable citations (found ${result.totalCitationsFound}, resolved ${result.totalResolved})`
-                        : `${result.totalSupported}/${result.totalResolved} cited sources supported the output`,
+                        ? `No citations judged (found ${result.totalCitationsFound}, resolved ${result.totalResolved}, judged 0)`
+                        : `${result.totalSupported}/${result.totalJudged} judged sources supported the output`,
                 },
             ],
-            suggestions: result.passed ? [] : [`Only ${result.totalSupported}/${result.totalResolved} cited sources actually supported the claim.`],
+            suggestions: result.passed ? [] : [`Only ${result.totalSupported}/${result.totalJudged} judged sources actually supported the claim.`],
             rules_evaluated: 1,
             rules_skipped: 0,
             insufficient_data: result.overallScore === null,
@@ -128,6 +129,7 @@ export function registerVerifyCitationsTool(server, storage) {
                         passed: result.passed,
                         total_citations_found: result.totalCitationsFound,
                         total_resolved: result.totalResolved,
+                        total_judged: result.totalJudged,
                         total_supported: result.totalSupported,
                         total_cost_usd: result.totalCostUsd,
                         citations: result.citations.map((c) => ({

package/dist/transport/http.js

@@ -56,9 +56,31 @@ export async function createHttpTransport(mcpServer, config, logger) {
      * `127.0.0.1:0` and reject every real request with a 403 that looks
      * exactly like an attack. Routes are registered immediately after, and
      * the port is not discoverable by any client until this function returns.
+     *
+     * The callback MUST inspect its error argument. Express 5 wires the
+     * listen callback as `server.once('error', done)` as well as the
+     * listening callback — so on EADDRINUSE it is invoked WITH the error.
+     * Ignoring that argument resolved this promise on a server that never
+     * bound: the caller then logged "HTTP transport listening on <port>"
+     * while another process owned the port, and the process idled forever.
+     * A CI health poll got 200 from the OTHER instance and shipped
+     * evaluations to a stranger's database. A bind failure must reject,
+     * name the port, and take the process down nonzero.
      */
-    const httpServer = await new Promise((resolve) => {
-        const server = app.listen(config.transport.port, config.transport.host, () => resolve(server));
+    const httpServer = await new Promise((resolve, reject) => {
+        const server = app.listen(config.transport.port, config.transport.host, (err) => {
+            if (err) {
+                const bind = `${config.transport.host}:${config.transport.port}`;
+                const code = err.code;
+                reject(code === 'EADDRINUSE'
+                    ? new Error(`HTTP transport failed to start: port ${config.transport.port} is already in use ` +
+                        `(EADDRINUSE on ${bind}). Another process — possibly another iris instance — owns it. ` +
+                        `Pass --port <other> (or set IRIS_PORT) or stop the other process.`)
+                    : new Error(`HTTP transport failed to bind ${bind}: ${err.message}`));
+                return;
+            }
+            resolve(server);
+        });
     });
     const address = httpServer.address();
     const port = typeof address === 'object' && address ? address.port : config.transport.port;

package/dist/types/config.d.ts

@@ -15,6 +15,13 @@ export interface IrisConfig {
     dashboard: {
         enabled: boolean;
         port: number;
+        /**
+         * Bind address. Defaults to loopback: the dashboard is unauthenticated
+         * by default (security.apiKey is undefined) and serves the full trace
+         * history, so binding it to every interface exposes agent inputs and
+         * outputs to the local network. Set explicitly to share it.
+         */
+        host: string;
     };
     eval: {
         defaultThreshold: number;

package/dist/types/decision-moment.d.ts

@@ -120,3 +120,15 @@ export interface MomentQueryResult {
     limit: number;
     offset: number;
 }
+export interface RankedFailure extends DecisionMoment {
+    /** Severity × recency-decay blend, 0-1. Higher = shown first. */
+    rankScore: number;
+}
+export interface FailureQueryResult {
+    failures: RankedFailure[];
+    /** How many recent traces were scanned to build the list. */
+    scanned: number;
+    /** Total traces matching the filter (pre-scan-cap). */
+    total: number;
+    limit: number;
+}

package/dist/types/eval.d.ts

@@ -4,6 +4,18 @@ export interface EvalRule {
     description: string;
     evalType: EvalType;
     weight: number;
+    /**
+     * Hard-fail marker. When a critical rule FAILS (and was not skipped), the
+     * overall eval reports passed=false regardless of the weighted score.
+     *
+     * Exists because the weighted average routinely outvotes a genuine
+     * violation: an output leaking a real SSN failed no_pii while the other
+     * safety rules passed, scoring ~0.765 — above the 0.7 threshold — so the
+     * one field every CI gate reads said passed:true about the product's
+     * flagship failure scenario. The score stays a quality gradient; `passed`
+     * is the verdict, and a critical violation must never be averaged away.
+     */
+    critical?: boolean;
     evaluate(context: EvalContext): EvalRuleResult;
 }
 export interface EvalContext {
@@ -23,6 +35,17 @@ export interface EvalContext {
     costUsd?: number;
     metadata?: Record<string, unknown>;
     customConfig?: Record<string, unknown>;
+    /**
+     * Per-evaluation regex circuit breaker, initialized by the engine (never
+     * by callers). Each sandbox budget breach increments `breaches`; once it
+     * reaches the cap, remaining regex rules in the SAME evaluation skip
+     * without running. Bounds how long a single hostile output can stall a
+     * request: without it, N regex rules × (budget + worker respawn) of
+     * main-thread stall scale linearly with N.
+     */
+    regexBudget?: {
+        breaches: number;
+    };
 }
 export interface EvalRuleResult {
     ruleName: string;
@@ -31,6 +54,8 @@ export interface EvalRuleResult {
     message: string;
     skipped?: boolean;
     skipReason?: string;
+    configInvalid?: boolean;
+    budgetExceeded?: boolean;
 }
 export interface EvalResult {
     id: string;
@@ -46,6 +71,13 @@ export interface EvalResult {
     rules_evaluated?: number;
     rules_skipped?: number;
     insufficient_data?: boolean;
+    /**
+     * Names of critical rules that failed (present only when non-empty).
+     * Any entry here forces passed=false regardless of the weighted score —
+     * this field is how a caller tells "failed the quality bar" apart from
+     * "committed a hard violation".
+     */
+    critical_failures?: string[];
 }
 export type CustomRuleType = 'regex_match' | 'regex_no_match' | 'min_length' | 'max_length' | 'contains_keywords' | 'excludes_keywords' | 'json_schema' | 'cost_threshold';
 export interface CustomRuleDefinition {

package/dist/types/query.d.ts

@@ -23,7 +23,7 @@ export interface TraceQueryResult {
     limit: number;
     offset: number;
 }
-export type EvalStatsPeriod = '24h' | '7d' | '30d' | 'all';
+export type EvalStatsPeriod = '24h' | '2d' | '7d' | '14d' | '30d' | '60d' | '90d' | '180d' | 'all';
 export interface EvalStats {
     passRate: number;
     avgScore: number;

package/dist/utils/iris-home.d.ts

@@ -0,0 +1 @@
+export declare function irisHome(): string;

package/dist/utils/iris-home.js

@@ -0,0 +1,21 @@
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+/*
+ * Single resolver for the iris home directory (default: ~/.iris).
+ *
+ * Every per-user file iris touches lives under this directory — the
+ * SQLite DB default, config.json, custom-rules.json, audit.log,
+ * preferences.json. Before this helper each module joined
+ * homedir() + '.iris' itself, which meant there was no way to point a
+ * spawned server at a scratch directory: the E2E suite isolated the DB
+ * via IRIS_DB_PATH but still wiped the real audit.log, deployed test
+ * rules into the real custom-rules.json, and overwrote the real
+ * preferences.json on every run.
+ *
+ * IRIS_HOME redirects all of them at once. Read at call time — not
+ * module load — so a test harness that sets the env var before
+ * spawning (or between in-process calls) always wins.
+ */
+export function irisHome() {
+    return process.env.IRIS_HOME ?? join(homedir(), '.iris');
+}