@iris-eval/mcp-server 0.4.4 → 0.4.5

package/README.md

@@ -41,7 +41,7 @@ Iris evaluates all of it.
 
 ## Quickstart
 
-Add Iris to your MCP config. Works with Claude Desktop, Cursor, Windsurf, and any MCP-compatible agent.
+Add Iris to your MCP config. Works with Claude Desktop, Claude Code, Cursor, Windsurf, Continue, VS Code, Cline, Zed, Codex CLI, Gemini CLI — and any other MCP-compatible agent.
 
 ```json
 {
@@ -102,6 +102,60 @@ Then restart the session (`/clear` or relaunch) for tools to load.
 
 Add to your workspace `.cursor/mcp.json` or global MCP settings using the JSON config above.
 
+#### VS Code (native MCP)
+
+Add to `.vscode/mcp.json` in your workspace (note: VS Code uses `servers`, not `mcpServers`):
+
+```json
+{
+  "servers": {
+    "iris-eval": {
+      "command": "npx",
+      "args": ["@iris-eval/mcp-server"]
+    }
+  }
+}
+```
+
+#### Cline
+
+Open Cline's MCP Servers panel → Configure MCP Servers, and add the `mcpServers` JSON config above to `cline_mcp_settings.json`.
+
+#### Zed
+
+Add to Zed `settings.json`:
+
+```json
+{
+  "context_servers": {
+    "iris-eval": {
+      "command": {
+        "path": "npx",
+        "args": ["@iris-eval/mcp-server"]
+      }
+    }
+  }
+}
+```
+
+#### OpenAI Codex CLI
+
+Add to `~/.codex/config.toml`:
+
+```toml
+[mcp_servers.iris-eval]
+command = "npx"
+args = ["@iris-eval/mcp-server"]
+```
+
+#### Gemini CLI
+
+Add the `mcpServers` JSON config above to `~/.gemini/settings.json`.
+
+#### Anything else that speaks MCP
+
+Iris is a standard stdio MCP server — one `npx @iris-eval/mcp-server` command, no SDK, no code changes. If your client supports MCP, it supports Iris. Client config formats change; when in doubt, check your client's MCP docs and point it at that command.
+
 </details>
 
 ### Other Install Methods

package/dist/audit-log-reader.js

@@ -28,7 +28,7 @@ const EntrySchema = z.object({
     user: z.string(),
     ruleId: z.string(),
     ruleName: z.string().optional(),
-    details: z.record(z.unknown()).optional(),
+    details: z.record(z.string(), z.unknown()).optional(),
 });
 function defaultAuditPath() {
     return join(homedir(), '.iris', 'audit.log');

package/dist/config/defaults.d.ts

@@ -1,2 +1,3 @@
 import type { IrisConfig } from '../types/index.js';
+export declare const PKG_VERSION: string;
 export declare const defaultConfig: IrisConfig;

package/dist/config/defaults.js

@@ -11,6 +11,9 @@ try {
 catch {
     // Fallback if package.json isn't resolvable at runtime
 }
+// The single runtime source for the server's own version — import this
+// instead of hardcoding a literal (OTel resource attrs, health, banners).
+export const PKG_VERSION = pkgVersion;
 export const defaultConfig = {
     storage: {
         type: 'sqlite',

package/dist/custom-rule-store.js

@@ -28,6 +28,8 @@ import { join, dirname } from 'node:path';
 import { homedir } from 'node:os';
 import { randomBytes } from 'node:crypto';
 import { z } from 'zod';
+import isSafeRegex from 'safe-regex2';
+import { CUSTOM_RULE_CONFIG_KEYS, readNumericConfig, describeKeys } from './eval/rules/config-keys.js';
 import { LOCAL_TENANT } from './types/tenant.js';
 const SEVERITY_VALUES = ['low', 'medium', 'high', 'critical'];
 const EVAL_TYPE_VALUES = ['completeness', 'relevance', 'safety', 'cost', 'custom'];
@@ -41,11 +43,116 @@ const RULE_TYPE_VALUES = [
     'json_schema',
     'cost_threshold',
 ];
-const DefinitionSchema = z.object({
+// Per-type config requirements, enforced at DEPLOY time.
+//
+// `config` was previously `z.record(z.unknown())` — any object passed. That
+// let a rule like {type:'min_length', config:{}} deploy successfully and then
+// fail on every single evaluation forever, silently dragging down aggregate
+// scores with no indication the RULE (not the agent) was broken. Validating
+// here means the failure surfaces once, at deploy, with an actionable message
+// — instead of quietly corrupting every eval that follows.
+const MAX_RULE_PATTERN_LENGTH = 1000;
+function requirePositiveNumber(config, type, ctx) {
+    const value = readNumericConfig(config, type);
+    if (value == null || value <= 0) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            path: ['config', CUSTOM_RULE_CONFIG_KEYS[type][0]],
+            message: `${type} rule requires ${describeKeys(type)} (positive number)`,
+        });
+    }
+}
+function requireNonEmptyStringArray(config, key, ctx, hint) {
+    const value = config[key];
+    if (!Array.isArray(value) || value.length === 0 || !value.every((v) => typeof v === 'string')) {
+        ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['config', key], message: hint });
+    }
+}
+const DefinitionSchema = z
+    .object({
     name: z.string().min(1).max(80),
     type: z.enum(RULE_TYPE_VALUES),
-    config: z.record(z.unknown()),
+    config: z.record(z.string(), z.unknown()),
     weight: z.number().positive().optional(),
+})
+    .superRefine((def, ctx) => {
+    const config = def.config ?? {};
+    switch (def.type) {
+        case 'regex_match':
+        case 'regex_no_match': {
+            const pattern = config.pattern;
+            if (typeof pattern !== 'string' || pattern.length === 0) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ['config', 'pattern'],
+                    message: `${def.type} rule requires config.pattern (non-empty string)`,
+                });
+                break;
+            }
+            if (pattern.length > MAX_RULE_PATTERN_LENGTH) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ['config', 'pattern'],
+                    message: `Regex pattern too long (${pattern.length} > ${MAX_RULE_PATTERN_LENGTH})`,
+                });
+                break;
+            }
+            // Strip a leading inline flag group the way the evaluator does, so a
+            // pattern that WILL run is not rejected here for syntax it tolerates.
+            const stripped = pattern.replace(/^\(\?[imsugy]+\)/, '');
+            // Syntax BEFORE safety: safe-regex2 returns false for anything it
+            // cannot parse, so checking it first reports a plainly broken pattern
+            // like `(` as "catastrophic backtracking" — an error that sends the
+            // author looking for a performance problem they do not have.
+            try {
+                new RegExp(stripped, typeof config.flags === 'string' ? config.flags : '');
+            }
+            catch (e) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ['config', 'pattern'],
+                    message: `Invalid regex syntax: ${e instanceof Error ? e.message : 'unknown error'}`,
+                });
+                break;
+            }
+            if (!isSafeRegex(stripped)) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ['config', 'pattern'],
+                    message: 'Regex pattern rejected: potentially unsafe (catastrophic backtracking)',
+                });
+            }
+            break;
+        }
+        case 'min_length':
+            requirePositiveNumber(config, 'min_length', ctx);
+            break;
+        case 'max_length':
+            requirePositiveNumber(config, 'max_length', ctx);
+            break;
+        case 'contains_keywords':
+            requireNonEmptyStringArray(config, 'keywords', ctx, 'contains_keywords rule requires config.keywords (non-empty string array)');
+            break;
+        case 'excludes_keywords':
+            requireNonEmptyStringArray(config, 'keywords', ctx, 'excludes_keywords rule requires config.keywords (non-empty string array)');
+            break;
+        case 'cost_threshold': {
+            // 0 is a legitimate threshold ("must be free"), so only reject
+            // missing / non-numeric / negative.
+            const max = readNumericConfig(config, 'cost_threshold');
+            if (max == null || max < 0) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ['config', CUSTOM_RULE_CONFIG_KEYS.cost_threshold[0]],
+                    message: `cost_threshold rule requires ${describeKeys('cost_threshold')} (non-negative number)`,
+                });
+            }
+            break;
+        }
+        case 'json_schema':
+            // No required config — validity is judged against the output itself.
+            break;
+    }
 });
 const DeployedRuleSchema = z.object({
     id: z.string().min(1),