@iris-eval/mcp-server 0.4.4 → 0.4.6

package/README.md

@@ -41,7 +41,7 @@ Iris evaluates all of it.
 
 ## Quickstart
 
-Add Iris to your MCP config. Works with Claude Desktop, Cursor, Windsurf, and any MCP-compatible agent.
+Add Iris to your MCP config. Works with Claude Desktop, Claude Code, Cursor, Windsurf, Continue, VS Code, Cline, Zed, Codex CLI, Gemini CLI — and any other MCP-compatible agent.
 
 ```json
 {
@@ -102,6 +102,60 @@ Then restart the session (`/clear` or relaunch) for tools to load.
 
 Add to your workspace `.cursor/mcp.json` or global MCP settings using the JSON config above.
 
+#### VS Code (native MCP)
+
+Add to `.vscode/mcp.json` in your workspace (note: VS Code uses `servers`, not `mcpServers`):
+
+```json
+{
+  "servers": {
+    "iris-eval": {
+      "command": "npx",
+      "args": ["@iris-eval/mcp-server"]
+    }
+  }
+}
+```
+
+#### Cline
+
+Open Cline's MCP Servers panel → Configure MCP Servers, and add the `mcpServers` JSON config above to `cline_mcp_settings.json`.
+
+#### Zed
+
+Add to Zed `settings.json`:
+
+```json
+{
+  "context_servers": {
+    "iris-eval": {
+      "command": {
+        "path": "npx",
+        "args": ["@iris-eval/mcp-server"]
+      }
+    }
+  }
+}
+```
+
+#### OpenAI Codex CLI
+
+Add to `~/.codex/config.toml`:
+
+```toml
+[mcp_servers.iris-eval]
+command = "npx"
+args = ["@iris-eval/mcp-server"]
+```
+
+#### Gemini CLI
+
+Add the `mcpServers` JSON config above to `~/.gemini/settings.json`.
+
+#### Anything else that speaks MCP
+
+Iris is a standard stdio MCP server — one `npx @iris-eval/mcp-server` command, no SDK, no code changes. If your client supports MCP, it supports Iris. Client config formats change; when in doubt, check your client's MCP docs and point it at that command.
+
 </details>
 
 ### Other Install Methods

package/dist/audit-log-reader.js

@@ -12,7 +12,7 @@
  */
 import { readFileSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
-import { homedir } from 'node:os';
+import { irisHome } from './utils/iris-home.js';
 import { z } from 'zod';
 const AUDIT_ACTIONS = ['rule.deploy', 'rule.delete', 'rule.toggle', 'rule.update'];
 const EntrySchema = z.object({
@@ -28,10 +28,10 @@ const EntrySchema = z.object({
     user: z.string(),
     ruleId: z.string(),
     ruleName: z.string().optional(),
-    details: z.record(z.unknown()).optional(),
+    details: z.record(z.string(), z.unknown()).optional(),
 });
 function defaultAuditPath() {
-    return join(homedir(), '.iris', 'audit.log');
+    return join(irisHome(), 'audit.log');
 }
 export function readAuditLog(opts) {
     const filePath = opts?.filePath ?? defaultAuditPath();

package/dist/config/defaults.d.ts

@@ -1,2 +1,3 @@
 import type { IrisConfig } from '../types/index.js';
+export declare const PKG_VERSION: string;
 export declare const defaultConfig: IrisConfig;

package/dist/config/defaults.js

@@ -1,7 +1,6 @@
 import { join } from 'node:path';
 import { readFileSync } from 'node:fs';
-import { homedir } from 'node:os';
-const irisHome = join(homedir(), '.iris');
+import { irisHome } from '../utils/iris-home.js';
 // Read version from package.json to avoid hardcoded drift
 let pkgVersion = '0.1.8';
 try {
@@ -11,10 +10,13 @@ try {
 catch {
     // Fallback if package.json isn't resolvable at runtime
 }
+// The single runtime source for the server's own version — import this
+// instead of hardcoding a literal (OTel resource attrs, health, banners).
+export const PKG_VERSION = pkgVersion;
 export const defaultConfig = {
     storage: {
         type: 'sqlite',
-        path: join(irisHome, 'iris.db'),
+        path: join(irisHome(), 'iris.db'),
     },
     server: {
         name: 'iris-eval',
@@ -28,6 +30,7 @@ export const defaultConfig = {
     dashboard: {
         enabled: false,
         port: 6920,
+        host: '127.0.0.1',
     },
     eval: {
         defaultThreshold: 0.7,

package/dist/config/index.d.ts

@@ -6,6 +6,7 @@ export interface CliArgs {
     dbPath?: string;
     dashboard?: boolean;
     dashboardPort?: number;
+    dashboardHost?: string;
     apiKey?: string;
 }
 export declare function loadConfig(cliArgs?: CliArgs): IrisConfig;

package/dist/config/index.js

@@ -1,7 +1,7 @@
 import { readFileSync, mkdirSync, existsSync } from 'node:fs';
 import { join, dirname } from 'node:path';
-import { homedir } from 'node:os';
 import { defaultConfig } from './defaults.js';
+import { irisHome } from '../utils/iris-home.js';
 function deepMerge(target, source) {
     const result = { ...target };
     for (const key of Object.keys(source)) {
@@ -71,6 +71,12 @@ function loadEnvVars() {
             port: parsePortEnv(process.env.IRIS_DASHBOARD_PORT, 'IRIS_DASHBOARD_PORT'),
         };
     }
+    if (process.env.IRIS_DASHBOARD_HOST) {
+        config.dashboard = {
+            ...config.dashboard,
+            host: process.env.IRIS_DASHBOARD_HOST,
+        };
+    }
     if (process.env.IRIS_API_KEY) {
         config.security = { ...config.security, apiKey: process.env.IRIS_API_KEY };
     }
@@ -99,17 +105,20 @@ function cliArgsToConfig(args) {
     if (args.dashboardPort) {
         config.dashboard = { ...config.dashboard, port: args.dashboardPort };
     }
+    if (args.dashboardHost) {
+        config.dashboard = { ...config.dashboard, host: args.dashboardHost };
+    }
     if (args.apiKey) {
         config.security = { ...config.security, apiKey: args.apiKey };
     }
     return config;
 }
 export function loadConfig(cliArgs) {
-    const irisHome = join(homedir(), '.iris');
-    if (!existsSync(irisHome)) {
-        mkdirSync(irisHome, { recursive: true });
+    const home = irisHome();
+    if (!existsSync(home)) {
+        mkdirSync(home, { recursive: true });
     }
-    const configPath = cliArgs?.config ?? join(irisHome, 'config.json');
+    const configPath = cliArgs?.config ?? join(home, 'config.json');
     const fileConfig = loadConfigFile(configPath);
     const envConfig = loadEnvVars();
     const argsConfig = cliArgs ? cliArgsToConfig(cliArgs) : {};

package/dist/custom-rule-store.js

@@ -23,11 +23,15 @@
  * For now we use atomic write-via-rename so a crashed write doesn't
  * leave a half-file.
  */
-import { mkdirSync, readFileSync, writeFileSync, existsSync, renameSync, appendFileSync } from 'node:fs';
+import { mkdirSync, readFileSync, existsSync, appendFileSync } from 'node:fs';
+import { writeAtomic } from './utils/write-atomic.js';
+import { irisHome } from './utils/iris-home.js';
 import { join, dirname } from 'node:path';
-import { homedir } from 'node:os';
 import { randomBytes } from 'node:crypto';
 import { z } from 'zod';
+import isSafeRegex from 'safe-regex2';
+import { regexBacktrackingBudgetExceeded } from './eval/rules/regex-budget.js';
+import { CUSTOM_RULE_CONFIG_KEYS, readNumericConfig, describeKeys } from './eval/rules/config-keys.js';
 import { LOCAL_TENANT } from './types/tenant.js';
 const SEVERITY_VALUES = ['low', 'medium', 'high', 'critical'];
 const EVAL_TYPE_VALUES = ['completeness', 'relevance', 'safety', 'cost', 'custom'];
@@ -41,11 +45,131 @@ const RULE_TYPE_VALUES = [
     'json_schema',
     'cost_threshold',
 ];
-const DefinitionSchema = z.object({
+// Per-type config requirements, enforced at DEPLOY time.
+//
+// `config` was previously `z.record(z.unknown())` — any object passed. That
+// let a rule like {type:'min_length', config:{}} deploy successfully and then
+// fail on every single evaluation forever, silently dragging down aggregate
+// scores with no indication the RULE (not the agent) was broken. Validating
+// here means the failure surfaces once, at deploy, with an actionable message
+// — instead of quietly corrupting every eval that follows.
+const MAX_RULE_PATTERN_LENGTH = 1000;
+function requirePositiveNumber(config, type, ctx) {
+    const value = readNumericConfig(config, type);
+    if (value == null || value <= 0) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            path: ['config', CUSTOM_RULE_CONFIG_KEYS[type][0]],
+            message: `${type} rule requires ${describeKeys(type)} (positive number)`,
+        });
+    }
+}
+function requireNonEmptyStringArray(config, key, ctx, hint) {
+    const value = config[key];
+    if (!Array.isArray(value) || value.length === 0 || !value.every((v) => typeof v === 'string')) {
+        ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['config', key], message: hint });
+    }
+}
+const DefinitionSchema = z
+    .object({
     name: z.string().min(1).max(80),
     type: z.enum(RULE_TYPE_VALUES),
-    config: z.record(z.unknown()),
+    config: z.record(z.string(), z.unknown()),
     weight: z.number().positive().optional(),
+})
+    .superRefine((def, ctx) => {
+    const config = def.config ?? {};
+    switch (def.type) {
+        case 'regex_match':
+        case 'regex_no_match': {
+            const pattern = config.pattern;
+            if (typeof pattern !== 'string' || pattern.length === 0) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ['config', 'pattern'],
+                    message: `${def.type} rule requires config.pattern (non-empty string)`,
+                });
+                break;
+            }
+            if (pattern.length > MAX_RULE_PATTERN_LENGTH) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ['config', 'pattern'],
+                    message: `Regex pattern too long (${pattern.length} > ${MAX_RULE_PATTERN_LENGTH})`,
+                });
+                break;
+            }
+            // Strip a leading inline flag group the way the evaluator does, so a
+            // pattern that WILL run is not rejected here for syntax it tolerates.
+            const stripped = pattern.replace(/^\(\?[imsugy]+\)/, '');
+            // Syntax BEFORE safety: safe-regex2 returns false for anything it
+            // cannot parse, so checking it first reports a plainly broken pattern
+            // like `(` as "catastrophic backtracking" — an error that sends the
+            // author looking for a performance problem they do not have.
+            try {
+                new RegExp(stripped, typeof config.flags === 'string' ? config.flags : '');
+            }
+            catch (e) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ['config', 'pattern'],
+                    message: `Invalid regex syntax: ${e instanceof Error ? e.message : 'unknown error'}`,
+                });
+                break;
+            }
+            if (!isSafeRegex(stripped)) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ['config', 'pattern'],
+                    message: 'Regex pattern rejected: potentially unsafe (catastrophic backtracking)',
+                });
+                break;
+            }
+            // safe-regex2 is a star-height heuristic — it catches EXPONENTIAL
+            // blowup only. Polynomial patterns pass it: a*a*a*a*a*b is judged
+            // safe and takes 156ms on 40 characters. Measure what the static
+            // check cannot see.
+            {
+                const budgetIssue = regexBacktrackingBudgetExceeded(stripped, typeof config.flags === 'string' ? config.flags : '');
+                if (budgetIssue) {
+                    ctx.addIssue({
+                        code: z.ZodIssueCode.custom,
+                        path: ['config', 'pattern'],
+                        message: budgetIssue,
+                    });
+                }
+            }
+            break;
+        }
+        case 'min_length':
+            requirePositiveNumber(config, 'min_length', ctx);
+            break;
+        case 'max_length':
+            requirePositiveNumber(config, 'max_length', ctx);
+            break;
+        case 'contains_keywords':
+            requireNonEmptyStringArray(config, 'keywords', ctx, 'contains_keywords rule requires config.keywords (non-empty string array)');
+            break;
+        case 'excludes_keywords':
+            requireNonEmptyStringArray(config, 'keywords', ctx, 'excludes_keywords rule requires config.keywords (non-empty string array)');
+            break;
+        case 'cost_threshold': {
+            // 0 is a legitimate threshold ("must be free"), so only reject
+            // missing / non-numeric / negative.
+            const max = readNumericConfig(config, 'cost_threshold');
+            if (max == null || max < 0) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ['config', CUSTOM_RULE_CONFIG_KEYS.cost_threshold[0]],
+                    message: `cost_threshold rule requires ${describeKeys('cost_threshold')} (non-negative number)`,
+                });
+            }
+            break;
+        }
+        case 'json_schema':
+            // No required config — validity is judged against the output itself.
+            break;
+    }
 });
 const DeployedRuleSchema = z.object({
     id: z.string().min(1),
@@ -60,26 +184,22 @@ const DeployedRuleSchema = z.object({
     sourceMomentId: z.string().optional(),
     version: z.number().int().positive(),
 });
-const FileSchema = z.object({
-    version: z.literal(1),
-    rules: z.array(DeployedRuleSchema),
-});
 /**
  * Default file path for a tenant. LOCAL_TENANT keeps the v0.4 path
  * (zero migration); others get a per-tenant suffix.
  */
 function defaultPathFor(tenantId) {
     if (tenantId === LOCAL_TENANT) {
-        return join(homedir(), '.iris', 'custom-rules.json');
+        return join(irisHome(), 'custom-rules.json');
     }
     // Sanitize tenant id for filesystem safety. TenantId is branded but
     // could in principle contain odd chars on Cloud — limit to a known-safe
     // alphabet so we never write outside the .iris directory.
     const safe = String(tenantId).replace(/[^a-zA-Z0-9._-]/g, '_');
-    return join(homedir(), '.iris', `custom-rules-${safe}.json`);
+    return join(irisHome(), `custom-rules-${safe}.json`);
 }
 function defaultAuditPath() {
-    return join(homedir(), '.iris', 'audit.log');
+    return join(irisHome(), 'audit.log');
 }
 function generateRuleId() {
     return `rule-${randomBytes(4).toString('hex')}`;
@@ -94,45 +214,80 @@ function appendAudit(auditPath, entry) {
         // still succeeds; the operator just loses the audit trail.
     }
 }
-function writeAtomic(targetPath, contents) {
-    mkdirSync(dirname(targetPath), { recursive: true });
-    const tmp = `${targetPath}.tmp.${process.pid}`;
-    writeFileSync(tmp, contents, 'utf-8');
-    renameSync(tmp, targetPath);
-}
+/*
+ * Read leniently, one rule at a time.
+ *
+ * This used to validate the whole array with a single safeParse and return
+ * [] if ANY element failed. The empty result was then cached, and the next
+ * deploy/delete/toggle called persist(), which wrote {version:1, rules:[]}
+ * over the file — permanently destroying every valid rule alongside the
+ * bad one. The old comment ("do NOT overwrite the file") described an
+ * intent the write path did not honour.
+ *
+ * It was reachable, not theoretical: DefinitionSchema's superRefine now
+ * runs on READ as well as WRITE, and eval/rules/custom.ts notes that rules
+ * predating that validation — e.g. {type:'min_length', config:{}} — are
+ * already sitting in users' files.
+ */
 function loadRulesFromDisk(rulesPath) {
     if (!existsSync(rulesPath))
-        return [];
+        return { rules: [], quarantined: [], readable: true };
+    let parsedJson;
     try {
-        const raw = readFileSync(rulesPath, 'utf-8');
-        const parsed = FileSchema.safeParse(JSON.parse(raw));
-        if (parsed.success)
-            return parsed.data.rules;
-        // Malformed: leave rules empty; do NOT overwrite the file.
-        return [];
+        parsedJson = JSON.parse(readFileSync(rulesPath, 'utf-8'));
     }
     catch {
-        // Unreadable: leave rules empty.
-        return [];
+        return { rules: [], quarantined: [], readable: false };
+    }
+    const envelope = z.object({ rules: z.array(z.unknown()).optional() }).safeParse(parsedJson);
+    if (!envelope.success)
+        return { rules: [], quarantined: [], readable: false };
+    const rules = [];
+    const quarantined = [];
+    for (const entry of envelope.data.rules ?? []) {
+        const rule = DeployedRuleSchema.safeParse(entry);
+        if (rule.success)
+            rules.push(rule.data);
+        else
+            quarantined.push(entry);
     }
+    return { rules, quarantined, readable: true };
 }
 export function createCustomRuleStore(opts) {
     const pathFor = opts?.pathFor ?? defaultPathFor;
     const auditPath = opts?.auditPath ?? defaultAuditPath();
     // In-memory cache keyed by tenant. Lazy-loaded on first access per
     // tenant; subsequent calls hit the cache.
-    const tenantRules = new Map();
-    function load(tenantId) {
-        let rules = tenantRules.get(tenantId);
-        if (rules === undefined) {
-            rules = loadRulesFromDisk(pathFor(tenantId));
-            tenantRules.set(tenantId, rules);
+    const tenantState = new Map();
+    function state(tenantId) {
+        let loaded = tenantState.get(tenantId);
+        if (loaded === undefined) {
+            loaded = loadRulesFromDisk(pathFor(tenantId));
+            tenantState.set(tenantId, loaded);
         }
-        return rules;
+        return loaded;
+    }
+    function load(tenantId) {
+        return state(tenantId).rules;
     }
     function persist(tenantId) {
-        const rules = tenantRules.get(tenantId) ?? [];
-        const file = { version: 1, rules };
+        const loaded = state(tenantId);
+        if (!loaded.readable) {
+            /*
+             * The file exists but never parsed. Overwriting it would replace
+             * content we could not read — exactly the data loss this store used
+             * to cause silently. Fail loudly so the caller surfaces a 500 and
+             * the operator can fix or move the file.
+             */
+            throw new Error(`Refusing to write ${pathFor(tenantId)}: the existing file could not be parsed. ` +
+                `Fix or move it, then retry — writing now would destroy its contents.`);
+        }
+        // Quarantined entries ride along untouched so a deploy never deletes
+        // rules this version could not validate.
+        const file = {
+            version: 1,
+            rules: [...loaded.rules, ...loaded.quarantined],
+        };
         writeAtomic(pathFor(tenantId), JSON.stringify(file, null, 2));
     }
     return {