@iris-eval/mcp-server 0.3.0 → 0.4.0-rc.1

package/README.md

@@ -31,7 +31,7 @@ Iris evaluates all of it.
 | | |
 |---|---|
 | **Trace Logging** | Hierarchical span trees with per-tool-call latency, token usage, and cost in USD. Stored in SQLite, queryable instantly. |
-| **Output Evaluation** | 12 built-in rules across 4 categories: completeness, relevance, safety, cost. PII detection, prompt injection patterns, hallucination markers. Add custom rules with Zod schemas. |
+| **Output Evaluation** | 13 built-in rules across 4 categories: completeness, relevance, safety, cost. PII detection (10 patterns: SSN, credit card, phone, email, IBAN, DOB, MRN, IP, API key, passport), prompt injection (13 patterns), stub-output detection, hallucination markers (17 hedging phrases + fabricated-citation heuristic). Add custom rules with Zod schemas. |
 | **Cost Visibility** | Aggregate cost across all agents over any time window. Set budget thresholds. Get flagged when agents overspend. |
 | **Web Dashboard** | Real-time dark-mode UI with trace visualization, eval results, and cost breakdowns. |
 
@@ -117,11 +117,19 @@ docker run -p 3000:3000 -v iris-data:/data ghcr.io/iris-eval/mcp-server
 
 ## MCP Tools
 
-Iris registers three tools that any MCP-compatible agent can invoke:
+Iris registers nine tools that any MCP-compatible agent can invoke — full rule + trace lifecycle + LLM-as-judge + semantic citation verification:
 
 - **`log_trace`** — Log an agent execution with spans, tool calls, token usage, and cost
-- **`evaluate_output`** — Score output quality against completeness, relevance, safety, and cost rules
+- **`evaluate_output`** — Score output quality against completeness, relevance, safety, and cost rules (heuristic, deterministic, free)
 - **`get_traces`** — Query stored traces with filtering, pagination, and time-range support
+- **`list_rules`** — Enumerate deployed custom eval rules (read-only)
+- **`deploy_rule`** — Register a new custom eval rule so it fires on every `evaluate_output` of that category
+- **`delete_rule`** — Remove a deployed custom rule (destructive, idempotent)
+- **`delete_trace`** — Remove a single stored trace by ID (destructive, tenant-scoped)
+- **`evaluate_with_llm_judge`** — Semantic eval via LLM (Anthropic or OpenAI). Five templates: accuracy, helpfulness, safety, correctness, faithfulness. Cost-capped, per-eval pricing disclosed.
+- **`verify_citations`** — Extract citations from output (numbered, author-year, URLs, DOIs), fetch sources behind an SSRF-guarded + domain-allowlisted resolver, and use an LLM judge to check whether each source actually supports the cited claim. Opt-in outbound HTTP.
+
+When `IRIS_OTEL_ENDPOINT` is configured, `log_trace` calls also emit a best-effort OTLP/HTTP JSON export to any OpenTelemetry collector (Jaeger, Grafana Tempo, Datadog OTLP, Honeycomb, etc). See [docs/otel-integration.md](docs/otel-integration.md).
 
 Full tool schemas and configuration: [iris-eval.com](https://iris-eval.com)
 

package/dist/audit-log-reader.d.ts

@@ -0,0 +1,24 @@
+import type { AuditLogEntry } from './types/custom-rule.js';
+declare const AUDIT_ACTIONS: readonly ["rule.deploy", "rule.delete", "rule.toggle", "rule.update"];
+export interface AuditQueryFilter {
+    action?: typeof AUDIT_ACTIONS[number];
+    /** ISO timestamp; only entries at or after. */
+    since?: string;
+    /** Optional substring match against ruleId or ruleName. */
+    search?: string;
+}
+export interface AuditQueryResult {
+    entries: AuditLogEntry[];
+    total: number;
+    limit: number;
+    offset: number;
+    /** Absolute path to the audit log file (for diagnostics). */
+    path: string;
+}
+export declare function readAuditLog(opts?: {
+    filePath?: string;
+    filter?: AuditQueryFilter;
+    limit?: number;
+    offset?: number;
+}): AuditQueryResult;
+export {};

package/dist/audit-log-reader.js

@@ -0,0 +1,87 @@
+/*
+ * audit-log-reader — read + parse ~/.iris/audit.log (JSONL append-only).
+ *
+ * The file is written by custom-rule-store.ts (rule.deploy / rule.delete /
+ * rule.toggle / rule.update). This reader hydrates entries for the
+ * dashboard's GET /api/v1/audit endpoint.
+ *
+ * Performance: the audit log can grow over time. We read the whole file,
+ * parse line-by-line, and return only the latest N entries (newest first).
+ * For audit volumes < 10k entries this is comfortably sub-50ms; v0.5 will
+ * tail-read or paginate via byte offsets if it ever matters.
+ */
+import { readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import { z } from 'zod';
+const AUDIT_ACTIONS = ['rule.deploy', 'rule.delete', 'rule.toggle', 'rule.update'];
+const EntrySchema = z.object({
+    ts: z.string(),
+    /*
+     * tenantId added in v0.4.0. Optional for backward compatibility:
+     * entries written by v0.3.x don't have it; readers MUST treat them
+     * as 'local' (single-tenant OSS assumption). See threat model §4.3
+     * finding TM-4 for the Cloud-side enforcement plan.
+     */
+    tenantId: z.string().optional(),
+    action: z.enum(AUDIT_ACTIONS),
+    user: z.string(),
+    ruleId: z.string(),
+    ruleName: z.string().optional(),
+    details: z.record(z.unknown()).optional(),
+});
+function defaultAuditPath() {
+    return join(homedir(), '.iris', 'audit.log');
+}
+export function readAuditLog(opts) {
+    const filePath = opts?.filePath ?? defaultAuditPath();
+    const limit = Math.min(Math.max(opts?.limit ?? 100, 1), 1000);
+    const offset = Math.max(opts?.offset ?? 0, 0);
+    if (!existsSync(filePath)) {
+        return { entries: [], total: 0, limit, offset, path: filePath };
+    }
+    let raw;
+    try {
+        raw = readFileSync(filePath, 'utf-8');
+    }
+    catch {
+        return { entries: [], total: 0, limit, offset, path: filePath };
+    }
+    // Parse line-by-line, drop malformed rows silently. The audit log is
+    // append-only so a partially-flushed last line is the only realistic
+    // source of malformation and we shouldn't fail the whole query for it.
+    const all = [];
+    for (const line of raw.split(/\r?\n/)) {
+        if (!line.trim())
+            continue;
+        try {
+            const parsed = EntrySchema.safeParse(JSON.parse(line));
+            if (parsed.success)
+                all.push(parsed.data);
+        }
+        catch {
+            // Skip malformed line
+        }
+    }
+    // Filter
+    const filter = opts?.filter ?? {};
+    const filtered = all.filter((entry) => {
+        if (filter.action && entry.action !== filter.action)
+            return false;
+        if (filter.since && entry.ts < filter.since)
+            return false;
+        if (filter.search) {
+            const needle = filter.search.toLowerCase();
+            const matchId = entry.ruleId.toLowerCase().includes(needle);
+            const matchName = entry.ruleName?.toLowerCase().includes(needle) ?? false;
+            if (!matchId && !matchName)
+                return false;
+        }
+        return true;
+    });
+    // Sort newest first by ts
+    filtered.sort((a, b) => b.ts.localeCompare(a.ts));
+    const total = filtered.length;
+    const entries = filtered.slice(offset, offset + limit);
+    return { entries, total, limit, offset, path: filePath };
+}

package/dist/config/defaults.js

@@ -50,7 +50,13 @@ export const defaultConfig = {
         apiKey: undefined,
         allowedOrigins: ['http://localhost:*'],
         rateLimit: {
-            api: 100,
+            /* Dashboard polls ~6 endpoints every 5–10s (Health view alone hits
+             * stats + trend + audit + 2× moments + priorStats). At 100/min the
+             * dashboard exhausts its own quota and surfaces 429s on /rules etc.
+             * 600/min ≈ 10/s — still rejects abusive crawlers, but accommodates
+             * a polling-heavy first-party dashboard with headroom for navigation
+             * bursts. */
+            api: 600,
             mcp: 20,
         },
         requestSizeLimit: '1mb',

package/dist/custom-rule-store.d.ts

@@ -0,0 +1,27 @@
+import type { DeployedCustomRule, RuleSeverity } from './types/custom-rule.js';
+import type { CustomRuleDefinition, EvalType } from './types/eval.js';
+export interface CustomRuleStore {
+    list(): DeployedCustomRule[];
+    get(id: string): DeployedCustomRule | undefined;
+    deploy(input: DeployRuleInput): DeployedCustomRule;
+    delete(id: string, user?: string): boolean;
+    setEnabled(id: string, enabled: boolean, user?: string): DeployedCustomRule | undefined;
+    /** All ENABLED rules in deploy order — what the engine should register. */
+    enabledRules(): DeployedCustomRule[];
+    /** Path on disk for diagnostics. */
+    filePath: string;
+    auditPath: string;
+}
+export interface DeployRuleInput {
+    name: string;
+    description?: string;
+    evalType: EvalType;
+    severity?: RuleSeverity;
+    definition: CustomRuleDefinition;
+    sourceMomentId?: string;
+    user?: string;
+}
+export declare function createCustomRuleStore(opts?: {
+    rulesPath?: string;
+    auditPath?: string;
+}): CustomRuleStore;

package/dist/custom-rule-store.js

@@ -0,0 +1,188 @@
+/*
+ * custom-rule-store — file-based persistence for deployed custom rules.
+ *
+ * Lives at ~/.iris/custom-rules.json with the schema in
+ * src/types/custom-rule.ts. Audit log lives at ~/.iris/audit.log
+ * (append-only JSONL). Both files are created on first write.
+ *
+ * The v0.4 cut is single-user local. Concurrent writes from multiple
+ * iris-mcp instances are not protected — that's a v0.5 concern when
+ * multi-tenancy lands. For now we use atomic write-via-rename so a
+ * crashed write doesn't leave a half-file.
+ */
+import { mkdirSync, readFileSync, writeFileSync, existsSync, renameSync, appendFileSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { homedir } from 'node:os';
+import { randomBytes } from 'node:crypto';
+import { z } from 'zod';
+const SEVERITY_VALUES = ['low', 'medium', 'high', 'critical'];
+const EVAL_TYPE_VALUES = ['completeness', 'relevance', 'safety', 'cost', 'custom'];
+const RULE_TYPE_VALUES = [
+    'regex_match',
+    'regex_no_match',
+    'min_length',
+    'max_length',
+    'contains_keywords',
+    'excludes_keywords',
+    'json_schema',
+    'cost_threshold',
+];
+const DefinitionSchema = z.object({
+    name: z.string().min(1).max(80),
+    type: z.enum(RULE_TYPE_VALUES),
+    config: z.record(z.unknown()),
+    weight: z.number().positive().optional(),
+});
+const DeployedRuleSchema = z.object({
+    id: z.string().min(1),
+    name: z.string().min(1).max(80),
+    description: z.string().max(500),
+    evalType: z.enum(EVAL_TYPE_VALUES),
+    severity: z.enum(SEVERITY_VALUES),
+    definition: DefinitionSchema,
+    enabled: z.boolean(),
+    createdAt: z.string(),
+    updatedAt: z.string(),
+    sourceMomentId: z.string().optional(),
+    version: z.number().int().positive(),
+});
+const FileSchema = z.object({
+    version: z.literal(1),
+    rules: z.array(DeployedRuleSchema),
+});
+function defaultRulesPath() {
+    return join(homedir(), '.iris', 'custom-rules.json');
+}
+function defaultAuditPath() {
+    return join(homedir(), '.iris', 'audit.log');
+}
+function generateRuleId() {
+    return `rule-${randomBytes(4).toString('hex')}`;
+}
+function appendAudit(auditPath, entry) {
+    try {
+        mkdirSync(dirname(auditPath), { recursive: true });
+        appendFileSync(auditPath, `${JSON.stringify(entry)}\n`, 'utf-8');
+    }
+    catch {
+        // Audit best-effort. If filesystem is read-only or full, the deploy
+        // still succeeds; the operator just loses the audit trail.
+    }
+}
+function writeAtomic(targetPath, contents) {
+    mkdirSync(dirname(targetPath), { recursive: true });
+    const tmp = `${targetPath}.tmp.${process.pid}`;
+    writeFileSync(tmp, contents, 'utf-8');
+    renameSync(tmp, targetPath);
+}
+export function createCustomRuleStore(opts) {
+    const rulesPath = opts?.rulesPath ?? defaultRulesPath();
+    const auditPath = opts?.auditPath ?? defaultAuditPath();
+    // In-memory copy that mirrors the file. Writes update both.
+    let rules = [];
+    // Initial load
+    if (existsSync(rulesPath)) {
+        try {
+            const raw = readFileSync(rulesPath, 'utf-8');
+            const parsed = FileSchema.safeParse(JSON.parse(raw));
+            if (parsed.success) {
+                rules = parsed.data.rules;
+            }
+            // Malformed: leave rules empty; do NOT overwrite the file.
+        }
+        catch {
+            // Unreadable: leave rules empty.
+        }
+    }
+    function persist() {
+        const file = { version: 1, rules };
+        writeAtomic(rulesPath, JSON.stringify(file, null, 2));
+    }
+    return {
+        filePath: rulesPath,
+        auditPath,
+        list() {
+            return [...rules];
+        },
+        get(id) {
+            return rules.find((r) => r.id === id);
+        },
+        enabledRules() {
+            return rules.filter((r) => r.enabled);
+        },
+        deploy(input) {
+            const now = new Date().toISOString();
+            const id = generateRuleId();
+            const rule = {
+                id,
+                name: input.name,
+                description: input.description ?? '',
+                evalType: input.evalType,
+                severity: input.severity ?? 'medium',
+                definition: input.definition,
+                enabled: true,
+                createdAt: now,
+                updatedAt: now,
+                sourceMomentId: input.sourceMomentId,
+                version: 1,
+            };
+            // Validate before persisting.
+            const validated = DeployedRuleSchema.parse(rule);
+            rules.push(validated);
+            persist();
+            /* Tenant scoping: OSS single-tenant emits 'local'. Cloud replaces
+             * the entire custom-rule-store with a DB-backed service that reads
+             * the tenant from the authenticated session; that service will
+             * compute tenantId per-call. Hard-coded here for OSS. */
+            appendAudit(auditPath, {
+                ts: now,
+                tenantId: 'local',
+                action: 'rule.deploy',
+                user: input.user ?? 'local',
+                ruleId: id,
+                ruleName: rule.name,
+                details: input.sourceMomentId
+                    ? { sourceMomentId: input.sourceMomentId, severity: rule.severity }
+                    : { severity: rule.severity },
+            });
+            return validated;
+        },
+        delete(id, user = 'local') {
+            const idx = rules.findIndex((r) => r.id === id);
+            if (idx === -1)
+                return false;
+            const removed = rules[idx];
+            rules.splice(idx, 1);
+            persist();
+            appendAudit(auditPath, {
+                ts: new Date().toISOString(),
+                tenantId: 'local',
+                action: 'rule.delete',
+                user,
+                ruleId: id,
+                ruleName: removed.name,
+            });
+            return true;
+        },
+        setEnabled(id, enabled, user = 'local') {
+            const rule = rules.find((r) => r.id === id);
+            if (!rule)
+                return undefined;
+            if (rule.enabled === enabled)
+                return rule;
+            rule.enabled = enabled;
+            rule.updatedAt = new Date().toISOString();
+            persist();
+            appendAudit(auditPath, {
+                ts: rule.updatedAt,
+                tenantId: 'local',
+                action: 'rule.toggle',
+                user,
+                ruleId: id,
+                ruleName: rule.name,
+                details: { enabled },
+            });
+            return rule;
+        },
+    };
+}

package/dist/dashboard/assets/index-BEG5FYWH.css

@@ -0,0 +1 @@
+@import "https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@500..700&family=Manrope:wght@400..700&family=JetBrains+Mono:wght@400..700&display=swap";:root{--lightningcss-light:initial;--lightningcss-dark: ;color-scheme:light dark;--iris-50:#f0fdfa;--iris-100:#ccfbf1;--iris-200:#99f6e4;--iris-300:#5eead4;--iris-400:#2dd4bf;--iris-500:#14b8a6;--iris-600:#0d9488;--iris-700:#0f766e;--iris-800:#115e59;--iris-900:#134e4a;--iris-950:#042f2e;--eval-pass:#22c55e;--eval-warn:#eab308;--eval-fail:#ef4444;--eval-tool:#3b82f6;--eval-llm:#a855f7;--eval-skipped:#71717a}@media (prefers-color-scheme:dark){:root{--lightningcss-light: ;--lightningcss-dark:initial}}:root,[data-theme=dark]{--bg-base:#050508;--bg-raised:#08080e;--bg-surface:#0d0d15;--bg-card:#101018;--bg-card-hover:#16161f;--border-subtle:#ffffff0d;--border-default:#ffffff14;--border-strong:#ffffff24;--border-glow:#14b8a680;--text-primary:#f0f0f5;--text-secondary:#9494a8;--text-muted:#5e5e72;--text-accent:var(--iris-400);--glow-primary:#14b8a61f;--glow-strong:#14b8a640;--shadow-sm:0 1px 2px #0000004d;--shadow-md:0 4px 6px #0006;--shadow-lg:0 10px 15px #00000080}[data-theme=light]{--bg-base:#fafcfc;--bg-raised:#f1f5f5;--bg-surface:#e8eded;--bg-card:#fff;--bg-card-hover:#f4f8f8;--border-subtle:#0000000a;--border-default:#00000014;--border-strong:#00000024;--border-glow:#0d948859;--text-primary:#0a0f0e;--text-secondary:#3d5250;--text-muted:#7a908e;--text-accent:var(--iris-700);--glow-primary:#0d94880f;--glow-strong:#0d94881f;--shadow-sm:0 1px 2px #0000000f;--shadow-md:0 4px 6px #00000014;--shadow-lg:0 10px 15px #0000001a}:root{--font-display:"Space Grotesk", -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;--font-body:"Manrope", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;--font-mono:"JetBrains Mono", "Fira Code", ui-monospace, monospace;--font-sans:var(--font-body);--text-caption-xs:11px;--text-caption:12px;--text-body-sm:13px;--text-body:14px;--text-body-lg:15px;--text-heading-sm:16px;--text-heading:20px;--text-display-sm:28px;--text-display:40px;--font-size-xs:var(--text-caption);--font-size-sm:var(--text-body-sm);--font-size-base:var(--text-body);--font-size-lg:var(--text-body-lg);--font-size-xl:var(--text-heading-sm);--font-size-2xl:var(--text-heading);--font-size-3xl:var(--text-display-sm);--leading-body:1.5;--leading-heading:1.2;--leading-display:1.1;--leading-mono:1.4;--space-0_5:2px;--space-1:4px;--space-1_5:6px;--space-2:8px;--space-2_5:10px;--space-3:12px;--space-4:16px;--space-5:20px;--space-6:24px;--space-8:32px;--space-10:40px;--space-12:48px;--space-16:64px;--space-20:80px;--space-24:96px}:root,[data-density=compact]{--density-row:32px;--density-padding:var(--space-3);--density-body:var(--text-body-sm)}[data-density=comfortable]{--density-row:44px;--density-padding:var(--space-4);--density-body:var(--text-body)}:root{--sidebar-width-expanded:256px;--sidebar-width-collapsed:64px;--header-height:56px;--page-toolbar-height:40px;--radius-xs:4px;--radius-sm:6px;--radius:8px;--radius-lg:12px;--radius-xl:16px;--radius-pill:999px;--border-radius:var(--radius);--border-radius-sm:var(--radius-xs);--border-radius-lg:var(--radius-lg);--transition-instant:.1s ease;--transition-fast:.15s ease;--transition-base:.2s ease;--transition-slow:.3s ease;--ease-iris:cubic-bezier(.25, .4, .25, 1);--bg-primary:var(--bg-base);--bg-secondary:var(--bg-raised);--bg-tertiary:var(--bg-surface);--bg-hover:var(--bg-card-hover);--border-color:var(--border-default);--accent-primary:var(--iris-500);--accent-primary-hover:var(--iris-400);--accent-success:var(--eval-pass);--accent-error:var(--eval-fail);--accent-warning:var(--eval-warn);--accent-tool:var(--eval-tool);--accent-llm:var(--eval-llm)}html{transition:background-color var(--transition-base), color var(--transition-base)}*,:before,:after{box-sizing:border-box;margin:0;padding:0}html,body,#root{width:100%;height:100%}body{font-family:var(--font-body);font-size:var(--text-body);color:var(--text-primary);background-color:var(--bg-base);line-height:var(--leading-body);-webkit-font-smoothing:antialiased;font-feature-settings:"cv11", "ss01"}h1,h2,h3,h4,.display{font-family:var(--font-display);letter-spacing:-.01em;font-weight:600}a{color:var(--text-accent);text-decoration:none}a:hover{color:var(--iris-300)}button{cursor:pointer;font-family:inherit}input,select{font-family:inherit;font-size:inherit}code,pre{font-family:var(--font-mono)}a:focus-visible,button:focus-visible,input:focus-visible,select:focus-visible,[role=button]:focus-visible{outline:2px solid var(--iris-500);outline-offset:2px;border-radius:var(--radius-xs)}::selection{background:var(--iris-600);color:#fff}::-webkit-scrollbar{width:8px;height:8px}::-webkit-scrollbar-track{background:var(--bg-base)}::-webkit-scrollbar-thumb{background:var(--border-strong);border-radius:var(--radius-xs)}::-webkit-scrollbar-thumb:hover{background:var(--text-muted)}html{scrollbar-color:var(--border-strong) transparent;scrollbar-width:thin}@keyframes pulse-ring{0%{opacity:.5;transform:scale(1)}to{opacity:0;transform:scale(2.5)}}.pulse-dot{position:relative}.pulse-dot:after{content:"";background:var(--iris-500);border-radius:50%;animation:2s ease-out infinite pulse-ring;position:absolute;inset:-2px}@media (prefers-reduced-motion:reduce){*,:before,:after{scroll-behavior:auto!important;transition-duration:.01ms!important;animation-duration:.01ms!important;animation-iteration-count:1!important}}@media (width<=767px){aside[aria-label=Main\ navigation]{width:160px}main{overflow-x:auto}}@media print{body{color:#000!important;background:#fff!important}aside[aria-label=Main\ navigation],header,[role=region][aria-label=Welcome],[role=region][aria-label=Bulk\ actions],[role=status],[role=dialog]{display:none!important}body,#root,main{height:auto!important;overflow:visible!important}main{padding:0!important}tr,pre,code{page-break-inside:avoid}h1,h2,h3{page-break-after:avoid}[aria-label*=violation],[aria-label*=spike],[aria-label*=collision],[aria-label*=Pass],[aria-label*=Fail]{border:1px solid #000!important}a{color:#000!important;text-decoration:underline!important}a[href^=http]:after{content:" (" attr(href) ")";color:#555;font-size:80%}}