@iris-eval/mcp-server 0.8.1 → 0.9.0

package/dist/self-test.js

@@ -48,6 +48,7 @@ import { irisHome } from './utils/iris-home.js';
 import { EvalEngine } from './eval/engine.js';
 import { generateTraceId } from './utils/ids.js';
 import { LOCAL_TENANT } from './types/tenant.js';
+import { judgeState, judgeStateLine } from './judge-enablement.js';
 const CHECK = '✓';
 const CROSS = '✗';
 /*
@@ -57,6 +58,7 @@ const CROSS = '✗';
  */
 export const SELF_TEST_STEPS = {
     configuredHome: 'configured IRIS_HOME is writable',
+    judge: 'judge key in this shell',
     tempHome: 'create isolated temp home',
     storage: 'initialize storage',
     trace: 'log a trace',
@@ -238,6 +240,18 @@ export async function runSelfTest(write = stdoutLine) {
     await step(SELF_TEST_STEPS.configuredHome, () => probeConfiguredHome(userHome, userStoragePath), {
         independent: true,
     });
+    /*
+     * The judge line, read from THIS shell's environment before the scrub
+     * (the judge variables are not scrubbed, but the order keeps the claim
+     * honest). Informational — a missing key is not a failed install — and
+     * it never calls a provider. The sentence about the client's env block
+     * is the whole point: a key exported here can print enabled while the
+     * process an MCP client spawns never receives it.
+     */
+    await step(SELF_TEST_STEPS.judge, () => {
+        const state = judgeState();
+        return `${judgeStateLine(state)}; your MCP client passes only what its config env block lists — confirm with iris://capabilities from inside the client`;
+    }, { independent: true });
     await step(SELF_TEST_STEPS.tempHome, () => {
         tempHome = mkdtempSync(join(tmpdir(), 'iris-self-test-'));
         for (const key of SCRUBBED_ENV_VARS) {

package/dist/server.d.ts

@@ -3,9 +3,18 @@ import type { IrisConfig } from './types/index.js';
 import type { IStorageAdapter } from './types/query.js';
 import type { CustomRuleStore } from './custom-rule-store.js';
 import { EvalEngine } from './eval/engine.js';
+import { type Capabilities } from './capabilities.js';
 export interface IrisServer {
     mcpServer: McpServer;
     evalEngine: EvalEngine;
     customRuleStore: CustomRuleStore;
+    /** The instructions the client received at initialize — built from this server's runtime state. */
+    instructions: string;
+    /** What this server can do, as iris://capabilities and /api/v1/capabilities serve it. */
+    capabilities: () => Capabilities;
 }
-export declare function createIrisServer(config: IrisConfig, storage: IStorageAdapter, customRuleStore?: CustomRuleStore): IrisServer;
+export interface IrisServerOptions {
+    /** `demo` when the server runs against the disposable demo database. */
+    mode?: 'real' | 'demo';
+}
+export declare function createIrisServer(config: IrisConfig, storage: IStorageAdapter, customRuleStore?: CustomRuleStore, options?: IrisServerOptions): IrisServer;

package/dist/server.js

@@ -1,20 +1,47 @@
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { EvalEngine } from './eval/engine.js';
+import { rulesByType } from './eval/rules/index.js';
+import { builtInRuleRoster } from './eval/criticality.js';
 import { registerAllTools } from './tools/index.js';
 import { registerAllResources } from './resources/index.js';
+import { registerPrompts } from './prompts.js';
 import { createCustomRuleStore } from './custom-rule-store.js';
-export function createIrisServer(config, storage, customRuleStore) {
-    const mcpServer = new McpServer({
-        name: config.server.name,
-        version: config.server.version,
-    });
+import { buildInstructions } from './instructions.js';
+import { buildCapabilities } from './capabilities.js';
+import { judgeState } from './judge-enablement.js';
+export function createIrisServer(config, storage, customRuleStore, options) {
     const evalEngine = new EvalEngine(config.eval.defaultThreshold, config.eval.ruleThresholds, config.eval);
     // Caller can inject a shared rule store (e.g. index.ts passes the
     // same instance the HTTP dashboard uses, so a rule deployed via MCP
     // is immediately visible in the dashboard without a restart). If
     // none provided, create a fresh one loading from the default path.
     const ruleStore = customRuleStore ?? createCustomRuleStore();
+    /*
+     * The instructions are built from what THIS process will do: the
+     * roster and bundles from the registry, the critical list after this
+     * config's promotions and demotions, and whether a judge key reached
+     * this environment. The key is read here once more at boot only to
+     * describe the state; the judge tools resolve it again per call, and
+     * both reads see the same environment because a process's environment
+     * is fixed when its client spawns it.
+     */
+    const roster = builtInRuleRoster((rule) => evalEngine.effectiveCriticality(rule));
+    const instructions = buildInstructions({
+        ruleCount: roster.length,
+        categories: Object.entries(rulesByType)
+            .filter(([, rules]) => rules.length > 0)
+            .map(([category]) => category),
+        threshold: config.eval.defaultThreshold,
+        critical: roster.filter((r) => r.critical).map((r) => r.name),
+        judge: judgeState(),
+    });
+    const mcpServer = new McpServer({
+        name: config.server.name,
+        version: config.server.version,
+    }, { instructions });
+    const capabilities = () => buildCapabilities({ config, evalEngine, customRuleStore: ruleStore, mode: options?.mode });
     registerAllTools(mcpServer, storage, evalEngine, ruleStore);
-    registerAllResources(mcpServer, storage);
-    return { mcpServer, evalEngine, customRuleStore: ruleStore };
+    registerAllResources(mcpServer, storage, capabilities);
+    registerPrompts(mcpServer, config.server.version);
+    return { mcpServer, evalEngine, customRuleStore: ruleStore, instructions, capabilities };
 }

package/dist/storage/index.js

@@ -2,7 +2,7 @@ import { SqliteAdapter } from './sqlite-adapter.js';
 export function createStorage(config) {
     switch (config.storage.type) {
         case 'sqlite':
-            return new SqliteAdapter(config.storage.path);
+            return new SqliteAdapter(config.storage.path, { redact: config.storage.redact ?? 'none' });
         default:
             throw new Error(`Unsupported storage type: ${config.storage.type} (supported: sqlite)`);
     }

package/dist/storage/migrations/007-eval-provenance.d.ts

@@ -0,0 +1,3 @@
+import type Database from 'better-sqlite3';
+export declare const id = "007-eval-provenance";
+export declare function up(db: Database.Database): void;

package/dist/storage/migrations/007-eval-provenance.js

@@ -0,0 +1,30 @@
+export const id = '007-eval-provenance';
+/*
+ * What an evaluation cannot reconstruct about itself after the fact: the
+ * Iris version, the ruleset and configuration hashes and the threshold that
+ * produced it (arc zero: "why did this pass on that day" was unanswerable
+ * from Iris alone), the judge's spend (the tool description said it was
+ * kept; the write path stored none of it), and the erasure stamp the
+ * right-to-erasure fix sets when a trace is deleted. `writer_version` on the
+ * migration ledger lets a downgraded binary refuse a database it cannot
+ * read instead of reading half a schema.
+ *
+ * `verdict`, `coverage` and `critical_skipped` are NOT columns: every one of
+ * them is derived on read from the stored rule_results plus the threshold
+ * kept here, so rows written before this migration read back the same way
+ * without a backfill.
+ */
+export function up(db) {
+    db.exec(`
+    ALTER TABLE eval_results ADD COLUMN provenance TEXT;
+    ALTER TABLE eval_results ADD COLUMN engine_version TEXT;
+    ALTER TABLE eval_results ADD COLUMN ruleset_hash TEXT;
+    ALTER TABLE eval_results ADD COLUMN config_hash TEXT;
+    ALTER TABLE eval_results ADD COLUMN threshold REAL;
+    ALTER TABLE eval_results ADD COLUMN eval_cost_usd REAL;
+    ALTER TABLE eval_results ADD COLUMN eval_tokens INTEGER;
+    ALTER TABLE eval_results ADD COLUMN erased_at TEXT;
+    ALTER TABLE _iris_migrations ADD COLUMN writer_version TEXT;
+    CREATE INDEX IF NOT EXISTS idx_eval_results_tenant_engine ON eval_results(tenant_id, engine_version, ruleset_hash);
+  `);
+}

package/dist/storage/migrations/index.js

@@ -4,6 +4,8 @@ import * as migration003 from './003-eval-passed-index.js';
 import * as migration004 from './004-tenant-id.js';
 import * as migration005 from './005-normalize-created-at.js';
 import * as migration006 from './006-eval-critical-failures.js';
+import * as migration007 from './007-eval-provenance.js';
+import { PKG_VERSION } from '../../config/defaults.js';
 const migrations = [
     migration001,
     migration002,
@@ -11,6 +13,7 @@ const migrations = [
     migration004,
     migration005,
     migration006,
+    migration007,
 ];
 export function runMigrations(db) {
     db.exec(`
@@ -19,10 +22,24 @@ export function runMigrations(db) {
       applied_at TEXT NOT NULL DEFAULT (datetime('now'))
     )
   `);
-    const applied = new Set(db
-        .prepare('SELECT id FROM _iris_migrations')
-        .all()
-        .map((row) => row.id));
+    const known = new Set(migrations.map((m) => m.id));
+    const hasWriterVersion = db.prepare("PRAGMA table_info('_iris_migrations')").all().some((c) => c.name === 'writer_version');
+    const appliedRows = db
+        .prepare(hasWriterVersion ? 'SELECT id, writer_version FROM _iris_migrations' : 'SELECT id, NULL AS writer_version FROM _iris_migrations')
+        .all();
+    /*
+     * A downgrade guard (0.9.0). Before it, a binary that did not know a
+     * migration silently ignored it and read a schema newer than itself —
+     * half the columns, none of the meaning. Now an applied id this build has
+     * never heard of refuses to start, naming the version that wrote it, so
+     * the operator upgrades instead of corrupting.
+     */
+    const unknown = appliedRows.filter((r) => !known.has(r.id));
+    if (unknown.length > 0) {
+        const writers = [...new Set(unknown.map((r) => r.writer_version ?? 'an unknown version'))].join(', ');
+        throw new Error(`This database was migrated by a newer Iris (${writers}) — migration(s) ${unknown.map((r) => r.id).join(', ')} are unknown to v${PKG_VERSION}. Upgrade Iris, or point IRIS_DB_PATH at a database this version wrote.`);
+    }
+    const applied = new Set(appliedRows.map((r) => r.id));
     for (const migration of migrations) {
         if (!applied.has(migration.id)) {
             db.transaction(() => {
@@ -31,4 +48,7 @@ export function runMigrations(db) {
             })();
         }
     }
+    // Every applied migration names the binary that applied it (this one, for
+    // rows written before the column existed — the closest true statement).
+    db.prepare('UPDATE _iris_migrations SET writer_version = ? WHERE writer_version IS NULL').run(PKG_VERSION);
 }

package/dist/storage/sqlite-adapter.d.ts

@@ -2,10 +2,19 @@ import type { IStorageAdapter, DashboardSummary, TraceQueryOptions, TraceQueryRe
 import type { Trace, Span } from '../types/trace.js';
 import type { EvalResult } from '../types/eval.js';
 import type { TenantId } from '../types/tenant.js';
+export type RedactMode = 'none' | 'critical_spans';
+export interface SqliteAdapterOptions {
+    /** storage.redact — replace the spans a critical detector flagged in the stored text. */
+    redact?: RedactMode;
+}
+/** What every text field of an erased evaluation reads afterwards. */
+export declare const ERASED_MESSAGE = "erased with the trace";
 export declare class SqliteAdapter implements IStorageAdapter {
     private db;
     private readonly dbPath;
-    constructor(dbPath: string);
+    /** storage.redact — see SqliteAdapterOptions. */
+    private readonly redact;
+    constructor(dbPath: string, options?: SqliteAdapterOptions);
     initialize(): Promise<void>;
     close(): Promise<void>;
     insertTrace(tenantId: TenantId, trace: Trace): Promise<void>;
@@ -15,6 +24,7 @@ export declare class SqliteAdapter implements IStorageAdapter {
     getSpansByTraceId(tenantId: TenantId, traceId: string): Promise<Span[]>;
     insertEvalResult(tenantId: TenantId, result: EvalResult): Promise<void>;
     getEvalsByTraceId(tenantId: TenantId, traceId: string): Promise<EvalResult[]>;
+    getEvalById(tenantId: TenantId, id: string): Promise<EvalResult | null>;
     queryEvalResults(tenantId: TenantId, options: {
         eval_type?: string;
         passed?: boolean;
@@ -41,6 +51,21 @@ export declare class SqliteAdapter implements IStorageAdapter {
     }>;
     checkpoint(): Promise<void>;
     deleteTrace(tenantId: TenantId, traceId: string): Promise<boolean>;
+    /**
+     * Blank every text field of the evaluations linked to these traces and
+     * stamp erased_at (migration 007). Verdict, scores, criticality and the
+     * evidence OFFSETS stay — they carry no text — so history and drift
+     * analytics keep working over an erased row.
+     */
+    private eraseEvaluationsOfTraces;
+    /**
+     * storage.redact = 'critical_spans': the spans a critical detector fired
+     * on are replaced in the STORED text by [REDACTED:<pattern>]. The
+     * evidence offsets are left as computed — they index the text the caller
+     * saw, which is what a reader of the evidence needs — and the option's
+     * documentation says so.
+     */
+    private storedOutputText;
     getDistinctValues(tenantId: TenantId, column: string): Promise<string[]>;
     private rowToTrace;
     private rowToSpan;

package/dist/storage/sqlite-adapter.js

@@ -21,6 +21,7 @@
  */
 import Database from 'better-sqlite3';
 import { ensureOwnerOnly } from '../utils/write-atomic.js';
+import { deriveCoverage, deriveCriticalSkipped, deriveVerdict } from '../eval/verdict.js';
 import { TenantContextRequiredError } from '../types/tenant.js';
 import { runMigrations } from './migrations/index.js';
 const ALLOWED_SORT_COLUMNS = new Set(['timestamp', 'latency_ms', 'cost_usd']);
@@ -34,11 +35,16 @@ function assertTenant(tenantId) {
         throw new TenantContextRequiredError('SqliteAdapter method invoked without a valid TenantId; refusing to query');
     }
 }
+/** What every text field of an erased evaluation reads afterwards. */
+export const ERASED_MESSAGE = 'erased with the trace';
 export class SqliteAdapter {
     db;
     dbPath;
-    constructor(dbPath) {
+    /** storage.redact — see SqliteAdapterOptions. */
+    redact;
+    constructor(dbPath, options) {
         this.dbPath = dbPath;
+        this.redact = options?.redact ?? 'none';
         this.db = new Database(dbPath);
     }
     async initialize() {
@@ -55,7 +61,14 @@ export class SqliteAdapter {
          * alternative is the whole point of #372.
          */
         this.db.pragma('secure_delete = ON');
-        runMigrations(this.db);
+        try {
+            runMigrations(this.db);
+        }
+        catch (err) {
+            // A refused boot (a newer writer, a failed migration) must not leak the handle.
+            this.db.close();
+            throw err;
+        }
         /*
          * iris.db holds agent inputs and outputs verbatim, and a tool that
          * detects PII necessarily stores the PII it found. better-sqlite3
@@ -203,10 +216,16 @@ export class SqliteAdapter {
          * no surface could filter, count, or explain the release's flagship
          * behaviour. NULL when nothing vetoed.
          */
+        /*
+         * Provenance (migration 007) is the part of the receipt a row cannot
+         * reconstruct: the release, the ruleset and config hashes, the threshold.
+         * verdict / coverage / critical_skipped are derived on every read from
+         * rule_results plus that threshold, so they are not columns.
+         */
         this.db.prepare(`
-      INSERT INTO eval_results (tenant_id, id, trace_id, eval_type, output_text, expected_text, score, passed, rule_results, suggestions, rules_evaluated, rules_skipped, insufficient_data, critical_failures, created_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `).run(tenantId, result.id, result.trace_id ?? null, result.eval_type, result.output_text, result.expected_text ?? null, result.score, result.passed ? 1 : 0, JSON.stringify(result.rule_results), JSON.stringify(result.suggestions), result.rules_evaluated ?? null, result.rules_skipped ?? null, result.insufficient_data ? 1 : 0, result.critical_failures?.length ? JSON.stringify(result.critical_failures) : null, new Date().toISOString());
+      INSERT INTO eval_results (tenant_id, id, trace_id, eval_type, output_text, expected_text, score, passed, rule_results, suggestions, rules_evaluated, rules_skipped, insufficient_data, critical_failures, created_at, provenance, engine_version, ruleset_hash, config_hash, threshold, eval_cost_usd, eval_tokens)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(tenantId, result.id, result.trace_id ?? null, result.eval_type, this.storedOutputText(result), result.expected_text ?? null, result.score, result.passed ? 1 : 0, JSON.stringify(result.rule_results), JSON.stringify(result.suggestions), result.rules_evaluated ?? null, result.rules_skipped ?? null, result.insufficient_data ? 1 : 0, result.critical_failures?.length ? JSON.stringify(result.critical_failures) : null, new Date().toISOString(), result.provenance ? JSON.stringify(result.provenance) : null, result.provenance?.irisVersion ?? null, result.provenance?.rulesetHash ?? null, result.provenance?.configHash ?? null, result.provenance?.thresholds.default ?? null, result.eval_cost_usd ?? null, result.eval_tokens ?? null);
     }
     async getEvalsByTraceId(tenantId, traceId) {
         assertTenant(tenantId);
@@ -215,6 +234,13 @@ export class SqliteAdapter {
             .all(tenantId, traceId);
         return rows.map((row) => this.rowToEvalResult(row));
     }
+    async getEvalById(tenantId, id) {
+        assertTenant(tenantId);
+        const row = this.db
+            .prepare('SELECT * FROM eval_results WHERE tenant_id = ? AND id = ?')
+            .get(tenantId, id);
+        return row ? this.rowToEvalResult(row) : null;
+    }
     async queryEvalResults(tenantId, options) {
         assertTenant(tenantId);
         const conditions = ['tenant_id = ?'];
@@ -504,10 +530,14 @@ export class SqliteAdapter {
     async deleteTracesOlderThan(tenantId, days) {
         assertTenant(tenantId);
         const cutoff = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString();
-        const result = this.db
-            .prepare('DELETE FROM traces WHERE tenant_id = ? AND timestamp < ?')
-            .run(tenantId, cutoff);
-        return result.changes;
+        // Same erasure as deleteTrace: an evaluation younger than the window
+        // whose trace is swept keeps its verdict and loses its text.
+        const run = this.db.transaction((tid, cut) => {
+            const ids = this.db.prepare('SELECT trace_id FROM traces WHERE tenant_id = ? AND timestamp < ?').all(tid, cut).map((r) => r.trace_id);
+            this.eraseEvaluationsOfTraces(tid, ids);
+            return this.db.prepare('DELETE FROM traces WHERE tenant_id = ? AND timestamp < ?').run(tid, cut).changes;
+        });
+        return run(tenantId, cutoff);
     }
     async deleteEvalResultsOlderThan(tenantId, days) {
         assertTenant(tenantId);
@@ -563,10 +593,80 @@ export class SqliteAdapter {
         // Tenant-scoped: a trace id owned by a different tenant is
         // untouchable from this call. Cross-tenant deletions are not just
         // denied — they're invisible (no indication the id even exists).
-        const result = this.db
-            .prepare('DELETE FROM traces WHERE tenant_id = ? AND trace_id = ?')
-            .run(tenantId, traceId);
-        return result.changes > 0;
+        //
+        // The right-to-erasure fix: eval_results.trace_id is ON DELETE SET
+        // NULL, so the delete alone left every linked evaluation behind with
+        // output_text verbatim — including what no_pii had flagged — orphaned
+        // and readable. The text is erased in the same transaction, BEFORE the
+        // FK can orphan the rows.
+        const run = this.db.transaction((tid, id) => {
+            const exists = this.db.prepare('SELECT 1 FROM traces WHERE tenant_id = ? AND trace_id = ?').get(tid, id);
+            if (!exists)
+                return 0;
+            this.eraseEvaluationsOfTraces(tid, [id]);
+            return this.db.prepare('DELETE FROM traces WHERE tenant_id = ? AND trace_id = ?').run(tid, id).changes;
+        });
+        return run(tenantId, traceId) > 0;
+    }
+    /**
+     * Blank every text field of the evaluations linked to these traces and
+     * stamp erased_at (migration 007). Verdict, scores, criticality and the
+     * evidence OFFSETS stay — they carry no text — so history and drift
+     * analytics keep working over an erased row.
+     */
+    eraseEvaluationsOfTraces(tenantId, traceIds) {
+        if (traceIds.length === 0)
+            return 0;
+        const now = new Date().toISOString();
+        const select = this.db.prepare('SELECT id, rule_results FROM eval_results WHERE tenant_id = ? AND trace_id = ?');
+        const update = this.db.prepare('UPDATE eval_results SET output_text = ?, expected_text = NULL, suggestions = ?, rule_results = ?, erased_at = ? WHERE tenant_id = ? AND id = ?');
+        let erased = 0;
+        for (const traceId of traceIds) {
+            for (const row of select.all(tenantId, traceId)) {
+                let rules = [];
+                try {
+                    rules = JSON.parse(row.rule_results);
+                }
+                catch {
+                    rules = [];
+                }
+                const erasedRules = rules.map((r) => ({
+                    ...r,
+                    message: ERASED_MESSAGE,
+                    ...(r.skipReason ? { skipReason: ERASED_MESSAGE } : {}),
+                }));
+                update.run('', '[]', JSON.stringify(erasedRules), now, tenantId, row.id);
+                erased += 1;
+            }
+        }
+        return erased;
+    }
+    /**
+     * storage.redact = 'critical_spans': the spans a critical detector fired
+     * on are replaced in the STORED text by [REDACTED:<pattern>]. The
+     * evidence offsets are left as computed — they index the text the caller
+     * saw, which is what a reader of the evidence needs — and the option's
+     * documentation says so.
+     */
+    storedOutputText(result) {
+        const text = result.output_text;
+        if (this.redact !== 'critical_spans' || !text)
+            return text ?? null;
+        const spans = result.rule_results
+            .filter((r) => r.critical === true && !r.passed && !r.skipped)
+            .flatMap((r) => (r.evidence ?? []).filter((e) => e.type === 'span' && e.source === 'output'));
+        if (spans.length === 0)
+            return text;
+        const seen = new Set();
+        let out = text;
+        for (const s of [...spans].sort((a, b) => b.start - a.start)) {
+            const key = `${s.start}:${s.end}`;
+            if (seen.has(key) || s.start >= s.end || s.end > out.length)
+                continue;
+            seen.add(key);
+            out = `${out.slice(0, s.start)}[REDACTED:${s.label}]${out.slice(s.end)}`;
+        }
+        return out;
     }
     async getDistinctValues(tenantId, column) {
         assertTenant(tenantId);
@@ -613,7 +713,7 @@ export class SqliteAdapter {
         };
     }
     rowToEvalResult(row) {
-        return {
+        const result = {
             id: row.id,
             trace_id: row.trace_id,
             eval_type: row.eval_type,
@@ -623,7 +723,8 @@ export class SqliteAdapter {
              * regroup the per-bundle breakdown from what IS stored.
              */
             output_text: row.output_text,
-            expected_text: row.expected_text,
+            expected_text: row.expected_text ?? undefined,
+            ...(row.erased_at ? { erased_at: row.erased_at } : {}),
             score: row.score,
             passed: row.passed === 1,
             rule_results: JSON.parse(row.rule_results),
@@ -640,6 +741,25 @@ export class SqliteAdapter {
             ...(row.critical_failures != null
                 ? { critical_failures: JSON.parse(row.critical_failures) }
                 : {}),
+            ...(row.eval_cost_usd != null ? { eval_cost_usd: row.eval_cost_usd } : {}),
+            ...(row.eval_tokens != null ? { eval_tokens: row.eval_tokens } : {}),
+            ...(row.provenance != null ? { provenance: JSON.parse(row.provenance) } : {}),
         };
+        /*
+         * Derived on every read, never stored (0.9.0): the critical rules that
+         * skipped (from the stamped flags — absent for rows older than those
+         * flags, never []), the coverage by question, and the verdict with its
+         * basis — the last only when the row carries the threshold it was judged
+         * against, because a basis guessed against today's threshold would be a
+         * fabrication about that day.
+         */
+        const criticalSkipped = deriveCriticalSkipped(result.rule_results);
+        if (criticalSkipped)
+            result.critical_skipped = criticalSkipped;
+        if (result.rule_results.some((r) => r.question !== undefined))
+            result.coverage = deriveCoverage(result.rule_results);
+        if (result.provenance)
+            result.verdict = deriveVerdict(result, result.provenance.thresholds.default);
+        return result;
     }
 }

package/dist/tools/delete-rule.d.ts

@@ -1,4 +1,12 @@
+import { z } from 'zod';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { CustomRuleStore } from '../custom-rule-store.js';
 import type { EvalEngine } from '../eval/engine.js';
+export declare const deleteRuleOutputSchema: z.ZodObject<{
+    deleted: z.ZodBoolean;
+    rule_id: z.ZodString;
+    toggled: z.ZodOptional<z.ZodBoolean>;
+    enabled: z.ZodOptional<z.ZodBoolean>;
+    rule: z.ZodOptional<z.ZodObject<{}, z.core.$loose>>;
+}, z.core.$loose>;
 export declare function registerDeleteRuleTool(server: McpServer, customRuleStore: CustomRuleStore, evalEngine: EvalEngine): void;

package/dist/tools/delete-rule.js

@@ -22,6 +22,8 @@ import { z } from 'zod';
 import { createCustomRule } from '../eval/rules/custom.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { strictInput } from './strict-input.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { guarded, respond } from './respond.js';
 const inputSchema = {
     rule_id: z
         .string()
@@ -32,41 +34,45 @@ const inputSchema = {
         .optional()
         .describe('When present the rule is NOT deleted: false DISABLES it (kept in the store, stops firing immediately, history and provenance preserved); true RE-ENABLES a disabled rule. Omit to delete'),
 };
+export const deleteRuleOutputSchema = z.looseObject({
+    deleted: z.boolean().describe('true when a rule was removed; always false on a toggle'),
+    rule_id: z.string().describe('the id that was asked for'),
+    toggled: z.boolean().optional().describe('toggle only: true when the rule exists (also when it was already in the requested state)'),
+    enabled: z.boolean().optional().describe('toggle only: the rule\'s state after the call'),
+    rule: z.looseObject({}).optional().describe('toggle only: the rule as stored'),
+});
 export function registerDeleteRuleTool(server, customRuleStore, evalEngine) {
     server.registerTool('delete_rule', {
         title: 'Delete or Disable Custom Rule',
-        description: [
-            'Remove a deployed custom evaluation rule — or, with `enabled`, disable / re-enable it without removing it. Either way the change takes effect on the next evaluate_output call; past eval_results that referenced the rule are preserved.',
-            '',
-            'Sibling tools — deploy_rule adds custom rules, list_rules enumerates them (including disabled ones, with `enabled: false`), evaluate_output runs them. delete_trace handles trace deletion (separate concern); log_trace / get_traces handle trace I/O. delete_rule is the DESTRUCTIVE remove path for the custom-rule store and the only MCP path that toggles a rule; it does NOT touch traces, eval_results, or built-in (non-custom) rules.',
-            '',
-            'Behavior. Without `enabled`: DESTRUCTIVE — rewrites ~/.iris/custom-rules.json without the deleted row and appends a `rule.delete` entry to the audit log (~/.iris/audit.log). Not idempotent: deleting an already-deleted rule returns `deleted: false` rather than re-emitting the audit row. The rule stops firing immediately on the live process. With `enabled`: NOT destructive — the rule row stays, its `enabled` flag and `updatedAt` change, a `rule.toggle` audit entry is appended (none if the flag was already in that state), and the live engine unregisters (false) or re-registers (true) the rule so the change is immediate; a disabled rule is not loaded at the next boot either. Historical eval_results that reference this rule_id stay in the database — drift analytics + audit trail remain valid. Tenant-scoped in Cloud tier; OSS operates on LOCAL_TENANT. Rate-limited to 20 req/min on HTTP MCP.',
-            '',
-            'Output shape. Delete: `{ "deleted": boolean, "rule_id": string }` — `deleted=true` if a row was removed; `deleted=false` if no rule with that id existed. Toggle (enabled given): `{ "deleted": false, "toggled": boolean, "rule_id": string, "enabled"?: boolean, "rule"?: { ...the rule } }` — `toggled=true` with the rule\'s current state when the id exists (also when it was already in the requested state), `toggled=false` and no `rule` when it does not.',
-            '',
-            "Use when a custom rule is obsolete (behavior changed, false positives unacceptable, replaced by a better rule). Typical flow: list_rules → identify the stale one → delete_rule(id). Combine with deploy_rule to replace: delete_rule(oldId) + deploy_rule(newDefinition), or deploy_rule with the same name and replace:true. To temporarily PAUSE a rule — false positives to investigate, a rollout to stage — pass `enabled: false` instead of deleting; it keeps the id, the provenance and the history, and `enabled: true` brings it back with the same id.",
-            '',
-            "Don't use on built-in (non-custom) rules — the rule_id format checks for `rule-<hex>` custom ids; built-ins aren't in the store. Don't use to delete a trace or eval result (use delete_trace for traces; eval_results deletion is not exposed per row — they fall under data retention and `--purge`).",
-            '',
-            'Parameters. rule_id must match `rule-<lowercase-hex>` format (Zod regex). Format mismatch fails Zod with 400 BEFORE the store is touched. Cross-tenant rule_ids return `deleted: false` / `toggled: false` silently — they\'re invisible to the caller\'s tenant rather than producing a not-found error (prevents enumeration attacks). The rule_id you pass is exactly what list_rules returned in `id` or what deploy_rule returned in `rule.id`. enabled is optional: omit to delete, false to disable, true to re-enable.',
-            '',
-            "Error modes. Throws 400 on malformed rule_id (wrong prefix) or an unknown argument. Returns `{deleted: false}` (or `{toggled: false}`) if rule_id doesn't match any deployed rule (not an error — idempotent-ish). Returns 429 on HTTP rate limit. File-write failures propagate as 500.",
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Remove a deployed custom rule — or, with enabled, disable or re-enable it without removing it — effective on the next evaluate_output call.',
+            does: 'Without enabled: deletes the rule from ~/.iris/custom-rules.json, appends a rule.delete audit entry and unregisters it from the running engine; deleted is false when no rule has that id (already gone, or not this tenant\'s), and no audit row is written twice. ' +
+                'With enabled: the rule stays with its history and provenance; false stops it firing at once and keeps it off across restarts, true brings it back under the same id; a rule.toggle audit entry is written unless the flag was already in that state. ' +
+                'Past evaluations that referenced the rule are untouched either way.',
+            whenNot: 'On built-in rules: they are not in the store and cannot be deleted or disabled. To delete a trace (delete_trace). To replace a rule: deploy_rule with the same name and replace: true.',
+            returns: deleteRuleOutputSchema,
+            errors: 'IRIS_STORAGE_ERROR when the store cannot be written. A malformed rule_id (not rule-<hex>) or an unknown argument is refused before the handler runs. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                deploy_rule: 'add or replace a rule',
+                list_rules: 'find the id',
+                evaluate_output: 'where the rule fires',
+            },
+        }),
         inputSchema: strictInput(inputSchema),
+        outputSchema: deleteRuleOutputSchema,
         annotations: {
             readOnlyHint: false,
             destructiveHint: true,
             idempotentHint: false,
             openWorldHint: false,
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         // OSS: MCP tools operate under LOCAL_TENANT. See list-rules.ts for context.
         if (args.enabled !== undefined) {
             const rule = customRuleStore.setEnabled(LOCAL_TENANT, args.rule_id, args.enabled, 'mcp');
             if (!rule) {
-                return {
-                    content: [{ type: 'text', text: JSON.stringify({ deleted: false, toggled: false, rule_id: args.rule_id }) }],
-                };
+                return respond(deleteRuleOutputSchema, { deleted: false, toggled: false, rule_id: args.rule_id });
             }
             // Mirror the store in the live engine, so the toggle is immediate
             // (registerRule is idempotent by id — re-enabling an already-live
@@ -77,14 +83,7 @@ export function registerDeleteRuleTool(server, customRuleStore, evalEngine) {
             else {
                 evalEngine.unregisterRule(rule.id);
             }
-            return {
-                content: [
-                    {
-                        type: 'text',
-                        text: JSON.stringify({ deleted: false, toggled: true, rule_id: args.rule_id, enabled: rule.enabled, rule }),
-                    },
-                ],
-            };
+            return respond(deleteRuleOutputSchema, { deleted: false, toggled: true, rule_id: args.rule_id, enabled: rule.enabled, rule });
         }
         const deleted = customRuleStore.delete(LOCAL_TENANT, args.rule_id, 'mcp');
         if (deleted) {
@@ -94,13 +93,6 @@ export function registerDeleteRuleTool(server, customRuleStore, evalEngine) {
             // rule was never registered in this process.
             evalEngine.unregisterRule(args.rule_id);
         }
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({ deleted, rule_id: args.rule_id }),
-                },
-            ],
-        };
-    });
+        return respond(deleteRuleOutputSchema, { deleted, rule_id: args.rule_id });
+    }));
 }

package/dist/tools/delete-trace.d.ts

@@ -1,3 +1,8 @@
+import { z } from 'zod';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
+export declare const deleteTraceOutputSchema: z.ZodObject<{
+    deleted: z.ZodBoolean;
+    trace_id: z.ZodString;
+}, z.core.$loose>;
 export declare function registerDeleteTraceTool(server: McpServer, storage: IStorageAdapter): void;