@iris-eval/mcp-server 0.4.5 → 0.5.0

package/dist/index.js

@@ -12,8 +12,11 @@ import { loadOrInitPreferences, shouldAutoLaunchDashboard, createPreferenceStore
 import { openBrowser } from './utils/open-browser.js';
 import { createCustomRuleStore } from './custom-rule-store.js';
 import { createCustomRule } from './eval/rules/custom.js';
+import { EvalEngine } from './eval/engine.js';
 import { LOCAL_TENANT } from './types/tenant.js';
 import { validatePortConfig } from './utils/validate-port-config.js';
+import { irisHome } from './utils/iris-home.js';
+import { seedDemoData, clearDemoData, demoDbPath, demoPreferencesPath, demoCustomRulesPath, demoAuditLogPath, } from './dashboard/seed-demo-data.js';
 const PortSchema = z
     .string()
     .regex(/^\d+$/, 'must be a positive integer')
@@ -28,6 +31,10 @@ const CliSchema = z
     'api-key': z.string().min(1).optional(),
     dashboard: z.boolean().optional(),
     'dashboard-port': PortSchema.optional(),
+    'dashboard-host': z.string().min(1).optional(),
+    demo: z.boolean().optional(),
+    'demo-clear': z.boolean().optional(),
+    'self-test': z.boolean().optional(),
     help: z.boolean().optional(),
 })
     .strict();
@@ -40,8 +47,24 @@ try {
             config: { type: 'string' },
             'db-path': { type: 'string' },
             'api-key': { type: 'string' },
-            dashboard: { type: 'boolean', default: false },
+            /*
+             * No `default: false` here, unlike the other booleans. The value
+             * flows into loadConfig, and cliArgsToConfig writes the CLI layer
+             * whenever `dashboard !== undefined` — so a default made the flag's
+             * ABSENCE indistinguishable from an explicitly disabled dashboard and stamped
+             * `enabled: false` over the env and config-file layers, which merge
+             * before it. IRIS_DASHBOARD=true and `dashboard.enabled` in
+             * config.json therefore did nothing at all. Left undefined, the
+             * lower layers survive and `--dashboard` still wins when passed.
+             * The other boolean flags are read for truthiness only and never
+             * reach the merge, so their defaults are harmless.
+             */
+            dashboard: { type: 'boolean' },
             'dashboard-port': { type: 'string' },
+            'dashboard-host': { type: 'string' },
+            demo: { type: 'boolean', default: false },
+            'demo-clear': { type: 'boolean', default: false },
+            'self-test': { type: 'boolean', default: false },
             help: { type: 'boolean', short: 'h', default: false },
         },
         strict: true,
@@ -74,16 +97,32 @@ Options:
   --api-key <key>          API key for HTTP authentication
   --dashboard              Enable web dashboard
   --dashboard-port <port>  Dashboard port 1-65535 (default: 6920)
+  --dashboard-host <host>  Dashboard bind address (default: 127.0.0.1). The dashboard is
+                           unauthenticated unless --api-key is set — binding it beyond
+                           loopback exposes your full trace history to the network.
+  --demo                   Seed a demo database and serve the dashboard against it —
+                           see the dashboard working before wiring up your agent.
+                           Demo data lives in its own files (demo.db) and never mixes
+                           with your real traces. Serves the dashboard only (no MCP
+                           transport). Idempotent: re-running reuses the seeded data.
+  --demo-clear             Delete the demo database (and its sidecar files), then exit.
+                           Your real traces are not touched.
+  --self-test              Run the offline install diagnostic and exit: storage round-trip,
+                           deterministic evals, dashboard + rebinding guard — all inside an
+                           isolated temp home. Exit code 0 = healthy, 1 = a check failed.
   -h, --help               Show this help message
 
 Environment variables (CLI flags take precedence):
   IRIS_TRANSPORT                       stdio | http
   IRIS_HOST                            Bind address for HTTP transport (default: 127.0.0.1)
   IRIS_PORT                            HTTP transport port (1-65535)
-  IRIS_DB_PATH                         SQLite database path
+  IRIS_HOME                            Directory for all per-user files: config.json, iris.db, custom-rules.json,
+                                       audit.log, preferences.json (default: ~/.iris)
+  IRIS_DB_PATH                         SQLite database path (overrides IRIS_HOME for the DB only)
   IRIS_LOG_LEVEL                       debug | info | warn | error
-  IRIS_DASHBOARD                       true to enable web dashboard
+  IRIS_DASHBOARD                       true/1/yes/on enables the web dashboard; false/0/no/off disables it (overrides config.json)
   IRIS_DASHBOARD_PORT                  Dashboard port (1-65535, default: 6920)
+  IRIS_DASHBOARD_HOST                  Dashboard bind address (default: 127.0.0.1)
   IRIS_API_KEY                         API key for HTTP authentication
   IRIS_ALLOWED_ORIGINS                 Comma-separated origin allowlist. Dashboard: CORS headers (supports globs, e.g. http://localhost:*).
                                        HTTP transport: exact-match Origin allowlist for DNS-rebinding protection (globs ignored;
@@ -105,14 +144,53 @@ Dashboard preferences (~/.iris/preferences.json):
 `);
     process.exit(0);
 }
+/*
+ * --self-test exits BEFORE loadConfig() runs at module scope below —
+ * deliberately. The diagnostic builds its own isolated IRIS_HOME and
+ * scrubs the IRIS_* env layer (src/self-test.ts), so the normal boot
+ * path's config (and the user's real ~/.iris) must never load first.
+ */
+if (values['self-test']) {
+    const { runSelfTest } = await import('./self-test.js');
+    process.exit(await runSelfTest());
+}
+/*
+ * Demo-mode flag validation happens before loadConfig so a refused
+ * combination exits without touching the filesystem.
+ */
+if (values.demo && values['demo-clear']) {
+    process.stderr.write('iris-mcp: --demo and --demo-clear cannot be combined.\nRun `iris-mcp --help` for usage.\n');
+    process.exit(2);
+}
+if (values.demo && values['db-path']) {
+    process.stderr.write('iris-mcp: --demo always serves its own database (demo.db under your iris home) and cannot be combined with --db-path.\n' +
+        'Run `iris-mcp --demo` alone, or drop --demo to use your own database.\n');
+    process.exit(2);
+}
+if (values['demo-clear']) {
+    const { removed } = clearDemoData();
+    if (removed.length === 0) {
+        process.stderr.write(`iris-mcp: no demo data found under "${irisHome()}" — nothing to remove.\n`);
+    }
+    else {
+        for (const path of removed) {
+            process.stderr.write(`iris-mcp: removed "${path}"\n`);
+        }
+        process.stderr.write('iris-mcp: demo data cleared. Your real traces were not touched.\n');
+    }
+    process.exit(0);
+}
 const config = loadConfig({
     transport: values.transport,
     port: values.port,
     config: values.config,
-    dbPath: values['db-path'],
+    // Demo mode serves the dashboard against the dedicated demo database —
+    // never the real store — and always with the dashboard enabled.
+    dbPath: values.demo ? demoDbPath() : values['db-path'],
     apiKey: values['api-key'],
-    dashboard: values.dashboard,
+    dashboard: values.demo ? true : values.dashboard,
     dashboardPort: values['dashboard-port'],
+    dashboardHost: values['dashboard-host'],
 });
 const logger = createLogger(config);
 async function main() {
@@ -136,7 +214,9 @@ async function main() {
     // and would need per-tenant rule registration).
     const enabled = customRuleStore.enabledRules(LOCAL_TENANT);
     for (const rule of enabled) {
-        evalEngine.registerRule(rule.evalType, createCustomRule(rule.definition));
+        // Severity rides along: high/critical deployed rules hard-fail the
+        // evals they lose (createCustomRule sets EvalRule.critical from it).
+        evalEngine.registerRule(rule.evalType, createCustomRule(rule.definition, rule.severity), rule.id);
     }
     if (enabled.length > 0) {
         logger.info(`Loaded ${enabled.length} deployed custom rule(s) from ${customRuleStore.pathFor(LOCAL_TENANT)}`);
@@ -176,7 +256,18 @@ async function main() {
             logger.info(`Tip: run with --dashboard to open the web dashboard on port ${config.dashboard.port}`);
         }
     }
-    if (config.dashboard.enabled || config.transport.type === 'http') {
+    /*
+     * The dashboard starts ONLY when explicitly enabled (--dashboard,
+     * IRIS_DASHBOARD=true, or dashboard.enabled in config.json). It used to
+     * also start implicitly whenever `--transport http` was chosen — which
+     * contradicted the README ("off by default"), and worse: if its default
+     * port 6920 was busy on a shared CI runner, the WHOLE process exited 1
+     * after the transport had already bound successfully. An unrequested
+     * server must never be able to kill the requested one. Users who relied
+     * on the ingest endpoint riding along get a loud pointer below instead
+     * of a silent 404.
+     */
+    if (config.dashboard.enabled) {
         const preferenceStore = createPreferenceStore();
         const dashboardServer = createDashboardServer(storage, config, logger, {
             customRuleStore,
@@ -189,19 +280,25 @@ async function main() {
         // dashboard in the user's default browser. Skipped in CI, when the
         // user has previously set autoLaunch=false in ~/.iris/preferences.json,
         // or when IRIS_NO_AUTO_LAUNCH=1 is set.
-        if (config.dashboard.enabled) {
-            const prefState = loadOrInitPreferences();
-            if (prefState.isFirstRun && shouldAutoLaunchDashboard(prefState)) {
-                const url = `http://localhost:${config.dashboard.port}`;
-                logger.info(`First run detected — opening dashboard at ${url}`);
-                logger.info(`(To disable auto-launch: set IRIS_NO_AUTO_LAUNCH=1 or edit ${prefState.path})`);
-                openBrowser(url);
-            }
-            else if (prefState.isFirstRun) {
-                logger.info(`First run detected — skipping auto-launch (CI/IRIS_NO_AUTO_LAUNCH set). Dashboard at http://localhost:${config.dashboard.port}`);
-            }
+        const prefState = loadOrInitPreferences();
+        if (prefState.isFirstRun && shouldAutoLaunchDashboard(prefState)) {
+            const url = `http://localhost:${config.dashboard.port}`;
+            logger.info(`First run detected — opening dashboard at ${url}`);
+            logger.info(`(To disable auto-launch: set IRIS_NO_AUTO_LAUNCH=1 or edit ${prefState.path})`);
+            openBrowser(url);
+        }
+        else if (prefState.isFirstRun) {
+            logger.info(`First run detected — skipping auto-launch (CI/IRIS_NO_AUTO_LAUNCH set). Dashboard at http://localhost:${config.dashboard.port}`);
         }
     }
+    else if (config.transport.type === 'http') {
+        // Loud, because this used to start implicitly: anyone who relied on the
+        // ingest endpoint riding along with --transport http must learn how to
+        // get it back from this line, not from a connection refused.
+        logger.info(`Dashboard not started (off by default). The dashboard — and the HTTP ingest endpoint ` +
+            `POST /api/v1/traces — start on port ${config.dashboard.port} when you pass --dashboard, ` +
+            `set IRIS_DASHBOARD=true, or set dashboard.enabled=true in config.json.`);
+    }
     if (config.security.apiKey) {
         logger.info('API key authentication enabled');
     }
@@ -222,7 +319,97 @@ async function main() {
     process.on('SIGINT', shutdown);
     process.on('SIGTERM', shutdown);
 }
-main().catch((err) => {
+function printDemoBanner(summary, url) {
+    const line = '='.repeat(60);
+    const counts = summary.alreadySeeded
+        ? `  Reusing the existing demo database (${summary.traceCount} traces, ${summary.evalCount} evaluations).`
+        : `  Seeded ${summary.traceCount} traces / ${summary.evalCount} evaluations across the last 7 days.`;
+    process.stderr.write(`
+${line}
+  IRIS DEMO MODE — everything on screen is demo data
+${line}
+
+${counts}
+  Demo database: "${summary.dbPath}"
+  Your real trace database is untouched — demo data never mixes with it.
+
+  Worth clicking into:
+    - a PII leak (a synthetic SSN in an agent reply) caught by the safety rules
+    - a prompt-injection attempt flagged in summarized forum posts
+    - a failed LLM-judge score, with the judge's rationale
+
+  Dashboard: ${url}
+
+  Remove the demo data with one command:
+    npx @iris-eval/mcp-server --demo-clear
+
+  Press Ctrl+C to stop.
+
+`);
+}
+/*
+ * Demo mode (--demo): seed the dedicated demo database (idempotent) and
+ * serve the dashboard against it. No MCP transport is started — demo mode
+ * exists to put something real on screen before an agent is wired up.
+ *
+ * Isolation: everything demo mode writes lives in demo-scoped files under
+ * irisHome() (demo.db, demo-preferences.json, demo-custom-rules.json,
+ * demo-audit.log). A rule deployed from the demo dashboard lands in the
+ * demo rule store, and --demo-clear removes all of it. The real iris.db,
+ * custom-rules.json, audit.log and preferences.json are never touched.
+ */
+async function runDemo() {
+    logger.info(`Starting Iris demo mode v${config.server.version}`);
+    const seedSummary = await seedDemoData();
+    if (seedSummary.alreadySeeded) {
+        logger.info(`Demo database already seeded (${seedSummary.traceCount} traces) — reusing it`);
+    }
+    else {
+        logger.info(`Seeded demo database with ${seedSummary.traceCount} traces at ${seedSummary.dbPath}`);
+    }
+    const storage = createStorage(config);
+    await storage.initialize();
+    const customRuleStore = createCustomRuleStore({
+        pathFor: () => demoCustomRulesPath(),
+        auditPath: demoAuditLogPath(),
+    });
+    const evalEngine = new EvalEngine(config.eval.defaultThreshold, config.eval.ruleThresholds);
+    for (const rule of customRuleStore.enabledRules(LOCAL_TENANT)) {
+        evalEngine.registerRule(rule.evalType, createCustomRule(rule.definition, rule.severity), rule.id);
+    }
+    const preferenceStore = createPreferenceStore(demoPreferencesPath());
+    const dashboardServer = createDashboardServer(storage, config, logger, {
+        customRuleStore,
+        evalEngine,
+        preferenceStore,
+    });
+    const server = dashboardServer.start();
+    server.on('listening', () => {
+        // Use the port actually bound (supports --dashboard-port 0 in tests).
+        const addr = server.address();
+        const port = typeof addr === 'object' && addr ? addr.port : config.dashboard.port;
+        const url = `http://localhost:${port}`;
+        printDemoBanner(seedSummary, url);
+        const prefState = loadOrInitPreferences(demoPreferencesPath());
+        if (shouldAutoLaunchDashboard(prefState)) {
+            openBrowser(url);
+        }
+    });
+    const shutdown = async () => {
+        logger.info('Shutting down gracefully...');
+        await Promise.race([
+            new Promise((resolve) => server.close(() => resolve())),
+            new Promise((resolve) => setTimeout(resolve, 10_000)),
+        ]);
+        await storage.close();
+        logger.info('Shutdown complete');
+        process.exit(0);
+    };
+    process.on('SIGINT', shutdown);
+    process.on('SIGTERM', shutdown);
+}
+const run = values.demo ? runDemo : main;
+run().catch((err) => {
     logger.error(`Fatal error: ${err instanceof Error ? err.message : err}`, {
         stack: err instanceof Error ? err.stack : undefined,
     });

package/dist/middleware/rebinding-guard.d.ts

@@ -0,0 +1,21 @@
+import type { RequestHandler } from 'express';
+export declare function isLoopbackHost(host: string): boolean;
+/** Concrete origins/hosts this server answers to on `port`. */
+export declare function loopbackOriginsFor(port: number): string[];
+export declare function loopbackHostsFor(port: number): string[];
+export interface RebindingGuardOptions {
+    /**
+     * Port the server is actually bound to. Accepts a resolver because the
+     * middleware is registered BEFORE listen() — and the configured port is
+     * 0 whenever the caller wants an ephemeral one (tests and embedders do
+     * this). Baking 0 into the allowlist would produce `http://localhost:0`
+     * and reject every real request with a 403 that looks exactly like an
+     * attack. Same trap the MCP transport documents at transport/http.ts.
+     */
+    port: number | (() => number);
+    /** Bind address, used to decide whether Host validation applies. */
+    host: string;
+    /** Operator's configured origins; glob entries are ignored (see above). */
+    allowedOrigins?: string[];
+}
+export declare function createRebindingGuard(options: RebindingGuardOptions): RequestHandler;

package/dist/middleware/rebinding-guard.js

@@ -0,0 +1,77 @@
+/*
+ * DNS-rebinding protection for the dashboard HTTP API.
+ *
+ * v0.4.5 closed this hole on the MCP transport (/mcp) by handing
+ * allowedOrigins + allowedHosts to the SDK. The dashboard — same data,
+ * plus every mutating endpoint — never got the equivalent, and it starts
+ * implicitly alongside `--transport http`. So a browser on any page could
+ * POST a rule deployment to http://localhost:6920 and the server would
+ * execute it.
+ *
+ * CORS does not substitute, for the reason already written down in
+ * transport/http.ts: the browser withholds the RESPONSE, but the write has
+ * already happened. The request has to be REJECTED.
+ *
+ * Two checks, mirroring the SDK's semantics:
+ *
+ *   Origin — enforced whenever the header is present. Absent means a
+ *   non-browser client (curl, an MCP client, a health probe), which is not
+ *   the threat model here; browsers always send it on cross-origin
+ *   requests. Exact match only — glob patterns from the CORS allowlist are
+ *   meaningless against a single concrete Origin and are dropped rather
+ *   than left in the list looking effective.
+ *
+ *   Host — enforced only when bound to loopback. A non-loopback bind is a
+ *   deliberate network deployment, usually behind a proxy that rewrites
+ *   Host, and an exact-match list would break it.
+ */
+export function isLoopbackHost(host) {
+    return host === '127.0.0.1' || host === 'localhost' || host === '::1' || host === '[::1]';
+}
+/** Concrete origins/hosts this server answers to on `port`. */
+export function loopbackOriginsFor(port) {
+    return [`http://127.0.0.1:${port}`, `http://localhost:${port}`, `http://[::1]:${port}`];
+}
+export function loopbackHostsFor(port) {
+    return [`127.0.0.1:${port}`, `localhost:${port}`, `[::1]:${port}`];
+}
+export function createRebindingGuard(options) {
+    const { port, host, allowedOrigins = [] } = options;
+    const exactConfigured = allowedOrigins.filter((o) => !o.includes('*'));
+    const enforceHost = isLoopbackHost(host);
+    let cache;
+    function listsFor(resolvedPort) {
+        if (cache?.port !== resolvedPort) {
+            cache = {
+                port: resolvedPort,
+                origins: new Set([...loopbackOriginsFor(resolvedPort), ...exactConfigured]),
+                /*
+                 * `[::1]:port` is the form Node actually puts in the Host header
+                 * for an IPv6 loopback request — brackets included. A guard
+                 * written against the bare '::1' would be inert, which is exactly
+                 * how the citation-fetch SSRF guard was silently dead before
+                 * v0.4.5 (URL.hostname returns '[::1]', never '::1').
+                 */
+                hosts: new Set(loopbackHostsFor(resolvedPort)),
+            };
+        }
+        return cache;
+    }
+    return (req, res, next) => {
+        const resolvedPort = typeof port === 'function' ? port() : port;
+        const { origins, hosts } = listsFor(resolvedPort);
+        const origin = req.headers.origin;
+        if (origin && !origins.has(origin)) {
+            res.status(403).json({ error: 'Forbidden: invalid Origin header' });
+            return;
+        }
+        if (enforceHost) {
+            const hostHeader = req.headers.host;
+            if (hostHeader && !hosts.has(hostHeader)) {
+                res.status(403).json({ error: 'Forbidden: invalid Host header' });
+                return;
+            }
+        }
+        next();
+    };
+}

package/dist/preferences.js

@@ -18,9 +18,10 @@
  * /moments), dismissedTours, archivedMoments. These let the dashboard
  * remember the user's last view across reloads + across iris-mcp restarts.
  */
-import { mkdirSync, readFileSync, writeFileSync, existsSync, renameSync } from 'node:fs';
-import { join, dirname } from 'node:path';
-import { homedir } from 'node:os';
+import { readFileSync, existsSync } from 'node:fs';
+import { writeAtomic } from './utils/write-atomic.js';
+import { irisHome } from './utils/iris-home.js';
+import { join } from 'node:path';
 import { z } from 'zod';
 const MomentFiltersSchema = z
     .object({
@@ -61,7 +62,7 @@ export const PreferencesSchema = z
 })
     .passthrough();
 function defaultPreferencesPath() {
-    return join(homedir(), '.iris', 'preferences.json');
+    return join(irisHome(), 'preferences.json');
 }
 function freshPreferences() {
     return PreferencesSchema.parse({
@@ -74,12 +75,6 @@ function freshPreferences() {
         archivedMoments: [],
     });
 }
-function writeAtomic(targetPath, contents) {
-    mkdirSync(dirname(targetPath), { recursive: true });
-    const tmp = `${targetPath}.tmp.${process.pid}`;
-    writeFileSync(tmp, contents, 'utf-8');
-    renameSync(tmp, targetPath);
-}
 export function loadOrInitPreferences(customPath) {
     const path = customPath ?? defaultPreferencesPath();
     if (!existsSync(path)) {

package/dist/self-test.d.ts

@@ -0,0 +1,18 @@
+export declare const SELF_TEST_STEPS: {
+    readonly tempHome: "create isolated temp home";
+    readonly storage: "initialize storage";
+    readonly trace: "log a trace";
+    readonly piiEval: "eval: PII positive (planted SSN)";
+    readonly injectionEval: "eval: injection positive (planted override text)";
+    readonly cleanEval: "eval: clean output passes";
+    readonly readBack: "read back persisted results";
+    readonly dashboard: "start dashboard on ephemeral loopback port";
+    readonly health: "health endpoint answers";
+    readonly stats: "stats endpoint answers";
+    readonly rebindingGuard: "rebinding guard rejects hostile Origin";
+    readonly cleanup: "clean up temp home";
+};
+export declare const SELF_TEST_PASS_VERDICT = "\u2713 PASS \u2014 this install works";
+export declare const SELF_TEST_FAIL_VERDICT = "\u2717 FAIL";
+export type WriteLine = (line: string) => void;
+export declare function runSelfTest(write?: WriteLine): Promise<number>;