@iris-eval/mcp-server 0.3.1 → 0.4.0

package/dist/dashboard/server.js

@@ -7,13 +7,18 @@ import { createAuthMiddleware } from '../middleware/auth.js';
 import { createCorsMiddleware } from '../middleware/cors.js';
 import { createErrorHandler } from '../middleware/error-handler.js';
 import { createApiRateLimiter } from '../middleware/rate-limit.js';
+import { createTenantMiddleware } from '../middleware/tenant.js';
 import { registerTraceRoutes } from './routes/traces.js';
 import { registerSummaryRoutes } from './routes/summary.js';
 import { registerEvaluationRoutes } from './routes/evaluations.js';
 import { registerFilterRoutes } from './routes/filters.js';
 import { registerEvalStatsRoutes } from './routes/eval-stats.js';
 import { registerHealthRoutes } from './routes/health.js';
-export function createDashboardServer(storage, config, logger) {
+import { registerMomentRoutes } from './routes/moments.js';
+import { registerRuleRoutes } from './routes/rules.js';
+import { registerPreferencesRoutes } from './routes/preferences.js';
+import { registerAuditRoutes } from './routes/audit.js';
+export function createDashboardServer(storage, config, logger, options) {
     const app = express();
     // Security headers
     app.use(helmet({
@@ -21,7 +26,15 @@ export function createDashboardServer(storage, config, logger) {
             directives: {
                 defaultSrc: ["'self'"],
                 scriptSrc: ["'self'"],
-                styleSrc: ["'self'", "'unsafe-inline'"],
+                // 'self' covers our bundled CSS. fonts.googleapis.com hosts the
+                // brand fonts (Space Grotesk + Manrope + JetBrains Mono) loaded
+                // via @import in tokens.css. Without this, the @import gets
+                // blocked and the entire stylesheet is dropped by the browser.
+                // v0.4.1 will self-host these fonts and let us tighten this back
+                // to 'self' only.
+                styleSrc: ["'self'", "'unsafe-inline'", "https://fonts.googleapis.com"],
+                // The fontFaces in those stylesheets resolve to fonts.gstatic.com.
+                fontSrc: ["'self'", "https://fonts.gstatic.com", "data:"],
                 connectSrc: ["'self'"],
             },
         },
@@ -32,6 +45,10 @@ export function createDashboardServer(storage, config, logger) {
     app.use(createCorsMiddleware(config.security.allowedOrigins));
     // Authentication
     app.use(createAuthMiddleware(config));
+    // Tenant resolution — attaches req.tenantId to every request.
+    // OSS: always resolves to LOCAL_TENANT. Cloud: swaps for an auth-aware
+    // resolver that reads the authenticated session. See middleware/tenant.ts.
+    app.use(createTenantMiddleware());
     // API routes with rate limiting
     const router = express.Router();
     router.use(createApiRateLimiter(config));
@@ -41,6 +58,19 @@ export function createDashboardServer(storage, config, logger) {
     registerEvalStatsRoutes(router, storage);
     registerFilterRoutes(router, storage);
     registerHealthRoutes(router, storage, config.server.version);
+    registerMomentRoutes(router, storage);
+    if (options?.customRuleStore && options?.evalEngine) {
+        registerRuleRoutes(router, storage, {
+            customRuleStore: options.customRuleStore,
+            evalEngine: options.evalEngine,
+        });
+    }
+    if (options?.preferenceStore) {
+        registerPreferencesRoutes(router, options.preferenceStore);
+    }
+    // Audit always available — falls back to default ~/.iris/audit.log path
+    // when no custom rule store is provided (read-only access).
+    registerAuditRoutes(router, options?.customRuleStore);
     app.use('/api/v1', router);
     // Serve static dashboard files if built (rate limited)
     const currentDir = dirname(fileURLToPath(import.meta.url));
@@ -57,9 +87,28 @@ export function createDashboardServer(storage, config, logger) {
     return {
         app,
         start() {
-            return app.listen(config.dashboard.port, () => {
+            const server = app.listen(config.dashboard.port, () => {
                 logger.info(`Dashboard available at http://localhost:${config.dashboard.port}`);
             });
+            /*
+             * F-006: surface listen() errors instead of swallowing them.
+             * Without this handler, EADDRINUSE (port already bound, typically
+             * by the MCP HTTP transport) goes to the default Node 'error'
+             * handler which emits a warning but doesn't crash — so the process
+             * keeps running in a broken state. We log the specific cause then
+             * exit(1) so the user sees the actual problem.
+             */
+            server.on('error', (err) => {
+                if (err.code === 'EADDRINUSE') {
+                    logger.error(`Dashboard failed to start: port ${config.dashboard.port} is already in use. ` +
+                        `If running HTTP transport on the same port, use --dashboard-port <other>.`);
+                }
+                else {
+                    logger.error(`Dashboard server error: ${err.message}`);
+                }
+                process.exit(1);
+            });
+            return server;
         },
     };
 }

package/dist/eval/citation-verify/extract.d.ts

@@ -0,0 +1,11 @@
+export interface ExtractedCitation {
+    raw: string;
+    kind: 'numbered' | 'author_year' | 'url' | 'doi';
+    offsetStart: number;
+    offsetEnd: number;
+    contextWindow: string;
+    identifier: string;
+}
+export declare function extractCitations(output: string): ExtractedCitation[];
+export declare function hasAnyCitations(output: string): boolean;
+export declare function countCitationsByKind(output: string): Record<ExtractedCitation['kind'], number>;

package/dist/eval/citation-verify/extract.js

@@ -0,0 +1,102 @@
+// Citation extraction — finds references inside an AI-generated output.
+// No network, no judgment on validity; the resolver + verifier do that.
+// This file's only job is "what did the model claim to cite, and what's
+// the textual context around each citation?"
+//
+// Supported citation forms:
+//   [1], [2], [3]            — numbered citations (bare)
+//   [1][2]                   — consecutive numbered citations
+//   (Author, 2021)           — author-year parenthetical
+//   (Author et al., 2021)    — multi-author parenthetical
+//   https://example.com/...  — bare URLs
+//   10.1234/foo.bar          — DOIs (bare or with https://doi.org/)
+//
+// We intentionally do NOT try to parse footnote DEFINITIONS here — the
+// extractor pairs each INLINE citation with a local sentence/paragraph
+// context window. The verifier will match (citation, context) pairs
+// against resolved sources.
+const URL_PATTERN = /https?:\/\/[^\s)\]]+/g;
+// Basic DOI — 10. prefix + registrant + slash + suffix
+const DOI_PATTERN = /\b10\.\d{4,9}\/[^\s)\];,"'<>]+/g;
+// [1], [2], [12]. Avoid matching [INSERT], [FIXME] etc. by requiring digits only.
+const NUMBERED_PATTERN = /\[(\d{1,3})\]/g;
+// (Author, 2021), (Smith et al., 2019), (van der Berg, 2023)
+// Restrictive enough to avoid matching any parenthetical aside:
+// - 1-40 chars inside parens
+// - ends with ", YYYY" where YYYY is 1800-2099
+const AUTHOR_YEAR_PATTERN = /\(([^()]{1,50}?,\s*(?:18|19|20)\d{2})\)/g;
+function makeContext(text, start, end) {
+    const radius = 200;
+    const contextStart = Math.max(0, start - radius);
+    const contextEnd = Math.min(text.length, end + radius);
+    let ctx = text.slice(contextStart, contextEnd);
+    if (contextStart > 0)
+        ctx = '…' + ctx;
+    if (contextEnd < text.length)
+        ctx = ctx + '…';
+    return ctx;
+}
+export function extractCitations(output) {
+    if (!output)
+        return [];
+    const results = [];
+    for (const match of output.matchAll(NUMBERED_PATTERN)) {
+        results.push({
+            raw: match[0],
+            kind: 'numbered',
+            offsetStart: match.index ?? 0,
+            offsetEnd: (match.index ?? 0) + match[0].length,
+            contextWindow: makeContext(output, match.index ?? 0, (match.index ?? 0) + match[0].length),
+            identifier: match[1],
+        });
+    }
+    for (const match of output.matchAll(URL_PATTERN)) {
+        // Strip trailing punctuation that shouldn't be part of the URL.
+        let cleaned = match[0].replace(/[.,;:!?)"'>]+$/, '');
+        results.push({
+            raw: match[0],
+            kind: 'url',
+            offsetStart: match.index ?? 0,
+            offsetEnd: (match.index ?? 0) + cleaned.length,
+            contextWindow: makeContext(output, match.index ?? 0, (match.index ?? 0) + cleaned.length),
+            identifier: cleaned,
+        });
+    }
+    for (const match of output.matchAll(DOI_PATTERN)) {
+        // Skip DOIs that are inside a URL (already captured as a URL).
+        const inUrl = /https?:\/\/[^\s]*$/.test(output.slice(0, match.index ?? 0));
+        if (inUrl)
+            continue;
+        const cleaned = match[0].replace(/[.,;:!?)"'>]+$/, '');
+        results.push({
+            raw: match[0],
+            kind: 'doi',
+            offsetStart: match.index ?? 0,
+            offsetEnd: (match.index ?? 0) + cleaned.length,
+            contextWindow: makeContext(output, match.index ?? 0, (match.index ?? 0) + cleaned.length),
+            identifier: cleaned,
+        });
+    }
+    for (const match of output.matchAll(AUTHOR_YEAR_PATTERN)) {
+        results.push({
+            raw: match[0],
+            kind: 'author_year',
+            offsetStart: match.index ?? 0,
+            offsetEnd: (match.index ?? 0) + match[0].length,
+            contextWindow: makeContext(output, match.index ?? 0, (match.index ?? 0) + match[0].length),
+            identifier: match[1].trim(),
+        });
+    }
+    // Sort by position — dashboards expect reading order.
+    return results.sort((a, b) => a.offsetStart - b.offsetStart);
+}
+export function hasAnyCitations(output) {
+    return extractCitations(output).length > 0;
+}
+export function countCitationsByKind(output) {
+    const counts = { numbered: 0, author_year: 0, url: 0, doi: 0 };
+    for (const c of extractCitations(output)) {
+        counts[c.kind]++;
+    }
+    return counts;
+}

package/dist/eval/citation-verify/resolve.d.ts

@@ -0,0 +1,26 @@
+export interface ResolveOptions {
+    allowFetch: boolean;
+    timeoutMs?: number;
+    maxBytes?: number;
+    domainAllowlist?: readonly string[];
+    maxRedirects?: number;
+}
+export interface ResolvedSource {
+    url: string;
+    requestedUrl: string;
+    status: number;
+    contentType: string;
+    text: string;
+    truncated: boolean;
+    fetchedAt: string;
+    bytesFetched: number;
+    fromCache: boolean;
+}
+export declare class CitationResolveError extends Error {
+    readonly kind: 'bad_scheme' | 'ssrf' | 'not_allowed_domain' | 'timeout' | 'too_large' | 'bad_status' | 'redirect_loop' | 'not_text' | 'fetch_disabled';
+    readonly details?: string | undefined;
+    constructor(message: string, kind: 'bad_scheme' | 'ssrf' | 'not_allowed_domain' | 'timeout' | 'too_large' | 'bad_status' | 'redirect_loop' | 'not_text' | 'fetch_disabled', details?: string | undefined);
+}
+export declare function isSafeHost(host: string): boolean;
+export declare function __clearCitationCacheForTests(): void;
+export declare function resolveSource(identifier: string, opts: ResolveOptions): Promise<ResolvedSource>;

package/dist/eval/citation-verify/resolve.js

@@ -0,0 +1,237 @@
+// Citation source resolver — fetches URLs and DOIs so the verifier can
+// feed them to the LLM judge. This is the security-sensitive piece: we
+// are fetching URLs that appeared in model-generated output, which is
+// effectively user-controlled input.
+//
+// Defense layers (in order):
+//   1. Scheme allowlist — http/https only; refuse file:/javascript:/etc.
+//   2. SSRF host check — refuse localhost, link-local, private ranges,
+//      and cloud metadata (AWS/GCP/Azure/DigitalOcean) IP literals.
+//   3. Optional domain allowlist — IRIS_CITATION_DOMAINS=doi.org,arxiv.org
+//      restricts to a curated set; empty/unset = open web (still SSRF-guarded).
+//   4. Timeout + size cap — 10s default, 5MB cap on response body.
+//   5. Redirect chase cap — follow max 3 redirects, each re-checked.
+//   6. Cache — in-process LRU (100 entries) so retries don't re-fetch.
+//
+// This is opt-in: calls require passing {allowFetch: true} so an agent
+// can't trick Iris into fetching random URLs without operator consent
+// (consent granted via tool param or env IRIS_CITATION_ALLOW_FETCH=1).
+export class CitationResolveError extends Error {
+    kind;
+    details;
+    constructor(message, kind, details) {
+        super(message);
+        this.kind = kind;
+        this.details = details;
+        this.name = 'CitationResolveError';
+    }
+}
+// Private IP ranges + localhost + link-local + cloud metadata.
+const BLOCKED_IPV4 = [
+    // Localhost
+    /^127\./,
+    // Link-local
+    /^169\.254\./,
+    // Private RFC 1918
+    /^10\./,
+    /^192\.168\./,
+    /^172\.(1[6-9]|2\d|3[01])\./,
+    // Cloud metadata
+    /^169\.254\.169\.254$/,
+    // Broadcast
+    /^255\.255\.255\.255$/,
+    // This-network
+    /^0\./,
+];
+const BLOCKED_IPV6 = [
+    /^::1$/, // localhost
+    /^fc|^fd/i, // unique local
+    /^fe80/i, // link-local
+    /^::ffff:127\./i, // IPv4-mapped localhost
+];
+const BLOCKED_HOST_SUBSTRINGS = ['localhost', 'internal', '.local', 'metadata.google', 'metadata.azure'];
+function isIpv4(host) {
+    return /^\d{1,3}(\.\d{1,3}){3}$/.test(host);
+}
+function isIpv6(host) {
+    return host.includes(':');
+}
+export function isSafeHost(host) {
+    const hostLower = host.toLowerCase();
+    for (const sub of BLOCKED_HOST_SUBSTRINGS) {
+        if (hostLower === sub || hostLower.endsWith(sub))
+            return false;
+    }
+    if (isIpv4(host)) {
+        for (const re of BLOCKED_IPV4) {
+            if (re.test(host))
+                return false;
+        }
+    }
+    if (isIpv6(host)) {
+        for (const re of BLOCKED_IPV6) {
+            if (re.test(host))
+                return false;
+        }
+    }
+    return true;
+}
+function matchesAllowlist(host, allowlist) {
+    if (!allowlist || allowlist.length === 0)
+        return true;
+    const hostLower = host.toLowerCase();
+    for (const allowed of allowlist) {
+        const a = allowed.toLowerCase();
+        if (hostLower === a || hostLower.endsWith('.' + a))
+            return true;
+    }
+    return false;
+}
+// Tiny LRU — short-circuits duplicate fetches in a single batch of
+// citations (e.g. 5 citations in one output all pointing to the same
+// source). Not durable; every process start is fresh.
+const CACHE_MAX = 100;
+const cache = new Map();
+function cacheGet(key) {
+    const hit = cache.get(key);
+    if (!hit)
+        return undefined;
+    // Re-insert to bump LRU order
+    cache.delete(key);
+    cache.set(key, hit);
+    return { ...hit, fromCache: true };
+}
+function cacheSet(key, value) {
+    if (cache.size >= CACHE_MAX) {
+        const firstKey = cache.keys().next().value;
+        if (firstKey !== undefined)
+            cache.delete(firstKey);
+    }
+    cache.set(key, value);
+}
+export function __clearCitationCacheForTests() {
+    cache.clear();
+}
+function normalizeDoiToUrl(doiOrUrl) {
+    const trimmed = doiOrUrl.trim();
+    if (/^https?:\/\//i.test(trimmed))
+        return trimmed;
+    if (/^10\.\d{4,9}\//.test(trimmed))
+        return `https://doi.org/${trimmed}`;
+    return trimmed;
+}
+async function doFetch(url, opts, redirectsLeft) {
+    const timeoutMs = opts.timeoutMs ?? 10_000;
+    const maxBytes = opts.maxBytes ?? 5 * 1024 * 1024;
+    const parsed = new URL(url);
+    if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
+        throw new CitationResolveError(`Refusing non-http(s) scheme: ${parsed.protocol}`, 'bad_scheme', parsed.protocol);
+    }
+    if (!isSafeHost(parsed.hostname)) {
+        throw new CitationResolveError(`Refusing SSRF-blocked host: ${parsed.hostname}`, 'ssrf', parsed.hostname);
+    }
+    if (!matchesAllowlist(parsed.hostname, opts.domainAllowlist)) {
+        throw new CitationResolveError(`Host ${parsed.hostname} not in IRIS_CITATION_DOMAINS allowlist`, 'not_allowed_domain', parsed.hostname);
+    }
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    let res;
+    try {
+        res = await fetch(url, {
+            signal: controller.signal,
+            redirect: 'manual',
+            headers: {
+                'user-agent': 'iris-mcp-citation-verifier/0.4 (+https://iris-eval.com)',
+                accept: 'text/html, text/plain, application/pdf, application/xhtml+xml, */*;q=0.1',
+            },
+        });
+    }
+    catch (err) {
+        if (err instanceof Error && err.name === 'AbortError') {
+            throw new CitationResolveError(`Fetch timed out after ${timeoutMs}ms`, 'timeout');
+        }
+        throw err;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+    // Manual redirect chase — we re-check each target against SSRF rules.
+    if (res.status >= 300 && res.status < 400) {
+        const loc = res.headers.get('location');
+        if (!loc) {
+            throw new CitationResolveError(`Redirect with no Location header (${res.status})`, 'bad_status');
+        }
+        if (redirectsLeft <= 0) {
+            throw new CitationResolveError(`Exceeded max redirects (${opts.maxRedirects ?? 3})`, 'redirect_loop');
+        }
+        const next = new URL(loc, url).toString();
+        return doFetch(next, opts, redirectsLeft - 1);
+    }
+    if (res.status < 200 || res.status >= 300) {
+        throw new CitationResolveError(`Bad status ${res.status}`, 'bad_status', String(res.status));
+    }
+    const contentType = (res.headers.get('content-type') ?? '').toLowerCase();
+    // We only extract text. PDFs could be supported later with pdf-parse
+    // but that's an opt-in + heavy dep. For now we refuse non-text.
+    const textLike = contentType.includes('text/') ||
+        contentType.includes('xml') ||
+        contentType.includes('json') ||
+        contentType === '';
+    if (!textLike) {
+        throw new CitationResolveError(`Refusing non-text content-type: ${contentType}`, 'not_text', contentType);
+    }
+    // Stream with a byte cap so we don't DoS ourselves on a huge body.
+    if (!res.body) {
+        throw new CitationResolveError('Response had no body stream', 'bad_status');
+    }
+    const reader = res.body.getReader();
+    const chunks = [];
+    let total = 0;
+    let truncated = false;
+    for (;;) {
+        const { value, done } = await reader.read();
+        if (done)
+            break;
+        total += value.byteLength;
+        if (total > maxBytes) {
+            truncated = true;
+            chunks.push(value.slice(0, Math.max(0, maxBytes - (total - value.byteLength))));
+            await reader.cancel();
+            break;
+        }
+        chunks.push(value);
+    }
+    // Concat + decode. UTF-8 is good enough for this use case — any weird
+    // encoding will surface as mojibake for the LLM judge, still safe.
+    const buf = new Uint8Array(total > maxBytes ? maxBytes : total);
+    let off = 0;
+    for (const c of chunks) {
+        buf.set(c, off);
+        off += c.byteLength;
+    }
+    const text = new TextDecoder('utf-8', { fatal: false }).decode(buf);
+    return {
+        url: res.url || url,
+        requestedUrl: url,
+        status: res.status,
+        contentType,
+        text,
+        truncated,
+        fetchedAt: new Date().toISOString(),
+        bytesFetched: Math.min(total, maxBytes),
+        fromCache: false,
+    };
+}
+export async function resolveSource(identifier, opts) {
+    if (!opts.allowFetch) {
+        throw new CitationResolveError('Citation fetch is disabled. Pass allowFetch:true or set IRIS_CITATION_ALLOW_FETCH=1.', 'fetch_disabled');
+    }
+    const url = normalizeDoiToUrl(identifier);
+    const cacheKey = url;
+    const cached = cacheGet(cacheKey);
+    if (cached)
+        return cached;
+    const fetched = await doFetch(url, opts, opts.maxRedirects ?? 3);
+    cacheSet(cacheKey, fetched);
+    return fetched;
+}

package/dist/eval/citation-verify/verifier.d.ts

@@ -0,0 +1,43 @@
+import { type LLMProvider } from '../llm-judge/client.js';
+import { type ExtractedCitation } from './extract.js';
+import { type ResolvedSource } from './resolve.js';
+export interface VerifyCitationsParams {
+    output: string;
+    provider: LLMProvider;
+    model: string;
+    apiKey: string;
+    allowFetch: boolean;
+    domainAllowlist?: readonly string[];
+    maxCostUsdTotal?: number;
+    perSourceTimeoutMs?: number;
+    perSourceMaxBytes?: number;
+    maxCitations?: number;
+}
+export interface VerifiedCitation {
+    citation: ExtractedCitation;
+    resolveStatus: 'ok' | 'skipped' | 'error';
+    resolveError?: {
+        kind: string;
+        message: string;
+    };
+    source?: Pick<ResolvedSource, 'url' | 'status' | 'contentType' | 'bytesFetched' | 'truncated'>;
+    judge?: {
+        supported: boolean;
+        confidence: number;
+        rationale: string;
+        inputTokens: number;
+        outputTokens: number;
+        costUsd: number | null;
+        latencyMs: number;
+    };
+}
+export interface VerifyCitationsResult {
+    overallScore: number | null;
+    passed: boolean;
+    citations: VerifiedCitation[];
+    totalCostUsd: number;
+    totalCitationsFound: number;
+    totalResolved: number;
+    totalSupported: number;
+}
+export declare function verifyCitations(params: VerifyCitationsParams): Promise<VerifyCitationsResult>;