npm - @iris-eval/mcp-server - Versions diffs - 0.3.1 → 0.4.0 - Mend

@iris-eval/mcp-server 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (100) hide show

package/README.md +11 -3
package/dist/audit-log-reader.d.ts +24 -0
package/dist/audit-log-reader.js +87 -0
package/dist/config/defaults.js +7 -1
package/dist/custom-rule-store.d.ts +27 -0
package/dist/custom-rule-store.js +188 -0
package/dist/dashboard/assets/index-BEG5FYWH.css +1 -0
package/dist/dashboard/assets/index-D9JHfSB2.js +12 -0
package/dist/dashboard/index.html +2 -2
package/dist/dashboard/routes/audit.d.ts +3 -0
package/dist/dashboard/routes/audit.js +36 -0
package/dist/dashboard/routes/eval-stats.js +9 -4
package/dist/dashboard/routes/evaluations.js +3 -1
package/dist/dashboard/routes/filters.js +5 -3
package/dist/dashboard/routes/health.js +8 -1
package/dist/dashboard/routes/index.d.ts +4 -0
package/dist/dashboard/routes/index.js +4 -0
package/dist/dashboard/routes/moments.d.ts +3 -0
package/dist/dashboard/routes/moments.js +115 -0
package/dist/dashboard/routes/preferences.d.ts +3 -0
package/dist/dashboard/routes/preferences.js +52 -0
package/dist/dashboard/routes/rules.d.ts +10 -0
package/dist/dashboard/routes/rules.js +169 -0
package/dist/dashboard/routes/summary.js +3 -1
package/dist/dashboard/routes/traces.js +7 -4
package/dist/dashboard/server.d.ts +9 -1
package/dist/dashboard/server.js +52 -3
package/dist/eval/citation-verify/extract.d.ts +11 -0
package/dist/eval/citation-verify/extract.js +102 -0
package/dist/eval/citation-verify/resolve.d.ts +26 -0
package/dist/eval/citation-verify/resolve.js +237 -0
package/dist/eval/citation-verify/verifier.d.ts +43 -0
package/dist/eval/citation-verify/verifier.js +203 -0
package/dist/eval/decision-moment.d.ts +12 -0
package/dist/eval/decision-moment.js +181 -0
package/dist/eval/llm-judge/client.d.ts +28 -0
package/dist/eval/llm-judge/client.js +183 -0
package/dist/eval/llm-judge/evaluator.d.ts +32 -0
package/dist/eval/llm-judge/evaluator.js +138 -0
package/dist/eval/llm-judge/pricing.d.ts +9 -0
package/dist/eval/llm-judge/pricing.js +31 -0
package/dist/eval/llm-judge/templates/index.d.ts +20 -0
package/dist/eval/llm-judge/templates/index.js +170 -0
package/dist/eval/rules/custom.js +13 -2
package/dist/index.js +77 -14
package/dist/middleware/index.d.ts +1 -0
package/dist/middleware/index.js +1 -0
package/dist/middleware/tenant.d.ts +17 -0
package/dist/middleware/tenant.js +26 -0
package/dist/otel/exporter.d.ts +24 -0
package/dist/otel/exporter.js +116 -0
package/dist/otel/lazy.d.ts +5 -0
package/dist/otel/lazy.js +31 -0
package/dist/otel/mapper.d.ts +24 -0
package/dist/otel/mapper.js +208 -0
package/dist/preferences.d.ts +129 -0
package/dist/preferences.js +152 -0
package/dist/resources/dashboard-summary.js +3 -1
package/dist/resources/trace-detail.js +5 -3
package/dist/server.d.ts +3 -1
package/dist/server.js +9 -3
package/dist/storage/migrations/004-tenant-id.d.ts +3 -0
package/dist/storage/migrations/004-tenant-id.js +40 -0
package/dist/storage/migrations/index.js +2 -1
package/dist/storage/sqlite-adapter.d.ts +17 -15
package/dist/storage/sqlite-adapter.js +130 -79
package/dist/tools/delete-rule.d.ts +3 -0
package/dist/tools/delete-rule.js +53 -0
package/dist/tools/delete-trace.d.ts +3 -0
package/dist/tools/delete-trace.js +54 -0
package/dist/tools/deploy-rule.d.ts +3 -0
package/dist/tools/deploy-rule.js +91 -0
package/dist/tools/evaluate-output.js +23 -2
package/dist/tools/evaluate-with-llm-judge.d.ts +3 -0
package/dist/tools/evaluate-with-llm-judge.js +147 -0
package/dist/tools/get-traces.js +23 -3
package/dist/tools/index.d.ts +2 -1
package/dist/tools/index.js +13 -1
package/dist/tools/list-rules.d.ts +3 -0
package/dist/tools/list-rules.js +66 -0
package/dist/tools/log-trace.js +30 -2
package/dist/tools/verify-citations.d.ts +3 -0
package/dist/tools/verify-citations.js +157 -0
package/dist/types/custom-rule.d.ts +70 -0
package/dist/types/custom-rule.js +1 -0
package/dist/types/decision-moment.d.ts +122 -0
package/dist/types/decision-moment.js +17 -0
package/dist/types/index.d.ts +2 -0
package/dist/types/index.js +1 -1
package/dist/types/query.d.ts +25 -15
package/dist/types/tenant.d.ts +26 -0
package/dist/types/tenant.js +58 -0
package/dist/utils/open-browser.d.ts +1 -0
package/dist/utils/open-browser.js +45 -0
package/dist/utils/validate-port-config.d.ts +2 -0
package/dist/utils/validate-port-config.js +9 -0
package/package.json +4 -1
package/server.json +2 -2
package/dist/dashboard/assets/index-CnDg6bYi.js +0 -43
package/dist/dashboard/assets/index-ZsBou2-c.css +0 -1

package/dist/dashboard/index.html CHANGED Viewed

@@ -5,8 +5,8 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <title>Iris — Agent Eval & Observability</title>
-    <script type="module" crossorigin src="/assets/index-CnDg6bYi.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-ZsBou2-c.css">
+    <script type="module" crossorigin src="/assets/index-D9JHfSB2.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-BEG5FYWH.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/dashboard/routes/audit.d.ts ADDED Viewed

@@ -0,0 +1,3 @@
+import { Router } from 'express';
+import type { CustomRuleStore } from '../../custom-rule-store.js';
+export declare function registerAuditRoutes(router: Router, store?: CustomRuleStore): void;

package/dist/dashboard/routes/audit.js ADDED Viewed

@@ -0,0 +1,36 @@
+import { z } from 'zod';
+import { readAuditLog } from '../../audit-log-reader.js';
+const QuerySchema = z.object({
+    action: z.enum(['rule.deploy', 'rule.delete', 'rule.toggle', 'rule.update']).optional(),
+    since: z.string().datetime({ offset: true }).optional(),
+    search: z.string().max(200).optional(),
+    limit: z.coerce.number().int().min(1).max(1000).default(100),
+    offset: z.coerce.number().int().min(0).default(0),
+});
+export function registerAuditRoutes(router, store) {
+    router.get('/audit', (req, res) => {
+        try {
+            const query = QuerySchema.parse(req.query);
+            const result = readAuditLog({
+                // CustomRuleStore controls the audit file path; defer to it when
+                // available so tests + production match the same target.
+                filePath: store?.auditPath,
+                filter: {
+                    action: query.action,
+                    since: query.since,
+                    search: query.search,
+                },
+                limit: query.limit,
+                offset: query.offset,
+            });
+            res.json(result);
+        }
+        catch (err) {
+            if (err instanceof z.ZodError) {
+                res.status(400).json({ error: 'Invalid audit query', details: err.issues });
+                return;
+            }
+            throw err;
+        }
+    });
+}

package/dist/dashboard/routes/eval-stats.js CHANGED Viewed

@@ -1,3 +1,4 @@
+import { requireTenant } from '../../middleware/tenant.js';
 import { evalStatsPeriodSchema, evalStatsFailuresSchema } from '../validation.js';
 export function registerEvalStatsRoutes(router, storage) {
     /**
@@ -6,8 +7,9 @@ export function registerEvalStatsRoutes(router, storage) {
      */
     router.get('/eval-stats', async (req, res) => {
         try {
+            const tenantId = requireTenant(req);
             const { period } = evalStatsPeriodSchema.parse(req.query);
-            const stats = await storage.getEvalStats(period);
+            const stats = await storage.getEvalStats(tenantId, period);
             res.json(stats);
         }
         catch (err) {
@@ -24,8 +26,9 @@ export function registerEvalStatsRoutes(router, storage) {
      */
     router.get('/eval-stats/trend', async (req, res) => {
         try {
+            const tenantId = requireTenant(req);
             const { period } = evalStatsPeriodSchema.parse(req.query);
-            const trend = await storage.getEvalStatsTrend(period);
+            const trend = await storage.getEvalStatsTrend(tenantId, period);
             res.json(trend);
         }
         catch (err) {
@@ -43,8 +46,9 @@ export function registerEvalStatsRoutes(router, storage) {
      */
     router.get('/eval-stats/rules', async (req, res) => {
         try {
+            const tenantId = requireTenant(req);
             const { period } = evalStatsPeriodSchema.parse(req.query);
-            const rules = await storage.getEvalStatsRules(period);
+            const rules = await storage.getEvalStatsRules(tenantId, period);
             res.json(rules);
         }
         catch (err) {
@@ -61,8 +65,9 @@ export function registerEvalStatsRoutes(router, storage) {
      */
     router.get('/eval-stats/failures', async (req, res) => {
         try {
+            const tenantId = requireTenant(req);
             const query = evalStatsFailuresSchema.parse(req.query);
-            const failures = await storage.getEvalStatsFailures(query.period, query.limit);
+            const failures = await storage.getEvalStatsFailures(tenantId, query.period, query.limit);
             res.json(failures);
         }
         catch (err) {

package/dist/dashboard/routes/evaluations.js CHANGED Viewed

@@ -1,9 +1,11 @@
+import { requireTenant } from '../../middleware/tenant.js';
 import { evalQuerySchema } from '../validation.js';
 export function registerEvaluationRoutes(router, storage) {
     router.get('/evaluations', async (req, res) => {
         try {
+            const tenantId = requireTenant(req);
             const query = evalQuerySchema.parse(req.query);
-            const result = await storage.queryEvalResults({
+            const result = await storage.queryEvalResults(tenantId, {
                 eval_type: query.eval_type,
                 passed: query.passed,
                 since: query.since,

package/dist/dashboard/routes/filters.js CHANGED Viewed

@@ -1,8 +1,10 @@
+import { requireTenant } from '../../middleware/tenant.js';
 export function registerFilterRoutes(router, storage) {
-    router.get('/filters', async (_req, res) => {
+    router.get('/filters', async (req, res) => {
+        const tenantId = requireTenant(req);
         const [agentNames, frameworks] = await Promise.all([
-            storage.getDistinctValues('agent_name'),
-            storage.getDistinctValues('framework'),
+            storage.getDistinctValues(tenantId, 'agent_name'),
+            storage.getDistinctValues(tenantId, 'framework'),
         ]);
         res.json({ agent_names: agentNames, frameworks });
     });

package/dist/dashboard/routes/health.js CHANGED Viewed

@@ -1,3 +1,4 @@
+import { LOCAL_TENANT } from '../../types/tenant.js';
 const startTime = Date.now();
 export function registerHealthRoutes(router, storage, version) {
     const serverVersion = version ?? 'unknown';
@@ -5,7 +6,13 @@ export function registerHealthRoutes(router, storage, version) {
         const uptime_seconds = Math.floor((Date.now() - startTime) / 1000);
         if (storage) {
             try {
-                const summary = await storage.getDashboardSummary(1);
+                /* Health probes use LOCAL_TENANT directly — the health endpoint
+                 * is pre-tenant-resolution (runs for unauthenticated callers on
+                 * Cloud too) and reports server-level stats. This is the ONE
+                 * place storage is called with an explicit LOCAL_TENANT rather
+                 * than a resolved tenantId; deliberate, documented here so it
+                 * stays the only exception. */
+                const summary = await storage.getDashboardSummary(LOCAL_TENANT, 1);
                 res.json({
                     status: 'ok',
                     version: serverVersion,

package/dist/dashboard/routes/index.d.ts CHANGED Viewed

@@ -4,3 +4,7 @@ export { registerEvaluationRoutes } from './evaluations.js';
 export { registerEvalStatsRoutes } from './eval-stats.js';
 export { registerFilterRoutes } from './filters.js';
 export { registerHealthRoutes } from './health.js';
+export { registerMomentRoutes } from './moments.js';
+export { registerRuleRoutes } from './rules.js';
+export { registerPreferencesRoutes } from './preferences.js';
+export { registerAuditRoutes } from './audit.js';

package/dist/dashboard/routes/index.js CHANGED Viewed

@@ -4,3 +4,7 @@ export { registerEvaluationRoutes } from './evaluations.js';
 export { registerEvalStatsRoutes } from './eval-stats.js';
 export { registerFilterRoutes } from './filters.js';
 export { registerHealthRoutes } from './health.js';
+export { registerMomentRoutes } from './moments.js';
+export { registerRuleRoutes } from './rules.js';
+export { registerPreferencesRoutes } from './preferences.js';
+export { registerAuditRoutes } from './audit.js';

package/dist/dashboard/routes/moments.d.ts ADDED Viewed

@@ -0,0 +1,3 @@
+import { Router } from 'express';
+import type { IStorageAdapter } from '../../types/query.js';
+export declare function registerMomentRoutes(router: Router, storage: IStorageAdapter): void;

package/dist/dashboard/routes/moments.js ADDED Viewed

@@ -0,0 +1,115 @@
+import { z } from 'zod';
+import { requireTenant } from '../../middleware/tenant.js';
+import { deriveMoment, deriveMomentDetail } from '../../eval/decision-moment.js';
+const VERDICT_VALUES = ['pass', 'fail', 'partial', 'unevaluated'];
+const SIGNIFICANCE_KINDS = [
+    'safety-violation',
+    'cost-spike',
+    'first-failure',
+    'novel-pattern',
+    'rule-collision',
+    'normal-pass',
+    'normal-fail',
+];
+const momentQuerySchema = z.object({
+    agent_name: z.string().min(1).max(200).optional(),
+    verdict: z.enum(VERDICT_VALUES).optional(),
+    min_significance: z.coerce.number().min(0).max(1).optional(),
+    significance_kind: z
+        .enum(SIGNIFICANCE_KINDS)
+        .optional(),
+    since: z.string().datetime({ offset: true }).optional(),
+    until: z.string().datetime({ offset: true }).optional(),
+    limit: z.coerce.number().int().min(1).max(200).default(50),
+    offset: z.coerce.number().int().min(0).default(0),
+    sort_order: z.enum(['asc', 'desc']).default('desc'),
+});
+export function registerMomentRoutes(router, storage) {
+    router.get('/moments', async (req, res) => {
+        try {
+            const tenantId = requireTenant(req);
+            const query = momentQuerySchema.parse(req.query);
+            // Pull the underlying traces. We over-fetch when post-filtering by
+            // significance to give the moment classifier headroom, then trim.
+            const wantsSignificanceFilter = query.min_significance !== undefined || query.significance_kind !== undefined;
+            const fetchLimit = wantsSignificanceFilter
+                ? Math.min(query.limit * 4, 200)
+                : query.limit;
+            const traceResult = await storage.queryTraces(tenantId, {
+                filter: {
+                    agent_name: query.agent_name,
+                    since: query.since,
+                    until: query.until,
+                },
+                limit: fetchLimit,
+                offset: query.offset,
+                sort_by: 'timestamp',
+                sort_order: query.sort_order,
+            });
+            // Hydrate moments by fetching evals per trace. Acceptable for limit ≤ 200;
+            // batching is a v0.4.1 optimization once we have moment-volume data.
+            const moments = [];
+            for (const trace of traceResult.traces) {
+                const evals = await storage.getEvalsByTraceId(tenantId, trace.trace_id);
+                const moment = deriveMoment(trace, evals);
+                if (query.verdict && moment.verdict !== query.verdict)
+                    continue;
+                if (query.min_significance !== undefined &&
+                    moment.significance.score < query.min_significance)
+                    continue;
+                if (query.significance_kind &&
+                    moment.significance.kind !== query.significance_kind)
+                    continue;
+                moments.push(moment);
+                if (moments.length >= query.limit)
+                    break;
+            }
+            const result = {
+                moments,
+                // total reflects the underlying trace count (pre-filter) — significance-
+                // filtered totals would require materializing the full set, which we
+                // avoid for now. Clients should treat this as "at least this many."
+                total: traceResult.total,
+                limit: query.limit,
+                offset: query.offset,
+            };
+            res.json(result);
+        }
+        catch (err) {
+            if (err instanceof Error && err.name === 'ZodError') {
+                res.status(400).json({
+                    error: 'Invalid query parameters',
+                    details: err.issues,
+                });
+                return;
+            }
+            throw err;
+        }
+    });
+    router.get('/moments/:id', async (req, res) => {
+        try {
+            const tenantId = requireTenant(req);
+            const trace = await storage.getTrace(tenantId, req.params.id);
+            if (!trace) {
+                res.status(404).json({ error: 'Decision moment not found' });
+                return;
+            }
+            const [evals, spans] = await Promise.all([
+                storage.getEvalsByTraceId(tenantId, req.params.id),
+                storage.getSpansByTraceId(tenantId, req.params.id),
+            ]);
+            const detail = deriveMomentDetail(trace, evals, spans);
+            res.json(detail);
+        }
+        catch (err) {
+            if (err instanceof Error && err.name === 'ZodError') {
+                res.status(400).json({
+                    error: 'Invalid query parameters',
+                    details: err.issues,
+                });
+                return;
+            }
+            throw err;
+        }
+    });
+}

package/dist/dashboard/routes/preferences.d.ts ADDED Viewed

@@ -0,0 +1,3 @@
+import { Router } from 'express';
+import type { PreferenceStore } from '../../preferences.js';
+export declare function registerPreferencesRoutes(router: Router, store: PreferenceStore): void;

package/dist/dashboard/routes/preferences.js ADDED Viewed

@@ -0,0 +1,52 @@
+import { z } from 'zod';
+/* PATCH body schema. Every field optional + strict (rejects unknown keys
+ * to catch typos at the API boundary). Inner schemas mirror preferences.ts. */
+const MomentFiltersPatchSchema = z
+    .object({
+    agentName: z.string().max(200).optional(),
+    verdict: z.enum(['pass', 'fail', 'partial', 'unevaluated']).optional(),
+    significanceKind: z
+        .enum([
+        'safety-violation',
+        'cost-spike',
+        'first-failure',
+        'novel-pattern',
+        'rule-collision',
+        'normal-pass',
+        'normal-fail',
+    ])
+        .optional(),
+})
+    .strict();
+const PatchSchema = z
+    .object({
+    autoLaunch: z.boolean().optional(),
+    dismissedBanners: z.array(z.string().max(80)).optional(),
+    theme: z.enum(['dark', 'light', 'system']).optional(),
+    momentFilters: MomentFiltersPatchSchema.optional(),
+    dismissedTours: z.array(z.string().max(80)).optional(),
+    archivedMoments: z.array(z.string().max(200)).optional(),
+    density: z.enum(['compact', 'comfortable']).optional(),
+    sidebarCollapsed: z.boolean().optional(),
+    notificationsLastSeen: z.string().datetime({ offset: true }).optional(),
+})
+    .strict();
+export function registerPreferencesRoutes(router, store) {
+    router.get('/preferences', (_req, res) => {
+        res.json({ preferences: store.read(), path: store.path });
+    });
+    router.patch('/preferences', (req, res) => {
+        try {
+            const patch = PatchSchema.parse(req.body);
+            const updated = store.patch(patch);
+            res.json({ preferences: updated, path: store.path });
+        }
+        catch (err) {
+            if (err instanceof z.ZodError) {
+                res.status(400).json({ error: 'Invalid preferences patch', details: err.issues });
+                return;
+            }
+            throw err;
+        }
+    });
+}

package/dist/dashboard/routes/rules.d.ts ADDED Viewed

@@ -0,0 +1,10 @@
+import { Router } from 'express';
+import type { IStorageAdapter } from '../../types/query.js';
+import type { CustomRuleStore } from '../../custom-rule-store.js';
+import type { EvalEngine } from '../../eval/engine.js';
+interface RoutesOptions {
+    customRuleStore: CustomRuleStore;
+    evalEngine: EvalEngine;
+}
+export declare function registerRuleRoutes(router: Router, storage: IStorageAdapter, opts: RoutesOptions): void;
+export {};

package/dist/dashboard/routes/rules.js ADDED Viewed

@@ -0,0 +1,169 @@
+import { z } from 'zod';
+import { createCustomRule } from '../../eval/rules/custom.js';
+import { requireTenant } from '../../middleware/tenant.js';
+const SeveritySchema = z.enum(['low', 'medium', 'high', 'critical']);
+const EvalTypeSchema = z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']);
+const RuleTypeSchema = z.enum([
+    'regex_match',
+    'regex_no_match',
+    'min_length',
+    'max_length',
+    'contains_keywords',
+    'excludes_keywords',
+    'json_schema',
+    'cost_threshold',
+]);
+const DefinitionSchema = z.object({
+    name: z.string().min(1).max(80),
+    type: RuleTypeSchema,
+    config: z.record(z.unknown()),
+    weight: z.number().positive().optional(),
+});
+const DeploySchema = z.object({
+    name: z.string().min(1).max(80).regex(/^[a-z0-9._-]+$/i, 'Use letters, digits, dot, dash, underscore'),
+    description: z.string().max(500).optional(),
+    evalType: EvalTypeSchema,
+    severity: SeveritySchema.optional(),
+    definition: DefinitionSchema,
+    sourceMomentId: z.string().optional(),
+});
+const PreviewSchema = z.object({
+    definition: DefinitionSchema,
+    evalType: EvalTypeSchema.default('custom'),
+    /** Window in days; default 7. Hard cap at 30 to keep memory bounded. */
+    windowDays: z.coerce.number().int().min(1).max(30).default(7),
+    /** Trace cap; default 1000. */
+    maxTraces: z.coerce.number().int().min(1).max(5000).default(1000),
+});
+export function registerRuleRoutes(router, storage, opts) {
+    router.get('/rules/custom', (_req, res) => {
+        const rules = opts.customRuleStore.list();
+        res.json({ rules });
+    });
+    router.post('/rules/custom', async (req, res) => {
+        try {
+            const input = DeploySchema.parse(req.body);
+            // Server overrides the inner definition's `name` so it always matches the
+            // user-facing rule name. Avoids confusion when the rule name and the
+            // inner definition's check name diverge.
+            const definition = {
+                ...input.definition,
+                name: input.name,
+            };
+            const rule = opts.customRuleStore.deploy({
+                name: input.name,
+                description: input.description,
+                evalType: input.evalType,
+                severity: input.severity,
+                definition,
+                sourceMomentId: input.sourceMomentId,
+            });
+            // Register the new rule with the live engine so it fires on subsequent
+            // evaluate_output calls without requiring a server restart.
+            opts.evalEngine.registerRule(rule.evalType, createCustomRule(rule.definition));
+            res.status(201).json({ rule });
+        }
+        catch (err) {
+            if (err instanceof z.ZodError) {
+                res.status(400).json({ error: 'Invalid rule definition', details: err.issues });
+                return;
+            }
+            throw err;
+        }
+    });
+    router.delete('/rules/custom/:id', (req, res) => {
+        const removed = opts.customRuleStore.delete(req.params.id);
+        if (!removed) {
+            res.status(404).json({ error: 'Rule not found' });
+            return;
+        }
+        // Note: removing from the live engine requires a registry reset, which
+        // the engine doesn't expose in v0.4. The deleted rule continues to fire
+        // until the next iris-mcp restart. Documented behavior; v0.4.1 adds
+        // engine.unregisterRule for hot-removal.
+        res.status(204).end();
+    });
+    router.post('/rules/custom/preview', async (req, res) => {
+        try {
+            const tenantId = requireTenant(req);
+            const input = PreviewSchema.parse(req.body);
+            const result = await previewRule(tenantId, input, storage);
+            res.json(result);
+        }
+        catch (err) {
+            if (err instanceof z.ZodError) {
+                res.status(400).json({ error: 'Invalid preview request', details: err.issues });
+                return;
+            }
+            const status = err.status;
+            if (status === 422) {
+                res.status(422).json({ error: 'Rule definition rejected', message: err.message });
+                return;
+            }
+            throw err;
+        }
+    });
+}
+async function previewRule(tenantId, input, storage) {
+    const since = new Date(Date.now() - input.windowDays * 24 * 60 * 60 * 1000).toISOString();
+    const traceResult = await storage.queryTraces(tenantId, {
+        filter: { since },
+        limit: input.maxTraces,
+        sort_by: 'timestamp',
+        sort_order: 'desc',
+    });
+    const rule = createCustomRule(input.definition);
+    // Sanity-probe the rule against an empty input. Compile-time errors
+    // (invalid regex, pattern too long, ReDoS rejection) surface here as a
+    // failed result with a recognizable error prefix. Surface as a 422 so
+    // the UI can show the error instead of a misleading "5 traces fail."
+    const probe = rule.evaluate({ output: '' });
+    if (!probe.skipped &&
+        !probe.passed &&
+        /^(?:Invalid regex|Regex pattern (?:too long|rejected))/.test(probe.message)) {
+        const err = new Error(probe.message);
+        err.status = 422;
+        throw err;
+    }
+    let wouldPass = 0;
+    let wouldFail = 0;
+    let wouldSkip = 0;
+    const examples = [];
+    for (const trace of traceResult.traces) {
+        if (trace.output === undefined) {
+            wouldSkip++;
+            continue;
+        }
+        const result = rule.evaluate({
+            output: trace.output,
+            input: trace.input,
+            costUsd: trace.cost_usd,
+            tokenUsage: trace.token_usage,
+        });
+        if (result.skipped) {
+            wouldSkip++;
+        }
+        else if (result.passed) {
+            wouldPass++;
+        }
+        else {
+            wouldFail++;
+            if (examples.length < 5) {
+                examples.push({
+                    traceId: trace.trace_id,
+                    agentName: trace.agent_name,
+                    timestamp: trace.timestamp,
+                    outputPreview: trace.output.slice(0, 200),
+                });
+            }
+        }
+    }
+    return {
+        tracesEvaluated: traceResult.traces.length,
+        wouldFail,
+        wouldPass,
+        wouldSkip,
+        examples,
+        windowSinceIso: since,
+    };
+}

package/dist/dashboard/routes/summary.js CHANGED Viewed

@@ -1,9 +1,11 @@
+import { requireTenant } from '../../middleware/tenant.js';
 import { summaryQuerySchema } from '../validation.js';
 export function registerSummaryRoutes(router, storage) {
     router.get('/summary', async (req, res) => {
         try {
+            const tenantId = requireTenant(req);
             const query = summaryQuerySchema.parse(req.query);
-            const summary = await storage.getDashboardSummary(query.hours);
+            const summary = await storage.getDashboardSummary(tenantId, query.hours);
             res.json(summary);
         }
         catch (err) {

package/dist/dashboard/routes/traces.js CHANGED Viewed

@@ -1,9 +1,11 @@
+import { requireTenant } from '../../middleware/tenant.js';
 import { traceQuerySchema } from '../validation.js';
 export function registerTraceRoutes(router, storage) {
     router.get('/traces', async (req, res) => {
         try {
+            const tenantId = requireTenant(req);
             const query = traceQuerySchema.parse(req.query);
-            const result = await storage.queryTraces({
+            const result = await storage.queryTraces(tenantId, {
                 filter: {
                     agent_name: query.agent_name,
                     framework: query.framework,
@@ -27,13 +29,14 @@ export function registerTraceRoutes(router, storage) {
     });
     router.get('/traces/:id', async (req, res) => {
         try {
-            const trace = await storage.getTrace(req.params.id);
+            const tenantId = requireTenant(req);
+            const trace = await storage.getTrace(tenantId, req.params.id);
             if (!trace) {
                 res.status(404).json({ error: 'Trace not found' });
                 return;
             }
-            const spans = await storage.getSpansByTraceId(req.params.id);
-            const evals = await storage.getEvalsByTraceId(req.params.id);
+            const spans = await storage.getSpansByTraceId(tenantId, req.params.id);
+            const evals = await storage.getEvalsByTraceId(tenantId, req.params.id);
             res.json({ trace, spans, evals });
         }
         catch (err) {

package/dist/dashboard/server.d.ts CHANGED Viewed

@@ -3,8 +3,16 @@ import type { Server } from 'node:http';
 import type { IStorageAdapter } from '../types/query.js';
 import type { IrisConfig } from '../types/config.js';
 import type { Logger } from '../utils/logger.js';
+import type { CustomRuleStore } from '../custom-rule-store.js';
+import type { EvalEngine } from '../eval/engine.js';
+import type { PreferenceStore } from '../preferences.js';
 export interface DashboardServer {
     app: express.Application;
     start(): Server;
 }
-export declare function createDashboardServer(storage: IStorageAdapter, config: IrisConfig, logger: Logger): DashboardServer;
+export interface DashboardServerOptions {
+    customRuleStore?: CustomRuleStore;
+    evalEngine?: EvalEngine;
+    preferenceStore?: PreferenceStore;
+}
+export declare function createDashboardServer(storage: IStorageAdapter, config: IrisConfig, logger: Logger, options?: DashboardServerOptions): DashboardServer;