npm - @iris-eval/mcp-server - Versions diffs - 0.4.0 → 0.4.1 - Mend

@iris-eval/mcp-server 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (27) hide show

package/README.md +2 -2
package/dist/custom-rule-store.d.ts +18 -10
package/dist/custom-rule-store.js +81 -45
package/dist/dashboard/assets/{index-BEG5FYWH.css → index-B4Aw6ozt.css} +1 -1
package/dist/dashboard/assets/index-YnFsPfd6.js +12 -0
package/dist/dashboard/index.html +2 -2
package/dist/dashboard/routes/rules.js +18 -5
package/dist/eval/citation-verify/resolve.d.ts +7 -0
package/dist/eval/citation-verify/resolve.js +42 -7
package/dist/index.js +5 -2
package/dist/lib/claims.d.ts +111 -0
package/dist/lib/claims.js +52 -0
package/dist/middleware/auth.js +12 -1
package/dist/middleware/cors.js +7 -1
package/dist/middleware/tenant.d.ts +5 -3
package/dist/tools/delete-rule.js +7 -1
package/dist/tools/delete-trace.js +4 -0
package/dist/tools/deploy-rule.js +7 -1
package/dist/tools/evaluate-output.js +12 -8
package/dist/tools/evaluate-with-llm-judge.js +4 -0
package/dist/tools/get-traces.js +15 -11
package/dist/tools/list-rules.js +9 -1
package/dist/tools/log-trace.js +15 -11
package/dist/tools/verify-citations.js +4 -0
package/package.json +5 -1
package/server.json +2 -2
package/dist/dashboard/assets/index-D9JHfSB2.js +0 -12

package/dist/dashboard/index.html CHANGED Viewed

@@ -5,8 +5,8 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <title>Iris — Agent Eval & Observability</title>
-    <script type="module" crossorigin src="/assets/index-D9JHfSB2.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-BEG5FYWH.css">
+    <script type="module" crossorigin src="/assets/index-YnFsPfd6.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-B4Aw6ozt.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/dashboard/routes/rules.js CHANGED Viewed

@@ -36,12 +36,22 @@ const PreviewSchema = z.object({
     maxTraces: z.coerce.number().int().min(1).max(5000).default(1000),
 });
 export function registerRuleRoutes(router, storage, opts) {
-    router.get('/rules/custom', (_req, res) => {
-        const rules = opts.customRuleStore.list();
+    /*
+     * Every /rules/custom route resolves the tenant via requireTenant(req)
+     * and threads it through to the store. In OSS the tenant middleware
+     * always sets LOCAL_TENANT, so all rules continue to live at
+     * ~/.iris/custom-rules.json (the v0.4 path — zero migration). In Cloud,
+     * each tenant's rules will live in their own file (per-tenant partition)
+     * and writes will only ever touch the resolved tenant's data.
+     */
+    router.get('/rules/custom', (req, res) => {
+        const tenantId = requireTenant(req);
+        const rules = opts.customRuleStore.list(tenantId);
         res.json({ rules });
     });
     router.post('/rules/custom', async (req, res) => {
         try {
+            const tenantId = requireTenant(req);
             const input = DeploySchema.parse(req.body);
             // Server overrides the inner definition's `name` so it always matches the
             // user-facing rule name. Avoids confusion when the rule name and the
@@ -50,7 +60,7 @@ export function registerRuleRoutes(router, storage, opts) {
                 ...input.definition,
                 name: input.name,
             };
-            const rule = opts.customRuleStore.deploy({
+            const rule = opts.customRuleStore.deploy(tenantId, {
                 name: input.name,
                 description: input.description,
                 evalType: input.evalType,
@@ -59,7 +69,9 @@ export function registerRuleRoutes(router, storage, opts) {
                 sourceMomentId: input.sourceMomentId,
             });
             // Register the new rule with the live engine so it fires on subsequent
-            // evaluate_output calls without requiring a server restart.
+            // evaluate_output calls without requiring a server restart. The engine
+            // is process-global in v0.4 — Cloud multi-tenant engine wiring is a
+            // v0.5 architectural item.
             opts.evalEngine.registerRule(rule.evalType, createCustomRule(rule.definition));
             res.status(201).json({ rule });
         }
@@ -72,7 +84,8 @@ export function registerRuleRoutes(router, storage, opts) {
         }
     });
     router.delete('/rules/custom/:id', (req, res) => {
-        const removed = opts.customRuleStore.delete(req.params.id);
+        const tenantId = requireTenant(req);
+        const removed = opts.customRuleStore.delete(tenantId, req.params.id);
         if (!removed) {
             res.status(404).json({ error: 'Rule not found' });
             return;

package/dist/eval/citation-verify/resolve.d.ts CHANGED Viewed

@@ -22,5 +22,12 @@ export declare class CitationResolveError extends Error {
     constructor(message: string, kind: 'bad_scheme' | 'ssrf' | 'not_allowed_domain' | 'timeout' | 'too_large' | 'bad_status' | 'redirect_loop' | 'not_text' | 'fetch_disabled', details?: string | undefined);
 }
 export declare function isSafeHost(host: string): boolean;
+type DnsLookupAll = (host: string) => Promise<Array<{
+    address: string;
+    family: 4 | 6;
+}>>;
+export declare function __setDnsLookupForTests(impl: DnsLookupAll | null): void;
+export declare function resolveAndCheckHost(host: string): Promise<void>;
 export declare function __clearCitationCacheForTests(): void;
 export declare function resolveSource(identifier: string, opts: ResolveOptions): Promise<ResolvedSource>;
+export {};

package/dist/eval/citation-verify/resolve.js CHANGED Viewed

@@ -7,15 +7,25 @@
 //   1. Scheme allowlist — http/https only; refuse file:/javascript:/etc.
 //   2. SSRF host check — refuse localhost, link-local, private ranges,
 //      and cloud metadata (AWS/GCP/Azure/DigitalOcean) IP literals.
-//   3. Optional domain allowlist — IRIS_CITATION_DOMAINS=doi.org,arxiv.org
+//   3. DNS pre-resolve — every public hostname is resolved via
+//      dns.lookup({all:true}) and EVERY returned IP is re-checked against
+//      the IP blocklist. Defeats DNS-rebinding via public records pointing
+//      at private space (e.g. `*.localtest.me` resolving to 127.0.0.1).
+//      Residual TOCTOU window between this lookup and the socket connect
+//      is acknowledged — closing it requires a custom undici dispatcher;
+//      queued for follow-up if exploitation is observed.
+//   4. Optional domain allowlist — IRIS_CITATION_DOMAINS=doi.org,arxiv.org
 //      restricts to a curated set; empty/unset = open web (still SSRF-guarded).
-//   4. Timeout + size cap — 10s default, 5MB cap on response body.
-//   5. Redirect chase cap — follow max 3 redirects, each re-checked.
-//   6. Cache — in-process LRU (100 entries) so retries don't re-fetch.
+//   5. Timeout + size cap — 10s default, 5MB cap on response body.
+//   6. Redirect chase cap — follow max 3 redirects, each re-checked.
+//   7. Cache — in-process LRU (100 entries) so retries don't re-fetch.
 //
 // This is opt-in: calls require passing {allowFetch: true} so an agent
 // can't trick Iris into fetching random URLs without operator consent
 // (consent granted via tool param or env IRIS_CITATION_ALLOW_FETCH=1).
+import { lookup as dnsLookupCb } from 'node:dns';
+import { promisify } from 'node:util';
+const dnsLookupAll = promisify(dnsLookupCb);
 export class CitationResolveError extends Error {
     kind;
     details;
@@ -76,6 +86,33 @@ export function isSafeHost(host) {
     }
     return true;
 }
+let dnsLookupImpl = (host) => dnsLookupAll(host, { all: true });
+export function __setDnsLookupForTests(impl) {
+    dnsLookupImpl = impl ?? ((host) => dnsLookupAll(host, { all: true }));
+}
+export async function resolveAndCheckHost(host) {
+    if (!isSafeHost(host)) {
+        throw new CitationResolveError(`Refusing SSRF-blocked host: ${host}`, 'ssrf', host);
+    }
+    // IP literals already passed isSafeHost — skip DNS (would just re-resolve to self).
+    if (isIpv4(host) || isIpv6(host))
+        return;
+    let addresses;
+    try {
+        addresses = await dnsLookupImpl(host);
+    }
+    catch (err) {
+        throw new CitationResolveError(`DNS resolution failed for ${host}: ${err instanceof Error ? err.message : String(err)}`, 'ssrf', host);
+    }
+    if (addresses.length === 0) {
+        throw new CitationResolveError(`DNS returned no addresses for ${host}`, 'ssrf', host);
+    }
+    for (const { address } of addresses) {
+        if (!isSafeHost(address)) {
+            throw new CitationResolveError(`Refusing SSRF-blocked address ${address} (resolved from ${host})`, 'ssrf', host);
+        }
+    }
+}
 function matchesAllowlist(host, allowlist) {
     if (!allowlist || allowlist.length === 0)
         return true;
@@ -127,9 +164,7 @@ async function doFetch(url, opts, redirectsLeft) {
     if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
         throw new CitationResolveError(`Refusing non-http(s) scheme: ${parsed.protocol}`, 'bad_scheme', parsed.protocol);
     }
-    if (!isSafeHost(parsed.hostname)) {
-        throw new CitationResolveError(`Refusing SSRF-blocked host: ${parsed.hostname}`, 'ssrf', parsed.hostname);
-    }
+    await resolveAndCheckHost(parsed.hostname);
     if (!matchesAllowlist(parsed.hostname, opts.domainAllowlist)) {
         throw new CitationResolveError(`Host ${parsed.hostname} not in IRIS_CITATION_DOMAINS allowlist`, 'not_allowed_domain', parsed.hostname);
     }

package/dist/index.js CHANGED Viewed

@@ -129,12 +129,15 @@ async function main() {
     // Load deployed custom rules from ~/.iris/custom-rules.json (B3 — workflow inversion).
     // Each enabled rule is registered with the engine under its evalType so it fires on
     // every evaluate_output call of that category. Persistence via custom-rule-store.
-    const enabled = customRuleStore.enabledRules();
+    // OSS single-tenant: register rules under LOCAL_TENANT only. Cloud multi-tenant
+    // engine wiring is a v0.5 architectural item (the engine is a process singleton
+    // and would need per-tenant rule registration).
+    const enabled = customRuleStore.enabledRules(LOCAL_TENANT);
     for (const rule of enabled) {
         evalEngine.registerRule(rule.evalType, createCustomRule(rule.definition));
     }
     if (enabled.length > 0) {
-        logger.info(`Loaded ${enabled.length} deployed custom rule(s) from ${customRuleStore.filePath}`);
+        logger.info(`Loaded ${enabled.length} deployed custom rule(s) from ${customRuleStore.pathFor(LOCAL_TENANT)}`);
     }
     const httpServers = [];
     // Run data retention cleanup on startup.

package/dist/lib/claims.d.ts ADDED Viewed

@@ -0,0 +1,111 @@
+export declare const CLAIMS: {
+    $schema: string;
+    brand: {
+        categoryName: string;
+        coinedTerms: string[];
+        npmPackage: string;
+        publicRepoUrl: string;
+        securityEmail: string;
+        supportEmail: string;
+        tagline: string;
+        websiteUrl: string;
+    };
+    evalRules: {
+        builtInCount: number;
+        categories: string[];
+        categoryCount: number;
+        hallucinationMarkers: number;
+        injectionPatterns: number;
+        names: string[];
+        piiPatterns: number;
+        stubMarkers: string[];
+    };
+    generatedAt: string;
+    generatedFromCommit: string;
+    generatorVersion: string;
+    llmJudgeTemplates: {
+        count: number;
+        names: string[];
+        supportedModels: string[];
+        supportedProviders: string[];
+    };
+    mcpTools: {
+        annotations: {
+            destructiveHintCount: number;
+            openWorldHintCount: number;
+            readOnlyHintCount: number;
+        };
+        count: number;
+        names: string[];
+    };
+    release: {
+        currentReleaseDate: string;
+        currentReleaseVersion: string;
+        nextPlannedScope: string;
+        nextPlannedVersion: string;
+    };
+    schemaVersion: number;
+    tests: {
+        integration: {
+            failed: null;
+            passed: null;
+            total: null;
+        };
+        playwrightE2E: {
+            browsers: never[];
+            failed: null;
+            passed: null;
+            total: null;
+        };
+        totalCombined: number;
+        vitestDashboard: {
+            failed: number;
+            passed: number;
+            total: number;
+        };
+        vitestRoot: {
+            failed: number;
+            passed: number;
+            total: number;
+        };
+    };
+    version: {
+        dashboardPackage: string;
+        initPackage: string;
+        langchainPackage: string;
+        mcpServer: string;
+        websitePackage: string;
+    };
+};
+export declare const VERSION_MCP_SERVER: string;
+export declare const VERSION_LANGCHAIN_PACKAGE: string | null;
+export declare const VERSION_WEBSITE_PACKAGE: string | null;
+export declare const VERSION_DASHBOARD_PACKAGE: string | null;
+export declare const TEST_COUNT_VITEST_ROOT: number | null;
+export declare const TEST_COUNT_VITEST_DASHBOARD: number | null;
+export declare const TEST_COUNT_INTEGRATION: number | null;
+export declare const TEST_COUNT_PLAYWRIGHT_E2E: number | null;
+export declare const TEST_COUNT_TOTAL: number | null;
+export declare const MCP_TOOL_COUNT: number;
+export declare const MCP_TOOL_NAMES: readonly string[];
+export declare const RULE_COUNT_BUILT_IN: number;
+export declare const RULE_CATEGORIES: readonly string[];
+export declare const RULE_CATEGORY_COUNT: number;
+export declare const RULE_NAMES: readonly string[];
+export declare const PII_PATTERN_COUNT: number | null;
+export declare const INJECTION_PATTERN_COUNT: number | null;
+export declare const HALLUCINATION_MARKER_COUNT: number | null;
+export declare const LLM_JUDGE_TEMPLATE_COUNT: number;
+export declare const LLM_JUDGE_TEMPLATE_NAMES: readonly string[];
+export declare const TAGLINE: string;
+export declare const CATEGORY_NAME: string;
+export declare const COINED_TERMS: readonly string[];
+export declare const WEBSITE_URL: string;
+export declare const PUBLIC_REPO_URL: string;
+export declare const NPM_PACKAGE: string;
+export declare const SUPPORT_EMAIL: string;
+export declare const SECURITY_EMAIL: string;
+export declare const CURRENT_RELEASE_VERSION: string | null;
+export declare const CURRENT_RELEASE_DATE: string | null;
+export declare const NEXT_PLANNED_VERSION: string | null;
+export declare const NEXT_PLANNED_SCOPE: string | null;

package/dist/lib/claims.js ADDED Viewed

@@ -0,0 +1,52 @@
+// Truthbase reader — single import surface for facts about Iris.
+//
+// Source of truth: iris/.claims.json (regenerated from canonical artifacts
+// via `npm run claims:generate`). Surfaces import the named consts below
+// rather than hardcoding values inline.
+//
+// When a fact changes (a new test lands, a new MCP tool registers, the
+// version bumps), the regenerator updates .claims.json on the next commit
+// and every surface that imports from this reader updates simultaneously.
+// No drift.
+import claimsRaw from '../../.claims.json' with { type: 'json' };
+// Re-export the raw object for surfaces that need composite values.
+export const CLAIMS = claimsRaw;
+// Versions
+export const VERSION_MCP_SERVER = claimsRaw.version.mcpServer;
+export const VERSION_LANGCHAIN_PACKAGE = claimsRaw.version.langchainPackage;
+export const VERSION_WEBSITE_PACKAGE = claimsRaw.version.websitePackage;
+export const VERSION_DASHBOARD_PACKAGE = claimsRaw.version.dashboardPackage;
+// Tests
+export const TEST_COUNT_VITEST_ROOT = claimsRaw.tests.vitestRoot.total;
+export const TEST_COUNT_VITEST_DASHBOARD = claimsRaw.tests.vitestDashboard.total;
+export const TEST_COUNT_INTEGRATION = claimsRaw.tests.integration.total;
+export const TEST_COUNT_PLAYWRIGHT_E2E = claimsRaw.tests.playwrightE2E.total;
+export const TEST_COUNT_TOTAL = claimsRaw.tests.totalCombined;
+// MCP tools
+export const MCP_TOOL_COUNT = claimsRaw.mcpTools.count;
+export const MCP_TOOL_NAMES = claimsRaw.mcpTools.names;
+// Eval rules
+export const RULE_COUNT_BUILT_IN = claimsRaw.evalRules.builtInCount;
+export const RULE_CATEGORIES = claimsRaw.evalRules.categories;
+export const RULE_CATEGORY_COUNT = claimsRaw.evalRules.categoryCount;
+export const RULE_NAMES = claimsRaw.evalRules.names;
+export const PII_PATTERN_COUNT = claimsRaw.evalRules.piiPatterns;
+export const INJECTION_PATTERN_COUNT = claimsRaw.evalRules.injectionPatterns;
+export const HALLUCINATION_MARKER_COUNT = claimsRaw.evalRules.hallucinationMarkers;
+// LLM-judge templates
+export const LLM_JUDGE_TEMPLATE_COUNT = claimsRaw.llmJudgeTemplates.count;
+export const LLM_JUDGE_TEMPLATE_NAMES = claimsRaw.llmJudgeTemplates.names;
+// Brand
+export const TAGLINE = claimsRaw.brand.tagline;
+export const CATEGORY_NAME = claimsRaw.brand.categoryName;
+export const COINED_TERMS = claimsRaw.brand.coinedTerms;
+export const WEBSITE_URL = claimsRaw.brand.websiteUrl;
+export const PUBLIC_REPO_URL = claimsRaw.brand.publicRepoUrl;
+export const NPM_PACKAGE = claimsRaw.brand.npmPackage;
+export const SUPPORT_EMAIL = claimsRaw.brand.supportEmail;
+export const SECURITY_EMAIL = claimsRaw.brand.securityEmail;
+// Release
+export const CURRENT_RELEASE_VERSION = claimsRaw.release.currentReleaseVersion;
+export const CURRENT_RELEASE_DATE = claimsRaw.release.currentReleaseDate;
+export const NEXT_PLANNED_VERSION = claimsRaw.release.nextPlannedVersion;
+export const NEXT_PLANNED_SCOPE = claimsRaw.release.nextPlannedScope;

package/dist/middleware/auth.js CHANGED Viewed

@@ -5,6 +5,7 @@ export function createAuthMiddleware(config) {
         return (_req, _res, next) => next();
     }
     const keyBuffer = Buffer.from(apiKey);
+    const keyLen = keyBuffer.length;
     return (req, res, next) => {
         if (req.path === '/health' || req.path === '/api/v1/health') {
             return next();
@@ -14,8 +15,18 @@ export function createAuthMiddleware(config) {
             res.status(401).json({ error: 'Missing or invalid Authorization header' });
             return;
         }
+        // Pad the incoming token to the configured-key length and run
+        // timingSafeEqual on same-size buffers. The byte-compare and the
+        // length-equality check are computed independently before being
+        // combined, so the request takes the same compare path regardless
+        // of whether the token's length matches — eliminating the precise
+        // length-equality fast-path the original code had.
         const tokenBuffer = Buffer.from(authHeader.slice(7));
-        if (tokenBuffer.length !== keyBuffer.length || !timingSafeEqual(tokenBuffer, keyBuffer)) {
+        const candidate = Buffer.alloc(keyLen);
+        tokenBuffer.copy(candidate, 0, 0, keyLen);
+        const cmpEq = timingSafeEqual(candidate, keyBuffer);
+        const lenEq = tokenBuffer.length === keyLen;
+        if (!(cmpEq && lenEq)) {
             res.status(403).json({ error: 'Invalid API key' });
             return;
         }

package/dist/middleware/cors.js CHANGED Viewed

@@ -1,10 +1,16 @@
+// Wildcard `*` in an origin pattern matches a SINGLE label only — no dots,
+// colons, or slashes. Previously substituted `.*`, which let
+// `http://localhost:*` match `http://localhost:8080.evil.com` (the `.*`
+// happily consumed `8080.evil.com`). Single-label match prevents the
+// label-crossing bypass while still supporting `*.example.com` for
+// per-subdomain allowlists and `localhost:*` for ephemeral dev ports.
 function isOriginAllowed(origin, allowedOrigins) {
     for (const pattern of allowedOrigins) {
         if (pattern === '*')
             return true;
         if (pattern === origin)
             return true;
-        const regex = new RegExp('^' + pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$');
+        const regex = new RegExp('^' + pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '[^.:/]+') + '$');
         if (regex.test(origin))
             return true;
     }

package/dist/middleware/tenant.d.ts CHANGED Viewed

@@ -1,8 +1,10 @@
 import type { RequestHandler, Request } from 'express';
 import type { TenantId } from '../types/tenant.js';
-declare module 'express-serve-static-core' {
-    interface Request {
-        tenantId?: TenantId;
+declare global {
+    namespace Express {
+        interface Request {
+            tenantId?: TenantId;
+        }
     }
 }
 /** Read `req.tenantId` with a fail-safe guarantee for downstream code. */

package/dist/tools/delete-rule.js CHANGED Viewed

@@ -10,6 +10,7 @@
  * log row is the permanent record that the rule ever existed.
  */
 import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
 const inputSchema = {
     rule_id: z
         .string()
@@ -22,6 +23,8 @@ export function registerDeleteRuleTool(server, customRuleStore) {
         description: [
             'Remove a deployed custom evaluation rule. The rule stops firing on future evaluate_output calls; past eval_results that referenced it are preserved.',
             '',
+            'Sibling tools — deploy_rule adds custom rules, list_rules enumerates them, evaluate_output runs them. delete_trace handles trace deletion (separate concern); log_trace / get_traces handle trace I/O. delete_rule is the DESTRUCTIVE remove path for the custom-rule store; it does NOT touch traces, eval_results, or built-in (non-custom) rules.',
+            '',
             'Behavior. DESTRUCTIVE — rewrites ~/.iris/custom-rules.json without the deleted row and appends a `rule.delete` entry to the audit log (~/.iris/audit.log). Not idempotent: deleting an already-deleted rule returns `deleted: false` rather than re-emitting the audit row. The rule stops firing immediately on the live process. Historical eval_results that reference this rule_id stay in the database — drift analytics + audit trail remain valid. Tenant-scoped in Cloud tier; OSS operates on LOCAL_TENANT. Rate-limited to 20 req/min on HTTP MCP.',
             '',
             'Output shape. Returns JSON: `{ "deleted": boolean, "rule_id": string }`. `deleted=true` if a row was removed; `deleted=false` if no rule with that id existed.',
@@ -30,6 +33,8 @@ export function registerDeleteRuleTool(server, customRuleStore) {
             '',
             "Don't use to pause a rule (toggle in the dashboard preserves history better). Don't use on built-in (non-custom) rules — the rule_id format checks for `rule-<hex>` custom ids; built-ins aren't in the store. Don't use to delete a trace or eval result (use delete_trace for traces; eval_results deletion is not exposed in v0.4 — they fall under data retention).",
             '',
+            'Parameters. rule_id is the only parameter; must match `rule-<lowercase-hex>` format (Zod regex). Format mismatch fails Zod with 400 BEFORE the store is touched. Cross-tenant rule_ids return `deleted: false` silently — they\'re invisible to the caller\'s tenant rather than producing a not-found error (prevents enumeration attacks). The rule_id you pass is exactly what list_rules returned in `id` or what deploy_rule returned in `rule.id`.',
+            '',
             "Error modes. Throws 400 on malformed rule_id (wrong prefix). Returns `{deleted: false}` if rule_id doesn't match any deployed rule (not an error — idempotent-ish). Returns 429 on HTTP rate limit. File-write failures propagate as 500.",
         ].join('\n'),
         inputSchema,
@@ -40,7 +45,8 @@ export function registerDeleteRuleTool(server, customRuleStore) {
             openWorldHint: false,
         },
     }, async (args) => {
-        const deleted = customRuleStore.delete(args.rule_id, 'mcp');
+        // OSS: MCP tools operate under LOCAL_TENANT. See list-rules.ts for context.
+        const deleted = customRuleStore.delete(LOCAL_TENANT, args.rule_id, 'mcp');
         return {
             content: [
                 {

package/dist/tools/delete-trace.js CHANGED Viewed

@@ -23,6 +23,8 @@ export function registerDeleteTraceTool(server, storage) {
         description: [
             'Remove a single trace by id. Cascades to spans; eval_results keep the score history with trace_id NULLed.',
             '',
+            'Sibling tools — log_trace creates traces, get_traces queries them, evaluate_output / evaluate_with_llm_judge / verify_citations score them. delete_rule handles custom-rule deletion (separate concern); list_rules / deploy_rule manage the custom-rule lifecycle. delete_trace is the DESTRUCTIVE single-row remove for traces; it does NOT touch eval_results (preserved for audit + drift analytics), spans cascade automatically.',
+            '',
             'Behavior. DESTRUCTIVE — SQL DELETE scoped to the caller\'s tenant_id. Cascades: spans belonging to this trace are deleted (FK ON DELETE CASCADE); eval_results that referenced this trace have their trace_id set to NULL (FK ON DELETE SET NULL) so aggregate dashboards + historical scores remain valid even after the trace is gone. Not idempotent: deleting an already-deleted trace returns `deleted: false`. Does not emit an audit log entry in v0.4 — traces are user-scope data, not policy changes. Rate-limited to 20 req/min on HTTP MCP.',
             '',
             'Output shape. Returns JSON: `{ "deleted": boolean, "trace_id": string }`. `deleted=true` if a row was removed; `deleted=false` if no trace with that id existed (or it belonged to a different tenant — cross-tenant deletes silently fail).',
@@ -31,6 +33,8 @@ export function registerDeleteTraceTool(server, storage) {
             '',
             "Don't use to clean up OLD data in bulk (use retention config with --retention-days). Don't use to PAUSE a trace — traces are immutable once stored; there's nothing to pause. Don't use to delete eval_results — eval_results survive their trace's deletion intentionally (for audit + drift analysis); they're pruned only by retention.",
             '',
+            'Parameters. trace_id is the only parameter; must match 32-char lowercase hex (Zod regex). The trace_id you pass is exactly what log_trace returned in its response, or what get_traces returned per row. Format mismatch fails Zod with 400 BEFORE the storage layer is touched. Cross-tenant trace_ids return `deleted: false` silently — they\'re invisible to the caller\'s tenant (prevents enumeration attacks; matches delete_rule\'s tenant-isolation contract).',
+            '',
             "Error modes. Throws 400 on malformed trace_id (wrong format: not 32-char lowercase hex). Returns `{deleted: false}` when the id doesn't exist in the caller's tenant (not an error — the trace may simply have been deleted already). Returns 429 on HTTP rate limit. Storage failures propagate as 500.",
         ].join('\n'),
         inputSchema,

package/dist/tools/deploy-rule.js CHANGED Viewed

@@ -11,6 +11,7 @@
  * eval_type.
  */
 import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
 const CustomRuleDefinitionSchema = z.object({
     name: z.string(),
     type: z.enum([
@@ -52,6 +53,8 @@ export function registerDeployRuleTool(server, customRuleStore) {
         description: [
             'Deploy a new custom evaluation rule that will fire on every future evaluate_output call of its eval category.',
             '',
+            'Sibling tools — list_rules enumerates deployed rules, delete_rule removes them, evaluate_output runs them. log_trace / get_traces / delete_trace handle the trace lifecycle separately; evaluate_with_llm_judge / verify_citations run semantic scoring (not heuristic-rule-driven). deploy_rule is the WRITE path that grows the custom-rule library.',
+            '',
             'Behavior. Writes a row to ~/.iris/custom-rules.json (atomic write via temp file + rename) and appends a `rule.deploy` entry to the audit log (~/.iris/audit.log). The rule activates immediately for the running process and persists across restarts. Each call mints a fresh rule_id; not idempotent (deploying twice creates two rules). Tenant-scoped in Cloud tier; OSS rules are owned by LOCAL_TENANT. Rate-limited to 20 req/min on HTTP MCP.',
             '',
             'Output shape. Returns JSON: `{ "rule": { "id": "rule-XXXX", "name", "description", "evalType", "severity", "definition", "enabled": true, "createdAt", "updatedAt", "version": 1, "sourceMomentId?" } }`. The returned rule is the canonical persisted form; save the `id` if you plan to update or delete later.',
@@ -60,6 +63,8 @@ export function registerDeployRuleTool(server, customRuleStore) {
             '',
             "Don't use to VALIDATE a rule before committing — deploy writes immediately. Use the dashboard's preview endpoint (POST /api/v1/rules/custom/preview) for dry-run validation against sample output. Don't use to EDIT an existing rule — this call only creates; edits require a dedicated flow (coming in v0.5). To update a rule today: delete_rule then deploy_rule with the new definition.",
             '',
+            'Parameters. name is 1-120 chars (Zod-enforced min/max); appears in eval_result rule_results so make it human-readable. description is optional, max 500 chars (used in dashboard tooltips). evalType determines WHEN the rule fires (must match the eval_type your evaluate_output calls use; e.g., a "completeness" rule fires on every evaluate_output where eval_type="completeness" OR eval_type="custom"). severity affects dashboard sort + audit log signal but does NOT affect scoring (scoring uses the rule\'s weight). definition.type and definition.config must match (e.g., regex_match needs config.pattern; cost_threshold needs config.max_usd; min_length needs config.min). sourceMomentId is optional but recommended (preserves workflow-inversion provenance from Make-This-A-Rule composer). Defaults: severity="medium".',
+            '',
             "Error modes. Throws 400 on invalid definition (Zod rejects — e.g., regex that fails safe-regex2 ReDoS check, or length > 1000 chars). Throws 400 on empty `name`. Throws 400 if the eval category mismatches the definition type. Returns 429 when HTTP rate limit exceeded. File-write failures (disk full, read-only fs) propagate as 500; the audit log is best-effort and does not block deploy.",
         ].join('\n'),
         inputSchema,
@@ -70,7 +75,8 @@ export function registerDeployRuleTool(server, customRuleStore) {
             openWorldHint: false,
         },
     }, async (args) => {
-        const rule = customRuleStore.deploy({
+        // OSS: MCP tools operate under LOCAL_TENANT. See list-rules.ts for context.
+        const rule = customRuleStore.deploy(LOCAL_TENANT, {
             name: args.name,
             description: args.description,
             evalType: args.evalType,

package/dist/tools/evaluate-output.js CHANGED Viewed

@@ -10,18 +10,18 @@ const CustomRuleSchema = z.object({
     weight: z.number().optional(),
 });
 const inputSchema = {
-    output: z.string().describe('The output text to evaluate'),
-    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']).default('completeness').describe('Type of evaluation'),
-    expected: z.string().optional().describe('Expected output for comparison'),
-    input: z.string().optional().describe('Original input for context'),
-    trace_id: z.string().optional().describe('Link evaluation to a trace'),
-    custom_rules: z.array(CustomRuleSchema).optional().describe('Custom evaluation rules'),
-    cost_usd: z.number().optional().describe('Cost for cost evaluation'),
+    output: z.string().describe('The output text to evaluate (the agent\'s response that gets scored against rules)'),
+    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']).default('completeness').describe('Rule bundle to apply: completeness | relevance | safety | cost | custom — picks which built-in rules fire'),
+    expected: z.string().optional().describe('Expected output for comparison — REQUIRED when eval_type="relevance" (used as keyword-overlap target)'),
+    input: z.string().optional().describe('Original input for context — improves relevance scoring (keyword overlap vs input)'),
+    trace_id: z.string().optional().describe('Link evaluation to a trace — surfaces this eval in the dashboard\'s trace drill-through'),
+    custom_rules: z.array(CustomRuleSchema).optional().describe('Custom evaluation rules — fires REGARDLESS of eval_type; pass eval_type="custom" if you want ONLY these'),
+    cost_usd: z.number().optional().describe('Cost in USD — only consulted when eval_type="cost" (compared against cost_threshold rules)'),
     token_usage: z.object({
         prompt_tokens: z.number().optional(),
         completion_tokens: z.number().optional(),
         total_tokens: z.number().optional(),
-    }).optional().describe('Token usage for cost evaluation'),
+    }).optional().describe('Token usage breakdown — only consulted when eval_type="cost" (used for token-budget rules)'),
 };
 export function registerEvaluateOutputTool(server, storage, evalEngine) {
     server.registerTool('evaluate_output', {
@@ -29,6 +29,8 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
         description: [
             'Score agent output against configurable eval rules and return a 0..1 score + per-rule breakdown.',
             '',
+            'Sibling tools — evaluate_with_llm_judge runs semantic LLM-based scoring (slower, costs money; this tool is heuristic, free, deterministic), verify_citations checks citation grounding specifically, log_trace records executions, get_traces queries them, list_rules / deploy_rule / delete_rule manage the custom-rule lifecycle. evaluate_output is the FAST PATH for length / keyword / PII / injection / cost-threshold checks where rules are sufficient.',
+            '',
             'Behavior. Deterministic, in-process scoring — same inputs always produce the same result. Writes one eval_result row to Iris storage (linked to trace_id if provided; unlinked otherwise). No external network calls in heuristic mode (v0.4 adds an llm_as_judge eval_type that DOES call LLM APIs; see the separate evaluate_with_llm_judge tool for that). Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Runs in ~5-50ms for rule-based evaluation.',
             '',
             'Output shape. Returns JSON: `{ "id": "<uuid>", "score": 0..1, "passed": boolean, "rule_results": [{ "ruleName", "passed", "score", "message", "skipped?" }], "suggestions": string[], "rules_evaluated": number, "rules_skipped": number, "insufficient_data": boolean }`. `insufficient_data=true` means no applicable rules fired (e.g., safety eval with only cost data).',
@@ -37,6 +39,8 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
             '',
             'Don\'t use when the output is empty or has no applicable rules — the eval_type decides which rules apply, and invalid combinations return score=0 + insufficient_data=true (not an error, but not actionable). Don\'t use to VALIDATE JSON schemas directly (use your language\'s JSON Schema validator — Iris\'s `json_schema` custom rule type is for output-shape assertions, not arbitrary validation).',
             '',
+            'Parameters. expected is REQUIRED when eval_type="relevance" (used as the comparison target for keyword overlap + topic consistency); ignored for other eval_types. cost_usd + token_usage are ONLY consulted when eval_type="cost" (ignored otherwise). custom_rules ALWAYS fires regardless of eval_type — pass eval_type="custom" if you want ONLY your rules to run (otherwise both your rules AND the eval_type bundle run together). trace_id is optional but recommended (linking the eval to its trace surfaces it in the dashboard\'s drill-through). input adds context to keyword-overlap relevance checks; ignored otherwise. Defaults: eval_type="completeness".',
+            '',
             'Error modes. Throws on malformed custom_rules (Zod rejects). Returns 400 on regex patterns that fail safe-regex2 ReDoS check or exceed 1000-char limit. Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. The eval itself never throws — failing rules report `passed: false` with a message, they don\'t bubble exceptions.',
         ].join('\n'),
         inputSchema,

package/dist/tools/evaluate-with-llm-judge.js CHANGED Viewed

@@ -59,6 +59,8 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
         description: [
             'Score agent output using an LLM as the judge (Anthropic or OpenAI). Returns a calibrated 0..1 score with rationale, per-dimension breakdown, and exact cost.',
             '',
+            'Sibling tools — evaluate_output runs heuristic rules (free, deterministic, ~ms latency, no API key needed); this tool runs LLM-based semantic scoring (paid, 1-10s latency, requires API key). verify_citations is a SPECIALIZED form of LLM judging that focuses on citation grounding only. log_trace / get_traces handle trace I/O; list_rules / deploy_rule / delete_rule manage heuristic-rule lifecycle. evaluate_with_llm_judge is the GENERAL semantic-scoring path.',
+            '',
             'Behavior. Calls an external LLM API (Anthropic or OpenAI) — costs money per call, takes 1-10 seconds, respects an IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL cap. Non-deterministic at temperature > 0; default temperature=0 gives near-deterministic scores. Writes one eval_result row to Iris storage (linked to trace_id if provided) plus captures provider response id + latency + token counts + cost in the rule_results payload. Rate-limited to 20 req/min on HTTP MCP; your LLM provider also enforces its own rate limits (we transparently retry once on 429).',
             '',
             'Output shape. Returns JSON: `{ "id": "<uuid>", "score": 0..1, "passed": boolean, "rationale": string, "dimensions": {...}, "model": string, "provider": "anthropic"|"openai", "template": string, "input_tokens": number, "output_tokens": number, "cost_usd": number, "latency_ms": number }`. `dimensions` has per-dimension sub-scores (e.g., accuracy template returns `{factual_claims, citations, internal_consistency}`).',
@@ -67,6 +69,8 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
             '',
             "Don't use for simple regex/length/keyword checks (use evaluate_output with heuristic rules — they're free, deterministic, 1000x faster). Don't use without an API key set (IRIS_ANTHROPIC_API_KEY or IRIS_OPENAI_API_KEY). Don't use on very large outputs (>8K tokens) without raising max_cost_usd — the pre-check will refuse the call.",
             '',
+            'Parameters. model is required (no default — pick consciously since cost varies 100x across models). provider is auto-detected from the model name; override only for ambiguous IDs. expected is REQUIRED when template="correctness" (the reference answer to compare against); ignored for other templates. source_material is REQUIRED when template="faithfulness" (the RAG sources to ground against); ignored otherwise. input is optional but improves scoring on helpfulness/safety templates (gives the judge the user prompt that produced the output). max_cost_usd defaults to env var IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or $0.25 — the worst-case cost is computed BEFORE the call (input_tokens × prompt_price + max_output_tokens × completion_price); call refused upfront if it would exceed. max_output_tokens caps the judge response (default 512, max 4096); higher = more rationale detail + more cost. temperature default 0 (deterministic). timeout_ms default 60000. trace_id optional but recommended (links eval to trace in dashboard). Defaults: temperature=0, max_output_tokens=512, max_cost_usd=$0.25, timeout_ms=60000.',
+            '',
             'Error modes. Throws when the required API key env var is missing. Throws when the estimated worst-case cost exceeds max_cost_usd (raise the cap or trim prompts). Throws LLMJudgeError on provider errors — kind=`auth` on 401/403, `rate_limit` on 429 (auto-retried once), `server_error` on 5xx, `timeout` on abort, `malformed_response` when the judge fails to emit valid JSON on both attempts. Throws "Unknown model" for unsupported model IDs — update src/eval/llm-judge/pricing.ts first.',
         ].join('\n'),
         inputSchema,

package/dist/tools/get-traces.js CHANGED Viewed

@@ -1,17 +1,17 @@
 import { z } from 'zod';
 import { LOCAL_TENANT } from '../types/tenant.js';
 const inputSchema = {
-    agent_name: z.string().optional().describe('Filter by agent name'),
-    framework: z.string().optional().describe('Filter by framework'),
-    since: z.string().optional().describe('ISO timestamp lower bound'),
-    until: z.string().optional().describe('ISO timestamp upper bound'),
-    min_score: z.number().optional().describe('Minimum eval score filter'),
-    max_score: z.number().optional().describe('Maximum eval score filter'),
-    limit: z.number().default(50).describe('Results per page'),
-    offset: z.number().default(0).describe('Pagination offset'),
-    sort_by: z.enum(['timestamp', 'latency_ms', 'cost_usd']).default('timestamp').describe('Sort field'),
-    sort_order: z.enum(['asc', 'desc']).default('desc').describe('Sort order'),
-    include_summary: z.boolean().default(false).describe('Include dashboard summary stats'),
+    agent_name: z.string().optional().describe('Filter by agent name — exact match (no wildcards in v0.4)'),
+    framework: z.string().optional().describe('Filter by agent framework — exact match (e.g., langchain, autogen)'),
+    since: z.string().optional().describe('ISO timestamp lower bound — return traces with timestamp >= this'),
+    until: z.string().optional().describe('ISO timestamp upper bound — return traces with timestamp < this'),
+    min_score: z.number().optional().describe('Minimum eval score filter (0..1) — applied to LATEST eval per trace, not all evals'),
+    max_score: z.number().optional().describe('Maximum eval score filter (0..1) — applied to LATEST eval per trace'),
+    limit: z.number().default(50).describe('Results per page (default 50, max 1000 — values >1000 return 400)'),
+    offset: z.number().default(0).describe('Zero-based pagination offset — skip first N results'),
+    sort_by: z.enum(['timestamp', 'latency_ms', 'cost_usd']).default('timestamp').describe('Sort by timestamp | latency_ms | cost_usd (default timestamp)'),
+    sort_order: z.enum(['asc', 'desc']).default('desc').describe('Sort order: asc | desc (default desc — most recent / highest first)'),
+    include_summary: z.boolean().default(false).describe('Include dashboard summary stats in same response — saves a round-trip when ingesting for dashboards'),
 };
 export function registerGetTracesTool(server, storage) {
     server.registerTool('get_traces', {
@@ -19,6 +19,8 @@ export function registerGetTracesTool(server, storage) {
         description: [
             'Query stored agent-execution traces with filters, pagination, and optional dashboard summary.',
             '',
+            'Sibling tools — log_trace creates traces, delete_trace removes a single trace, evaluate_output / evaluate_with_llm_judge / verify_citations score them, list_rules / deploy_rule / delete_rule manage the custom-rule lifecycle. get_traces is the READ path for historical agent executions — never mutates anything.',
+            '',
             'Behavior. Read-only: never mutates storage, never calls external services. Idempotent: repeated calls with the same args return consistent results (new traces logged after the call obviously show up on subsequent calls). Tenant-scoped: queries only the caller\'s tenant rows (LOCAL_TENANT in OSS). Paginates results (default limit 50, max 1000). Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio.',
             '',
             'Output shape. Returns JSON: `{ "traces": [{...traceRow}], "total": number, "limit": number, "offset": number, "summary"?: { total_traces, avg_latency_ms, total_cost_usd, error_rate, eval_pass_rate, traces_per_hour, top_agents } }`. Each trace row includes trace_id, agent_name, framework, input, output, tool_calls, latency_ms, token_usage, cost_usd, metadata, timestamp. `summary` only included when `include_summary: true`.',
@@ -27,6 +29,8 @@ export function registerGetTracesTool(server, storage) {
             '',
             'Don\'t use to score a trace (use evaluate_output). Don\'t use to create a trace (use log_trace). Don\'t use as a live event stream — it\'s a query, not a subscription; poll with exponential backoff or use the dashboard\'s SSE endpoint for real-time.',
             '',
+            'Parameters. limit defaults to 50, max 1000 (anything higher returns 400). offset is zero-based pagination. min_score / max_score filter on the LATEST eval per trace, not all evals (so a trace with one failing + one passing eval may or may not match depending on which landed last). Combining since + sort_by="latency_ms" + sort_order="desc" is the canonical "find slow recent traces" query. include_summary returns dashboard-style aggregates in the SAME response (saves a round-trip; use true for dashboard ingest, false for analytics queries that don\'t need them). agent_name and framework are exact-match (no wildcards in v0.4). Defaults: limit=50, offset=0, sort_by="timestamp", sort_order="desc", include_summary=false.',
+            '',
             'Error modes. Returns 400 on invalid sort_by / sort_order (Zod enum). Returns 400 if limit > 1000. Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. Empty result with `total: 0` on no matches (not an error).',
         ].join('\n'),
         inputSchema,