npm - @iris-eval/mcp-server - Versions diffs - 0.6.0 → 0.8.0 - Mend

@iris-eval/mcp-server 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (38) hide show

package/README.md +7 -3
package/dist/config/defaults.js +17 -1
package/dist/config/index.js +8 -0
package/dist/dashboard/assets/{index-CshLgDRB.js → index-DTA8DzF_.js} +1 -1
package/dist/dashboard/index.html +1 -1
package/dist/dashboard/routes/rules.d.ts +8 -11
package/dist/dashboard/routes/rules.js +9 -16
package/dist/dashboard/routes/traces.js +14 -2
package/dist/dashboard/validation.d.ts +4 -4
package/dist/dashboard/validation.js +6 -2
package/dist/eval/criticality.d.ts +67 -0
package/dist/eval/criticality.js +154 -0
package/dist/eval/engine.d.ts +33 -2
package/dist/eval/engine.js +64 -8
package/dist/eval/rules/cost.d.ts +9 -0
package/dist/eval/rules/cost.js +97 -1
package/dist/eval/rules/relevance.d.ts +13 -0
package/dist/eval/rules/relevance.js +185 -21
package/dist/eval/rules/safety.d.ts +13 -1
package/dist/eval/rules/safety.js +273 -21
package/dist/eval/rules/trajectory.d.ts +91 -0
package/dist/eval/rules/trajectory.js +297 -0
package/dist/index.js +1 -1
package/dist/self-test.js +1 -1
package/dist/server.js +1 -1
package/dist/tools/evaluate-output.js +38 -23
package/dist/tools/index.js +1 -1
package/dist/tools/list-rules.d.ts +2 -1
package/dist/tools/list-rules.js +22 -4
package/dist/tools/log-trace.d.ts +8 -1
package/dist/tools/log-trace.js +19 -4
package/dist/tools/strict-input.js +2 -2
package/dist/tools/trace-link.d.ts +10 -0
package/dist/tools/trace-link.js +13 -1
package/dist/types/config.d.ts +14 -0
package/dist/types/eval.d.ts +39 -7
package/package.json +8 -1
package/server.json +2 -2

package/dist/dashboard/index.html CHANGED Viewed

@@ -6,7 +6,7 @@
     <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <!-- Stop shipping agents on vibes is filled from .claims.json brand.tagline at build time (vite.config.ts) — never restate the tagline here. -->
     <title>Iris — Stop shipping agents on vibes</title>
-    <script type="module" crossorigin src="/assets/index-CshLgDRB.js"></script>
+    <script type="module" crossorigin src="/assets/index-DTA8DzF_.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-D0cFfBqn.css">
   </head>
   <body>

package/dist/dashboard/routes/rules.d.ts CHANGED Viewed

@@ -2,20 +2,18 @@ import { Router } from 'express';
 import type { IStorageAdapter } from '../../types/query.js';
 import type { CustomRuleStore } from '../../custom-rule-store.js';
 import type { EvalEngine } from '../../eval/engine.js';
-import type { EvalType } from '../../types/eval.js';
+import { type BuiltInRuleMeta } from '../../eval/criticality.js';
 interface RoutesOptions {
     customRuleStore: CustomRuleStore;
     evalEngine: EvalEngine;
 }
-/** Built-in rule metadata as the dashboard sees it — derived from the engine, never restated. */
-export interface BuiltInRuleMeta {
-    name: string;
-    category: EvalType;
-    description: string;
-    weight: number;
-    critical: boolean;
-}
-export declare function listBuiltInRules(): BuiltInRuleMeta[];
+export type { BuiltInRuleMeta };
+/**
+ * The roster as the dashboard asks for it. Passing the engine is what makes
+ * `critical` the value this server will actually apply; omitting it returns
+ * the rules' own declarations, reported as source 'default'.
+ */
+export declare function listBuiltInRules(engine?: EvalEngine): BuiltInRuleMeta[];
 export declare function registerRuleRoutes(router: Router, storage: IStorageAdapter, opts: RoutesOptions): void;
 /** Verdict of the proposed rule against the caller's own sample text. */
 export interface SamplePreview {
@@ -25,4 +23,3 @@ export interface SamplePreview {
     skipped: boolean;
     skipReason?: string;
 }
-export {};

package/dist/dashboard/routes/rules.js CHANGED Viewed

@@ -1,6 +1,6 @@
 import { z } from 'zod';
 import { createCustomRule } from '../../eval/rules/custom.js';
-import { rulesByType } from '../../eval/rules/index.js';
+import { builtInRuleRoster } from '../../eval/criticality.js';
 import { requireTenant } from '../../middleware/tenant.js';
 import { DuplicateRuleNameError, replacedRulesWarning, retireSameNamedRules } from '../../tools/deploy-rule.js';
 import { strictBody } from '../validation.js';
@@ -77,20 +77,13 @@ const PreviewSchema = strictBody({
 const ToggleSchema = strictBody({
     enabled: z.boolean(),
 });
-export function listBuiltInRules() {
-    const out = [];
-    for (const [category, rules] of Object.entries(rulesByType)) {
-        for (const rule of rules) {
-            out.push({
-                name: rule.name,
-                category,
-                description: rule.description,
-                weight: rule.weight,
-                critical: rule.critical === true,
-            });
-        }
-    }
-    return out;
+/**
+ * The roster as the dashboard asks for it. Passing the engine is what makes
+ * `critical` the value this server will actually apply; omitting it returns
+ * the rules' own declarations, reported as source 'default'.
+ */
+export function listBuiltInRules(engine) {
+    return builtInRuleRoster(engine ? (rule) => engine.effectiveCriticality(rule) : undefined);
 }
 export function registerRuleRoutes(router, storage, opts) {
     /*
@@ -112,7 +105,7 @@ export function registerRuleRoutes(router, storage, opts) {
      * are process-global.
      */
     router.get('/rules/builtin', (_req, res) => {
-        res.json({ rules: listBuiltInRules() });
+        res.json({ rules: listBuiltInRules(opts.evalEngine) });
     });
     router.get('/rules/custom', (req, res) => {
         const tenantId = requireTenant(req);

package/dist/dashboard/routes/traces.js CHANGED Viewed

@@ -2,6 +2,7 @@ import { requireTenant } from '../../middleware/tenant.js';
 import { generateTraceId, generateSpanId } from '../../utils/ids.js';
 import { bestEffortExport } from '../../otel/lazy.js';
 import { traceQuerySchema, ingestTraceSchema } from '../validation.js';
+import { DEFAULT_EVAL_TYPE, DEFAULT_EVAL_TYPE_NOTE } from '../../eval/engine.js';
 export function registerTraceRoutes(router, storage, options) {
     /*
      * Deterministic capture over HTTP. MCP tool calls are model-
@@ -67,10 +68,20 @@ export function registerTraceRoutes(router, storage, options) {
                 input: body.input,
                 costUsd: body.cost_usd,
                 tokenUsage: body.token_usage,
+                // The trajectory the SAME request just stored. This body already
+                // carries what the agent did; not forwarding it made every
+                // trajectory rule skip on the one path where the data was
+                // guaranteed present — an ingest that captured a failed tool call
+                // and then evaluated as though it had never been told.
+                toolCalls: body.tool_calls,
             };
-            const evaluation = body.eval_type === 'all'
+            // An omitted eval_type runs every bundle — the same default, from the
+            // same constant, as the MCP tool — and says so in the response.
+            const evalTypeOmitted = body.eval_type === undefined;
+            const evalType = body.eval_type ?? DEFAULT_EVAL_TYPE;
+            const evaluation = evalType === 'all'
                 ? options.evalEngine.evaluateAll(context)
-                : options.evalEngine.evaluate(body.eval_type, context);
+                : options.evalEngine.evaluate(evalType, context);
             evaluation.trace_id = traceId;
             await storage.insertEvalResult(tenantId, evaluation);
             res.status(201).json({
@@ -103,6 +114,7 @@ export function registerTraceRoutes(router, storage, options) {
                         : {}),
                     // Per-bundle breakdown — eval_type="all" only.
                     ...(evaluation.categories ? { categories: evaluation.categories } : {}),
+                    ...(evalTypeOmitted ? { note: DEFAULT_EVAL_TYPE_NOTE } : {}),
                 },
             });
         }

package/dist/dashboard/validation.d.ts CHANGED Viewed

@@ -4,7 +4,7 @@ export declare function strictBody<T extends z.ZodRawShape>(shape: T, opts?: {
 }): z.ZodObject<{ -readonly [P in keyof T]: T[P]; }, z.core.$strict>;
 export declare const ingestTraceSchema: z.ZodObject<{
     evaluate: z.ZodDefault<z.ZodBoolean>;
-    eval_type: z.ZodDefault<z.ZodEnum<{
+    eval_type: z.ZodOptional<z.ZodEnum<{
         completeness: "completeness";
         relevance: "relevance";
         safety: "safety";
@@ -22,7 +22,7 @@ export declare const ingestTraceSchema: z.ZodObject<{
         output: z.ZodOptional<z.ZodUnknown>;
         latency_ms: z.ZodOptional<z.ZodNumber>;
         error: z.ZodOptional<z.ZodString>;
-    }, z.core.$strip>>>;
+    }, z.core.$strict>>>;
     latency_ms: z.ZodOptional<z.ZodNumber>;
     token_usage: z.ZodOptional<z.ZodObject<{
         prompt_tokens: z.ZodOptional<z.ZodNumber>;
@@ -83,9 +83,9 @@ export declare const traceQuerySchema: z.ZodObject<{
 export declare const evalQuerySchema: z.ZodObject<{
     eval_type: z.ZodOptional<z.ZodString>;
     passed: z.ZodOptional<z.ZodPipe<z.ZodEnum<{
-        true: "true";
         false: "false";
-    }>, z.ZodTransform<boolean, "true" | "false">>>;
+        true: "true";
+    }>, z.ZodTransform<boolean, "false" | "true">>>;
     since: z.ZodOptional<z.ZodString>;
     until: z.ZodOptional<z.ZodString>;
     limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;

package/dist/dashboard/validation.js CHANGED Viewed

@@ -52,8 +52,12 @@ export const ingestTraceSchema = strictBody({
     // Same bundle list evaluate_output accepts, "all" included — the
     // ingest path used to stop one short and run the single-bundle engine
     // no matter what, so an HTTP caller could not get the per-category
-    // verdict the MCP tool returns.
-    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom', 'all']).default('completeness'),
+    // verdict the MCP tool returns. Optional rather than defaulted HERE so
+    // the route can tell "chose all" from "never chose": the effective
+    // default is every bundle (DEFAULT_EVAL_TYPE in eval/engine.ts, the
+    // same constant evaluate_output reads), and an omitted eval_type gets a
+    // note in the response saying the default ran.
+    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom', 'all']).optional(),
 }, {
     reserved: {
         trace_id: 'trace_id is minted by the server on every ingest and cannot be supplied by the client — ' +

package/dist/eval/criticality.d.ts ADDED Viewed

@@ -0,0 +1,67 @@
+import type { EvalRule, EvalType } from '../types/eval.js';
+/** Where a rule's EFFECTIVE criticality came from. */
+export type CriticalitySource = 'default' | 'config';
+export interface CriticalityOverrides {
+    /** Built-in rule names promoted to critical. */
+    criticalRules?: string[];
+    /** Built-in rule names demoted from critical. */
+    nonCriticalRules?: string[];
+}
+export interface EffectiveCriticality {
+    critical: boolean;
+    /** 'config' when one of the two lists decided this; 'default' otherwise. */
+    source: CriticalitySource;
+}
+/** Every built-in rule, in bundle order. The registry the lists are checked against. */
+export declare function builtInRules(): EvalRule[];
+/** Built-in rule names, sorted — the "valid list" an error message prints. */
+export declare function builtInRuleNames(): string[];
+/**
+ * Every problem with a pair of override lists, as human sentences.
+ * Empty means valid. Separate from the thrower so a caller that wants to
+ * report several at once (a config linter, a test) can.
+ */
+export declare function criticalityIssues(overrides: CriticalityOverrides | undefined): string[];
+/**
+ * Throw on any problem, with every problem named at once.
+ *
+ * Called from loadConfig (so a bad config.json fails at startup, before a
+ * single evaluation runs) and from the EvalEngine constructor (so no code
+ * path can build an engine that silently ignores an override).
+ */
+export declare function assertValidCriticality(overrides: CriticalityOverrides | undefined): void;
+/**
+ * The effective criticality of one rule.
+ *
+ * A rule the overrides do not name keeps its definition's `critical` and
+ * reports source 'default' — which for a deployed custom rule means the
+ * severity it was deployed with.
+ */
+export declare function resolveCriticality(rule: EvalRule, overrides: CriticalityOverrides | undefined, builtIns: ReadonlySet<EvalRule>): EffectiveCriticality;
+/** A resolver bound to one set of overrides, so callers don't rebuild the identity set per rule. */
+export declare function criticalityResolver(overrides: CriticalityOverrides | undefined): (rule: EvalRule) => EffectiveCriticality;
+/** Built-in rule metadata as every roster surface reports it — derived from the registry, never restated. */
+export interface BuiltInRuleMeta {
+    name: string;
+    category: EvalType;
+    description: string;
+    weight: number;
+    /** EFFECTIVE criticality, after eval.criticalRules / eval.nonCriticalRules. */
+    critical: boolean;
+    /** Who decided it: the rule's own declaration, or one of the config lists. */
+    criticalSource: CriticalitySource;
+}
+/**
+ * The whole built-in roster, one entry per rule.
+ *
+ * `resolve` is what makes `critical` EFFECTIVE rather than declared. Reading
+ * `rule.critical` on a roster while the engine applies an override would
+ * show an operator a list that disagrees with the verdicts the same process
+ * is producing — they would see `no_silent_tool_failure` reported
+ * non-critical while it vetoed their pipeline. Callers that only want names
+ * and weights may omit it and get the declarations, reported as 'default'.
+ *
+ * Typed on a function, not on EvalEngine, so this module stays free of the
+ * engine it is imported by.
+ */
+export declare function builtInRuleRoster(resolve?: (rule: EvalRule) => EffectiveCriticality): BuiltInRuleMeta[];

package/dist/eval/criticality.js ADDED Viewed

@@ -0,0 +1,154 @@
+/*
+ * Which built-in rules VETO — a deployment's decision, not ours.
+ *
+ * `critical` is a property of each rule's definition, and until now that was
+ * the whole story: a failing critical rule forced `passed: false`, a failing
+ * non-critical one only moved the score, and nobody running Iris could
+ * change either. That default is a judgement about acceptable error, and it
+ * is not ours to make for everyone. The trajectory release is the plain
+ * case: `no_silent_tool_failure` catches an agent answering over a tool that
+ * errored, which is exactly what a team gating deploys wants to block — but
+ * its precision on a 30-case family carries a 95% lower bound of 77.2%, so
+ * shipping it as a veto for everyone would force false failures on people
+ * who never asked for that trade. A team that HAS looked at the number can
+ * make the call for their own pipeline; the config keys below are how.
+ *
+ * Two lists, both optional, both naming BUILT-IN rules:
+ *   eval.criticalRules     — promote: these rules veto `passed`.
+ *   eval.nonCriticalRules  — demote: these rules stop vetoing.
+ *
+ * Every name is checked against the rule registry when the config loads and
+ * again when an engine is constructed. An unknown name is a startup error
+ * naming the valid list, never a silent no-op: a typo in `criticalRules`
+ * that quietly did nothing would leave an operator believing a gate exists
+ * when it does not, which is the same "detection that reports an all-clear"
+ * failure the critical veto was built to stop.
+ *
+ * Overrides are matched by rule IDENTITY, not by name. Deployed custom rules
+ * do not enforce unique names — one can legitimately be called `no_pii` —
+ * and a name-keyed override would silently reach it. A custom rule's
+ * severity stays its own definition's business.
+ */
+import { rulesByType } from './rules/index.js';
+/** Every built-in rule, in bundle order. The registry the lists are checked against. */
+export function builtInRules() {
+    return ['completeness', 'relevance', 'safety', 'cost'].flatMap((t) => rulesByType[t]);
+}
+/** Built-in rule names, sorted — the "valid list" an error message prints. */
+export function builtInRuleNames() {
+    return builtInRules()
+        .map((r) => r.name)
+        .sort();
+}
+/**
+ * Every problem with a pair of override lists, as human sentences.
+ * Empty means valid. Separate from the thrower so a caller that wants to
+ * report several at once (a config linter, a test) can.
+ */
+export function criticalityIssues(overrides) {
+    if (!overrides)
+        return [];
+    const issues = [];
+    const valid = new Set(builtInRuleNames());
+    const seen = {
+        criticalRules: new Set(),
+        nonCriticalRules: new Set(),
+    };
+    for (const key of ['criticalRules', 'nonCriticalRules']) {
+        const list = overrides[key];
+        if (list === undefined)
+            continue;
+        if (!Array.isArray(list)) {
+            issues.push(`eval.${key} must be an array of built-in rule names.`);
+            continue;
+        }
+        for (const entry of list) {
+            if (typeof entry !== 'string' || entry.trim().length === 0) {
+                issues.push(`eval.${key} contains ${JSON.stringify(entry)}, which is not a rule name.`);
+                continue;
+            }
+            if (!valid.has(entry)) {
+                issues.push(`eval.${key} names "${entry}", which is not a built-in rule. ` +
+                    `Valid names: ${builtInRuleNames().join(', ')}. ` +
+                    'Deployed custom rules carry their own severity and are not set here.');
+                continue;
+            }
+            seen[key].add(entry);
+        }
+    }
+    for (const name of seen.criticalRules) {
+        if (seen.nonCriticalRules.has(name)) {
+            issues.push(`"${name}" is in BOTH eval.criticalRules and eval.nonCriticalRules, so the config does not say whether it should veto. Remove it from one of them.`);
+        }
+    }
+    return issues;
+}
+/**
+ * Throw on any problem, with every problem named at once.
+ *
+ * Called from loadConfig (so a bad config.json fails at startup, before a
+ * single evaluation runs) and from the EvalEngine constructor (so no code
+ * path can build an engine that silently ignores an override).
+ */
+export function assertValidCriticality(overrides) {
+    const issues = criticalityIssues(overrides);
+    if (issues.length === 0)
+        return;
+    throw new Error(`Invalid eval criticality configuration:\n  - ${issues.join('\n  - ')}\n` +
+        'Set eval.criticalRules / eval.nonCriticalRules in your Iris config.json (see docs/api-reference.md § Rule criticality).');
+}
+/**
+ * The effective criticality of one rule.
+ *
+ * A rule the overrides do not name keeps its definition's `critical` and
+ * reports source 'default' — which for a deployed custom rule means the
+ * severity it was deployed with.
+ */
+export function resolveCriticality(rule, overrides, builtIns) {
+    const declared = rule.critical === true;
+    if (!overrides || !builtIns.has(rule))
+        return { critical: declared, source: 'default' };
+    if (overrides.criticalRules?.includes(rule.name))
+        return { critical: true, source: 'config' };
+    if (overrides.nonCriticalRules?.includes(rule.name))
+        return { critical: false, source: 'config' };
+    return { critical: declared, source: 'default' };
+}
+/** A resolver bound to one set of overrides, so callers don't rebuild the identity set per rule. */
+export function criticalityResolver(overrides) {
+    assertValidCriticality(overrides);
+    const builtIns = new Set(builtInRules());
+    return (rule) => resolveCriticality(rule, overrides, builtIns);
+}
+/**
+ * The whole built-in roster, one entry per rule.
+ *
+ * `resolve` is what makes `critical` EFFECTIVE rather than declared. Reading
+ * `rule.critical` on a roster while the engine applies an override would
+ * show an operator a list that disagrees with the verdicts the same process
+ * is producing — they would see `no_silent_tool_failure` reported
+ * non-critical while it vetoed their pipeline. Callers that only want names
+ * and weights may omit it and get the declarations, reported as 'default'.
+ *
+ * Typed on a function, not on EvalEngine, so this module stays free of the
+ * engine it is imported by.
+ */
+export function builtInRuleRoster(resolve) {
+    const out = [];
+    for (const [category, rules] of Object.entries(rulesByType)) {
+        for (const rule of rules) {
+            const effective = resolve
+                ? resolve(rule)
+                : { critical: rule.critical === true, source: 'default' };
+            out.push({
+                name: rule.name,
+                category,
+                description: rule.description,
+                weight: rule.weight,
+                critical: effective.critical,
+                criticalSource: effective.source,
+            });
+        }
+    }
+    return out;
+}

package/dist/eval/engine.d.ts CHANGED Viewed

@@ -1,4 +1,5 @@
-import type { EvalRule, EvalContext, EvalResult, EvalType, CustomRuleDefinition } from '../types/eval.js';
+import type { EvalRule, EvalContext, EvalResult, EvalResultType, EvalType, CustomRuleDefinition } from '../types/eval.js';
+import { type CriticalityOverrides, type EffectiveCriticality } from './criticality.js';
 /**
  * Every bundle eval_type="all" walks, in the order their categories are
  * reported. 'custom' is last: it holds only deployed rules registered under
@@ -6,6 +7,22 @@ import type { EvalRule, EvalContext, EvalResult, EvalType, CustomRuleDefinition
  * from the breakdown when neither exists.
  */
 export declare const ALL_EVAL_TYPES: readonly EvalType[];
+/**
+ * What runs when a caller never chose a bundle. It used to be
+ * 'completeness', so a CI gate keyed on `passed` skipped PII and injection
+ * unless the caller knew to set eval_type — six of seven UAT personas read
+ * passed:true on PII-laden text with nothing in the payload saying the
+ * safety bundle had not run. Every bundle is the only default under which
+ * an omitted argument cannot silently narrow the verdict. The MCP tool and
+ * the HTTP ingest route both read this constant, so the two surfaces
+ * cannot default differently.
+ */
+export declare const DEFAULT_EVAL_TYPE: EvalResultType;
+/**
+ * The one-line note both surfaces attach when the default ran, so a reader
+ * of the response knows the bundle was chosen for them and how to narrow it.
+ */
+export declare const DEFAULT_EVAL_TYPE_NOTE = "eval_type was omitted, so the default ran every bundle \u2014 completeness, relevance, safety, cost and any custom rules \u2014 the same as eval_type=\"all\"; pass a single bundle name to narrow the run.";
 export declare class EvalEngine {
     private additionalRules;
     /**
@@ -22,7 +39,21 @@ export declare class EvalEngine {
     private idByRule;
     private threshold;
     private ruleThresholds?;
-    constructor(threshold?: number, ruleThresholds?: Record<string, unknown>);
+    /**
+     * Effective criticality per rule, bound to this engine's config overrides.
+     * Every veto decision reads THIS, never `rule.critical` directly, so a
+     * promotion or demotion cannot apply on one code path and not another.
+     */
+    private criticality;
+    /**
+     * `criticalityOverrides` are `config.eval` — the criticalRules /
+     * nonCriticalRules lists. Validated here as well as in loadConfig, so an
+     * engine built directly (a test, an embedder) cannot silently ignore a
+     * misspelled rule name.
+     */
+    constructor(threshold?: number, ruleThresholds?: Record<string, unknown>, criticalityOverrides?: CriticalityOverrides);
+    /** The effective criticality of one rule under this engine's config. Read by the rule roster surfaces. */
+    effectiveCriticality(rule: EvalRule): EffectiveCriticality;
     /**
      * Register a rule under a bundle. When `ruleId` is given the registration
      * is IDEMPOTENT by id: registering an id that is already live replaces the

package/dist/eval/engine.js CHANGED Viewed

@@ -1,4 +1,5 @@
 import { getRulesForType, createCustomRule } from './rules/index.js';
+import { criticalityResolver } from './criticality.js';
 import { generateEvalId } from '../utils/ids.js';
 /**
  * Every bundle eval_type="all" walks, in the order their categories are
@@ -7,6 +8,22 @@ import { generateEvalId } from '../utils/ids.js';
  * from the breakdown when neither exists.
  */
 export const ALL_EVAL_TYPES = ['completeness', 'relevance', 'safety', 'cost', 'custom'];
+/**
+ * What runs when a caller never chose a bundle. It used to be
+ * 'completeness', so a CI gate keyed on `passed` skipped PII and injection
+ * unless the caller knew to set eval_type — six of seven UAT personas read
+ * passed:true on PII-laden text with nothing in the payload saying the
+ * safety bundle had not run. Every bundle is the only default under which
+ * an omitted argument cannot silently narrow the verdict. The MCP tool and
+ * the HTTP ingest route both read this constant, so the two surfaces
+ * cannot default differently.
+ */
+export const DEFAULT_EVAL_TYPE = 'all';
+/**
+ * The one-line note both surfaces attach when the default ran, so a reader
+ * of the response knows the bundle was chosen for them and how to narrow it.
+ */
+export const DEFAULT_EVAL_TYPE_NOTE = 'eval_type was omitted, so the default ran every bundle — completeness, relevance, safety, cost and any custom rules — the same as eval_type="all"; pass a single bundle name to narrow the run.';
 export class EvalEngine {
     additionalRules = new Map();
     /**
@@ -23,9 +40,26 @@ export class EvalEngine {
     idByRule = new Map();
     threshold;
     ruleThresholds;
-    constructor(threshold = 0.7, ruleThresholds) {
+    /**
+     * Effective criticality per rule, bound to this engine's config overrides.
+     * Every veto decision reads THIS, never `rule.critical` directly, so a
+     * promotion or demotion cannot apply on one code path and not another.
+     */
+    criticality;
+    /**
+     * `criticalityOverrides` are `config.eval` — the criticalRules /
+     * nonCriticalRules lists. Validated here as well as in loadConfig, so an
+     * engine built directly (a test, an embedder) cannot silently ignore a
+     * misspelled rule name.
+     */
+    constructor(threshold = 0.7, ruleThresholds, criticalityOverrides) {
         this.threshold = threshold;
         this.ruleThresholds = ruleThresholds;
+        this.criticality = criticalityResolver(criticalityOverrides);
+    }
+    /** The effective criticality of one rule under this engine's config. Read by the rule roster surfaces. */
+    effectiveCriticality(rule) {
+        return this.criticality(rule);
     }
     /**
      * Register a rule under a bundle. When `ruleId` is given the registration
@@ -158,8 +192,16 @@ export class EvalEngine {
             const raw = rule.evaluate(evalContext);
             const ruleId = this.idByRule.get(rule);
             const category = categories?.[i];
-            if (ruleId === undefined && category === undefined)
-                return raw;
+            /*
+             * Every result says whether THIS rule vetoes and who decided that.
+             * Without it, a reader holding a failed evaluation cannot tell a
+             * hard violation from a merely low score without knowing the rule
+             * library by heart — and once eval.criticalRules exists, cannot tell
+             * a shipped default from their own promotion at all. Stamped here, on
+             * the one path every evaluation takes, so a surface cannot render the
+             * declared criticality where the engine applied a configured one.
+             */
+            const { critical, source } = this.criticality(rule);
             // ruleId / category sit right after the name so a reader scanning
             // rule_results sees WHICH deployed rule (and which bundle) spoke.
             const { ruleName, ...rest } = raw;
@@ -167,6 +209,8 @@ export class EvalEngine {
                 ruleName,
                 ...(ruleId !== undefined ? { ruleId } : {}),
                 ...(category !== undefined ? { category } : {}),
+                critical,
+                criticalSource: source,
                 ...rest,
             };
         });
@@ -285,7 +329,7 @@ export class EvalEngine {
             }
         }
         const criticalSkipped = skippedIndices
-            .filter((i) => rules[i].critical === true)
+            .filter((i) => this.criticality(rules[i]).critical)
             .map((i) => ruleResults[i].ruleName);
         if (evaluatedIndices.length === 0) {
             return {
@@ -306,7 +350,7 @@ export class EvalEngine {
         const rawScore = totalWeight > 0 ? weightedScore / totalWeight : 0;
         const score = Number.isFinite(rawScore) ? rawScore : 0;
         const criticalFailures = evaluatedIndices
-            .filter((i) => rules[i].critical === true && !ruleResults[i].passed)
+            .filter((i) => this.criticality(rules[i]).critical && !ruleResults[i].passed)
             .map((i) => ruleResults[i].ruleName);
         return {
             score,
@@ -325,12 +369,24 @@ export class EvalEngine {
             if (indices.length === 0)
                 continue;
             const verdict = this.summarize(indices.map((i) => rules[i]), indices.map((i) => ruleResults[i]));
+            /*
+             * A bundle whose every rule skipped was not judged (#406). Reporting
+             * it as passed:false / score:0 read as "failing" to anyone regrouping
+             * by category — cost "failed" on a call that carried no cost data.
+             * Inside the breakdown, null is the honest value: neither passing
+             * nor failing, and it never counted toward the overall verdict
+             * (summarize() already excludes skipped rules). The TOP-LEVEL
+             * `passed` is deliberately not made nullable — it is the verdict a
+             * gate keys on, and a gate must fail closed when nothing was judged;
+             * `insufficient_data: true` is the "unknown" marker at that level.
+             */
+            const judged = verdict.rulesEvaluated > 0;
             breakdown[type] = {
-                score: Math.round(verdict.score * 1000) / 1000,
-                passed: verdict.passed,
+                score: judged ? Math.round(verdict.score * 1000) / 1000 : null,
+                passed: judged ? verdict.passed : null,
                 rules_evaluated: verdict.rulesEvaluated,
                 rules_skipped: verdict.rulesSkipped,
-                insufficient_data: verdict.rulesEvaluated === 0,
+                insufficient_data: !judged,
                 ...(verdict.criticalFailures.length > 0 ? { critical_failures: verdict.criticalFailures } : {}),
                 ...(verdict.criticalSkipped.length > 0 ? { critical_skipped: verdict.criticalSkipped } : {}),
             };

package/dist/eval/rules/cost.d.ts CHANGED Viewed

@@ -1,4 +1,13 @@
 import type { EvalRule } from '../../types/eval.js';
 export declare const costUnderThreshold: EvalRule;
 export declare const tokenEfficiency: EvalRule;
+/** Default for config key `max_tool_repeats`: how many identical calls are tolerated. */
+export declare const DEFAULT_MAX_TOOL_REPEATS = 3;
+/**
+ * How many complete A,B,A,B cycles are tolerated before the alternation is
+ * a loop. Three cycles is six consecutive calls that between them made two
+ * distinct requests.
+ */
+export declare const MAX_TWO_CALL_CYCLES = 2;
+export declare const noToolLoop: EvalRule;
 export declare const costRules: EvalRule[];