npm - @iris-eval/mcp-server - Versions diffs - 0.7.0 → 0.8.1 - Mend

@iris-eval/mcp-server 0.7.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (35) hide show

package/README.md +4 -3
package/dist/config/defaults.js +14 -0
package/dist/config/index.js +8 -0
package/dist/dashboard/assets/index-BfMShR3p.js +10 -0
package/dist/dashboard/index.html +1 -1
package/dist/dashboard/routes/rules.d.ts +8 -11
package/dist/dashboard/routes/rules.js +9 -16
package/dist/dashboard/routes/traces.js +6 -0
package/dist/dashboard/validation.d.ts +3 -3
package/dist/eval/criticality.d.ts +67 -0
package/dist/eval/criticality.js +154 -0
package/dist/eval/engine.d.ts +16 -1
package/dist/eval/engine.js +33 -5
package/dist/eval/rules/cost.d.ts +9 -0
package/dist/eval/rules/cost.js +97 -1
package/dist/eval/rules/safety.d.ts +1 -0
package/dist/eval/rules/safety.js +73 -1
package/dist/eval/rules/trajectory.d.ts +91 -0
package/dist/eval/rules/trajectory.js +297 -0
package/dist/index.js +1 -1
package/dist/self-test.js +1 -1
package/dist/server.js +1 -1
package/dist/tools/evaluate-output.js +21 -8
package/dist/tools/index.js +1 -1
package/dist/tools/list-rules.d.ts +2 -1
package/dist/tools/list-rules.js +22 -4
package/dist/tools/log-trace.d.ts +8 -1
package/dist/tools/log-trace.js +19 -4
package/dist/tools/trace-link.d.ts +10 -0
package/dist/tools/trace-link.js +13 -1
package/dist/types/config.d.ts +14 -0
package/dist/types/eval.d.ts +30 -5
package/package.json +2 -1
package/server.json +51 -2
package/dist/dashboard/assets/index-CKs2Wbd_.js +0 -10

package/dist/dashboard/index.html CHANGED Viewed

@@ -6,7 +6,7 @@
     <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <!-- Stop shipping agents on vibes is filled from .claims.json brand.tagline at build time (vite.config.ts) — never restate the tagline here. -->
     <title>Iris — Stop shipping agents on vibes</title>
-    <script type="module" crossorigin src="/assets/index-CKs2Wbd_.js"></script>
+    <script type="module" crossorigin src="/assets/index-BfMShR3p.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-D0cFfBqn.css">
   </head>
   <body>

package/dist/dashboard/routes/rules.d.ts CHANGED Viewed

@@ -2,20 +2,18 @@ import { Router } from 'express';
 import type { IStorageAdapter } from '../../types/query.js';
 import type { CustomRuleStore } from '../../custom-rule-store.js';
 import type { EvalEngine } from '../../eval/engine.js';
-import type { EvalType } from '../../types/eval.js';
+import { type BuiltInRuleMeta } from '../../eval/criticality.js';
 interface RoutesOptions {
     customRuleStore: CustomRuleStore;
     evalEngine: EvalEngine;
 }
-/** Built-in rule metadata as the dashboard sees it — derived from the engine, never restated. */
-export interface BuiltInRuleMeta {
-    name: string;
-    category: EvalType;
-    description: string;
-    weight: number;
-    critical: boolean;
-}
-export declare function listBuiltInRules(): BuiltInRuleMeta[];
+export type { BuiltInRuleMeta };
+/**
+ * The roster as the dashboard asks for it. Passing the engine is what makes
+ * `critical` the value this server will actually apply; omitting it returns
+ * the rules' own declarations, reported as source 'default'.
+ */
+export declare function listBuiltInRules(engine?: EvalEngine): BuiltInRuleMeta[];
 export declare function registerRuleRoutes(router: Router, storage: IStorageAdapter, opts: RoutesOptions): void;
 /** Verdict of the proposed rule against the caller's own sample text. */
 export interface SamplePreview {
@@ -25,4 +23,3 @@ export interface SamplePreview {
     skipped: boolean;
     skipReason?: string;
 }
-export {};

package/dist/dashboard/routes/rules.js CHANGED Viewed

@@ -1,6 +1,6 @@
 import { z } from 'zod';
 import { createCustomRule } from '../../eval/rules/custom.js';
-import { rulesByType } from '../../eval/rules/index.js';
+import { builtInRuleRoster } from '../../eval/criticality.js';
 import { requireTenant } from '../../middleware/tenant.js';
 import { DuplicateRuleNameError, replacedRulesWarning, retireSameNamedRules } from '../../tools/deploy-rule.js';
 import { strictBody } from '../validation.js';
@@ -77,20 +77,13 @@ const PreviewSchema = strictBody({
 const ToggleSchema = strictBody({
     enabled: z.boolean(),
 });
-export function listBuiltInRules() {
-    const out = [];
-    for (const [category, rules] of Object.entries(rulesByType)) {
-        for (const rule of rules) {
-            out.push({
-                name: rule.name,
-                category,
-                description: rule.description,
-                weight: rule.weight,
-                critical: rule.critical === true,
-            });
-        }
-    }
-    return out;
+/**
+ * The roster as the dashboard asks for it. Passing the engine is what makes
+ * `critical` the value this server will actually apply; omitting it returns
+ * the rules' own declarations, reported as source 'default'.
+ */
+export function listBuiltInRules(engine) {
+    return builtInRuleRoster(engine ? (rule) => engine.effectiveCriticality(rule) : undefined);
 }
 export function registerRuleRoutes(router, storage, opts) {
     /*
@@ -112,7 +105,7 @@ export function registerRuleRoutes(router, storage, opts) {
      * are process-global.
      */
     router.get('/rules/builtin', (_req, res) => {
-        res.json({ rules: listBuiltInRules() });
+        res.json({ rules: listBuiltInRules(opts.evalEngine) });
     });
     router.get('/rules/custom', (req, res) => {
         const tenantId = requireTenant(req);

package/dist/dashboard/routes/traces.js CHANGED Viewed

@@ -68,6 +68,12 @@ export function registerTraceRoutes(router, storage, options) {
                 input: body.input,
                 costUsd: body.cost_usd,
                 tokenUsage: body.token_usage,
+                // The trajectory the SAME request just stored. This body already
+                // carries what the agent did; not forwarding it made every
+                // trajectory rule skip on the one path where the data was
+                // guaranteed present — an ingest that captured a failed tool call
+                // and then evaluated as though it had never been told.
+                toolCalls: body.tool_calls,
             };
             // An omitted eval_type runs every bundle — the same default, from the
             // same constant, as the MCP tool — and says so in the response.

package/dist/dashboard/validation.d.ts CHANGED Viewed

@@ -22,7 +22,7 @@ export declare const ingestTraceSchema: z.ZodObject<{
         output: z.ZodOptional<z.ZodUnknown>;
         latency_ms: z.ZodOptional<z.ZodNumber>;
         error: z.ZodOptional<z.ZodString>;
-    }, z.core.$strip>>>;
+    }, z.core.$strict>>>;
     latency_ms: z.ZodOptional<z.ZodNumber>;
     token_usage: z.ZodOptional<z.ZodObject<{
         prompt_tokens: z.ZodOptional<z.ZodNumber>;
@@ -83,9 +83,9 @@ export declare const traceQuerySchema: z.ZodObject<{
 export declare const evalQuerySchema: z.ZodObject<{
     eval_type: z.ZodOptional<z.ZodString>;
     passed: z.ZodOptional<z.ZodPipe<z.ZodEnum<{
-        true: "true";
         false: "false";
-    }>, z.ZodTransform<boolean, "true" | "false">>>;
+        true: "true";
+    }>, z.ZodTransform<boolean, "false" | "true">>>;
     since: z.ZodOptional<z.ZodString>;
     until: z.ZodOptional<z.ZodString>;
     limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;

package/dist/eval/criticality.d.ts ADDED Viewed

@@ -0,0 +1,67 @@
+import type { EvalRule, EvalType } from '../types/eval.js';
+/** Where a rule's EFFECTIVE criticality came from. */
+export type CriticalitySource = 'default' | 'config';
+export interface CriticalityOverrides {
+    /** Built-in rule names promoted to critical. */
+    criticalRules?: string[];
+    /** Built-in rule names demoted from critical. */
+    nonCriticalRules?: string[];
+}
+export interface EffectiveCriticality {
+    critical: boolean;
+    /** 'config' when one of the two lists decided this; 'default' otherwise. */
+    source: CriticalitySource;
+}
+/** Every built-in rule, in bundle order. The registry the lists are checked against. */
+export declare function builtInRules(): EvalRule[];
+/** Built-in rule names, sorted — the "valid list" an error message prints. */
+export declare function builtInRuleNames(): string[];
+/**
+ * Every problem with a pair of override lists, as human sentences.
+ * Empty means valid. Separate from the thrower so a caller that wants to
+ * report several at once (a config linter, a test) can.
+ */
+export declare function criticalityIssues(overrides: CriticalityOverrides | undefined): string[];
+/**
+ * Throw on any problem, with every problem named at once.
+ *
+ * Called from loadConfig (so a bad config.json fails at startup, before a
+ * single evaluation runs) and from the EvalEngine constructor (so no code
+ * path can build an engine that silently ignores an override).
+ */
+export declare function assertValidCriticality(overrides: CriticalityOverrides | undefined): void;
+/**
+ * The effective criticality of one rule.
+ *
+ * A rule the overrides do not name keeps its definition's `critical` and
+ * reports source 'default' — which for a deployed custom rule means the
+ * severity it was deployed with.
+ */
+export declare function resolveCriticality(rule: EvalRule, overrides: CriticalityOverrides | undefined, builtIns: ReadonlySet<EvalRule>): EffectiveCriticality;
+/** A resolver bound to one set of overrides, so callers don't rebuild the identity set per rule. */
+export declare function criticalityResolver(overrides: CriticalityOverrides | undefined): (rule: EvalRule) => EffectiveCriticality;
+/** Built-in rule metadata as every roster surface reports it — derived from the registry, never restated. */
+export interface BuiltInRuleMeta {
+    name: string;
+    category: EvalType;
+    description: string;
+    weight: number;
+    /** EFFECTIVE criticality, after eval.criticalRules / eval.nonCriticalRules. */
+    critical: boolean;
+    /** Who decided it: the rule's own declaration, or one of the config lists. */
+    criticalSource: CriticalitySource;
+}
+/**
+ * The whole built-in roster, one entry per rule.
+ *
+ * `resolve` is what makes `critical` EFFECTIVE rather than declared. Reading
+ * `rule.critical` on a roster while the engine applies an override would
+ * show an operator a list that disagrees with the verdicts the same process
+ * is producing — they would see `no_silent_tool_failure` reported
+ * non-critical while it vetoed their pipeline. Callers that only want names
+ * and weights may omit it and get the declarations, reported as 'default'.
+ *
+ * Typed on a function, not on EvalEngine, so this module stays free of the
+ * engine it is imported by.
+ */
+export declare function builtInRuleRoster(resolve?: (rule: EvalRule) => EffectiveCriticality): BuiltInRuleMeta[];

package/dist/eval/criticality.js ADDED Viewed

@@ -0,0 +1,154 @@
+/*
+ * Which built-in rules VETO — a deployment's decision, not ours.
+ *
+ * `critical` is a property of each rule's definition, and until now that was
+ * the whole story: a failing critical rule forced `passed: false`, a failing
+ * non-critical one only moved the score, and nobody running Iris could
+ * change either. That default is a judgement about acceptable error, and it
+ * is not ours to make for everyone. The trajectory release is the plain
+ * case: `no_silent_tool_failure` catches an agent answering over a tool that
+ * errored, which is exactly what a team gating deploys wants to block — but
+ * its precision on a 30-case family carries a 95% lower bound of 77.2%, so
+ * shipping it as a veto for everyone would force false failures on people
+ * who never asked for that trade. A team that HAS looked at the number can
+ * make the call for their own pipeline; the config keys below are how.
+ *
+ * Two lists, both optional, both naming BUILT-IN rules:
+ *   eval.criticalRules     — promote: these rules veto `passed`.
+ *   eval.nonCriticalRules  — demote: these rules stop vetoing.
+ *
+ * Every name is checked against the rule registry when the config loads and
+ * again when an engine is constructed. An unknown name is a startup error
+ * naming the valid list, never a silent no-op: a typo in `criticalRules`
+ * that quietly did nothing would leave an operator believing a gate exists
+ * when it does not, which is the same "detection that reports an all-clear"
+ * failure the critical veto was built to stop.
+ *
+ * Overrides are matched by rule IDENTITY, not by name. Deployed custom rules
+ * do not enforce unique names — one can legitimately be called `no_pii` —
+ * and a name-keyed override would silently reach it. A custom rule's
+ * severity stays its own definition's business.
+ */
+import { rulesByType } from './rules/index.js';
+/** Every built-in rule, in bundle order. The registry the lists are checked against. */
+export function builtInRules() {
+    return ['completeness', 'relevance', 'safety', 'cost'].flatMap((t) => rulesByType[t]);
+}
+/** Built-in rule names, sorted — the "valid list" an error message prints. */
+export function builtInRuleNames() {
+    return builtInRules()
+        .map((r) => r.name)
+        .sort();
+}
+/**
+ * Every problem with a pair of override lists, as human sentences.
+ * Empty means valid. Separate from the thrower so a caller that wants to
+ * report several at once (a config linter, a test) can.
+ */
+export function criticalityIssues(overrides) {
+    if (!overrides)
+        return [];
+    const issues = [];
+    const valid = new Set(builtInRuleNames());
+    const seen = {
+        criticalRules: new Set(),
+        nonCriticalRules: new Set(),
+    };
+    for (const key of ['criticalRules', 'nonCriticalRules']) {
+        const list = overrides[key];
+        if (list === undefined)
+            continue;
+        if (!Array.isArray(list)) {
+            issues.push(`eval.${key} must be an array of built-in rule names.`);
+            continue;
+        }
+        for (const entry of list) {
+            if (typeof entry !== 'string' || entry.trim().length === 0) {
+                issues.push(`eval.${key} contains ${JSON.stringify(entry)}, which is not a rule name.`);
+                continue;
+            }
+            if (!valid.has(entry)) {
+                issues.push(`eval.${key} names "${entry}", which is not a built-in rule. ` +
+                    `Valid names: ${builtInRuleNames().join(', ')}. ` +
+                    'Deployed custom rules carry their own severity and are not set here.');
+                continue;
+            }
+            seen[key].add(entry);
+        }
+    }
+    for (const name of seen.criticalRules) {
+        if (seen.nonCriticalRules.has(name)) {
+            issues.push(`"${name}" is in BOTH eval.criticalRules and eval.nonCriticalRules, so the config does not say whether it should veto. Remove it from one of them.`);
+        }
+    }
+    return issues;
+}
+/**
+ * Throw on any problem, with every problem named at once.
+ *
+ * Called from loadConfig (so a bad config.json fails at startup, before a
+ * single evaluation runs) and from the EvalEngine constructor (so no code
+ * path can build an engine that silently ignores an override).
+ */
+export function assertValidCriticality(overrides) {
+    const issues = criticalityIssues(overrides);
+    if (issues.length === 0)
+        return;
+    throw new Error(`Invalid eval criticality configuration:\n  - ${issues.join('\n  - ')}\n` +
+        'Set eval.criticalRules / eval.nonCriticalRules in your Iris config.json (see docs/api-reference.md § Rule criticality).');
+}
+/**
+ * The effective criticality of one rule.
+ *
+ * A rule the overrides do not name keeps its definition's `critical` and
+ * reports source 'default' — which for a deployed custom rule means the
+ * severity it was deployed with.
+ */
+export function resolveCriticality(rule, overrides, builtIns) {
+    const declared = rule.critical === true;
+    if (!overrides || !builtIns.has(rule))
+        return { critical: declared, source: 'default' };
+    if (overrides.criticalRules?.includes(rule.name))
+        return { critical: true, source: 'config' };
+    if (overrides.nonCriticalRules?.includes(rule.name))
+        return { critical: false, source: 'config' };
+    return { critical: declared, source: 'default' };
+}
+/** A resolver bound to one set of overrides, so callers don't rebuild the identity set per rule. */
+export function criticalityResolver(overrides) {
+    assertValidCriticality(overrides);
+    const builtIns = new Set(builtInRules());
+    return (rule) => resolveCriticality(rule, overrides, builtIns);
+}
+/**
+ * The whole built-in roster, one entry per rule.
+ *
+ * `resolve` is what makes `critical` EFFECTIVE rather than declared. Reading
+ * `rule.critical` on a roster while the engine applies an override would
+ * show an operator a list that disagrees with the verdicts the same process
+ * is producing — they would see `no_silent_tool_failure` reported
+ * non-critical while it vetoed their pipeline. Callers that only want names
+ * and weights may omit it and get the declarations, reported as 'default'.
+ *
+ * Typed on a function, not on EvalEngine, so this module stays free of the
+ * engine it is imported by.
+ */
+export function builtInRuleRoster(resolve) {
+    const out = [];
+    for (const [category, rules] of Object.entries(rulesByType)) {
+        for (const rule of rules) {
+            const effective = resolve
+                ? resolve(rule)
+                : { critical: rule.critical === true, source: 'default' };
+            out.push({
+                name: rule.name,
+                category,
+                description: rule.description,
+                weight: rule.weight,
+                critical: effective.critical,
+                criticalSource: effective.source,
+            });
+        }
+    }
+    return out;
+}

package/dist/eval/engine.d.ts CHANGED Viewed

@@ -1,4 +1,5 @@
 import type { EvalRule, EvalContext, EvalResult, EvalResultType, EvalType, CustomRuleDefinition } from '../types/eval.js';
+import { type CriticalityOverrides, type EffectiveCriticality } from './criticality.js';
 /**
  * Every bundle eval_type="all" walks, in the order their categories are
  * reported. 'custom' is last: it holds only deployed rules registered under
@@ -38,7 +39,21 @@ export declare class EvalEngine {
     private idByRule;
     private threshold;
     private ruleThresholds?;
-    constructor(threshold?: number, ruleThresholds?: Record<string, unknown>);
+    /**
+     * Effective criticality per rule, bound to this engine's config overrides.
+     * Every veto decision reads THIS, never `rule.critical` directly, so a
+     * promotion or demotion cannot apply on one code path and not another.
+     */
+    private criticality;
+    /**
+     * `criticalityOverrides` are `config.eval` — the criticalRules /
+     * nonCriticalRules lists. Validated here as well as in loadConfig, so an
+     * engine built directly (a test, an embedder) cannot silently ignore a
+     * misspelled rule name.
+     */
+    constructor(threshold?: number, ruleThresholds?: Record<string, unknown>, criticalityOverrides?: CriticalityOverrides);
+    /** The effective criticality of one rule under this engine's config. Read by the rule roster surfaces. */
+    effectiveCriticality(rule: EvalRule): EffectiveCriticality;
     /**
      * Register a rule under a bundle. When `ruleId` is given the registration
      * is IDEMPOTENT by id: registering an id that is already live replaces the

package/dist/eval/engine.js CHANGED Viewed

@@ -1,4 +1,5 @@
 import { getRulesForType, createCustomRule } from './rules/index.js';
+import { criticalityResolver } from './criticality.js';
 import { generateEvalId } from '../utils/ids.js';
 /**
  * Every bundle eval_type="all" walks, in the order their categories are
@@ -39,9 +40,26 @@ export class EvalEngine {
     idByRule = new Map();
     threshold;
     ruleThresholds;
-    constructor(threshold = 0.7, ruleThresholds) {
+    /**
+     * Effective criticality per rule, bound to this engine's config overrides.
+     * Every veto decision reads THIS, never `rule.critical` directly, so a
+     * promotion or demotion cannot apply on one code path and not another.
+     */
+    criticality;
+    /**
+     * `criticalityOverrides` are `config.eval` — the criticalRules /
+     * nonCriticalRules lists. Validated here as well as in loadConfig, so an
+     * engine built directly (a test, an embedder) cannot silently ignore a
+     * misspelled rule name.
+     */
+    constructor(threshold = 0.7, ruleThresholds, criticalityOverrides) {
         this.threshold = threshold;
         this.ruleThresholds = ruleThresholds;
+        this.criticality = criticalityResolver(criticalityOverrides);
+    }
+    /** The effective criticality of one rule under this engine's config. Read by the rule roster surfaces. */
+    effectiveCriticality(rule) {
+        return this.criticality(rule);
     }
     /**
      * Register a rule under a bundle. When `ruleId` is given the registration
@@ -174,8 +192,16 @@ export class EvalEngine {
             const raw = rule.evaluate(evalContext);
             const ruleId = this.idByRule.get(rule);
             const category = categories?.[i];
-            if (ruleId === undefined && category === undefined)
-                return raw;
+            /*
+             * Every result says whether THIS rule vetoes and who decided that.
+             * Without it, a reader holding a failed evaluation cannot tell a
+             * hard violation from a merely low score without knowing the rule
+             * library by heart — and once eval.criticalRules exists, cannot tell
+             * a shipped default from their own promotion at all. Stamped here, on
+             * the one path every evaluation takes, so a surface cannot render the
+             * declared criticality where the engine applied a configured one.
+             */
+            const { critical, source } = this.criticality(rule);
             // ruleId / category sit right after the name so a reader scanning
             // rule_results sees WHICH deployed rule (and which bundle) spoke.
             const { ruleName, ...rest } = raw;
@@ -183,6 +209,8 @@ export class EvalEngine {
                 ruleName,
                 ...(ruleId !== undefined ? { ruleId } : {}),
                 ...(category !== undefined ? { category } : {}),
+                critical,
+                criticalSource: source,
                 ...rest,
             };
         });
@@ -301,7 +329,7 @@ export class EvalEngine {
             }
         }
         const criticalSkipped = skippedIndices
-            .filter((i) => rules[i].critical === true)
+            .filter((i) => this.criticality(rules[i]).critical)
             .map((i) => ruleResults[i].ruleName);
         if (evaluatedIndices.length === 0) {
             return {
@@ -322,7 +350,7 @@ export class EvalEngine {
         const rawScore = totalWeight > 0 ? weightedScore / totalWeight : 0;
         const score = Number.isFinite(rawScore) ? rawScore : 0;
         const criticalFailures = evaluatedIndices
-            .filter((i) => rules[i].critical === true && !ruleResults[i].passed)
+            .filter((i) => this.criticality(rules[i]).critical && !ruleResults[i].passed)
             .map((i) => ruleResults[i].ruleName);
         return {
             score,

package/dist/eval/rules/cost.d.ts CHANGED Viewed

@@ -1,4 +1,13 @@
 import type { EvalRule } from '../../types/eval.js';
 export declare const costUnderThreshold: EvalRule;
 export declare const tokenEfficiency: EvalRule;
+/** Default for config key `max_tool_repeats`: how many identical calls are tolerated. */
+export declare const DEFAULT_MAX_TOOL_REPEATS = 3;
+/**
+ * How many complete A,B,A,B cycles are tolerated before the alternation is
+ * a loop. Three cycles is six consecutive calls that between them made two
+ * distinct requests.
+ */
+export declare const MAX_TWO_CALL_CYCLES = 2;
+export declare const noToolLoop: EvalRule;
 export declare const costRules: EvalRule[];

package/dist/eval/rules/cost.js CHANGED Viewed

@@ -1,3 +1,4 @@
+import { callKey, describeInput, skipWithoutTrajectory } from './trajectory.js';
 export const costUnderThreshold = {
     name: 'cost_under_threshold',
     description: 'Total cost must be under a configurable USD threshold',
@@ -44,4 +45,99 @@ export const tokenEfficiency = {
         };
     },
 };
-export const costRules = [costUnderThreshold, tokenEfficiency];
+/** Default for config key `max_tool_repeats`: how many identical calls are tolerated. */
+export const DEFAULT_MAX_TOOL_REPEATS = 3;
+/**
+ * How many complete A,B,A,B cycles are tolerated before the alternation is
+ * a loop. Three cycles is six consecutive calls that between them made two
+ * distinct requests.
+ */
+export const MAX_TWO_CALL_CYCLES = 2;
+/** The longest run of alternating A,B calls, and how many complete cycles it holds. */
+function longestTwoCallCycle(keys) {
+    let best = null;
+    for (let start = 0; start + 3 < keys.length; start++) {
+        const a = keys[start];
+        const b = keys[start + 1];
+        if (a === b)
+            continue;
+        let len = 2;
+        while (start + len < keys.length && keys[start + len] === (len % 2 === 0 ? a : b))
+            len++;
+        const cycles = Math.floor(len / 2);
+        if (cycles >= 2 && (best === null || cycles > best.cycles))
+            best = { a, b, cycles };
+    }
+    return best;
+}
+/*
+ * The other half of what the trajectory shows: not a wrong answer, a wasted
+ * one.
+ *
+ * Transcript t-16 answers the question correctly, and gets there by running
+ * the identical `ls src/tools` five times with five identical results. Four
+ * of those turns bought nothing, and each one resent the whole context —
+ * 18,918 prompt tokens for a directory listing. cost_under_threshold cannot
+ * see it: the bill still comes to $0.0621, well under the $0.10 default. The
+ * loop is only visible in the sequence of calls.
+ *
+ * Cost, not completeness or safety, because that is where the harm lands —
+ * spend and latency, on an answer that was already available. Non-critical
+ * for the same reason: a repetitive agent is wasteful, not unsafe, and
+ * `passed` should not be vetoed by a behavioural signal.
+ */
+export const noToolLoop = {
+    name: 'no_tool_loop',
+    description: 'The agent must not repeat itself. Fails when one tool is called with an identical input (object keys sorted, whitespace collapsed) more than max_tool_repeats times — default 3, config key `max_tool_repeats` — or when two calls alternate for more than two complete A,B,A,B cycles. Skips when no tool calls are provided, so an evaluation with no trajectory reports "not judged" rather than clean. Catches the wasted spend a cost threshold cannot see: five identical calls can still bill under a per-evaluation cost limit',
+    evalType: 'cost',
+    weight: 1,
+    evaluate(context) {
+        const skip = skipWithoutTrajectory('no_tool_loop', context);
+        if (skip)
+            return skip;
+        const calls = context.toolCalls ?? [];
+        const configured = context.customConfig?.max_tool_repeats;
+        const maxRepeats = typeof configured === 'number' && Number.isFinite(configured) && configured >= 1
+            ? Math.floor(configured)
+            : DEFAULT_MAX_TOOL_REPEATS;
+        const keys = calls.map(callKey);
+        const counts = new Map();
+        for (const key of keys)
+            counts.set(key, (counts.get(key) ?? 0) + 1);
+        let worstKey = '';
+        let worstCount = 0;
+        for (const [key, count] of counts) {
+            if (count > worstCount) {
+                worstKey = key;
+                worstCount = count;
+            }
+        }
+        if (worstCount > maxRepeats) {
+            const call = calls[keys.indexOf(worstKey)];
+            return {
+                ruleName: 'no_tool_loop',
+                passed: false,
+                score: Math.max(0, 1 - (worstCount - maxRepeats) * 0.25),
+                message: `Tool loop: ${call.tool_name} called ${worstCount} times with the same input — ${describeInput(call.input)} — over ${calls.length} call${calls.length === 1 ? '' : 's'} (max ${maxRepeats})`,
+            };
+        }
+        const cycle = longestTwoCallCycle(keys);
+        if (cycle !== null && cycle.cycles > MAX_TWO_CALL_CYCLES) {
+            const a = calls[keys.indexOf(cycle.a)];
+            const b = calls[keys.indexOf(cycle.b)];
+            return {
+                ruleName: 'no_tool_loop',
+                passed: false,
+                score: Math.max(0, 1 - (cycle.cycles - MAX_TWO_CALL_CYCLES) * 0.25),
+                message: `Tool loop: ${a.tool_name} (${describeInput(a.input)}) and ${b.tool_name} (${describeInput(b.input)}) alternate for ${cycle.cycles} cycles (max ${MAX_TWO_CALL_CYCLES})`,
+            };
+        }
+        return {
+            ruleName: 'no_tool_loop',
+            passed: true,
+            score: 1,
+            message: `No repeated tool call (${calls.length} call${calls.length === 1 ? '' : 's'}; most repeated ran ${worstCount}×, max ${maxRepeats})`,
+        };
+    },
+};
+export const costRules = [costUnderThreshold, tokenEfficiency, noToolLoop];

package/dist/eval/rules/safety.d.ts CHANGED Viewed

@@ -37,4 +37,5 @@ export interface HallucinationSignal {
 }
 export declare const HALLUCINATION_MARKERS: ReadonlyArray<HallucinationSignal>;
 export declare const noHallucinationMarkers: EvalRule;
+export declare const noSilentToolFailure: EvalRule;
 export declare const safetyRules: EvalRule[];