npm - @iris-eval/mcp-server - Versions diffs - 0.9.0 → 0.10.0 - Mend

@iris-eval/mcp-server 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (48) hide show

package/README.md +1 -1
package/dist/config/defaults.js +15 -0
package/dist/dashboard/assets/{index-Cz8_oOqG.js → index-CeJbaq6m.js} +1 -1
package/dist/dashboard/index.html +1 -1
package/dist/dashboard/routes/traces.js +2 -2
package/dist/dashboard/seed-demo-data.js +1 -1
package/dist/eval/citation-verify/verifier.d.ts +16 -1
package/dist/eval/citation-verify/verifier.js +14 -4
package/dist/eval/compose.d.ts +57 -0
package/dist/eval/compose.js +179 -0
package/dist/eval/criticality.d.ts +7 -0
package/dist/eval/decision-moment.js +33 -4
package/dist/eval/engine.d.ts +5 -2
package/dist/eval/engine.js +81 -13
package/dist/eval/llm-judge/evaluator.d.ts +20 -0
package/dist/eval/llm-judge/evaluator.js +10 -1
package/dist/eval/published-accuracy.d.ts +22 -22
package/dist/eval/published-accuracy.js +11 -11
package/dist/eval/risk.d.ts +60 -0
package/dist/eval/risk.js +187 -0
package/dist/eval/rules/completeness.js +5 -1
package/dist/eval/rules/cost.d.ts +1 -1
package/dist/eval/rules/cost.js +6 -6
package/dist/eval/rules/custom.js +1 -0
package/dist/eval/rules/relevance.js +7 -2
package/dist/eval/rules/safety.d.ts +6 -2
package/dist/eval/rules/safety.js +55 -59
package/dist/eval/seeded-random.d.ts +4 -0
package/dist/eval/seeded-random.js +36 -0
package/dist/eval/stamp.d.ts +1 -1
package/dist/eval/stamp.js +1 -0
package/dist/eval/text/checksums.d.ts +23 -0
package/dist/eval/text/checksums.js +97 -0
package/dist/eval/text/normalise.d.ts +30 -0
package/dist/eval/text/normalise.js +265 -0
package/dist/eval/text/sentences.d.ts +15 -0
package/dist/eval/text/sentences.js +149 -0
package/dist/self-test.js +3 -3
package/dist/storage/sqlite-adapter.js +16 -2
package/dist/tools/evaluate-output.js +2 -2
package/dist/tools/evaluate-with-llm-judge.d.ts +3 -0
package/dist/tools/evaluate-with-llm-judge.js +29 -1
package/dist/tools/verify-citations.d.ts +2 -1
package/dist/tools/verify-citations.js +25 -4
package/dist/types/config.d.ts +35 -0
package/dist/types/eval.d.ts +51 -0
package/package.json +1 -1
package/server.json +2 -2

package/dist/dashboard/index.html CHANGED Viewed

@@ -6,7 +6,7 @@
     <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <!-- Stop shipping agents on vibes is filled from .claims.json brand.tagline at build time (vite.config.ts) — never restate the tagline here. -->
     <title>Iris — Stop shipping agents on vibes</title>
-    <script type="module" crossorigin src="/assets/index-Cz8_oOqG.js"></script>
+    <script type="module" crossorigin src="/assets/index-CeJbaq6m.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-D0cFfBqn.css">
   </head>
   <body>

package/dist/dashboard/routes/traces.js CHANGED Viewed

@@ -82,8 +82,8 @@ export function registerTraceRoutes(router, storage, options) {
             const evalTypeOmitted = body.eval_type === undefined;
             const evalType = body.eval_type ?? DEFAULT_EVAL_TYPE;
             const evaluation = evalType === 'all'
-                ? options.evalEngine.evaluateAll(context)
-                : options.evalEngine.evaluate(evalType, context);
+                ? await options.evalEngine.evaluateAll(context)
+                : await options.evalEngine.evaluate(evalType, context);
             evaluation.trace_id = traceId;
             await storage.insertEvalResult(tenantId, evaluation);
             // The same serializer as evaluate_output (src/eval/response.ts): the

package/dist/dashboard/seed-demo-data.js CHANGED Viewed

@@ -574,7 +574,7 @@ function simulateCostEval(costUsd, tokenUsage, shouldPass) {
             : `Cost ($${costUsd.toFixed(4)}) exceeds threshold ($${threshold.toFixed(4)})`,
     };
     const r2 = {
-        ruleName: 'token_efficiency',
+        ruleName: 'verbosity_ratio',
         passed: ratio <= maxRatio,
         score: ratio <= maxRatio ? 1 : Math.max(0, 1 - (ratio - maxRatio) / maxRatio),
         message: ratio <= maxRatio

package/dist/eval/citation-verify/verifier.d.ts CHANGED Viewed

@@ -33,7 +33,22 @@ export interface VerifiedCitation {
 }
 export interface VerifyCitationsResult {
     overallScore: number | null;
-    passed: boolean;
+    /**
+     * The verdict, and it is **null when nothing was judged**.
+     *
+     * Until 0.10.0 this was `true` in that case: no citation resolved, or the
+     * judge failed on every one, and the tool said the output passed. A
+     * caller reading `passed` shipped an answer whose sources had not been
+     * checked at all. There is no verdict when nothing was verified, and null
+     * is what says so.
+     *
+     * When citations WERE judged, the rule is counts and not a proportion:
+     * every judged citation must be supported. A proportion let one
+     * fabricated source among three real ones score 0.67 and pass.
+     */
+    passed: boolean | null;
+    /** Judged citations the judge ruled unsupported. The number the verdict turns on. */
+    totalUnsupported: number;
     citations: VerifiedCitation[];
     totalCostUsd: number;
     totalCitationsFound: number;

package/dist/eval/citation-verify/verifier.js CHANGED Viewed

@@ -247,13 +247,23 @@ export async function verifyCitations(params) {
     // unsupported would make a judge outage on 5 of 10 supported citations
     // score 0.5, indistinguishable from fabrication.
     const overallScore = totalJudged > 0 ? Math.round((totalSupported / totalJudged) * 100) / 100 : null;
-    // Fail if >= 50% of judged sources don't support the claim. When no
-    // citations, none resolved, or none judged, we don't fail — there's
-    // nothing to score, we just report that.
-    const passed = overallScore === null ? true : overallScore >= 0.5;
+    const totalUnsupported = totalJudged - totalSupported;
+    /*
+     * Counts, not a proportion, and null when nothing was judged.
+     *
+     * The old rule was `overallScore >= 0.5`, with `true` when the score was
+     * null. Both halves were wrong. A proportion let one fabricated source
+     * among three real ones score 0.67 and pass — a citation either supports
+     * the claim or it does not, and one that does not is the finding. And
+     * "nothing was judged" was reported as a pass, so an output whose sources
+     * never resolved, or whose every judge call failed, came back looking
+     * verified. There is no verdict when nothing was verified.
+     */
+    const passed = totalJudged === 0 ? null : totalUnsupported === 0;
     return {
         overallScore,
         passed,
+        totalUnsupported,
         citations: out,
         totalCostUsd: Math.round(totalCost * 1_000_000) / 1_000_000,
         totalCitationsFound: totalFound,

package/dist/eval/compose.d.ts ADDED Viewed

@@ -0,0 +1,57 @@
+import type { EvalResult, EvalRuleResult, Interpretation, Need, Verdict } from '../types/eval.js';
+import { type PriorMode } from './risk.js';
+export interface ComposeConfig {
+    /** `risk` composes by kind; `legacy` runs the pre-0.10.0 arithmetic. */
+    composer: 'risk' | 'legacy';
+    /** How many wrongly blocked builds one shipped failure is worth. τ = 1 / (1 + c). */
+    falsePassCost: number;
+    /** What a critical rule that could not answer does to the verdict. */
+    onCriticalSkipped: 'unknown' | 'fail' | 'pass';
+    /** Inputs the deployment insists every evaluation carries; absent ones make the verdict unknown. */
+    requiredEvidence: readonly Need[];
+    /** Whether a shipped default threshold decides the verdict, or only advises. */
+    defaultsGate: boolean;
+    /** The prior that an output is bad, before any rule speaks. */
+    prior: number;
+    /** How that prior is spread over the failure classes the detectors examine. */
+    priorMode: PriorMode;
+}
+export declare const DEFAULT_COMPOSE: ComposeConfig;
+/** The risk threshold a loss ratio implies: block when the expected loss of passing exceeds that of blocking. */
+export declare function tau(falsePassCost: number): number;
+/**
+ * Whether a policy rule DECIDES the verdict here, or only advises.
+ *
+ * "A default is not your policy." A shipped threshold — a cost ceiling of
+ * $0.50, a length floor of 50 characters — is our guess about a deployment
+ * we have never seen, and stopping someone's build on it is presumptuous.
+ * A threshold the deployment SET is their decision and gates.
+ *
+ * The distinction is not a list of rule names. For a BUILT-IN policy it is
+ * whether the number the rule compared against is one we chose, which every
+ * result already records as `thresholdSource` on its count evidence (0.9.0);
+ * a policy with no number at all — "the output is empty" — is structural,
+ * has no guess in it, and gates.
+ *
+ * A CUSTOM rule is different: its severity is the deployment's own statement
+ * of how much it matters, made when the rule was deployed. High and critical
+ * gate (they resolve to critical); medium and low advise, which is the
+ * contract `deploy_rule` has always had. An inline rule passed in the call
+ * carries no severity and advises, for the same reason.
+ */
+export declare function decides(r: EvalRuleResult, defaultsGate: boolean): boolean;
+/**
+ * The verdict for one evaluation. The weighted mean is never consulted: it
+ * survives as a quality gradient on the score field and is never re-meant.
+ */
+export declare function compose(result: Pick<EvalResult, 'rule_results' | 'score' | 'insufficient_data' | 'rules_evaluated'>, cfg: ComposeConfig): Verdict;
+/**
+ * The sentences a reader needs that the verdict alone does not carry.
+ *
+ * The one that must exist: when a rule visibly FIRED and the verdict still
+ * passed, say why and name the one setting that would change it. Without
+ * it, "cost_under_threshold failed" beside "passed: true" reads as a bug,
+ * and that is the first thing a builder who never opens a config file will
+ * meet.
+ */
+export declare function interpretations(result: Pick<EvalResult, 'rule_results'>, verdict: Verdict, cfg: ComposeConfig): Interpretation[];

package/dist/eval/compose.js ADDED Viewed

@@ -0,0 +1,179 @@
+import { riskEstimate, DEFAULT_PRIOR, DEFAULT_PRIOR_MODE, DEFAULT_FALSE_PASS_COST } from './risk.js';
+export const DEFAULT_COMPOSE = {
+    composer: 'risk',
+    falsePassCost: DEFAULT_FALSE_PASS_COST,
+    onCriticalSkipped: 'unknown',
+    requiredEvidence: [],
+    defaultsGate: false,
+    prior: DEFAULT_PRIOR,
+    priorMode: DEFAULT_PRIOR_MODE,
+};
+/** The risk threshold a loss ratio implies: block when the expected loss of passing exceeds that of blocking. */
+export function tau(falsePassCost) {
+    return 1 / (1 + falsePassCost);
+}
+const isCritical = (r) => r.critical === true;
+const fired = (r) => !r.skipped && r.passed === false;
+/**
+ * Whether a policy rule DECIDES the verdict here, or only advises.
+ *
+ * "A default is not your policy." A shipped threshold — a cost ceiling of
+ * $0.50, a length floor of 50 characters — is our guess about a deployment
+ * we have never seen, and stopping someone's build on it is presumptuous.
+ * A threshold the deployment SET is their decision and gates.
+ *
+ * The distinction is not a list of rule names. For a BUILT-IN policy it is
+ * whether the number the rule compared against is one we chose, which every
+ * result already records as `thresholdSource` on its count evidence (0.9.0);
+ * a policy with no number at all — "the output is empty" — is structural,
+ * has no guess in it, and gates.
+ *
+ * A CUSTOM rule is different: its severity is the deployment's own statement
+ * of how much it matters, made when the rule was deployed. High and critical
+ * gate (they resolve to critical); medium and low advise, which is the
+ * contract `deploy_rule` has always had. An inline rule passed in the call
+ * carries no severity and advises, for the same reason.
+ */
+export function decides(r, defaultsGate) {
+    if (isCritical(r))
+        return true;
+    if (defaultsGate)
+        return true;
+    if (r.origin === 'custom')
+        return false;
+    const ourDefault = (r.evidence ?? []).some((e) => e.type === 'count' && e.threshold !== undefined && (e.thresholdSource ?? 'default') === 'default');
+    return !ourDefault;
+}
+/** The inputs at least one evaluated rule actually read. */
+function inputsSeen(rows) {
+    const seen = new Set();
+    for (const r of rows)
+        if (!r.skipped)
+            for (const n of r.saw ?? [])
+                seen.add(n);
+    return seen;
+}
+/**
+ * The verdict for one evaluation. The weighted mean is never consulted: it
+ * survives as a quality gradient on the score field and is never re-meant.
+ */
+export function compose(result, cfg) {
+    const rows = result.rule_results;
+    const evaluated = result.rules_evaluated ?? rows.filter((r) => !r.skipped).length;
+    if (result.insufficient_data || evaluated === 0) {
+        return { state: 'unknown', passed: false, basis: 'no_rules', by: [], risk: null };
+    }
+    /*
+     * 1. Gates: a policy whose author has already decided — and a JUDGMENT,
+     * for the same reason. Nobody runs a judge by accident: the caller chose
+     * the template, supplied the key and paid for the answer, so a failing
+     * judgment decides rather than being weighed against anything. It also
+     * cannot be weighed: a judgment carries no published error rate until a
+     * measured run exists for its template and model, so the risk layer would
+     * drop it silently and a paid-for "fail" would read as clean.
+     */
+    const gates = rows.filter((r) => fired(r) && ((r.kind === 'policy' && decides(r, cfg.defaultsGate)) || r.kind === 'judgment'));
+    if (gates.length > 0) {
+        return { state: 'fail', passed: false, basis: 'policy_gate', by: gates.map((r) => r.ruleName), risk: null };
+    }
+    /*
+     * 2. Vetoes: an effectively-critical rule that is not a policy. Keyed on
+     * "not a policy" rather than on the two detecting kinds, so a rule built
+     * by hand without metadata — a test double, an embedder's own rule —
+     * still vetoes when it is marked critical. Silently ignoring a critical
+     * rule because it forgot to declare its kind is the failure mode this
+     * composer exists to remove, not one to introduce.
+     */
+    const vetoes = rows.filter((r) => r.kind !== 'policy' && fired(r) && isCritical(r));
+    if (vetoes.length > 0) {
+        return { state: 'fail', passed: false, basis: 'detector_veto', by: vetoes.map((r) => r.ruleName), risk: null };
+    }
+    /*
+     * 3. Asked and could not answer. `not_applicable` is NEVER this: a
+     * trajectory rule with no tool calls was not asked, and treating that as
+     * unknown would make every text-only evaluation unknown, which is worse
+     * than the fail-open it replaces.
+     */
+    const unknown = rows.filter((r) => isCritical(r) && r.skipped === true && r.skipClass !== undefined && r.skipClass !== 'not_applicable');
+    if (unknown.length > 0 && cfg.onCriticalSkipped !== 'pass') {
+        const by = unknown.map((r) => r.ruleName);
+        return cfg.onCriticalSkipped === 'fail'
+            ? { state: 'fail', passed: false, basis: 'critical_unknown', by, risk: null }
+            : { state: 'unknown', passed: false, basis: 'critical_unknown', by, risk: null };
+    }
+    // 4. Evidence the deployment insists on.
+    if (cfg.requiredEvidence.length > 0) {
+        const seen = inputsSeen(rows);
+        const missing = cfg.requiredEvidence.filter((n) => !seen.has(n));
+        if (missing.length > 0) {
+            return { state: 'unknown', passed: false, basis: 'required_evidence_missing', by: [...missing], risk: null };
+        }
+    }
+    // 5. Everything that carries a published error rate, as one probability.
+    const risk = riskEstimate(result, cfg.prior, cfg.priorMode);
+    if (risk === null) {
+        return { state: 'pass', passed: true, basis: 'clean', by: [], risk: null };
+    }
+    const t = tau(cfg.falsePassCost);
+    const confidence = risk.lo <= t && t <= risk.hi ? 'marginal' : 'decisive';
+    if (risk.pBad > t) {
+        const by = Object.entries(risk.perClass)
+            .filter(([, q]) => q !== null && q !== undefined && q > 0.5)
+            .map(([cls]) => cls);
+        return { state: 'fail', passed: false, basis: 'risk_over_loss', by, risk, confidence };
+    }
+    return { state: 'pass', passed: true, basis: 'clean', by: [], risk, confidence };
+}
+/**
+ * The sentences a reader needs that the verdict alone does not carry.
+ *
+ * The one that must exist: when a rule visibly FIRED and the verdict still
+ * passed, say why and name the one setting that would change it. Without
+ * it, "cost_under_threshold failed" beside "passed: true" reads as a bug,
+ * and that is the first thing a builder who never opens a config file will
+ * meet.
+ */
+export function interpretations(result, verdict, cfg) {
+    const out = [];
+    for (const r of result.rule_results) {
+        if (!fired(r))
+            continue;
+        if (verdict.by.includes(r.ruleName))
+            continue;
+        if (r.kind === 'policy' && !decides(r, cfg.defaultsGate)) {
+            out.push({
+                severity: 'warn',
+                addressee: 'operator',
+                rule: r.ruleName,
+                text: `${r.ruleName} failed against a threshold Iris ships, not one you set, so it did not decide this verdict. Set it in your configuration to make it a gate, or set eval.defaultsGate to true to make every shipped default gate.`,
+                configKey: 'eval.defaultsGate',
+            });
+            continue;
+        }
+        if (verdict.state === 'pass') {
+            out.push({
+                severity: 'note',
+                addressee: 'operator',
+                rule: r.ruleName,
+                text: `${r.ruleName} failed but the verdict passed: on its published accuracy this rule alone does not carry the risk past your loss threshold. Lower eval.falsePassCost to block on weaker evidence.`,
+                configKey: 'eval.falsePassCost',
+            });
+        }
+    }
+    if (verdict.basis === 'critical_unknown') {
+        out.push({
+            severity: 'block',
+            addressee: 'operator',
+            text: `A critical check was asked and could not answer (${verdict.by.join(', ')}), so this verdict is unknown rather than clean. Set eval.onCriticalSkipped to "pass" to accept that risk, or to "fail" to treat it as a failure.`,
+            configKey: 'eval.onCriticalSkipped',
+        });
+    }
+    if (verdict.confidence === 'marginal') {
+        out.push({
+            severity: 'note',
+            addressee: 'operator',
+            text: 'The credible interval on this risk estimate straddles your threshold, so this verdict could go either way on the evidence available. Treat it as a close call rather than a clear one.',
+        });
+    }
+    return out;
+}

package/dist/eval/criticality.d.ts CHANGED Viewed

@@ -6,6 +6,13 @@ export interface CriticalityOverrides {
     criticalRules?: string[];
     /** Built-in rule names demoted from critical. */
     nonCriticalRules?: string[];
+    composer?: 'risk' | 'legacy';
+    falsePassCost?: number;
+    onCriticalSkipped?: 'unknown' | 'fail' | 'pass';
+    requiredEvidence?: string[];
+    defaultsGate?: boolean;
+    prior?: number;
+    priorMode?: 'per-output' | 'per-class';
 }
 export interface EffectiveCriticality {
     critical: boolean;

package/dist/eval/decision-moment.js CHANGED Viewed

@@ -104,14 +104,34 @@ function computeRuleSnapshot(evals) {
     }
     return { failed, skipped, passedCount, totalCount };
 }
+/*
+ * The moment SHOWS the verdict each evaluation reached; it does not compute
+ * a second one.
+ *
+ * It used to count failed rules: no failures meant pass, no passes meant
+ * fail, anything else meant partial. From 0.10.0 those two answers diverge,
+ * and the divergence is the whole point of the composer. An evaluation can
+ * pass with a rule visibly failed — a shipped default that only advises, or
+ * evidence too weak to carry the risk past the deployment's loss threshold
+ * — and the old arithmetic would have called that "partial", contradicting
+ * the verdict the tool returned for the same evaluation.
+ *
+ * "partial" now means what it says: several evaluations of one trace and
+ * they did not agree. An `unknown` verdict reads as unevaluated, because
+ * that is what it is — asked, and unable to answer.
+ */
 function computeVerdict(evals, snapshot) {
     if (evals.length === 0)
         return 'unevaluated';
     if (snapshot.totalCount - snapshot.skipped.length === 0)
         return 'unevaluated';
-    if (snapshot.failed.length === 0)
+    const decided = evals.filter((e) => e.verdict === undefined || e.verdict.state !== 'unknown');
+    if (decided.length === 0)
+        return 'unevaluated';
+    const passed = decided.filter((e) => e.passed).length;
+    if (passed === decided.length)
         return 'pass';
-    if (snapshot.passedCount === 0)
+    if (passed === 0)
         return 'fail';
     return 'partial';
 }
@@ -122,8 +142,17 @@ function computeOverallScore(evals) {
     return sum / evals.length;
 }
 function classifySignificance({ trace, evals, ruleSnapshot, verdict, }) {
-    // 1. Safety violation — any safety rule failed → top priority.
-    const safetyFailed = ruleSnapshot.failed.filter((name) => SAFETY_RULE_NAMES.has(name));
+    /*
+     * 1. Safety violation — a rule that VETOES failed, or a safety-bundle rule
+     * did. Bundle membership alone was the old test, and it is the weaker
+     * one: from 0.10.0 which rules veto is the deployment's call
+     * (eval.criticalRules), so a rule promoted to critical outside the safety
+     * bundle is exactly as serious and used to rank as a plain failure. The
+     * bundle list stays as well, because a safety rule that a deployment
+     * DEMOTED still describes what it found.
+     */
+    const vetoed = new Set(evals.flatMap((e) => e.rule_results.filter((r) => !r.skipped && r.passed === false && r.role === 'veto').map((r) => r.ruleName)));
+    const safetyFailed = ruleSnapshot.failed.filter((name) => SAFETY_RULE_NAMES.has(name) || vetoed.has(name));
     if (safetyFailed.length > 0) {
         return {
             kind: 'safety-violation',

package/dist/eval/engine.d.ts CHANGED Viewed

@@ -46,6 +46,8 @@ export declare class EvalEngine {
      * promotion or demotion cannot apply on one code path and not another.
      */
     private criticality;
+    /** The verdict's six defaults, resolved once from the config this engine was built with. */
+    private compose;
     /**
      * `criticalityOverrides` are `config.eval` — the criticalRules /
      * nonCriticalRules lists. Validated here as well as in loadConfig, so an
@@ -53,6 +55,7 @@ export declare class EvalEngine {
      * misspelled rule name.
      */
     constructor(threshold?: number, ruleThresholds?: Record<string, unknown>, criticalityOverrides?: CriticalityOverrides);
+    private decide;
     /** The effective criticality of one rule under this engine's config. Read by the rule roster surfaces. */
     effectiveCriticality(rule: EvalRule): EffectiveCriticality;
     /**
@@ -72,7 +75,7 @@ export declare class EvalEngine {
     unregisterRule(ruleId: string): boolean;
     /** Whether a deployed rule id is currently registered (and therefore firing). */
     hasRule(ruleId: string): boolean;
-    evaluate(evalType: EvalType, context: EvalContext, customRules?: CustomRuleDefinition[]): EvalResult;
+    evaluate(evalType: EvalType, context: EvalContext, customRules?: CustomRuleDefinition[]): Promise<EvalResult>;
     /**
      * eval_type="all" (#370): every built-in bundle, each with the deployed
      * rules registered under it, plus the rules deployed under "custom" and
@@ -82,7 +85,7 @@ export declare class EvalEngine {
      * that ran (weighted score against the threshold, critical veto across
      * all bundles); `categories` carries the same arithmetic per bundle.
      */
-    evaluateAll(context: EvalContext, customRules?: CustomRuleDefinition[]): EvalResult;
+    evaluateAll(context: EvalContext, customRules?: CustomRuleDefinition[]): Promise<EvalResult>;
     private run;
     /**
      * Weighted average over the rules that ran, plus the critical veto.

package/dist/eval/engine.js CHANGED Viewed

@@ -1,5 +1,6 @@
 import { getRulesForType, createCustomRule } from './rules/index.js';
 import { criticalityResolver } from './criticality.js';
+import { compose, interpretations, DEFAULT_COMPOSE } from './compose.js';
 import { inputsPresent, stampRuleResult } from './stamp.js';
 import { buildProvenance, configHash, deriveCoverage, deriveVerdict, rulesetHash } from './verdict.js';
 import { PKG_VERSION } from '../config/defaults.js';
@@ -50,6 +51,8 @@ export class EvalEngine {
      * promotion or demotion cannot apply on one code path and not another.
      */
     criticality;
+    /** The verdict's six defaults, resolved once from the config this engine was built with. */
+    compose;
     /**
      * `criticalityOverrides` are `config.eval` — the criticalRules /
      * nonCriticalRules lists. Validated here as well as in loadConfig, so an
@@ -61,6 +64,33 @@ export class EvalEngine {
         this.ruleThresholds = ruleThresholds;
         this.criticalityOverrides = criticalityOverrides;
         this.criticality = criticalityResolver(criticalityOverrides);
+        this.compose = {
+            composer: criticalityOverrides?.composer ?? DEFAULT_COMPOSE.composer,
+            falsePassCost: criticalityOverrides?.falsePassCost ?? DEFAULT_COMPOSE.falsePassCost,
+            onCriticalSkipped: criticalityOverrides?.onCriticalSkipped ?? DEFAULT_COMPOSE.onCriticalSkipped,
+            requiredEvidence: criticalityOverrides?.requiredEvidence ?? DEFAULT_COMPOSE.requiredEvidence,
+            defaultsGate: criticalityOverrides?.defaultsGate ?? DEFAULT_COMPOSE.defaultsGate,
+            prior: criticalityOverrides?.prior ?? DEFAULT_COMPOSE.prior,
+            priorMode: criticalityOverrides?.priorMode ?? DEFAULT_COMPOSE.priorMode,
+        };
+    }
+    /*
+     * The verdict, and `passed` with it.
+     *
+     * From 0.10.0 `passed` IS `verdict.state === 'pass'` — one definition, on
+     * every surface. The weighted `score` survives untouched as a quality
+     * gradient over the rules that ran, and is never re-meant: a reader who
+     * was using it as a gradient keeps it, and a reader who was using it as a
+     * safety signal was reading a number that arc zero measured as inert.
+     */
+    decide(result) {
+        const verdict = this.compose.composer === 'legacy' ? deriveVerdict(result, this.threshold) : compose(result, this.compose);
+        result.verdict = verdict;
+        result.passed = verdict.passed;
+        const notes = interpretations(result, verdict, this.compose);
+        if (notes.length > 0)
+            result.interpretations = notes;
+        return result;
     }
     /** The effective criticality of one rule under this engine's config. Read by the rule roster surfaces. */
     effectiveCriticality(rule) {
@@ -109,7 +139,7 @@ export class EvalEngine {
     hasRule(ruleId) {
         return this.rulesById.has(ruleId);
     }
-    evaluate(evalType, context, customRules) {
+    async evaluate(evalType, context, customRules) {
         /*
          * Inline custom_rules are ADDITIVE, which is what evaluate_output's
          * description promises in two places: "fires REGARDLESS of eval_type"
@@ -146,7 +176,7 @@ export class EvalEngine {
      * that ran (weighted score against the threshold, critical veto across
      * all bundles); `categories` carries the same arithmetic per bundle.
      */
-    evaluateAll(context, customRules) {
+    async evaluateAll(context, customRules) {
         const rules = [];
         const categories = [];
         for (const type of ALL_EVAL_TYPES) {
@@ -161,7 +191,16 @@ export class EvalEngine {
         }
         return this.run('all', rules, categories, context);
     }
-    run(evalType, rules, categories, context) {
+    /*
+     * Async from 0.10.0. Nothing it awaits yet: every rule the package ships
+     * is synchronous, and `EvalRule.evaluate` stays synchronous so the type
+     * system keeps proving that a deterministic rule cannot reach the network
+     * — which is what makes "evaluate_output never spends" a compile-time
+     * fact rather than a test. The signature moves first, in one mechanical
+     * change, so the judgment rule that DOES call a provider can be added
+     * without re-touching every caller a second time.
+     */
+    async run(evalType, rules, categories, context) {
         // Merge system-level thresholds into customConfig (user-provided values take precedence)
         if (this.ruleThresholds) {
             context = {
@@ -194,10 +233,41 @@ export class EvalEngine {
          * rule it carries.
          */
         const evalContext = { ...context, regexBudget: { breaches: 0 } };
-        const ruleResults = rules.map((rule, i) => {
-            const raw = rule.evaluate(evalContext);
+        /*
+         * Sequential, and it must stay sequential when a rule becomes awaitable:
+         * every rule in one evaluation shares the regex circuit breaker above,
+         * and running them concurrently would race the breach count that bounds
+         * a hostile output.
+         */
+        const ruleResults = [];
+        for (const [i, rule] of rules.entries()) {
+            /*
+             * A judgment rule calls a provider and costs money. It runs only when
+             * the caller has said this evaluation may spend — which the free
+             * evaluation path never does. Enforced here, on the one path every
+             * evaluation takes, so no tool can forget it and no future rule can
+             * quietly opt itself in.
+             */
+            const raw = rule.kind === 'judgment' && evalContext.allowPaid !== true
+                ? {
+                    ruleName: rule.name,
+                    passed: false,
+                    score: 0,
+                    message: 'Judgment rules are not run on this path: it may not call a paid provider.',
+                    skipped: true,
+                    skipReason: 'this evaluation may not spend (context.allowPaid is not set)',
+                }
+                : rule.evaluate(evalContext);
             const ruleId = this.idByRule.get(rule);
-            const category = categories?.[i];
+            /*
+             * The bundle this rule ran under. `categories` is only supplied for
+             * eval_type="all"; for a single bundle the rule's own evalType is the
+             * answer and is just as true. It used to be left off, which meant a
+             * single-bundle call could not tell a custom rule from a built-in one
+             * — and the composer needs that, because a custom rule's severity is
+             * the deployment's own statement of how much it matters.
+             */
+            const category = categories?.[i] ?? (evalType === 'all' ? rule.evalType : evalType);
             /*
              * Every result says whether THIS rule vetoes and who decided that.
              * Without it, a reader holding a failed evaluation cannot tell a
@@ -221,7 +291,7 @@ export class EvalEngine {
              * so no surface can show a result without its receipt. It changes no
              * verdict: summarize() below still decides passed exactly as before.
              */
-            return {
+            ruleResults.push({
                 ruleName,
                 ...(ruleId !== undefined ? { ruleId } : {}),
                 ...(category !== undefined ? { category } : {}),
@@ -229,8 +299,8 @@ export class EvalEngine {
                 criticalSource: source,
                 ...rest,
                 ...stampRuleResult(rule, raw, context, effective),
-            };
-        });
+            });
+        }
         const overall = this.summarize(rules, ruleResults);
         const perCategory = categories ? this.categorize(rules, ruleResults, categories) : undefined;
         /*
@@ -283,8 +353,7 @@ export class EvalEngine {
                 coverage,
                 provenance,
             };
-            unknown.verdict = deriveVerdict(unknown, this.threshold);
-            return unknown;
+            return this.decide(unknown);
         }
         const suggestions = [];
         for (const result of ruleResults) {
@@ -333,8 +402,7 @@ export class EvalEngine {
             coverage,
             provenance,
         };
-        result.verdict = deriveVerdict(result, this.threshold);
-        return result;
+        return this.decide(result);
     }
     /**
      * Weighted average over the rules that ran, plus the critical veto.

package/dist/eval/llm-judge/evaluator.d.ts CHANGED Viewed

@@ -26,8 +26,28 @@ export interface LLMJudgeEvaluateParams {
     maxInputTokensEstimate?: number;
 }
 export interface LLMJudgeEvaluationResult {
+    /**
+     * The verdict, and it is the THRESHOLD's, not the model's.
+     *
+     * Until 0.10.0 the model's own `passed` boolean won whenever it supplied
+     * one, and the template's threshold was a fallback the product rarely
+     * reached. That let a judge return `score: 0.2` with `passed: true` and
+     * be believed — a scoring rubric whose score did not decide anything.
+     * Now the score is the measurement and the threshold is the rule.
+     */
     passed: boolean;
     score: number;
+    /** The threshold the score was read against, so a reader can check the arithmetic. */
+    passThreshold: number;
+    /** What the model said about passing, when it said anything. Recorded, never obeyed. */
+    selfReportedPass?: boolean;
+    /**
+     * True when the model's own boolean disagrees with the threshold verdict.
+     * Worth surfacing: a judge that scores 0.95 and says "fail", or scores
+     * 0.2 and says "pass", is telling you its rubric and its judgement have
+     * come apart on this output.
+     */
+    disagreement?: boolean;
     rationale: string;
     dimensions: Record<string, number>;
     model: string;

package/dist/eval/llm-judge/evaluator.js CHANGED Viewed

@@ -151,10 +151,19 @@ export async function evaluateWithLLMJudge(params) {
         latencyMs += raw.latencyMs;
         parsed = parseJudgeResponse(raw.content);
     }
-    const passed = parsed.passed ?? parsed.score >= template.passThreshold;
+    /*
+     * The threshold decides. The model's own boolean is evidence about the
+     * model, not about the output, and it is recorded beside the verdict
+     * rather than substituted for it.
+     */
+    const passed = parsed.score >= template.passThreshold;
+    const disagreement = parsed.passed !== undefined && parsed.passed !== passed;
     const costUsd = estimateCostUsd(params.model, inputTokens, outputTokens);
     return {
         passed,
+        passThreshold: template.passThreshold,
+        ...(parsed.passed !== undefined ? { selfReportedPass: parsed.passed } : {}),
+        ...(disagreement ? { disagreement: true } : {}),
         score: parsed.score,
         rationale: parsed.rationale,
         dimensions: parsed.dimensions,