npm - @iris-eval/mcp-server - Versions diffs - 0.8.2 → 0.10.0 - Mend

@iris-eval/mcp-server 0.8.2 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (131) hide show

package/README.md +9 -2
package/dist/capabilities.d.ts +64 -0
package/dist/capabilities.js +65 -0
package/dist/config/defaults.js +17 -0
package/dist/custom-rule-store.d.ts +4 -0
package/dist/custom-rule-store.js +8 -3
package/dist/dashboard/assets/{index-CyzO6OC7.js → index-CeJbaq6m.js} +1 -1
package/dist/dashboard/index.html +1 -1
package/dist/dashboard/routes/capabilities.d.ts +3 -0
package/dist/dashboard/routes/capabilities.js +11 -0
package/dist/dashboard/routes/health.d.ts +5 -1
package/dist/dashboard/routes/health.js +15 -3
package/dist/dashboard/routes/rules.js +4 -1
package/dist/dashboard/routes/traces.d.ts +3 -0
package/dist/dashboard/routes/traces.js +11 -30
package/dist/dashboard/seed-demo-data.js +1 -1
package/dist/dashboard/server.d.ts +2 -0
package/dist/dashboard/server.js +6 -2
package/dist/eval/accuracy.d.ts +41 -0
package/dist/eval/accuracy.js +97 -0
package/dist/eval/citation-verify/verifier.d.ts +16 -1
package/dist/eval/citation-verify/verifier.js +14 -4
package/dist/eval/compose.d.ts +57 -0
package/dist/eval/compose.js +179 -0
package/dist/eval/criticality.d.ts +15 -1
package/dist/eval/criticality.js +6 -0
package/dist/eval/decision-moment.js +33 -4
package/dist/eval/dormant.d.ts +4 -0
package/dist/eval/dormant.js +22 -0
package/dist/eval/engine.d.ts +6 -2
package/dist/eval/engine.js +126 -12
package/dist/eval/failure-classes.d.ts +8 -0
package/dist/eval/failure-classes.js +18 -0
package/dist/eval/llm-judge/evaluator.d.ts +30 -0
package/dist/eval/llm-judge/evaluator.js +26 -2
package/dist/eval/published-accuracy.d.ts +230 -0
package/dist/eval/published-accuracy.js +86 -0
package/dist/eval/questions.d.ts +12 -0
package/dist/eval/questions.js +14 -0
package/dist/eval/response-schema.d.ts +652 -0
package/dist/eval/response-schema.js +130 -0
package/dist/eval/response.d.ts +12 -0
package/dist/eval/response.js +30 -0
package/dist/eval/risk.d.ts +60 -0
package/dist/eval/risk.js +187 -0
package/dist/eval/rules/completeness.js +36 -1
package/dist/eval/rules/cost.d.ts +2 -2
package/dist/eval/rules/cost.js +50 -6
package/dist/eval/rules/custom.d.ts +0 -12
package/dist/eval/rules/custom.js +22 -0
package/dist/eval/rules/relevance.js +23 -2
package/dist/eval/rules/safety.d.ts +6 -2
package/dist/eval/rules/safety.js +224 -51
package/dist/eval/seeded-random.d.ts +4 -0
package/dist/eval/seeded-random.js +36 -0
package/dist/eval/stamp.d.ts +14 -0
package/dist/eval/stamp.js +89 -0
package/dist/eval/stats.d.ts +33 -0
package/dist/eval/stats.js +109 -0
package/dist/eval/text/checksums.d.ts +23 -0
package/dist/eval/text/checksums.js +97 -0
package/dist/eval/text/normalise.d.ts +30 -0
package/dist/eval/text/normalise.js +265 -0
package/dist/eval/text/sentences.d.ts +15 -0
package/dist/eval/text/sentences.js +149 -0
package/dist/eval/verdict.d.ts +34 -0
package/dist/eval/verdict.js +131 -0
package/dist/index.js +5 -28
package/dist/instructions.d.ts +17 -0
package/dist/instructions.js +53 -0
package/dist/judge-enablement.d.ts +34 -0
package/dist/judge-enablement.js +78 -0
package/dist/judge-enablement.json +10 -0
package/dist/preferences.d.ts +1 -1
package/dist/prompts.d.ts +3 -0
package/dist/prompts.js +29 -0
package/dist/resources/index.d.ts +5 -2
package/dist/resources/index.js +65 -5
package/dist/resources/uris.d.ts +12 -0
package/dist/resources/uris.js +24 -0
package/dist/retention.d.ts +20 -0
package/dist/retention.js +44 -0
package/dist/self-test.d.ts +1 -0
package/dist/self-test.js +17 -3
package/dist/server.d.ts +10 -1
package/dist/server.js +34 -7
package/dist/storage/index.js +1 -1
package/dist/storage/migrations/007-eval-provenance.d.ts +3 -0
package/dist/storage/migrations/007-eval-provenance.js +30 -0
package/dist/storage/migrations/index.js +24 -4
package/dist/storage/sqlite-adapter.d.ts +26 -1
package/dist/storage/sqlite-adapter.js +149 -15
package/dist/tools/delete-rule.d.ts +8 -0
package/dist/tools/delete-rule.js +30 -38
package/dist/tools/delete-trace.d.ts +5 -0
package/dist/tools/delete-trace.js +24 -27
package/dist/tools/deploy-rule.d.ts +13 -1
package/dist/tools/deploy-rule.js +37 -34
package/dist/tools/describe.d.ts +20 -0
package/dist/tools/describe.js +36 -0
package/dist/tools/errors.d.ts +36 -0
package/dist/tools/errors.js +134 -0
package/dist/tools/evaluate-output.d.ts +8 -1
package/dist/tools/evaluate-output.js +39 -60
package/dist/tools/evaluate-with-llm-judge.d.ts +34 -0
package/dist/tools/evaluate-with-llm-judge.js +124 -69
package/dist/tools/get-traces.d.ts +9 -0
package/dist/tools/get-traces.js +29 -28
package/dist/tools/index.d.ts +8 -0
package/dist/tools/index.js +22 -1
package/dist/tools/list-rules.d.ts +13 -0
package/dist/tools/list-rules.js +43 -46
package/dist/tools/log-trace.d.ts +4 -0
package/dist/tools/log-trace.js +31 -29
package/dist/tools/respond.d.ts +42 -0
package/dist/tools/respond.js +90 -0
package/dist/tools/strict-input.js +1 -1
package/dist/tools/trace-link.d.ts +2 -0
package/dist/tools/trace-link.js +13 -2
package/dist/tools/verify-citations.d.ts +18 -2
package/dist/tools/verify-citations.js +122 -96
package/dist/types/config.d.ts +44 -0
package/dist/types/eval.d.ts +309 -0
package/dist/types/eval.js +2 -1
package/dist/types/query.d.ts +2 -0
package/package.json +1 -1
package/server.json +2 -2
package/dist/resources/dashboard-summary.d.ts +0 -3
package/dist/resources/dashboard-summary.js +0 -16
package/dist/resources/trace-detail.d.ts +0 -3
package/dist/resources/trace-detail.js +0 -30

package/dist/dashboard/index.html CHANGED Viewed

@@ -6,7 +6,7 @@
     <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <!-- Stop shipping agents on vibes is filled from .claims.json brand.tagline at build time (vite.config.ts) — never restate the tagline here. -->
     <title>Iris — Stop shipping agents on vibes</title>
-    <script type="module" crossorigin src="/assets/index-CyzO6OC7.js"></script>
+    <script type="module" crossorigin src="/assets/index-CeJbaq6m.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-D0cFfBqn.css">
   </head>
   <body>

package/dist/dashboard/routes/capabilities.d.ts ADDED Viewed

@@ -0,0 +1,3 @@
+import { Router } from 'express';
+import type { Capabilities } from '../../capabilities.js';
+export declare function registerCapabilitiesRoutes(router: Router, build: () => Capabilities): void;

package/dist/dashboard/routes/capabilities.js ADDED Viewed

@@ -0,0 +1,11 @@
+/*
+ * GET /api/v1/capabilities — the same object iris://capabilities serves,
+ * for a caller on the HTTP path (an agent driving Iris through the ingest
+ * endpoint, a dashboard, a health probe that wants more than "ok").
+ * Provider name only, never a key.
+ */
+export function registerCapabilitiesRoutes(router, build) {
+    router.get('/capabilities', (_req, res) => {
+        res.json(build());
+    });
+}

package/dist/dashboard/routes/health.d.ts CHANGED Viewed

@@ -1,3 +1,7 @@
 import { Router } from 'express';
 import type { IStorageAdapter } from '../../types/query.js';
-export declare function registerHealthRoutes(router: Router, storage?: IStorageAdapter, version?: string): void;
+export interface HealthOptions {
+    /** `demo` when serving the disposable demo database. */
+    mode?: 'real' | 'demo';
+}
+export declare function registerHealthRoutes(router: Router, storage?: IStorageAdapter, version?: string, options?: HealthOptions): void;

package/dist/dashboard/routes/health.js CHANGED Viewed

@@ -1,9 +1,19 @@
 import { LOCAL_TENANT } from '../../types/tenant.js';
+import { judgeState } from '../../judge-enablement.js';
 const startTime = Date.now();
-export function registerHealthRoutes(router, storage, version) {
+export function registerHealthRoutes(router, storage, version, options) {
     const serverVersion = version ?? 'unknown';
+    const mode = options?.mode ?? 'real';
     router.get('/health', async (_req, res) => {
         const uptime_seconds = Math.floor((Date.now() - startTime) / 1000);
+        /*
+         * The judge state, provider name only — never the key. Read per
+         * request rather than at boot so a test (or an operator) that sets
+         * the variable in this process sees it here; the process a client
+         * spawns has a fixed environment anyway, so both reads agree there.
+         */
+        const judge = judgeState();
+        const judgeField = { enabled: judge.enabled, provider: judge.provider };
         if (storage) {
             try {
                 /* Health probes use LOCAL_TENANT directly — the health endpoint
@@ -26,14 +36,16 @@ export function registerHealthRoutes(router, storage, version) {
                     uptime_seconds,
                     trace_count: total,
                     storage: 'connected',
+                    judge: judgeField,
+                    mode,
                 });
             }
             catch {
-                res.status(503).json({ status: 'degraded', version: serverVersion, uptime_seconds, storage: 'disconnected' });
+                res.status(503).json({ status: 'degraded', version: serverVersion, uptime_seconds, storage: 'disconnected', judge: judgeField, mode });
             }
         }
         else {
-            res.json({ status: 'ok', version: serverVersion, uptime_seconds });
+            res.json({ status: 'ok', version: serverVersion, uptime_seconds, judge: judgeField, mode });
         }
     });
 }

package/dist/dashboard/routes/rules.js CHANGED Viewed

@@ -110,7 +110,9 @@ export function registerRuleRoutes(router, storage, opts) {
     router.get('/rules/custom', (req, res) => {
         const tenantId = requireTenant(req);
         const rules = opts.customRuleStore.list(tenantId);
-        res.json({ rules });
+        // Quarantined entries ride along: they are on disk, never registered,
+        // and a gating one is named on every verdict as coverage.dormant.
+        res.json({ rules, quarantined: opts.customRuleStore.quarantined(tenantId) });
     });
     router.post('/rules/custom', async (req, res) => {
         try {
@@ -133,6 +135,7 @@ export function registerRuleRoutes(router, storage, opts) {
              */
             const replaced = retireSameNamedRules(opts.customRuleStore, opts.evalEngine, tenantId, input.name, input.replace, 'local');
             const rule = opts.customRuleStore.deploy(tenantId, {
+                replaces: replaced.map((r) => r.id),
                 name: input.name,
                 description: input.description,
                 evalType: input.evalType,

package/dist/dashboard/routes/traces.d.ts CHANGED Viewed

@@ -1,4 +1,5 @@
 import { Router } from 'express';
+import type { CustomRuleStore } from '../../custom-rule-store.js';
 import type { IStorageAdapter } from '../../types/query.js';
 import type { EvalEngine } from '../../eval/engine.js';
 export interface TraceRouteOptions {
@@ -10,5 +11,7 @@ export interface TraceRouteOptions {
      * as a success.
      */
     evalEngine?: EvalEngine;
+    /** The custom-rule store, so an evaluation over HTTP carries coverage.dormant like the tool does. */
+    customRuleStore?: CustomRuleStore;
 }
 export declare function registerTraceRoutes(router: Router, storage: IStorageAdapter, options?: TraceRouteOptions): void;

package/dist/dashboard/routes/traces.js CHANGED Viewed

@@ -1,3 +1,5 @@
+import { toEvaluationResponse } from '../../eval/response.js';
+import { dormantRulesFrom } from '../../eval/dormant.js';
 import { requireTenant } from '../../middleware/tenant.js';
 import { generateTraceId, generateSpanId } from '../../utils/ids.js';
 import { bestEffortExport } from '../../otel/lazy.js';
@@ -80,42 +82,21 @@ export function registerTraceRoutes(router, storage, options) {
             const evalTypeOmitted = body.eval_type === undefined;
             const evalType = body.eval_type ?? DEFAULT_EVAL_TYPE;
             const evaluation = evalType === 'all'
-                ? options.evalEngine.evaluateAll(context)
-                : options.evalEngine.evaluate(evalType, context);
+                ? await options.evalEngine.evaluateAll(context)
+                : await options.evalEngine.evaluate(evalType, context);
             evaluation.trace_id = traceId;
             await storage.insertEvalResult(tenantId, evaluation);
+            // The same serializer as evaluate_output (src/eval/response.ts): the
+            // veto reason, the skipped criticals, the verdict basis, coverage and
+            // provenance travel the ingest path exactly as they travel the tool.
             res.status(201).json({
                 trace_id: traceId,
                 status: 'stored',
-                evaluation: {
-                    id: evaluation.id,
-                    eval_type: evaluation.eval_type,
-                    score: evaluation.score,
-                    passed: evaluation.passed,
-                    rule_results: evaluation.rule_results,
-                    suggestions: evaluation.suggestions,
-                    rules_evaluated: evaluation.rules_evaluated,
-                    rules_skipped: evaluation.rules_skipped,
-                    insufficient_data: evaluation.insufficient_data,
-                    /*
-                     * The veto reason travels the ingest path too. Omitting it left an
-                     * HTTP caller seeing passed:false beside a high score with no way
-                     * to learn that a critical rule — not the weighted average —
-                     * produced the verdict. Absent when nothing vetoed.
-                     */
-                    ...(evaluation.critical_failures?.length
-                        ? { critical_failures: evaluation.critical_failures }
-                        : {}),
-                    // The other half of the veto contract, same as evaluate_output: a
-                    // critical rule that SKIPPED did not judge the output and cannot
-                    // veto, so a fail-closed gate needs to see it named.
-                    ...(evaluation.critical_skipped?.length
-                        ? { critical_skipped: evaluation.critical_skipped }
-                        : {}),
-                    // Per-bundle breakdown — eval_type="all" only.
-                    ...(evaluation.categories ? { categories: evaluation.categories } : {}),
+                evaluation: toEvaluationResponse(evaluation, {
+                    traceId,
+                    dormant: options?.customRuleStore ? dormantRulesFrom(options.customRuleStore.quarantined(tenantId)) : undefined,
                     ...(evalTypeOmitted ? { note: DEFAULT_EVAL_TYPE_NOTE } : {}),
-                },
+                }),
             });
         }
         catch (err) {

package/dist/dashboard/seed-demo-data.js CHANGED Viewed

@@ -574,7 +574,7 @@ function simulateCostEval(costUsd, tokenUsage, shouldPass) {
             : `Cost ($${costUsd.toFixed(4)}) exceeds threshold ($${threshold.toFixed(4)})`,
     };
     const r2 = {
-        ruleName: 'token_efficiency',
+        ruleName: 'verbosity_ratio',
         passed: ratio <= maxRatio,
         score: ratio <= maxRatio ? 1 : Math.max(0, 1 - (ratio - maxRatio) / maxRatio),
         message: ratio <= maxRatio

package/dist/dashboard/server.d.ts CHANGED Viewed

@@ -14,5 +14,7 @@ export interface DashboardServerOptions {
     customRuleStore?: CustomRuleStore;
     evalEngine?: EvalEngine;
     preferenceStore?: PreferenceStore;
+    /**  when serving the disposable demo database; reported on /api/v1/capabilities. */
+    mode?: 'real' | 'demo';
 }
 export declare function createDashboardServer(storage: IStorageAdapter, config: IrisConfig, logger: Logger, options?: DashboardServerOptions): DashboardServer;

package/dist/dashboard/server.js CHANGED Viewed

@@ -4,6 +4,8 @@ import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
 import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
 import { irisHome } from '../utils/iris-home.js';
+import { buildCapabilities } from '../capabilities.js';
+import { registerCapabilitiesRoutes } from './routes/capabilities.js';
 import { createAuthMiddleware } from '../middleware/auth.js';
 import { createCorsMiddleware } from '../middleware/cors.js';
 import { createErrorHandler } from '../middleware/error-handler.js';
@@ -75,12 +77,14 @@ export function createDashboardServer(storage, config, logger, options) {
     // API routes with rate limiting
     const router = express.Router();
     router.use(createApiRateLimiter(config));
-    registerTraceRoutes(router, storage, { evalEngine: options?.evalEngine });
+    registerTraceRoutes(router, storage, { evalEngine: options?.evalEngine, customRuleStore: options?.customRuleStore });
     registerSummaryRoutes(router, storage);
     registerEvaluationRoutes(router, storage);
     registerEvalStatsRoutes(router, storage);
     registerFilterRoutes(router, storage);
-    registerHealthRoutes(router, storage, config.server.version);
+    registerHealthRoutes(router, storage, config.server.version, { mode: options?.mode });
+    // The same object iris://capabilities serves, for the HTTP path.
+    registerCapabilitiesRoutes(router, () => buildCapabilities({ config, evalEngine: options?.evalEngine, customRuleStore: options?.customRuleStore, mode: options?.mode }));
     registerMomentRoutes(router, storage);
     registerFailureRoutes(router, storage);
     if (options?.customRuleStore && options?.evalEngine) {

package/dist/eval/accuracy.d.ts ADDED Viewed

@@ -0,0 +1,41 @@
+import { type Confusion } from './stats.js';
+export interface Interval {
+    point: number;
+    lo: number;
+    hi: number;
+}
+export interface PublishedRuleAccuracy extends Confusion {
+    n: number;
+    precision: number | null;
+    recall: number | null;
+    f1: number | null;
+    ci95: {
+        precision: readonly [number, number] | null;
+        recall: readonly [number, number] | null;
+        f1: readonly [number, number] | null;
+    };
+}
+export interface PublishedProvenance {
+    corpusVersion: string;
+    release: string;
+    labelling: 'same-model' | 'human-verified';
+}
+export declare const DEFAULT_PREVALENCE = 0.5;
+export declare const INTERVAL_DRAWS = 2000;
+export declare function publishedProvenance(): PublishedProvenance;
+/** The published numbers for a built-in rule, or null for a rule with no family (a custom rule, a rule added before its proof). */
+export declare function publishedAccuracyFor(ruleName: string): PublishedRuleAccuracy | null;
+/** Every rule name with published numbers, in the order the proof emitted them. */
+export declare function publishedRuleNames(): string[];
+/**
+ * PPV at a prevalence with a 95% credible interval, for a rule that FIRED.
+ * Null when the rule has no published family or its family has no positives
+ * or no negatives (nothing to estimate from).
+ */
+export declare function ppvInterval(ruleName: string, prevalence?: number): Interval | null;
+/** P(violation | the rule did not fire) at a prevalence with a 95% credible interval, for a rule that did NOT fire. */
+export declare function missRateInterval(ruleName: string, prevalence?: number): Interval | null;
+/** PPV at several prevalences — the field-prevalence table a reader needs beside a published precision. */
+export declare function ppvAt(ruleName: string, prevalences?: readonly number[]): Record<string, number | null>;
+/** Test hook: clear the memo so a seeded interval can be recomputed. */
+export declare function resetAccuracyMemo(): void;

package/dist/eval/accuracy.js ADDED Viewed

@@ -0,0 +1,97 @@
+/*
+ * The published accuracy, read at runtime.
+ *
+ * `npm run proof` measures every built-in rule on its labelled family and
+ * writes the numbers to proof/results.json — and, since 0.9.0, to
+ * src/eval/published-accuracy.ts, a generated module that ships inside the
+ * package (the npm `files` list carries dist/ only, so a runtime read of
+ * proof/ would find nothing). `npm run proof -- --check` diffs the generated
+ * module too, so the numbers a verdict carries are the numbers on /proof.
+ *
+ * What this module adds: the arithmetic that turns a confusion matrix into
+ * "how often is this fire right for YOU". The published precision is the
+ * positive predictive value at the corpus prevalence, roughly one half; a
+ * deployment whose traffic carries one violation in a hundred sees a very
+ * different number from the same rule. Both the point and its interval are
+ * computed here — the interval by seeded Monte Carlo over the Beta posteriors
+ * of sensitivity and specificity (Jeffreys prior, ½ pseudo-count per cell),
+ * two thousand draws, memoised per (rule, prevalence to three decimals) so
+ * the draws run once per process.
+ *
+ * Every number carries its provenance: the corpus version, the release it
+ * was generated for, and the labelling ('same-model' until the founder's
+ * blind label lands). A surface that drops the labelling tag is a truth
+ * defect, not a formatting choice.
+ */
+import { PUBLISHED_ACCURACY, PUBLISHED_ACCURACY_CORPUS_VERSION, PUBLISHED_ACCURACY_LABELLING, PUBLISHED_ACCURACY_RELEASE } from './published-accuracy.js';
+import { beta, fnv1a, missRate, mulberry32, percentile95, ppv, round4, sensitivity, specificity } from './stats.js';
+export const DEFAULT_PREVALENCE = 0.5;
+export const INTERVAL_DRAWS = 2000;
+export function publishedProvenance() {
+    return { corpusVersion: PUBLISHED_ACCURACY_CORPUS_VERSION, release: PUBLISHED_ACCURACY_RELEASE, labelling: PUBLISHED_ACCURACY_LABELLING };
+}
+/** The published numbers for a built-in rule, or null for a rule with no family (a custom rule, a rule added before its proof). */
+export function publishedAccuracyFor(ruleName) {
+    const entry = PUBLISHED_ACCURACY[ruleName];
+    return entry ?? null;
+}
+/** Every rule name with published numbers, in the order the proof emitted them. */
+export function publishedRuleNames() {
+    return Object.keys(PUBLISHED_ACCURACY);
+}
+const memo = new Map();
+const key = (ruleName, prevalence, which) => `${which}:${ruleName}:${prevalence.toFixed(3)}`;
+function sampleInterval(counts, prevalence, seed, fn) {
+    const sens = sensitivity(counts);
+    const spec = specificity(counts);
+    if (sens === null || spec === null)
+        return null;
+    const rng = mulberry32(fnv1a(seed));
+    const draws = [];
+    for (let i = 0; i < INTERVAL_DRAWS; i++) {
+        const s = beta(counts.tp + 0.5, counts.fn + 0.5, rng);
+        const p = beta(counts.tn + 0.5, counts.fp + 0.5, rng);
+        draws.push(fn(s, p, prevalence));
+    }
+    const [lo, hi] = percentile95(draws);
+    return { point: round4(fn(sens, spec, prevalence)), lo: round4(lo), hi: round4(hi) };
+}
+/**
+ * PPV at a prevalence with a 95% credible interval, for a rule that FIRED.
+ * Null when the rule has no published family or its family has no positives
+ * or no negatives (nothing to estimate from).
+ */
+export function ppvInterval(ruleName, prevalence = DEFAULT_PREVALENCE) {
+    const k = key(ruleName, prevalence, 'ppv');
+    if (memo.has(k))
+        return memo.get(k) ?? null;
+    const counts = publishedAccuracyFor(ruleName);
+    const result = counts ? sampleInterval(counts, prevalence, `ppv:${ruleName}:${PUBLISHED_ACCURACY_CORPUS_VERSION}`, ppv) : null;
+    memo.set(k, result);
+    return result;
+}
+/** P(violation | the rule did not fire) at a prevalence with a 95% credible interval, for a rule that did NOT fire. */
+export function missRateInterval(ruleName, prevalence = DEFAULT_PREVALENCE) {
+    const k = key(ruleName, prevalence, 'miss');
+    if (memo.has(k))
+        return memo.get(k) ?? null;
+    const counts = publishedAccuracyFor(ruleName);
+    const result = counts ? sampleInterval(counts, prevalence, `miss:${ruleName}:${PUBLISHED_ACCURACY_CORPUS_VERSION}`, missRate) : null;
+    memo.set(k, result);
+    return result;
+}
+/** PPV at several prevalences — the field-prevalence table a reader needs beside a published precision. */
+export function ppvAt(ruleName, prevalences = [0.01, 0.05, 0.2, 0.5]) {
+    const counts = publishedAccuracyFor(ruleName);
+    const out = {};
+    for (const p of prevalences) {
+        const sens = counts ? sensitivity(counts) : null;
+        const spec = counts ? specificity(counts) : null;
+        out[p.toFixed(2)] = sens === null || spec === null ? null : round4(ppv(sens, spec, p));
+    }
+    return out;
+}
+/** Test hook: clear the memo so a seeded interval can be recomputed. */
+export function resetAccuracyMemo() {
+    memo.clear();
+}

package/dist/eval/citation-verify/verifier.d.ts CHANGED Viewed

@@ -33,7 +33,22 @@ export interface VerifiedCitation {
 }
 export interface VerifyCitationsResult {
     overallScore: number | null;
-    passed: boolean;
+    /**
+     * The verdict, and it is **null when nothing was judged**.
+     *
+     * Until 0.10.0 this was `true` in that case: no citation resolved, or the
+     * judge failed on every one, and the tool said the output passed. A
+     * caller reading `passed` shipped an answer whose sources had not been
+     * checked at all. There is no verdict when nothing was verified, and null
+     * is what says so.
+     *
+     * When citations WERE judged, the rule is counts and not a proportion:
+     * every judged citation must be supported. A proportion let one
+     * fabricated source among three real ones score 0.67 and pass.
+     */
+    passed: boolean | null;
+    /** Judged citations the judge ruled unsupported. The number the verdict turns on. */
+    totalUnsupported: number;
     citations: VerifiedCitation[];
     totalCostUsd: number;
     totalCitationsFound: number;

package/dist/eval/citation-verify/verifier.js CHANGED Viewed

@@ -247,13 +247,23 @@ export async function verifyCitations(params) {
     // unsupported would make a judge outage on 5 of 10 supported citations
     // score 0.5, indistinguishable from fabrication.
     const overallScore = totalJudged > 0 ? Math.round((totalSupported / totalJudged) * 100) / 100 : null;
-    // Fail if >= 50% of judged sources don't support the claim. When no
-    // citations, none resolved, or none judged, we don't fail — there's
-    // nothing to score, we just report that.
-    const passed = overallScore === null ? true : overallScore >= 0.5;
+    const totalUnsupported = totalJudged - totalSupported;
+    /*
+     * Counts, not a proportion, and null when nothing was judged.
+     *
+     * The old rule was `overallScore >= 0.5`, with `true` when the score was
+     * null. Both halves were wrong. A proportion let one fabricated source
+     * among three real ones score 0.67 and pass — a citation either supports
+     * the claim or it does not, and one that does not is the finding. And
+     * "nothing was judged" was reported as a pass, so an output whose sources
+     * never resolved, or whose every judge call failed, came back looking
+     * verified. There is no verdict when nothing was verified.
+     */
+    const passed = totalJudged === 0 ? null : totalUnsupported === 0;
     return {
         overallScore,
         passed,
+        totalUnsupported,
         citations: out,
         totalCostUsd: Math.round(totalCost * 1_000_000) / 1_000_000,
         totalCitationsFound: totalFound,

package/dist/eval/compose.d.ts ADDED Viewed

@@ -0,0 +1,57 @@
+import type { EvalResult, EvalRuleResult, Interpretation, Need, Verdict } from '../types/eval.js';
+import { type PriorMode } from './risk.js';
+export interface ComposeConfig {
+    /** `risk` composes by kind; `legacy` runs the pre-0.10.0 arithmetic. */
+    composer: 'risk' | 'legacy';
+    /** How many wrongly blocked builds one shipped failure is worth. τ = 1 / (1 + c). */
+    falsePassCost: number;
+    /** What a critical rule that could not answer does to the verdict. */
+    onCriticalSkipped: 'unknown' | 'fail' | 'pass';
+    /** Inputs the deployment insists every evaluation carries; absent ones make the verdict unknown. */
+    requiredEvidence: readonly Need[];
+    /** Whether a shipped default threshold decides the verdict, or only advises. */
+    defaultsGate: boolean;
+    /** The prior that an output is bad, before any rule speaks. */
+    prior: number;
+    /** How that prior is spread over the failure classes the detectors examine. */
+    priorMode: PriorMode;
+}
+export declare const DEFAULT_COMPOSE: ComposeConfig;
+/** The risk threshold a loss ratio implies: block when the expected loss of passing exceeds that of blocking. */
+export declare function tau(falsePassCost: number): number;
+/**
+ * Whether a policy rule DECIDES the verdict here, or only advises.
+ *
+ * "A default is not your policy." A shipped threshold — a cost ceiling of
+ * $0.50, a length floor of 50 characters — is our guess about a deployment
+ * we have never seen, and stopping someone's build on it is presumptuous.
+ * A threshold the deployment SET is their decision and gates.
+ *
+ * The distinction is not a list of rule names. For a BUILT-IN policy it is
+ * whether the number the rule compared against is one we chose, which every
+ * result already records as `thresholdSource` on its count evidence (0.9.0);
+ * a policy with no number at all — "the output is empty" — is structural,
+ * has no guess in it, and gates.
+ *
+ * A CUSTOM rule is different: its severity is the deployment's own statement
+ * of how much it matters, made when the rule was deployed. High and critical
+ * gate (they resolve to critical); medium and low advise, which is the
+ * contract `deploy_rule` has always had. An inline rule passed in the call
+ * carries no severity and advises, for the same reason.
+ */
+export declare function decides(r: EvalRuleResult, defaultsGate: boolean): boolean;
+/**
+ * The verdict for one evaluation. The weighted mean is never consulted: it
+ * survives as a quality gradient on the score field and is never re-meant.
+ */
+export declare function compose(result: Pick<EvalResult, 'rule_results' | 'score' | 'insufficient_data' | 'rules_evaluated'>, cfg: ComposeConfig): Verdict;
+/**
+ * The sentences a reader needs that the verdict alone does not carry.
+ *
+ * The one that must exist: when a rule visibly FIRED and the verdict still
+ * passed, say why and name the one setting that would change it. Without
+ * it, "cost_under_threshold failed" beside "passed: true" reads as a bug,
+ * and that is the first thing a builder who never opens a config file will
+ * meet.
+ */
+export declare function interpretations(result: Pick<EvalResult, 'rule_results'>, verdict: Verdict, cfg: ComposeConfig): Interpretation[];

package/dist/eval/compose.js ADDED Viewed

@@ -0,0 +1,179 @@
+import { riskEstimate, DEFAULT_PRIOR, DEFAULT_PRIOR_MODE, DEFAULT_FALSE_PASS_COST } from './risk.js';
+export const DEFAULT_COMPOSE = {
+    composer: 'risk',
+    falsePassCost: DEFAULT_FALSE_PASS_COST,
+    onCriticalSkipped: 'unknown',
+    requiredEvidence: [],
+    defaultsGate: false,
+    prior: DEFAULT_PRIOR,
+    priorMode: DEFAULT_PRIOR_MODE,
+};
+/** The risk threshold a loss ratio implies: block when the expected loss of passing exceeds that of blocking. */
+export function tau(falsePassCost) {
+    return 1 / (1 + falsePassCost);
+}
+const isCritical = (r) => r.critical === true;
+const fired = (r) => !r.skipped && r.passed === false;
+/**
+ * Whether a policy rule DECIDES the verdict here, or only advises.
+ *
+ * "A default is not your policy." A shipped threshold — a cost ceiling of
+ * $0.50, a length floor of 50 characters — is our guess about a deployment
+ * we have never seen, and stopping someone's build on it is presumptuous.
+ * A threshold the deployment SET is their decision and gates.
+ *
+ * The distinction is not a list of rule names. For a BUILT-IN policy it is
+ * whether the number the rule compared against is one we chose, which every
+ * result already records as `thresholdSource` on its count evidence (0.9.0);
+ * a policy with no number at all — "the output is empty" — is structural,
+ * has no guess in it, and gates.
+ *
+ * A CUSTOM rule is different: its severity is the deployment's own statement
+ * of how much it matters, made when the rule was deployed. High and critical
+ * gate (they resolve to critical); medium and low advise, which is the
+ * contract `deploy_rule` has always had. An inline rule passed in the call
+ * carries no severity and advises, for the same reason.
+ */
+export function decides(r, defaultsGate) {
+    if (isCritical(r))
+        return true;
+    if (defaultsGate)
+        return true;
+    if (r.origin === 'custom')
+        return false;
+    const ourDefault = (r.evidence ?? []).some((e) => e.type === 'count' && e.threshold !== undefined && (e.thresholdSource ?? 'default') === 'default');
+    return !ourDefault;
+}
+/** The inputs at least one evaluated rule actually read. */
+function inputsSeen(rows) {
+    const seen = new Set();
+    for (const r of rows)
+        if (!r.skipped)
+            for (const n of r.saw ?? [])
+                seen.add(n);
+    return seen;
+}
+/**
+ * The verdict for one evaluation. The weighted mean is never consulted: it
+ * survives as a quality gradient on the score field and is never re-meant.
+ */
+export function compose(result, cfg) {
+    const rows = result.rule_results;
+    const evaluated = result.rules_evaluated ?? rows.filter((r) => !r.skipped).length;
+    if (result.insufficient_data || evaluated === 0) {
+        return { state: 'unknown', passed: false, basis: 'no_rules', by: [], risk: null };
+    }
+    /*
+     * 1. Gates: a policy whose author has already decided — and a JUDGMENT,
+     * for the same reason. Nobody runs a judge by accident: the caller chose
+     * the template, supplied the key and paid for the answer, so a failing
+     * judgment decides rather than being weighed against anything. It also
+     * cannot be weighed: a judgment carries no published error rate until a
+     * measured run exists for its template and model, so the risk layer would
+     * drop it silently and a paid-for "fail" would read as clean.
+     */
+    const gates = rows.filter((r) => fired(r) && ((r.kind === 'policy' && decides(r, cfg.defaultsGate)) || r.kind === 'judgment'));
+    if (gates.length > 0) {
+        return { state: 'fail', passed: false, basis: 'policy_gate', by: gates.map((r) => r.ruleName), risk: null };
+    }
+    /*
+     * 2. Vetoes: an effectively-critical rule that is not a policy. Keyed on
+     * "not a policy" rather than on the two detecting kinds, so a rule built
+     * by hand without metadata — a test double, an embedder's own rule —
+     * still vetoes when it is marked critical. Silently ignoring a critical
+     * rule because it forgot to declare its kind is the failure mode this
+     * composer exists to remove, not one to introduce.
+     */
+    const vetoes = rows.filter((r) => r.kind !== 'policy' && fired(r) && isCritical(r));
+    if (vetoes.length > 0) {
+        return { state: 'fail', passed: false, basis: 'detector_veto', by: vetoes.map((r) => r.ruleName), risk: null };
+    }
+    /*
+     * 3. Asked and could not answer. `not_applicable` is NEVER this: a
+     * trajectory rule with no tool calls was not asked, and treating that as
+     * unknown would make every text-only evaluation unknown, which is worse
+     * than the fail-open it replaces.
+     */
+    const unknown = rows.filter((r) => isCritical(r) && r.skipped === true && r.skipClass !== undefined && r.skipClass !== 'not_applicable');
+    if (unknown.length > 0 && cfg.onCriticalSkipped !== 'pass') {
+        const by = unknown.map((r) => r.ruleName);
+        return cfg.onCriticalSkipped === 'fail'
+            ? { state: 'fail', passed: false, basis: 'critical_unknown', by, risk: null }
+            : { state: 'unknown', passed: false, basis: 'critical_unknown', by, risk: null };
+    }
+    // 4. Evidence the deployment insists on.
+    if (cfg.requiredEvidence.length > 0) {
+        const seen = inputsSeen(rows);
+        const missing = cfg.requiredEvidence.filter((n) => !seen.has(n));
+        if (missing.length > 0) {
+            return { state: 'unknown', passed: false, basis: 'required_evidence_missing', by: [...missing], risk: null };
+        }
+    }
+    // 5. Everything that carries a published error rate, as one probability.
+    const risk = riskEstimate(result, cfg.prior, cfg.priorMode);
+    if (risk === null) {
+        return { state: 'pass', passed: true, basis: 'clean', by: [], risk: null };
+    }
+    const t = tau(cfg.falsePassCost);
+    const confidence = risk.lo <= t && t <= risk.hi ? 'marginal' : 'decisive';
+    if (risk.pBad > t) {
+        const by = Object.entries(risk.perClass)
+            .filter(([, q]) => q !== null && q !== undefined && q > 0.5)
+            .map(([cls]) => cls);
+        return { state: 'fail', passed: false, basis: 'risk_over_loss', by, risk, confidence };
+    }
+    return { state: 'pass', passed: true, basis: 'clean', by: [], risk, confidence };
+}
+/**
+ * The sentences a reader needs that the verdict alone does not carry.
+ *
+ * The one that must exist: when a rule visibly FIRED and the verdict still
+ * passed, say why and name the one setting that would change it. Without
+ * it, "cost_under_threshold failed" beside "passed: true" reads as a bug,
+ * and that is the first thing a builder who never opens a config file will
+ * meet.
+ */
+export function interpretations(result, verdict, cfg) {
+    const out = [];
+    for (const r of result.rule_results) {
+        if (!fired(r))
+            continue;
+        if (verdict.by.includes(r.ruleName))
+            continue;
+        if (r.kind === 'policy' && !decides(r, cfg.defaultsGate)) {
+            out.push({
+                severity: 'warn',
+                addressee: 'operator',
+                rule: r.ruleName,
+                text: `${r.ruleName} failed against a threshold Iris ships, not one you set, so it did not decide this verdict. Set it in your configuration to make it a gate, or set eval.defaultsGate to true to make every shipped default gate.`,
+                configKey: 'eval.defaultsGate',
+            });
+            continue;
+        }
+        if (verdict.state === 'pass') {
+            out.push({
+                severity: 'note',
+                addressee: 'operator',
+                rule: r.ruleName,
+                text: `${r.ruleName} failed but the verdict passed: on its published accuracy this rule alone does not carry the risk past your loss threshold. Lower eval.falsePassCost to block on weaker evidence.`,
+                configKey: 'eval.falsePassCost',
+            });
+        }
+    }
+    if (verdict.basis === 'critical_unknown') {
+        out.push({
+            severity: 'block',
+            addressee: 'operator',
+            text: `A critical check was asked and could not answer (${verdict.by.join(', ')}), so this verdict is unknown rather than clean. Set eval.onCriticalSkipped to "pass" to accept that risk, or to "fail" to treat it as a failure.`,
+            configKey: 'eval.onCriticalSkipped',
+        });
+    }
+    if (verdict.confidence === 'marginal') {
+        out.push({
+            severity: 'note',
+            addressee: 'operator',
+            text: 'The credible interval on this risk estimate straddles your threshold, so this verdict could go either way on the evidence available. Treat it as a close call rather than a clear one.',
+        });
+    }
+    return out;
+}