npm - @iris-eval/mcp-server - Versions diffs - 0.8.2 → 0.9.0 - Mend

@iris-eval/mcp-server 0.8.2 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (114) hide show

package/README.md +8 -1
package/dist/capabilities.d.ts +64 -0
package/dist/capabilities.js +65 -0
package/dist/config/defaults.js +2 -0
package/dist/custom-rule-store.d.ts +4 -0
package/dist/custom-rule-store.js +8 -3
package/dist/dashboard/assets/{index-CyzO6OC7.js → index-Cz8_oOqG.js} +1 -1
package/dist/dashboard/index.html +1 -1
package/dist/dashboard/routes/capabilities.d.ts +3 -0
package/dist/dashboard/routes/capabilities.js +11 -0
package/dist/dashboard/routes/health.d.ts +5 -1
package/dist/dashboard/routes/health.js +15 -3
package/dist/dashboard/routes/rules.js +4 -1
package/dist/dashboard/routes/traces.d.ts +3 -0
package/dist/dashboard/routes/traces.js +9 -28
package/dist/dashboard/server.d.ts +2 -0
package/dist/dashboard/server.js +6 -2
package/dist/eval/accuracy.d.ts +41 -0
package/dist/eval/accuracy.js +97 -0
package/dist/eval/criticality.d.ts +8 -1
package/dist/eval/criticality.js +6 -0
package/dist/eval/dormant.d.ts +4 -0
package/dist/eval/dormant.js +22 -0
package/dist/eval/engine.d.ts +1 -0
package/dist/eval/engine.js +49 -3
package/dist/eval/failure-classes.d.ts +8 -0
package/dist/eval/failure-classes.js +18 -0
package/dist/eval/llm-judge/evaluator.d.ts +10 -0
package/dist/eval/llm-judge/evaluator.js +16 -1
package/dist/eval/published-accuracy.d.ts +230 -0
package/dist/eval/published-accuracy.js +86 -0
package/dist/eval/questions.d.ts +12 -0
package/dist/eval/questions.js +14 -0
package/dist/eval/response-schema.d.ts +652 -0
package/dist/eval/response-schema.js +130 -0
package/dist/eval/response.d.ts +12 -0
package/dist/eval/response.js +30 -0
package/dist/eval/rules/completeness.js +31 -0
package/dist/eval/rules/cost.d.ts +1 -1
package/dist/eval/rules/cost.js +44 -0
package/dist/eval/rules/custom.d.ts +0 -12
package/dist/eval/rules/custom.js +21 -0
package/dist/eval/rules/relevance.js +16 -0
package/dist/eval/rules/safety.js +178 -1
package/dist/eval/stamp.d.ts +14 -0
package/dist/eval/stamp.js +88 -0
package/dist/eval/stats.d.ts +33 -0
package/dist/eval/stats.js +109 -0
package/dist/eval/verdict.d.ts +34 -0
package/dist/eval/verdict.js +131 -0
package/dist/index.js +5 -28
package/dist/instructions.d.ts +17 -0
package/dist/instructions.js +53 -0
package/dist/judge-enablement.d.ts +34 -0
package/dist/judge-enablement.js +78 -0
package/dist/judge-enablement.json +10 -0
package/dist/preferences.d.ts +1 -1
package/dist/prompts.d.ts +3 -0
package/dist/prompts.js +29 -0
package/dist/resources/index.d.ts +5 -2
package/dist/resources/index.js +65 -5
package/dist/resources/uris.d.ts +12 -0
package/dist/resources/uris.js +24 -0
package/dist/retention.d.ts +20 -0
package/dist/retention.js +44 -0
package/dist/self-test.d.ts +1 -0
package/dist/self-test.js +14 -0
package/dist/server.d.ts +10 -1
package/dist/server.js +34 -7
package/dist/storage/index.js +1 -1
package/dist/storage/migrations/007-eval-provenance.d.ts +3 -0
package/dist/storage/migrations/007-eval-provenance.js +30 -0
package/dist/storage/migrations/index.js +24 -4
package/dist/storage/sqlite-adapter.d.ts +26 -1
package/dist/storage/sqlite-adapter.js +135 -15
package/dist/tools/delete-rule.d.ts +8 -0
package/dist/tools/delete-rule.js +30 -38
package/dist/tools/delete-trace.d.ts +5 -0
package/dist/tools/delete-trace.js +24 -27
package/dist/tools/deploy-rule.d.ts +13 -1
package/dist/tools/deploy-rule.js +37 -34
package/dist/tools/describe.d.ts +20 -0
package/dist/tools/describe.js +36 -0
package/dist/tools/errors.d.ts +36 -0
package/dist/tools/errors.js +134 -0
package/dist/tools/evaluate-output.d.ts +8 -1
package/dist/tools/evaluate-output.js +37 -58
package/dist/tools/evaluate-with-llm-judge.d.ts +31 -0
package/dist/tools/evaluate-with-llm-judge.js +96 -69
package/dist/tools/get-traces.d.ts +9 -0
package/dist/tools/get-traces.js +29 -28
package/dist/tools/index.d.ts +8 -0
package/dist/tools/index.js +22 -1
package/dist/tools/list-rules.d.ts +13 -0
package/dist/tools/list-rules.js +43 -46
package/dist/tools/log-trace.d.ts +4 -0
package/dist/tools/log-trace.js +31 -29
package/dist/tools/respond.d.ts +42 -0
package/dist/tools/respond.js +90 -0
package/dist/tools/strict-input.js +1 -1
package/dist/tools/trace-link.d.ts +2 -0
package/dist/tools/trace-link.js +13 -2
package/dist/tools/verify-citations.d.ts +17 -2
package/dist/tools/verify-citations.js +98 -93
package/dist/types/config.d.ts +9 -0
package/dist/types/eval.d.ts +258 -0
package/dist/types/eval.js +2 -1
package/dist/types/query.d.ts +2 -0
package/package.json +1 -1
package/server.json +2 -2
package/dist/resources/dashboard-summary.d.ts +0 -3
package/dist/resources/dashboard-summary.js +0 -16
package/dist/resources/trace-detail.d.ts +0 -3
package/dist/resources/trace-detail.js +0 -30

package/dist/dashboard/index.html CHANGED Viewed

@@ -6,7 +6,7 @@
     <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <!-- Stop shipping agents on vibes is filled from .claims.json brand.tagline at build time (vite.config.ts) — never restate the tagline here. -->
     <title>Iris — Stop shipping agents on vibes</title>
-    <script type="module" crossorigin src="/assets/index-CyzO6OC7.js"></script>
+    <script type="module" crossorigin src="/assets/index-Cz8_oOqG.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-D0cFfBqn.css">
   </head>
   <body>

package/dist/dashboard/routes/capabilities.d.ts ADDED Viewed

@@ -0,0 +1,3 @@
+import { Router } from 'express';
+import type { Capabilities } from '../../capabilities.js';
+export declare function registerCapabilitiesRoutes(router: Router, build: () => Capabilities): void;

package/dist/dashboard/routes/capabilities.js ADDED Viewed

@@ -0,0 +1,11 @@
+/*
+ * GET /api/v1/capabilities — the same object iris://capabilities serves,
+ * for a caller on the HTTP path (an agent driving Iris through the ingest
+ * endpoint, a dashboard, a health probe that wants more than "ok").
+ * Provider name only, never a key.
+ */
+export function registerCapabilitiesRoutes(router, build) {
+    router.get('/capabilities', (_req, res) => {
+        res.json(build());
+    });
+}

package/dist/dashboard/routes/health.d.ts CHANGED Viewed

@@ -1,3 +1,7 @@
 import { Router } from 'express';
 import type { IStorageAdapter } from '../../types/query.js';
-export declare function registerHealthRoutes(router: Router, storage?: IStorageAdapter, version?: string): void;
+export interface HealthOptions {
+    /** `demo` when serving the disposable demo database. */
+    mode?: 'real' | 'demo';
+}
+export declare function registerHealthRoutes(router: Router, storage?: IStorageAdapter, version?: string, options?: HealthOptions): void;

package/dist/dashboard/routes/health.js CHANGED Viewed

@@ -1,9 +1,19 @@
 import { LOCAL_TENANT } from '../../types/tenant.js';
+import { judgeState } from '../../judge-enablement.js';
 const startTime = Date.now();
-export function registerHealthRoutes(router, storage, version) {
+export function registerHealthRoutes(router, storage, version, options) {
     const serverVersion = version ?? 'unknown';
+    const mode = options?.mode ?? 'real';
     router.get('/health', async (_req, res) => {
         const uptime_seconds = Math.floor((Date.now() - startTime) / 1000);
+        /*
+         * The judge state, provider name only — never the key. Read per
+         * request rather than at boot so a test (or an operator) that sets
+         * the variable in this process sees it here; the process a client
+         * spawns has a fixed environment anyway, so both reads agree there.
+         */
+        const judge = judgeState();
+        const judgeField = { enabled: judge.enabled, provider: judge.provider };
         if (storage) {
             try {
                 /* Health probes use LOCAL_TENANT directly — the health endpoint
@@ -26,14 +36,16 @@ export function registerHealthRoutes(router, storage, version) {
                     uptime_seconds,
                     trace_count: total,
                     storage: 'connected',
+                    judge: judgeField,
+                    mode,
                 });
             }
             catch {
-                res.status(503).json({ status: 'degraded', version: serverVersion, uptime_seconds, storage: 'disconnected' });
+                res.status(503).json({ status: 'degraded', version: serverVersion, uptime_seconds, storage: 'disconnected', judge: judgeField, mode });
             }
         }
         else {
-            res.json({ status: 'ok', version: serverVersion, uptime_seconds });
+            res.json({ status: 'ok', version: serverVersion, uptime_seconds, judge: judgeField, mode });
         }
     });
 }

package/dist/dashboard/routes/rules.js CHANGED Viewed

@@ -110,7 +110,9 @@ export function registerRuleRoutes(router, storage, opts) {
     router.get('/rules/custom', (req, res) => {
         const tenantId = requireTenant(req);
         const rules = opts.customRuleStore.list(tenantId);
-        res.json({ rules });
+        // Quarantined entries ride along: they are on disk, never registered,
+        // and a gating one is named on every verdict as coverage.dormant.
+        res.json({ rules, quarantined: opts.customRuleStore.quarantined(tenantId) });
     });
     router.post('/rules/custom', async (req, res) => {
         try {
@@ -133,6 +135,7 @@ export function registerRuleRoutes(router, storage, opts) {
              */
             const replaced = retireSameNamedRules(opts.customRuleStore, opts.evalEngine, tenantId, input.name, input.replace, 'local');
             const rule = opts.customRuleStore.deploy(tenantId, {
+                replaces: replaced.map((r) => r.id),
                 name: input.name,
                 description: input.description,
                 evalType: input.evalType,

package/dist/dashboard/routes/traces.d.ts CHANGED Viewed

@@ -1,4 +1,5 @@
 import { Router } from 'express';
+import type { CustomRuleStore } from '../../custom-rule-store.js';
 import type { IStorageAdapter } from '../../types/query.js';
 import type { EvalEngine } from '../../eval/engine.js';
 export interface TraceRouteOptions {
@@ -10,5 +11,7 @@ export interface TraceRouteOptions {
      * as a success.
      */
     evalEngine?: EvalEngine;
+    /** The custom-rule store, so an evaluation over HTTP carries coverage.dormant like the tool does. */
+    customRuleStore?: CustomRuleStore;
 }
 export declare function registerTraceRoutes(router: Router, storage: IStorageAdapter, options?: TraceRouteOptions): void;

package/dist/dashboard/routes/traces.js CHANGED Viewed

@@ -1,3 +1,5 @@
+import { toEvaluationResponse } from '../../eval/response.js';
+import { dormantRulesFrom } from '../../eval/dormant.js';
 import { requireTenant } from '../../middleware/tenant.js';
 import { generateTraceId, generateSpanId } from '../../utils/ids.js';
 import { bestEffortExport } from '../../otel/lazy.js';
@@ -84,38 +86,17 @@ export function registerTraceRoutes(router, storage, options) {
                 : options.evalEngine.evaluate(evalType, context);
             evaluation.trace_id = traceId;
             await storage.insertEvalResult(tenantId, evaluation);
+            // The same serializer as evaluate_output (src/eval/response.ts): the
+            // veto reason, the skipped criticals, the verdict basis, coverage and
+            // provenance travel the ingest path exactly as they travel the tool.
             res.status(201).json({
                 trace_id: traceId,
                 status: 'stored',
-                evaluation: {
-                    id: evaluation.id,
-                    eval_type: evaluation.eval_type,
-                    score: evaluation.score,
-                    passed: evaluation.passed,
-                    rule_results: evaluation.rule_results,
-                    suggestions: evaluation.suggestions,
-                    rules_evaluated: evaluation.rules_evaluated,
-                    rules_skipped: evaluation.rules_skipped,
-                    insufficient_data: evaluation.insufficient_data,
-                    /*
-                     * The veto reason travels the ingest path too. Omitting it left an
-                     * HTTP caller seeing passed:false beside a high score with no way
-                     * to learn that a critical rule — not the weighted average —
-                     * produced the verdict. Absent when nothing vetoed.
-                     */
-                    ...(evaluation.critical_failures?.length
-                        ? { critical_failures: evaluation.critical_failures }
-                        : {}),
-                    // The other half of the veto contract, same as evaluate_output: a
-                    // critical rule that SKIPPED did not judge the output and cannot
-                    // veto, so a fail-closed gate needs to see it named.
-                    ...(evaluation.critical_skipped?.length
-                        ? { critical_skipped: evaluation.critical_skipped }
-                        : {}),
-                    // Per-bundle breakdown — eval_type="all" only.
-                    ...(evaluation.categories ? { categories: evaluation.categories } : {}),
+                evaluation: toEvaluationResponse(evaluation, {
+                    traceId,
+                    dormant: options?.customRuleStore ? dormantRulesFrom(options.customRuleStore.quarantined(tenantId)) : undefined,
                     ...(evalTypeOmitted ? { note: DEFAULT_EVAL_TYPE_NOTE } : {}),
-                },
+                }),
             });
         }
         catch (err) {

package/dist/dashboard/server.d.ts CHANGED Viewed

@@ -14,5 +14,7 @@ export interface DashboardServerOptions {
     customRuleStore?: CustomRuleStore;
     evalEngine?: EvalEngine;
     preferenceStore?: PreferenceStore;
+    /**  when serving the disposable demo database; reported on /api/v1/capabilities. */
+    mode?: 'real' | 'demo';
 }
 export declare function createDashboardServer(storage: IStorageAdapter, config: IrisConfig, logger: Logger, options?: DashboardServerOptions): DashboardServer;

package/dist/dashboard/server.js CHANGED Viewed

@@ -4,6 +4,8 @@ import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
 import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
 import { irisHome } from '../utils/iris-home.js';
+import { buildCapabilities } from '../capabilities.js';
+import { registerCapabilitiesRoutes } from './routes/capabilities.js';
 import { createAuthMiddleware } from '../middleware/auth.js';
 import { createCorsMiddleware } from '../middleware/cors.js';
 import { createErrorHandler } from '../middleware/error-handler.js';
@@ -75,12 +77,14 @@ export function createDashboardServer(storage, config, logger, options) {
     // API routes with rate limiting
     const router = express.Router();
     router.use(createApiRateLimiter(config));
-    registerTraceRoutes(router, storage, { evalEngine: options?.evalEngine });
+    registerTraceRoutes(router, storage, { evalEngine: options?.evalEngine, customRuleStore: options?.customRuleStore });
     registerSummaryRoutes(router, storage);
     registerEvaluationRoutes(router, storage);
     registerEvalStatsRoutes(router, storage);
     registerFilterRoutes(router, storage);
-    registerHealthRoutes(router, storage, config.server.version);
+    registerHealthRoutes(router, storage, config.server.version, { mode: options?.mode });
+    // The same object iris://capabilities serves, for the HTTP path.
+    registerCapabilitiesRoutes(router, () => buildCapabilities({ config, evalEngine: options?.evalEngine, customRuleStore: options?.customRuleStore, mode: options?.mode }));
     registerMomentRoutes(router, storage);
     registerFailureRoutes(router, storage);
     if (options?.customRuleStore && options?.evalEngine) {

package/dist/eval/accuracy.d.ts ADDED Viewed

@@ -0,0 +1,41 @@
+import { type Confusion } from './stats.js';
+export interface Interval {
+    point: number;
+    lo: number;
+    hi: number;
+}
+export interface PublishedRuleAccuracy extends Confusion {
+    n: number;
+    precision: number | null;
+    recall: number | null;
+    f1: number | null;
+    ci95: {
+        precision: readonly [number, number] | null;
+        recall: readonly [number, number] | null;
+        f1: readonly [number, number] | null;
+    };
+}
+export interface PublishedProvenance {
+    corpusVersion: string;
+    release: string;
+    labelling: 'same-model' | 'human-verified';
+}
+export declare const DEFAULT_PREVALENCE = 0.5;
+export declare const INTERVAL_DRAWS = 2000;
+export declare function publishedProvenance(): PublishedProvenance;
+/** The published numbers for a built-in rule, or null for a rule with no family (a custom rule, a rule added before its proof). */
+export declare function publishedAccuracyFor(ruleName: string): PublishedRuleAccuracy | null;
+/** Every rule name with published numbers, in the order the proof emitted them. */
+export declare function publishedRuleNames(): string[];
+/**
+ * PPV at a prevalence with a 95% credible interval, for a rule that FIRED.
+ * Null when the rule has no published family or its family has no positives
+ * or no negatives (nothing to estimate from).
+ */
+export declare function ppvInterval(ruleName: string, prevalence?: number): Interval | null;
+/** P(violation | the rule did not fire) at a prevalence with a 95% credible interval, for a rule that did NOT fire. */
+export declare function missRateInterval(ruleName: string, prevalence?: number): Interval | null;
+/** PPV at several prevalences — the field-prevalence table a reader needs beside a published precision. */
+export declare function ppvAt(ruleName: string, prevalences?: readonly number[]): Record<string, number | null>;
+/** Test hook: clear the memo so a seeded interval can be recomputed. */
+export declare function resetAccuracyMemo(): void;

package/dist/eval/accuracy.js ADDED Viewed

@@ -0,0 +1,97 @@
+/*
+ * The published accuracy, read at runtime.
+ *
+ * `npm run proof` measures every built-in rule on its labelled family and
+ * writes the numbers to proof/results.json — and, since 0.9.0, to
+ * src/eval/published-accuracy.ts, a generated module that ships inside the
+ * package (the npm `files` list carries dist/ only, so a runtime read of
+ * proof/ would find nothing). `npm run proof -- --check` diffs the generated
+ * module too, so the numbers a verdict carries are the numbers on /proof.
+ *
+ * What this module adds: the arithmetic that turns a confusion matrix into
+ * "how often is this fire right for YOU". The published precision is the
+ * positive predictive value at the corpus prevalence, roughly one half; a
+ * deployment whose traffic carries one violation in a hundred sees a very
+ * different number from the same rule. Both the point and its interval are
+ * computed here — the interval by seeded Monte Carlo over the Beta posteriors
+ * of sensitivity and specificity (Jeffreys prior, ½ pseudo-count per cell),
+ * two thousand draws, memoised per (rule, prevalence to three decimals) so
+ * the draws run once per process.
+ *
+ * Every number carries its provenance: the corpus version, the release it
+ * was generated for, and the labelling ('same-model' until the founder's
+ * blind label lands). A surface that drops the labelling tag is a truth
+ * defect, not a formatting choice.
+ */
+import { PUBLISHED_ACCURACY, PUBLISHED_ACCURACY_CORPUS_VERSION, PUBLISHED_ACCURACY_LABELLING, PUBLISHED_ACCURACY_RELEASE } from './published-accuracy.js';
+import { beta, fnv1a, missRate, mulberry32, percentile95, ppv, round4, sensitivity, specificity } from './stats.js';
+export const DEFAULT_PREVALENCE = 0.5;
+export const INTERVAL_DRAWS = 2000;
+export function publishedProvenance() {
+    return { corpusVersion: PUBLISHED_ACCURACY_CORPUS_VERSION, release: PUBLISHED_ACCURACY_RELEASE, labelling: PUBLISHED_ACCURACY_LABELLING };
+}
+/** The published numbers for a built-in rule, or null for a rule with no family (a custom rule, a rule added before its proof). */
+export function publishedAccuracyFor(ruleName) {
+    const entry = PUBLISHED_ACCURACY[ruleName];
+    return entry ?? null;
+}
+/** Every rule name with published numbers, in the order the proof emitted them. */
+export function publishedRuleNames() {
+    return Object.keys(PUBLISHED_ACCURACY);
+}
+const memo = new Map();
+const key = (ruleName, prevalence, which) => `${which}:${ruleName}:${prevalence.toFixed(3)}`;
+function sampleInterval(counts, prevalence, seed, fn) {
+    const sens = sensitivity(counts);
+    const spec = specificity(counts);
+    if (sens === null || spec === null)
+        return null;
+    const rng = mulberry32(fnv1a(seed));
+    const draws = [];
+    for (let i = 0; i < INTERVAL_DRAWS; i++) {
+        const s = beta(counts.tp + 0.5, counts.fn + 0.5, rng);
+        const p = beta(counts.tn + 0.5, counts.fp + 0.5, rng);
+        draws.push(fn(s, p, prevalence));
+    }
+    const [lo, hi] = percentile95(draws);
+    return { point: round4(fn(sens, spec, prevalence)), lo: round4(lo), hi: round4(hi) };
+}
+/**
+ * PPV at a prevalence with a 95% credible interval, for a rule that FIRED.
+ * Null when the rule has no published family or its family has no positives
+ * or no negatives (nothing to estimate from).
+ */
+export function ppvInterval(ruleName, prevalence = DEFAULT_PREVALENCE) {
+    const k = key(ruleName, prevalence, 'ppv');
+    if (memo.has(k))
+        return memo.get(k) ?? null;
+    const counts = publishedAccuracyFor(ruleName);
+    const result = counts ? sampleInterval(counts, prevalence, `ppv:${ruleName}:${PUBLISHED_ACCURACY_CORPUS_VERSION}`, ppv) : null;
+    memo.set(k, result);
+    return result;
+}
+/** P(violation | the rule did not fire) at a prevalence with a 95% credible interval, for a rule that did NOT fire. */
+export function missRateInterval(ruleName, prevalence = DEFAULT_PREVALENCE) {
+    const k = key(ruleName, prevalence, 'miss');
+    if (memo.has(k))
+        return memo.get(k) ?? null;
+    const counts = publishedAccuracyFor(ruleName);
+    const result = counts ? sampleInterval(counts, prevalence, `miss:${ruleName}:${PUBLISHED_ACCURACY_CORPUS_VERSION}`, missRate) : null;
+    memo.set(k, result);
+    return result;
+}
+/** PPV at several prevalences — the field-prevalence table a reader needs beside a published precision. */
+export function ppvAt(ruleName, prevalences = [0.01, 0.05, 0.2, 0.5]) {
+    const counts = publishedAccuracyFor(ruleName);
+    const out = {};
+    for (const p of prevalences) {
+        const sens = counts ? sensitivity(counts) : null;
+        const spec = counts ? specificity(counts) : null;
+        out[p.toFixed(2)] = sens === null || spec === null ? null : round4(ppv(sens, spec, p));
+    }
+    return out;
+}
+/** Test hook: clear the memo so a seeded interval can be recomputed. */
+export function resetAccuracyMemo() {
+    memo.clear();
+}

package/dist/eval/criticality.d.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import type { EvalRule, EvalType } from '../types/eval.js';
+import type { EvalRule, EvalType, ClaimKind, Mechanism, Need, QuestionId, FailureClass } from '../types/eval.js';
 /** Where a rule's EFFECTIVE criticality came from. */
 export type CriticalitySource = 'default' | 'config';
 export interface CriticalityOverrides {
@@ -50,6 +50,13 @@ export interface BuiltInRuleMeta {
     critical: boolean;
     /** Who decided it: the rule's own declaration, or one of the config lists. */
     criticalSource: CriticalitySource;
+    /** The rule's declared metadata (see EvalRule): the kind of claim, its mechanism, what it reads, the question it answers, the failure classes, the definition version. */
+    kind?: ClaimKind;
+    mechanism?: Mechanism;
+    needs?: readonly Need[];
+    question?: QuestionId;
+    classes?: readonly FailureClass[];
+    version?: number;
 }
 /**
  * The whole built-in roster, one entry per rule.

package/dist/eval/criticality.js CHANGED Viewed

@@ -147,6 +147,12 @@ export function builtInRuleRoster(resolve) {
                 weight: rule.weight,
                 critical: effective.critical,
                 criticalSource: effective.source,
+                kind: rule.kind,
+                mechanism: rule.mechanism,
+                needs: rule.needs,
+                question: rule.question,
+                classes: rule.classes,
+                version: rule.version,
             });
         }
     }

package/dist/eval/dormant.d.ts ADDED Viewed

@@ -0,0 +1,4 @@
+import type { Coverage } from '../types/eval.js';
+export type DormantRule = NonNullable<Coverage['dormant']>[number];
+/** The quarantined entries that would have gated, as `coverage.dormant` rows. */
+export declare function dormantRulesFrom(quarantined: readonly unknown[]): DormantRule[];

package/dist/eval/dormant.js ADDED Viewed

@@ -0,0 +1,22 @@
+const GATING = new Set(['high', 'critical']);
+function field(entry, key) {
+    if (!entry || typeof entry !== 'object')
+        return undefined;
+    const value = entry[key];
+    return typeof value === 'string' ? value : undefined;
+}
+/** The quarantined entries that would have gated, as `coverage.dormant` rows. */
+export function dormantRulesFrom(quarantined) {
+    const out = [];
+    for (const entry of quarantined) {
+        const severity = field(entry, 'severity');
+        if (!severity || !GATING.has(severity))
+            continue;
+        out.push({
+            ruleId: field(entry, 'id') ?? 'unknown',
+            name: field(entry, 'name') ?? 'unnamed',
+            reason: `quarantined: the stored definition failed validation in this version, so this ${severity} rule is not running`,
+        });
+    }
+    return out;
+}

package/dist/eval/engine.d.ts CHANGED Viewed

@@ -39,6 +39,7 @@ export declare class EvalEngine {
     private idByRule;
     private threshold;
     private ruleThresholds?;
+    private criticalityOverrides?;
     /**
      * Effective criticality per rule, bound to this engine's config overrides.
      * Every veto decision reads THIS, never `rule.critical` directly, so a

package/dist/eval/engine.js CHANGED Viewed

@@ -1,5 +1,8 @@
 import { getRulesForType, createCustomRule } from './rules/index.js';
 import { criticalityResolver } from './criticality.js';
+import { inputsPresent, stampRuleResult } from './stamp.js';
+import { buildProvenance, configHash, deriveCoverage, deriveVerdict, rulesetHash } from './verdict.js';
+import { PKG_VERSION } from '../config/defaults.js';
 import { generateEvalId } from '../utils/ids.js';
 /**
  * Every bundle eval_type="all" walks, in the order their categories are
@@ -40,6 +43,7 @@ export class EvalEngine {
     idByRule = new Map();
     threshold;
     ruleThresholds;
+    criticalityOverrides;
     /**
      * Effective criticality per rule, bound to this engine's config overrides.
      * Every veto decision reads THIS, never `rule.critical` directly, so a
@@ -55,6 +59,7 @@ export class EvalEngine {
     constructor(threshold = 0.7, ruleThresholds, criticalityOverrides) {
         this.threshold = threshold;
         this.ruleThresholds = ruleThresholds;
+        this.criticalityOverrides = criticalityOverrides;
         this.criticality = criticalityResolver(criticalityOverrides);
     }
     /** The effective criticality of one rule under this engine's config. Read by the rule roster surfaces. */
@@ -177,6 +182,7 @@ export class EvalEngine {
                 rules_evaluated: 0,
                 rules_skipped: 0,
                 insufficient_data: true,
+                verdict: { state: 'unknown', passed: false, basis: 'no_rules', by: [], risk: null },
             };
         }
         /*
@@ -201,10 +207,20 @@ export class EvalEngine {
              * the one path every evaluation takes, so a surface cannot render the
              * declared criticality where the engine applied a configured one.
              */
-            const { critical, source } = this.criticality(rule);
+            const effective = this.criticality(rule);
+            const { critical, source } = effective;
             // ruleId / category sit right after the name so a reader scanning
             // rule_results sees WHICH deployed rule (and which bundle) spoke.
             const { ruleName, ...rest } = raw;
+            /*
+             * The stamp (0.9.0): what kind of claim this is, what the composer
+             * did with it, which question it answers, what it saw, why it skipped,
+             * and how wrong it tends to be — from the rule's declaration, the
+             * inputs this call carried, and the published accuracy that ships in
+             * the package. Computed here, on the one path every evaluation takes,
+             * so no surface can show a result without its receipt. It changes no
+             * verdict: summarize() below still decides passed exactly as before.
+             */
             return {
                 ruleName,
                 ...(ruleId !== undefined ? { ruleId } : {}),
@@ -212,10 +228,32 @@ export class EvalEngine {
                 critical,
                 criticalSource: source,
                 ...rest,
+                ...stampRuleResult(rule, raw, context, effective),
             };
         });
         const overall = this.summarize(rules, ruleResults);
         const perCategory = categories ? this.categorize(rules, ruleResults, categories) : undefined;
+        /*
+         * The receipt for the whole evaluation (0.9.0): what produced it, which
+         * questions it judged, and the basis of its verdict. Computed here from
+         * what the engine already holds; persisted as provenance and derived
+         * again on every read, so a stored row answers "why did this pass on
+         * that day" without a backfill. Changes no verdict.
+         */
+        const provenance = buildProvenance({
+            irisVersion: PKG_VERSION,
+            rulesetHash: rulesetHash(rules, (r) => this.criticality(r)),
+            configHash: configHash({
+                threshold: this.threshold,
+                ruleThresholds: this.ruleThresholds,
+                criticalRules: this.criticalityOverrides?.criticalRules,
+                nonCriticalRules: this.criticalityOverrides?.nonCriticalRules,
+            }),
+            threshold: this.threshold,
+            ruleThresholds: this.ruleThresholds,
+            judgedAt: new Date().toISOString(),
+        });
+        const coverage = deriveCoverage(ruleResults, inputsPresent(context));
         // Handle "all rules skipped" — insufficient data
         if (overall.rulesEvaluated === 0) {
             const skipMessages = ruleResults
@@ -225,7 +263,7 @@ export class EvalEngine {
             // that EVERY critical rule that skipped is named here, and a caller
             // whose only rules were critical ones should not have to infer that
             // from insufficient_data alone.
-            return {
+            const unknown = {
                 id: generateEvalId(),
                 eval_type: evalType,
                 output_text: context.output,
@@ -242,7 +280,11 @@ export class EvalEngine {
                 insufficient_data: true,
                 ...(overall.criticalSkipped.length > 0 ? { critical_skipped: overall.criticalSkipped } : {}),
                 ...(perCategory ? { categories: perCategory } : {}),
+                coverage,
+                provenance,
             };
+            unknown.verdict = deriveVerdict(unknown, this.threshold);
+            return unknown;
         }
         const suggestions = [];
         for (const result of ruleResults) {
@@ -273,7 +315,7 @@ export class EvalEngine {
                 'checks, not "clean"; a gate that must fail closed should treat critical_skipped ' +
                 'as a failure.');
         }
-        return {
+        const result = {
             id: generateEvalId(),
             eval_type: evalType,
             output_text: context.output,
@@ -288,7 +330,11 @@ export class EvalEngine {
             ...(overall.criticalFailures.length > 0 ? { critical_failures: overall.criticalFailures } : {}),
             ...(overall.criticalSkipped.length > 0 ? { critical_skipped: overall.criticalSkipped } : {}),
             ...(perCategory ? { categories: perCategory } : {}),
+            coverage,
+            provenance,
         };
+        result.verdict = deriveVerdict(result, this.threshold);
+        return result;
     }
     /**
      * Weighted average over the rules that ran, plus the critical veto.

package/dist/eval/failure-classes.d.ts ADDED Viewed

@@ -0,0 +1,8 @@
+import type { FailureClass, Need } from '../types/eval.js';
+export interface FailureClassEntry {
+    id: FailureClass;
+    text: string;
+}
+export declare const FAILURE_CLASSES: readonly FailureClassEntry[];
+export declare const FAILURE_CLASS_IDS: readonly FailureClass[];
+export declare const NEEDS: readonly Need[];

package/dist/eval/failure-classes.js ADDED Viewed

@@ -0,0 +1,18 @@
+export const FAILURE_CLASSES = [
+    { id: 'pii_leak', text: 'Personal data in the output (SSN, card, phone, email, date of birth, address …)' },
+    { id: 'credential_leak', text: 'A secret in the output (API key, token, private key, seed phrase)' },
+    { id: 'injection', text: 'Injection-shaped content in the output: attack phrasing or a directive aimed at an evaluator or a downstream system' },
+    { id: 'injection_compliance', text: 'The agent read an injected instruction in a tool result and obeyed it' },
+    { id: 'silent_tool_failure', text: 'A tool call failed and the output never acknowledges it' },
+    { id: 'tool_loop', text: 'The same call repeated past the configured limit with the same result' },
+    { id: 'stub', text: 'A placeholder, a deferral or a promise instead of the work' },
+    { id: 'fabrication', text: 'A claim that contradicts the material the agent was given' },
+    { id: 'ungrounded', text: 'An identifier, number or citation in the output that nothing the agent read supports' },
+    { id: 'incomplete_ask', text: 'Part of a multi-part ask was not answered' },
+    { id: 'off_task', text: 'The output does not address what was asked' },
+    { id: 'over_budget', text: 'The run cost more than the deployment allows, in money or tokens' },
+    { id: 'format', text: 'The output fails a shape or size requirement (empty, too short, too few sentences, invalid JSON)' },
+    { id: 'invalid_tool_call', text: 'A tool was called that is not in the catalogue, or with arguments its schema rejects' },
+];
+export const FAILURE_CLASS_IDS = FAILURE_CLASSES.map((c) => c.id);
+export const NEEDS = ['output', 'input', 'expected', 'tool_calls', 'tool_outputs', 'tools_catalogue', 'cost', 'tokens', 'citations'];

package/dist/eval/llm-judge/evaluator.d.ts CHANGED Viewed

@@ -1,5 +1,15 @@
 import { type LLMProvider } from './client.js';
 import { type TemplateName } from './templates/index.js';
+/**
+ * The pre-check refused the call: the worst-case spend (two attempts) would
+ * exceed the cap. Typed so the tool can answer IRIS_BUDGET_EXCEEDED with both
+ * numbers; nothing was spent.
+ */
+export declare class CostCapError extends Error {
+    readonly estimatedUsd: number;
+    readonly capUsd: number;
+    constructor(estimatedUsd: number, capUsd: number);
+}
 export interface LLMJudgeEvaluateParams {
     output: string;
     template: TemplateName;

package/dist/eval/llm-judge/evaluator.js CHANGED Viewed

@@ -1,6 +1,21 @@
 import { callLLMJudge, estimateInputTokens, LLMJudgeError } from './client.js';
 import { estimateCostUsd, findPricing } from './pricing.js';
 import { getTemplate } from './templates/index.js';
+/**
+ * The pre-check refused the call: the worst-case spend (two attempts) would
+ * exceed the cap. Typed so the tool can answer IRIS_BUDGET_EXCEEDED with both
+ * numbers; nothing was spent.
+ */
+export class CostCapError extends Error {
+    estimatedUsd;
+    capUsd;
+    constructor(estimatedUsd, capUsd) {
+        super(`Estimated max cost ${estimatedUsd.toFixed(4)} USD (including one retry on a malformed judge reply) exceeds cap ${capUsd.toFixed(4)} USD — refusing to call. Raise IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or max_cost_usd, or trim prompts/maxOutputTokens.`);
+        this.estimatedUsd = estimatedUsd;
+        this.capUsd = capUsd;
+        this.name = 'CostCapError';
+    }
+}
 // Malformed judge response — retried once by `evaluate`, surfaced as
 // LLMJudgeError('malformed_response') if the retry also fails.
 function parseJudgeResponse(raw) {
@@ -88,7 +103,7 @@ export async function evaluateWithLLMJudge(params) {
     const retryCost = estimateCostUsd(params.model, estimateInputTokens(strictSystem, userPrompt), retryMaxOutputTokens);
     const estimatedCost = firstAttemptCost === null || retryCost === null ? null : firstAttemptCost + retryCost;
     if (estimatedCost !== null && estimatedCost > maxCost) {
-        throw new Error(`Estimated max cost ${estimatedCost.toFixed(4)} USD (including one retry on a malformed judge reply) exceeds cap ${maxCost.toFixed(4)} USD — refusing to call. Raise IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or trim prompts/maxOutputTokens.`);
+        throw new CostCapError(estimatedCost, maxCost);
     }
     // First attempt
     let raw = await callLLMJudge({