@iris-eval/mcp-server 0.6.0 → 0.8.0

package/dist/tools/trace-link.d.ts

@@ -1,7 +1,17 @@
 import type { IStorageAdapter } from '../types/query.js';
 import type { EvalResult } from '../types/eval.js';
+import type { Trace } from '../types/trace.js';
 import type { TenantId } from '../types/tenant.js';
 export declare function unknownTraceMessage(traceId: string): string;
+/**
+ * The same refuse-before-any-work check, returning the row it already read.
+ *
+ * evaluate_output needs the trace itself (its `tool_calls`, so a caller who
+ * has already logged the trajectory does not have to resend it), and the
+ * existence check had to load the row anyway. Fetching it twice would be
+ * two reads for one fact — and two chances for them to disagree.
+ */
+export declare function getTraceOrThrow(storage: IStorageAdapter, tenantId: TenantId, traceId: string): Promise<Trace>;
 export declare function assertTraceExists(storage: IStorageAdapter, tenantId: TenantId, traceId: string): Promise<void>;
 /** insertEvalResult with the foreign-key race translated into the same clear message. */
 export declare function insertLinkedEvalResult(storage: IStorageAdapter, tenantId: TenantId, result: EvalResult): Promise<void>;

package/dist/tools/trace-link.js

@@ -18,10 +18,22 @@ export function unknownTraceMessage(traceId) {
         'Nothing was evaluated or written. Pass the trace_id returned by log_trace (or listed by get_traces), ' +
         'or omit trace_id to store an unlinked evaluation.');
 }
-export async function assertTraceExists(storage, tenantId, traceId) {
+/**
+ * The same refuse-before-any-work check, returning the row it already read.
+ *
+ * evaluate_output needs the trace itself (its `tool_calls`, so a caller who
+ * has already logged the trajectory does not have to resend it), and the
+ * existence check had to load the row anyway. Fetching it twice would be
+ * two reads for one fact — and two chances for them to disagree.
+ */
+export async function getTraceOrThrow(storage, tenantId, traceId) {
     const trace = await storage.getTrace(tenantId, traceId);
     if (!trace)
         throw new Error(unknownTraceMessage(traceId));
+    return trace;
+}
+export async function assertTraceExists(storage, tenantId, traceId) {
+    await getTraceOrThrow(storage, tenantId, traceId);
 }
 /** insertEvalResult with the foreign-key race translated into the same clear message. */
 export async function insertLinkedEvalResult(storage, tenantId, result) {

package/dist/types/config.d.ts

@@ -32,7 +32,21 @@ export interface IrisConfig {
             topic_consistency?: number;
             cost_threshold?: number;
             max_token_ratio?: number;
+            max_tool_repeats?: number;
         };
+        /**
+         * Built-in rule names promoted to CRITICAL — a failure vetoes `passed`
+         * regardless of the weighted score. Validated against the rule registry
+         * when the config loads; an unknown name is a startup error naming the
+         * valid list, never a silent no-op.
+         */
+        criticalRules?: string[];
+        /**
+         * Built-in rule names demoted from critical — they still score and still
+         * report a failure, but they stop vetoing `passed`. Same validation. A
+         * name in both lists is a config error: it does not say what you want.
+         */
+        nonCriticalRules?: string[];
     };
     logging: {
         level: 'debug' | 'info' | 'warn' | 'error';

package/dist/types/eval.d.ts

@@ -1,3 +1,4 @@
+import type { ToolCallRecord } from './trace.js';
 export type EvalType = 'completeness' | 'relevance' | 'safety' | 'cost' | 'custom';
 /**
  * What an EvalResult can be tagged as: a single bundle (EvalType), or
@@ -30,11 +31,19 @@ export interface EvalContext {
     output: string;
     expected?: string;
     input?: string;
-    toolCalls?: Array<{
-        tool_name: string;
-        input?: unknown;
-        output?: unknown;
-    }>;
+    /**
+     * The agent's trajectory — what it actually DID, in call order.
+     *
+     * Deliberately the SAME record the capture path stores (ToolCallRecord =
+     * log_trace's `tool_calls[]`), not a narrower local shape. It used to be
+     * a three-field inline type without `error`, so a rule could see that a
+     * tool was called but never that it FAILED: the acceptance pass found
+     * three real transcripts that answered confidently after a grep exited 1,
+     * an ls hit a missing directory and a node -e threw, and no rule could
+     * reach the fact. Re-declaring a subset here would reintroduce exactly
+     * that gap the next time a field is added to the capture shape.
+     */
+    toolCalls?: ToolCallRecord[];
     tokenUsage?: {
         prompt_tokens?: number;
         completion_tokens?: number;
@@ -71,6 +80,22 @@ export interface EvalRuleResult {
      * regroup them.
      */
     category?: EvalType;
+    /**
+     * Whether this rule VETOES the verdict — its EFFECTIVE criticality, after
+     * `eval.criticalRules` / `eval.nonCriticalRules` are applied, not the
+     * value on the rule's definition. A reader holding a failed evaluation
+     * could otherwise not tell a hard violation from a low score without
+     * knowing the rule library by heart.
+     */
+    critical?: boolean;
+    /**
+     * Who decided that: 'default' is the rule's own declaration (for a
+     * deployed custom rule, the severity it was deployed with); 'config' means
+     * one of the two override lists named it. The distinction is the point of
+     * making criticality configurable — an operator reading a verdict must be
+     * able to see that their own promotion caused it.
+     */
+    criticalSource?: 'default' | 'config';
     passed: boolean;
     score: number;
     message: string;
@@ -83,10 +108,17 @@ export interface EvalRuleResult {
  * Per-bundle verdict inside an eval_type="all" result. Same semantics as a
  * single-bundle EvalResult (threshold + critical veto), computed over that
  * bundle's rules only.
+ *
+ * `score` and `passed` are null when the bundle evaluated no rule (every
+ * rule skipped for missing context — cost without cost_usd, relevance
+ * without input). Such a bundle was not judged: it is neither passing nor
+ * failing, `insufficient_data` is true, and it never counted toward the
+ * overall verdict (#406). The top-level EvalResult keeps a boolean
+ * `passed` on purpose — a gate keyed on it must fail closed.
  */
 export interface EvalCategoryResult {
-    score: number;
-    passed: boolean;
+    score: number | null;
+    passed: boolean | null;
     rules_evaluated: number;
     rules_skipped: number;
     insufficient_data: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "description": "Stop shipping agents on vibes. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",
@@ -28,8 +28,15 @@
     "version:sync": "node scripts/sync-versions.mjs",
     "claims:capture-tests": "node scripts/claims/capture-tests.mjs",
     "claims:generate": "node scripts/claims/generate.mjs",
+    "claims:generate:live": "node scripts/claims/generate.mjs --live",
     "claims:check": "node scripts/claims/generate.mjs --check",
     "claims:check-hardcoded": "node scripts/claims/check-no-hardcoded.mjs",
+    "proof": "tsx proof/run.ts",
+    "proof:typecheck": "tsc -p proof/tsconfig.json",
+    "llms:render": "node scripts/claims/render-llms.mjs",
+    "llms:check": "node scripts/claims/render-llms.mjs --check",
+    "proof:judge": "tsx proof/judge/run.ts",
+    "proof:judge:typecheck": "tsc -p proof/judge/tsconfig.json",
     "clean": "rm -rf dist coverage",
     "seed:demo": "tsx scripts/seed-demo-data.ts",
     "demo": "tsx scripts/demo.ts"

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.6.0",
+  "version": "0.8.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.6.0",
+      "version": "0.8.0",
       "transport": {
         "type": "stdio"
       },