@iris-eval/mcp-server 0.5.1 → 0.7.0

package/dist/types/eval.d.ts

@@ -1,4 +1,12 @@
 export type EvalType = 'completeness' | 'relevance' | 'safety' | 'cost' | 'custom';
+/**
+ * What an EvalResult can be tagged as: a single bundle (EvalType), or
+ * 'all' — evaluate_output's eval_type="all", which runs every bundle in one
+ * pass and reports a per-category breakdown beside the overall verdict.
+ * Kept apart from EvalType on purpose: rules are deployed and registered
+ * under a real bundle, never under 'all'.
+ */
+export type EvalResultType = EvalType | 'all';
 export interface EvalRule {
     name: string;
     description: string;
@@ -49,6 +57,20 @@ export interface EvalContext {
 }
 export interface EvalRuleResult {
     ruleName: string;
+    /**
+     * Deployed rule id (rule-<hex>) when the rule came from the custom-rule
+     * store. Absent for built-in rules and for inline custom_rules. Names are
+     * not unique — a same-name redeploy with replace:true mints a new id, and
+     * stores written before the same-name guard may hold duplicates — so this
+     * is the field that tells two same-named results apart (#373).
+     */
+    ruleId?: string;
+    /**
+     * The bundle this rule belongs to. Present only on eval_type="all"
+     * results, where rule_results spans every bundle and a reader needs to
+     * regroup them.
+     */
+    category?: EvalType;
     passed: boolean;
     score: number;
     message: string;
@@ -57,10 +79,31 @@ export interface EvalRuleResult {
     configInvalid?: boolean;
     budgetExceeded?: boolean;
 }
+/**
+ * Per-bundle verdict inside an eval_type="all" result. Same semantics as a
+ * single-bundle EvalResult (threshold + critical veto), computed over that
+ * bundle's rules only.
+ *
+ * `score` and `passed` are null when the bundle evaluated no rule (every
+ * rule skipped for missing context — cost without cost_usd, relevance
+ * without input). Such a bundle was not judged: it is neither passing nor
+ * failing, `insufficient_data` is true, and it never counted toward the
+ * overall verdict (#406). The top-level EvalResult keeps a boolean
+ * `passed` on purpose — a gate keyed on it must fail closed.
+ */
+export interface EvalCategoryResult {
+    score: number | null;
+    passed: boolean | null;
+    rules_evaluated: number;
+    rules_skipped: number;
+    insufficient_data: boolean;
+    critical_failures?: string[];
+    critical_skipped?: string[];
+}
 export interface EvalResult {
     id: string;
     trace_id?: string;
-    eval_type: EvalType;
+    eval_type: EvalResultType;
     output_text: string;
     expected_text?: string;
     score: number;
@@ -93,6 +136,14 @@ export interface EvalResult {
      * non-empty `critical_skipped` as "unknown", not as "clean".
      */
     critical_skipped?: string[];
+    /**
+     * Per-bundle breakdown, present only when eval_type is 'all'. Keyed by
+     * bundle; a bundle with no rules at all (nothing deployed under "custom"
+     * and no inline custom_rules) is absent rather than reported as
+     * insufficient. Response-only — not persisted as a column; the stored
+     * rule_results carry a `category` per rule so a reader can regroup.
+     */
+    categories?: Partial<Record<EvalType, EvalCategoryResult>>;
 }
 export type CustomRuleType = 'regex_match' | 'regex_no_match' | 'min_length' | 'max_length' | 'contains_keywords' | 'excludes_keywords' | 'json_schema' | 'cost_threshold';
 export interface CustomRuleDefinition {

package/dist/types/index.d.ts

@@ -1,5 +1,5 @@
 export type { SpanKind, SpanStatus, SpanEvent, ToolCallRecord, TokenUsage, Span, Trace, } from './trace.js';
-export type { EvalType, EvalRule, EvalContext, EvalRuleResult, EvalResult, CustomRuleType, CustomRuleDefinition, } from './eval.js';
+export type { EvalType, EvalResultType, EvalRule, EvalContext, EvalRuleResult, EvalResult, EvalCategoryResult, CustomRuleType, CustomRuleDefinition, } from './eval.js';
 export type { TraceFilter, TraceQueryOptions, TraceQueryResult, DashboardSummary, EvalStatsPeriod, EvalStats, EvalStatsTrendBucket, EvalStatsRuleBreakdown, EvalStatsFailure, IStorageAdapter, } from './query.js';
 export type { IrisConfig } from './config.js';
 export type { TenantId } from './tenant.js';

package/dist/types/query.d.ts

@@ -95,6 +95,31 @@ export interface IStorageAdapter {
     }>;
     getDashboardSummary(tenantId: TenantId, sinceHours?: number): Promise<DashboardSummary>;
     deleteTracesOlderThan(tenantId: TenantId, days: number): Promise<number>;
+    /**
+     * Retention twin of deleteTracesOlderThan for eval_results (#372).
+     * Deleting a trace only NULLs the trace_id on its evaluations (FK ON
+     * DELETE SET NULL), so every eval row — output_text verbatim, including
+     * whatever no_pii flagged — outlived the retention window until this
+     * existed. Cutoff is on created_at.
+     */
+    deleteEvalResultsOlderThan(tenantId: TenantId, days: number): Promise<number>;
+    /**
+     * Delete EVERY trace, span and eval result for the tenant, then compact
+     * the database so the deleted text does not linger in free pages or in
+     * the write-ahead log. Returns what was removed. Deployed rules, the
+     * audit log and preferences are not storage rows and are untouched.
+     */
+    purge(tenantId: TenantId): Promise<{
+        traces: number;
+        evalResults: number;
+    }>;
+    /**
+     * Fold the write-ahead log into the main file and truncate it
+     * (wal_checkpoint TRUNCATE). Best-effort; called after a retention sweep
+     * so rows deleted at startup do not survive as readable text in
+     * iris.db-wal.
+     */
+    checkpoint(): Promise<void>;
     /**
      * Delete a single trace by id. Cascades to spans via FK ON DELETE
      * CASCADE; eval_results get their trace_id set to NULL (so score

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.5.1",
+  "version": "0.7.0",
   "description": "Stop shipping agents on vibes. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",
@@ -28,8 +28,15 @@
     "version:sync": "node scripts/sync-versions.mjs",
     "claims:capture-tests": "node scripts/claims/capture-tests.mjs",
     "claims:generate": "node scripts/claims/generate.mjs",
+    "claims:generate:live": "node scripts/claims/generate.mjs --live",
     "claims:check": "node scripts/claims/generate.mjs --check",
     "claims:check-hardcoded": "node scripts/claims/check-no-hardcoded.mjs",
+    "proof": "tsx proof/run.ts",
+    "proof:typecheck": "tsc -p proof/tsconfig.json",
+    "llms:render": "node scripts/claims/render-llms.mjs",
+    "llms:check": "node scripts/claims/render-llms.mjs --check",
+    "proof:judge": "tsx proof/judge/run.ts",
+    "proof:judge:typecheck": "tsc -p proof/judge/tsconfig.json",
     "clean": "rm -rf dist coverage",
     "seed:demo": "tsx scripts/seed-demo-data.ts",
     "demo": "tsx scripts/demo.ts"

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.5.1",
+  "version": "0.7.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.5.1",
+      "version": "0.7.0",
       "transport": {
         "type": "stdio"
       },