@iris-eval/mcp-server 0.8.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +9 -2
  2. package/dist/capabilities.d.ts +64 -0
  3. package/dist/capabilities.js +65 -0
  4. package/dist/config/defaults.js +2 -0
  5. package/dist/custom-rule-store.d.ts +4 -0
  6. package/dist/custom-rule-store.js +8 -3
  7. package/dist/dashboard/assets/{index-BfMShR3p.js → index-Cz8_oOqG.js} +1 -1
  8. package/dist/dashboard/index.html +1 -1
  9. package/dist/dashboard/routes/capabilities.d.ts +3 -0
  10. package/dist/dashboard/routes/capabilities.js +11 -0
  11. package/dist/dashboard/routes/health.d.ts +5 -1
  12. package/dist/dashboard/routes/health.js +15 -3
  13. package/dist/dashboard/routes/rules.js +4 -1
  14. package/dist/dashboard/routes/traces.d.ts +3 -0
  15. package/dist/dashboard/routes/traces.js +9 -28
  16. package/dist/dashboard/server.d.ts +2 -0
  17. package/dist/dashboard/server.js +6 -2
  18. package/dist/eval/accuracy.d.ts +41 -0
  19. package/dist/eval/accuracy.js +97 -0
  20. package/dist/eval/criticality.d.ts +8 -1
  21. package/dist/eval/criticality.js +6 -0
  22. package/dist/eval/dormant.d.ts +4 -0
  23. package/dist/eval/dormant.js +22 -0
  24. package/dist/eval/engine.d.ts +1 -0
  25. package/dist/eval/engine.js +49 -3
  26. package/dist/eval/failure-classes.d.ts +8 -0
  27. package/dist/eval/failure-classes.js +18 -0
  28. package/dist/eval/llm-judge/evaluator.d.ts +10 -0
  29. package/dist/eval/llm-judge/evaluator.js +16 -1
  30. package/dist/eval/published-accuracy.d.ts +230 -0
  31. package/dist/eval/published-accuracy.js +86 -0
  32. package/dist/eval/questions.d.ts +12 -0
  33. package/dist/eval/questions.js +14 -0
  34. package/dist/eval/response-schema.d.ts +652 -0
  35. package/dist/eval/response-schema.js +130 -0
  36. package/dist/eval/response.d.ts +12 -0
  37. package/dist/eval/response.js +30 -0
  38. package/dist/eval/rules/completeness.js +31 -0
  39. package/dist/eval/rules/cost.d.ts +1 -1
  40. package/dist/eval/rules/cost.js +44 -0
  41. package/dist/eval/rules/custom.d.ts +0 -12
  42. package/dist/eval/rules/custom.js +21 -0
  43. package/dist/eval/rules/relevance.js +16 -0
  44. package/dist/eval/rules/safety.js +178 -1
  45. package/dist/eval/stamp.d.ts +14 -0
  46. package/dist/eval/stamp.js +88 -0
  47. package/dist/eval/stats.d.ts +33 -0
  48. package/dist/eval/stats.js +109 -0
  49. package/dist/eval/verdict.d.ts +34 -0
  50. package/dist/eval/verdict.js +131 -0
  51. package/dist/index.js +5 -28
  52. package/dist/instructions.d.ts +17 -0
  53. package/dist/instructions.js +53 -0
  54. package/dist/judge-enablement.d.ts +34 -0
  55. package/dist/judge-enablement.js +78 -0
  56. package/dist/judge-enablement.json +10 -0
  57. package/dist/preferences.d.ts +1 -1
  58. package/dist/prompts.d.ts +3 -0
  59. package/dist/prompts.js +29 -0
  60. package/dist/resources/index.d.ts +5 -2
  61. package/dist/resources/index.js +65 -5
  62. package/dist/resources/uris.d.ts +12 -0
  63. package/dist/resources/uris.js +24 -0
  64. package/dist/retention.d.ts +20 -0
  65. package/dist/retention.js +44 -0
  66. package/dist/self-test.d.ts +1 -0
  67. package/dist/self-test.js +14 -0
  68. package/dist/server.d.ts +10 -1
  69. package/dist/server.js +34 -7
  70. package/dist/storage/index.js +1 -1
  71. package/dist/storage/migrations/007-eval-provenance.d.ts +3 -0
  72. package/dist/storage/migrations/007-eval-provenance.js +30 -0
  73. package/dist/storage/migrations/index.js +24 -4
  74. package/dist/storage/sqlite-adapter.d.ts +26 -1
  75. package/dist/storage/sqlite-adapter.js +135 -15
  76. package/dist/tools/delete-rule.d.ts +8 -0
  77. package/dist/tools/delete-rule.js +30 -38
  78. package/dist/tools/delete-trace.d.ts +5 -0
  79. package/dist/tools/delete-trace.js +24 -27
  80. package/dist/tools/deploy-rule.d.ts +13 -1
  81. package/dist/tools/deploy-rule.js +37 -34
  82. package/dist/tools/describe.d.ts +20 -0
  83. package/dist/tools/describe.js +36 -0
  84. package/dist/tools/errors.d.ts +36 -0
  85. package/dist/tools/errors.js +134 -0
  86. package/dist/tools/evaluate-output.d.ts +8 -1
  87. package/dist/tools/evaluate-output.js +37 -58
  88. package/dist/tools/evaluate-with-llm-judge.d.ts +31 -0
  89. package/dist/tools/evaluate-with-llm-judge.js +96 -69
  90. package/dist/tools/get-traces.d.ts +9 -0
  91. package/dist/tools/get-traces.js +30 -29
  92. package/dist/tools/index.d.ts +8 -0
  93. package/dist/tools/index.js +22 -1
  94. package/dist/tools/list-rules.d.ts +13 -0
  95. package/dist/tools/list-rules.js +43 -46
  96. package/dist/tools/log-trace.d.ts +4 -0
  97. package/dist/tools/log-trace.js +31 -29
  98. package/dist/tools/respond.d.ts +42 -0
  99. package/dist/tools/respond.js +90 -0
  100. package/dist/tools/strict-input.js +1 -1
  101. package/dist/tools/trace-link.d.ts +2 -0
  102. package/dist/tools/trace-link.js +13 -2
  103. package/dist/tools/verify-citations.d.ts +17 -2
  104. package/dist/tools/verify-citations.js +98 -93
  105. package/dist/types/config.d.ts +9 -0
  106. package/dist/types/eval.d.ts +258 -0
  107. package/dist/types/eval.js +2 -1
  108. package/dist/types/query.d.ts +2 -0
  109. package/package.json +1 -1
  110. package/server.json +72 -2
  111. package/dist/resources/dashboard-summary.d.ts +0 -3
  112. package/dist/resources/dashboard-summary.js +0 -16
  113. package/dist/resources/trace-detail.d.ts +0 -3
  114. package/dist/resources/trace-detail.js +0 -30
@@ -0,0 +1,652 @@
1
+ import { z } from 'zod';
2
+ export declare const intervalSchema: z.ZodObject<{
3
+ point: z.ZodNumber;
4
+ lo: z.ZodNumber;
5
+ hi: z.ZodNumber;
6
+ }, z.core.$loose>;
7
+ export declare const uncertaintySchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
8
+ basis: z.ZodLiteral<"published_accuracy">;
9
+ fired: z.ZodBoolean;
10
+ ppv: z.ZodOptional<z.ZodObject<{
11
+ point: z.ZodNumber;
12
+ lo: z.ZodNumber;
13
+ hi: z.ZodNumber;
14
+ }, z.core.$loose>>;
15
+ missRate: z.ZodOptional<z.ZodObject<{
16
+ point: z.ZodNumber;
17
+ lo: z.ZodNumber;
18
+ hi: z.ZodNumber;
19
+ }, z.core.$loose>>;
20
+ prior: z.ZodObject<{
21
+ pi: z.ZodNumber;
22
+ source: z.ZodEnum<{
23
+ default: "default";
24
+ config: "config";
25
+ estimated: "estimated";
26
+ }>;
27
+ }, z.core.$loose>;
28
+ corpus: z.ZodObject<{
29
+ n: z.ZodNumber;
30
+ tp: z.ZodNumber;
31
+ fp: z.ZodNumber;
32
+ fn: z.ZodNumber;
33
+ tn: z.ZodNumber;
34
+ version: z.ZodString;
35
+ release: z.ZodString;
36
+ labelling: z.ZodEnum<{
37
+ "same-model": "same-model";
38
+ "human-verified": "human-verified";
39
+ }>;
40
+ }, z.core.$loose>;
41
+ }, z.core.$loose>, z.ZodObject<{
42
+ basis: z.ZodLiteral<"definition">;
43
+ conformance: z.ZodObject<{
44
+ n: z.ZodNumber;
45
+ matched: z.ZodNumber;
46
+ }, z.core.$loose>;
47
+ }, z.core.$loose>, z.ZodObject<{
48
+ basis: z.ZodLiteral<"self_consistency">;
49
+ samples: z.ZodNumber;
50
+ voteFraction: z.ZodNumber;
51
+ scoreSd: z.ZodNumber;
52
+ }, z.core.$loose>, z.ZodObject<{
53
+ basis: z.ZodLiteral<"local_labels">;
54
+ precision: z.ZodObject<{
55
+ point: z.ZodNumber;
56
+ lo: z.ZodNumber;
57
+ hi: z.ZodNumber;
58
+ }, z.core.$loose>;
59
+ n: z.ZodNumber;
60
+ }, z.core.$loose>, z.ZodObject<{
61
+ basis: z.ZodLiteral<"policy">;
62
+ }, z.core.$loose>, z.ZodObject<{
63
+ basis: z.ZodLiteral<"unmeasured">;
64
+ why: z.ZodString;
65
+ }, z.core.$loose>], "basis">;
66
+ export declare const evidenceSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
67
+ type: z.ZodLiteral<"span">;
68
+ source: z.ZodString;
69
+ start: z.ZodNumber;
70
+ end: z.ZodNumber;
71
+ label: z.ZodString;
72
+ }, z.core.$loose>, z.ZodObject<{
73
+ type: z.ZodLiteral<"pattern">;
74
+ name: z.ZodString;
75
+ count: z.ZodNumber;
76
+ }, z.core.$loose>, z.ZodObject<{
77
+ type: z.ZodLiteral<"toolCall">;
78
+ index: z.ZodNumber;
79
+ toolName: z.ZodString;
80
+ label: z.ZodString;
81
+ }, z.core.$loose>, z.ZodObject<{
82
+ type: z.ZodLiteral<"citation">;
83
+ url: z.ZodString;
84
+ status: z.ZodEnum<{
85
+ resolved: "resolved";
86
+ dead: "dead";
87
+ unverifiable: "unverifiable";
88
+ supported: "supported";
89
+ unsupported: "unsupported";
90
+ }>;
91
+ }, z.core.$loose>, z.ZodObject<{
92
+ type: z.ZodLiteral<"count">;
93
+ stat: z.ZodString;
94
+ unit: z.ZodString;
95
+ value: z.ZodNumber;
96
+ threshold: z.ZodOptional<z.ZodNumber>;
97
+ thresholdSource: z.ZodOptional<z.ZodEnum<{
98
+ default: "default";
99
+ config: "config";
100
+ call: "call";
101
+ rule: "rule";
102
+ }>>;
103
+ }, z.core.$loose>], "type">;
104
+ export declare const measuredValueSchema: z.ZodObject<{
105
+ stat: z.ZodString;
106
+ unit: z.ZodString;
107
+ value: z.ZodNumber;
108
+ }, z.core.$loose>;
109
+ export declare const claimKindSchema: z.ZodEnum<{
110
+ measurement: "measurement";
111
+ detection: "detection";
112
+ inference: "inference";
113
+ judgment: "judgment";
114
+ policy: "policy";
115
+ verification: "verification";
116
+ }>;
117
+ export declare const roleSchema: z.ZodEnum<{
118
+ gate: "gate";
119
+ veto: "veto";
120
+ risk: "risk";
121
+ advisory: "advisory";
122
+ term: "term";
123
+ }>;
124
+ export declare const skipClassSchema: z.ZodEnum<{
125
+ not_applicable: "not_applicable";
126
+ defeated: "defeated";
127
+ config_invalid: "config_invalid";
128
+ }>;
129
+ export declare const needSchema: z.ZodEnum<{
130
+ cost: "cost";
131
+ output: "output";
132
+ input: "input";
133
+ expected: "expected";
134
+ tool_calls: "tool_calls";
135
+ tool_outputs: "tool_outputs";
136
+ tools_catalogue: "tools_catalogue";
137
+ tokens: "tokens";
138
+ citations: "citations";
139
+ }>;
140
+ export declare const questionIdSchema: z.ZodEnum<{
141
+ safe_output: "safe_output";
142
+ grounded: "grounded";
143
+ complete: "complete";
144
+ relevant: "relevant";
145
+ task_completed: "task_completed";
146
+ tool_use_correct: "tool_use_correct";
147
+ within_budget: "within_budget";
148
+ }>;
149
+ export declare const evalRuleResultSchema: z.ZodObject<{
150
+ ruleName: z.ZodString;
151
+ ruleId: z.ZodOptional<z.ZodString>;
152
+ category: z.ZodOptional<z.ZodEnum<{
153
+ completeness: "completeness";
154
+ relevance: "relevance";
155
+ safety: "safety";
156
+ cost: "cost";
157
+ custom: "custom";
158
+ }>>;
159
+ critical: z.ZodOptional<z.ZodBoolean>;
160
+ criticalSource: z.ZodOptional<z.ZodEnum<{
161
+ default: "default";
162
+ config: "config";
163
+ }>>;
164
+ passed: z.ZodBoolean;
165
+ score: z.ZodNumber;
166
+ message: z.ZodString;
167
+ skipped: z.ZodOptional<z.ZodBoolean>;
168
+ skipReason: z.ZodOptional<z.ZodString>;
169
+ configInvalid: z.ZodOptional<z.ZodBoolean>;
170
+ budgetExceeded: z.ZodOptional<z.ZodBoolean>;
171
+ kind: z.ZodOptional<z.ZodEnum<{
172
+ measurement: "measurement";
173
+ detection: "detection";
174
+ inference: "inference";
175
+ judgment: "judgment";
176
+ policy: "policy";
177
+ verification: "verification";
178
+ }>>;
179
+ role: z.ZodOptional<z.ZodEnum<{
180
+ gate: "gate";
181
+ veto: "veto";
182
+ risk: "risk";
183
+ advisory: "advisory";
184
+ term: "term";
185
+ }>>;
186
+ question: z.ZodOptional<z.ZodEnum<{
187
+ safe_output: "safe_output";
188
+ grounded: "grounded";
189
+ complete: "complete";
190
+ relevant: "relevant";
191
+ task_completed: "task_completed";
192
+ tool_use_correct: "tool_use_correct";
193
+ within_budget: "within_budget";
194
+ }>>;
195
+ classes: z.ZodOptional<z.ZodArray<z.ZodString>>;
196
+ ruleVersion: z.ZodOptional<z.ZodNumber>;
197
+ saw: z.ZodOptional<z.ZodArray<z.ZodEnum<{
198
+ cost: "cost";
199
+ output: "output";
200
+ input: "input";
201
+ expected: "expected";
202
+ tool_calls: "tool_calls";
203
+ tool_outputs: "tool_outputs";
204
+ tools_catalogue: "tools_catalogue";
205
+ tokens: "tokens";
206
+ citations: "citations";
207
+ }>>>;
208
+ skipClass: z.ZodOptional<z.ZodEnum<{
209
+ not_applicable: "not_applicable";
210
+ defeated: "defeated";
211
+ config_invalid: "config_invalid";
212
+ }>>;
213
+ uncertainty: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
214
+ basis: z.ZodLiteral<"published_accuracy">;
215
+ fired: z.ZodBoolean;
216
+ ppv: z.ZodOptional<z.ZodObject<{
217
+ point: z.ZodNumber;
218
+ lo: z.ZodNumber;
219
+ hi: z.ZodNumber;
220
+ }, z.core.$loose>>;
221
+ missRate: z.ZodOptional<z.ZodObject<{
222
+ point: z.ZodNumber;
223
+ lo: z.ZodNumber;
224
+ hi: z.ZodNumber;
225
+ }, z.core.$loose>>;
226
+ prior: z.ZodObject<{
227
+ pi: z.ZodNumber;
228
+ source: z.ZodEnum<{
229
+ default: "default";
230
+ config: "config";
231
+ estimated: "estimated";
232
+ }>;
233
+ }, z.core.$loose>;
234
+ corpus: z.ZodObject<{
235
+ n: z.ZodNumber;
236
+ tp: z.ZodNumber;
237
+ fp: z.ZodNumber;
238
+ fn: z.ZodNumber;
239
+ tn: z.ZodNumber;
240
+ version: z.ZodString;
241
+ release: z.ZodString;
242
+ labelling: z.ZodEnum<{
243
+ "same-model": "same-model";
244
+ "human-verified": "human-verified";
245
+ }>;
246
+ }, z.core.$loose>;
247
+ }, z.core.$loose>, z.ZodObject<{
248
+ basis: z.ZodLiteral<"definition">;
249
+ conformance: z.ZodObject<{
250
+ n: z.ZodNumber;
251
+ matched: z.ZodNumber;
252
+ }, z.core.$loose>;
253
+ }, z.core.$loose>, z.ZodObject<{
254
+ basis: z.ZodLiteral<"self_consistency">;
255
+ samples: z.ZodNumber;
256
+ voteFraction: z.ZodNumber;
257
+ scoreSd: z.ZodNumber;
258
+ }, z.core.$loose>, z.ZodObject<{
259
+ basis: z.ZodLiteral<"local_labels">;
260
+ precision: z.ZodObject<{
261
+ point: z.ZodNumber;
262
+ lo: z.ZodNumber;
263
+ hi: z.ZodNumber;
264
+ }, z.core.$loose>;
265
+ n: z.ZodNumber;
266
+ }, z.core.$loose>, z.ZodObject<{
267
+ basis: z.ZodLiteral<"policy">;
268
+ }, z.core.$loose>, z.ZodObject<{
269
+ basis: z.ZodLiteral<"unmeasured">;
270
+ why: z.ZodString;
271
+ }, z.core.$loose>], "basis">>;
272
+ evidence: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
273
+ type: z.ZodLiteral<"span">;
274
+ source: z.ZodString;
275
+ start: z.ZodNumber;
276
+ end: z.ZodNumber;
277
+ label: z.ZodString;
278
+ }, z.core.$loose>, z.ZodObject<{
279
+ type: z.ZodLiteral<"pattern">;
280
+ name: z.ZodString;
281
+ count: z.ZodNumber;
282
+ }, z.core.$loose>, z.ZodObject<{
283
+ type: z.ZodLiteral<"toolCall">;
284
+ index: z.ZodNumber;
285
+ toolName: z.ZodString;
286
+ label: z.ZodString;
287
+ }, z.core.$loose>, z.ZodObject<{
288
+ type: z.ZodLiteral<"citation">;
289
+ url: z.ZodString;
290
+ status: z.ZodEnum<{
291
+ resolved: "resolved";
292
+ dead: "dead";
293
+ unverifiable: "unverifiable";
294
+ supported: "supported";
295
+ unsupported: "unsupported";
296
+ }>;
297
+ }, z.core.$loose>, z.ZodObject<{
298
+ type: z.ZodLiteral<"count">;
299
+ stat: z.ZodString;
300
+ unit: z.ZodString;
301
+ value: z.ZodNumber;
302
+ threshold: z.ZodOptional<z.ZodNumber>;
303
+ thresholdSource: z.ZodOptional<z.ZodEnum<{
304
+ default: "default";
305
+ config: "config";
306
+ call: "call";
307
+ rule: "rule";
308
+ }>>;
309
+ }, z.core.$loose>], "type">>>;
310
+ value: z.ZodOptional<z.ZodObject<{
311
+ stat: z.ZodString;
312
+ unit: z.ZodString;
313
+ value: z.ZodNumber;
314
+ }, z.core.$loose>>;
315
+ }, z.core.$loose>;
316
+ export declare const evalCategoryResultSchema: z.ZodObject<{
317
+ score: z.ZodNullable<z.ZodNumber>;
318
+ passed: z.ZodNullable<z.ZodBoolean>;
319
+ rules_evaluated: z.ZodNumber;
320
+ rules_skipped: z.ZodNumber;
321
+ insufficient_data: z.ZodBoolean;
322
+ critical_failures: z.ZodOptional<z.ZodArray<z.ZodString>>;
323
+ critical_skipped: z.ZodOptional<z.ZodArray<z.ZodString>>;
324
+ }, z.core.$loose>;
325
+ export declare const coverageSchema: z.ZodObject<{
326
+ inputs: z.ZodRecord<z.ZodString, z.ZodBoolean>;
327
+ questions: z.ZodArray<z.ZodObject<{
328
+ id: z.ZodEnum<{
329
+ safe_output: "safe_output";
330
+ grounded: "grounded";
331
+ complete: "complete";
332
+ relevant: "relevant";
333
+ task_completed: "task_completed";
334
+ tool_use_correct: "tool_use_correct";
335
+ within_budget: "within_budget";
336
+ }>;
337
+ status: z.ZodEnum<{
338
+ not_applicable: "not_applicable";
339
+ judged: "judged";
340
+ unjudged: "unjudged";
341
+ }>;
342
+ why: z.ZodOptional<z.ZodString>;
343
+ }, z.core.$loose>>;
344
+ dormant: z.ZodOptional<z.ZodArray<z.ZodObject<{
345
+ ruleId: z.ZodString;
346
+ name: z.ZodString;
347
+ reason: z.ZodString;
348
+ }, z.core.$loose>>>;
349
+ }, z.core.$loose>;
350
+ export declare const verdictSchema: z.ZodObject<{
351
+ state: z.ZodEnum<{
352
+ pass: "pass";
353
+ fail: "fail";
354
+ unknown: "unknown";
355
+ }>;
356
+ passed: z.ZodBoolean;
357
+ basis: z.ZodEnum<{
358
+ policy_gate: "policy_gate";
359
+ detector_veto: "detector_veto";
360
+ critical_unknown: "critical_unknown";
361
+ required_evidence_missing: "required_evidence_missing";
362
+ risk_over_loss: "risk_over_loss";
363
+ score_below_threshold: "score_below_threshold";
364
+ clean: "clean";
365
+ no_rules: "no_rules";
366
+ }>;
367
+ by: z.ZodArray<z.ZodString>;
368
+ risk: z.ZodNullable<z.ZodObject<{
369
+ pBad: z.ZodNumber;
370
+ lo: z.ZodNumber;
371
+ hi: z.ZodNumber;
372
+ }, z.core.$loose>>;
373
+ confidence: z.ZodOptional<z.ZodEnum<{
374
+ decisive: "decisive";
375
+ marginal: "marginal";
376
+ }>>;
377
+ }, z.core.$loose>;
378
+ export declare const provenanceSchema: z.ZodObject<{
379
+ irisVersion: z.ZodString;
380
+ rulesetHash: z.ZodString;
381
+ configHash: z.ZodString;
382
+ thresholds: z.ZodObject<{
383
+ default: z.ZodNumber;
384
+ perRule: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
385
+ }, z.core.$loose>;
386
+ corpusVersion: z.ZodString;
387
+ judgedAt: z.ZodString;
388
+ }, z.core.$loose>;
389
+ /** The `evaluate_output` response — the same object the engine returns plus the tool's own fields. */
390
+ export declare const evaluateOutputResponseSchema: z.ZodObject<{
391
+ id: z.ZodString;
392
+ trace_id: z.ZodOptional<z.ZodString>;
393
+ verdict: z.ZodOptional<z.ZodObject<{
394
+ state: z.ZodEnum<{
395
+ pass: "pass";
396
+ fail: "fail";
397
+ unknown: "unknown";
398
+ }>;
399
+ passed: z.ZodBoolean;
400
+ basis: z.ZodEnum<{
401
+ policy_gate: "policy_gate";
402
+ detector_veto: "detector_veto";
403
+ critical_unknown: "critical_unknown";
404
+ required_evidence_missing: "required_evidence_missing";
405
+ risk_over_loss: "risk_over_loss";
406
+ score_below_threshold: "score_below_threshold";
407
+ clean: "clean";
408
+ no_rules: "no_rules";
409
+ }>;
410
+ by: z.ZodArray<z.ZodString>;
411
+ risk: z.ZodNullable<z.ZodObject<{
412
+ pBad: z.ZodNumber;
413
+ lo: z.ZodNumber;
414
+ hi: z.ZodNumber;
415
+ }, z.core.$loose>>;
416
+ confidence: z.ZodOptional<z.ZodEnum<{
417
+ decisive: "decisive";
418
+ marginal: "marginal";
419
+ }>>;
420
+ }, z.core.$loose>>;
421
+ coverage: z.ZodOptional<z.ZodObject<{
422
+ inputs: z.ZodRecord<z.ZodString, z.ZodBoolean>;
423
+ questions: z.ZodArray<z.ZodObject<{
424
+ id: z.ZodEnum<{
425
+ safe_output: "safe_output";
426
+ grounded: "grounded";
427
+ complete: "complete";
428
+ relevant: "relevant";
429
+ task_completed: "task_completed";
430
+ tool_use_correct: "tool_use_correct";
431
+ within_budget: "within_budget";
432
+ }>;
433
+ status: z.ZodEnum<{
434
+ not_applicable: "not_applicable";
435
+ judged: "judged";
436
+ unjudged: "unjudged";
437
+ }>;
438
+ why: z.ZodOptional<z.ZodString>;
439
+ }, z.core.$loose>>;
440
+ dormant: z.ZodOptional<z.ZodArray<z.ZodObject<{
441
+ ruleId: z.ZodString;
442
+ name: z.ZodString;
443
+ reason: z.ZodString;
444
+ }, z.core.$loose>>>;
445
+ }, z.core.$loose>>;
446
+ provenance: z.ZodOptional<z.ZodObject<{
447
+ irisVersion: z.ZodString;
448
+ rulesetHash: z.ZodString;
449
+ configHash: z.ZodString;
450
+ thresholds: z.ZodObject<{
451
+ default: z.ZodNumber;
452
+ perRule: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
453
+ }, z.core.$loose>;
454
+ corpusVersion: z.ZodString;
455
+ judgedAt: z.ZodString;
456
+ }, z.core.$loose>>;
457
+ erased_at: z.ZodOptional<z.ZodString>;
458
+ eval_type: z.ZodEnum<{
459
+ completeness: "completeness";
460
+ relevance: "relevance";
461
+ safety: "safety";
462
+ cost: "cost";
463
+ custom: "custom";
464
+ all: "all";
465
+ }>;
466
+ score: z.ZodNumber;
467
+ passed: z.ZodBoolean;
468
+ rule_results: z.ZodArray<z.ZodObject<{
469
+ ruleName: z.ZodString;
470
+ ruleId: z.ZodOptional<z.ZodString>;
471
+ category: z.ZodOptional<z.ZodEnum<{
472
+ completeness: "completeness";
473
+ relevance: "relevance";
474
+ safety: "safety";
475
+ cost: "cost";
476
+ custom: "custom";
477
+ }>>;
478
+ critical: z.ZodOptional<z.ZodBoolean>;
479
+ criticalSource: z.ZodOptional<z.ZodEnum<{
480
+ default: "default";
481
+ config: "config";
482
+ }>>;
483
+ passed: z.ZodBoolean;
484
+ score: z.ZodNumber;
485
+ message: z.ZodString;
486
+ skipped: z.ZodOptional<z.ZodBoolean>;
487
+ skipReason: z.ZodOptional<z.ZodString>;
488
+ configInvalid: z.ZodOptional<z.ZodBoolean>;
489
+ budgetExceeded: z.ZodOptional<z.ZodBoolean>;
490
+ kind: z.ZodOptional<z.ZodEnum<{
491
+ measurement: "measurement";
492
+ detection: "detection";
493
+ inference: "inference";
494
+ judgment: "judgment";
495
+ policy: "policy";
496
+ verification: "verification";
497
+ }>>;
498
+ role: z.ZodOptional<z.ZodEnum<{
499
+ gate: "gate";
500
+ veto: "veto";
501
+ risk: "risk";
502
+ advisory: "advisory";
503
+ term: "term";
504
+ }>>;
505
+ question: z.ZodOptional<z.ZodEnum<{
506
+ safe_output: "safe_output";
507
+ grounded: "grounded";
508
+ complete: "complete";
509
+ relevant: "relevant";
510
+ task_completed: "task_completed";
511
+ tool_use_correct: "tool_use_correct";
512
+ within_budget: "within_budget";
513
+ }>>;
514
+ classes: z.ZodOptional<z.ZodArray<z.ZodString>>;
515
+ ruleVersion: z.ZodOptional<z.ZodNumber>;
516
+ saw: z.ZodOptional<z.ZodArray<z.ZodEnum<{
517
+ cost: "cost";
518
+ output: "output";
519
+ input: "input";
520
+ expected: "expected";
521
+ tool_calls: "tool_calls";
522
+ tool_outputs: "tool_outputs";
523
+ tools_catalogue: "tools_catalogue";
524
+ tokens: "tokens";
525
+ citations: "citations";
526
+ }>>>;
527
+ skipClass: z.ZodOptional<z.ZodEnum<{
528
+ not_applicable: "not_applicable";
529
+ defeated: "defeated";
530
+ config_invalid: "config_invalid";
531
+ }>>;
532
+ uncertainty: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
533
+ basis: z.ZodLiteral<"published_accuracy">;
534
+ fired: z.ZodBoolean;
535
+ ppv: z.ZodOptional<z.ZodObject<{
536
+ point: z.ZodNumber;
537
+ lo: z.ZodNumber;
538
+ hi: z.ZodNumber;
539
+ }, z.core.$loose>>;
540
+ missRate: z.ZodOptional<z.ZodObject<{
541
+ point: z.ZodNumber;
542
+ lo: z.ZodNumber;
543
+ hi: z.ZodNumber;
544
+ }, z.core.$loose>>;
545
+ prior: z.ZodObject<{
546
+ pi: z.ZodNumber;
547
+ source: z.ZodEnum<{
548
+ default: "default";
549
+ config: "config";
550
+ estimated: "estimated";
551
+ }>;
552
+ }, z.core.$loose>;
553
+ corpus: z.ZodObject<{
554
+ n: z.ZodNumber;
555
+ tp: z.ZodNumber;
556
+ fp: z.ZodNumber;
557
+ fn: z.ZodNumber;
558
+ tn: z.ZodNumber;
559
+ version: z.ZodString;
560
+ release: z.ZodString;
561
+ labelling: z.ZodEnum<{
562
+ "same-model": "same-model";
563
+ "human-verified": "human-verified";
564
+ }>;
565
+ }, z.core.$loose>;
566
+ }, z.core.$loose>, z.ZodObject<{
567
+ basis: z.ZodLiteral<"definition">;
568
+ conformance: z.ZodObject<{
569
+ n: z.ZodNumber;
570
+ matched: z.ZodNumber;
571
+ }, z.core.$loose>;
572
+ }, z.core.$loose>, z.ZodObject<{
573
+ basis: z.ZodLiteral<"self_consistency">;
574
+ samples: z.ZodNumber;
575
+ voteFraction: z.ZodNumber;
576
+ scoreSd: z.ZodNumber;
577
+ }, z.core.$loose>, z.ZodObject<{
578
+ basis: z.ZodLiteral<"local_labels">;
579
+ precision: z.ZodObject<{
580
+ point: z.ZodNumber;
581
+ lo: z.ZodNumber;
582
+ hi: z.ZodNumber;
583
+ }, z.core.$loose>;
584
+ n: z.ZodNumber;
585
+ }, z.core.$loose>, z.ZodObject<{
586
+ basis: z.ZodLiteral<"policy">;
587
+ }, z.core.$loose>, z.ZodObject<{
588
+ basis: z.ZodLiteral<"unmeasured">;
589
+ why: z.ZodString;
590
+ }, z.core.$loose>], "basis">>;
591
+ evidence: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
592
+ type: z.ZodLiteral<"span">;
593
+ source: z.ZodString;
594
+ start: z.ZodNumber;
595
+ end: z.ZodNumber;
596
+ label: z.ZodString;
597
+ }, z.core.$loose>, z.ZodObject<{
598
+ type: z.ZodLiteral<"pattern">;
599
+ name: z.ZodString;
600
+ count: z.ZodNumber;
601
+ }, z.core.$loose>, z.ZodObject<{
602
+ type: z.ZodLiteral<"toolCall">;
603
+ index: z.ZodNumber;
604
+ toolName: z.ZodString;
605
+ label: z.ZodString;
606
+ }, z.core.$loose>, z.ZodObject<{
607
+ type: z.ZodLiteral<"citation">;
608
+ url: z.ZodString;
609
+ status: z.ZodEnum<{
610
+ resolved: "resolved";
611
+ dead: "dead";
612
+ unverifiable: "unverifiable";
613
+ supported: "supported";
614
+ unsupported: "unsupported";
615
+ }>;
616
+ }, z.core.$loose>, z.ZodObject<{
617
+ type: z.ZodLiteral<"count">;
618
+ stat: z.ZodString;
619
+ unit: z.ZodString;
620
+ value: z.ZodNumber;
621
+ threshold: z.ZodOptional<z.ZodNumber>;
622
+ thresholdSource: z.ZodOptional<z.ZodEnum<{
623
+ default: "default";
624
+ config: "config";
625
+ call: "call";
626
+ rule: "rule";
627
+ }>>;
628
+ }, z.core.$loose>], "type">>>;
629
+ value: z.ZodOptional<z.ZodObject<{
630
+ stat: z.ZodString;
631
+ unit: z.ZodString;
632
+ value: z.ZodNumber;
633
+ }, z.core.$loose>>;
634
+ }, z.core.$loose>>;
635
+ suggestions: z.ZodArray<z.ZodString>;
636
+ rules_evaluated: z.ZodNumber;
637
+ rules_skipped: z.ZodNumber;
638
+ insufficient_data: z.ZodBoolean;
639
+ critical_failures: z.ZodOptional<z.ZodArray<z.ZodString>>;
640
+ critical_skipped: z.ZodOptional<z.ZodArray<z.ZodString>>;
641
+ categories: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
642
+ score: z.ZodNullable<z.ZodNumber>;
643
+ passed: z.ZodNullable<z.ZodBoolean>;
644
+ rules_evaluated: z.ZodNumber;
645
+ rules_skipped: z.ZodNumber;
646
+ insufficient_data: z.ZodBoolean;
647
+ critical_failures: z.ZodOptional<z.ZodArray<z.ZodString>>;
648
+ critical_skipped: z.ZodOptional<z.ZodArray<z.ZodString>>;
649
+ }, z.core.$loose>>>;
650
+ note: z.ZodOptional<z.ZodString>;
651
+ }, z.core.$loose>;
652
+ export type EvaluateOutputResponse = z.infer<typeof evaluateOutputResponseSchema>;