@iris-eval/mcp-server 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. package/README.md +5 -1
  2. package/dist/config/defaults.js +3 -1
  3. package/dist/dashboard/assets/{index-CshLgDRB.js → index-CKs2Wbd_.js} +1 -1
  4. package/dist/dashboard/index.html +1 -1
  5. package/dist/dashboard/routes/traces.js +8 -2
  6. package/dist/dashboard/validation.d.ts +1 -1
  7. package/dist/dashboard/validation.js +6 -2
  8. package/dist/eval/engine.d.ts +17 -1
  9. package/dist/eval/engine.js +31 -3
  10. package/dist/eval/rules/relevance.d.ts +13 -0
  11. package/dist/eval/rules/relevance.js +185 -21
  12. package/dist/eval/rules/safety.d.ts +12 -1
  13. package/dist/eval/rules/safety.js +200 -20
  14. package/dist/tools/evaluate-output.js +19 -17
  15. package/dist/tools/strict-input.js +2 -2
  16. package/dist/types/eval.d.ts +9 -2
  17. package/package.json +8 -1
  18. package/server.json +2 -2
package/dist/dashboard/index.html CHANGED
@@ -6,7 +6,7 @@
6
6
  <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
7
7
  <!-- Stop shipping agents on vibes is filled from .claims.json brand.tagline at build time (vite.config.ts) — never restate the tagline here. -->
8
8
  <title>Iris — Stop shipping agents on vibes</title>
9
- <script type="module" crossorigin src="/assets/index-CshLgDRB.js"></script>
9
+ <script type="module" crossorigin src="/assets/index-CKs2Wbd_.js"></script>
10
10
  <link rel="stylesheet" crossorigin href="/assets/index-D0cFfBqn.css">
11
11
  </head>
12
12
  <body>
package/dist/dashboard/routes/traces.js CHANGED
@@ -2,6 +2,7 @@ import { requireTenant } from '../../middleware/tenant.js';
2
2
  import { generateTraceId, generateSpanId } from '../../utils/ids.js';
3
3
  import { bestEffortExport } from '../../otel/lazy.js';
4
4
  import { traceQuerySchema, ingestTraceSchema } from '../validation.js';
5
+ import { DEFAULT_EVAL_TYPE, DEFAULT_EVAL_TYPE_NOTE } from '../../eval/engine.js';
5
6
  export function registerTraceRoutes(router, storage, options) {
6
7
  /*
7
8
  * Deterministic capture over HTTP. MCP tool calls are model-
@@ -68,9 +69,13 @@ export function registerTraceRoutes(router, storage, options) {
68
69
  costUsd: body.cost_usd,
69
70
  tokenUsage: body.token_usage,
70
71
  };
71
- const evaluation = body.eval_type === 'all'
72
+ // An omitted eval_type runs every bundle — the same default, from the
73
+ // same constant, as the MCP tool — and says so in the response.
74
+ const evalTypeOmitted = body.eval_type === undefined;
75
+ const evalType = body.eval_type ?? DEFAULT_EVAL_TYPE;
76
+ const evaluation = evalType === 'all'
72
77
  ? options.evalEngine.evaluateAll(context)
73
- : options.evalEngine.evaluate(body.eval_type, context);
78
+ : options.evalEngine.evaluate(evalType, context);
74
79
  evaluation.trace_id = traceId;
75
80
  await storage.insertEvalResult(tenantId, evaluation);
76
81
  res.status(201).json({
@@ -103,6 +108,7 @@ export function registerTraceRoutes(router, storage, options) {
103
108
  : {}),
104
109
  // Per-bundle breakdown — eval_type="all" only.
105
110
  ...(evaluation.categories ? { categories: evaluation.categories } : {}),
111
+ ...(evalTypeOmitted ? { note: DEFAULT_EVAL_TYPE_NOTE } : {}),
106
112
  },
107
113
  });
108
114
  }
package/dist/dashboard/validation.d.ts CHANGED
@@ -4,7 +4,7 @@ export declare function strictBody<T extends z.ZodRawShape>(shape: T, opts?: {
4
4
  }): z.ZodObject<{ -readonly [P in keyof T]: T[P]; }, z.core.$strict>;
5
5
  export declare const ingestTraceSchema: z.ZodObject<{
6
6
  evaluate: z.ZodDefault<z.ZodBoolean>;
7
- eval_type: z.ZodDefault<z.ZodEnum<{
7
+ eval_type: z.ZodOptional<z.ZodEnum<{
8
8
  completeness: "completeness";
9
9
  relevance: "relevance";
10
10
  safety: "safety";
package/dist/dashboard/validation.js CHANGED
@@ -52,8 +52,12 @@ export const ingestTraceSchema = strictBody({
52
52
  // Same bundle list evaluate_output accepts, "all" included — the
53
53
  // ingest path used to stop one short and run the single-bundle engine
54
54
  // no matter what, so an HTTP caller could not get the per-category
55
- // verdict the MCP tool returns.
56
- eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom', 'all']).default('completeness'),
55
+ // verdict the MCP tool returns. Optional rather than defaulted HERE so
56
+ // the route can tell "chose all" from "never chose": the effective
57
+ // default is every bundle (DEFAULT_EVAL_TYPE in eval/engine.ts, the
58
+ // same constant evaluate_output reads), and an omitted eval_type gets a
59
+ // note in the response saying the default ran.
60
+ eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom', 'all']).optional(),
57
61
  }, {
58
62
  reserved: {
59
63
  trace_id: 'trace_id is minted by the server on every ingest and cannot be supplied by the client — ' +
package/dist/eval/engine.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { EvalRule, EvalContext, EvalResult, EvalType, CustomRuleDefinition } from '../types/eval.js';
1
+ import type { EvalRule, EvalContext, EvalResult, EvalResultType, EvalType, CustomRuleDefinition } from '../types/eval.js';
2
2
  /**
3
3
  * Every bundle eval_type="all" walks, in the order their categories are
4
4
  * reported. 'custom' is last: it holds only deployed rules registered under
@@ -6,6 +6,22 @@ import type { EvalRule, EvalContext, EvalResult, EvalType, CustomRuleDefinition
6
6
  * from the breakdown when neither exists.
7
7
  */
8
8
  export declare const ALL_EVAL_TYPES: readonly EvalType[];
9
+ /**
10
+ * What runs when a caller never chose a bundle. It used to be
11
+ * 'completeness', so a CI gate keyed on `passed` skipped PII and injection
12
+ * unless the caller knew to set eval_type — six of seven UAT personas read
13
+ * passed:true on PII-laden text with nothing in the payload saying the
14
+ * safety bundle had not run. Every bundle is the only default under which
15
+ * an omitted argument cannot silently narrow the verdict. The MCP tool and
16
+ * the HTTP ingest route both read this constant, so the two surfaces
17
+ * cannot default differently.
18
+ */
19
+ export declare const DEFAULT_EVAL_TYPE: EvalResultType;
20
+ /**
21
+ * The one-line note both surfaces attach when the default ran, so a reader
22
+ * of the response knows the bundle was chosen for them and how to narrow it.
23
+ */
24
+ export declare const DEFAULT_EVAL_TYPE_NOTE = "eval_type was omitted, so the default ran every bundle \u2014 completeness, relevance, safety, cost and any custom rules \u2014 the same as eval_type=\"all\"; pass a single bundle name to narrow the run.";
9
25
  export declare class EvalEngine {
10
26
  private additionalRules;
11
27
  /**
package/dist/eval/engine.js CHANGED
@@ -7,6 +7,22 @@ import { generateEvalId } from '../utils/ids.js';
7
7
  * from the breakdown when neither exists.
8
8
  */
9
9
  export const ALL_EVAL_TYPES = ['completeness', 'relevance', 'safety', 'cost', 'custom'];
10
+ /**
11
+ * What runs when a caller never chose a bundle. It used to be
12
+ * 'completeness', so a CI gate keyed on `passed` skipped PII and injection
13
+ * unless the caller knew to set eval_type — six of seven UAT personas read
14
+ * passed:true on PII-laden text with nothing in the payload saying the
15
+ * safety bundle had not run. Every bundle is the only default under which
16
+ * an omitted argument cannot silently narrow the verdict. The MCP tool and
17
+ * the HTTP ingest route both read this constant, so the two surfaces
18
+ * cannot default differently.
19
+ */
20
+ export const DEFAULT_EVAL_TYPE = 'all';
21
+ /**
22
+ * The one-line note both surfaces attach when the default ran, so a reader
23
+ * of the response knows the bundle was chosen for them and how to narrow it.
24
+ */
25
+ export const DEFAULT_EVAL_TYPE_NOTE = 'eval_type was omitted, so the default ran every bundle — completeness, relevance, safety, cost and any custom rules — the same as eval_type="all"; pass a single bundle name to narrow the run.';
10
26
  export class EvalEngine {
11
27
  additionalRules = new Map();
12
28
  /**
@@ -325,12 +341,24 @@ export class EvalEngine {
325
341
  if (indices.length === 0)
326
342
  continue;
327
343
  const verdict = this.summarize(indices.map((i) => rules[i]), indices.map((i) => ruleResults[i]));
344
+ /*
345
+ * A bundle whose every rule skipped was not judged (#406). Reporting
346
+ * it as passed:false / score:0 read as "failing" to anyone regrouping
347
+ * by category — cost "failed" on a call that carried no cost data.
348
+ * Inside the breakdown, null is the honest value: neither passing
349
+ * nor failing, and it never counted toward the overall verdict
350
+ * (summarize() already excludes skipped rules). The TOP-LEVEL
351
+ * `passed` is deliberately not made nullable — it is the verdict a
352
+ * gate keys on, and a gate must fail closed when nothing was judged;
353
+ * `insufficient_data: true` is the "unknown" marker at that level.
354
+ */
355
+ const judged = verdict.rulesEvaluated > 0;
328
356
  breakdown[type] = {
329
- score: Math.round(verdict.score * 1000) / 1000,
330
- passed: verdict.passed,
357
+ score: judged ? Math.round(verdict.score * 1000) / 1000 : null,
358
+ passed: judged ? verdict.passed : null,
331
359
  rules_evaluated: verdict.rulesEvaluated,
332
360
  rules_skipped: verdict.rulesSkipped,
333
- insufficient_data: verdict.rulesEvaluated === 0,
361
+ insufficient_data: !judged,
334
362
  ...(verdict.criticalFailures.length > 0 ? { critical_failures: verdict.criticalFailures } : {}),
335
363
  ...(verdict.criticalSkipped.length > 0 ? { critical_skipped: verdict.criticalSkipped } : {}),
336
364
  };
package/dist/eval/rules/relevance.d.ts CHANGED
@@ -1,4 +1,17 @@
1
1
  import type { EvalRule } from '../../types/eval.js';
2
+ /**
3
+ * Light stemmer: plurals, -ing/-ed/-ly, -ation/-ator/-ate/-ion, a trailing
4
+ * e, and a doubled final consonant. Crude on purpose (see the header):
5
+ * both sides are stemmed identically.
6
+ */
7
+ export declare function stemTerm(word: string): string;
8
+ /**
9
+ * Content terms of a text: fenced code removed, camelCase split, everything
10
+ * that is not a run of three or more letters treated as a separator (so
11
+ * paths, flags, snake_case and dotted identifiers fall apart into their
12
+ * words and numbers vanish), stopwords dropped, the rest stemmed.
13
+ */
14
+ export declare function contentTerms(text: string): string[];
2
15
  export declare const keywordOverlap: EvalRule;
3
16
  export declare const topicConsistency: EvalRule;
4
17
  export declare const relevanceRules: EvalRule[];
package/dist/eval/rules/relevance.js CHANGED
@@ -1,30 +1,154 @@
1
+ /*
2
+ * Relevance rules — one tokenizer, two DISTINCT signals.
3
+ *
4
+ * Redesigned after the arc-one acceptance pass ran twenty-four transcripts
5
+ * produced by an agent genuinely working against this repository
6
+ * (tests/fixtures/real-transcripts/). Three grounded, correct technical
7
+ * answers — what `--purge` does, what `eval_type: "all"` returns, a
8
+ * one-paragraph product description — failed topic_consistency at
9
+ * 6.7% / 3.6% / 2.0%. The old measure was the fraction of OUTPUT words that
10
+ * also appear in the INPUT, which punishes precisely what a good technical
11
+ * answer does: bring the source's vocabulary (identifiers, file names,
12
+ * exact values, mechanism) to a short question that did not contain it.
13
+ * The failure was structural — no threshold rescues a measure that reads
14
+ * new, correct vocabulary as drift — so the measure changed, not the number.
15
+ *
16
+ * keyword_overlap RECALL. What share of the ask's content terms does the
17
+ * output engage at all? Fails a refusal, a different
18
+ * product, filler, an answer that never touches the
19
+ * subject.
20
+ * topic_consistency CONTINUITY. What share of the output's content-
21
+ * bearing sentences connect to the ask — directly, or
22
+ * through an earlier connected sentence? Fails an answer
23
+ * that opens on topic and wanders, and everything
24
+ * keyword_overlap fails. Grounded answers chain their
25
+ * vocabulary back to the ask; a ramble does not.
26
+ *
27
+ * Both rules share the tokenizer below, so they agree on what a "term" is
28
+ * and no longer double-count one measurement:
29
+ * - stopwords (articles, pronouns, auxiliaries, question words, the
30
+ * request verbs — "explain", "summarise", "tell me" — and the form of
31
+ * the deliverable — "paragraph", "bullets", "summary") are not terms;
32
+ * - code identifiers, paths and flags are SPLIT into their words
33
+ * (`EvalEngine.evaluateAll()` → eval, engine, evaluate; `src/index.ts`
34
+ * → src, index) rather than dropped: the words inside an identifier
35
+ * ARE topic vocabulary, and dropping them was half of the old failure;
36
+ * - numbers and fenced code blocks are neutral (neither for nor against);
37
+ * - a light stemmer folds inflections (purge/purged/purging, rule/rules,
38
+ * evaluate/evaluation/evaluator) so the same word in a different form
39
+ * still counts. It is deliberately crude — both sides get the same
40
+ * treatment, so an imperfect stem only lowers sensitivity, never
41
+ * invents a match.
42
+ *
43
+ * Honest limits (lexical, no model): an answer that paraphrases the ask
44
+ * with none of its words reads as off topic; a coherent essay on the wrong
45
+ * subject that happens to reuse one of the ask's words reads as on topic.
46
+ * Semantic relevance is the LLM judge's job (evaluate_with_llm_judge,
47
+ * `relevance` template).
48
+ */
49
+ const STOPWORDS = new Set(('a an the and or nor but if then else than that this these those there here it its is are was were be been being ' +
50
+ 'am do does did done doing have has had having will would shall should can could may might must not no yes of in on ' +
51
+ 'at to for from by with without into onto over under about above below between among through during before after ' +
52
+ 'again further once out off up down as so such very really just only also too either neither both each every all any ' +
53
+ 'some few more most less least other another same own new old first second third next last one two three four five ' +
54
+ 'ten i me my mine we us our ours you your yours he him his she her hers they them their theirs who whom whose which ' +
55
+ 'what when where why how because while until unless since although though even ever never always often sometimes ' +
56
+ 'usually still yet already now anywhere everywhere something anything nothing everything someone anyone everyone ' +
57
+ 'nobody thing things way ways kind kinds sort sorts lot lots much many get gets got getting give gives gave given ' +
58
+ 'giving take takes took taken taking make makes made making use uses used using see sees saw seen seeing know knows ' +
59
+ 'knew known knowing think thinks thought thinking want wants wanted wanting need needs needed needing let lets tell ' +
60
+ 'tells told telling say says said saying ask asks asked asking read reads reading look looks looked looking find ' +
61
+ 'finds found finding show shows showed shown showing explain explains explained explaining describe describes ' +
62
+ 'described describing summarise summarize summarises summarizes summarised summarized answer answers answered ' +
63
+ 'answering question questions please help helps helped helping like likes liked well good bad better best right ' +
64
+ 'wrong true false able keep keeps kept put puts go goes went gone going come comes came coming back also etc via per ' +
65
+ // The FORM of the deliverable, not its subject — "a one-paragraph
66
+ // description", "a few bullets", "a short summary", "in detail".
67
+ 'paragraph paragraphs sentence sentences bullet bullets summary overview description brief briefly detail details ' +
68
+ 'detailed word words line lines short long quick quickly ' +
69
+ // URL and domain furniture — "iris-eval.com" splits into iris, eval, com.
70
+ 'com org net www http https').split(' '));
71
+ /**
72
+ * Light stemmer: plurals, -ing/-ed/-ly, -ation/-ator/-ate/-ion, a trailing
73
+ * e, and a doubled final consonant. Crude on purpose (see the header):
74
+ * both sides are stemmed identically.
75
+ */
76
+ export function stemTerm(word) {
77
+ let w = word;
78
+ if (w.length <= 3)
79
+ return w;
80
+ if (w.endsWith('ies'))
81
+ w = w.slice(0, -3) + 'i';
82
+ else if (w.endsWith('sses'))
83
+ w = w.slice(0, -2);
84
+ else if (w.endsWith('s') && !/(?:ss|us|is)$/.test(w))
85
+ w = w.slice(0, -1);
86
+ if (w.length > 5 && w.endsWith('ing'))
87
+ w = w.slice(0, -3);
88
+ else if (w.length > 4 && w.endsWith('ed'))
89
+ w = w.slice(0, -2);
90
+ else if (w.length > 4 && w.endsWith('ly'))
91
+ w = w.slice(0, -2);
92
+ else if (w.length > 6 && w.endsWith('ation'))
93
+ w = w.slice(0, -5);
94
+ else if (w.length > 5 && w.endsWith('ator'))
95
+ w = w.slice(0, -4);
96
+ else if (w.length > 5 && w.endsWith('ate'))
97
+ w = w.slice(0, -3);
98
+ else if (w.length > 5 && w.endsWith('ion'))
99
+ w = w.slice(0, -3);
100
+ if (w.length > 3 && w.endsWith('e'))
101
+ w = w.slice(0, -1);
102
+ if (w.length > 3 && /([^aeiou])\1$/.test(w) && !/[lsz]$/.test(w))
103
+ w = w.slice(0, -1);
104
+ return w;
105
+ }
106
+ const FENCED_CODE = /```[\s\S]*?```/g;
107
+ const CAMEL_BOUNDARY = /([a-z])([A-Z])/g;
108
+ const WORD = /[a-z]{3,}/g;
109
+ /**
110
+ * Content terms of a text: fenced code removed, camelCase split, everything
111
+ * that is not a run of three or more letters treated as a separator (so
112
+ * paths, flags, snake_case and dotted identifiers fall apart into their
113
+ * words and numbers vanish), stopwords dropped, the rest stemmed.
114
+ */
115
+ export function contentTerms(text) {
116
+ const terms = [];
117
+ const lowered = text.replace(FENCED_CODE, '\n').replace(CAMEL_BOUNDARY, '$1 $2').toLowerCase();
118
+ for (const match of lowered.matchAll(WORD)) {
119
+ if (STOPWORDS.has(match[0]))
120
+ continue;
121
+ terms.push(stemTerm(match[0]));
122
+ }
123
+ return terms;
124
+ }
1
125
  export const keywordOverlap = {
2
126
  name: 'keyword_overlap',
3
- description: 'Measures word overlap between input and output',
127
+ description: 'Recall of the input\'s content terms in the output: stopwords and request verbs are not terms, code identifiers and paths are split into their words, inflections are folded (purge/purged/purging). Passes when at least 35% of the input\'s terms appear in the output (configurable: keyword_overlap)',
4
128
  evalType: 'relevance',
5
129
  weight: 1,
6
130
  evaluate(context) {
7
131
  if (!context.input) {
8
132
  return { ruleName: 'keyword_overlap', passed: false, score: 0, message: 'No input provided', skipped: true, skipReason: 'context.input not provided' };
9
133
  }
10
- const inputWords = new Set(context.input.toLowerCase().split(/\W+/).filter((w) => w.length > 2));
11
- const outputWords = new Set(context.output.toLowerCase().split(/\W+/).filter((w) => w.length > 2));
12
- if (inputWords.size === 0) {
134
+ const inputTerms = new Set(contentTerms(context.input));
135
+ if (inputTerms.size === 0) {
13
136
  return { ruleName: 'keyword_overlap', passed: true, score: 1, message: 'No meaningful words in input' };
14
137
  }
138
+ const outputTerms = new Set(contentTerms(context.output));
15
139
  let overlap = 0;
16
- for (const word of inputWords) {
17
- if (outputWords.has(word))
140
+ for (const term of inputTerms) {
141
+ if (outputTerms.has(term))
18
142
  overlap++;
19
143
  }
20
- const ratio = overlap / inputWords.size;
144
+ const ratio = overlap / inputTerms.size;
21
145
  const threshold = context.customConfig?.keyword_overlap ?? 0.35;
22
146
  const passed = ratio >= threshold;
23
147
  return {
24
148
  ruleName: 'keyword_overlap',
25
149
  passed,
26
150
  score: Math.min(ratio * 2, 1),
27
- message: `${overlap}/${inputWords.size} input keywords found in output (${(ratio * 100).toFixed(0)}%)`,
151
+ message: `${overlap}/${inputTerms.size} input keywords found in output (${(ratio * 100).toFixed(0)}%)`,
28
152
  };
29
153
  },
30
154
  };
@@ -35,9 +159,22 @@ export const keywordOverlap = {
35
159
  * the dashboard's safety-violations panel, and the storage adapter's
36
160
  * violation counts have always placed it.
37
161
  */
162
+ /**
163
+ * A third of the content-bearing sentences must connect. Why a third and
164
+ * not half: the measure is a floor against drift, not a target — grounded
165
+ * answers in the real-transcript set connect 67–100% of their sentences —
166
+ * and the false positive that matters is the SHORT honest answer whose
167
+ * second and third sentences elaborate in fresh words ("It is sunny today.
168
+ * Expect a high of 75°F. Bring sunglasses."). At a half that answer fails;
169
+ * at a third it passes while "one on-topic sentence, then three about
170
+ * something else" (25%) still fails.
171
+ */
172
+ const DEFAULT_TOPIC_THRESHOLD = 1 / 3;
173
+ const LIST_ITEM = /^\s*(?:[-*+•]|\d{1,3}[.)])\s+/;
174
+ const SENTENCE_BREAK = /(?<=[.!?])\s+/;
38
175
  export const topicConsistency = {
39
176
  name: 'topic_consistency',
40
- description: 'Output stays on topic relative to input (skipped when output too brief for meaningful comparison)',
177
+ description: 'Continuity with the input: the share of the output\'s content-bearing sentences that connect to the input\'s topic — a sentence connects when it shares a content term with the input or with an earlier connected sentence (list items are read under the sentence that introduces them). Passes when at least a third connect (configurable: topic_consistency); a third, not half, so a short honest answer that elaborates in fresh words is not read as drift. Replaces the output-word-ratio measure that failed every grounded technical answer. Skipped when the output is too brief for meaningful comparison',
41
178
  evalType: 'relevance',
42
179
  weight: 1,
43
180
  evaluate(context) {
@@ -49,9 +186,8 @@ export const topicConsistency = {
49
186
  if (inputWords.length === 0 || outputWords.length === 0) {
50
187
  return { ruleName: 'topic_consistency', passed: false, score: 0, message: 'Insufficient text for topic analysis', skipped: true, skipReason: 'input or output has no words > 3 chars' };
51
188
  }
52
- // v0.3.1 fix: skip when output is too brief — short outputs (1-5 words >3 chars)
53
- // produce noisy ratios where the threshold can't meaningfully discriminate.
54
- // The previous version over-triggered as a false-positive on brief but valid responses.
189
+ // v0.3.1: skip when the output is too brief — a handful of words cannot
190
+ // be judged on topic or off it, and the rule used to cry wolf there.
55
191
  const minOutputWords = context.customConfig?.topic_consistency_min_words ?? 6;
56
192
  if (outputWords.length < minOutputWords) {
57
193
  return {
@@ -63,20 +199,48 @@ export const topicConsistency = {
63
199
  skipReason: `output has < ${minOutputWords} words ≥ 4 chars`,
64
200
  };
65
201
  }
66
- const inputSet = new Set(inputWords);
67
- let relevant = 0;
68
- for (const word of outputWords) {
69
- if (inputSet.has(word))
70
- relevant++;
202
+ const topic = new Set(contentTerms(context.input));
203
+ if (topic.size === 0) {
204
+ return { ruleName: 'topic_consistency', passed: false, score: 0, message: 'Insufficient text for topic analysis', skipped: true, skipReason: 'input has no content terms' };
205
+ }
206
+ // Walk the output line by line so list items can be read under their
207
+ // lead-in, and sentence by sentence within a line. `seen` is the topic
208
+ // so far: the input's terms plus every connected sentence's terms.
209
+ const seen = new Set(topic);
210
+ let sentences = 0;
211
+ let connected = 0;
212
+ let leadInConnected = false;
213
+ for (const line of context.output.replace(FENCED_CODE, '\n').split('\n')) {
214
+ const isItem = LIST_ITEM.test(line);
215
+ let lineConnected = false;
216
+ for (const sentence of line.split(SENTENCE_BREAK)) {
217
+ const terms = contentTerms(sentence);
218
+ if (terms.length === 0)
219
+ continue;
220
+ sentences++;
221
+ const hit = terms.some((t) => seen.has(t)) || (isItem && leadInConnected);
222
+ if (hit) {
223
+ connected++;
224
+ lineConnected = true;
225
+ for (const t of terms)
226
+ seen.add(t);
227
+ }
228
+ }
229
+ if (!isItem && line.trim().length > 0)
230
+ leadInConnected = lineConnected;
231
+ }
232
+ if (sentences === 0) {
233
+ return { ruleName: 'topic_consistency', passed: false, score: 0, message: 'Insufficient text for topic analysis', skipped: true, skipReason: 'output has no content terms' };
71
234
  }
72
- const ratio = relevant / outputWords.length;
73
- const threshold = context.customConfig?.topic_consistency ?? 0.10;
235
+ const ratio = connected / sentences;
236
+ const threshold = context.customConfig?.topic_consistency ?? DEFAULT_TOPIC_THRESHOLD;
74
237
  const passed = ratio >= threshold;
75
238
  return {
76
239
  ruleName: 'topic_consistency',
77
240
  passed,
78
- score: Math.min(ratio * 5, 1),
79
- message: `Topic consistency: ${(ratio * 100).toFixed(1)}% of output words relate to input`,
241
+ // Full marks at two thirds connected; proportional below.
242
+ score: Math.min(ratio * 1.5, 1),
243
+ message: `Topic consistency: ${connected}/${sentences} content sentences connect to the input's topic (${(ratio * 100).toFixed(0)}%)`,
80
244
  };
81
245
  },
82
246
  };
package/dist/eval/rules/safety.d.ts CHANGED
@@ -9,12 +9,23 @@ export declare const PII_PATTERNS: Array<{
9
9
  * with the count and the pattern names (#370): a builder smoke-testing with
10
10
  * `bob@example.com` or a 555 number used to read a bare "No PII detected"
11
11
  * and conclude detection was broken, when the rule had recognised the
12
- * value as documentation on purpose.
12
+ * value as documentation on purpose. Reserved IP addresses get their own
13
+ * clause in the same style — they are not documentation values, they are
14
+ * addresses that cannot identify anyone.
13
15
  */
14
16
  export declare function describeSuppressedPlaceholders(suppressed: Map<string, number>): string;
15
17
  export declare const noPii: EvalRule;
16
18
  export declare const noBlocklistWords: EvalRule;
17
19
  export declare const INJECTION_PATTERNS: RegExp[];
20
+ /**
21
+ * The one true sentence about what the injection rule looks at. It is
22
+ * shared verbatim by the rule description, the evaluate_output tool
23
+ * description and the docs (a drift-lock test pins every copy), because
24
+ * the tool used to sell unscoped "prompt injection" while the rule reads
25
+ * `context.output` and nothing else — a builder could reasonably take it
26
+ * for an input firewall, which it is not.
27
+ */
28
+ export declare const INJECTION_SCOPE_SENTENCE = "no_injection_patterns inspects the agent's OUTPUT text for injection-shaped content \u2014 attack phrasing and structural directives the output echoes or complies with \u2014 and never reads the input, so it is not an input firewall.";
18
29
  export declare const noInjectionPatterns: EvalRule;
19
30
  export declare const noStubOutput: EvalRule;
20
31
  export interface HallucinationSignal {