@iris-eval/mcp-server 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -36
- package/dist/config/index.d.ts +10 -0
- package/dist/config/index.js +33 -7
- package/dist/dashboard/assets/index-CshLgDRB.js +10 -0
- package/dist/dashboard/assets/{index-UffZ-aEJ.css → index-D0cFfBqn.css} +1 -1
- package/dist/dashboard/index.html +4 -3
- package/dist/dashboard/routes/health.js +10 -3
- package/dist/dashboard/routes/moments.js +1 -1
- package/dist/dashboard/routes/preferences.d.ts +1 -0
- package/dist/dashboard/routes/preferences.js +31 -3
- package/dist/dashboard/routes/rules.d.ts +18 -0
- package/dist/dashboard/routes/rules.js +160 -6
- package/dist/dashboard/routes/traces.js +30 -3
- package/dist/dashboard/seed-demo-data.js +14 -3
- package/dist/dashboard/server.js +13 -3
- package/dist/dashboard/session-auth.d.ts +8 -0
- package/dist/dashboard/session-auth.js +237 -0
- package/dist/dashboard/validation.d.ts +9 -3
- package/dist/dashboard/validation.js +69 -11
- package/dist/eval/citation-verify/verifier.d.ts +17 -0
- package/dist/eval/citation-verify/verifier.js +68 -15
- package/dist/eval/decision-moment.js +17 -9
- package/dist/eval/engine.d.ts +62 -0
- package/dist/eval/engine.js +196 -58
- package/dist/eval/llm-judge/evaluator.js +50 -33
- package/dist/eval/llm-judge/templates/index.d.ts +4 -0
- package/dist/eval/llm-judge/templates/index.js +10 -4
- package/dist/eval/rules/custom.js +59 -6
- package/dist/eval/rules/relevance.js +1 -1
- package/dist/eval/rules/safety.d.ts +8 -0
- package/dist/eval/rules/safety.js +63 -18
- package/dist/index.js +102 -16
- package/dist/middleware/rate-limit.d.ts +25 -0
- package/dist/middleware/rate-limit.js +54 -2
- package/dist/self-test.d.ts +14 -0
- package/dist/self-test.js +97 -13
- package/dist/storage/demo-guard.d.ts +8 -0
- package/dist/storage/demo-guard.js +53 -0
- package/dist/storage/migrations/006-eval-critical-failures.d.ts +3 -0
- package/dist/storage/migrations/006-eval-critical-failures.js +23 -0
- package/dist/storage/migrations/index.js +2 -0
- package/dist/storage/sqlite-adapter.d.ts +6 -0
- package/dist/storage/sqlite-adapter.js +91 -4
- package/dist/tools/delete-rule.js +49 -11
- package/dist/tools/deploy-rule.d.ts +33 -0
- package/dist/tools/deploy-rule.js +130 -27
- package/dist/tools/evaluate-output.js +50 -24
- package/dist/tools/evaluate-with-llm-judge.js +11 -4
- package/dist/tools/get-traces.d.ts +27 -0
- package/dist/tools/get-traces.js +60 -8
- package/dist/tools/list-rules.js +2 -2
- package/dist/tools/log-trace.js +5 -4
- package/dist/tools/strict-input.d.ts +1 -0
- package/dist/tools/strict-input.js +25 -0
- package/dist/tools/trace-link.d.ts +7 -0
- package/dist/tools/trace-link.js +39 -0
- package/dist/tools/verify-citations.d.ts +19 -0
- package/dist/tools/verify-citations.js +42 -5
- package/dist/types/decision-moment.d.ts +8 -0
- package/dist/types/eval.d.ts +60 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/query.d.ts +25 -0
- package/package.json +1 -1
- package/server.json +2 -2
- package/dist/dashboard/assets/index-BZZt8bVh.js +0 -10
|
@@ -33,6 +33,20 @@ function configError(definition, message) {
|
|
|
33
33
|
function safeRegexResult(definition, message) {
|
|
34
34
|
return configError(definition, message);
|
|
35
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* `config.keywords` as a non-empty array of strings, or undefined when it
|
|
38
|
+
* is anything else. Element types are checked at runtime because the
|
|
39
|
+
* inline schema accepts any config value: `keywords: [1, 2]` passed the
|
|
40
|
+
* old Array.isArray check and then threw from `.toLowerCase()` mid-eval.
|
|
41
|
+
*/
|
|
42
|
+
function readKeywordList(config) {
|
|
43
|
+
const value = config.keywords;
|
|
44
|
+
if (!Array.isArray(value) || value.length === 0)
|
|
45
|
+
return undefined;
|
|
46
|
+
if (!value.every((k) => typeof k === 'string'))
|
|
47
|
+
return undefined;
|
|
48
|
+
return value;
|
|
49
|
+
}
|
|
36
50
|
/**
|
|
37
51
|
* Converts a leading inline flag group like `(?i)` or `(?im)` into a real
|
|
38
52
|
* flags argument. Node's RegExp engine does not support inline flag groups,
|
|
@@ -65,7 +79,26 @@ export function normalizeRegexSource(patternStr, flags) {
|
|
|
65
79
|
* general. The sandbox's hard deadline is the boundary.
|
|
66
80
|
*/
|
|
67
81
|
function validateRegex(definition) {
|
|
68
|
-
|
|
82
|
+
/*
|
|
83
|
+
* Runtime shape check, not just a compile-time cast. evaluate_output's
|
|
84
|
+
* inline custom_rules schema accepts any config record, so
|
|
85
|
+
* `{type: "regex_match", config: {}}` (or a null / numeric pattern)
|
|
86
|
+
* reaches this point; the old `as string` cast was a no-op at runtime
|
|
87
|
+
* and normalizeRegexSource threw a TypeError out of the engine — the
|
|
88
|
+
* whole evaluate_output call failed, contradicting its own description
|
|
89
|
+
* ("the eval itself never throws"). Deploy-time validation already
|
|
90
|
+
* rejects these; this is the same configError contract for the inline
|
|
91
|
+
* path and for rules persisted before that validation existed.
|
|
92
|
+
*/
|
|
93
|
+
const rawPattern = definition.config.pattern;
|
|
94
|
+
if (typeof rawPattern !== 'string' || rawPattern.length === 0) {
|
|
95
|
+
return safeRegexResult(definition, `${definition.type} rule requires config.pattern (non-empty string)`);
|
|
96
|
+
}
|
|
97
|
+
const rawFlags = definition.config.flags;
|
|
98
|
+
if (rawFlags !== undefined && rawFlags !== null && typeof rawFlags !== 'string') {
|
|
99
|
+
return safeRegexResult(definition, `${definition.type} rule config.flags must be a string when present`);
|
|
100
|
+
}
|
|
101
|
+
const { pattern: patternStr, flags } = normalizeRegexSource(rawPattern, rawFlags ?? '');
|
|
69
102
|
if (patternStr.length > MAX_PATTERN_LENGTH) {
|
|
70
103
|
return safeRegexResult(definition, `Regex pattern too long (${patternStr.length} > ${MAX_PATTERN_LENGTH})`);
|
|
71
104
|
}
|
|
@@ -233,8 +266,8 @@ export function createCustomRule(definition, severity) {
|
|
|
233
266
|
return { ruleName: definition.name, passed, score: passed ? 1 : max / context.output.length, message: passed ? `Length (${context.output.length}) within maximum (${max})` : `Length (${context.output.length}) exceeds maximum (${max})` };
|
|
234
267
|
}
|
|
235
268
|
case 'contains_keywords': {
|
|
236
|
-
const keywords = definition.config
|
|
237
|
-
if (!keywords
|
|
269
|
+
const keywords = readKeywordList(definition.config);
|
|
270
|
+
if (!keywords) {
|
|
238
271
|
return configError(definition, 'contains_keywords rule requires config.keywords (non-empty string array)');
|
|
239
272
|
}
|
|
240
273
|
const lower = context.output.toLowerCase();
|
|
@@ -244,8 +277,8 @@ export function createCustomRule(definition, severity) {
|
|
|
244
277
|
return { ruleName: definition.name, passed, score: ratio, message: `Found ${found.length}/${keywords.length} required keywords` };
|
|
245
278
|
}
|
|
246
279
|
case 'excludes_keywords': {
|
|
247
|
-
const keywords = definition.config
|
|
248
|
-
if (!keywords
|
|
280
|
+
const keywords = readKeywordList(definition.config);
|
|
281
|
+
if (!keywords) {
|
|
249
282
|
return configError(definition, 'excludes_keywords rule requires config.keywords (non-empty string array)');
|
|
250
283
|
}
|
|
251
284
|
const lower = context.output.toLowerCase();
|
|
@@ -267,7 +300,27 @@ export function createCustomRule(definition, severity) {
|
|
|
267
300
|
if (max == null || max < 0) {
|
|
268
301
|
return configError(definition, `cost_threshold rule requires ${describeKeys('cost_threshold')} (non-negative number)`);
|
|
269
302
|
}
|
|
270
|
-
|
|
303
|
+
/*
|
|
304
|
+
* No cost data → SKIP, exactly like the built-in
|
|
305
|
+
* cost_under_threshold (cost.ts). The old `context.costUsd ?? 0`
|
|
306
|
+
* read a missing cost as free, so a rule deployed at severity
|
|
307
|
+
* critical to hard-fail evaluations over $0.50 reported
|
|
308
|
+
* passed:true, score:1 on every evaluate_output call that simply
|
|
309
|
+
* omitted cost_usd — the veto never fired on evidence it never
|
|
310
|
+
* had. A skipped critical rule is reported in critical_skipped
|
|
311
|
+
* instead, so a fail-closed gate can see the rule did not run.
|
|
312
|
+
*/
|
|
313
|
+
if (context.costUsd === undefined || context.costUsd === null) {
|
|
314
|
+
return {
|
|
315
|
+
ruleName: definition.name,
|
|
316
|
+
passed: false,
|
|
317
|
+
score: 0,
|
|
318
|
+
message: 'Cost data not provided',
|
|
319
|
+
skipped: true,
|
|
320
|
+
skipReason: 'context.costUsd not provided',
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
const cost = context.costUsd;
|
|
271
324
|
const passed = cost <= max;
|
|
272
325
|
return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? `Cost ($${cost}) within threshold ($${max})` : `Cost ($${cost}) exceeds threshold ($${max})` };
|
|
273
326
|
}
|
|
@@ -30,7 +30,7 @@ export const keywordOverlap = {
|
|
|
30
30
|
};
|
|
31
31
|
/*
|
|
32
32
|
* no_hallucination_markers moved to the safety bundle (safety.ts) in
|
|
33
|
-
* v0.
|
|
33
|
+
* v0.5.0 — its rewrite is context-grounded fabrication/contradiction
|
|
34
34
|
* detection, and the safety bundle is where the evaluate_output docs,
|
|
35
35
|
* the dashboard's safety-violations panel, and the storage adapter's
|
|
36
36
|
* violation counts have always placed it.
|
|
@@ -4,6 +4,14 @@ export declare const PII_PATTERNS: Array<{
|
|
|
4
4
|
pattern: RegExp;
|
|
5
5
|
placeholders?: RegExp[];
|
|
6
6
|
}>;
|
|
7
|
+
/**
|
|
8
|
+
* The pass message when placeholders were ignored. Says so explicitly,
|
|
9
|
+
* with the count and the pattern names (#370): a builder smoke-testing with
|
|
10
|
+
* `bob@example.com` or a 555 number used to read a bare "No PII detected"
|
|
11
|
+
* and conclude detection was broken, when the rule had recognised the
|
|
12
|
+
* value as documentation on purpose.
|
|
13
|
+
*/
|
|
14
|
+
export declare function describeSuppressedPlaceholders(suppressed: Map<string, number>): string;
|
|
7
15
|
export declare const noPii: EvalRule;
|
|
8
16
|
export declare const noBlocklistWords: EvalRule;
|
|
9
17
|
export declare const INJECTION_PATTERNS: RegExp[];
|
|
@@ -8,9 +8,10 @@
|
|
|
8
8
|
*
|
|
9
9
|
* `placeholders` suppresses documentation values that are PII-shaped but by
|
|
10
10
|
* definition not PII: RFC 2606 example domains, the reserved 555 fictional
|
|
11
|
-
* phone block and toll-free lines, published payment test cards,
|
|
12
|
-
*
|
|
13
|
-
*
|
|
11
|
+
* phone block and toll-free lines, published payment test cards, masked
|
|
12
|
+
* keys, and 10-digit runs with no separators (Unix timestamps, JWTs and
|
|
13
|
+
* rate-limit headers read as "phone numbers"). The canonical documentation
|
|
14
|
+
* SSN is deliberately NOT suppressed — see the SSN entry below (#362).
|
|
14
15
|
* A pattern only fails the rule when at least one of its matches is NOT
|
|
15
16
|
* covered by a placeholder — so real PII beside a placeholder still fails.
|
|
16
17
|
*/
|
|
@@ -104,10 +105,28 @@ export const PII_PATTERNS = [
|
|
|
104
105
|
// v0.3.1 additions
|
|
105
106
|
// IBAN: 2 letters + 2 digits + 1-30 alphanumeric (international bank account number)
|
|
106
107
|
{ name: 'IBAN', pattern: /\b[A-Z]{2}\d{2}[A-Z0-9]{10,30}\b/ },
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
108
|
+
/*
|
|
109
|
+
* US passport — CONTEXT-ANCHORED, like DOB and MRN below. A legacy
|
|
110
|
+
* passport number is nine bare digits and the modern (2021+) format is
|
|
111
|
+
* one letter + eight digits; neither shape has internal structure to
|
|
112
|
+
* anchor on. The old `\b[A-Z]?\d{9}\b` fired on ANY nine-digit run —
|
|
113
|
+
* order IDs, EINs, routing numbers, nine-digit Unix timestamps — and
|
|
114
|
+
* because no_pii is critical, "Order ID: 123456789" vetoed the whole
|
|
115
|
+
* evaluation. It also never matched the modern C12345678 shape its own
|
|
116
|
+
* comment promised: the optional letter still demanded nine digits after
|
|
117
|
+
* it. Now the number must follow the word "passport" within a short
|
|
118
|
+
* window, which is what docs/api-reference.md has described all along.
|
|
119
|
+
* The window is bounded ({0,40}) so the scan stays linear in the input.
|
|
120
|
+
*/
|
|
121
|
+
{ name: 'Passport', pattern: /\bpassports?\b[\s\S]{0,40}?\b(?:[A-Z]\d{8}|\d{9})\b/i },
|
|
122
|
+
// Date of birth contextual — DOB or "Born:" / "Birthday:" + a date in
|
|
123
|
+
// either US/EU numeric form (03/15/1987, 15.03.87) or ISO form
|
|
124
|
+
// (1987-03-15). The ISO alternative is listed first: it is the shape
|
|
125
|
+
// `Date of birth: 1987-03-15` takes in any structured record, and the
|
|
126
|
+
// label-anchored pattern used to miss exactly that while catching the
|
|
127
|
+
// slash form (#374). Both alternatives are fixed-width per position, so
|
|
128
|
+
// the scan stays linear.
|
|
129
|
+
{ name: 'DOB', pattern: /\b(?:DOB|D\.O\.B\.|Date of Birth|Born|Birthday)\s{0,8}[:.]?\s{0,8}(?:\d{4}-\d{2}-\d{2}|\d{1,2}[\/\-.]\d{1,2}[\/\-.](?:\d{2}|\d{4}))\b/i },
|
|
111
130
|
// Medical record number — MRN: + alphanumeric (common format)
|
|
112
131
|
{ name: 'Medical Record Number', pattern: /\b(?:MRN|Medical Record (?:Number|No\.?|#))\s{0,8}[:.]?\s{0,8}[A-Z0-9]{6,12}\b/i },
|
|
113
132
|
// IPv4 address
|
|
@@ -137,19 +156,40 @@ export const PII_PATTERNS = [
|
|
|
137
156
|
{ name: 'Seed Phrase', pattern: /\b(?:[Ss]eed|[Rr]ecovery|[Mm]nemonic)\s(?:[Pp]hrase|[Ww]ords)\b[\s\S]{0,120}?\b(?:[a-z]{3,8}\s{1,4}){11}[a-z]{3,8}\b/ },
|
|
138
157
|
];
|
|
139
158
|
/**
|
|
140
|
-
*
|
|
141
|
-
* the pattern's documented placeholder values
|
|
142
|
-
*
|
|
159
|
+
* `fired` is true when `pattern` has at least one match in `output` that is
|
|
160
|
+
* not one of the pattern's documented placeholder values; `suppressed`
|
|
161
|
+
* counts the matches that WERE placeholders. Patterns without a
|
|
162
|
+
* `placeholders` list keep the plain test() fast path, and the scan stops
|
|
163
|
+
* at the first real match — the suppressed count is only complete (and only
|
|
164
|
+
* reported) when nothing real fired.
|
|
143
165
|
*/
|
|
144
|
-
function
|
|
166
|
+
function piiPatternMatches(output, pattern, placeholders) {
|
|
145
167
|
if (!placeholders)
|
|
146
|
-
return pattern.test(output);
|
|
168
|
+
return { fired: pattern.test(output), suppressed: 0 };
|
|
147
169
|
const global = new RegExp(pattern.source, pattern.flags.includes('g') ? pattern.flags : `${pattern.flags}g`);
|
|
170
|
+
let suppressed = 0;
|
|
148
171
|
for (const match of output.matchAll(global)) {
|
|
149
172
|
if (!placeholders.some((placeholder) => placeholder.test(match[0])))
|
|
150
|
-
return true;
|
|
173
|
+
return { fired: true, suppressed };
|
|
174
|
+
suppressed++;
|
|
151
175
|
}
|
|
152
|
-
return false;
|
|
176
|
+
return { fired: false, suppressed };
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* The pass message when placeholders were ignored. Says so explicitly,
|
|
180
|
+
* with the count and the pattern names (#370): a builder smoke-testing with
|
|
181
|
+
* `bob@example.com` or a 555 number used to read a bare "No PII detected"
|
|
182
|
+
* and conclude detection was broken, when the rule had recognised the
|
|
183
|
+
* value as documentation on purpose.
|
|
184
|
+
*/
|
|
185
|
+
export function describeSuppressedPlaceholders(suppressed) {
|
|
186
|
+
const total = [...suppressed.values()].reduce((sum, n) => sum + n, 0);
|
|
187
|
+
if (total === 0)
|
|
188
|
+
return 'No PII detected';
|
|
189
|
+
const parts = [...suppressed.entries()].map(([name, n]) => (n > 1 ? `${name} ×${n}` : name));
|
|
190
|
+
return (`No PII detected (${total} documentation placeholder${total === 1 ? '' : 's'} ignored: ${parts.join(', ')} — ` +
|
|
191
|
+
'example.com/.org/.net addresses, 555 and toll-free phone numbers, published payment test cards ' +
|
|
192
|
+
'and masked keys are never counted as PII; real values in the same shapes still fail)');
|
|
153
193
|
}
|
|
154
194
|
export const noPii = {
|
|
155
195
|
name: 'no_pii',
|
|
@@ -166,17 +206,22 @@ export const noPii = {
|
|
|
166
206
|
critical: true,
|
|
167
207
|
evaluate(context) {
|
|
168
208
|
const found = [];
|
|
209
|
+
const suppressed = new Map();
|
|
169
210
|
for (const { name, pattern, placeholders } of PII_PATTERNS) {
|
|
170
|
-
|
|
211
|
+
const { fired, suppressed: ignored } = piiPatternMatches(context.output, pattern, placeholders);
|
|
212
|
+
if (fired) {
|
|
171
213
|
found.push(name);
|
|
172
214
|
}
|
|
215
|
+
else if (ignored > 0) {
|
|
216
|
+
suppressed.set(name, ignored);
|
|
217
|
+
}
|
|
173
218
|
}
|
|
174
219
|
const passed = found.length === 0;
|
|
175
220
|
return {
|
|
176
221
|
ruleName: 'no_pii',
|
|
177
222
|
passed,
|
|
178
223
|
score: passed ? 1 : 0,
|
|
179
|
-
message: passed ?
|
|
224
|
+
message: passed ? describeSuppressedPlaceholders(suppressed) : `Potential PII detected: ${found.join(', ')}`,
|
|
180
225
|
};
|
|
181
226
|
},
|
|
182
227
|
};
|
|
@@ -742,7 +787,7 @@ export const noStubOutput = {
|
|
|
742
787
|
},
|
|
743
788
|
};
|
|
744
789
|
/*
|
|
745
|
-
* Hallucination detection — rewritten v0.
|
|
790
|
+
* Hallucination detection — rewritten v0.5.0, moved here from the relevance
|
|
746
791
|
* bundle in the same change.
|
|
747
792
|
*
|
|
748
793
|
* The previous incarnation matched 17 refusal-boilerplate phrases ("as an
|
|
@@ -1150,7 +1195,7 @@ function detectUngroundedDate(output, input) {
|
|
|
1150
1195
|
}
|
|
1151
1196
|
/*
|
|
1152
1197
|
* Parse a markdown table row by splitting on '|' — never by regexing the
|
|
1153
|
-
* whole line. The v0.
|
|
1198
|
+
* whole line. The v0.5.0 first cut used /^\s*\|\s*([^|]+?)\s*\|(.+)\|?\s*$/,
|
|
1154
1199
|
* where the greedy \s* and lazy [^|]+? both match a run of spaces: on a
|
|
1155
1200
|
* line of '|' + N spaces with no closing pipe the engine has ~N ways to
|
|
1156
1201
|
* split the run, each failing late — super-quadratic backtracking (~7.5×
|
package/dist/index.js
CHANGED
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
import { parseArgs } from 'node:util';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { loadConfig } from './config/index.js';
|
|
5
|
+
import { PKG_VERSION } from './config/defaults.js';
|
|
5
6
|
import { createStorage } from './storage/index.js';
|
|
7
|
+
import { withDemoIngestGuard } from './storage/demo-guard.js';
|
|
6
8
|
import { createIrisServer } from './server.js';
|
|
7
9
|
import { createStdioTransport } from './transport/stdio.js';
|
|
8
10
|
import { createHttpTransport } from './transport/http.js';
|
|
@@ -35,6 +37,8 @@ const CliSchema = z
|
|
|
35
37
|
demo: z.boolean().optional(),
|
|
36
38
|
'demo-clear': z.boolean().optional(),
|
|
37
39
|
'self-test': z.boolean().optional(),
|
|
40
|
+
purge: z.boolean().optional(),
|
|
41
|
+
version: z.boolean().optional(),
|
|
38
42
|
help: z.boolean().optional(),
|
|
39
43
|
})
|
|
40
44
|
.strict();
|
|
@@ -65,6 +69,8 @@ try {
|
|
|
65
69
|
demo: { type: 'boolean', default: false },
|
|
66
70
|
'demo-clear': { type: 'boolean', default: false },
|
|
67
71
|
'self-test': { type: 'boolean', default: false },
|
|
72
|
+
purge: { type: 'boolean', default: false },
|
|
73
|
+
version: { type: 'boolean', default: false },
|
|
68
74
|
help: { type: 'boolean', short: 'h', default: false },
|
|
69
75
|
},
|
|
70
76
|
strict: true,
|
|
@@ -74,6 +80,16 @@ catch (err) {
|
|
|
74
80
|
process.stderr.write(`iris-mcp: ${err.message}\nRun \`iris-mcp --help\` for usage.\n`);
|
|
75
81
|
process.exit(2);
|
|
76
82
|
}
|
|
83
|
+
/*
|
|
84
|
+
* --version answers on stdout, bare, before anything else: the README's
|
|
85
|
+
* "check your version" recipe pointed at a flag that did not exist and a
|
|
86
|
+
* --help banner that printed no version (#369). Bare so `iris-mcp
|
|
87
|
+
* --version` composes in scripts the way `npm --version` does.
|
|
88
|
+
*/
|
|
89
|
+
if (parsed.values.version) {
|
|
90
|
+
process.stdout.write(`${PKG_VERSION}\n`);
|
|
91
|
+
process.exit(0);
|
|
92
|
+
}
|
|
77
93
|
const validation = CliSchema.safeParse(parsed.values);
|
|
78
94
|
if (!validation.success) {
|
|
79
95
|
const issues = validation.error.issues
|
|
@@ -85,7 +101,7 @@ if (!validation.success) {
|
|
|
85
101
|
const values = validation.data;
|
|
86
102
|
if (values.help) {
|
|
87
103
|
process.stderr.write(`
|
|
88
|
-
Iris — MCP-Native Agent Eval Server
|
|
104
|
+
Iris — MCP-Native Agent Eval Server v${PKG_VERSION}
|
|
89
105
|
|
|
90
106
|
Usage: iris-mcp [options]
|
|
91
107
|
|
|
@@ -107,9 +123,16 @@ Options:
|
|
|
107
123
|
transport). Idempotent: re-running reuses the seeded data.
|
|
108
124
|
--demo-clear Delete the demo database (and its sidecar files), then exit.
|
|
109
125
|
Your real traces are not touched.
|
|
110
|
-
--self-test Run the offline install diagnostic and exit:
|
|
111
|
-
|
|
126
|
+
--self-test Run the offline install diagnostic and exit: the configured IRIS_HOME
|
|
127
|
+
is created and probed for writability, then storage round-trip,
|
|
128
|
+
deterministic evals, dashboard + rebinding guard run inside an
|
|
112
129
|
isolated temp home. Exit code 0 = healthy, 1 = a check failed.
|
|
130
|
+
--purge Delete EVERY stored trace, span and evaluation from the configured
|
|
131
|
+
database, compact the file and truncate the write-ahead log so the
|
|
132
|
+
deleted text does not linger on disk, then exit. Deployed rules, the
|
|
133
|
+
audit log and preferences are kept. Not reversible. Stop any running
|
|
134
|
+
Iris server first — the file is compacted in place.
|
|
135
|
+
--version Print the version and exit
|
|
113
136
|
-h, --help Show this help message
|
|
114
137
|
|
|
115
138
|
Environment variables (CLI flags take precedence):
|
|
@@ -139,11 +162,31 @@ Environment variables (CLI flags take precedence):
|
|
|
139
162
|
IRIS_OTEL_TIMEOUT_MS Per-export timeout (default: 15000)
|
|
140
163
|
RATE_LIMIT_SALT (waitlist API only — required when website is deployed)
|
|
141
164
|
|
|
142
|
-
Dashboard preferences (~/.iris/preferences.json):
|
|
165
|
+
Dashboard preferences ($IRIS_HOME/preferences.json, default ~/.iris/preferences.json):
|
|
143
166
|
Edit autoLaunch: false to permanently disable first-run dashboard auto-launch.
|
|
144
167
|
`);
|
|
145
168
|
process.exit(0);
|
|
146
169
|
}
|
|
170
|
+
/*
|
|
171
|
+
* The mode flags are mutually exclusive, and the check runs before any of
|
|
172
|
+
* them so a refused combination exits without touching the filesystem —
|
|
173
|
+
* `--self-test --purge` must not quietly run only the first one it sees.
|
|
174
|
+
*/
|
|
175
|
+
const modeFlags = ['demo', 'demo-clear', 'self-test', 'purge'].filter((flag) => values[flag]);
|
|
176
|
+
if (modeFlags.length > 1) {
|
|
177
|
+
process.stderr.write(`iris-mcp: ${modeFlags.map((flag) => `--${flag}`).join(' and ')} cannot be combined.\nRun \`iris-mcp --help\` for usage.\n`);
|
|
178
|
+
process.exit(2);
|
|
179
|
+
}
|
|
180
|
+
/*
|
|
181
|
+
* `--purge --dashboard` used to run the purge and drop the dashboard flag
|
|
182
|
+
* on the floor. A flag that changes nothing is the same fault as a
|
|
183
|
+
* misspelled tool argument: the caller asked for something and got no
|
|
184
|
+
* sign it was ignored. (v0.6.0 acceptance pass.)
|
|
185
|
+
*/
|
|
186
|
+
if (values.purge && values.dashboard) {
|
|
187
|
+
process.stderr.write('iris-mcp: --purge exits as soon as the purge finishes and cannot be combined with --dashboard (nothing would be served).\nRun `iris-mcp --help` for usage.\n');
|
|
188
|
+
process.exit(2);
|
|
189
|
+
}
|
|
147
190
|
/*
|
|
148
191
|
* --self-test exits BEFORE loadConfig() runs at module scope below —
|
|
149
192
|
* deliberately. The diagnostic builds its own isolated IRIS_HOME and
|
|
@@ -154,14 +197,6 @@ if (values['self-test']) {
|
|
|
154
197
|
const { runSelfTest } = await import('./self-test.js');
|
|
155
198
|
process.exit(await runSelfTest());
|
|
156
199
|
}
|
|
157
|
-
/*
|
|
158
|
-
* Demo-mode flag validation happens before loadConfig so a refused
|
|
159
|
-
* combination exits without touching the filesystem.
|
|
160
|
-
*/
|
|
161
|
-
if (values.demo && values['demo-clear']) {
|
|
162
|
-
process.stderr.write('iris-mcp: --demo and --demo-clear cannot be combined.\nRun `iris-mcp --help` for usage.\n');
|
|
163
|
-
process.exit(2);
|
|
164
|
-
}
|
|
165
200
|
if (values.demo && values['db-path']) {
|
|
166
201
|
process.stderr.write('iris-mcp: --demo always serves its own database (demo.db under your iris home) and cannot be combined with --db-path.\n' +
|
|
167
202
|
'Run `iris-mcp --demo` alone, or drop --demo to use your own database.\n');
|
|
@@ -193,6 +228,39 @@ const config = loadConfig({
|
|
|
193
228
|
dashboardHost: values['dashboard-host'],
|
|
194
229
|
});
|
|
195
230
|
const logger = createLogger(config);
|
|
231
|
+
/*
|
|
232
|
+
* --purge (#372): the retention sweep only ever trimmed by age, and
|
|
233
|
+
* deleting iris.db by hand left every row readable in iris.db-wal. This
|
|
234
|
+
* is the one-command answer to "remove everything Iris stored about my
|
|
235
|
+
* agents" — every trace, span and evaluation for the local tenant,
|
|
236
|
+
* followed by VACUUM + a TRUNCATE checkpoint so the text is gone from the
|
|
237
|
+
* main file and the write-ahead log alike. Rules, audit log and
|
|
238
|
+
* preferences are not storage rows and stay. Prints what it removed and
|
|
239
|
+
* exits 0; runs against the configured database, never the demo one
|
|
240
|
+
* (--demo-clear handles that).
|
|
241
|
+
*/
|
|
242
|
+
async function runPurge() {
|
|
243
|
+
const storage = createStorage(config);
|
|
244
|
+
await storage.initialize();
|
|
245
|
+
try {
|
|
246
|
+
const { traces, evalResults } = await storage.purge(LOCAL_TENANT);
|
|
247
|
+
process.stderr.write(`iris-mcp: purged ${traces} trace(s) and ${evalResults} evaluation(s) from "${config.storage.path}" ` +
|
|
248
|
+
'(database compacted, write-ahead log truncated). Deployed rules, audit log and preferences were kept.\n');
|
|
249
|
+
}
|
|
250
|
+
finally {
|
|
251
|
+
await storage.close();
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
if (values.purge) {
|
|
255
|
+
try {
|
|
256
|
+
await runPurge();
|
|
257
|
+
process.exit(0);
|
|
258
|
+
}
|
|
259
|
+
catch (err) {
|
|
260
|
+
process.stderr.write(`iris-mcp: purge failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
261
|
+
process.exit(1);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
196
264
|
async function main() {
|
|
197
265
|
logger.info(`Starting Iris MCP server v${config.server.version}`);
|
|
198
266
|
// F-006: fail fast on HTTP+dashboard port collision. See validatePortConfig.
|
|
@@ -231,9 +299,19 @@ async function main() {
|
|
|
231
299
|
// cleanly.
|
|
232
300
|
if (config.retention.days > 0) {
|
|
233
301
|
try {
|
|
234
|
-
const
|
|
235
|
-
|
|
236
|
-
|
|
302
|
+
const deletedTraces = await storage.deleteTracesOlderThan(LOCAL_TENANT, config.retention.days);
|
|
303
|
+
/*
|
|
304
|
+
* Evaluations too (#372). Deleting a trace only NULLs trace_id on
|
|
305
|
+
* its evaluations, so every eval row — output_text verbatim,
|
|
306
|
+
* including whatever no_pii flagged — used to outlive the retention
|
|
307
|
+
* window indefinitely while the traces around it were swept.
|
|
308
|
+
*/
|
|
309
|
+
const deletedEvals = await storage.deleteEvalResultsOlderThan(LOCAL_TENANT, config.retention.days);
|
|
310
|
+
if (deletedTraces + deletedEvals > 0) {
|
|
311
|
+
// Fold the WAL into the main file and truncate it, so the swept
|
|
312
|
+
// rows do not survive as readable text in iris.db-wal.
|
|
313
|
+
await storage.checkpoint();
|
|
314
|
+
logger.info(`Retention cleanup: deleted ${deletedTraces} trace(s) and ${deletedEvals} evaluation(s) older than ${config.retention.days} days`);
|
|
237
315
|
}
|
|
238
316
|
}
|
|
239
317
|
catch (err) {
|
|
@@ -332,6 +410,8 @@ ${line}
|
|
|
332
410
|
${counts}
|
|
333
411
|
Demo database: "${summary.dbPath}"
|
|
334
412
|
Your real trace database is untouched — demo data never mixes with it.
|
|
413
|
+
Trace ingest (POST /api/v1/traces) is refused in demo mode: start the
|
|
414
|
+
real server (iris-mcp --dashboard) to store your own traces.
|
|
335
415
|
|
|
336
416
|
Worth clicking into:
|
|
337
417
|
- a PII leak (a synthetic SSN in an agent reply) caught by the safety rules
|
|
@@ -367,7 +447,13 @@ async function runDemo() {
|
|
|
367
447
|
else {
|
|
368
448
|
logger.info(`Seeded demo database with ${seedSummary.traceCount} traces at ${seedSummary.dbPath}`);
|
|
369
449
|
}
|
|
370
|
-
|
|
450
|
+
/*
|
|
451
|
+
* Ingest is refused in demo mode: demo.db is disposable by design and
|
|
452
|
+
* --demo-clear deletes it wholesale, so a capture client pointed at the
|
|
453
|
+
* demo dashboard's port would have its real traces silently stored next
|
|
454
|
+
* to the fake ones and later destroyed (storage/demo-guard.ts).
|
|
455
|
+
*/
|
|
456
|
+
const storage = withDemoIngestGuard(createStorage(config));
|
|
371
457
|
await storage.initialize();
|
|
372
458
|
const customRuleStore = createCustomRuleStore({
|
|
373
459
|
pathFor: () => demoCustomRulesPath(),
|
|
@@ -1,3 +1,28 @@
|
|
|
1
1
|
import type { IrisConfig } from '../types/config.js';
|
|
2
2
|
export declare function createApiRateLimiter(config: Pick<IrisConfig, 'security'>): import("express-rate-limit").RateLimitRequestHandler;
|
|
3
|
+
/**
|
|
4
|
+
* Mounted directly in front of the session layer, so every request that
|
|
5
|
+
* is about to be AUTHORIZED — cookie check, Bearer check, the `?key=`
|
|
6
|
+
* exchange — has passed a per-IP ceiling first (CodeQL
|
|
7
|
+
* js/missing-rate-limiting on the session middleware). Same figure as the
|
|
8
|
+
* API limiter with its own counter; static assets share it, which a
|
|
9
|
+
* dashboard page load (a few dozen requests) never approaches.
|
|
10
|
+
*/
|
|
11
|
+
export declare function createAuthGateRateLimiter(config: Pick<IrisConfig, 'security'>): import("express-rate-limit").RateLimitRequestHandler;
|
|
12
|
+
/**
|
|
13
|
+
* JSON-RPC 2.0 application error code for "rate limited". The reserved
|
|
14
|
+
* server range is -32000..-32099; the MCP SDK uses -32000/-32001 for
|
|
15
|
+
* connection-closed and request-timeout, so this sits clear of both.
|
|
16
|
+
* Mirrors HTTP 429 in the low digits so a log line reads at a glance.
|
|
17
|
+
*/
|
|
18
|
+
export declare const JSON_RPC_RATE_LIMITED = -32029;
|
|
19
|
+
/**
|
|
20
|
+
* The MCP endpoint speaks JSON-RPC, so its 429 must too (#373). The stock
|
|
21
|
+
* express-rate-limit body — `{ "error": "Too many requests" }` — is not a
|
|
22
|
+
* JSON-RPC message: a strict client (the reference SDK included) fails to
|
|
23
|
+
* parse the response and surfaces a PROTOCOL error, and the one thing the
|
|
24
|
+
* caller needed to learn — wait, then retry — is exactly what got lost. The
|
|
25
|
+
* envelope below echoes the request id when the body carried one, names
|
|
26
|
+
* the limit and the wait, and points at the config key that raises it.
|
|
27
|
+
*/
|
|
3
28
|
export declare function createMcpRateLimiter(config: Pick<IrisConfig, 'security'>): import("express-rate-limit").RateLimitRequestHandler;
|
|
@@ -8,12 +8,64 @@ export function createApiRateLimiter(config) {
|
|
|
8
8
|
message: { error: 'Too many requests, please try again later' },
|
|
9
9
|
});
|
|
10
10
|
}
|
|
11
|
-
|
|
11
|
+
/**
|
|
12
|
+
* Mounted directly in front of the session layer, so every request that
|
|
13
|
+
* is about to be AUTHORIZED — cookie check, Bearer check, the `?key=`
|
|
14
|
+
* exchange — has passed a per-IP ceiling first (CodeQL
|
|
15
|
+
* js/missing-rate-limiting on the session middleware). Same figure as the
|
|
16
|
+
* API limiter with its own counter; static assets share it, which a
|
|
17
|
+
* dashboard page load (a few dozen requests) never approaches.
|
|
18
|
+
*/
|
|
19
|
+
export function createAuthGateRateLimiter(config) {
|
|
12
20
|
return rateLimit({
|
|
13
21
|
windowMs: 60_000,
|
|
14
|
-
limit: config.security.rateLimit.
|
|
22
|
+
limit: config.security.rateLimit.api,
|
|
15
23
|
standardHeaders: 'draft-7',
|
|
16
24
|
legacyHeaders: false,
|
|
17
25
|
message: { error: 'Too many requests, please try again later' },
|
|
18
26
|
});
|
|
19
27
|
}
|
|
28
|
+
/**
|
|
29
|
+
* JSON-RPC 2.0 application error code for "rate limited". The reserved
|
|
30
|
+
* server range is -32000..-32099; the MCP SDK uses -32000/-32001 for
|
|
31
|
+
* connection-closed and request-timeout, so this sits clear of both.
|
|
32
|
+
* Mirrors HTTP 429 in the low digits so a log line reads at a glance.
|
|
33
|
+
*/
|
|
34
|
+
export const JSON_RPC_RATE_LIMITED = -32029;
|
|
35
|
+
/**
|
|
36
|
+
* The MCP endpoint speaks JSON-RPC, so its 429 must too (#373). The stock
|
|
37
|
+
* express-rate-limit body — `{ "error": "Too many requests" }` — is not a
|
|
38
|
+
* JSON-RPC message: a strict client (the reference SDK included) fails to
|
|
39
|
+
* parse the response and surfaces a PROTOCOL error, and the one thing the
|
|
40
|
+
* caller needed to learn — wait, then retry — is exactly what got lost. The
|
|
41
|
+
* envelope below echoes the request id when the body carried one, names
|
|
42
|
+
* the limit and the wait, and points at the config key that raises it.
|
|
43
|
+
*/
|
|
44
|
+
export function createMcpRateLimiter(config) {
|
|
45
|
+
const limit = config.security.rateLimit.mcp;
|
|
46
|
+
return rateLimit({
|
|
47
|
+
windowMs: 60_000,
|
|
48
|
+
limit,
|
|
49
|
+
standardHeaders: 'draft-7',
|
|
50
|
+
legacyHeaders: false,
|
|
51
|
+
handler: (req, res) => {
|
|
52
|
+
const body = req.body;
|
|
53
|
+
const requestId = body !== null && typeof body === 'object' && !Array.isArray(body) && 'id' in body
|
|
54
|
+
? body.id
|
|
55
|
+
: null;
|
|
56
|
+
const id = typeof requestId === 'string' || typeof requestId === 'number' ? requestId : null;
|
|
57
|
+
const resetTime = req.rateLimit?.resetTime;
|
|
58
|
+
const retryAfterSeconds = resetTime instanceof Date ? Math.max(1, Math.ceil((resetTime.getTime() - Date.now()) / 1000)) : 60;
|
|
59
|
+
res.status(429).json({
|
|
60
|
+
jsonrpc: '2.0',
|
|
61
|
+
id,
|
|
62
|
+
error: {
|
|
63
|
+
code: JSON_RPC_RATE_LIMITED,
|
|
64
|
+
message: `Rate limit exceeded: this MCP endpoint allows ${limit} requests per minute. ` +
|
|
65
|
+
`Retry in ${retryAfterSeconds}s, or raise security.rateLimit.mcp in config.json for an interactive session.`,
|
|
66
|
+
data: { limit, windowMs: 60_000, retryAfterSeconds },
|
|
67
|
+
},
|
|
68
|
+
});
|
|
69
|
+
},
|
|
70
|
+
});
|
|
71
|
+
}
|
package/dist/self-test.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export declare const SELF_TEST_STEPS: {
|
|
2
|
+
readonly configuredHome: "configured IRIS_HOME is writable";
|
|
2
3
|
readonly tempHome: "create isolated temp home";
|
|
3
4
|
readonly storage: "initialize storage";
|
|
4
5
|
readonly trace: "log a trace";
|
|
@@ -15,4 +16,17 @@ export declare const SELF_TEST_STEPS: {
|
|
|
15
16
|
export declare const SELF_TEST_PASS_VERDICT = "\u2713 PASS \u2014 this install works";
|
|
16
17
|
export declare const SELF_TEST_FAIL_VERDICT = "\u2717 FAIL";
|
|
17
18
|
export type WriteLine = (line: string) => void;
|
|
19
|
+
/**
|
|
20
|
+
* The configured-home probe (#371). Exercises the exact calls the real
|
|
21
|
+
* server makes at startup, in order: create IRIS_HOME (same helper and
|
|
22
|
+
* mode as loadConfig), create the database directory when IRIS_DB_PATH
|
|
23
|
+
* points elsewhere, write-and-unlink a probe file in each, and — only if
|
|
24
|
+
* the real database already exists — open it and take a write lock
|
|
25
|
+
* (BEGIN IMMEDIATE … ROLLBACK), which fails on a read-only file or a
|
|
26
|
+
* non-database exactly as the first INSERT would, without migrating or
|
|
27
|
+
* changing anything. A missing database is not created: the server
|
|
28
|
+
* creates it on first run, and the writable-directory probe is what
|
|
29
|
+
* proves that it can.
|
|
30
|
+
*/
|
|
31
|
+
export declare function probeConfiguredHome(home: string, dbPath: string): string;
|
|
18
32
|
export declare function runSelfTest(write?: WriteLine): Promise<number>;
|