@iris-eval/mcp-server 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/README.md +99 -36
  2. package/dist/config/index.d.ts +10 -0
  3. package/dist/config/index.js +33 -7
  4. package/dist/dashboard/assets/index-CshLgDRB.js +10 -0
  5. package/dist/dashboard/assets/{index-UffZ-aEJ.css → index-D0cFfBqn.css} +1 -1
  6. package/dist/dashboard/index.html +4 -3
  7. package/dist/dashboard/routes/health.js +10 -3
  8. package/dist/dashboard/routes/moments.js +1 -1
  9. package/dist/dashboard/routes/preferences.d.ts +1 -0
  10. package/dist/dashboard/routes/preferences.js +31 -3
  11. package/dist/dashboard/routes/rules.d.ts +18 -0
  12. package/dist/dashboard/routes/rules.js +160 -6
  13. package/dist/dashboard/routes/traces.js +30 -3
  14. package/dist/dashboard/seed-demo-data.js +14 -3
  15. package/dist/dashboard/server.js +13 -3
  16. package/dist/dashboard/session-auth.d.ts +8 -0
  17. package/dist/dashboard/session-auth.js +237 -0
  18. package/dist/dashboard/validation.d.ts +9 -3
  19. package/dist/dashboard/validation.js +69 -11
  20. package/dist/eval/citation-verify/verifier.d.ts +17 -0
  21. package/dist/eval/citation-verify/verifier.js +68 -15
  22. package/dist/eval/decision-moment.js +17 -9
  23. package/dist/eval/engine.d.ts +62 -0
  24. package/dist/eval/engine.js +196 -58
  25. package/dist/eval/llm-judge/evaluator.js +50 -33
  26. package/dist/eval/llm-judge/templates/index.d.ts +4 -0
  27. package/dist/eval/llm-judge/templates/index.js +10 -4
  28. package/dist/eval/rules/custom.js +59 -6
  29. package/dist/eval/rules/relevance.js +1 -1
  30. package/dist/eval/rules/safety.d.ts +8 -0
  31. package/dist/eval/rules/safety.js +63 -18
  32. package/dist/index.js +102 -16
  33. package/dist/middleware/rate-limit.d.ts +25 -0
  34. package/dist/middleware/rate-limit.js +54 -2
  35. package/dist/self-test.d.ts +14 -0
  36. package/dist/self-test.js +97 -13
  37. package/dist/storage/demo-guard.d.ts +8 -0
  38. package/dist/storage/demo-guard.js +53 -0
  39. package/dist/storage/migrations/006-eval-critical-failures.d.ts +3 -0
  40. package/dist/storage/migrations/006-eval-critical-failures.js +23 -0
  41. package/dist/storage/migrations/index.js +2 -0
  42. package/dist/storage/sqlite-adapter.d.ts +6 -0
  43. package/dist/storage/sqlite-adapter.js +91 -4
  44. package/dist/tools/delete-rule.js +49 -11
  45. package/dist/tools/deploy-rule.d.ts +33 -0
  46. package/dist/tools/deploy-rule.js +130 -27
  47. package/dist/tools/evaluate-output.js +50 -24
  48. package/dist/tools/evaluate-with-llm-judge.js +11 -4
  49. package/dist/tools/get-traces.d.ts +27 -0
  50. package/dist/tools/get-traces.js +60 -8
  51. package/dist/tools/list-rules.js +2 -2
  52. package/dist/tools/log-trace.js +5 -4
  53. package/dist/tools/strict-input.d.ts +1 -0
  54. package/dist/tools/strict-input.js +25 -0
  55. package/dist/tools/trace-link.d.ts +7 -0
  56. package/dist/tools/trace-link.js +39 -0
  57. package/dist/tools/verify-citations.d.ts +19 -0
  58. package/dist/tools/verify-citations.js +42 -5
  59. package/dist/types/decision-moment.d.ts +8 -0
  60. package/dist/types/eval.d.ts +60 -1
  61. package/dist/types/index.d.ts +1 -1
  62. package/dist/types/query.d.ts +25 -0
  63. package/package.json +1 -1
  64. package/server.json +2 -2
  65. package/dist/dashboard/assets/index-BZZt8bVh.js +0 -10
@@ -33,6 +33,20 @@ function configError(definition, message) {
33
33
  function safeRegexResult(definition, message) {
34
34
  return configError(definition, message);
35
35
  }
36
+ /**
37
+ * `config.keywords` as a non-empty array of strings, or undefined when it
38
+ * is anything else. Element types are checked at runtime because the
39
+ * inline schema accepts any config value: `keywords: [1, 2]` passed the
40
+ * old Array.isArray check and then threw from `.toLowerCase()` mid-eval.
41
+ */
42
+ function readKeywordList(config) {
43
+ const value = config.keywords;
44
+ if (!Array.isArray(value) || value.length === 0)
45
+ return undefined;
46
+ if (!value.every((k) => typeof k === 'string'))
47
+ return undefined;
48
+ return value;
49
+ }
36
50
  /**
37
51
  * Converts a leading inline flag group like `(?i)` or `(?im)` into a real
38
52
  * flags argument. Node's RegExp engine does not support inline flag groups,
@@ -65,7 +79,26 @@ export function normalizeRegexSource(patternStr, flags) {
65
79
  * general. The sandbox's hard deadline is the boundary.
66
80
  */
67
81
  function validateRegex(definition) {
68
- const { pattern: patternStr, flags } = normalizeRegexSource(definition.config.pattern, definition.config.flags ?? '');
82
+ /*
83
+ * Runtime shape check, not just a compile-time cast. evaluate_output's
84
+ * inline custom_rules schema accepts any config record, so
85
+ * `{type: "regex_match", config: {}}` (or a null / numeric pattern)
86
+ * reaches this point; the old `as string` cast was a no-op at runtime
87
+ * and normalizeRegexSource threw a TypeError out of the engine — the
88
+ * whole evaluate_output call failed, contradicting its own description
89
+ * ("the eval itself never throws"). Deploy-time validation already
90
+ * rejects these; this is the same configError contract for the inline
91
+ * path and for rules persisted before that validation existed.
92
+ */
93
+ const rawPattern = definition.config.pattern;
94
+ if (typeof rawPattern !== 'string' || rawPattern.length === 0) {
95
+ return safeRegexResult(definition, `${definition.type} rule requires config.pattern (non-empty string)`);
96
+ }
97
+ const rawFlags = definition.config.flags;
98
+ if (rawFlags !== undefined && rawFlags !== null && typeof rawFlags !== 'string') {
99
+ return safeRegexResult(definition, `${definition.type} rule config.flags must be a string when present`);
100
+ }
101
+ const { pattern: patternStr, flags } = normalizeRegexSource(rawPattern, rawFlags ?? '');
69
102
  if (patternStr.length > MAX_PATTERN_LENGTH) {
70
103
  return safeRegexResult(definition, `Regex pattern too long (${patternStr.length} > ${MAX_PATTERN_LENGTH})`);
71
104
  }
@@ -233,8 +266,8 @@ export function createCustomRule(definition, severity) {
233
266
  return { ruleName: definition.name, passed, score: passed ? 1 : max / context.output.length, message: passed ? `Length (${context.output.length}) within maximum (${max})` : `Length (${context.output.length}) exceeds maximum (${max})` };
234
267
  }
235
268
  case 'contains_keywords': {
236
- const keywords = definition.config.keywords;
237
- if (!keywords || !Array.isArray(keywords) || keywords.length === 0) {
269
+ const keywords = readKeywordList(definition.config);
270
+ if (!keywords) {
238
271
  return configError(definition, 'contains_keywords rule requires config.keywords (non-empty string array)');
239
272
  }
240
273
  const lower = context.output.toLowerCase();
@@ -244,8 +277,8 @@ export function createCustomRule(definition, severity) {
244
277
  return { ruleName: definition.name, passed, score: ratio, message: `Found ${found.length}/${keywords.length} required keywords` };
245
278
  }
246
279
  case 'excludes_keywords': {
247
- const keywords = definition.config.keywords;
248
- if (!keywords || !Array.isArray(keywords) || keywords.length === 0) {
280
+ const keywords = readKeywordList(definition.config);
281
+ if (!keywords) {
249
282
  return configError(definition, 'excludes_keywords rule requires config.keywords (non-empty string array)');
250
283
  }
251
284
  const lower = context.output.toLowerCase();
@@ -267,7 +300,27 @@ export function createCustomRule(definition, severity) {
267
300
  if (max == null || max < 0) {
268
301
  return configError(definition, `cost_threshold rule requires ${describeKeys('cost_threshold')} (non-negative number)`);
269
302
  }
270
- const cost = context.costUsd ?? 0;
303
+ /*
304
+ * No cost data → SKIP, exactly like the built-in
305
+ * cost_under_threshold (cost.ts). The old `context.costUsd ?? 0`
306
+ * read a missing cost as free, so a rule deployed at severity
307
+ * critical to hard-fail evaluations over $0.50 reported
308
+ * passed:true, score:1 on every evaluate_output call that simply
309
+ * omitted cost_usd — the veto never fired on evidence it never
310
+ * had. A skipped critical rule is reported in critical_skipped
311
+ * instead, so a fail-closed gate can see the rule did not run.
312
+ */
313
+ if (context.costUsd === undefined || context.costUsd === null) {
314
+ return {
315
+ ruleName: definition.name,
316
+ passed: false,
317
+ score: 0,
318
+ message: 'Cost data not provided',
319
+ skipped: true,
320
+ skipReason: 'context.costUsd not provided',
321
+ };
322
+ }
323
+ const cost = context.costUsd;
271
324
  const passed = cost <= max;
272
325
  return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? `Cost ($${cost}) within threshold ($${max})` : `Cost ($${cost}) exceeds threshold ($${max})` };
273
326
  }
@@ -30,7 +30,7 @@ export const keywordOverlap = {
30
30
  };
31
31
  /*
32
32
  * no_hallucination_markers moved to the safety bundle (safety.ts) in
33
- * v0.4.7 — its rewrite is context-grounded fabrication/contradiction
33
+ * v0.5.0 — its rewrite is context-grounded fabrication/contradiction
34
34
  * detection, and the safety bundle is where the evaluate_output docs,
35
35
  * the dashboard's safety-violations panel, and the storage adapter's
36
36
  * violation counts have always placed it.
@@ -4,6 +4,14 @@ export declare const PII_PATTERNS: Array<{
4
4
  pattern: RegExp;
5
5
  placeholders?: RegExp[];
6
6
  }>;
7
+ /**
8
+ * The pass message when placeholders were ignored. Says so explicitly,
9
+ * with the count and the pattern names (#370): a builder smoke-testing with
10
+ * `bob@example.com` or a 555 number used to read a bare "No PII detected"
11
+ * and conclude detection was broken, when the rule had recognised the
12
+ * value as documentation on purpose.
13
+ */
14
+ export declare function describeSuppressedPlaceholders(suppressed: Map<string, number>): string;
7
15
  export declare const noPii: EvalRule;
8
16
  export declare const noBlocklistWords: EvalRule;
9
17
  export declare const INJECTION_PATTERNS: RegExp[];
@@ -8,9 +8,10 @@
8
8
  *
9
9
  * `placeholders` suppresses documentation values that are PII-shaped but by
10
10
  * definition not PII: RFC 2606 example domains, the reserved 555 fictional
11
- * phone block and toll-free lines, published payment test cards, the
12
- * never-issued docs SSN, masked keys, and 10-digit runs with no separators
13
- * (Unix timestamps, JWTs and rate-limit headers read as "phone numbers").
11
+ * phone block and toll-free lines, published payment test cards, masked
12
+ * keys, and 10-digit runs with no separators (Unix timestamps, JWTs and
13
+ * rate-limit headers read as "phone numbers"). The canonical documentation
14
+ * SSN is deliberately NOT suppressed — see the SSN entry below (#362).
14
15
  * A pattern only fails the rule when at least one of its matches is NOT
15
16
  * covered by a placeholder — so real PII beside a placeholder still fails.
16
17
  */
@@ -104,10 +105,28 @@ export const PII_PATTERNS = [
104
105
  // v0.3.1 additions
105
106
  // IBAN: 2 letters + 2 digits + 1-30 alphanumeric (international bank account number)
106
107
  { name: 'IBAN', pattern: /\b[A-Z]{2}\d{2}[A-Z0-9]{10,30}\b/ },
107
- // US passport: 9 digits, optionally prefixed with letter (modern format C12345678)
108
- { name: 'Passport', pattern: /\b[A-Z]?\d{9}\b/ },
109
- // Date of birth contextual DOB or "Born:" / "Birthday:" + date
110
- { name: 'DOB', pattern: /\b(?:DOB|D\.O\.B\.|Date of Birth|Born|Birthday)\s{0,8}[:.]?\s{0,8}\d{1,2}[\/\-.]\d{1,2}[\/\-.](?:\d{2}|\d{4})\b/i },
108
+ /*
109
+ * US passport — CONTEXT-ANCHORED, like DOB and MRN below. A legacy
110
+ * passport number is nine bare digits and the modern (2021+) format is
111
+ * one letter + eight digits; neither shape has internal structure to
112
+ * anchor on. The old `\b[A-Z]?\d{9}\b` fired on ANY nine-digit run —
113
+ * order IDs, EINs, routing numbers, nine-digit Unix timestamps — and
114
+ * because no_pii is critical, "Order ID: 123456789" vetoed the whole
115
+ * evaluation. It also never matched the modern C12345678 shape its own
116
+ * comment promised: the optional letter still demanded nine digits after
117
+ * it. Now the number must follow the word "passport" within a short
118
+ * window, which is what docs/api-reference.md has described all along.
119
+ * The window is bounded ({0,40}) so the scan stays linear in the input.
120
+ */
121
+ { name: 'Passport', pattern: /\bpassports?\b[\s\S]{0,40}?\b(?:[A-Z]\d{8}|\d{9})\b/i },
122
+ // Date of birth contextual — DOB or "Born:" / "Birthday:" + a date in
123
+ // either US/EU numeric form (03/15/1987, 15.03.87) or ISO form
124
+ // (1987-03-15). The ISO alternative is listed first: it is the shape
125
+ // `Date of birth: 1987-03-15` takes in any structured record, and the
126
+ // label-anchored pattern used to miss exactly that while catching the
127
+ // slash form (#374). Both alternatives are fixed-width per position, so
128
+ // the scan stays linear.
129
+ { name: 'DOB', pattern: /\b(?:DOB|D\.O\.B\.|Date of Birth|Born|Birthday)\s{0,8}[:.]?\s{0,8}(?:\d{4}-\d{2}-\d{2}|\d{1,2}[\/\-.]\d{1,2}[\/\-.](?:\d{2}|\d{4}))\b/i },
111
130
  // Medical record number — MRN: + alphanumeric (common format)
112
131
  { name: 'Medical Record Number', pattern: /\b(?:MRN|Medical Record (?:Number|No\.?|#))\s{0,8}[:.]?\s{0,8}[A-Z0-9]{6,12}\b/i },
113
132
  // IPv4 address
@@ -137,19 +156,40 @@ export const PII_PATTERNS = [
137
156
  { name: 'Seed Phrase', pattern: /\b(?:[Ss]eed|[Rr]ecovery|[Mm]nemonic)\s(?:[Pp]hrase|[Ww]ords)\b[\s\S]{0,120}?\b(?:[a-z]{3,8}\s{1,4}){11}[a-z]{3,8}\b/ },
138
157
  ];
139
158
  /**
140
- * True when `pattern` has at least one match in `output` that is not one of
141
- * the pattern's documented placeholder values. Patterns without a
142
- * `placeholders` list keep the plain test() fast path.
159
+ * `fired` is true when `pattern` has at least one match in `output` that is
160
+ * not one of the pattern's documented placeholder values; `suppressed`
161
+ * counts the matches that WERE placeholders. Patterns without a
162
+ * `placeholders` list keep the plain test() fast path, and the scan stops
163
+ * at the first real match — the suppressed count is only complete (and only
164
+ * reported) when nothing real fired.
143
165
  */
144
- function piiPatternFires(output, pattern, placeholders) {
166
+ function piiPatternMatches(output, pattern, placeholders) {
145
167
  if (!placeholders)
146
- return pattern.test(output);
168
+ return { fired: pattern.test(output), suppressed: 0 };
147
169
  const global = new RegExp(pattern.source, pattern.flags.includes('g') ? pattern.flags : `${pattern.flags}g`);
170
+ let suppressed = 0;
148
171
  for (const match of output.matchAll(global)) {
149
172
  if (!placeholders.some((placeholder) => placeholder.test(match[0])))
150
- return true;
173
+ return { fired: true, suppressed };
174
+ suppressed++;
151
175
  }
152
- return false;
176
+ return { fired: false, suppressed };
177
+ }
178
+ /**
179
+ * The pass message when placeholders were ignored. Says so explicitly,
180
+ * with the count and the pattern names (#370): a builder smoke-testing with
181
+ * `bob@example.com` or a 555 number used to read a bare "No PII detected"
182
+ * and conclude detection was broken, when the rule had recognised the
183
+ * value as documentation on purpose.
184
+ */
185
+ export function describeSuppressedPlaceholders(suppressed) {
186
+ const total = [...suppressed.values()].reduce((sum, n) => sum + n, 0);
187
+ if (total === 0)
188
+ return 'No PII detected';
189
+ const parts = [...suppressed.entries()].map(([name, n]) => (n > 1 ? `${name} ×${n}` : name));
190
+ return (`No PII detected (${total} documentation placeholder${total === 1 ? '' : 's'} ignored: ${parts.join(', ')} — ` +
191
+ 'example.com/.org/.net addresses, 555 and toll-free phone numbers, published payment test cards ' +
192
+ 'and masked keys are never counted as PII; real values in the same shapes still fail)');
153
193
  }
154
194
  export const noPii = {
155
195
  name: 'no_pii',
@@ -166,17 +206,22 @@ export const noPii = {
166
206
  critical: true,
167
207
  evaluate(context) {
168
208
  const found = [];
209
+ const suppressed = new Map();
169
210
  for (const { name, pattern, placeholders } of PII_PATTERNS) {
170
- if (piiPatternFires(context.output, pattern, placeholders)) {
211
+ const { fired, suppressed: ignored } = piiPatternMatches(context.output, pattern, placeholders);
212
+ if (fired) {
171
213
  found.push(name);
172
214
  }
215
+ else if (ignored > 0) {
216
+ suppressed.set(name, ignored);
217
+ }
173
218
  }
174
219
  const passed = found.length === 0;
175
220
  return {
176
221
  ruleName: 'no_pii',
177
222
  passed,
178
223
  score: passed ? 1 : 0,
179
- message: passed ? 'No PII detected' : `Potential PII detected: ${found.join(', ')}`,
224
+ message: passed ? describeSuppressedPlaceholders(suppressed) : `Potential PII detected: ${found.join(', ')}`,
180
225
  };
181
226
  },
182
227
  };
@@ -742,7 +787,7 @@ export const noStubOutput = {
742
787
  },
743
788
  };
744
789
  /*
745
- * Hallucination detection — rewritten v0.4.7, moved here from the relevance
790
+ * Hallucination detection — rewritten v0.5.0, moved here from the relevance
746
791
  * bundle in the same change.
747
792
  *
748
793
  * The previous incarnation matched 17 refusal-boilerplate phrases ("as an
@@ -1150,7 +1195,7 @@ function detectUngroundedDate(output, input) {
1150
1195
  }
1151
1196
  /*
1152
1197
  * Parse a markdown table row by splitting on '|' — never by regexing the
1153
- * whole line. The v0.4.7 first cut used /^\s*\|\s*([^|]+?)\s*\|(.+)\|?\s*$/,
1198
+ * whole line. The v0.5.0 first cut used /^\s*\|\s*([^|]+?)\s*\|(.+)\|?\s*$/,
1154
1199
  * where the greedy \s* and lazy [^|]+? both match a run of spaces: on a
1155
1200
  * line of '|' + N spaces with no closing pipe the engine has ~N ways to
1156
1201
  * split the run, each failing late — super-quadratic backtracking (~7.5×
package/dist/index.js CHANGED
@@ -2,7 +2,9 @@
2
2
  import { parseArgs } from 'node:util';
3
3
  import { z } from 'zod';
4
4
  import { loadConfig } from './config/index.js';
5
+ import { PKG_VERSION } from './config/defaults.js';
5
6
  import { createStorage } from './storage/index.js';
7
+ import { withDemoIngestGuard } from './storage/demo-guard.js';
6
8
  import { createIrisServer } from './server.js';
7
9
  import { createStdioTransport } from './transport/stdio.js';
8
10
  import { createHttpTransport } from './transport/http.js';
@@ -35,6 +37,8 @@ const CliSchema = z
35
37
  demo: z.boolean().optional(),
36
38
  'demo-clear': z.boolean().optional(),
37
39
  'self-test': z.boolean().optional(),
40
+ purge: z.boolean().optional(),
41
+ version: z.boolean().optional(),
38
42
  help: z.boolean().optional(),
39
43
  })
40
44
  .strict();
@@ -65,6 +69,8 @@ try {
65
69
  demo: { type: 'boolean', default: false },
66
70
  'demo-clear': { type: 'boolean', default: false },
67
71
  'self-test': { type: 'boolean', default: false },
72
+ purge: { type: 'boolean', default: false },
73
+ version: { type: 'boolean', default: false },
68
74
  help: { type: 'boolean', short: 'h', default: false },
69
75
  },
70
76
  strict: true,
@@ -74,6 +80,16 @@ catch (err) {
74
80
  process.stderr.write(`iris-mcp: ${err.message}\nRun \`iris-mcp --help\` for usage.\n`);
75
81
  process.exit(2);
76
82
  }
83
+ /*
84
+ * --version answers on stdout, bare, before anything else: the README's
85
+ * "check your version" recipe pointed at a flag that did not exist and a
86
+ * --help banner that printed no version (#369). Bare so `iris-mcp
87
+ * --version` composes in scripts the way `npm --version` does.
88
+ */
89
+ if (parsed.values.version) {
90
+ process.stdout.write(`${PKG_VERSION}\n`);
91
+ process.exit(0);
92
+ }
77
93
  const validation = CliSchema.safeParse(parsed.values);
78
94
  if (!validation.success) {
79
95
  const issues = validation.error.issues
@@ -85,7 +101,7 @@ if (!validation.success) {
85
101
  const values = validation.data;
86
102
  if (values.help) {
87
103
  process.stderr.write(`
88
- Iris — MCP-Native Agent Eval Server
104
+ Iris — MCP-Native Agent Eval Server v${PKG_VERSION}
89
105
 
90
106
  Usage: iris-mcp [options]
91
107
 
@@ -107,9 +123,16 @@ Options:
107
123
  transport). Idempotent: re-running reuses the seeded data.
108
124
  --demo-clear Delete the demo database (and its sidecar files), then exit.
109
125
  Your real traces are not touched.
110
- --self-test Run the offline install diagnostic and exit: storage round-trip,
111
- deterministic evals, dashboard + rebinding guard all inside an
126
+ --self-test Run the offline install diagnostic and exit: the configured IRIS_HOME
127
+ is created and probed for writability, then storage round-trip,
128
+ deterministic evals, dashboard + rebinding guard run inside an
112
129
  isolated temp home. Exit code 0 = healthy, 1 = a check failed.
130
+ --purge Delete EVERY stored trace, span and evaluation from the configured
131
+ database, compact the file and truncate the write-ahead log so the
132
+ deleted text does not linger on disk, then exit. Deployed rules, the
133
+ audit log and preferences are kept. Not reversible. Stop any running
134
+ Iris server first — the file is compacted in place.
135
+ --version Print the version and exit
113
136
  -h, --help Show this help message
114
137
 
115
138
  Environment variables (CLI flags take precedence):
@@ -139,11 +162,31 @@ Environment variables (CLI flags take precedence):
139
162
  IRIS_OTEL_TIMEOUT_MS Per-export timeout (default: 15000)
140
163
  RATE_LIMIT_SALT (waitlist API only — required when website is deployed)
141
164
 
142
- Dashboard preferences (~/.iris/preferences.json):
165
+ Dashboard preferences ($IRIS_HOME/preferences.json, default ~/.iris/preferences.json):
143
166
  Edit autoLaunch: false to permanently disable first-run dashboard auto-launch.
144
167
  `);
145
168
  process.exit(0);
146
169
  }
170
+ /*
171
+ * The mode flags are mutually exclusive, and the check runs before any of
172
+ * them so a refused combination exits without touching the filesystem —
173
+ * `--self-test --purge` must not quietly run only the first one it sees.
174
+ */
175
+ const modeFlags = ['demo', 'demo-clear', 'self-test', 'purge'].filter((flag) => values[flag]);
176
+ if (modeFlags.length > 1) {
177
+ process.stderr.write(`iris-mcp: ${modeFlags.map((flag) => `--${flag}`).join(' and ')} cannot be combined.\nRun \`iris-mcp --help\` for usage.\n`);
178
+ process.exit(2);
179
+ }
180
+ /*
181
+ * `--purge --dashboard` used to run the purge and drop the dashboard flag
182
+ * on the floor. A flag that changes nothing is the same fault as a
183
+ * misspelled tool argument: the caller asked for something and got no
184
+ * sign it was ignored. (v0.6.0 acceptance pass.)
185
+ */
186
+ if (values.purge && values.dashboard) {
187
+ process.stderr.write('iris-mcp: --purge exits as soon as the purge finishes and cannot be combined with --dashboard (nothing would be served).\nRun `iris-mcp --help` for usage.\n');
188
+ process.exit(2);
189
+ }
147
190
  /*
148
191
  * --self-test exits BEFORE loadConfig() runs at module scope below —
149
192
  * deliberately. The diagnostic builds its own isolated IRIS_HOME and
@@ -154,14 +197,6 @@ if (values['self-test']) {
154
197
  const { runSelfTest } = await import('./self-test.js');
155
198
  process.exit(await runSelfTest());
156
199
  }
157
- /*
158
- * Demo-mode flag validation happens before loadConfig so a refused
159
- * combination exits without touching the filesystem.
160
- */
161
- if (values.demo && values['demo-clear']) {
162
- process.stderr.write('iris-mcp: --demo and --demo-clear cannot be combined.\nRun `iris-mcp --help` for usage.\n');
163
- process.exit(2);
164
- }
165
200
  if (values.demo && values['db-path']) {
166
201
  process.stderr.write('iris-mcp: --demo always serves its own database (demo.db under your iris home) and cannot be combined with --db-path.\n' +
167
202
  'Run `iris-mcp --demo` alone, or drop --demo to use your own database.\n');
@@ -193,6 +228,39 @@ const config = loadConfig({
193
228
  dashboardHost: values['dashboard-host'],
194
229
  });
195
230
  const logger = createLogger(config);
231
+ /*
232
+ * --purge (#372): the retention sweep only ever trimmed by age, and
233
+ * deleting iris.db by hand left every row readable in iris.db-wal. This
234
+ * is the one-command answer to "remove everything Iris stored about my
235
+ * agents" — every trace, span and evaluation for the local tenant,
236
+ * followed by VACUUM + a TRUNCATE checkpoint so the text is gone from the
237
+ * main file and the write-ahead log alike. Rules, audit log and
238
+ * preferences are not storage rows and stay. Prints what it removed and
239
+ * exits 0; runs against the configured database, never the demo one
240
+ * (--demo-clear handles that).
241
+ */
242
+ async function runPurge() {
243
+ const storage = createStorage(config);
244
+ await storage.initialize();
245
+ try {
246
+ const { traces, evalResults } = await storage.purge(LOCAL_TENANT);
247
+ process.stderr.write(`iris-mcp: purged ${traces} trace(s) and ${evalResults} evaluation(s) from "${config.storage.path}" ` +
248
+ '(database compacted, write-ahead log truncated). Deployed rules, audit log and preferences were kept.\n');
249
+ }
250
+ finally {
251
+ await storage.close();
252
+ }
253
+ }
254
+ if (values.purge) {
255
+ try {
256
+ await runPurge();
257
+ process.exit(0);
258
+ }
259
+ catch (err) {
260
+ process.stderr.write(`iris-mcp: purge failed: ${err instanceof Error ? err.message : String(err)}\n`);
261
+ process.exit(1);
262
+ }
263
+ }
196
264
  async function main() {
197
265
  logger.info(`Starting Iris MCP server v${config.server.version}`);
198
266
  // F-006: fail fast on HTTP+dashboard port collision. See validatePortConfig.
@@ -231,9 +299,19 @@ async function main() {
231
299
  // cleanly.
232
300
  if (config.retention.days > 0) {
233
301
  try {
234
- const deleted = await storage.deleteTracesOlderThan(LOCAL_TENANT, config.retention.days);
235
- if (deleted > 0) {
236
- logger.info(`Retention cleanup: deleted ${deleted} trace(s) older than ${config.retention.days} days`);
302
+ const deletedTraces = await storage.deleteTracesOlderThan(LOCAL_TENANT, config.retention.days);
303
+ /*
304
+ * Evaluations too (#372). Deleting a trace only NULLs trace_id on
305
+ * its evaluations, so every eval row — output_text verbatim,
306
+ * including whatever no_pii flagged — used to outlive the retention
307
+ * window indefinitely while the traces around it were swept.
308
+ */
309
+ const deletedEvals = await storage.deleteEvalResultsOlderThan(LOCAL_TENANT, config.retention.days);
310
+ if (deletedTraces + deletedEvals > 0) {
311
+ // Fold the WAL into the main file and truncate it, so the swept
312
+ // rows do not survive as readable text in iris.db-wal.
313
+ await storage.checkpoint();
314
+ logger.info(`Retention cleanup: deleted ${deletedTraces} trace(s) and ${deletedEvals} evaluation(s) older than ${config.retention.days} days`);
237
315
  }
238
316
  }
239
317
  catch (err) {
@@ -332,6 +410,8 @@ ${line}
332
410
  ${counts}
333
411
  Demo database: "${summary.dbPath}"
334
412
  Your real trace database is untouched — demo data never mixes with it.
413
+ Trace ingest (POST /api/v1/traces) is refused in demo mode: start the
414
+ real server (iris-mcp --dashboard) to store your own traces.
335
415
 
336
416
  Worth clicking into:
337
417
  - a PII leak (a synthetic SSN in an agent reply) caught by the safety rules
@@ -367,7 +447,13 @@ async function runDemo() {
367
447
  else {
368
448
  logger.info(`Seeded demo database with ${seedSummary.traceCount} traces at ${seedSummary.dbPath}`);
369
449
  }
370
- const storage = createStorage(config);
450
+ /*
451
+ * Ingest is refused in demo mode: demo.db is disposable by design and
452
+ * --demo-clear deletes it wholesale, so a capture client pointed at the
453
+ * demo dashboard's port would have its real traces silently stored next
454
+ * to the fake ones and later destroyed (storage/demo-guard.ts).
455
+ */
456
+ const storage = withDemoIngestGuard(createStorage(config));
371
457
  await storage.initialize();
372
458
  const customRuleStore = createCustomRuleStore({
373
459
  pathFor: () => demoCustomRulesPath(),
@@ -1,3 +1,28 @@
1
1
  import type { IrisConfig } from '../types/config.js';
2
2
  export declare function createApiRateLimiter(config: Pick<IrisConfig, 'security'>): import("express-rate-limit").RateLimitRequestHandler;
3
+ /**
4
+ * Mounted directly in front of the session layer, so every request that
5
+ * is about to be AUTHORIZED — cookie check, Bearer check, the `?key=`
6
+ * exchange — has passed a per-IP ceiling first (CodeQL
7
+ * js/missing-rate-limiting on the session middleware). Same figure as the
8
+ * API limiter with its own counter; static assets share it, which a
9
+ * dashboard page load (a few dozen requests) never approaches.
10
+ */
11
+ export declare function createAuthGateRateLimiter(config: Pick<IrisConfig, 'security'>): import("express-rate-limit").RateLimitRequestHandler;
12
+ /**
13
+ * JSON-RPC 2.0 application error code for "rate limited". The reserved
14
+ * server range is -32000..-32099; the MCP SDK uses -32000/-32001 for
15
+ * connection-closed and request-timeout, so this sits clear of both.
16
+ * Mirrors HTTP 429 in the low digits so a log line reads at a glance.
17
+ */
18
+ export declare const JSON_RPC_RATE_LIMITED = -32029;
19
+ /**
20
+ * The MCP endpoint speaks JSON-RPC, so its 429 must too (#373). The stock
21
+ * express-rate-limit body — `{ "error": "Too many requests" }` — is not a
22
+ * JSON-RPC message: a strict client (the reference SDK included) fails to
23
+ * parse the response and surfaces a PROTOCOL error, and the one thing the
24
+ * caller needed to learn — wait, then retry — is exactly what got lost. The
25
+ * envelope below echoes the request id when the body carried one, names
26
+ * the limit and the wait, and points at the config key that raises it.
27
+ */
3
28
  export declare function createMcpRateLimiter(config: Pick<IrisConfig, 'security'>): import("express-rate-limit").RateLimitRequestHandler;
@@ -8,12 +8,64 @@ export function createApiRateLimiter(config) {
8
8
  message: { error: 'Too many requests, please try again later' },
9
9
  });
10
10
  }
11
- export function createMcpRateLimiter(config) {
11
+ /**
12
+ * Mounted directly in front of the session layer, so every request that
13
+ * is about to be AUTHORIZED — cookie check, Bearer check, the `?key=`
14
+ * exchange — has passed a per-IP ceiling first (CodeQL
15
+ * js/missing-rate-limiting on the session middleware). Same figure as the
16
+ * API limiter with its own counter; static assets share it, which a
17
+ * dashboard page load (a few dozen requests) never approaches.
18
+ */
19
+ export function createAuthGateRateLimiter(config) {
12
20
  return rateLimit({
13
21
  windowMs: 60_000,
14
- limit: config.security.rateLimit.mcp,
22
+ limit: config.security.rateLimit.api,
15
23
  standardHeaders: 'draft-7',
16
24
  legacyHeaders: false,
17
25
  message: { error: 'Too many requests, please try again later' },
18
26
  });
19
27
  }
28
+ /**
29
+ * JSON-RPC 2.0 application error code for "rate limited". The reserved
30
+ * server range is -32000..-32099; the MCP SDK uses -32000/-32001 for
31
+ * connection-closed and request-timeout, so this sits clear of both.
32
+ * Mirrors HTTP 429 in the low digits so a log line reads at a glance.
33
+ */
34
+ export const JSON_RPC_RATE_LIMITED = -32029;
35
+ /**
36
+ * The MCP endpoint speaks JSON-RPC, so its 429 must too (#373). The stock
37
+ * express-rate-limit body — `{ "error": "Too many requests" }` — is not a
38
+ * JSON-RPC message: a strict client (the reference SDK included) fails to
39
+ * parse the response and surfaces a PROTOCOL error, and the one thing the
40
+ * caller needed to learn — wait, then retry — is exactly what got lost. The
41
+ * envelope below echoes the request id when the body carried one, names
42
+ * the limit and the wait, and points at the config key that raises it.
43
+ */
44
+ export function createMcpRateLimiter(config) {
45
+ const limit = config.security.rateLimit.mcp;
46
+ return rateLimit({
47
+ windowMs: 60_000,
48
+ limit,
49
+ standardHeaders: 'draft-7',
50
+ legacyHeaders: false,
51
+ handler: (req, res) => {
52
+ const body = req.body;
53
+ const requestId = body !== null && typeof body === 'object' && !Array.isArray(body) && 'id' in body
54
+ ? body.id
55
+ : null;
56
+ const id = typeof requestId === 'string' || typeof requestId === 'number' ? requestId : null;
57
+ const resetTime = req.rateLimit?.resetTime;
58
+ const retryAfterSeconds = resetTime instanceof Date ? Math.max(1, Math.ceil((resetTime.getTime() - Date.now()) / 1000)) : 60;
59
+ res.status(429).json({
60
+ jsonrpc: '2.0',
61
+ id,
62
+ error: {
63
+ code: JSON_RPC_RATE_LIMITED,
64
+ message: `Rate limit exceeded: this MCP endpoint allows ${limit} requests per minute. ` +
65
+ `Retry in ${retryAfterSeconds}s, or raise security.rateLimit.mcp in config.json for an interactive session.`,
66
+ data: { limit, windowMs: 60_000, retryAfterSeconds },
67
+ },
68
+ });
69
+ },
70
+ });
71
+ }
@@ -1,4 +1,5 @@
1
1
  export declare const SELF_TEST_STEPS: {
2
+ readonly configuredHome: "configured IRIS_HOME is writable";
2
3
  readonly tempHome: "create isolated temp home";
3
4
  readonly storage: "initialize storage";
4
5
  readonly trace: "log a trace";
@@ -15,4 +16,17 @@ export declare const SELF_TEST_STEPS: {
15
16
  export declare const SELF_TEST_PASS_VERDICT = "\u2713 PASS \u2014 this install works";
16
17
  export declare const SELF_TEST_FAIL_VERDICT = "\u2717 FAIL";
17
18
  export type WriteLine = (line: string) => void;
19
+ /**
20
+ * The configured-home probe (#371). Exercises the exact calls the real
21
+ * server makes at startup, in order: create IRIS_HOME (same helper and
22
+ * mode as loadConfig), create the database directory when IRIS_DB_PATH
23
+ * points elsewhere, write-and-unlink a probe file in each, and — only if
24
+ * the real database already exists — open it and take a write lock
25
+ * (BEGIN IMMEDIATE … ROLLBACK), which fails on a read-only file or a
26
+ * non-database exactly as the first INSERT would, without migrating or
27
+ * changing anything. A missing database is not created: the server
28
+ * creates it on first run, and the writable-directory probe is what
29
+ * proves that it can.
30
+ */
31
+ export declare function probeConfiguredHome(home: string, dbPath: string): string;
18
32
  export declare function runSelfTest(write?: WriteLine): Promise<number>;