@iris-eval/mcp-server 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +95 -33
  2. package/dist/config/index.d.ts +10 -0
  3. package/dist/config/index.js +33 -7
  4. package/dist/dashboard/assets/index-CshLgDRB.js +10 -0
  5. package/dist/dashboard/assets/{index-UffZ-aEJ.css → index-D0cFfBqn.css} +1 -1
  6. package/dist/dashboard/index.html +4 -3
  7. package/dist/dashboard/routes/health.js +10 -3
  8. package/dist/dashboard/routes/moments.js +1 -1
  9. package/dist/dashboard/routes/preferences.d.ts +1 -0
  10. package/dist/dashboard/routes/preferences.js +31 -3
  11. package/dist/dashboard/routes/rules.d.ts +18 -0
  12. package/dist/dashboard/routes/rules.js +160 -6
  13. package/dist/dashboard/routes/traces.js +21 -3
  14. package/dist/dashboard/seed-demo-data.js +11 -0
  15. package/dist/dashboard/server.js +13 -3
  16. package/dist/dashboard/session-auth.d.ts +8 -0
  17. package/dist/dashboard/session-auth.js +237 -0
  18. package/dist/dashboard/validation.d.ts +9 -3
  19. package/dist/dashboard/validation.js +69 -11
  20. package/dist/eval/engine.d.ts +62 -0
  21. package/dist/eval/engine.js +188 -82
  22. package/dist/eval/rules/safety.d.ts +8 -0
  23. package/dist/eval/rules/safety.js +43 -11
  24. package/dist/index.js +102 -16
  25. package/dist/middleware/rate-limit.d.ts +25 -0
  26. package/dist/middleware/rate-limit.js +54 -2
  27. package/dist/self-test.d.ts +14 -0
  28. package/dist/self-test.js +97 -13
  29. package/dist/storage/demo-guard.d.ts +8 -0
  30. package/dist/storage/demo-guard.js +53 -0
  31. package/dist/storage/sqlite-adapter.d.ts +6 -0
  32. package/dist/storage/sqlite-adapter.js +72 -1
  33. package/dist/tools/delete-rule.js +49 -11
  34. package/dist/tools/deploy-rule.d.ts +33 -0
  35. package/dist/tools/deploy-rule.js +130 -27
  36. package/dist/tools/evaluate-output.js +41 -22
  37. package/dist/tools/evaluate-with-llm-judge.js +10 -3
  38. package/dist/tools/get-traces.d.ts +27 -0
  39. package/dist/tools/get-traces.js +60 -8
  40. package/dist/tools/list-rules.js +2 -2
  41. package/dist/tools/log-trace.js +4 -3
  42. package/dist/tools/strict-input.d.ts +1 -0
  43. package/dist/tools/strict-input.js +25 -0
  44. package/dist/tools/trace-link.d.ts +7 -0
  45. package/dist/tools/trace-link.js +39 -0
  46. package/dist/tools/verify-citations.d.ts +19 -0
  47. package/dist/tools/verify-citations.js +41 -4
  48. package/dist/types/eval.d.ts +45 -1
  49. package/dist/types/index.d.ts +1 -1
  50. package/dist/types/query.d.ts +25 -0
  51. package/package.json +1 -1
  52. package/server.json +2 -2
  53. package/dist/dashboard/assets/index-VI_nbMfN.js +0 -10
package/README.md CHANGED
@@ -8,7 +8,7 @@
8
8
  [![CI](https://github.com/iris-eval/mcp-server/actions/workflows/ci.yml/badge.svg)](https://github.com/iris-eval/mcp-server/actions/workflows/ci.yml)
9
9
  [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/iris-eval/mcp-server/badge)](https://securityscorecards.dev/viewer/?uri=github.com/iris-eval/mcp-server)
10
10
  [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/12849/badge)](https://www.bestpractices.dev/projects/12849)
11
- [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
11
+ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/iris-eval/mcp-server/blob/main/LICENSE)
12
12
  [![Docker](https://img.shields.io/badge/Docker-ghcr.io-blue?logo=docker)](https://github.com/iris-eval/mcp-server/pkgs/container/mcp-server)
13
13
  [![PulseMCP](https://img.shields.io/badge/PulseMCP-Listed-blue?style=flat-square)](https://www.pulsemcp.com/servers/iris-eval)
14
14
  [![mcp.so](https://img.shields.io/badge/mcp.so-Listed-blue?style=flat-square)](https://mcp.so/server/iris/iris-eval)
@@ -56,11 +56,11 @@ Your agent discovers Iris's nine tools on connect, and the dashboard serves at *
56
56
 
57
57
  The trace lands on the dashboard with its scores. Prefer the MCP server headless? Drop `--dashboard` from the args — you can open the same dashboard any time with `npx @iris-eval/mcp-server --dashboard`.
58
58
 
59
- **One thing worth knowing up front:** MCP tools are called when the model decides to call them. Iris doesn't intercept your agent, so traces are logged when your agent asks it to log them — either because you told it to, or because your code calls the tools directly. Ask your agent to "log this to Iris and evaluate it" and it will. If you want capture that doesn't depend on the model choosing, `POST /api/v1/traces` does exactly that — your code sends the trace over plain HTTP, no model in the loop (see [docs/http-ingest.md](docs/http-ingest.md)). The CLI and SDKs on the [roadmap](docs/roadmap.md) will be thin clients over the same endpoint.
59
+ **One thing worth knowing up front:** MCP tools are called when the model decides to call them. Iris doesn't intercept your agent, so traces are logged when your agent asks it to log them — either because you told it to, or because your code calls the tools directly. Ask your agent to "log this to Iris and evaluate it" and it will. If you want capture that doesn't depend on the model choosing, `POST /api/v1/traces` does exactly that — your code sends the trace over plain HTTP, no model in the loop (see [docs/http-ingest.md](https://github.com/iris-eval/mcp-server/blob/main/docs/http-ingest.md)). The CLI and SDKs on the [roadmap](https://github.com/iris-eval/mcp-server/blob/main/docs/roadmap.md) will be thin clients over the same endpoint.
60
60
 
61
61
  ### Capture over HTTP (no model in the loop)
62
62
 
63
- With the dashboard running, anything that can send an HTTP request can log a trace — and optionally run the deterministic evals in the same request:
63
+ The ingest endpoint lives on the **dashboard port** — `6920` by default, not the MCP transport port — and it exists only while the dashboard is running. Pass `--dashboard` (or set `IRIS_DASHBOARD=true`); `--transport http` on its own does **not** start it, and a request to the transport port returns `404`. With the dashboard up, anything that can send an HTTP request can log a trace — and optionally run the deterministic evals in the same request:
64
64
 
65
65
  ```bash
66
66
  curl -s -X POST "http://127.0.0.1:6920/api/v1/traces" \
@@ -74,15 +74,16 @@ curl -s -X POST "http://127.0.0.1:6920/api/v1/traces" \
74
74
  }'
75
75
  ```
76
76
 
77
- Returns `201` with the stored `trace_id` and the evaluation result. The endpoint accepts the same body as the `log_trace` tool and sits behind the same loopback-only middleware stack as the rest of the dashboard. Full contract, field reference, and error semantics: [docs/http-ingest.md](docs/http-ingest.md).
77
+ Returns `201` with the stored `trace_id` and the evaluation result (in `--demo` mode the endpoint refuses writes with `403`, so demo data never mixes with yours). The endpoint accepts the same body as the `log_trace` tool and sits behind the same middleware stack as the rest of the dashboard: loopback bind and the DNS-rebinding guard by default, plus Bearer auth when you set one. **Two plain facts about it:** it accepts unauthenticated writes unless Iris was started with `--api-key` (or `IRIS_API_KEY`) — the loopback bind is what keeps it to your machine by default, so set a key before binding beyond loopback; and what it stores is verbatim — `input` and `output` land in `iris.db` exactly as sent, including any text `no_pii` goes on to flag. Full contract, field reference, and error semantics: [docs/http-ingest.md](https://github.com/iris-eval/mcp-server/blob/main/docs/http-ingest.md).
78
78
 
79
- ### Check the install
79
+ ### Verify your install
80
80
 
81
81
  ```bash
82
- npx @iris-eval/mcp-server --self-test
82
+ npx @iris-eval/mcp-server --self-test # offline diagnostic; exit 0 = healthy, 1 = a check failed
83
+ npx @iris-eval/mcp-server --version # prints the bare version, e.g. 0.5.1
83
84
  ```
84
85
 
85
- An offline install diagnostic: storage round-trip, deterministic evals, dashboard + DNS-rebinding guard — all inside an isolated temp home, so your real database is never opened. Exit code 0 = healthy, 1 = a check failed.
86
+ `--self-test` first creates your Iris home if it is missing and checks that it is writable (exit 1, naming the path, if it is not), then runs its checks — storage round-trip, a planted SSN and a planted injection caught by the safety rules, dashboard boot, the DNS-rebinding guard — inside an isolated temp home, so your real database is never opened. Everything Iris writes lives under one directory, your **Iris home**: `~/.iris` by default (`%USERPROFILE%\.iris` on Windows), or wherever `IRIS_HOME` points. That is where `iris.db`, `config.json`, `custom-rules.json`, `audit.log`, `preferences.json` and the demo files live; point `IRIS_HOME` at a scratch directory to try Iris without touching your real data.
86
87
 
87
88
  <details>
88
89
  <summary><strong>Setup by tool</strong></summary>
@@ -190,7 +191,7 @@ docker run -p 3000:3000 -p 6920:6920 -v iris-data:/data ghcr.io/iris-eval/mcp-se
190
191
  | **Web Dashboard** | Real-time dark-mode UI that lands on the failures, worst and newest first — trace visualization, eval results, cost breakdowns, and a command palette (⌘K) that searches your own rules, traces, and evals. |
191
192
  | **Local-first** | Everything lives in SQLite on your disk. No account, no sign-up, no telemetry. Outbound HTTP happens only where you opt in: your own LLM-judge key, citation fetching, or an OTel exporter you configure. |
192
193
 
193
- Where this is going next: [the roadmap](docs/roadmap.md).
194
+ Where this is going next: [the roadmap](https://github.com/iris-eval/mcp-server/blob/main/docs/roadmap.md).
194
195
 
195
196
  ## MCP Tools
196
197
 
@@ -206,7 +207,7 @@ Iris registers nine tools that any MCP-compatible agent can invoke — full rule
206
207
  - **`evaluate_with_llm_judge`** — Semantic eval via LLM (Anthropic or OpenAI). Five templates: accuracy, helpfulness, safety, correctness, faithfulness. Cost-capped, per-eval pricing disclosed. **Bring your own API key** (`IRIS_ANTHROPIC_API_KEY` or `IRIS_OPENAI_API_KEY`) — Iris doesn't proxy or relay LLM calls.
207
208
  - **`verify_citations`** — Extract citations from output (numbered, author-year, URLs, DOIs), fetch sources behind an SSRF-guarded + domain-allowlisted resolver, and use an LLM judge to check whether each source actually supports the cited claim. Opt-in outbound HTTP. Same BYOK requirement as `evaluate_with_llm_judge`.
208
209
 
209
- When `IRIS_OTEL_ENDPOINT` is configured, `log_trace` calls also emit a best-effort OTLP/HTTP JSON export to any OpenTelemetry collector (Jaeger, Grafana Tempo, Datadog OTLP, Honeycomb, etc). See [docs/otel-integration.md](docs/otel-integration.md).
210
+ When `IRIS_OTEL_ENDPOINT` is configured, `log_trace` calls also emit a best-effort OTLP/HTTP JSON export to any OpenTelemetry collector (Jaeger, Grafana Tempo, Datadog OTLP, Honeycomb, etc). See [docs/otel-integration.md](https://github.com/iris-eval/mcp-server/blob/main/docs/otel-integration.md).
210
211
 
211
212
  ### How `passed` is decided
212
213
 
@@ -215,10 +216,45 @@ When `IRIS_OTEL_ENDPOINT` is configured, `log_trace` calls also emit a best-effo
215
216
  - **`score`** (0..1) is the weighted average across the rules that ran — a quality gradient.
216
217
  - **`passed`** is the ship/no-ship verdict: `true` only when the score clears the pass threshold (default **0.7**) **and no critical rule failed**.
217
218
 
218
- Genuine safety violations hard-fail. `no_pii`, `no_injection_patterns`, and `no_blocklist_words` are **critical rules**: if one fails, the eval reports `passed: false` no matter how well the other rules scored, and the response names the culprits in `critical_failures`. A leaked SSN can't be averaged away. Custom rules deployed with `severity: "high"` or `"critical"` hard-fail the same way; `low`/`medium` severities only affect the score. One boundary to know: a critical rule that **skipped** (missing context, or any other cause of a skip) has not judged the output and does not veto — `rule_results` shows every skip and its reason, so a gate that must fail closed on non-verdicts can.
219
+ Genuine safety violations hard-fail. `no_pii`, `no_injection_patterns`, and `no_blocklist_words` are **critical rules**: if one fails, the eval reports `passed: false` no matter how well the other rules scored, and the response names the culprits in `critical_failures`. A leaked SSN can't be averaged away. Custom rules deployed with `severity: "high"` or `"critical"` hard-fail the same way; `low`/`medium` severities only affect the score. One boundary to know: a critical rule that **skipped** (missing context, or any other cause of a skip) has not judged the output and does not veto — it is listed in `critical_skipped`, and `rule_results` shows every skip and its reason, so a gate that must fail closed on non-verdicts can.
219
220
 
220
221
  One gotcha for CI gates: if you omit `eval_type`, the default `completeness` bundle runs — **safety rules don't**. The response echoes `eval_type` (plus a `note` when it was defaulted) so your gate can verify which bundle actually ran. Key on `passed` for the verdict and `eval_type: "safety"` for coverage.
221
222
 
223
+ ### Authoring a custom rule
224
+
225
+ Two ways to add a rule. **Inline** rules ride along on one `evaluate_output` call (`custom_rules`, up to 10 per call); they fire alongside whatever `eval_type` bundle you chose, or alone with `eval_type: "custom"`. **Deployed** rules are registered once with `deploy_rule`, persist in `custom-rules.json` under your Iris home, and fire on every future `evaluate_output` of their `evalType`. The definition is the same shape either way:
226
+
227
+ | Field | Required | What it is |
228
+ |---|---|---|
229
+ | `name` | yes | 1–80 characters; appears as `ruleName` in results |
230
+ | `type` | yes | one of `regex_match` · `regex_no_match` · `min_length` · `max_length` · `contains_keywords` · `excludes_keywords` · `json_schema` · `cost_threshold` |
231
+ | `config` | yes | the keys for that type: `pattern` (+ optional `flags`) for the two regex types · `min_length` / `max_length` (a character count) · `keywords` (+ optional `threshold`, 0–1, default `1` = all must appear) for the two keyword types · `{}` for `json_schema` · `max_cost` in USD for `cost_threshold` |
232
+ | `weight` | no | weight in the score; default `1` |
233
+
234
+ `deploy_rule` wraps the definition with `name`, an optional `description`, `evalType` (`completeness` · `relevance` · `safety` · `cost` · `custom`) and `severity`. Severity says what a **failure** means: `low`/`medium` only lower the score; `high`/`critical` hard-fail the evaluation — `passed: false`, the rule named in `critical_failures` — whatever the weighted score says. A rule that skips (a `cost_threshold` rule with no `cost_usd`, or a regex killed at the 100 ms sandbox budget) has not judged the output and is listed in `critical_skipped` instead. Deploy a critical rule that forbids internal hostnames in anything the agent says:
235
+
236
+ ```json
237
+ {
238
+ "name": "no_internal_hostnames",
239
+ "description": "Output must not mention internal hostnames.",
240
+ "evalType": "safety",
241
+ "severity": "critical",
242
+ "definition": {
243
+ "name": "no_internal_hostnames",
244
+ "type": "regex_no_match",
245
+ "config": { "pattern": "\\b[a-z0-9-]+\\.internal\\.example\\b", "flags": "i" }
246
+ }
247
+ }
248
+ ```
249
+
250
+ The response is the persisted rule — keep the `id` for `delete_rule`:
251
+
252
+ ```json
253
+ { "rule": { "id": "rule-588823d0", "name": "no_internal_hostnames", "evalType": "safety", "severity": "critical", "enabled": true, "version": 1, "definition": { "…": "…" } } }
254
+ ```
255
+
256
+ From the very next `evaluate_output` with `eval_type: "safety"`, an output that mentions `db-primary.internal.example` comes back `passed: false` with `critical_failures: ["no_internal_hostnames"]` — even though all five built-in safety rules passed and the weighted score is 0.895. Regex patterns must pass a ReDoS check at deploy time and always run in a sandbox worker under a hard 100 ms deadline. `list_rules` shows what is deployed; the dashboard's rule composer builds the same shape from a failure you clicked on. Full reference, scoring per type, and worked examples: [docs/custom-rules.md](https://github.com/iris-eval/mcp-server/blob/main/docs/custom-rules.md).
257
+
222
258
  Full tool schemas and configuration: [iris-eval.com](https://iris-eval.com)
223
259
 
224
260
  ## Hosted features
@@ -231,19 +267,19 @@ Two commitments hold regardless: **nothing that is free today will move behind a
231
267
 
232
268
  ## Examples
233
269
 
234
- - [Claude Desktop setup](examples/claude-desktop/) — MCP config for stdio and HTTP modes
235
- - [TypeScript — MCP SDK client](examples/typescript/basic-usage.ts) — connect and invoke tools
236
- - [HTTP transport (TS + Python)](examples/http-transport/) — full client code for REST-style integration
237
- - [LangChain instrumentation (Python, conceptual)](examples/langchain/observe-agent.py) — scaffold showing the shape; needs your agent code to be runnable
238
- - [CrewAI instrumentation (Python, conceptual)](examples/crewai/observe-crew.py) — scaffold; same caveat
270
+ - [Claude Desktop setup](https://github.com/iris-eval/mcp-server/tree/main/examples/claude-desktop) — MCP config for stdio and HTTP modes
271
+ - [TypeScript — MCP SDK client](https://github.com/iris-eval/mcp-server/blob/main/examples/typescript/basic-usage.ts) — connect and invoke tools
272
+ - [HTTP transport (TS + Python)](https://github.com/iris-eval/mcp-server/tree/main/examples/http-transport) — full client code for REST-style integration
273
+ - [LangChain instrumentation (Python, conceptual)](https://github.com/iris-eval/mcp-server/blob/main/examples/langchain/observe-agent.py) — scaffold showing the shape; needs your agent code to be runnable
274
+ - [CrewAI instrumentation (Python, conceptual)](https://github.com/iris-eval/mcp-server/blob/main/examples/crewai/observe-crew.py) — scaffold; same caveat
239
275
 
240
276
  ## Community
241
277
 
242
278
  - [GitHub Issues](https://github.com/iris-eval/mcp-server/issues) — Bug reports and feature requests
243
279
  - [GitHub Discussions](https://github.com/iris-eval/mcp-server/discussions) — Questions and ideas
244
- - [Contributing Guide](CONTRIBUTING.md) — How to contribute
245
- - [HTTP Ingest](docs/http-ingest.md) — Deterministic trace capture via `POST /api/v1/traces`
246
- - [Roadmap](docs/roadmap.md) — What's coming next
280
+ - [Contributing Guide](https://github.com/iris-eval/mcp-server/blob/main/CONTRIBUTING.md) — How to contribute
281
+ - [HTTP Ingest](https://github.com/iris-eval/mcp-server/blob/main/docs/http-ingest.md) — Deterministic trace capture via `POST /api/v1/traces`
282
+ - [Roadmap](https://github.com/iris-eval/mcp-server/blob/main/docs/roadmap.md) — What's coming next
247
283
 
248
284
  <details>
249
285
  <summary><strong>Configuration & Security</strong></summary>
@@ -256,37 +292,50 @@ Two commitments hold regardless: **nothing that is free today will move behind a
256
292
  | `--port` | `3000` | HTTP transport port |
257
293
  | `--db-path` | `~/.iris/iris.db` | SQLite database path |
258
294
  | `--config` | `~/.iris/config.json` | Config file path |
259
- | `--api-key` | — | API key for HTTP authentication |
260
- | `--dashboard` | `false` | Enable web dashboard |
295
+ | `--api-key` | — | API key for HTTP authentication (transport and dashboard, including `POST /api/v1/traces`) |
296
+ | `--dashboard` | `false` | Enable web dashboard. Also the only way the `POST /api/v1/traces` ingest endpoint starts — it never starts implicitly with `--transport http` |
261
297
  | `--dashboard-port` | `6920` | Dashboard port |
262
298
  | `--dashboard-host` | `127.0.0.1` | Dashboard bind address. Loopback by default — the dashboard is unauthenticated unless `--api-key` is set, so binding beyond loopback exposes your full trace history |
263
299
  | `--demo` | `false` | Seed a demo database (separate from your real traces) and serve the dashboard against it |
264
300
  | `--demo-clear` | `false` | Delete the demo database and exit |
265
301
  | `--self-test` | `false` | Run the offline install diagnostic in an isolated temp home, then exit (0 = healthy, 1 = a check failed) |
302
+ | `--purge` | `false` | Delete **every** stored trace, span and evaluation from the configured database, compact the file and truncate the write-ahead log so the deleted text does not linger on disk, then exit. Deployed rules, the audit log and preferences are kept. Not reversible. Stop any running Iris server first — the file is compacted in place. Refuses to combine with `--demo`, `--demo-clear` or `--self-test` |
303
+ | `--version` | — | Print the bare version (e.g. `0.5.1`) to stdout and exit 0. Reads nothing under your Iris home |
266
304
 
267
305
  ### Environment Variables
268
306
 
307
+ Every variable `--help` documents. CLI flags take precedence over environment variables when both are set.
308
+
269
309
  | Variable | Description |
270
310
  |----------|-------------|
271
311
  | `IRIS_TRANSPORT` | Transport type (`stdio` or `http`) |
272
- | `IRIS_PORT` | HTTP transport port |
273
- | `IRIS_HOST` | HTTP transport host (default `127.0.0.1`) |
312
+ | `IRIS_HOST` | HTTP transport bind address (default `127.0.0.1`) |
313
+ | `IRIS_PORT` | HTTP transport port (1-65535, default `3000`) |
274
314
  | `IRIS_HOME` | Directory for all per-user files: `config.json`, `iris.db`, `custom-rules.json`, `audit.log`, `preferences.json` (default `~/.iris`) |
275
315
  | `IRIS_DB_PATH` | SQLite database path (overrides `IRIS_HOME` for the DB only) |
276
316
  | `IRIS_LOG_LEVEL` | Log level: `debug`, `info`, `warn`, `error` |
277
- | `IRIS_DASHBOARD` | Enable web dashboard (`true`/`false`; `false` also overrides `dashboard.enabled` in config.json) |
278
- | `IRIS_DASHBOARD_PORT` | Dashboard port (default `6920`) |
317
+ | `IRIS_DASHBOARD` | `true`/`1`/`yes`/`on` enables the web dashboard; `false`/`0`/`no`/`off` disables it (also overrides `dashboard.enabled` in `config.json`) |
318
+ | `IRIS_DASHBOARD_PORT` | Dashboard port (1-65535, default `6920`) |
279
319
  | `IRIS_DASHBOARD_HOST` | Dashboard bind address (default `127.0.0.1`) |
280
320
  | `IRIS_API_KEY` | API key for HTTP authentication |
281
- | `IRIS_ALLOWED_ORIGINS` | Comma-separated allowed CORS origins |
282
-
283
- CLI flags take precedence over environment variables when both are set.
321
+ | `IRIS_ALLOWED_ORIGINS` | Comma-separated origin allowlist. Dashboard: CORS headers (supports globs, e.g. `http://localhost:*`). HTTP transport: exact-match `Origin` allowlist for DNS-rebinding protection (globs ignored; the server's own loopback origins are always allowed) |
322
+ | `IRIS_NO_AUTO_LAUNCH` | Set to `1` to disable the first-run dashboard auto-launch |
323
+ | `IRIS_ANTHROPIC_API_KEY` | Required by `evaluate_with_llm_judge` + `verify_citations` with `provider=anthropic` |
324
+ | `IRIS_OPENAI_API_KEY` | Required by `evaluate_with_llm_judge` + `verify_citations` with `provider=openai` |
325
+ | `IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL` | Hard cost cap per LLM judge call (default `0.25`) |
326
+ | `IRIS_CITATION_ALLOW_FETCH` | Set to `1` to permit outbound HTTP in `verify_citations` (off by default) |
327
+ | `IRIS_CITATION_DOMAINS` | Comma-separated hostname allowlist for `verify_citations` (suffix match) |
328
+ | `IRIS_OTEL_ENDPOINT` | Enable best-effort OTLP/HTTP JSON trace export to this collector URL |
329
+ | `IRIS_OTEL_SERVICE_NAME` | `service.name` resource attribute for OTel export (default `iris-mcp`) |
330
+ | `IRIS_OTEL_HEADERS` | Comma-separated `k=v` headers for OTel export (e.g. `authorization=Bearer abc`) |
331
+ | `IRIS_OTEL_TIMEOUT_MS` | Per-export timeout (default `15000`) |
332
+ | `RATE_LIMIT_SALT` | Website waitlist API only — required when the iris-eval.com site is deployed; the server never reads it |
284
333
 
285
334
  ### Security
286
335
 
287
336
  When using HTTP transport, Iris includes:
288
337
 
289
- - API key authentication with timing-safe comparison
338
+ - API key authentication with timing-safe comparison (Bearer for API clients; browser sign-in to the dashboard via `?key=`)
290
339
  - CORS restricted to localhost by default
291
340
  - Rate limiting (600 req/min dashboard API, 20 req/min MCP)
292
341
  - Helmet security headers
@@ -299,6 +348,12 @@ When using HTTP transport, Iris includes:
299
348
  iris-mcp --transport http --port 3000 --api-key "$(openssl rand -hex 32)" --dashboard
300
349
  ```
301
350
 
351
+ With a key set, API clients — MCP clients, capture SDKs, `POST /api/v1/traces` — send `Authorization: Bearer <key>`. To open the dashboard in a browser, append the key once to any dashboard URL, `http://localhost:6920/?key=<api key>`: Iris exchanges it for an HttpOnly, SameSite=Lax session cookie and redirects to the same page with the key removed from the address bar. A page opened without a session shows a sign-in form that does the same exchange. The key is never stored in the browser, and sessions live only in the server process.
352
+
353
+ ### Your data on disk
354
+
355
+ Everything Iris stores lives under your Iris home (`~/.iris`, or `IRIS_HOME`). `iris.db` keeps every trace's `input` and `output` **verbatim** — including any text `no_pii` goes on to flag; detection does not redact. At startup, traces and evaluations older than `retention.days` (default `30`, `0` disables, set in `config.json`) are deleted and the write-ahead log is checkpointed. To remove everything now, stop the server and run `--purge`: it deletes every stored trace, span and evaluation, compacts the database and truncates the write-ahead log so the text is gone from disk, and keeps your deployed rules, audit log and preferences.
356
+
302
357
  </details>
303
358
 
304
359
  <details>
@@ -326,20 +381,27 @@ Or install globally to avoid cache issues entirely:
326
381
  npm install -g @iris-eval/mcp-server@latest
327
382
  ```
328
383
 
384
+ ### `npm install --ignore-scripts` broke the SQLite binding
385
+
386
+ Iris stores traces with `better-sqlite3`, a native module that fetches or compiles its binding in an install script. If that script was skipped — `--ignore-scripts` on the command line, `ignore-scripts=true` in an `.npmrc` (common on corporate machines), or a registry mirror that strips postinstall — startup fails with a long "Could not locate the bindings file" dump listing a dozen paths it tried. Rebuild that one module:
387
+
388
+ ```bash
389
+ npm rebuild better-sqlite3
390
+ # for a global install:
391
+ npm rebuild -g better-sqlite3
392
+ ```
393
+
329
394
  ### Tools not showing up in Claude Code
330
395
 
331
396
  MCP tools only load at session start. After adding iris-eval, restart the session with `/clear` or relaunch the terminal.
332
397
 
333
398
  ### Version check
334
399
 
335
- Iris logs its version on the first startup line:
336
-
337
400
  ```bash
338
- npx @iris-eval/mcp-server --dashboard
339
- # First log line: "Starting Iris MCP server vX.Y.Z"
401
+ npx @iris-eval/mcp-server --version
340
402
  ```
341
403
 
342
- For a global install, `npm ls -g @iris-eval/mcp-server` shows the installed version.
404
+ The first startup log line also carries it (`Starting Iris MCP server vX.Y.Z`), and `--self-test` prints it in its summary. For a global install, `npm ls -g @iris-eval/mcp-server` shows the installed version.
343
405
 
344
406
  ### Updating
345
407
 
@@ -1,4 +1,14 @@
1
1
  import type { IrisConfig } from '../types/index.js';
2
+ export declare const IRIS_HOME_DIR_MODE = 448;
3
+ /**
4
+ * Create a directory Iris needs, or fail with ONE line that names the path
5
+ * and the permission problem (#371). Before this, an unwritable IRIS_HOME
6
+ * surfaced as a raw EPERM stack trace from deep inside mkdirSync — the
7
+ * first thing a new user saw, with nothing telling them which variable to
8
+ * change. Exported so --self-test probes the configured home through the
9
+ * exact same call the server will make.
10
+ */
11
+ export declare function ensureIrisDirectory(path: string, what: string): void;
2
12
  export interface CliArgs {
3
13
  transport?: string;
4
14
  port?: number;
@@ -2,6 +2,37 @@ import { readFileSync, mkdirSync, existsSync } from 'node:fs';
2
2
  import { join, dirname } from 'node:path';
3
3
  import { defaultConfig } from './defaults.js';
4
4
  import { irisHome } from '../utils/iris-home.js';
5
+ /*
6
+ * Owner-only (0700) for the iris home directory, matching the 0600 the data
7
+ * FILES inside it already get (write-atomic.ts, sqlite-adapter.ts). iris.db
8
+ * holds agent inputs and outputs verbatim — a PII detector necessarily
9
+ * stores the PII it found — and a 0755 directory beside 0600 files still
10
+ * lets every local account list what is there and read anything a library
11
+ * happens to create with the default umask (#372). The mode applies only
12
+ * when THIS process creates the directory; a pre-existing IRIS_HOME keeps
13
+ * whatever permissions its owner chose. No-op on Windows (ACLs govern).
14
+ */
15
+ export const IRIS_HOME_DIR_MODE = 0o700;
16
+ /**
17
+ * Create a directory Iris needs, or fail with ONE line that names the path
18
+ * and the permission problem (#371). Before this, an unwritable IRIS_HOME
19
+ * surfaced as a raw EPERM stack trace from deep inside mkdirSync — the
20
+ * first thing a new user saw, with nothing telling them which variable to
21
+ * change. Exported so --self-test probes the configured home through the
22
+ * exact same call the server will make.
23
+ */
24
+ export function ensureIrisDirectory(path, what) {
25
+ if (existsSync(path))
26
+ return;
27
+ try {
28
+ mkdirSync(path, { recursive: true, mode: IRIS_HOME_DIR_MODE });
29
+ }
30
+ catch (err) {
31
+ const code = err.code ?? (err instanceof Error ? err.message : String(err));
32
+ throw new Error(`Cannot create ${what} "${path}" (${code}). Iris keeps its database, config and rule files there. ` +
33
+ 'Point IRIS_HOME at a directory this user can write, or fix the permissions on that path.');
34
+ }
35
+ }
5
36
  function deepMerge(target, source) {
6
37
  const result = { ...target };
7
38
  for (const key of Object.keys(source)) {
@@ -132,9 +163,7 @@ function cliArgsToConfig(args) {
132
163
  }
133
164
  export function loadConfig(cliArgs) {
134
165
  const home = irisHome();
135
- if (!existsSync(home)) {
136
- mkdirSync(home, { recursive: true });
137
- }
166
+ ensureIrisDirectory(home, 'IRIS_HOME');
138
167
  const configPath = cliArgs?.config ?? join(home, 'config.json');
139
168
  const fileConfig = loadConfigFile(configPath);
140
169
  const envConfig = loadEnvVars();
@@ -142,9 +171,6 @@ export function loadConfig(cliArgs) {
142
171
  let config = deepMerge(defaultConfig, fileConfig);
143
172
  config = deepMerge(config, envConfig);
144
173
  config = deepMerge(config, argsConfig);
145
- const dbDir = dirname(config.storage.path);
146
- if (!existsSync(dbDir)) {
147
- mkdirSync(dbDir, { recursive: true });
148
- }
174
+ ensureIrisDirectory(dirname(config.storage.path), 'the database directory (IRIS_DB_PATH / --db-path)');
149
175
  return config;
150
176
  }