@iris-eval/mcp-server 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -36
- package/dist/config/index.d.ts +10 -0
- package/dist/config/index.js +33 -7
- package/dist/dashboard/assets/index-CshLgDRB.js +10 -0
- package/dist/dashboard/assets/{index-UffZ-aEJ.css → index-D0cFfBqn.css} +1 -1
- package/dist/dashboard/index.html +4 -3
- package/dist/dashboard/routes/health.js +10 -3
- package/dist/dashboard/routes/moments.js +1 -1
- package/dist/dashboard/routes/preferences.d.ts +1 -0
- package/dist/dashboard/routes/preferences.js +31 -3
- package/dist/dashboard/routes/rules.d.ts +18 -0
- package/dist/dashboard/routes/rules.js +160 -6
- package/dist/dashboard/routes/traces.js +30 -3
- package/dist/dashboard/seed-demo-data.js +14 -3
- package/dist/dashboard/server.js +13 -3
- package/dist/dashboard/session-auth.d.ts +8 -0
- package/dist/dashboard/session-auth.js +237 -0
- package/dist/dashboard/validation.d.ts +9 -3
- package/dist/dashboard/validation.js +69 -11
- package/dist/eval/citation-verify/verifier.d.ts +17 -0
- package/dist/eval/citation-verify/verifier.js +68 -15
- package/dist/eval/decision-moment.js +17 -9
- package/dist/eval/engine.d.ts +62 -0
- package/dist/eval/engine.js +196 -58
- package/dist/eval/llm-judge/evaluator.js +50 -33
- package/dist/eval/llm-judge/templates/index.d.ts +4 -0
- package/dist/eval/llm-judge/templates/index.js +10 -4
- package/dist/eval/rules/custom.js +59 -6
- package/dist/eval/rules/relevance.js +1 -1
- package/dist/eval/rules/safety.d.ts +8 -0
- package/dist/eval/rules/safety.js +63 -18
- package/dist/index.js +102 -16
- package/dist/middleware/rate-limit.d.ts +25 -0
- package/dist/middleware/rate-limit.js +54 -2
- package/dist/self-test.d.ts +14 -0
- package/dist/self-test.js +97 -13
- package/dist/storage/demo-guard.d.ts +8 -0
- package/dist/storage/demo-guard.js +53 -0
- package/dist/storage/migrations/006-eval-critical-failures.d.ts +3 -0
- package/dist/storage/migrations/006-eval-critical-failures.js +23 -0
- package/dist/storage/migrations/index.js +2 -0
- package/dist/storage/sqlite-adapter.d.ts +6 -0
- package/dist/storage/sqlite-adapter.js +91 -4
- package/dist/tools/delete-rule.js +49 -11
- package/dist/tools/deploy-rule.d.ts +33 -0
- package/dist/tools/deploy-rule.js +130 -27
- package/dist/tools/evaluate-output.js +50 -24
- package/dist/tools/evaluate-with-llm-judge.js +11 -4
- package/dist/tools/get-traces.d.ts +27 -0
- package/dist/tools/get-traces.js +60 -8
- package/dist/tools/list-rules.js +2 -2
- package/dist/tools/log-trace.js +5 -4
- package/dist/tools/strict-input.d.ts +1 -0
- package/dist/tools/strict-input.js +25 -0
- package/dist/tools/trace-link.d.ts +7 -0
- package/dist/tools/trace-link.js +39 -0
- package/dist/tools/verify-citations.d.ts +19 -0
- package/dist/tools/verify-citations.js +42 -5
- package/dist/types/decision-moment.d.ts +8 -0
- package/dist/types/eval.d.ts +60 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/query.d.ts +25 -0
- package/package.json +1 -1
- package/server.json +2 -2
- package/dist/dashboard/assets/index-BZZt8bVh.js +0 -10
package/README.md
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
[](https://github.com/iris-eval/mcp-server/actions/workflows/ci.yml)
|
|
9
9
|
[](https://securityscorecards.dev/viewer/?uri=github.com/iris-eval/mcp-server)
|
|
10
10
|
[](https://www.bestpractices.dev/projects/12849)
|
|
11
|
-
[](LICENSE)
|
|
11
|
+
[](https://github.com/iris-eval/mcp-server/blob/main/LICENSE)
|
|
12
12
|
[](https://github.com/iris-eval/mcp-server/pkgs/container/mcp-server)
|
|
13
13
|
[](https://www.pulsemcp.com/servers/iris-eval)
|
|
14
14
|
[](https://mcp.so/server/iris/iris-eval)
|
|
@@ -56,11 +56,11 @@ Your agent discovers Iris's nine tools on connect, and the dashboard serves at *
|
|
|
56
56
|
|
|
57
57
|
The trace lands on the dashboard with its scores. Prefer the MCP server headless? Drop `--dashboard` from the args — you can open the same dashboard any time with `npx @iris-eval/mcp-server --dashboard`.
|
|
58
58
|
|
|
59
|
-
**One thing worth knowing up front:** MCP tools are called when the model decides to call them. Iris doesn't intercept your agent, so traces are logged when your agent asks it to log them — either because you told it to, or because your code calls the tools directly. Ask your agent to "log this to Iris and evaluate it" and it will. If you want capture that doesn't depend on the model choosing, `POST /api/v1/traces` does exactly that — your code sends the trace over plain HTTP, no model in the loop (see [docs/http-ingest.md](docs/http-ingest.md)). The CLI and SDKs on the [roadmap](docs/roadmap.md) will be thin clients over the same endpoint.
|
|
59
|
+
**One thing worth knowing up front:** MCP tools are called when the model decides to call them. Iris doesn't intercept your agent, so traces are logged when your agent asks it to log them — either because you told it to, or because your code calls the tools directly. Ask your agent to "log this to Iris and evaluate it" and it will. If you want capture that doesn't depend on the model choosing, `POST /api/v1/traces` does exactly that — your code sends the trace over plain HTTP, no model in the loop (see [docs/http-ingest.md](https://github.com/iris-eval/mcp-server/blob/main/docs/http-ingest.md)). The CLI and SDKs on the [roadmap](https://github.com/iris-eval/mcp-server/blob/main/docs/roadmap.md) will be thin clients over the same endpoint.
|
|
60
60
|
|
|
61
61
|
### Capture over HTTP (no model in the loop)
|
|
62
62
|
|
|
63
|
-
|
|
63
|
+
The ingest endpoint lives on the **dashboard port** — `6920` by default, not the MCP transport port — and it exists only while the dashboard is running. Pass `--dashboard` (or set `IRIS_DASHBOARD=true`); `--transport http` on its own does **not** start it, and a request to the transport port returns `404`. With the dashboard up, anything that can send an HTTP request can log a trace — and optionally run the deterministic evals in the same request:
|
|
64
64
|
|
|
65
65
|
```bash
|
|
66
66
|
curl -s -X POST "http://127.0.0.1:6920/api/v1/traces" \
|
|
@@ -74,15 +74,16 @@ curl -s -X POST "http://127.0.0.1:6920/api/v1/traces" \
|
|
|
74
74
|
}'
|
|
75
75
|
```
|
|
76
76
|
|
|
77
|
-
Returns `201` with the stored `trace_id` and the evaluation result. The endpoint accepts the same body as the `log_trace` tool and sits behind the same
|
|
77
|
+
Returns `201` with the stored `trace_id` and the evaluation result (in `--demo` mode the endpoint refuses writes with `403`, so demo data never mixes with yours). The endpoint accepts the same body as the `log_trace` tool and sits behind the same middleware stack as the rest of the dashboard: loopback bind and the DNS-rebinding guard by default, plus Bearer auth when you set one. **Two plain facts about it:** it accepts unauthenticated writes unless Iris was started with `--api-key` (or `IRIS_API_KEY`) — the loopback bind is what keeps it to your machine by default, so set a key before binding beyond loopback; and what it stores is verbatim — `input` and `output` land in `iris.db` exactly as sent, including any text `no_pii` goes on to flag. Full contract, field reference, and error semantics: [docs/http-ingest.md](https://github.com/iris-eval/mcp-server/blob/main/docs/http-ingest.md).
|
|
78
78
|
|
|
79
|
-
###
|
|
79
|
+
### Verify your install
|
|
80
80
|
|
|
81
81
|
```bash
|
|
82
|
-
npx @iris-eval/mcp-server --self-test
|
|
82
|
+
npx @iris-eval/mcp-server --self-test # offline diagnostic; exit 0 = healthy, 1 = a check failed
|
|
83
|
+
npx @iris-eval/mcp-server --version # prints the bare version, e.g. 0.5.1
|
|
83
84
|
```
|
|
84
85
|
|
|
85
|
-
|
|
86
|
+
`--self-test` first creates your Iris home if it is missing and checks that it is writable (exit 1, naming the path, if it is not), then runs its checks — storage round-trip, a planted SSN and a planted injection caught by the safety rules, dashboard boot, the DNS-rebinding guard — inside an isolated temp home, so your real database is never opened. Everything Iris writes lives under one directory, your **Iris home**: `~/.iris` by default (`%USERPROFILE%\.iris` on Windows), or wherever `IRIS_HOME` points. That is where `iris.db`, `config.json`, `custom-rules.json`, `audit.log`, `preferences.json` and the demo files live; point `IRIS_HOME` at a scratch directory to try Iris without touching your real data.
|
|
86
87
|
|
|
87
88
|
<details>
|
|
88
89
|
<summary><strong>Setup by tool</strong></summary>
|
|
@@ -172,8 +173,9 @@ Iris is a standard stdio MCP server — one `npx @iris-eval/mcp-server` command,
|
|
|
172
173
|
npm install -g @iris-eval/mcp-server
|
|
173
174
|
iris-mcp --dashboard
|
|
174
175
|
|
|
175
|
-
# Docker
|
|
176
|
-
|
|
176
|
+
# Docker — two servers, two ports: 3000 = MCP HTTP transport,
|
|
177
|
+
# 6920 = dashboard (which also serves the POST /api/v1/traces ingest endpoint)
|
|
178
|
+
docker run -p 3000:3000 -p 6920:6920 -v iris-data:/data ghcr.io/iris-eval/mcp-server
|
|
177
179
|
```
|
|
178
180
|
|
|
179
181
|
> **Tip:** Global install (`npm install -g`) stores traces persistently at `~/.iris/iris.db`. With `npx`, traces persist in the same location, but startup is slower due to package resolution.
|
|
@@ -189,7 +191,7 @@ docker run -p 3000:3000 -v iris-data:/data ghcr.io/iris-eval/mcp-server
|
|
|
189
191
|
| **Web Dashboard** | Real-time dark-mode UI that lands on the failures, worst and newest first — trace visualization, eval results, cost breakdowns, and a command palette (⌘K) that searches your own rules, traces, and evals. |
|
|
190
192
|
| **Local-first** | Everything lives in SQLite on your disk. No account, no sign-up, no telemetry. Outbound HTTP happens only where you opt in: your own LLM-judge key, citation fetching, or an OTel exporter you configure. |
|
|
191
193
|
|
|
192
|
-
Where this is going next: [the roadmap](docs/roadmap.md).
|
|
194
|
+
Where this is going next: [the roadmap](https://github.com/iris-eval/mcp-server/blob/main/docs/roadmap.md).
|
|
193
195
|
|
|
194
196
|
## MCP Tools
|
|
195
197
|
|
|
@@ -205,7 +207,7 @@ Iris registers nine tools that any MCP-compatible agent can invoke — full rule
|
|
|
205
207
|
- **`evaluate_with_llm_judge`** — Semantic eval via LLM (Anthropic or OpenAI). Five templates: accuracy, helpfulness, safety, correctness, faithfulness. Cost-capped, per-eval pricing disclosed. **Bring your own API key** (`IRIS_ANTHROPIC_API_KEY` or `IRIS_OPENAI_API_KEY`) — Iris doesn't proxy or relay LLM calls.
|
|
206
208
|
- **`verify_citations`** — Extract citations from output (numbered, author-year, URLs, DOIs), fetch sources behind an SSRF-guarded + domain-allowlisted resolver, and use an LLM judge to check whether each source actually supports the cited claim. Opt-in outbound HTTP. Same BYOK requirement as `evaluate_with_llm_judge`.
|
|
207
209
|
|
|
208
|
-
When `IRIS_OTEL_ENDPOINT` is configured, `log_trace` calls also emit a best-effort OTLP/HTTP JSON export to any OpenTelemetry collector (Jaeger, Grafana Tempo, Datadog OTLP, Honeycomb, etc). See [docs/otel-integration.md](docs/otel-integration.md).
|
|
210
|
+
When `IRIS_OTEL_ENDPOINT` is configured, `log_trace` calls also emit a best-effort OTLP/HTTP JSON export to any OpenTelemetry collector (Jaeger, Grafana Tempo, Datadog OTLP, Honeycomb, etc). See [docs/otel-integration.md](https://github.com/iris-eval/mcp-server/blob/main/docs/otel-integration.md).
|
|
209
211
|
|
|
210
212
|
### How `passed` is decided
|
|
211
213
|
|
|
@@ -214,10 +216,45 @@ When `IRIS_OTEL_ENDPOINT` is configured, `log_trace` calls also emit a best-effo
|
|
|
214
216
|
- **`score`** (0..1) is the weighted average across the rules that ran — a quality gradient.
|
|
215
217
|
- **`passed`** is the ship/no-ship verdict: `true` only when the score clears the pass threshold (default **0.7**) **and no critical rule failed**.
|
|
216
218
|
|
|
217
|
-
Genuine safety violations hard-fail. `no_pii`, `no_injection_patterns`, and `no_blocklist_words` are **critical rules**: if one fails, the eval reports `passed: false` no matter how well the other rules scored, and the response names the culprits in `critical_failures`. A leaked SSN can't be averaged away. Custom rules deployed with `severity: "high"` or `"critical"` hard-fail the same way; `low`/`medium` severities only affect the score. One boundary to know: a critical rule that **skipped** (missing context, or any other cause of a skip) has not judged the output and does not veto — `rule_results` shows every skip and its reason, so a gate that must fail closed on non-verdicts can.
|
|
219
|
+
Genuine safety violations hard-fail. `no_pii`, `no_injection_patterns`, and `no_blocklist_words` are **critical rules**: if one fails, the eval reports `passed: false` no matter how well the other rules scored, and the response names the culprits in `critical_failures`. A leaked SSN can't be averaged away. Custom rules deployed with `severity: "high"` or `"critical"` hard-fail the same way; `low`/`medium` severities only affect the score. One boundary to know: a critical rule that **skipped** (missing context, or any other cause of a skip) has not judged the output and does not veto — it is listed in `critical_skipped`, and `rule_results` shows every skip and its reason, so a gate that must fail closed on non-verdicts can.
|
|
218
220
|
|
|
219
221
|
One gotcha for CI gates: if you omit `eval_type`, the default `completeness` bundle runs — **safety rules don't**. The response echoes `eval_type` (plus a `note` when it was defaulted) so your gate can verify which bundle actually ran. Key on `passed` for the verdict and `eval_type: "safety"` for coverage.
|
|
220
222
|
|
|
223
|
+
### Authoring a custom rule
|
|
224
|
+
|
|
225
|
+
Two ways to add a rule. **Inline** rules ride along on one `evaluate_output` call (`custom_rules`, up to 10 per call); they fire alongside whatever `eval_type` bundle you chose, or alone with `eval_type: "custom"`. **Deployed** rules are registered once with `deploy_rule`, persist in `custom-rules.json` under your Iris home, and fire on every future `evaluate_output` of their `evalType`. The definition is the same shape either way:
|
|
226
|
+
|
|
227
|
+
| Field | Required | What it is |
|
|
228
|
+
|---|---|---|
|
|
229
|
+
| `name` | yes | 1–80 characters; appears as `ruleName` in results |
|
|
230
|
+
| `type` | yes | one of `regex_match` · `regex_no_match` · `min_length` · `max_length` · `contains_keywords` · `excludes_keywords` · `json_schema` · `cost_threshold` |
|
|
231
|
+
| `config` | yes | the keys for that type: `pattern` (+ optional `flags`) for the two regex types · `min_length` / `max_length` (a character count) · `keywords` (+ optional `threshold`, 0–1, default `1` = all must appear) for the two keyword types · `{}` for `json_schema` · `max_cost` in USD for `cost_threshold` |
|
|
232
|
+
| `weight` | no | weight in the score; default `1` |
|
|
233
|
+
|
|
234
|
+
`deploy_rule` wraps the definition with `name`, an optional `description`, `evalType` (`completeness` · `relevance` · `safety` · `cost` · `custom`) and `severity`. Severity says what a **failure** means: `low`/`medium` only lower the score; `high`/`critical` hard-fail the evaluation — `passed: false`, the rule named in `critical_failures` — whatever the weighted score says. A rule that skips (a `cost_threshold` rule with no `cost_usd`, or a regex killed at the 100 ms sandbox budget) has not judged the output and is listed in `critical_skipped` instead. Deploy a critical rule that forbids internal hostnames in anything the agent says:
|
|
235
|
+
|
|
236
|
+
```json
|
|
237
|
+
{
|
|
238
|
+
"name": "no_internal_hostnames",
|
|
239
|
+
"description": "Output must not mention internal hostnames.",
|
|
240
|
+
"evalType": "safety",
|
|
241
|
+
"severity": "critical",
|
|
242
|
+
"definition": {
|
|
243
|
+
"name": "no_internal_hostnames",
|
|
244
|
+
"type": "regex_no_match",
|
|
245
|
+
"config": { "pattern": "\\b[a-z0-9-]+\\.internal\\.example\\b", "flags": "i" }
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
The response is the persisted rule — keep the `id` for `delete_rule`:
|
|
251
|
+
|
|
252
|
+
```json
|
|
253
|
+
{ "rule": { "id": "rule-588823d0", "name": "no_internal_hostnames", "evalType": "safety", "severity": "critical", "enabled": true, "version": 1, "definition": { "…": "…" } } }
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
From the very next `evaluate_output` with `eval_type: "safety"`, an output that mentions `db-primary.internal.example` comes back `passed: false` with `critical_failures: ["no_internal_hostnames"]` — even though all five built-in safety rules passed and the weighted score is 0.895. Regex patterns must pass a ReDoS check at deploy time and always run in a sandbox worker under a hard 100 ms deadline. `list_rules` shows what is deployed; the dashboard's rule composer builds the same shape from a failure you clicked on. Full reference, scoring per type, and worked examples: [docs/custom-rules.md](https://github.com/iris-eval/mcp-server/blob/main/docs/custom-rules.md).
|
|
257
|
+
|
|
221
258
|
Full tool schemas and configuration: [iris-eval.com](https://iris-eval.com)
|
|
222
259
|
|
|
223
260
|
## Hosted features
|
|
@@ -230,19 +267,19 @@ Two commitments hold regardless: **nothing that is free today will move behind a
|
|
|
230
267
|
|
|
231
268
|
## Examples
|
|
232
269
|
|
|
233
|
-
- [Claude Desktop setup](examples/claude-desktop
|
|
234
|
-
- [TypeScript — MCP SDK client](examples/typescript/basic-usage.ts) — connect and invoke tools
|
|
235
|
-
- [HTTP transport (TS + Python)](examples/http-transport
|
|
236
|
-
- [LangChain instrumentation (Python, conceptual)](examples/langchain/observe-agent.py) — scaffold showing the shape; needs your agent code to be runnable
|
|
237
|
-
- [CrewAI instrumentation (Python, conceptual)](examples/crewai/observe-crew.py) — scaffold; same caveat
|
|
270
|
+
- [Claude Desktop setup](https://github.com/iris-eval/mcp-server/tree/main/examples/claude-desktop) — MCP config for stdio and HTTP modes
|
|
271
|
+
- [TypeScript — MCP SDK client](https://github.com/iris-eval/mcp-server/blob/main/examples/typescript/basic-usage.ts) — connect and invoke tools
|
|
272
|
+
- [HTTP transport (TS + Python)](https://github.com/iris-eval/mcp-server/tree/main/examples/http-transport) — full client code for REST-style integration
|
|
273
|
+
- [LangChain instrumentation (Python, conceptual)](https://github.com/iris-eval/mcp-server/blob/main/examples/langchain/observe-agent.py) — scaffold showing the shape; needs your agent code to be runnable
|
|
274
|
+
- [CrewAI instrumentation (Python, conceptual)](https://github.com/iris-eval/mcp-server/blob/main/examples/crewai/observe-crew.py) — scaffold; same caveat
|
|
238
275
|
|
|
239
276
|
## Community
|
|
240
277
|
|
|
241
278
|
- [GitHub Issues](https://github.com/iris-eval/mcp-server/issues) — Bug reports and feature requests
|
|
242
279
|
- [GitHub Discussions](https://github.com/iris-eval/mcp-server/discussions) — Questions and ideas
|
|
243
|
-
- [Contributing Guide](CONTRIBUTING.md) — How to contribute
|
|
244
|
-
- [HTTP Ingest](docs/http-ingest.md) — Deterministic trace capture via `POST /api/v1/traces`
|
|
245
|
-
- [Roadmap](docs/roadmap.md) — What's coming next
|
|
280
|
+
- [Contributing Guide](https://github.com/iris-eval/mcp-server/blob/main/CONTRIBUTING.md) — How to contribute
|
|
281
|
+
- [HTTP Ingest](https://github.com/iris-eval/mcp-server/blob/main/docs/http-ingest.md) — Deterministic trace capture via `POST /api/v1/traces`
|
|
282
|
+
- [Roadmap](https://github.com/iris-eval/mcp-server/blob/main/docs/roadmap.md) — What's coming next
|
|
246
283
|
|
|
247
284
|
<details>
|
|
248
285
|
<summary><strong>Configuration & Security</strong></summary>
|
|
@@ -255,39 +292,52 @@ Two commitments hold regardless: **nothing that is free today will move behind a
|
|
|
255
292
|
| `--port` | `3000` | HTTP transport port |
|
|
256
293
|
| `--db-path` | `~/.iris/iris.db` | SQLite database path |
|
|
257
294
|
| `--config` | `~/.iris/config.json` | Config file path |
|
|
258
|
-
| `--api-key` | — | API key for HTTP authentication |
|
|
259
|
-
| `--dashboard` | `false` | Enable web dashboard |
|
|
295
|
+
| `--api-key` | — | API key for HTTP authentication (transport and dashboard, including `POST /api/v1/traces`) |
|
|
296
|
+
| `--dashboard` | `false` | Enable web dashboard. Also the only way the `POST /api/v1/traces` ingest endpoint starts — it never starts implicitly with `--transport http` |
|
|
260
297
|
| `--dashboard-port` | `6920` | Dashboard port |
|
|
261
298
|
| `--dashboard-host` | `127.0.0.1` | Dashboard bind address. Loopback by default — the dashboard is unauthenticated unless `--api-key` is set, so binding beyond loopback exposes your full trace history |
|
|
262
299
|
| `--demo` | `false` | Seed a demo database (separate from your real traces) and serve the dashboard against it |
|
|
263
300
|
| `--demo-clear` | `false` | Delete the demo database and exit |
|
|
264
301
|
| `--self-test` | `false` | Run the offline install diagnostic in an isolated temp home, then exit (0 = healthy, 1 = a check failed) |
|
|
302
|
+
| `--purge` | `false` | Delete **every** stored trace, span and evaluation from the configured database, compact the file and truncate the write-ahead log so the deleted text does not linger on disk, then exit. Deployed rules, the audit log and preferences are kept. Not reversible. Stop any running Iris server first — the file is compacted in place. Refuses to combine with `--demo`, `--demo-clear` or `--self-test` |
|
|
303
|
+
| `--version` | — | Print the bare version (e.g. `0.5.1`) to stdout and exit 0. Reads nothing under your Iris home |
|
|
265
304
|
|
|
266
305
|
### Environment Variables
|
|
267
306
|
|
|
307
|
+
Every variable `--help` documents. CLI flags take precedence over environment variables when both are set.
|
|
308
|
+
|
|
268
309
|
| Variable | Description |
|
|
269
310
|
|----------|-------------|
|
|
270
311
|
| `IRIS_TRANSPORT` | Transport type (`stdio` or `http`) |
|
|
271
|
-
| `
|
|
272
|
-
| `
|
|
312
|
+
| `IRIS_HOST` | HTTP transport bind address (default `127.0.0.1`) |
|
|
313
|
+
| `IRIS_PORT` | HTTP transport port (1-65535, default `3000`) |
|
|
273
314
|
| `IRIS_HOME` | Directory for all per-user files: `config.json`, `iris.db`, `custom-rules.json`, `audit.log`, `preferences.json` (default `~/.iris`) |
|
|
274
315
|
| `IRIS_DB_PATH` | SQLite database path (overrides `IRIS_HOME` for the DB only) |
|
|
275
316
|
| `IRIS_LOG_LEVEL` | Log level: `debug`, `info`, `warn`, `error` |
|
|
276
|
-
| `IRIS_DASHBOARD` |
|
|
277
|
-
| `IRIS_DASHBOARD_PORT` | Dashboard port (default `6920`) |
|
|
317
|
+
| `IRIS_DASHBOARD` | `true`/`1`/`yes`/`on` enables the web dashboard; `false`/`0`/`no`/`off` disables it (also overrides `dashboard.enabled` in `config.json`) |
|
|
318
|
+
| `IRIS_DASHBOARD_PORT` | Dashboard port (1-65535, default `6920`) |
|
|
278
319
|
| `IRIS_DASHBOARD_HOST` | Dashboard bind address (default `127.0.0.1`) |
|
|
279
320
|
| `IRIS_API_KEY` | API key for HTTP authentication |
|
|
280
|
-
| `IRIS_ALLOWED_ORIGINS` | Comma-separated
|
|
281
|
-
|
|
282
|
-
|
|
321
|
+
| `IRIS_ALLOWED_ORIGINS` | Comma-separated origin allowlist. Dashboard: CORS headers (supports globs, e.g. `http://localhost:*`). HTTP transport: exact-match `Origin` allowlist for DNS-rebinding protection (globs ignored; the server's own loopback origins are always allowed) |
|
|
322
|
+
| `IRIS_NO_AUTO_LAUNCH` | Set to `1` to disable the first-run dashboard auto-launch |
|
|
323
|
+
| `IRIS_ANTHROPIC_API_KEY` | Required by `evaluate_with_llm_judge` + `verify_citations` with `provider=anthropic` |
|
|
324
|
+
| `IRIS_OPENAI_API_KEY` | Required by `evaluate_with_llm_judge` + `verify_citations` with `provider=openai` |
|
|
325
|
+
| `IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL` | Hard cost cap per LLM judge call (default `0.25`) |
|
|
326
|
+
| `IRIS_CITATION_ALLOW_FETCH` | Set to `1` to permit outbound HTTP in `verify_citations` (off by default) |
|
|
327
|
+
| `IRIS_CITATION_DOMAINS` | Comma-separated hostname allowlist for `verify_citations` (suffix match) |
|
|
328
|
+
| `IRIS_OTEL_ENDPOINT` | Enable best-effort OTLP/HTTP JSON trace export to this collector URL |
|
|
329
|
+
| `IRIS_OTEL_SERVICE_NAME` | `service.name` resource attribute for OTel export (default `iris-mcp`) |
|
|
330
|
+
| `IRIS_OTEL_HEADERS` | Comma-separated `k=v` headers for OTel export (e.g. `authorization=Bearer abc`) |
|
|
331
|
+
| `IRIS_OTEL_TIMEOUT_MS` | Per-export timeout (default `15000`) |
|
|
332
|
+
| `RATE_LIMIT_SALT` | Website waitlist API only — required when the iris-eval.com site is deployed; the server never reads it |
|
|
283
333
|
|
|
284
334
|
### Security
|
|
285
335
|
|
|
286
336
|
When using HTTP transport, Iris includes:
|
|
287
337
|
|
|
288
|
-
- API key authentication with timing-safe comparison
|
|
338
|
+
- API key authentication with timing-safe comparison (Bearer for API clients; browser sign-in to the dashboard via `?key=`)
|
|
289
339
|
- CORS restricted to localhost by default
|
|
290
|
-
- Rate limiting (
|
|
340
|
+
- Rate limiting (600 req/min dashboard API, 20 req/min MCP)
|
|
291
341
|
- Helmet security headers
|
|
292
342
|
- Zod input validation on all routes
|
|
293
343
|
- ReDoS-safe regex for custom eval rules
|
|
@@ -298,6 +348,12 @@ When using HTTP transport, Iris includes:
|
|
|
298
348
|
iris-mcp --transport http --port 3000 --api-key "$(openssl rand -hex 32)" --dashboard
|
|
299
349
|
```
|
|
300
350
|
|
|
351
|
+
With a key set, API clients — MCP clients, capture SDKs, `POST /api/v1/traces` — send `Authorization: Bearer <key>`. To open the dashboard in a browser, append the key once to any dashboard URL, `http://localhost:6920/?key=<api key>`: Iris exchanges it for an HttpOnly, SameSite=Lax session cookie and redirects to the same page with the key removed from the address bar. A page opened without a session shows a sign-in form that does the same exchange. The key is never stored in the browser, and sessions live only in the server process.
|
|
352
|
+
|
|
353
|
+
### Your data on disk
|
|
354
|
+
|
|
355
|
+
Everything Iris stores lives under your Iris home (`~/.iris`, or `IRIS_HOME`). `iris.db` keeps every trace's `input` and `output` **verbatim** — including any text `no_pii` goes on to flag; detection does not redact. At startup, traces and evaluations older than `retention.days` (default `30`, `0` disables, set in `config.json`) are deleted and the write-ahead log is checkpointed. To remove everything now, stop the server and run `--purge`: it deletes every stored trace, span and evaluation, compacts the database and truncates the write-ahead log so the text is gone from disk, and keeps your deployed rules, audit log and preferences.
|
|
356
|
+
|
|
301
357
|
</details>
|
|
302
358
|
|
|
303
359
|
<details>
|
|
@@ -325,20 +381,27 @@ Or install globally to avoid cache issues entirely:
|
|
|
325
381
|
npm install -g @iris-eval/mcp-server@latest
|
|
326
382
|
```
|
|
327
383
|
|
|
384
|
+
### `npm install --ignore-scripts` broke the SQLite binding
|
|
385
|
+
|
|
386
|
+
Iris stores traces with `better-sqlite3`, a native module that fetches or compiles its binding in an install script. If that script was skipped — `--ignore-scripts` on the command line, `ignore-scripts=true` in an `.npmrc` (common on corporate machines), or a registry mirror that strips postinstall — startup fails with a long "Could not locate the bindings file" dump listing a dozen paths it tried. Rebuild that one module:
|
|
387
|
+
|
|
388
|
+
```bash
|
|
389
|
+
npm rebuild better-sqlite3
|
|
390
|
+
# for a global install:
|
|
391
|
+
npm rebuild -g better-sqlite3
|
|
392
|
+
```
|
|
393
|
+
|
|
328
394
|
### Tools not showing up in Claude Code
|
|
329
395
|
|
|
330
396
|
MCP tools only load at session start. After adding iris-eval, restart the session with `/clear` or relaunch the terminal.
|
|
331
397
|
|
|
332
398
|
### Version check
|
|
333
399
|
|
|
334
|
-
Iris logs its version on the first startup line:
|
|
335
|
-
|
|
336
400
|
```bash
|
|
337
|
-
npx @iris-eval/mcp-server --
|
|
338
|
-
# First log line: "Starting Iris MCP server vX.Y.Z"
|
|
401
|
+
npx @iris-eval/mcp-server --version
|
|
339
402
|
```
|
|
340
403
|
|
|
341
|
-
For a global install, `npm ls -g @iris-eval/mcp-server` shows the installed version.
|
|
404
|
+
The first startup log line also carries it (`Starting Iris MCP server vX.Y.Z`), and `--self-test` prints it in its summary. For a global install, `npm ls -g @iris-eval/mcp-server` shows the installed version.
|
|
342
405
|
|
|
343
406
|
### Updating
|
|
344
407
|
|
package/dist/config/index.d.ts
CHANGED
|
@@ -1,4 +1,14 @@
|
|
|
1
1
|
import type { IrisConfig } from '../types/index.js';
|
|
2
|
+
export declare const IRIS_HOME_DIR_MODE = 448;
|
|
3
|
+
/**
|
|
4
|
+
* Create a directory Iris needs, or fail with ONE line that names the path
|
|
5
|
+
* and the permission problem (#371). Before this, an unwritable IRIS_HOME
|
|
6
|
+
* surfaced as a raw EPERM stack trace from deep inside mkdirSync — the
|
|
7
|
+
* first thing a new user saw, with nothing telling them which variable to
|
|
8
|
+
* change. Exported so --self-test probes the configured home through the
|
|
9
|
+
* exact same call the server will make.
|
|
10
|
+
*/
|
|
11
|
+
export declare function ensureIrisDirectory(path: string, what: string): void;
|
|
2
12
|
export interface CliArgs {
|
|
3
13
|
transport?: string;
|
|
4
14
|
port?: number;
|
package/dist/config/index.js
CHANGED
|
@@ -2,6 +2,37 @@ import { readFileSync, mkdirSync, existsSync } from 'node:fs';
|
|
|
2
2
|
import { join, dirname } from 'node:path';
|
|
3
3
|
import { defaultConfig } from './defaults.js';
|
|
4
4
|
import { irisHome } from '../utils/iris-home.js';
|
|
5
|
+
/*
|
|
6
|
+
* Owner-only (0700) for the iris home directory, matching the 0600 the data
|
|
7
|
+
* FILES inside it already get (write-atomic.ts, sqlite-adapter.ts). iris.db
|
|
8
|
+
* holds agent inputs and outputs verbatim — a PII detector necessarily
|
|
9
|
+
* stores the PII it found — and a 0755 directory beside 0600 files still
|
|
10
|
+
* lets every local account list what is there and read anything a library
|
|
11
|
+
* happens to create with the default umask (#372). The mode applies only
|
|
12
|
+
* when THIS process creates the directory; a pre-existing IRIS_HOME keeps
|
|
13
|
+
* whatever permissions its owner chose. No-op on Windows (ACLs govern).
|
|
14
|
+
*/
|
|
15
|
+
export const IRIS_HOME_DIR_MODE = 0o700;
|
|
16
|
+
/**
|
|
17
|
+
* Create a directory Iris needs, or fail with ONE line that names the path
|
|
18
|
+
* and the permission problem (#371). Before this, an unwritable IRIS_HOME
|
|
19
|
+
* surfaced as a raw EPERM stack trace from deep inside mkdirSync — the
|
|
20
|
+
* first thing a new user saw, with nothing telling them which variable to
|
|
21
|
+
* change. Exported so --self-test probes the configured home through the
|
|
22
|
+
* exact same call the server will make.
|
|
23
|
+
*/
|
|
24
|
+
export function ensureIrisDirectory(path, what) {
|
|
25
|
+
if (existsSync(path))
|
|
26
|
+
return;
|
|
27
|
+
try {
|
|
28
|
+
mkdirSync(path, { recursive: true, mode: IRIS_HOME_DIR_MODE });
|
|
29
|
+
}
|
|
30
|
+
catch (err) {
|
|
31
|
+
const code = err.code ?? (err instanceof Error ? err.message : String(err));
|
|
32
|
+
throw new Error(`Cannot create ${what} "${path}" (${code}). Iris keeps its database, config and rule files there. ` +
|
|
33
|
+
'Point IRIS_HOME at a directory this user can write, or fix the permissions on that path.');
|
|
34
|
+
}
|
|
35
|
+
}
|
|
5
36
|
function deepMerge(target, source) {
|
|
6
37
|
const result = { ...target };
|
|
7
38
|
for (const key of Object.keys(source)) {
|
|
@@ -132,9 +163,7 @@ function cliArgsToConfig(args) {
|
|
|
132
163
|
}
|
|
133
164
|
export function loadConfig(cliArgs) {
|
|
134
165
|
const home = irisHome();
|
|
135
|
-
|
|
136
|
-
mkdirSync(home, { recursive: true });
|
|
137
|
-
}
|
|
166
|
+
ensureIrisDirectory(home, 'IRIS_HOME');
|
|
138
167
|
const configPath = cliArgs?.config ?? join(home, 'config.json');
|
|
139
168
|
const fileConfig = loadConfigFile(configPath);
|
|
140
169
|
const envConfig = loadEnvVars();
|
|
@@ -142,9 +171,6 @@ export function loadConfig(cliArgs) {
|
|
|
142
171
|
let config = deepMerge(defaultConfig, fileConfig);
|
|
143
172
|
config = deepMerge(config, envConfig);
|
|
144
173
|
config = deepMerge(config, argsConfig);
|
|
145
|
-
|
|
146
|
-
if (!existsSync(dbDir)) {
|
|
147
|
-
mkdirSync(dbDir, { recursive: true });
|
|
148
|
-
}
|
|
174
|
+
ensureIrisDirectory(dirname(config.storage.path), 'the database directory (IRIS_DB_PATH / --db-path)');
|
|
149
175
|
return config;
|
|
150
176
|
}
|