@iris-eval/mcp-server 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -33
- package/dist/config/index.d.ts +10 -0
- package/dist/config/index.js +33 -7
- package/dist/dashboard/assets/index-CshLgDRB.js +10 -0
- package/dist/dashboard/assets/{index-UffZ-aEJ.css → index-D0cFfBqn.css} +1 -1
- package/dist/dashboard/index.html +4 -3
- package/dist/dashboard/routes/health.js +10 -3
- package/dist/dashboard/routes/moments.js +1 -1
- package/dist/dashboard/routes/preferences.d.ts +1 -0
- package/dist/dashboard/routes/preferences.js +31 -3
- package/dist/dashboard/routes/rules.d.ts +18 -0
- package/dist/dashboard/routes/rules.js +160 -6
- package/dist/dashboard/routes/traces.js +21 -3
- package/dist/dashboard/seed-demo-data.js +11 -0
- package/dist/dashboard/server.js +13 -3
- package/dist/dashboard/session-auth.d.ts +8 -0
- package/dist/dashboard/session-auth.js +237 -0
- package/dist/dashboard/validation.d.ts +9 -3
- package/dist/dashboard/validation.js +69 -11
- package/dist/eval/engine.d.ts +62 -0
- package/dist/eval/engine.js +188 -82
- package/dist/eval/rules/safety.d.ts +8 -0
- package/dist/eval/rules/safety.js +43 -11
- package/dist/index.js +102 -16
- package/dist/middleware/rate-limit.d.ts +25 -0
- package/dist/middleware/rate-limit.js +54 -2
- package/dist/self-test.d.ts +14 -0
- package/dist/self-test.js +97 -13
- package/dist/storage/demo-guard.d.ts +8 -0
- package/dist/storage/demo-guard.js +53 -0
- package/dist/storage/sqlite-adapter.d.ts +6 -0
- package/dist/storage/sqlite-adapter.js +72 -1
- package/dist/tools/delete-rule.js +49 -11
- package/dist/tools/deploy-rule.d.ts +33 -0
- package/dist/tools/deploy-rule.js +130 -27
- package/dist/tools/evaluate-output.js +41 -22
- package/dist/tools/evaluate-with-llm-judge.js +10 -3
- package/dist/tools/get-traces.d.ts +27 -0
- package/dist/tools/get-traces.js +60 -8
- package/dist/tools/list-rules.js +2 -2
- package/dist/tools/log-trace.js +4 -3
- package/dist/tools/strict-input.d.ts +1 -0
- package/dist/tools/strict-input.js +25 -0
- package/dist/tools/trace-link.d.ts +7 -0
- package/dist/tools/trace-link.js +39 -0
- package/dist/tools/verify-citations.d.ts +19 -0
- package/dist/tools/verify-citations.js +41 -4
- package/dist/types/eval.d.ts +45 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/query.d.ts +25 -0
- package/package.json +1 -1
- package/server.json +2 -2
- package/dist/dashboard/assets/index-VI_nbMfN.js +0 -10
package/README.md
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
[](https://github.com/iris-eval/mcp-server/actions/workflows/ci.yml)
|
|
9
9
|
[](https://securityscorecards.dev/viewer/?uri=github.com/iris-eval/mcp-server)
|
|
10
10
|
[](https://www.bestpractices.dev/projects/12849)
|
|
11
|
-
[](LICENSE)
|
|
11
|
+
[](https://github.com/iris-eval/mcp-server/blob/main/LICENSE)
|
|
12
12
|
[](https://github.com/iris-eval/mcp-server/pkgs/container/mcp-server)
|
|
13
13
|
[](https://www.pulsemcp.com/servers/iris-eval)
|
|
14
14
|
[](https://mcp.so/server/iris/iris-eval)
|
|
@@ -56,11 +56,11 @@ Your agent discovers Iris's nine tools on connect, and the dashboard serves at *
|
|
|
56
56
|
|
|
57
57
|
The trace lands on the dashboard with its scores. Prefer the MCP server headless? Drop `--dashboard` from the args — you can open the same dashboard any time with `npx @iris-eval/mcp-server --dashboard`.
|
|
58
58
|
|
|
59
|
-
**One thing worth knowing up front:** MCP tools are called when the model decides to call them. Iris doesn't intercept your agent, so traces are logged when your agent asks it to log them — either because you told it to, or because your code calls the tools directly. Ask your agent to "log this to Iris and evaluate it" and it will. If you want capture that doesn't depend on the model choosing, `POST /api/v1/traces` does exactly that — your code sends the trace over plain HTTP, no model in the loop (see [docs/http-ingest.md](docs/http-ingest.md)). The CLI and SDKs on the [roadmap](docs/roadmap.md) will be thin clients over the same endpoint.
|
|
59
|
+
**One thing worth knowing up front:** MCP tools are called when the model decides to call them. Iris doesn't intercept your agent, so traces are logged when your agent asks it to log them — either because you told it to, or because your code calls the tools directly. Ask your agent to "log this to Iris and evaluate it" and it will. If you want capture that doesn't depend on the model choosing, `POST /api/v1/traces` does exactly that — your code sends the trace over plain HTTP, no model in the loop (see [docs/http-ingest.md](https://github.com/iris-eval/mcp-server/blob/main/docs/http-ingest.md)). The CLI and SDKs on the [roadmap](https://github.com/iris-eval/mcp-server/blob/main/docs/roadmap.md) will be thin clients over the same endpoint.
|
|
60
60
|
|
|
61
61
|
### Capture over HTTP (no model in the loop)
|
|
62
62
|
|
|
63
|
-
|
|
63
|
+
The ingest endpoint lives on the **dashboard port** — `6920` by default, not the MCP transport port — and it exists only while the dashboard is running. Pass `--dashboard` (or set `IRIS_DASHBOARD=true`); `--transport http` on its own does **not** start it, and a request to the transport port returns `404`. With the dashboard up, anything that can send an HTTP request can log a trace — and optionally run the deterministic evals in the same request:
|
|
64
64
|
|
|
65
65
|
```bash
|
|
66
66
|
curl -s -X POST "http://127.0.0.1:6920/api/v1/traces" \
|
|
@@ -74,15 +74,16 @@ curl -s -X POST "http://127.0.0.1:6920/api/v1/traces" \
|
|
|
74
74
|
}'
|
|
75
75
|
```
|
|
76
76
|
|
|
77
|
-
Returns `201` with the stored `trace_id` and the evaluation result. The endpoint accepts the same body as the `log_trace` tool and sits behind the same
|
|
77
|
+
Returns `201` with the stored `trace_id` and the evaluation result (in `--demo` mode the endpoint refuses writes with `403`, so demo data never mixes with yours). The endpoint accepts the same body as the `log_trace` tool and sits behind the same middleware stack as the rest of the dashboard: loopback bind and the DNS-rebinding guard by default, plus Bearer auth when you set one. **Two plain facts about it:** it accepts unauthenticated writes unless Iris was started with `--api-key` (or `IRIS_API_KEY`) — the loopback bind is what keeps it to your machine by default, so set a key before binding beyond loopback; and what it stores is verbatim — `input` and `output` land in `iris.db` exactly as sent, including any text `no_pii` goes on to flag. Full contract, field reference, and error semantics: [docs/http-ingest.md](https://github.com/iris-eval/mcp-server/blob/main/docs/http-ingest.md).
|
|
78
78
|
|
|
79
|
-
###
|
|
79
|
+
### Verify your install
|
|
80
80
|
|
|
81
81
|
```bash
|
|
82
|
-
npx @iris-eval/mcp-server --self-test
|
|
82
|
+
npx @iris-eval/mcp-server --self-test # offline diagnostic; exit 0 = healthy, 1 = a check failed
|
|
83
|
+
npx @iris-eval/mcp-server --version # prints the bare version, e.g. 0.5.1
|
|
83
84
|
```
|
|
84
85
|
|
|
85
|
-
|
|
86
|
+
`--self-test` first creates your Iris home if it is missing and checks that it is writable (exit 1, naming the path, if it is not), then runs its checks — storage round-trip, a planted SSN and a planted injection caught by the safety rules, dashboard boot, the DNS-rebinding guard — inside an isolated temp home, so your real database is never opened. Everything Iris writes lives under one directory, your **Iris home**: `~/.iris` by default (`%USERPROFILE%\.iris` on Windows), or wherever `IRIS_HOME` points. That is where `iris.db`, `config.json`, `custom-rules.json`, `audit.log`, `preferences.json` and the demo files live; point `IRIS_HOME` at a scratch directory to try Iris without touching your real data.
|
|
86
87
|
|
|
87
88
|
<details>
|
|
88
89
|
<summary><strong>Setup by tool</strong></summary>
|
|
@@ -190,7 +191,7 @@ docker run -p 3000:3000 -p 6920:6920 -v iris-data:/data ghcr.io/iris-eval/mcp-se
|
|
|
190
191
|
| **Web Dashboard** | Real-time dark-mode UI that lands on the failures, worst and newest first — trace visualization, eval results, cost breakdowns, and a command palette (⌘K) that searches your own rules, traces, and evals. |
|
|
191
192
|
| **Local-first** | Everything lives in SQLite on your disk. No account, no sign-up, no telemetry. Outbound HTTP happens only where you opt in: your own LLM-judge key, citation fetching, or an OTel exporter you configure. |
|
|
192
193
|
|
|
193
|
-
Where this is going next: [the roadmap](docs/roadmap.md).
|
|
194
|
+
Where this is going next: [the roadmap](https://github.com/iris-eval/mcp-server/blob/main/docs/roadmap.md).
|
|
194
195
|
|
|
195
196
|
## MCP Tools
|
|
196
197
|
|
|
@@ -206,7 +207,7 @@ Iris registers nine tools that any MCP-compatible agent can invoke — full rule
|
|
|
206
207
|
- **`evaluate_with_llm_judge`** — Semantic eval via LLM (Anthropic or OpenAI). Five templates: accuracy, helpfulness, safety, correctness, faithfulness. Cost-capped, per-eval pricing disclosed. **Bring your own API key** (`IRIS_ANTHROPIC_API_KEY` or `IRIS_OPENAI_API_KEY`) — Iris doesn't proxy or relay LLM calls.
|
|
207
208
|
- **`verify_citations`** — Extract citations from output (numbered, author-year, URLs, DOIs), fetch sources behind an SSRF-guarded + domain-allowlisted resolver, and use an LLM judge to check whether each source actually supports the cited claim. Opt-in outbound HTTP. Same BYOK requirement as `evaluate_with_llm_judge`.
|
|
208
209
|
|
|
209
|
-
When `IRIS_OTEL_ENDPOINT` is configured, `log_trace` calls also emit a best-effort OTLP/HTTP JSON export to any OpenTelemetry collector (Jaeger, Grafana Tempo, Datadog OTLP, Honeycomb, etc). See [docs/otel-integration.md](docs/otel-integration.md).
|
|
210
|
+
When `IRIS_OTEL_ENDPOINT` is configured, `log_trace` calls also emit a best-effort OTLP/HTTP JSON export to any OpenTelemetry collector (Jaeger, Grafana Tempo, Datadog OTLP, Honeycomb, etc). See [docs/otel-integration.md](https://github.com/iris-eval/mcp-server/blob/main/docs/otel-integration.md).
|
|
210
211
|
|
|
211
212
|
### How `passed` is decided
|
|
212
213
|
|
|
@@ -215,10 +216,45 @@ When `IRIS_OTEL_ENDPOINT` is configured, `log_trace` calls also emit a best-effo
|
|
|
215
216
|
- **`score`** (0..1) is the weighted average across the rules that ran — a quality gradient.
|
|
216
217
|
- **`passed`** is the ship/no-ship verdict: `true` only when the score clears the pass threshold (default **0.7**) **and no critical rule failed**.
|
|
217
218
|
|
|
218
|
-
Genuine safety violations hard-fail. `no_pii`, `no_injection_patterns`, and `no_blocklist_words` are **critical rules**: if one fails, the eval reports `passed: false` no matter how well the other rules scored, and the response names the culprits in `critical_failures`. A leaked SSN can't be averaged away. Custom rules deployed with `severity: "high"` or `"critical"` hard-fail the same way; `low`/`medium` severities only affect the score. One boundary to know: a critical rule that **skipped** (missing context, or any other cause of a skip) has not judged the output and does not veto — `rule_results` shows every skip and its reason, so a gate that must fail closed on non-verdicts can.
|
|
219
|
+
Genuine safety violations hard-fail. `no_pii`, `no_injection_patterns`, and `no_blocklist_words` are **critical rules**: if one fails, the eval reports `passed: false` no matter how well the other rules scored, and the response names the culprits in `critical_failures`. A leaked SSN can't be averaged away. Custom rules deployed with `severity: "high"` or `"critical"` hard-fail the same way; `low`/`medium` severities only affect the score. One boundary to know: a critical rule that **skipped** (missing context, or any other cause of a skip) has not judged the output and does not veto — it is listed in `critical_skipped`, and `rule_results` shows every skip and its reason, so a gate that must fail closed on non-verdicts can.
|
|
219
220
|
|
|
220
221
|
One gotcha for CI gates: if you omit `eval_type`, the default `completeness` bundle runs — **safety rules don't**. The response echoes `eval_type` (plus a `note` when it was defaulted) so your gate can verify which bundle actually ran. Key on `passed` for the verdict and `eval_type: "safety"` for coverage.
|
|
221
222
|
|
|
223
|
+
### Authoring a custom rule
|
|
224
|
+
|
|
225
|
+
Two ways to add a rule. **Inline** rules ride along on one `evaluate_output` call (`custom_rules`, up to 10 per call); they fire alongside whatever `eval_type` bundle you chose, or alone with `eval_type: "custom"`. **Deployed** rules are registered once with `deploy_rule`, persist in `custom-rules.json` under your Iris home, and fire on every future `evaluate_output` of their `evalType`. The definition is the same shape either way:
|
|
226
|
+
|
|
227
|
+
| Field | Required | What it is |
|
|
228
|
+
|---|---|---|
|
|
229
|
+
| `name` | yes | 1–80 characters; appears as `ruleName` in results |
|
|
230
|
+
| `type` | yes | one of `regex_match` · `regex_no_match` · `min_length` · `max_length` · `contains_keywords` · `excludes_keywords` · `json_schema` · `cost_threshold` |
|
|
231
|
+
| `config` | yes | the keys for that type: `pattern` (+ optional `flags`) for the two regex types · `min_length` / `max_length` (a character count) · `keywords` (+ optional `threshold`, 0–1, default `1` = all must appear) for the two keyword types · `{}` for `json_schema` · `max_cost` in USD for `cost_threshold` |
|
|
232
|
+
| `weight` | no | weight in the score; default `1` |
|
|
233
|
+
|
|
234
|
+
`deploy_rule` wraps the definition with `name`, an optional `description`, `evalType` (`completeness` · `relevance` · `safety` · `cost` · `custom`) and `severity`. Severity says what a **failure** means: `low`/`medium` only lower the score; `high`/`critical` hard-fail the evaluation — `passed: false`, the rule named in `critical_failures` — whatever the weighted score says. A rule that skips (a `cost_threshold` rule with no `cost_usd`, or a regex killed at the 100 ms sandbox budget) has not judged the output and is listed in `critical_skipped` instead. Deploy a critical rule that forbids internal hostnames in anything the agent says:
|
|
235
|
+
|
|
236
|
+
```json
|
|
237
|
+
{
|
|
238
|
+
"name": "no_internal_hostnames",
|
|
239
|
+
"description": "Output must not mention internal hostnames.",
|
|
240
|
+
"evalType": "safety",
|
|
241
|
+
"severity": "critical",
|
|
242
|
+
"definition": {
|
|
243
|
+
"name": "no_internal_hostnames",
|
|
244
|
+
"type": "regex_no_match",
|
|
245
|
+
"config": { "pattern": "\\b[a-z0-9-]+\\.internal\\.example\\b", "flags": "i" }
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
The response is the persisted rule — keep the `id` for `delete_rule`:
|
|
251
|
+
|
|
252
|
+
```json
|
|
253
|
+
{ "rule": { "id": "rule-588823d0", "name": "no_internal_hostnames", "evalType": "safety", "severity": "critical", "enabled": true, "version": 1, "definition": { "…": "…" } } }
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
From the very next `evaluate_output` with `eval_type: "safety"`, an output that mentions `db-primary.internal.example` comes back `passed: false` with `critical_failures: ["no_internal_hostnames"]` — even though all five built-in safety rules passed and the weighted score is 0.895. Regex patterns must pass a ReDoS check at deploy time and always run in a sandbox worker under a hard 100 ms deadline. `list_rules` shows what is deployed; the dashboard's rule composer builds the same shape from a failure you clicked on. Full reference, scoring per type, and worked examples: [docs/custom-rules.md](https://github.com/iris-eval/mcp-server/blob/main/docs/custom-rules.md).
|
|
257
|
+
|
|
222
258
|
Full tool schemas and configuration: [iris-eval.com](https://iris-eval.com)
|
|
223
259
|
|
|
224
260
|
## Hosted features
|
|
@@ -231,19 +267,19 @@ Two commitments hold regardless: **nothing that is free today will move behind a
|
|
|
231
267
|
|
|
232
268
|
## Examples
|
|
233
269
|
|
|
234
|
-
- [Claude Desktop setup](examples/claude-desktop
|
|
235
|
-
- [TypeScript — MCP SDK client](examples/typescript/basic-usage.ts) — connect and invoke tools
|
|
236
|
-
- [HTTP transport (TS + Python)](examples/http-transport
|
|
237
|
-
- [LangChain instrumentation (Python, conceptual)](examples/langchain/observe-agent.py) — scaffold showing the shape; needs your agent code to be runnable
|
|
238
|
-
- [CrewAI instrumentation (Python, conceptual)](examples/crewai/observe-crew.py) — scaffold; same caveat
|
|
270
|
+
- [Claude Desktop setup](https://github.com/iris-eval/mcp-server/tree/main/examples/claude-desktop) — MCP config for stdio and HTTP modes
|
|
271
|
+
- [TypeScript — MCP SDK client](https://github.com/iris-eval/mcp-server/blob/main/examples/typescript/basic-usage.ts) — connect and invoke tools
|
|
272
|
+
- [HTTP transport (TS + Python)](https://github.com/iris-eval/mcp-server/tree/main/examples/http-transport) — full client code for REST-style integration
|
|
273
|
+
- [LangChain instrumentation (Python, conceptual)](https://github.com/iris-eval/mcp-server/blob/main/examples/langchain/observe-agent.py) — scaffold showing the shape; needs your agent code to be runnable
|
|
274
|
+
- [CrewAI instrumentation (Python, conceptual)](https://github.com/iris-eval/mcp-server/blob/main/examples/crewai/observe-crew.py) — scaffold; same caveat
|
|
239
275
|
|
|
240
276
|
## Community
|
|
241
277
|
|
|
242
278
|
- [GitHub Issues](https://github.com/iris-eval/mcp-server/issues) — Bug reports and feature requests
|
|
243
279
|
- [GitHub Discussions](https://github.com/iris-eval/mcp-server/discussions) — Questions and ideas
|
|
244
|
-
- [Contributing Guide](CONTRIBUTING.md) — How to contribute
|
|
245
|
-
- [HTTP Ingest](docs/http-ingest.md) — Deterministic trace capture via `POST /api/v1/traces`
|
|
246
|
-
- [Roadmap](docs/roadmap.md) — What's coming next
|
|
280
|
+
- [Contributing Guide](https://github.com/iris-eval/mcp-server/blob/main/CONTRIBUTING.md) — How to contribute
|
|
281
|
+
- [HTTP Ingest](https://github.com/iris-eval/mcp-server/blob/main/docs/http-ingest.md) — Deterministic trace capture via `POST /api/v1/traces`
|
|
282
|
+
- [Roadmap](https://github.com/iris-eval/mcp-server/blob/main/docs/roadmap.md) — What's coming next
|
|
247
283
|
|
|
248
284
|
<details>
|
|
249
285
|
<summary><strong>Configuration & Security</strong></summary>
|
|
@@ -256,37 +292,50 @@ Two commitments hold regardless: **nothing that is free today will move behind a
|
|
|
256
292
|
| `--port` | `3000` | HTTP transport port |
|
|
257
293
|
| `--db-path` | `~/.iris/iris.db` | SQLite database path |
|
|
258
294
|
| `--config` | `~/.iris/config.json` | Config file path |
|
|
259
|
-
| `--api-key` | — | API key for HTTP authentication |
|
|
260
|
-
| `--dashboard` | `false` | Enable web dashboard |
|
|
295
|
+
| `--api-key` | — | API key for HTTP authentication (transport and dashboard, including `POST /api/v1/traces`) |
|
|
296
|
+
| `--dashboard` | `false` | Enable web dashboard. Also the only way the `POST /api/v1/traces` ingest endpoint starts — it never starts implicitly with `--transport http` |
|
|
261
297
|
| `--dashboard-port` | `6920` | Dashboard port |
|
|
262
298
|
| `--dashboard-host` | `127.0.0.1` | Dashboard bind address. Loopback by default — the dashboard is unauthenticated unless `--api-key` is set, so binding beyond loopback exposes your full trace history |
|
|
263
299
|
| `--demo` | `false` | Seed a demo database (separate from your real traces) and serve the dashboard against it |
|
|
264
300
|
| `--demo-clear` | `false` | Delete the demo database and exit |
|
|
265
301
|
| `--self-test` | `false` | Run the offline install diagnostic in an isolated temp home, then exit (0 = healthy, 1 = a check failed) |
|
|
302
|
+
| `--purge` | `false` | Delete **every** stored trace, span and evaluation from the configured database, compact the file and truncate the write-ahead log so the deleted text does not linger on disk, then exit. Deployed rules, the audit log and preferences are kept. Not reversible. Stop any running Iris server first — the file is compacted in place. Refuses to combine with `--demo`, `--demo-clear` or `--self-test` |
|
|
303
|
+
| `--version` | — | Print the bare version (e.g. `0.5.1`) to stdout and exit 0. Reads nothing under your Iris home |
|
|
266
304
|
|
|
267
305
|
### Environment Variables
|
|
268
306
|
|
|
307
|
+
Every variable `--help` documents. CLI flags take precedence over environment variables when both are set.
|
|
308
|
+
|
|
269
309
|
| Variable | Description |
|
|
270
310
|
|----------|-------------|
|
|
271
311
|
| `IRIS_TRANSPORT` | Transport type (`stdio` or `http`) |
|
|
272
|
-
| `
|
|
273
|
-
| `
|
|
312
|
+
| `IRIS_HOST` | HTTP transport bind address (default `127.0.0.1`) |
|
|
313
|
+
| `IRIS_PORT` | HTTP transport port (1-65535, default `3000`) |
|
|
274
314
|
| `IRIS_HOME` | Directory for all per-user files: `config.json`, `iris.db`, `custom-rules.json`, `audit.log`, `preferences.json` (default `~/.iris`) |
|
|
275
315
|
| `IRIS_DB_PATH` | SQLite database path (overrides `IRIS_HOME` for the DB only) |
|
|
276
316
|
| `IRIS_LOG_LEVEL` | Log level: `debug`, `info`, `warn`, `error` |
|
|
277
|
-
| `IRIS_DASHBOARD` |
|
|
278
|
-
| `IRIS_DASHBOARD_PORT` | Dashboard port (default `6920`) |
|
|
317
|
+
| `IRIS_DASHBOARD` | `true`/`1`/`yes`/`on` enables the web dashboard; `false`/`0`/`no`/`off` disables it (also overrides `dashboard.enabled` in `config.json`) |
|
|
318
|
+
| `IRIS_DASHBOARD_PORT` | Dashboard port (1-65535, default `6920`) |
|
|
279
319
|
| `IRIS_DASHBOARD_HOST` | Dashboard bind address (default `127.0.0.1`) |
|
|
280
320
|
| `IRIS_API_KEY` | API key for HTTP authentication |
|
|
281
|
-
| `IRIS_ALLOWED_ORIGINS` | Comma-separated
|
|
282
|
-
|
|
283
|
-
|
|
321
|
+
| `IRIS_ALLOWED_ORIGINS` | Comma-separated origin allowlist. Dashboard: CORS headers (supports globs, e.g. `http://localhost:*`). HTTP transport: exact-match `Origin` allowlist for DNS-rebinding protection (globs ignored; the server's own loopback origins are always allowed) |
|
|
322
|
+
| `IRIS_NO_AUTO_LAUNCH` | Set to `1` to disable the first-run dashboard auto-launch |
|
|
323
|
+
| `IRIS_ANTHROPIC_API_KEY` | Required by `evaluate_with_llm_judge` + `verify_citations` with `provider=anthropic` |
|
|
324
|
+
| `IRIS_OPENAI_API_KEY` | Required by `evaluate_with_llm_judge` + `verify_citations` with `provider=openai` |
|
|
325
|
+
| `IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL` | Hard cost cap per LLM judge call (default `0.25`) |
|
|
326
|
+
| `IRIS_CITATION_ALLOW_FETCH` | Set to `1` to permit outbound HTTP in `verify_citations` (off by default) |
|
|
327
|
+
| `IRIS_CITATION_DOMAINS` | Comma-separated hostname allowlist for `verify_citations` (suffix match) |
|
|
328
|
+
| `IRIS_OTEL_ENDPOINT` | Enable best-effort OTLP/HTTP JSON trace export to this collector URL |
|
|
329
|
+
| `IRIS_OTEL_SERVICE_NAME` | `service.name` resource attribute for OTel export (default `iris-mcp`) |
|
|
330
|
+
| `IRIS_OTEL_HEADERS` | Comma-separated `k=v` headers for OTel export (e.g. `authorization=Bearer abc`) |
|
|
331
|
+
| `IRIS_OTEL_TIMEOUT_MS` | Per-export timeout (default `15000`) |
|
|
332
|
+
| `RATE_LIMIT_SALT` | Website waitlist API only — required when the iris-eval.com site is deployed; the server never reads it |
|
|
284
333
|
|
|
285
334
|
### Security
|
|
286
335
|
|
|
287
336
|
When using HTTP transport, Iris includes:
|
|
288
337
|
|
|
289
|
-
- API key authentication with timing-safe comparison
|
|
338
|
+
- API key authentication with timing-safe comparison (Bearer for API clients; browser sign-in to the dashboard via `?key=`)
|
|
290
339
|
- CORS restricted to localhost by default
|
|
291
340
|
- Rate limiting (600 req/min dashboard API, 20 req/min MCP)
|
|
292
341
|
- Helmet security headers
|
|
@@ -299,6 +348,12 @@ When using HTTP transport, Iris includes:
|
|
|
299
348
|
iris-mcp --transport http --port 3000 --api-key "$(openssl rand -hex 32)" --dashboard
|
|
300
349
|
```
|
|
301
350
|
|
|
351
|
+
With a key set, API clients — MCP clients, capture SDKs, `POST /api/v1/traces` — send `Authorization: Bearer <key>`. To open the dashboard in a browser, append the key once to any dashboard URL, `http://localhost:6920/?key=<api key>`: Iris exchanges it for an HttpOnly, SameSite=Lax session cookie and redirects to the same page with the key removed from the address bar. A page opened without a session shows a sign-in form that does the same exchange. The key is never stored in the browser, and sessions live only in the server process.
|
|
352
|
+
|
|
353
|
+
### Your data on disk
|
|
354
|
+
|
|
355
|
+
Everything Iris stores lives under your Iris home (`~/.iris`, or `IRIS_HOME`). `iris.db` keeps every trace's `input` and `output` **verbatim** — including any text `no_pii` goes on to flag; detection does not redact. At startup, traces and evaluations older than `retention.days` (default `30`, `0` disables, set in `config.json`) are deleted and the write-ahead log is checkpointed. To remove everything now, stop the server and run `--purge`: it deletes every stored trace, span and evaluation, compacts the database and truncates the write-ahead log so the text is gone from disk, and keeps your deployed rules, audit log and preferences.
|
|
356
|
+
|
|
302
357
|
</details>
|
|
303
358
|
|
|
304
359
|
<details>
|
|
@@ -326,20 +381,27 @@ Or install globally to avoid cache issues entirely:
|
|
|
326
381
|
npm install -g @iris-eval/mcp-server@latest
|
|
327
382
|
```
|
|
328
383
|
|
|
384
|
+
### `npm install --ignore-scripts` broke the SQLite binding
|
|
385
|
+
|
|
386
|
+
Iris stores traces with `better-sqlite3`, a native module that fetches or compiles its binding in an install script. If that script was skipped — `--ignore-scripts` on the command line, `ignore-scripts=true` in an `.npmrc` (common on corporate machines), or a registry mirror that strips postinstall — startup fails with a long "Could not locate the bindings file" dump listing a dozen paths it tried. Rebuild that one module:
|
|
387
|
+
|
|
388
|
+
```bash
|
|
389
|
+
npm rebuild better-sqlite3
|
|
390
|
+
# for a global install:
|
|
391
|
+
npm rebuild -g better-sqlite3
|
|
392
|
+
```
|
|
393
|
+
|
|
329
394
|
### Tools not showing up in Claude Code
|
|
330
395
|
|
|
331
396
|
MCP tools only load at session start. After adding iris-eval, restart the session with `/clear` or relaunch the terminal.
|
|
332
397
|
|
|
333
398
|
### Version check
|
|
334
399
|
|
|
335
|
-
Iris logs its version on the first startup line:
|
|
336
|
-
|
|
337
400
|
```bash
|
|
338
|
-
npx @iris-eval/mcp-server --
|
|
339
|
-
# First log line: "Starting Iris MCP server vX.Y.Z"
|
|
401
|
+
npx @iris-eval/mcp-server --version
|
|
340
402
|
```
|
|
341
403
|
|
|
342
|
-
For a global install, `npm ls -g @iris-eval/mcp-server` shows the installed version.
|
|
404
|
+
The first startup log line also carries it (`Starting Iris MCP server vX.Y.Z`), and `--self-test` prints it in its summary. For a global install, `npm ls -g @iris-eval/mcp-server` shows the installed version.
|
|
343
405
|
|
|
344
406
|
### Updating
|
|
345
407
|
|
package/dist/config/index.d.ts
CHANGED
|
@@ -1,4 +1,14 @@
|
|
|
1
1
|
import type { IrisConfig } from '../types/index.js';
|
|
2
|
+
export declare const IRIS_HOME_DIR_MODE = 448;
|
|
3
|
+
/**
|
|
4
|
+
* Create a directory Iris needs, or fail with ONE line that names the path
|
|
5
|
+
* and the permission problem (#371). Before this, an unwritable IRIS_HOME
|
|
6
|
+
* surfaced as a raw EPERM stack trace from deep inside mkdirSync — the
|
|
7
|
+
* first thing a new user saw, with nothing telling them which variable to
|
|
8
|
+
* change. Exported so --self-test probes the configured home through the
|
|
9
|
+
* exact same call the server will make.
|
|
10
|
+
*/
|
|
11
|
+
export declare function ensureIrisDirectory(path: string, what: string): void;
|
|
2
12
|
export interface CliArgs {
|
|
3
13
|
transport?: string;
|
|
4
14
|
port?: number;
|
package/dist/config/index.js
CHANGED
|
@@ -2,6 +2,37 @@ import { readFileSync, mkdirSync, existsSync } from 'node:fs';
|
|
|
2
2
|
import { join, dirname } from 'node:path';
|
|
3
3
|
import { defaultConfig } from './defaults.js';
|
|
4
4
|
import { irisHome } from '../utils/iris-home.js';
|
|
5
|
+
/*
|
|
6
|
+
* Owner-only (0700) for the iris home directory, matching the 0600 the data
|
|
7
|
+
* FILES inside it already get (write-atomic.ts, sqlite-adapter.ts). iris.db
|
|
8
|
+
* holds agent inputs and outputs verbatim — a PII detector necessarily
|
|
9
|
+
* stores the PII it found — and a 0755 directory beside 0600 files still
|
|
10
|
+
* lets every local account list what is there and read anything a library
|
|
11
|
+
* happens to create with the default umask (#372). The mode applies only
|
|
12
|
+
* when THIS process creates the directory; a pre-existing IRIS_HOME keeps
|
|
13
|
+
* whatever permissions its owner chose. No-op on Windows (ACLs govern).
|
|
14
|
+
*/
|
|
15
|
+
export const IRIS_HOME_DIR_MODE = 0o700;
|
|
16
|
+
/**
|
|
17
|
+
* Create a directory Iris needs, or fail with ONE line that names the path
|
|
18
|
+
* and the permission problem (#371). Before this, an unwritable IRIS_HOME
|
|
19
|
+
* surfaced as a raw EPERM stack trace from deep inside mkdirSync — the
|
|
20
|
+
* first thing a new user saw, with nothing telling them which variable to
|
|
21
|
+
* change. Exported so --self-test probes the configured home through the
|
|
22
|
+
* exact same call the server will make.
|
|
23
|
+
*/
|
|
24
|
+
export function ensureIrisDirectory(path, what) {
|
|
25
|
+
if (existsSync(path))
|
|
26
|
+
return;
|
|
27
|
+
try {
|
|
28
|
+
mkdirSync(path, { recursive: true, mode: IRIS_HOME_DIR_MODE });
|
|
29
|
+
}
|
|
30
|
+
catch (err) {
|
|
31
|
+
const code = err.code ?? (err instanceof Error ? err.message : String(err));
|
|
32
|
+
throw new Error(`Cannot create ${what} "${path}" (${code}). Iris keeps its database, config and rule files there. ` +
|
|
33
|
+
'Point IRIS_HOME at a directory this user can write, or fix the permissions on that path.');
|
|
34
|
+
}
|
|
35
|
+
}
|
|
5
36
|
function deepMerge(target, source) {
|
|
6
37
|
const result = { ...target };
|
|
7
38
|
for (const key of Object.keys(source)) {
|
|
@@ -132,9 +163,7 @@ function cliArgsToConfig(args) {
|
|
|
132
163
|
}
|
|
133
164
|
export function loadConfig(cliArgs) {
|
|
134
165
|
const home = irisHome();
|
|
135
|
-
|
|
136
|
-
mkdirSync(home, { recursive: true });
|
|
137
|
-
}
|
|
166
|
+
ensureIrisDirectory(home, 'IRIS_HOME');
|
|
138
167
|
const configPath = cliArgs?.config ?? join(home, 'config.json');
|
|
139
168
|
const fileConfig = loadConfigFile(configPath);
|
|
140
169
|
const envConfig = loadEnvVars();
|
|
@@ -142,9 +171,6 @@ export function loadConfig(cliArgs) {
|
|
|
142
171
|
let config = deepMerge(defaultConfig, fileConfig);
|
|
143
172
|
config = deepMerge(config, envConfig);
|
|
144
173
|
config = deepMerge(config, argsConfig);
|
|
145
|
-
|
|
146
|
-
if (!existsSync(dbDir)) {
|
|
147
|
-
mkdirSync(dbDir, { recursive: true });
|
|
148
|
-
}
|
|
174
|
+
ensureIrisDirectory(dirname(config.storage.path), 'the database directory (IRIS_DB_PATH / --db-path)');
|
|
149
175
|
return config;
|
|
150
176
|
}
|