@iris-eval/mcp-server 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/config/defaults.js +15 -0
- package/dist/dashboard/assets/{index-Cz8_oOqG.js → index-CeJbaq6m.js} +1 -1
- package/dist/dashboard/index.html +1 -1
- package/dist/dashboard/routes/traces.js +2 -2
- package/dist/dashboard/seed-demo-data.js +1 -1
- package/dist/eval/citation-verify/verifier.d.ts +16 -1
- package/dist/eval/citation-verify/verifier.js +14 -4
- package/dist/eval/compose.d.ts +57 -0
- package/dist/eval/compose.js +179 -0
- package/dist/eval/criticality.d.ts +7 -0
- package/dist/eval/decision-moment.js +33 -4
- package/dist/eval/engine.d.ts +5 -2
- package/dist/eval/engine.js +81 -13
- package/dist/eval/llm-judge/evaluator.d.ts +20 -0
- package/dist/eval/llm-judge/evaluator.js +10 -1
- package/dist/eval/published-accuracy.d.ts +22 -22
- package/dist/eval/published-accuracy.js +11 -11
- package/dist/eval/risk.d.ts +60 -0
- package/dist/eval/risk.js +187 -0
- package/dist/eval/rules/completeness.js +5 -1
- package/dist/eval/rules/cost.d.ts +1 -1
- package/dist/eval/rules/cost.js +6 -6
- package/dist/eval/rules/custom.js +1 -0
- package/dist/eval/rules/relevance.js +7 -2
- package/dist/eval/rules/safety.d.ts +6 -2
- package/dist/eval/rules/safety.js +55 -59
- package/dist/eval/seeded-random.d.ts +4 -0
- package/dist/eval/seeded-random.js +36 -0
- package/dist/eval/stamp.d.ts +1 -1
- package/dist/eval/stamp.js +1 -0
- package/dist/eval/text/checksums.d.ts +23 -0
- package/dist/eval/text/checksums.js +97 -0
- package/dist/eval/text/normalise.d.ts +30 -0
- package/dist/eval/text/normalise.js +265 -0
- package/dist/eval/text/sentences.d.ts +15 -0
- package/dist/eval/text/sentences.js +149 -0
- package/dist/self-test.js +3 -3
- package/dist/storage/sqlite-adapter.js +16 -2
- package/dist/tools/evaluate-output.js +2 -2
- package/dist/tools/evaluate-with-llm-judge.d.ts +3 -0
- package/dist/tools/evaluate-with-llm-judge.js +29 -1
- package/dist/tools/verify-citations.d.ts +2 -1
- package/dist/tools/verify-citations.js +25 -4
- package/dist/types/config.d.ts +35 -0
- package/dist/types/eval.d.ts +51 -0
- package/package.json +1 -1
- package/server.json +2 -2
package/README.md
CHANGED
|
@@ -60,7 +60,7 @@ The trace lands on the dashboard with its scores. Prefer the MCP server headless
|
|
|
60
60
|
|
|
61
61
|
### Capture over HTTP (no model in the loop)
|
|
62
62
|
|
|
63
|
-
The ingest endpoint lives on the **dashboard port** — `6920` by default, not the MCP transport port — and it exists only while the dashboard is running. Pass `--dashboard` (or set `IRIS_DASHBOARD=true`); `--transport http` on its own does **not** start it, and a request to the transport port returns `404`. With the dashboard up, anything that can send an HTTP request can log a trace — and optionally run the deterministic evals in the same request:
|
|
63
|
+
The ingest endpoint lives on the **dashboard port** — `6920` by default, not the MCP transport port — and it exists only while the dashboard is running. Pass `--dashboard` (or set `IRIS_DASHBOARD=true`); `--transport http` on its own does **not** start it, and a request to the transport port returns `404`. With the dashboard up, anything that can send an HTTP request can log a trace — and optionally run the deterministic evals in the same request. `GET /api/v1/capabilities` on the same port says what this server can judge, what each rule needs, the judge state with the steps that enable it, and the limits — the same object the MCP resource `iris://capabilities` serves — so an HTTP caller has the frame an MCP client gets at initialize:
|
|
64
64
|
|
|
65
65
|
```bash
|
|
66
66
|
curl -s -X POST "http://127.0.0.1:6920/api/v1/traces" \
|
package/dist/config/defaults.js
CHANGED
|
@@ -59,6 +59,21 @@ export const defaultConfig = {
|
|
|
59
59
|
*/
|
|
60
60
|
criticalRules: [],
|
|
61
61
|
nonCriticalRules: [],
|
|
62
|
+
/*
|
|
63
|
+
* The verdict's six defaults (0.10.0), each a config key so a ruling is
|
|
64
|
+
* a one-line change. Measured before shipped: on the held-out split of
|
|
65
|
+
* the composite corpus the risk composer is right about shipping 57.7%
|
|
66
|
+
* of the time against the legacy 38.5%, at an IDENTICAL false-block
|
|
67
|
+
* rate, missing 55.6% of bad outputs against 83.3%. See
|
|
68
|
+
* proof/COMPOSITE.md, which regenerates on every release.
|
|
69
|
+
*/
|
|
70
|
+
composer: 'risk',
|
|
71
|
+
falsePassCost: 1,
|
|
72
|
+
onCriticalSkipped: 'unknown',
|
|
73
|
+
requiredEvidence: [],
|
|
74
|
+
defaultsGate: false,
|
|
75
|
+
prior: 0.5,
|
|
76
|
+
priorMode: 'per-output',
|
|
62
77
|
},
|
|
63
78
|
logging: {
|
|
64
79
|
level: 'info',
|