@iris-eval/mcp-server 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +1 -1
  2. package/dist/config/defaults.js +15 -0
  3. package/dist/dashboard/assets/{index-Cz8_oOqG.js → index-CeJbaq6m.js} +1 -1
  4. package/dist/dashboard/index.html +1 -1
  5. package/dist/dashboard/routes/traces.js +2 -2
  6. package/dist/dashboard/seed-demo-data.js +1 -1
  7. package/dist/eval/citation-verify/verifier.d.ts +16 -1
  8. package/dist/eval/citation-verify/verifier.js +14 -4
  9. package/dist/eval/compose.d.ts +57 -0
  10. package/dist/eval/compose.js +179 -0
  11. package/dist/eval/criticality.d.ts +7 -0
  12. package/dist/eval/decision-moment.js +33 -4
  13. package/dist/eval/engine.d.ts +5 -2
  14. package/dist/eval/engine.js +81 -13
  15. package/dist/eval/llm-judge/evaluator.d.ts +20 -0
  16. package/dist/eval/llm-judge/evaluator.js +10 -1
  17. package/dist/eval/published-accuracy.d.ts +22 -22
  18. package/dist/eval/published-accuracy.js +11 -11
  19. package/dist/eval/risk.d.ts +60 -0
  20. package/dist/eval/risk.js +187 -0
  21. package/dist/eval/rules/completeness.js +5 -1
  22. package/dist/eval/rules/cost.d.ts +1 -1
  23. package/dist/eval/rules/cost.js +6 -6
  24. package/dist/eval/rules/custom.js +1 -0
  25. package/dist/eval/rules/relevance.js +7 -2
  26. package/dist/eval/rules/safety.d.ts +6 -2
  27. package/dist/eval/rules/safety.js +55 -59
  28. package/dist/eval/seeded-random.d.ts +4 -0
  29. package/dist/eval/seeded-random.js +36 -0
  30. package/dist/eval/stamp.d.ts +1 -1
  31. package/dist/eval/stamp.js +1 -0
  32. package/dist/eval/text/checksums.d.ts +23 -0
  33. package/dist/eval/text/checksums.js +97 -0
  34. package/dist/eval/text/normalise.d.ts +30 -0
  35. package/dist/eval/text/normalise.js +265 -0
  36. package/dist/eval/text/sentences.d.ts +15 -0
  37. package/dist/eval/text/sentences.js +149 -0
  38. package/dist/self-test.js +3 -3
  39. package/dist/storage/sqlite-adapter.js +16 -2
  40. package/dist/tools/evaluate-output.js +2 -2
  41. package/dist/tools/evaluate-with-llm-judge.d.ts +3 -0
  42. package/dist/tools/evaluate-with-llm-judge.js +29 -1
  43. package/dist/tools/verify-citations.d.ts +2 -1
  44. package/dist/tools/verify-citations.js +25 -4
  45. package/dist/types/config.d.ts +35 -0
  46. package/dist/types/eval.d.ts +51 -0
  47. package/package.json +1 -1
  48. package/server.json +2 -2
package/README.md CHANGED
@@ -60,7 +60,7 @@ The trace lands on the dashboard with its scores. Prefer the MCP server headless
60
60
 
61
61
  ### Capture over HTTP (no model in the loop)
62
62
 
63
- The ingest endpoint lives on the **dashboard port** — `6920` by default, not the MCP transport port — and it exists only while the dashboard is running. Pass `--dashboard` (or set `IRIS_DASHBOARD=true`); `--transport http` on its own does **not** start it, and a request to the transport port returns `404`. With the dashboard up, anything that can send an HTTP request can log a trace — and optionally run the deterministic evals in the same request:
63
+ The ingest endpoint lives on the **dashboard port** — `6920` by default, not the MCP transport port — and it exists only while the dashboard is running. Pass `--dashboard` (or set `IRIS_DASHBOARD=true`); `--transport http` on its own does **not** start it, and a request to the transport port returns `404`. With the dashboard up, anything that can send an HTTP request can log a trace — and optionally run the deterministic evals in the same request. `GET /api/v1/capabilities` on the same port says what this server can judge, what each rule needs, the judge state with the steps that enable it, and the limits — the same object the MCP resource `iris://capabilities` serves — so an HTTP caller has the frame an MCP client gets at initialize:
64
64
 
65
65
  ```bash
66
66
  curl -s -X POST "http://127.0.0.1:6920/api/v1/traces" \
@@ -59,6 +59,21 @@ export const defaultConfig = {
59
59
  */
60
60
  criticalRules: [],
61
61
  nonCriticalRules: [],
62
+ /*
63
+ * The verdict's six defaults (0.10.0), each a config key so a ruling is
64
+ * a one-line change. Measured before shipped: on the held-out split of
65
+ * the composite corpus the risk composer is right about shipping 57.7%
66
+ * of the time against the legacy 38.5%, at an IDENTICAL false-block
67
+ * rate, missing 55.6% of bad outputs against 83.3%. See
68
+ * proof/COMPOSITE.md, which regenerates on every release.
69
+ */
70
+ composer: 'risk',
71
+ falsePassCost: 1,
72
+ onCriticalSkipped: 'unknown',
73
+ requiredEvidence: [],
74
+ defaultsGate: false,
75
+ prior: 0.5,
76
+ priorMode: 'per-output',
62
77
  },
63
78
  logging: {
64
79
  level: 'info',