@graphorin/evals 0.6.0 → 0.6.1

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @graphorin/evals
 
+## 0.6.1
+
+### Patch Changes
+
+- [#142](https://github.com/o-stepper/graphorin/pull/142) [`436d6ca`](https://github.com/o-stepper/graphorin/commit/436d6ca5ebbd16df094e915682d3915c279a8430) Thanks [@o-stepper](https://github.com/o-stepper)! - Version constants and version-bearing strings now derive from each package's manifest at build time (`VERSION = pkg.version`; writer ids, client/server info, OTLP framework attributes, build-info metrics interpolate it). No behavioral change at the current version: the rendered strings are byte-identical. A release bump no longer edits source; the new `check-version-consistency` gate fails any reintroduced hardcoded framework version.
+
+- Updated dependencies [[`826ee59`](https://github.com/o-stepper/graphorin/commit/826ee5933ad38693b0dd2f20a110abfecba7d23d), [`436d6ca`](https://github.com/o-stepper/graphorin/commit/436d6ca5ebbd16df094e915682d3915c279a8430)]:
+  - @graphorin/observability@0.6.1
+  - @graphorin/core@0.6.1
+
 ## 0.6.0
 
 ### Minor Changes

package/README.md

@@ -7,14 +7,14 @@
 > bounded concurrency, and regression detection that compares the
 > current run against a stored baseline.
 >
-> Project Graphorin · v0.6.0 · MIT License · © 2026 Oleksiy Stepurenko ·
+> Project Graphorin · v0.6.1 · MIT License · © 2026 Oleksiy Stepurenko ·
 > <https://github.com/o-stepper/graphorin>
 
 ---
 
 ## Status
 
-- **Published:** v0.6.0 (optional sub-pack; the full orchestrator is
+- **Published:** v0.6.1 (optional sub-pack; the full orchestrator is
   decoupled from `@graphorin/observability` per RB-17 / DEC-152).
 
 ---
@@ -103,6 +103,25 @@ const report = await runEvals({
 });
 ```
 
+Every report summary carries a Wilson 95% confidence interval for the
+pass rate (`summary.passRateCi`), and, when `iterations > 1`,
+`summary.passHatK` - the pass^k estimate over the per-case iteration
+outcomes (the probability that all k iterations of a case pass).
+
+---
+
+## Statistics
+
+Sample-size-aware statistics ship as plain exported functions:
+
+```ts
+import { mean, sampleStddev, wilsonInterval, passHatK, pairedPassSignificance } from '@graphorin/evals';
+
+wilsonInterval(18, 20);            // { lo, hi } - 95% CI for 18/20 passes
+passHatK(outcomesByCase);          // pass^k over `-iter-N` outcome groups
+pairedPassSignificance(a, b);      // McNemar paired test between two runs
+```
+
 ---
 
 ## Regression detection
@@ -116,6 +135,9 @@ const regression = detectRegressions(report, baseline, {
   maxPassRateDropPct: 5,
   maxAvgScoreDrop: 0.05,
   maxAvgDurationIncreaseMs: 250,
+  // Opt-in: veto pass-rate-drop findings the McNemar paired test
+  // cannot distinguish from noise at the given alpha (default 0.05).
+  requireSignificance: true,
 });
 if (regression.hasRegressions) {
   for (const f of regression.findings) {
@@ -125,6 +147,10 @@ if (regression.hasRegressions) {
 exitOnFailures(report, regression);
 ```
 
+Pass-rate findings are annotated with the paired regressed / improved
+case counts and the McNemar p-value, so a 3-point drop on 20 cases
+reads differently from the same drop on 2000.
+
 ---
 
 ## Multi-format report writing
@@ -154,4 +180,4 @@ MIT © 2026 Oleksiy Stepurenko
 
 ---
 
-**Project Graphorin** · v0.6.0 · MIT License · © 2026 Oleksiy Stepurenko · <https://github.com/o-stepper/graphorin>
+**Project Graphorin** · v0.6.1 · MIT License · © 2026 Oleksiy Stepurenko · <https://github.com/o-stepper/graphorin>

package/dist/index.d.ts

@@ -60,8 +60,7 @@ import { PairedSignificance, mean, pairedPassSignificance, passByBaseCase, passH
  *
  * @packageDocumentation
  */
-/** Canonical version constant. Mirrors the `package.json` version. */
-declare const VERSION = "0.6.0";
+declare const VERSION: string;
 //#endregion
 export { type AgentLike, type ArgumentValidityOptions, type Case, type CorrectToolSelectedOptions, type Dataset, type EvalCaseResult, type EvalReport, type ExactMatchOptions, type FinalStateCorrectOptions, type FromTracesOptions, type JsonPathOptions, type LlmJudgeOptions, type LoadCsvOptions, type LoadDmrOptions, type LoadJsonlOptions, type LoadLocomoOptions, type LoadLongMemEvalOptions, type MemoryEvalAbility, type MemoryEvalInput, type MemoryEvalSession, type MemoryEvalTurn, type PairedSignificance, type PrebuiltScorerOptions, type PredicateOptions, type ProgressEvent, type RecoveryAfterErrorOptions, type RedundantCallDetectionOptions, type RegexMatchOptions, type RegressionFinding, type RegressionOptions, type RegressionReport, type ReporterFormat, type RunEvalOptions, type RunOptions, type ScoreResult, type Scorer, type TraceEvent, type Trajectory, type TrajectoryToolCall, VERSION, type WriteReportsOptions, type WrittenReport, argumentValidity, correctToolSelected, detectRegressions, detectRegressions as detectRegressionsFromReports, exactMatch, exitOnFailures, factualityScorer, fenceForJudge, finalStateCorrect, fromIterable, helpfulnessScorer, jsonPath, llmJudge, loadCsvDataset, loadDatasetFromTraces, loadDmrDataset, loadJsonlDataset, loadLocomoDataset, loadLongMemEvalDataset, mean, pairedPassSignificance, parseCsv, parseDmr, parseJsonl, parseLocomo, parseLongMemEval, parseScore, passByBaseCase, passHatK, predicate, recoveryAfterError, redundantCallDetection, regexMatch, renderHtmlReport, renderJsonReport, renderJunitReport, renderMarkdownReport, renderTerminalReport, runEvals, sampleStddev, scoreContract, stripIterationSuffix, toxicityScorer, wilsonInterval, writeReports };
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../src/index.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAgCa,OAAA"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../src/index.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAkCa"}

package/dist/index.js

@@ -1,3 +1,4 @@
+import { version } from "./package.js";
 import { mean, pairedPassSignificance, passByBaseCase, passHatK, sampleStddev, stripIterationSuffix, wilsonInterval } from "./stats.js";
 import { detectRegressions } from "./regression.js";
 import { renderHtmlReport } from "./reporters/html.js";
@@ -57,8 +58,8 @@ import { redundantCallDetection } from "./scorers/trajectory/redundant-call-dete
 *
 * @packageDocumentation
 */
-/** Canonical version constant. Mirrors the `package.json` version. */
-const VERSION = "0.6.0";
+/** Canonical version constant, derived from `package.json` at build time. */
+const VERSION = version;
 
 //#endregion
 export { VERSION, argumentValidity, correctToolSelected, detectRegressions, detectRegressions as detectRegressionsFromReports, exactMatch, exitOnFailures, factualityScorer, fenceForJudge, finalStateCorrect, fromIterable, helpfulnessScorer, jsonPath, llmJudge, loadCsvDataset, loadDatasetFromTraces, loadDmrDataset, loadJsonlDataset, loadLocomoDataset, loadLongMemEvalDataset, mean, pairedPassSignificance, parseCsv, parseDmr, parseJsonl, parseLocomo, parseLongMemEval, parseScore, passByBaseCase, passHatK, predicate, recoveryAfterError, redundantCallDetection, regexMatch, renderHtmlReport, renderJsonReport, renderJunitReport, renderMarkdownReport, renderTerminalReport, runEvals, sampleStddev, scoreContract, stripIterationSuffix, toxicityScorer, wilsonInterval, writeReports };

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":[],"sources":["../src/index.ts"],"sourcesContent":["/**\n * @graphorin/evals - eval framework for the Graphorin framework.\n *\n * Builds on the eval primitives shipped from `@graphorin/observability`\n * (RB-17 / DEC-152) - full orchestrator lives here, post-MVP, decoupled\n * from the core observability bundle so consumers do not pay the\n * dataset / reporter cost when only the inline runner is needed.\n *\n * ```ts\n * import {\n *   runEvals,\n *   loadJsonlDataset,\n *   exactMatch,\n *   renderTerminalReport,\n *   exitOnFailures,\n * } from '@graphorin/evals';\n *\n * const dataset = await loadJsonlDataset('./fixtures/golden.jsonl');\n * const report = await runEvals({\n *   agent,\n *   dataset,\n *   scorers: [exactMatch()],\n *   concurrency: 4,\n * });\n * console.log(renderTerminalReport(report));\n * exitOnFailures(report);\n * ```\n *\n * @packageDocumentation\n */\n\n/** Canonical version constant. Mirrors the `package.json` version. */\nexport const VERSION = '0.6.0';\n\nexport {\n  detectRegressions,\n  exitOnFailures,\n  type ReporterFormat,\n  type WriteReportsOptions,\n  type WrittenReport,\n  writeReports,\n} from './cli/index.js';\nexport type {\n  FromTracesOptions,\n  LoadDmrOptions,\n  LoadLocomoOptions,\n  LoadLongMemEvalOptions,\n  MemoryEvalAbility,\n  MemoryEvalInput,\n  MemoryEvalSession,\n  MemoryEvalTurn,\n  TraceEvent,\n} from './loaders/index.js';\nexport {\n  fromIterable,\n  type LoadCsvOptions,\n  type LoadJsonlOptions,\n  loadCsvDataset,\n  loadDatasetFromTraces,\n  loadDmrDataset,\n  loadJsonlDataset,\n  loadLocomoDataset,\n  loadLongMemEvalDataset,\n  parseCsv,\n  parseDmr,\n  parseJsonl,\n  parseLocomo,\n  parseLongMemEval,\n} from './loaders/index.js';\nexport { detectRegressions as detectRegressionsFromReports } from './regression.js';\nexport {\n  renderHtmlReport,\n  renderJsonReport,\n  renderJunitReport,\n  renderMarkdownReport,\n  renderTerminalReport,\n} from './reporters/index.js';\nexport { runEvals } from './runner.js';\nexport {\n  type ArgumentValidityOptions,\n  argumentValidity,\n  type CorrectToolSelectedOptions,\n  correctToolSelected,\n  type ExactMatchOptions,\n  exactMatch,\n  type FinalStateCorrectOptions,\n  factualityScorer,\n  fenceForJudge,\n  finalStateCorrect,\n  helpfulnessScorer,\n  type JsonPathOptions,\n  jsonPath,\n  type LlmJudgeOptions,\n  llmJudge,\n  type PrebuiltScorerOptions,\n  type PredicateOptions,\n  parseScore,\n  predicate,\n  type RecoveryAfterErrorOptions,\n  type RedundantCallDetectionOptions,\n  type RegexMatchOptions,\n  recoveryAfterError,\n  redundantCallDetection,\n  regexMatch,\n  scoreContract,\n  type Trajectory,\n  type TrajectoryToolCall,\n  toxicityScorer,\n} from './scorers/index.js';\nexport {\n  mean,\n  type PairedSignificance,\n  pairedPassSignificance,\n  passByBaseCase,\n  passHatK,\n  sampleStddev,\n  stripIterationSuffix,\n  wilsonInterval,\n} from './stats.js';\nexport type {\n  AgentLike,\n  Case,\n  Dataset,\n  EvalCaseResult,\n  EvalReport,\n  ProgressEvent,\n  RegressionFinding,\n  RegressionOptions,\n  RegressionReport,\n  RunEvalOptions,\n  RunOptions,\n  ScoreResult,\n  Scorer,\n} from './types.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCA,MAAa,UAAU"}
+{"version":3,"file":"index.js","names":["VERSION: string","pkg.version"],"sources":["../src/index.ts"],"sourcesContent":["/**\n * @graphorin/evals - eval framework for the Graphorin framework.\n *\n * Builds on the eval primitives shipped from `@graphorin/observability`\n * (RB-17 / DEC-152) - full orchestrator lives here, post-MVP, decoupled\n * from the core observability bundle so consumers do not pay the\n * dataset / reporter cost when only the inline runner is needed.\n *\n * ```ts\n * import {\n *   runEvals,\n *   loadJsonlDataset,\n *   exactMatch,\n *   renderTerminalReport,\n *   exitOnFailures,\n * } from '@graphorin/evals';\n *\n * const dataset = await loadJsonlDataset('./fixtures/golden.jsonl');\n * const report = await runEvals({\n *   agent,\n *   dataset,\n *   scorers: [exactMatch()],\n *   concurrency: 4,\n * });\n * console.log(renderTerminalReport(report));\n * exitOnFailures(report);\n * ```\n *\n * @packageDocumentation\n */\n\n/** Canonical version constant, derived from `package.json` at build time. */\nimport pkg from '../package.json' with { type: 'json' };\n\nexport const VERSION: string = pkg.version;\n\nexport {\n  detectRegressions,\n  exitOnFailures,\n  type ReporterFormat,\n  type WriteReportsOptions,\n  type WrittenReport,\n  writeReports,\n} from './cli/index.js';\nexport type {\n  FromTracesOptions,\n  LoadDmrOptions,\n  LoadLocomoOptions,\n  LoadLongMemEvalOptions,\n  MemoryEvalAbility,\n  MemoryEvalInput,\n  MemoryEvalSession,\n  MemoryEvalTurn,\n  TraceEvent,\n} from './loaders/index.js';\nexport {\n  fromIterable,\n  type LoadCsvOptions,\n  type LoadJsonlOptions,\n  loadCsvDataset,\n  loadDatasetFromTraces,\n  loadDmrDataset,\n  loadJsonlDataset,\n  loadLocomoDataset,\n  loadLongMemEvalDataset,\n  parseCsv,\n  parseDmr,\n  parseJsonl,\n  parseLocomo,\n  parseLongMemEval,\n} from './loaders/index.js';\nexport { detectRegressions as detectRegressionsFromReports } from './regression.js';\nexport {\n  renderHtmlReport,\n  renderJsonReport,\n  renderJunitReport,\n  renderMarkdownReport,\n  renderTerminalReport,\n} from './reporters/index.js';\nexport { runEvals } from './runner.js';\nexport {\n  type ArgumentValidityOptions,\n  argumentValidity,\n  type CorrectToolSelectedOptions,\n  correctToolSelected,\n  type ExactMatchOptions,\n  exactMatch,\n  type FinalStateCorrectOptions,\n  factualityScorer,\n  fenceForJudge,\n  finalStateCorrect,\n  helpfulnessScorer,\n  type JsonPathOptions,\n  jsonPath,\n  type LlmJudgeOptions,\n  llmJudge,\n  type PrebuiltScorerOptions,\n  type PredicateOptions,\n  parseScore,\n  predicate,\n  type RecoveryAfterErrorOptions,\n  type RedundantCallDetectionOptions,\n  type RegexMatchOptions,\n  recoveryAfterError,\n  redundantCallDetection,\n  regexMatch,\n  scoreContract,\n  type Trajectory,\n  type TrajectoryToolCall,\n  toxicityScorer,\n} from './scorers/index.js';\nexport {\n  mean,\n  type PairedSignificance,\n  pairedPassSignificance,\n  passByBaseCase,\n  passHatK,\n  sampleStddev,\n  stripIterationSuffix,\n  wilsonInterval,\n} from './stats.js';\nexport type {\n  AgentLike,\n  Case,\n  Dataset,\n  EvalCaseResult,\n  EvalReport,\n  ProgressEvent,\n  RegressionFinding,\n  RegressionOptions,\n  RegressionReport,\n  RunEvalOptions,\n  RunOptions,\n  ScoreResult,\n  Scorer,\n} from './types.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,MAAaA,UAAkBC"}

package/dist/package.js

@@ -0,0 +1,6 @@
+//#region package.json
+var version = "0.6.1";
+
+//#endregion
+export { version };
+//# sourceMappingURL=package.js.map

package/dist/package.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"package.js","names":[],"sources":["../package.json"],"sourcesContent":["{\n  \"name\": \"@graphorin/evals\",\n  \"version\": \"0.6.1\",\n  \"description\": \"Eval framework for the Graphorin framework. Ships scorer libraries (`code/` exact-match, regex, JSON path, predicates; `llm/` LLM-judge; `prebuilt/` toxicity / factuality / helpfulness), dataset loaders (JSONL, CSV, from-traces, generic iterable), reporters (terminal, markdown, JSON, JUnit XML), a parallel runner with `concurrency` config, and regression detection that compares the current run against a stored baseline. Builds on the eval primitives shipped from `@graphorin/observability` (RB-17 / DEC-152 - full orchestrator lives here, post-MVP, decoupled from the core observability bundle so consumers do not pay the dataset / reporter cost when only the inline runner is needed). Created and maintained by Oleksiy Stepurenko.\",\n  \"license\": \"MIT\",\n  \"author\": \"Oleksiy Stepurenko\",\n  \"homepage\": \"https://github.com/o-stepper/graphorin/tree/main/packages/evals\",\n  \"repository\": {\n    \"type\": \"git\",\n    \"url\": \"git+https://github.com/o-stepper/graphorin.git\",\n    \"directory\": \"packages/evals\"\n  },\n  \"bugs\": {\n    \"url\": \"https://github.com/o-stepper/graphorin/issues\"\n  },\n  \"keywords\": [\n    \"graphorin\",\n    \"ai\",\n    \"agents\",\n    \"framework\",\n    \"evals\",\n    \"evaluation\",\n    \"scorers\",\n    \"llm-judge\",\n    \"datasets\",\n    \"reporters\",\n    \"regression-detection\"\n  ],\n  \"type\": \"module\",\n  \"engines\": {\n    \"node\": \">=22.0.0\"\n  },\n  \"main\": \"./dist/index.js\",\n  \"module\": \"./dist/index.js\",\n  \"types\": \"./dist/index.d.ts\",\n  \"sideEffects\": false,\n  \"exports\": {\n    \".\": {\n      \"types\": \"./dist/index.d.ts\",\n      \"import\": \"./dist/index.js\"\n    },\n    \"./scorers\": {\n      \"types\": \"./dist/scorers/index.d.ts\",\n      \"import\": \"./dist/scorers/index.js\"\n    },\n    \"./loaders\": {\n      \"types\": \"./dist/loaders/index.d.ts\",\n      \"import\": \"./dist/loaders/index.js\"\n    },\n    \"./reporters\": {\n      \"types\": \"./dist/reporters/index.d.ts\",\n      \"import\": \"./dist/reporters/index.js\"\n    },\n    \"./cli\": {\n      \"types\": \"./dist/cli/index.d.ts\",\n      \"import\": \"./dist/cli/index.js\"\n    },\n    \"./package.json\": \"./package.json\"\n  },\n  \"files\": [\n    \"dist\",\n    \"README.md\",\n    \"CHANGELOG.md\",\n    \"LICENSE\"\n  ],\n  \"scripts\": {\n    \"build\": \"tsdown\",\n    \"typecheck\": \"tsc --noEmit && tsc -p tsconfig.tests.json\",\n    \"test\": \"vitest run\",\n    \"lint\": \"biome check .\",\n    \"clean\": \"rimraf dist .turbo *.tsbuildinfo\"\n  },\n  \"dependencies\": {\n    \"@graphorin/core\": \"workspace:*\",\n    \"@graphorin/observability\": \"workspace:*\"\n  },\n  \"publishConfig\": {\n    \"access\": \"public\",\n    \"provenance\": true\n  }\n}\n"],"mappings":";cAEa"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graphorin/evals",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "Eval framework for the Graphorin framework. Ships scorer libraries (`code/` exact-match, regex, JSON path, predicates; `llm/` LLM-judge; `prebuilt/` toxicity / factuality / helpfulness), dataset loaders (JSONL, CSV, from-traces, generic iterable), reporters (terminal, markdown, JSON, JUnit XML), a parallel runner with `concurrency` config, and regression detection that compares the current run against a stored baseline. Builds on the eval primitives shipped from `@graphorin/observability` (RB-17 / DEC-152 - full orchestrator lives here, post-MVP, decoupled from the core observability bundle so consumers do not pay the dataset / reporter cost when only the inline runner is needed). Created and maintained by Oleksiy Stepurenko.",
   "license": "MIT",
   "author": "Oleksiy Stepurenko",
@@ -64,8 +64,8 @@
     "LICENSE"
   ],
   "dependencies": {
-    "@graphorin/core": "0.6.0",
-    "@graphorin/observability": "0.6.0"
+    "@graphorin/core": "0.6.1",
+    "@graphorin/observability": "0.6.1"
   },
   "publishConfig": {
     "access": "public",