evalsutra 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/.claude-plugin/plugin.json +1 -1
  2. package/CHANGELOG.md +12 -0
  3. package/CITATION.cff +1 -1
  4. package/CODE_OF_CONDUCT.md +2 -0
  5. package/CONTRIBUTING.md +2 -0
  6. package/README.md +11 -10
  7. package/SUPPORT.md +5 -0
  8. package/docs/STATUS.md +1 -1
  9. package/documents/evalsutra/01-executive-summary.md +2 -2
  10. package/documents/evalsutra/02-functional-specification.md +1 -1
  11. package/documents/evalsutra/03-architecture.md +1 -1
  12. package/documents/evalsutra/04-technical-reference.md +12 -5
  13. package/documents/evalsutra/05-user-guide.md +4 -6
  14. package/documents/evalsutra/08-testing-and-evidence.md +1 -1
  15. package/documents/evalsutra/10-ci-cd-and-release.md +9 -7
  16. package/documents/evalsutra/11-faq-and-glossary.md +1 -1
  17. package/documents/evalsutra/12-contributor-handbook.md +1 -1
  18. package/documents/evalsutra/13-roadmap-and-status.md +2 -2
  19. package/documents/evalsutra/evidence/README.md +1 -1
  20. package/documents/evalsutra/evidence/samples/audit/audit-report.html +3 -3
  21. package/documents/evalsutra/evidence/samples/audit/audit-report.json +2 -2
  22. package/documents/evalsutra/evidence/samples/audit/audit-report.md +3 -3
  23. package/documents/evalsutra/evidence/samples/bad/report.html +1 -1
  24. package/documents/evalsutra/evidence/samples/bad/report.json +1 -1
  25. package/documents/evalsutra/evidence/samples/good/report.html +1 -1
  26. package/documents/evalsutra/evidence/samples/good/report.json +1 -1
  27. package/documents/evalsutra/evidence/test-run.txt +61 -59
  28. package/documents/evalsutra/evidence/test-summary.json +3 -3
  29. package/documents/evalsutra/html/01-executive-summary.html +2 -2
  30. package/documents/evalsutra/html/02-functional-specification.html +1 -1
  31. package/documents/evalsutra/html/03-architecture.html +1 -1
  32. package/documents/evalsutra/html/04-technical-reference.html +12 -6
  33. package/documents/evalsutra/html/05-user-guide.html +4 -6
  34. package/documents/evalsutra/html/08-testing-and-evidence.html +1 -1
  35. package/documents/evalsutra/html/10-ci-cd-and-release.html +10 -9
  36. package/documents/evalsutra/html/11-faq-and-glossary.html +1 -1
  37. package/documents/evalsutra/html/12-contributor-handbook.html +1 -1
  38. package/documents/evalsutra/html/13-roadmap-and-status.html +3 -3
  39. package/documents/evalsutra/html/index.html +1 -1
  40. package/llms.txt +1 -1
  41. package/package.json +3 -3
  42. package/packages/audit/audit.mjs +2 -2
  43. package/packages/cli/evalsutra.mjs +7 -1
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "evalsutra",
3
- "version": "0.3.0",
3
+ "version": "0.4.0",
4
4
  "description": "Evidence-first agent evaluation skill for Claude Code; offline evaluator CLI is separate",
5
5
  "author": {
6
6
  "name": "Vatsal Shah"
package/CHANGELOG.md CHANGED
@@ -1,5 +1,17 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.4.0 (2026-11-16)
4
+ - **Breaking:** minimum Node.js is now 22 (`engines`). Node 20 reached end of life in April 2026.
5
+ - CI matrix now Node 22, 24 and 26 on Ubuntu, Windows, macOS; Actions upgraded to checkout v7.0.1, setup-node v7.1.0, upload-artifact v7.0.2 and pinned to commit SHAs; `persist-credentials: false`.
6
+ - Added `.github/workflows/publish.yml` (npm trusted publishing with provenance on release; needs one-time npm setup) and Dependabot for Actions.
7
+ - Docs updated for supported Node versions and release process.
8
+
9
+ ## 0.3.1 (2026-11-07)
10
+ - Fixed: published 0.3.0 package failed audit check L4 (CONTRIBUTING.md and CODE_OF_CONDUCT.md were missing from `files`); SUPPORT.md also added.
11
+ - Added `evalsutra demo [--bad]`: runs the bundled demo from any folder. README quick start used repo-relative paths that failed under npx.
12
+ - Docs, README, llms.txt now use `npx evalsutra` (npm package published as 0.3.0).
13
+ - Tests: demo from a foreign working directory; package `files` completeness.
14
+
3
15
  ## 0.3.0 (2026-11-07)
4
16
  - Added `evalsutra audit`: 29-check self-audit with Markdown, HTML and JSON reports (grader accuracy on 36 known answers, evidence rules, reproducibility, security, skills, legal).
5
17
  - Reports now use a shared accessible design system (light/dark, no script, status as text plus symbol); HTML report shows provenance and confidence interval.
package/CITATION.cff CHANGED
@@ -2,7 +2,7 @@ cff-version: 1.2.0
2
2
  message: "If you use EvalSutra AI, please cite it as below."
3
3
  title: "EvalSutra AI: The Agent Evaluation & Reliability Intelligence Platform"
4
4
  type: software
5
- version: 0.3.0
5
+ version: 0.4.0
6
6
  date-released: 2026-11-07
7
7
  license: Apache-2.0
8
8
  repository-code: "https://github.com/shahvatsalofficial/evalsutra"
@@ -0,0 +1,2 @@
1
+ # Code of Conduct
2
+ Contributors must act respectfully and professionally. Harassment, discriminatory abuse, threats and hostile personal attacks are not welcome. Report concerns privately to maintainers through the eventual repository's private moderation channels. Enforcement procedures and contacts should be finalized before the public launch.
@@ -0,0 +1,2 @@
1
+ # Contributing
2
+ Thank you for improving EvalSutra AI, created by Vatsal Shah. Please open an issue describing the problem and supporting evidence before major API changes. Fork the project; create small PRs with reproducible tests; run `npm test` and `npm run validate`; update feature status and documentation. Never submit real secrets, private training data or unlicensed benchmarks. For security-sensitive reports, follow SECURITY.md rather than a public issue. Claims that an IDE or server integration works must include reproducible proof and host versions. Follow CODE_OF_CONDUCT.md.
package/README.md CHANGED
@@ -4,13 +4,14 @@
4
4
  *Test Intelligence. Measure Truth. Build Trust.*
5
5
 
6
6
  [![CI](https://github.com/shahvatsalofficial/evalsutra/actions/workflows/ci.yml/badge.svg)](https://github.com/shahvatsalofficial/evalsutra/actions/workflows/ci.yml)
7
+ [![npm](https://img.shields.io/npm/v/evalsutra.svg)](https://www.npmjs.com/package/evalsutra)
7
8
  [![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
8
- [![Node](https://img.shields.io/badge/node-%3E%3D20-brightgreen.svg)](package.json)
9
+ [![Node](https://img.shields.io/badge/node-%3E%3D22-brightgreen.svg)](package.json)
9
10
  [![Dependencies: 0](https://img.shields.io/badge/dependencies-0-success.svg)](THIRD_PARTY_NOTICES.md)
10
11
 
11
12
  Created by **Vatsal Shah**. Apache-2.0. Free for commercial use.
12
13
 
13
- EvalSutra tells you, with evidence, whether an AI agent did what it was supposed to do. It grades **recorded observations** (final output, tool calls, latency) against rules you wrote first, and it refuses to guess: missing evidence is `inconclusive`, never a pass; a model's opinion can never prove success. It needs only Node.js 20+, no API key, no account and no network.
14
+ EvalSutra tells you, with evidence, whether an AI agent did what it was supposed to do. It grades **recorded observations** (final output, tool calls, latency) against rules you wrote first, and it refuses to guess: missing evidence is `inconclusive`, never a pass; a model's opinion can never prove success. It needs only Node.js 22+, no API key, no account and no network.
14
15
 
15
16
  > **Honest scope:** EvalSutra does **not** run your agent and does not call any model. `GO_CANDIDATE` means "your recorded evidence met your thresholds", not "safe to ship". See [what is not built yet](#status).
16
17
 
@@ -21,13 +22,13 @@ An agent that writes "Done, all tests pass" may have run no tests. Reading prose
21
22
  ## 30-second demo (no key, no install)
22
23
 
23
24
  ```bash
24
- npx github:shahvatsalofficial/evalsutra run \
25
+ npx evalsutra run \
25
26
  --suite suites/demo.json \
26
27
  --observations examples/demo-observations-bad.json \
27
28
  --out artifacts/bad
28
29
  ```
29
30
 
30
- Real output from the bundled synthetic data (exit code 2):
31
+ Real output (exit code 2):
31
32
 
32
33
  ```text
33
34
  {
@@ -51,7 +52,7 @@ You get `report.html`, `report.md`, `report.json` and `junit.xml`. Open the HTML
51
52
 
52
53
  | Claim | Where to verify it |
53
54
  |---|---|
54
- | Tests pass on Ubuntu, Windows, macOS × Node 20, 22, 24 | [GitHub Actions](https://github.com/shahvatsalofficial/evalsutra/actions) |
55
+ | Tests pass on Ubuntu, Windows, macOS × Node 22, 24, 26 | [GitHub Actions](https://github.com/shahvatsalofficial/evalsutra/actions) |
55
56
  | Raw test output and counts | [test-run.txt](documents/evalsutra/evidence/test-run.txt), [summary](documents/evalsutra/evidence/README.md) |
56
57
  | 29-check self-audit, expected vs observed | [audit-report.md](documents/evalsutra/evidence/samples/audit/audit-report.md), [HTML](documents/evalsutra/evidence/samples/audit/audit-report.html) |
57
58
  | Grader has 0 false positives and 0 false negatives on 36 known-answer cases | audit checks G1 to G3 |
@@ -78,7 +79,7 @@ Re-create everything yourself: `npm test`, `node packages/cli/evalsutra.mjs audi
78
79
  ## Commands
79
80
 
80
81
  ```text
81
- evalsutra doctor | init | validate | run | compare | report | gate
82
+ evalsutra demo | doctor | init | validate | run | compare | report | gate
82
83
  evalsutra trace analyze | repro export | repro verify | audit
83
84
  evalsutra install | uninstall | list-skills | matrix
84
85
  ```
@@ -88,8 +89,8 @@ Every command has `--help`. Exit codes: `0` ok, `1` error, `2` NO_GO / regressio
88
89
  ## Install the skills in your editor
89
90
 
90
91
  ```bash
91
- npx github:shahvatsalofficial/evalsutra install --ide claude --project . --dry-run
92
- npx github:shahvatsalofficial/evalsutra install --ide claude --project .
92
+ npx evalsutra install --ide claude --project . --dry-run
93
+ npx evalsutra install --ide claude --project .
93
94
  ```
94
95
 
95
96
  Choose `claude`, `cursor`, `codex`, `gemini` or `copilot`. It writes only inside your project, never overwrites, refuses symlink escapes, and `uninstall` keeps files you edited. Only the file copy is tested; whether your editor version discovers the skills is **not verified**. Run `evalsutra matrix`.
@@ -133,9 +134,9 @@ Everything is in [`documents/evalsutra/`](documents/evalsutra/) as Markdown with
133
134
 
134
135
  ## Status
135
136
 
136
- **Implemented and tested (v0.3.0):** offline evaluator, statuses and gates, comparison, trace analysis, reports, repro capsule, installer, 18 skills, self-audit, documentation set.
137
+ **Implemented and tested (v0.4.0):** offline evaluator, statuses and gates, comparison, trace analysis, reports, repro capsule, installer, 18 skills, self-audit, documentation set.
137
138
 
138
- **Not implemented:** live agent runner, sandbox, server, dashboard, SARIF, OpenTelemetry, Promptfoo / Inspect AI / Harbor / MCP Inspector adapters, calibrated LLM judge, `audit-plugin`, JSON Schema files, npm publication.
139
+ **Not implemented:** live agent runner, sandbox, server, dashboard, SARIF, OpenTelemetry, Promptfoo / Inspect AI / Harbor / MCP Inspector adapters, calibrated LLM judge, `audit-plugin`, JSON Schema files.
139
140
 
140
141
  **Unverified:** skill discovery inside live Cursor, Codex, Gemini CLI and Copilot; Linux and macOS results other than the public CI history.
141
142
 
package/SUPPORT.md ADDED
@@ -0,0 +1,5 @@
1
+ # Support
2
+ - Questions and bugs: GitHub Issues at https://github.com/shahvatsalofficial/evalsutra/issues
3
+ - Security: see SECURITY.md (private reporting, not public issues)
4
+ - Docs: documents/evalsutra/ (Markdown and HTML)
5
+ This is a volunteer project; no support SLA is offered.
package/docs/STATUS.md CHANGED
@@ -1,4 +1,4 @@
1
- # Feature status (v0.3.0)
1
+ # Feature status (v0.4.0)
2
2
  Superseded in detail by documents/evalsutra/13-roadmap-and-status.md.
3
3
 
4
4
  Verified locally on Windows 11, Node 24 (`npm test`: 44 pass, 1 skipped symlink test that is POSIX-only). Linux/macOS results come only from real CI runs.
@@ -8,7 +8,7 @@ Teams ship AI agents and judge them by reading the agent's own words. An agent t
8
8
 
9
9
  ## What it is
10
10
 
11
- - A command-line tool (`evalsutra`) that needs only Node.js 20 or newer. No API key, no account, no network, no dependencies.
11
+ - A command-line tool (`evalsutra`) that needs only Node.js 22 or newer. No API key, no account, no network, no dependencies.
12
12
  - A set of 18 portable specialist **skills** (instruction files) that Claude Code, Cursor, Codex, Gemini CLI and GitHub Copilot can load, so the AI in your editor follows the same evidence rules.
13
13
  - Reports in JSON, Markdown, HTML and JUnit, plus an independent-style **self-audit** report that shows the tool's own accuracy.
14
14
 
@@ -39,7 +39,7 @@ Teams ship AI agents and judge them by reading the agent's own words. An agent t
39
39
 
40
40
  ## Current maturity
41
41
 
42
- Version 0.3.0 is an early release (0.x). The evaluator, reports, installer, trace analysis, repro capsule and self-audit are implemented and tested. Live agent execution, sandboxing, a server, a dashboard and third-party integrations are **planned**. See [Roadmap and status](13-roadmap-and-status.md).
42
+ Version 0.4.0 is an early release (0.x). The evaluator, reports, installer, trace analysis, repro capsule and self-audit are implemented and tested. Live agent execution, sandboxing, a server, a dashboard and third-party integrations are **planned**. See [Roadmap and status](13-roadmap-and-status.md).
43
43
 
44
44
  ## Who should read what
45
45
 
@@ -99,6 +99,6 @@ Every `run` writes `report.json`, `report.md`, `report.html`, `junit.xml`. HTML
99
99
  |---|---|
100
100
  | Works offline, no keys | no network APIs in source [S4] |
101
101
  | Deterministic | same inputs give identical output [R1, R2] |
102
- | Cross-platform | CI on Ubuntu, Windows, macOS; Node 20, 22, 24 |
102
+ | Cross-platform | CI on Ubuntu, Windows, macOS; Node 22, 24, 26 |
103
103
  | Small attack surface | zero dependencies, no lifecycle scripts [S5, S6] |
104
104
  | Accessible output | semantic HTML, WCAG-minded colour contrast, no colour-only status |
@@ -71,7 +71,7 @@ Add an assertion type in `evaluate.mjs` (validate, evaluate, test, audit row). A
71
71
 
72
72
  ## Deployment topology
73
73
 
74
- A single Node process on the user's machine or CI runner. Distribution: GitHub source (`npx github:shahvatsalofficial/evalsutra`), and npm once the maintainer publishes. Nothing is hosted.
74
+ A single Node process on the user's machine or CI runner. Distribution: npm (`npx evalsutra`) and GitHub source (`npx github:shahvatsalofficial/evalsutra`). Nothing is hosted.
75
75
 
76
76
  ## Observability
77
77
 
@@ -2,14 +2,20 @@
2
2
 
3
3
  ## Requirements
4
4
 
5
- Node.js 20 or newer. Tested on Node 20, 22 and 24 on Ubuntu, Windows and macOS in GitHub Actions.
5
+ Node.js 22 or newer (Node 20 reached end of life in April 2026). Tested on Node 22, 24 and 26 on Ubuntu, Windows and macOS in GitHub Actions.
6
6
 
7
7
  ## Install and run
8
8
 
9
9
  ```bash
10
- # no install: run straight from GitHub
10
+ # no install: run from npm
11
+ npx evalsutra doctor
12
+
13
+ # or straight from GitHub source
11
14
  npx github:shahvatsalofficial/evalsutra doctor
12
15
 
16
+ # or install globally (your choice)
17
+ npm install -g evalsutra
18
+
13
19
  # from a clone
14
20
  git clone https://github.com/shahvatsalofficial/evalsutra.git
15
21
  cd evalsutra
@@ -17,7 +23,7 @@ npm test
17
23
  node packages/cli/evalsutra.mjs doctor
18
24
  ```
19
25
 
20
- An npm package named `evalsutra` is configured but **not yet published**. Until it is, use the GitHub form above.
26
+ Published on npm as [`evalsutra`](https://www.npmjs.com/package/evalsutra) (Apache-2.0, zero dependencies). The GitHub form always tracks the latest commit.
21
27
 
22
28
  ## Exit codes
23
29
 
@@ -33,6 +39,7 @@ An npm package named `evalsutra` is configured but **not yet published**. Until
33
39
 
34
40
  | Command | Purpose |
35
41
  |---|---|
42
+ | `demo [--bad] [--out d]` | run the bundled demo suite on bundled synthetic data from any folder; `--bad` gives exit 2 |
36
43
  | `doctor` | environment facts as JSON; states `liveAgentExecution: false` |
37
44
  | `init [--dir d]` | write starter `suite.json` and `observations.json`; refuses to overwrite |
38
45
  | `validate --suite s` | check a suite; exit 4 on errors |
@@ -117,8 +124,8 @@ Digests use key-sorted JSON, so reordering keys does not change them. [R2, R3]
117
124
  ## Use in CI
118
125
 
119
126
  ```yaml
120
- - run: npx github:shahvatsalofficial/evalsutra run --suite suites/mine.json --observations out/obs.json --out out/eval
121
- - run: npx github:shahvatsalofficial/evalsutra gate --report out/eval/report.json --policy policies/default.json
127
+ - run: npx evalsutra run --suite suites/mine.json --observations out/obs.json --out out/eval
128
+ - run: npx evalsutra gate --report out/eval/report.json --policy policies/default.json
122
129
  - uses: actions/upload-artifact@v4
123
130
  with: { name: evalsutra-report, path: out/eval }
124
131
  ```
@@ -3,17 +3,15 @@
3
3
  ## 1. Five-minute tour
4
4
 
5
5
  ```bash
6
- git clone https://github.com/shahvatsalofficial/evalsutra.git
7
- cd evalsutra
8
- node packages/cli/evalsutra.mjs run --suite suites/demo.json --observations examples/demo-observations.json --out artifacts/demo
6
+ npx evalsutra demo
9
7
  ```
10
8
 
11
- You get `artifacts/demo/report.html` (open in a browser), `report.md`, `report.json`, `junit.xml`. The gate is `GO_CANDIDATE` because the sample evidence is clean.
9
+ You get `artifacts/demo/report.html` (in the folder where you ran it) (open in a browser), `report.md`, `report.json`, `junit.xml`. The gate is `GO_CANDIDATE` because the sample evidence is clean.
12
10
 
13
11
  Now the failing sample:
14
12
 
15
13
  ```bash
16
- node packages/cli/evalsutra.mjs run --suite suites/demo.json --observations examples/demo-observations-bad.json --out artifacts/bad
14
+ npx evalsutra demo --bad
17
15
  ```
18
16
 
19
17
  Exit code is 2 and the gate is `NO_GO`: the agent used a forbidden tool and leaked a canary value. One more case had no latency, so it is `inconclusive` rather than guessed.
@@ -93,4 +91,4 @@ See [Technical reference](04-technical-reference.md#use-in-ci).
93
91
  | `Unsupported --ide` | typo | use claude, cursor, codex, gemini or copilot |
94
92
  | `runner "x" is not_supported` | only replay exists | use `--runner replay` |
95
93
  | Gate stays `REVIEW` | an inconclusive case | add the missing observation field |
96
- | `npx` cannot find command | package not on npm yet | use `npx github:shahvatsalofficial/evalsutra` |
94
+ | `npx` cannot find command | package not on npm yet | use `npx evalsutra` |
@@ -15,7 +15,7 @@ Uses the built-in `node:test` runner. No test framework is installed.
15
15
 
16
16
  The numbers are generated, not typed. See [evidence/README.md](evidence/README.md) for the table, [test-run.txt](evidence/test-run.txt) for the full output and [test-summary.json](evidence/test-summary.json) for machine-readable counts.
17
17
 
18
- Operating-system coverage comes from GitHub Actions: Ubuntu, Windows, macOS × Node 20, 22, 24 (9 jobs). Check the live status at [Actions](https://github.com/shahvatsalofficial/evalsutra/actions). A badge is only trustworthy because it links to those real runs.
18
+ Operating-system coverage comes from GitHub Actions: Ubuntu, Windows, macOS × Node 22, 24, 26 (9 jobs). Check the live status at [Actions](https://github.com/shahvatsalofficial/evalsutra/actions). A badge is only trustworthy because it links to those real runs.
19
19
 
20
20
  ## What the tests cover
21
21
 
@@ -2,11 +2,11 @@
2
2
 
3
3
  ## Pipeline
4
4
 
5
- File: `.github/workflows/ci.yml`. Triggers: push, pull request. Permissions: `contents: read` only.
5
+ Files: `.github/workflows/ci.yml` (tests) and `publish.yml` (npm release). Triggers: push and pull request for CI; a published GitHub release for npm. CI permissions: `contents: read` only. Actions are pinned to full commit SHAs (checkout v7.0.1, setup-node v7.1.0, upload-artifact v7.0.2); Dependabot proposes weekly updates.
6
6
 
7
7
  | Job | Matrix | Steps |
8
8
  |---|---|---|
9
- | `tests` | Ubuntu, Windows, macOS × Node 20, 22, 24 | `npm test`, `npm run validate`, `npm run doctor`, `audit`, docs freshness check |
9
+ | `tests` | Ubuntu, Windows, macOS × Node 22, 24, 26 | `npm test`, `npm run validate`, `npm run doctor`, `audit`, docs freshness check |
10
10
 
11
11
  No secrets are used. No network is needed by the tests. A failing job blocks nothing automatically unless branch protection requires it; enable branch protection with the `tests` checks required.
12
12
 
@@ -14,6 +14,7 @@ No secrets are used. No network is needed by the tests. A failing job blocks not
14
14
 
15
15
  | Incident | Cause | Fix |
16
16
  |---|---|---|
17
+ | 0.3.0 npm package: README demo failed outside a clone; audit L4 failed | demo used repo-relative paths; CONTRIBUTING.md and CODE_OF_CONDUCT.md not in `files` | added `demo` command; added files; tests assert both |
17
18
  | Windows + Node 20 job failed with `Could not find ...tests\*.test.mjs` | PowerShell does not expand globs and Node 20 does not either | `npm test` now runs `node --test`, which discovers files itself |
18
19
 
19
20
  ## Release checklist
@@ -29,15 +30,16 @@ No secrets are used. No network is needed by the tests. A failing job blocks not
29
30
 
30
31
  ## npm publish (maintainer action)
31
32
 
32
- The package name `evalsutra` is set in `package.json`. Availability on npm was not verified. Do not publish before checking `npm view evalsutra`. Publishing needs the maintainer's own npm login with two-factor authentication.
33
+ Published as [`evalsutra`](https://www.npmjs.com/package/evalsutra). A manual `npm publish` needs the maintainer's login and two-factor code. The repository also contains `.github/workflows/publish.yml`, which publishes with npm **trusted publishing** (OIDC, provenance, no stored token) when a GitHub release is published. **One-time maintainer setup** on npmjs.com: package `evalsutra`, Settings, Trusted Publisher, GitHub Actions, owner `shahvatsalofficial`, repository `evalsutra`, workflow `publish.yml`. Until that is configured, the workflow fails and the manual route above is the one that works. The workflow has not yet run successfully.
33
34
 
34
35
  ```bash
35
- npm view evalsutra # expect E404 if the name is free
36
- npm pack --dry-run # inspect the file list
36
+ npm test && node packages/cli/evalsutra.mjs audit # must be green
37
+ npm pack --dry-run # inspect the file list
37
38
  npm publish --access public
39
+ npx -y evalsutra@X.Y.Z doctor # verify from a clean folder
38
40
  ```
39
41
 
40
- Prefer npm **trusted publishing** from GitHub Actions (OIDC, provenance) over a long-lived token once the first version exists.
42
+ Verify the published tarball, not just the repo: a clean-folder run of `npx evalsutra demo` and `npx evalsutra audit` found two packaging bugs in 0.3.0 (see Lessons). Prefer npm **trusted publishing** from GitHub Actions (OIDC, provenance) over a long-lived token.
41
43
 
42
44
  ## Versioning
43
45
 
@@ -45,4 +47,4 @@ Semantic versioning. 0.x means the CLI and report format may still change; chang
45
47
 
46
48
  ## Hardening backlog
47
49
 
48
- Pin Actions to commit SHAs, add dependency review, secret scanning (Gitleaks), CodeQL, OpenSSF Scorecard, SBOM, signed releases. None of these is claimed as done.
50
+ Done: Actions pinned to SHAs, Dependabot for Actions, provenance publish workflow (untested). Still open: dependency review, secret scanning (Gitleaks), CodeQL, OpenSSF Scorecard, SBOM, signed releases. None of the open items is claimed as done.
@@ -20,7 +20,7 @@
20
20
 
21
21
  **How do I add my own check?** See [Contributor handbook](12-contributor-handbook.md).
22
22
 
23
- **Is it on npm?** Not yet. Use `npx github:shahvatsalofficial/evalsutra`.
23
+ **Is it on npm?** Yes: `npx evalsutra` (package `evalsutra`). The GitHub form `npx github:shahvatsalofficial/evalsutra` also works.
24
24
 
25
25
  **How do I know the samples are not faked?** Every sample file is produced by `scripts/build-evidence.mjs` from the shipped CLI. Re-run it and diff.
26
26
 
@@ -8,7 +8,7 @@ cd evalsutra
8
8
  npm test
9
9
  ```
10
10
 
11
- Node 20+. No `npm install` is needed because there are no dependencies.
11
+ Node 22+. No `npm install` is needed because there are no dependencies.
12
12
 
13
13
  ## Layout
14
14
 
@@ -2,7 +2,7 @@
2
2
 
3
3
  Legend: **Done** = implemented and tested. **Planned** = not implemented. Nothing marked planned should be described to users as available.
4
4
 
5
- ## Status matrix (v0.3.0)
5
+ ## Status matrix (v0.4.0)
6
6
 
7
7
  | Capability | Status |
8
8
  |---|---|
@@ -28,7 +28,7 @@ Legend: **Done** = implemented and tested. **Planned** = not implemented. Nothin
28
28
  | Property-based and fuzz tests | Planned |
29
29
  | Server API and accessible dashboard | Planned |
30
30
  | Verified live skill discovery in each IDE | Planned |
31
- | npm publication | Pending maintainer action |
31
+ | npm publication | Done (`evalsutra` on npm); trusted publishing and provenance planned |
32
32
  | Signed releases, SBOM, Scorecard | Planned |
33
33
 
34
34
  ## Next milestones
@@ -6,7 +6,7 @@ Generated by `node scripts/build-evidence.mjs` on Node v24.15.0 (win32 x64). Not
6
6
 
7
7
  | Tests | Passed | Failed | Skipped | Cancelled | Exit code |
8
8
  |---|---|---|---|---|---|
9
- | 55 | 54 | 0 | 1 | 0 | 0 |
9
+ | 57 | 56 | 0 | 1 | 0 | 0 |
10
10
 
11
11
  Full output: [test-run.txt](test-run.txt). Machine-readable: [test-summary.json](test-summary.json).
12
12
 
@@ -38,9 +38,9 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
38
38
  @media (prefers-reduced-motion:reduce){html{scroll-behavior:auto}}
39
39
  </style></head><body><a class="skip" href="#main">Skip to content</a>
40
40
  <header class="site"><div><strong>EvalSutra AI</strong></div></header>
41
- <main id="main"><section class="hero"><p class="tag">Independent self-audit</p><h1>EvalSutra audit: PASS</h1><p>29 of 29 checks passed. Self-audit of the EvalSutra package files. It audits the tool, not any third-party agent.</p><p><small>evalsutra@0.3.0 · Node v24.15.0 · win32</small></p></section>
41
+ <main id="main"><section class="hero"><p class="tag">Independent self-audit</p><h1>EvalSutra audit: PASS</h1><p>29 of 29 checks passed. Self-audit of the EvalSutra package files. It audits the tool, not any third-party agent.</p><p><small>evalsutra@0.4.0 · Node v24.15.0 · win32</small></p></section>
42
42
  <h2>Summary by category</h2><div class="grid"><div class="card"><h3>Grader accuracy</h3><div class="stat">3/3<small>checks passed</small></div></div><div class="card"><h3>Evidence rules</h3><div class="stat">7/7<small>checks passed</small></div></div><div class="card"><h3>Reproducibility</h3><div class="stat">3/3<small>checks passed</small></div></div><div class="card"><h3>Trace analysis</h3><div class="stat">2/2<small>checks passed</small></div></div><div class="card"><h3>Security</h3><div class="stat">6/6<small>checks passed</small></div></div><div class="card"><h3>Skills</h3><div class="stat">4/4<small>checks passed</small></div></div><div class="card"><h3>Legal</h3><div class="stat">4/4<small>checks passed</small></div></div></div>
43
- <h2>All checks</h2><div class="tablewrap"><table><caption>Every check with expected and observed value. Result is text plus symbol.</caption><thead><tr><th>ID</th><th>Category</th><th>Check</th><th>Expected</th><th>Observed</th><th>Result</th></tr></thead><tbody><tr><th scope="row">G1</th><td>Grader accuracy</td><td>Known-answer table (36 cases)<br><small>all rows matched ground truth</small></td><td>0 wrong</td><td>0 wrong</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">G2</th><td>Grader accuracy</td><td>False positives (graded pass, truth not pass)</td><td>0</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">G3</th><td>Grader accuracy</td><td>False negatives (graded not-pass, truth pass)</td><td>0</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E1</th><td>Evidence rules</td><td>No observations at all must never pass</td><td>REVIEW, passRate null, coverage 0</td><td>REVIEW, null, 0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E2</th><td>Evidence rules</td><td>Model-judge-only evidence cannot reach GO_CANDIDATE</td><td>not GO_CANDIDATE</td><td>REVIEW</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E3</th><td>Evidence rules</td><td>All-skipped suite must not pass</td><td>not GO_CANDIDATE, 0 passed</td><td>REVIEW, 0 passed</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E4</th><td>Evidence rules</td><td>Forbidden tool + canary leak is NO_GO with a critical failure</td><td>NO_GO, 1 critical</td><td>NO_GO, 1 critical</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E5</th><td>Evidence rules</td><td>Clean demo evidence is GO_CANDIDATE (and only a candidate)</td><td>GO_CANDIDATE</td><td>GO_CANDIDATE</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E6</th><td>Evidence rules</td><td>Regression between good and bad run is detected</td><td>verdict FAIL</td><td>FAIL</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E7</th><td>Evidence rules</td><td>Case id &quot;constructor&quot; does not read inherited object members</td><td>inconclusive</td><td>inconclusive</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">R1</th><td>Reproducibility</td><td>Same inputs give byte-identical results</td><td>equal</td><td>equal</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">R2</th><td>Reproducibility</td><td>Input digest ignores JSON key order</td><td>equal</td><td>equal</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">R3</th><td>Reproducibility</td><td>Changing one value changes the digest</td><td>different</td><td>different</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">T1</th><td>Trace analysis</td><td>Success claim after failed last tool is flagged</td><td>UNSUPPORTED_SUCCESS_CLAIM</td><td>UNSUPPORTED_SUCCESS_CLAIM</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">T2</th><td>Trace analysis</td><td>Empty trace is inconclusive, never pass</td><td>inconclusive</td><td>inconclusive</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S1</th><td>Security</td><td>Report HTML/XML escape hostile suite names</td><td>no raw &lt;img</td><td>escaped</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S2</th><td>Security</td><td>Shell metacharacters in fixtures are inert data</td><td>valid, nothing executed</td><td>valid</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S3</th><td>Security</td><td>Secret scan over 136 text files</td><td>0 files with credential patterns</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S4</th><td>Security</td><td>No network/process/dynamic-code APIs in 10 source files</td><td>0 hits</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S5</th><td>Security</td><td>Zero runtime dependencies (no supply-chain surface)</td><td>0</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S6</th><td>Security</td><td>No install-time lifecycle scripts</td><td>0</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">K1</th><td>Skills</td><td>Eighteen specialist skills present</td><td>18</td><td>18</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">K2</th><td>Skills</td><td>Valid frontmatter (name matches folder, description present)</td><td>0 invalid</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">K3</th><td>Skills</td><td>Every skill states objective, procedure, contract and stop conditions</td><td>0 missing</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">K4</th><td>Skills</td><td>Plugin copy equals canonical copy</td><td>0 drift</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">L1</th><td>Legal</td><td>Apache-2.0 LICENSE text present</td><td>contains &quot;Apache License&quot; and &quot;Version 2.0&quot;</td><td>yes</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">L2</th><td>Legal</td><td>package.json license is Apache-2.0</td><td>Apache-2.0</td><td>Apache-2.0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">L3</th><td>Legal</td><td>NOTICE and THIRD_PARTY_NOTICES present</td><td>both exist</td><td>true/true</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">L4</th><td>Legal</td><td>SECURITY, CONTRIBUTING, CODE_OF_CONDUCT, CHANGELOG present</td><td>all exist</td><td>checked</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr></tbody></table></div>
43
+ <h2>All checks</h2><div class="tablewrap"><table><caption>Every check with expected and observed value. Result is text plus symbol.</caption><thead><tr><th>ID</th><th>Category</th><th>Check</th><th>Expected</th><th>Observed</th><th>Result</th></tr></thead><tbody><tr><th scope="row">G1</th><td>Grader accuracy</td><td>Known-answer table (36 cases)<br><small>all rows matched ground truth</small></td><td>0 wrong</td><td>0 wrong</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">G2</th><td>Grader accuracy</td><td>False positives (graded pass, truth not pass)</td><td>0</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">G3</th><td>Grader accuracy</td><td>False negatives (graded not-pass, truth pass)</td><td>0</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E1</th><td>Evidence rules</td><td>No observations at all must never pass</td><td>REVIEW, passRate null, coverage 0</td><td>REVIEW, null, 0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E2</th><td>Evidence rules</td><td>Model-judge-only evidence cannot reach GO_CANDIDATE</td><td>not GO_CANDIDATE</td><td>REVIEW</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E3</th><td>Evidence rules</td><td>All-skipped suite must not pass</td><td>not GO_CANDIDATE, 0 passed</td><td>REVIEW, 0 passed</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E4</th><td>Evidence rules</td><td>Forbidden tool + canary leak is NO_GO with a critical failure</td><td>NO_GO, 1 critical</td><td>NO_GO, 1 critical</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E5</th><td>Evidence rules</td><td>Clean demo evidence is GO_CANDIDATE (and only a candidate)</td><td>GO_CANDIDATE</td><td>GO_CANDIDATE</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E6</th><td>Evidence rules</td><td>Regression between good and bad run is detected</td><td>verdict FAIL</td><td>FAIL</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">E7</th><td>Evidence rules</td><td>Case id &quot;constructor&quot; does not read inherited object members</td><td>inconclusive</td><td>inconclusive</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">R1</th><td>Reproducibility</td><td>Same inputs give byte-identical results</td><td>equal</td><td>equal</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">R2</th><td>Reproducibility</td><td>Input digest ignores JSON key order</td><td>equal</td><td>equal</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">R3</th><td>Reproducibility</td><td>Changing one value changes the digest</td><td>different</td><td>different</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">T1</th><td>Trace analysis</td><td>Success claim after failed last tool is flagged</td><td>UNSUPPORTED_SUCCESS_CLAIM</td><td>UNSUPPORTED_SUCCESS_CLAIM</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">T2</th><td>Trace analysis</td><td>Empty trace is inconclusive, never pass</td><td>inconclusive</td><td>inconclusive</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S1</th><td>Security</td><td>Report HTML/XML escape hostile suite names</td><td>no raw &lt;img</td><td>escaped</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S2</th><td>Security</td><td>Shell metacharacters in fixtures are inert data</td><td>valid, nothing executed</td><td>valid</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S3</th><td>Security</td><td>Secret scan over 138 text files</td><td>0 files with credential patterns</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S4</th><td>Security</td><td>No network/process/dynamic-code APIs in 10 source files</td><td>0 hits</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S5</th><td>Security</td><td>Zero runtime dependencies (no supply-chain surface)</td><td>0</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">S6</th><td>Security</td><td>No install-time lifecycle scripts</td><td>0</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">K1</th><td>Skills</td><td>Eighteen specialist skills present</td><td>18</td><td>18</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">K2</th><td>Skills</td><td>Valid frontmatter (name matches folder, description present)</td><td>0 invalid</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">K3</th><td>Skills</td><td>Every skill states objective, procedure, contract and stop conditions</td><td>0 missing</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">K4</th><td>Skills</td><td>Plugin copy equals canonical copy</td><td>0 drift</td><td>0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">L1</th><td>Legal</td><td>Apache-2.0 LICENSE text present</td><td>contains &quot;Apache License&quot; and &quot;Version 2.0&quot;</td><td>yes</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">L2</th><td>Legal</td><td>package.json license is Apache-2.0</td><td>Apache-2.0</td><td>Apache-2.0</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">L3</th><td>Legal</td><td>NOTICE and THIRD_PARTY_NOTICES present</td><td>both exist</td><td>true/true</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr><tr><th scope="row">L4</th><td>Legal</td><td>SECURITY, CONTRIBUTING, CODE_OF_CONDUCT, CHANGELOG present</td><td>all exist</td><td>checked</td><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td></tr></tbody></table></div>
44
44
  <h2>Limits of this audit</h2><ul><li>Runs in-process with the code it audits, so it is a regression and integrity check, not an independent third-party attestation.</li><li>Known-answer table is finite; it proves listed behaviours only.</li><li>Does not test live IDEs, live models or other operating systems; see CI for OS results.</li></ul>
45
- <h2>Reproduce</h2><pre><code>npx github:shahvatsalofficial/evalsutra audit --out audit</code></pre></main>
45
+ <h2>Reproduce</h2><pre><code>npx evalsutra audit --out audit</code></pre></main>
46
46
  <footer>EvalSutra AI · Created by Vatsal Shah · Apache-2.0 · Generated offline; no scripts, no trackers, no network calls.</footer></body></html>
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "schemaVersion": 1,
3
- "tool": "evalsutra@0.3.0",
3
+ "tool": "evalsutra@0.4.0",
4
4
  "node": "v24.15.0",
5
5
  "platform": "win32",
6
6
  "scope": "Self-audit of the EvalSutra package files. It audits the tool, not any third-party agent.",
@@ -208,7 +208,7 @@
208
208
  {
209
209
  "category": "Security",
210
210
  "id": "S3",
211
- "title": "Secret scan over 136 text files",
211
+ "title": "Secret scan over 138 text files",
212
212
  "expected": "0 files with credential patterns",
213
213
  "observed": "0",
214
214
  "status": "pass",
@@ -1,7 +1,7 @@
1
1
  # EvalSutra self-audit report
2
2
 
3
3
  **Verdict: PASS** — 29/29 checks passed, 0 failed.
4
- Tool: `evalsutra@0.3.0` · Node v24.15.0 · win32
4
+ Tool: `evalsutra@0.4.0` · Node v24.15.0 · win32
5
5
 
6
6
  > Self-audit of the EvalSutra package files. It audits the tool, not any third-party agent.
7
7
 
@@ -38,7 +38,7 @@ Tool: `evalsutra@0.3.0` · Node v24.15.0 · win32
38
38
  | T2 | Trace analysis | Empty trace is inconclusive, never pass | inconclusive | inconclusive | PASS |
39
39
  | S1 | Security | Report HTML/XML escape hostile suite names | no raw <img | escaped | PASS |
40
40
  | S2 | Security | Shell metacharacters in fixtures are inert data | valid, nothing executed | valid | PASS |
41
- | S3 | Security | Secret scan over 136 text files | 0 files with credential patterns | 0 | PASS |
41
+ | S3 | Security | Secret scan over 138 text files | 0 files with credential patterns | 0 | PASS |
42
42
  | S4 | Security | No network/process/dynamic-code APIs in 10 source files | 0 hits | 0 | PASS |
43
43
  | S5 | Security | Zero runtime dependencies (no supply-chain surface) | 0 | 0 | PASS |
44
44
  | S6 | Security | No install-time lifecycle scripts | 0 | 0 | PASS |
@@ -60,5 +60,5 @@ Tool: `evalsutra@0.3.0` · Node v24.15.0 · win32
60
60
  ## Reproduce
61
61
 
62
62
  ```bash
63
- npx github:shahvatsalofficial/evalsutra audit --out audit
63
+ npx evalsutra audit --out audit
64
64
  ```
@@ -42,6 +42,6 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
42
42
  <div class="grid"><div class="card"><div class="stat">77.8%<small>pass rate (scored assertions only)</small></div></div><div class="card"><div class="stat">90.0%<small>evidence coverage</small></div></div><div class="card"><div class="stat">4/6<small>cases passed</small></div></div><div class="card"><div class="stat">1<small>failed</small></div></div><div class="card"><div class="stat">1<small>inconclusive / error</small></div></div><div class="card"><div class="stat">1<small>critical failures</small></div></div></div>
43
43
  <p>Case-level pass rate 95% Wilson interval: <strong>37.6% – 96.4%</strong> (n=5).</p>
44
44
  <h2>Cases</h2><div class="tablewrap"><table><caption>Status is shown as text and symbol.</caption><thead><tr><th>Case</th><th>Status</th><th>Critical</th><th>Evidence source</th><th>Assertions</th></tr></thead><tbody><tr><th scope="row">factual-answer</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>yes</td><td>test_fixture</td><td>normalized_equals: pass</td></tr><tr><th scope="row">json-contract</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>no</td><td>test_fixture</td><td>json_has_keys: pass</td></tr><tr><th scope="row">version-format</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>no</td><td>test_fixture</td><td>regex: pass</td></tr><tr><th scope="row">tool-order</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>no</td><td>test_fixture</td><td>tool_order: pass<br>max_steps: pass</td></tr><tr><th scope="row">forbidden-tool</th><td><span class="badge b-fail"><span aria-hidden="true">✖</span> FAIL</span></td><td>yes</td><td>test_fixture</td><td>forbidden_tools: fail (MISMATCH)<br>not_contains: fail (MISMATCH)</td></tr><tr><th scope="row">latency</th><td><span class="badge b-inconclusive"><span aria-hidden="true">?</span> INCONCLUSIVE</span></td><td>no</td><td>unspecified</td><td>max_latency_ms: inconclusive (LATENCY_MISSING)</td></tr></tbody></table></div>
45
- <h2>Provenance</h2><div class="tablewrap"><table><tbody><tr><th scope="row">source</th><td><code>supplied local JSON observations</code></td></tr><tr><th scope="row">suiteDigest</th><td><code>0294091e9fbba23c63d11d4c3b37f30126471c5f6f5f382760a6f39acb0b64fe</code></td></tr><tr><th scope="row">observationsDigest</th><td><code>7ca88745ddf4f82d01ce5054872e1871a7bb08aee48c3403c6c96d4062faf78d</code></td></tr><tr><th scope="row">tool</th><td><code>evalsutra@0.3.0</code></td></tr><tr><th scope="row">node</th><td><code>v24.15.0</code></td></tr><tr><th scope="row">platform</th><td><code>win32</code></td></tr><tr><th scope="row">scoring</th><td><code>scoring-v2</code></td></tr><tr><th scope="row">reportDigest</th><td><code>53bdcca24a31f3cd3d72d85cbc244d974f8b22d119a0b63e7e752ffd680a04d1</code></td></tr></tbody></table></div>
45
+ <h2>Provenance</h2><div class="tablewrap"><table><tbody><tr><th scope="row">source</th><td><code>supplied local JSON observations</code></td></tr><tr><th scope="row">suiteDigest</th><td><code>0294091e9fbba23c63d11d4c3b37f30126471c5f6f5f382760a6f39acb0b64fe</code></td></tr><tr><th scope="row">observationsDigest</th><td><code>7ca88745ddf4f82d01ce5054872e1871a7bb08aee48c3403c6c96d4062faf78d</code></td></tr><tr><th scope="row">tool</th><td><code>evalsutra@0.4.0</code></td></tr><tr><th scope="row">node</th><td><code>v24.15.0</code></td></tr><tr><th scope="row">platform</th><td><code>win32</code></td></tr><tr><th scope="row">scoring</th><td><code>scoring-v2</code></td></tr><tr><th scope="row">reportDigest</th><td><code>53bdcca24a31f3cd3d72d85cbc244d974f8b22d119a0b63e7e752ffd680a04d1</code></td></tr></tbody></table></div>
46
46
  <blockquote>GO_CANDIDATE is a recommendation on recorded evidence. It is not deployment approval. Human review remains required.</blockquote></main>
47
47
  <footer>EvalSutra AI · Created by Vatsal Shah · Apache-2.0 · Generated offline; no scripts, no trackers, no network calls.</footer></body></html>
@@ -8,7 +8,7 @@
8
8
  "source": "supplied local JSON observations",
9
9
  "suiteDigest": "0294091e9fbba23c63d11d4c3b37f30126471c5f6f5f382760a6f39acb0b64fe",
10
10
  "observationsDigest": "7ca88745ddf4f82d01ce5054872e1871a7bb08aee48c3403c6c96d4062faf78d",
11
- "tool": "evalsutra@0.3.0",
11
+ "tool": "evalsutra@0.4.0",
12
12
  "node": "v24.15.0",
13
13
  "platform": "win32"
14
14
  },
@@ -42,6 +42,6 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
42
42
  <div class="grid"><div class="card"><div class="stat">100.0%<small>pass rate (scored assertions only)</small></div></div><div class="card"><div class="stat">100.0%<small>evidence coverage</small></div></div><div class="card"><div class="stat">6/6<small>cases passed</small></div></div><div class="card"><div class="stat">0<small>failed</small></div></div><div class="card"><div class="stat">0<small>inconclusive / error</small></div></div><div class="card"><div class="stat">0<small>critical failures</small></div></div></div>
43
43
  <p>Case-level pass rate 95% Wilson interval: <strong>61.0% – 100.0%</strong> (n=6).</p>
44
44
  <h2>Cases</h2><div class="tablewrap"><table><caption>Status is shown as text and symbol.</caption><thead><tr><th>Case</th><th>Status</th><th>Critical</th><th>Evidence source</th><th>Assertions</th></tr></thead><tbody><tr><th scope="row">factual-answer</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>yes</td><td>test_fixture</td><td>normalized_equals: pass</td></tr><tr><th scope="row">json-contract</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>no</td><td>test_fixture</td><td>json_has_keys: pass</td></tr><tr><th scope="row">version-format</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>no</td><td>test_fixture</td><td>regex: pass</td></tr><tr><th scope="row">tool-order</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>no</td><td>test_fixture</td><td>tool_order: pass<br>max_steps: pass</td></tr><tr><th scope="row">forbidden-tool</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>yes</td><td>test_fixture</td><td>forbidden_tools: pass<br>not_contains: pass</td></tr><tr><th scope="row">latency</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>no</td><td>test_fixture</td><td>max_latency_ms: pass</td></tr></tbody></table></div>
45
- <h2>Provenance</h2><div class="tablewrap"><table><tbody><tr><th scope="row">source</th><td><code>supplied local JSON observations</code></td></tr><tr><th scope="row">suiteDigest</th><td><code>0294091e9fbba23c63d11d4c3b37f30126471c5f6f5f382760a6f39acb0b64fe</code></td></tr><tr><th scope="row">observationsDigest</th><td><code>c69a7a0788ab8499bdbbf71082aa1208c158cfa9e9ea2eb7f10712f0804d925b</code></td></tr><tr><th scope="row">tool</th><td><code>evalsutra@0.3.0</code></td></tr><tr><th scope="row">node</th><td><code>v24.15.0</code></td></tr><tr><th scope="row">platform</th><td><code>win32</code></td></tr><tr><th scope="row">scoring</th><td><code>scoring-v2</code></td></tr><tr><th scope="row">reportDigest</th><td><code>582cd02a454d1a733d7b41426a4b1bf40d3e96ffedfef20d7bf631fbf6d6fa92</code></td></tr></tbody></table></div>
45
+ <h2>Provenance</h2><div class="tablewrap"><table><tbody><tr><th scope="row">source</th><td><code>supplied local JSON observations</code></td></tr><tr><th scope="row">suiteDigest</th><td><code>0294091e9fbba23c63d11d4c3b37f30126471c5f6f5f382760a6f39acb0b64fe</code></td></tr><tr><th scope="row">observationsDigest</th><td><code>c69a7a0788ab8499bdbbf71082aa1208c158cfa9e9ea2eb7f10712f0804d925b</code></td></tr><tr><th scope="row">tool</th><td><code>evalsutra@0.4.0</code></td></tr><tr><th scope="row">node</th><td><code>v24.15.0</code></td></tr><tr><th scope="row">platform</th><td><code>win32</code></td></tr><tr><th scope="row">scoring</th><td><code>scoring-v2</code></td></tr><tr><th scope="row">reportDigest</th><td><code>582cd02a454d1a733d7b41426a4b1bf40d3e96ffedfef20d7bf631fbf6d6fa92</code></td></tr></tbody></table></div>
46
46
  <blockquote>GO_CANDIDATE is a recommendation on recorded evidence. It is not deployment approval. Human review remains required.</blockquote></main>
47
47
  <footer>EvalSutra AI · Created by Vatsal Shah · Apache-2.0 · Generated offline; no scripts, no trackers, no network calls.</footer></body></html>
@@ -8,7 +8,7 @@
8
8
  "source": "supplied local JSON observations",
9
9
  "suiteDigest": "0294091e9fbba23c63d11d4c3b37f30126471c5f6f5f382760a6f39acb0b64fe",
10
10
  "observationsDigest": "c69a7a0788ab8499bdbbf71082aa1208c158cfa9e9ea2eb7f10712f0804d925b",
11
- "tool": "evalsutra@0.3.0",
11
+ "tool": "evalsutra@0.4.0",
12
12
  "node": "v24.15.0",
13
13
  "platform": "win32"
14
14
  },
@@ -1,69 +1,71 @@
1
- EvalSutra 0.3.0 test run
1
+ EvalSutra 0.4.0 test run
2
2
  Node v24.15.0 on win32 x64
3
3
  Command: node --test --test-reporter=spec
4
4
  Exit code: 0
5
5
  Note: durations vary per run; this file is regenerated by scripts/build-evidence.mjs.
6
6
  ------------------------------------------------------------
7
- ✔ audit passes on the real tree (53.2171ms)
8
- ✔ audit has teeth: planted token fails S3 (1417.2926ms)
9
- ✔ audit has teeth: removed skill fails K1 (849.5589ms)
10
- ✔ audit has teeth: banned API in source fails S4 (1640.3106ms)
11
- ✔ audit has teeth: added dependency fails S5 (1180.8623ms)
12
- ✔ audit reports render, escape and list every check (75.0766ms)
13
- ✔ CLI audit writes md, html, json (990.2031ms)
14
- ✔ markdown converter escapes html and blocks javascript links (4.0612ms)
15
- ✔ docs: committed HTML matches Markdown (30.7872ms)
16
- ✔ docs: every relative link and anchor resolves (11.9032ms)
17
- ✔ doctor advertises offline replay, not live execution (1025.4897ms)
18
- ✔ CLI run writes attributable reports (833.9136ms)
19
- ✔ project-local install copies skill and rejects clobbering (1957.5333ms)
20
- ✔ unknown IDE returns error (721.0372ms)
21
- ✔ CLI compares candidate and baseline runs (1758.5176ms)
22
- ✔ suite validates (4.9299ms)
23
- ✔ invalid suite rejected (0.4849ms)
24
- ✔ duplicate ID rejected (0.2359ms)
25
- ✔ ground truth passes (1.2717ms)
26
- ✔ wrong answer fails (0.2097ms)
27
- ✔ missing evidence inconclusive (0.1768ms)
28
- ✔ required tool checks (0.2207ms)
29
- ✔ forbidden tool detection (0.2118ms)
30
- ✔ tool ordering (0.195ms)
31
- ✔ maximum steps (0.3329ms)
32
- ✔ latency absent inconclusive (0.1569ms)
33
- ✔ sample gate is candidate (1.2262ms)
34
- ✔ critical failure is no-go (0.4642ms)
35
- ✔ critical missing evidence requires review (0.3158ms)
36
- ✔ correct case counts (1.8779ms)
37
- ✔ missing observations input rejected (0.5001ms)
38
- ✔ regression detected (0.7858ms)
39
- ✔ inconclusive cannot get perfect coverage (0.2494ms)
40
- ✔ demo suite validates and passes (4.5936ms)
41
- ✔ forbidden tool + canary leak => NO_GO critical (0.5679ms)
42
- ✔ missing latency stays inconclusive (0.4796ms)
43
- ✔ normalized_equals (0.2353ms)
44
- ✔ regex and unsafe regex (0.6091ms)
45
- ✔ json_has_keys rejects invalid JSON (0.2972ms)
46
- ✔ model_judge source cannot prove a pass (0.5162ms)
47
- ✔ skipped case is not counted as pass (0.3752ms)
48
- ✔ wilson and passAtK (0.5858ms)
49
- ✔ redaction hides tokens and keys (0.6441ms)
50
- ✔ digest is key-order independent (1.0434ms)
51
- ✔ compare: FAIL on regression, INCONCLUSIVE with few pairs, PASS otherwise (0.9577ms)
52
- ✔ trace analysis flags loop, unsupported success claim, forbidden tool (1.1392ms)
53
- ✔ reporters escape HTML/XML (1.0637ms)
54
- ✔ CLI run writes 4 files, exit codes, dry-run writes none (2613.8431ms)
55
- ✔ CLI invalid evidence exits 4 (742.819ms)
56
- ✔ CLI gate applies policy (1708.3619ms)
57
- ✔ repro export/verify detects tampering (2988.5792ms)
58
- ✔ install all skills, dry-run, uninstall respects edits (3529.0866ms)
59
- ﹣ install refuses symlink escape (0.1871ms) # SKIP
60
- ✔ skills: canonical and plugin copies identical, valid frontmatter (625.3399ms)
61
- ✔ trace CLI exit code (665.1679ms)
62
- ℹ tests 55
7
+ ✔ audit passes on the real tree (153.3786ms)
8
+ ✔ audit has teeth: planted token fails S3 (1638.0384ms)
9
+ ✔ audit has teeth: removed skill fails K1 (2303.1733ms)
10
+ ✔ audit has teeth: banned API in source fails S4 (2634.4925ms)
11
+ ✔ audit has teeth: added dependency fails S5 (2874.2611ms)
12
+ ✔ audit reports render, escape and list every check (61.0174ms)
13
+ ✔ CLI audit writes md, html, json (3210.6888ms)
14
+ ✔ markdown converter escapes html and blocks javascript links (2.5446ms)
15
+ ✔ docs: committed HTML matches Markdown (16.2805ms)
16
+ ✔ docs: every relative link and anchor resolves (6.985ms)
17
+ ✔ demo command works from any working directory and honors --bad (5676.8772ms)
18
+ ✔ package.json files list ships everything the audit requires (0.905ms)
19
+ ✔ doctor advertises offline replay, not live execution (2707.9122ms)
20
+ ✔ CLI run writes attributable reports (3800.277ms)
21
+ ✔ project-local install copies skill and rejects clobbering (4707.7336ms)
22
+ ✔ unknown IDE returns error (2803.7346ms)
23
+ ✔ CLI compares candidate and baseline runs (4112.5781ms)
24
+ ✔ suite validates (5.2865ms)
25
+ ✔ invalid suite rejected (0.9028ms)
26
+ ✔ duplicate ID rejected (0.9906ms)
27
+ ✔ ground truth passes (1.0409ms)
28
+ ✔ wrong answer fails (0.3054ms)
29
+ ✔ missing evidence inconclusive (0.3011ms)
30
+ ✔ required tool checks (0.3529ms)
31
+ ✔ forbidden tool detection (0.3358ms)
32
+ ✔ tool ordering (0.3023ms)
33
+ ✔ maximum steps (0.8352ms)
34
+ ✔ latency absent inconclusive (0.3147ms)
35
+ ✔ sample gate is candidate (2.2649ms)
36
+ ✔ critical failure is no-go (0.5843ms)
37
+ ✔ critical missing evidence requires review (0.485ms)
38
+ ✔ correct case counts (2.5626ms)
39
+ ✔ missing observations input rejected (3.3601ms)
40
+ ✔ regression detected (2.0214ms)
41
+ ✔ inconclusive cannot get perfect coverage (0.4591ms)
42
+ ✔ demo suite validates and passes (17.4169ms)
43
+ ✔ forbidden tool + canary leak => NO_GO critical (2.3269ms)
44
+ ✔ missing latency stays inconclusive (2.0582ms)
45
+ ✔ normalized_equals (0.7367ms)
46
+ ✔ regex and unsafe regex (1.0229ms)
47
+ ✔ json_has_keys rejects invalid JSON (0.4701ms)
48
+ ✔ model_judge source cannot prove a pass (0.7675ms)
49
+ ✔ skipped case is not counted as pass (0.4935ms)
50
+ ✔ wilson and passAtK (0.8009ms)
51
+ ✔ redaction hides tokens and keys (0.94ms)
52
+ ✔ digest is key-order independent (1.3515ms)
53
+ ✔ compare: FAIL on regression, INCONCLUSIVE with few pairs, PASS otherwise (1.2807ms)
54
+ ✔ trace analysis flags loop, unsupported success claim, forbidden tool (1.4807ms)
55
+ ✔ reporters escape HTML/XML (1.4804ms)
56
+ ✔ CLI run writes 4 files, exit codes, dry-run writes none (10097.6744ms)
57
+ ✔ CLI invalid evidence exits 4 (2262.4824ms)
58
+ ✔ CLI gate applies policy (3032.7754ms)
59
+ ✔ repro export/verify detects tampering (8842.9296ms)
60
+ ✔ install all skills, dry-run, uninstall respects edits (5917.3008ms)
61
+ ﹣ install refuses symlink escape (0.223ms) # SKIP
62
+ ✔ skills: canonical and plugin copies identical, valid frontmatter (1360.3815ms)
63
+ ✔ trace CLI exit code (1774.2769ms)
64
+ ℹ tests 57
63
65
  ℹ suites 0
64
- ℹ pass 54
66
+ ℹ pass 56
65
67
  ℹ fail 0
66
68
  ℹ cancelled 0
67
69
  ℹ skipped 1
68
70
  ℹ todo 0
69
- ℹ duration_ms 15927.3127
71
+ ℹ duration_ms 41796.2335
@@ -1,10 +1,10 @@
1
1
  {
2
- "version": "0.3.0",
2
+ "version": "0.4.0",
3
3
  "node": "v24.15.0",
4
4
  "platform": "win32",
5
5
  "exitCode": 0,
6
- "tests": 55,
7
- "pass": 54,
6
+ "tests": 57,
7
+ "pass": 56,
8
8
  "fail": 0,
9
9
  "skipped": 1,
10
10
  "cancelled": 0
@@ -43,7 +43,7 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
43
43
  <h2 id="what-problem-it-solves">What problem it solves</h2>
44
44
  <p>Teams ship AI agents and judge them by reading the agent&#39;s own words. An agent that says &quot;Done, all tests pass&quot; may have run no tests. Prose is not proof. EvalSutra grades <strong>recorded evidence</strong> (final output, tool calls, timing) against rules you write in advance, and refuses to guess when evidence is missing.</p>
45
45
  <h2 id="what-it-is">What it is</h2>
46
- <ul><li>A command-line tool (<code>evalsutra</code>) that needs only Node.js 20 or newer. No API key, no account, no network, no dependencies.</li><li>A set of 18 portable specialist <strong>skills</strong> (instruction files) that Claude Code, Cursor, Codex, Gemini CLI and GitHub Copilot can load, so the AI in your editor follows the same evidence rules.</li><li>Reports in JSON, Markdown, HTML and JUnit, plus an independent-style <strong>self-audit</strong> report that shows the tool&#39;s own accuracy.</li></ul>
46
+ <ul><li>A command-line tool (<code>evalsutra</code>) that needs only Node.js 22 or newer. No API key, no account, no network, no dependencies.</li><li>A set of 18 portable specialist <strong>skills</strong> (instruction files) that Claude Code, Cursor, Codex, Gemini CLI and GitHub Copilot can load, so the AI in your editor follows the same evidence rules.</li><li>Reports in JSON, Markdown, HTML and JUnit, plus an independent-style <strong>self-audit</strong> report that shows the tool&#39;s own accuracy.</li></ul>
47
47
  <h2 id="what-it-is-not">What it is not</h2>
48
48
  <ul><li>It does <strong>not</strong> run your agent. It evaluates observations you supply. Live execution is on the roadmap, not in this release.</li><li>It does <strong>not</strong> certify safety. <code>GO_CANDIDATE</code> means &quot;recorded evidence met your thresholds&quot;, never &quot;approved for production&quot;.</li><li>It does <strong>not</strong> call any AI model, so it cannot be fooled by a persuasive answer.</li></ul>
49
49
  <h2 id="why-trust-it">Why trust it</h2>
@@ -51,7 +51,7 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
51
51
  <h2 id="business-value">Business value</h2>
52
52
  <ol><li><strong>Catch regressions before users do.</strong> Compare two runs on the same cases; a pass-to-fail change fails the build.</li><li><strong>Stop score inflation.</strong> Missing data is &quot;inconclusive&quot;, never a pass. Model-judge opinions cannot prove a pass.</li><li><strong>Cheap to adopt.</strong> One command, no vendor, no cost. Works inside CI and offline.</li><li><strong>Audit trail.</strong> Every report carries SHA-256 digests of its inputs, tool version, Node version and platform.</li></ol>
53
53
  <h2 id="current-maturity">Current maturity</h2>
54
- <p>Version 0.3.0 is an early release (0.x). The evaluator, reports, installer, trace analysis, repro capsule and self-audit are implemented and tested. Live agent execution, sandboxing, a server, a dashboard and third-party integrations are <strong>planned</strong>. See <a href="13-roadmap-and-status.html">Roadmap and status</a>.</p>
54
+ <p>Version 0.4.0 is an early release (0.x). The evaluator, reports, installer, trace analysis, repro capsule and self-audit are implemented and tested. Live agent execution, sandboxing, a server, a dashboard and third-party integrations are <strong>planned</strong>. See <a href="13-roadmap-and-status.html">Roadmap and status</a>.</p>
55
55
  <h2 id="who-should-read-what">Who should read what</h2>
56
56
  <div class="tablewrap"><table><thead><tr><th>Reader</th><th>Start here</th></tr></thead><tbody><tr><td>Executive, buyer</td><td>this page, then <a href="07-security-and-legal.html">Security and legal</a></td></tr><tr><td>Product, QA</td><td><a href="02-functional-specification.html">Functional specification</a>, <a href="05-user-guide.html">User guide</a></td></tr><tr><td>Engineer, architect</td><td><a href="03-architecture.html">Architecture</a>, <a href="04-technical-reference.html">Technical reference</a></td></tr><tr><td>Security, compliance</td><td><a href="07-security-and-legal.html">Security and legal</a>, <a href="09-audit-and-trust.html">Audit</a></td></tr><tr><td>Contributor</td><td><a href="12-contributor-handbook.html">Contributor handbook</a></td></tr></tbody></table></div></article></main>
57
57
  <footer>EvalSutra AI · Created by Vatsal Shah · Apache-2.0 · Generated offline; no scripts, no trackers, no network calls.</footer></body></html>
@@ -71,5 +71,5 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
71
71
  <h2 id="self-audit">Self-audit</h2>
72
72
  <p><code>audit</code> runs 29 checks on the tool itself and writes MD, HTML and JSON. See <a href="09-audit-and-trust.html">Audit and trust</a>.</p>
73
73
  <h2 id="non-functional-requirements">Non-functional requirements</h2>
74
- <div class="tablewrap"><table><thead><tr><th>Requirement</th><th>How it is met</th></tr></thead><tbody><tr><td>Works offline, no keys</td><td>no network APIs in source [S4]</td></tr><tr><td>Deterministic</td><td>same inputs give identical output [R1, R2]</td></tr><tr><td>Cross-platform</td><td>CI on Ubuntu, Windows, macOS; Node 20, 22, 24</td></tr><tr><td>Small attack surface</td><td>zero dependencies, no lifecycle scripts [S5, S6]</td></tr><tr><td>Accessible output</td><td>semantic HTML, WCAG-minded colour contrast, no colour-only status</td></tr></tbody></table></div></article></main>
74
+ <div class="tablewrap"><table><thead><tr><th>Requirement</th><th>How it is met</th></tr></thead><tbody><tr><td>Works offline, no keys</td><td>no network APIs in source [S4]</td></tr><tr><td>Deterministic</td><td>same inputs give identical output [R1, R2]</td></tr><tr><td>Cross-platform</td><td>CI on Ubuntu, Windows, macOS; Node 22, 24, 26</td></tr><tr><td>Small attack surface</td><td>zero dependencies, no lifecycle scripts [S5, S6]</td></tr><tr><td>Accessible output</td><td>semantic HTML, WCAG-minded colour contrast, no colour-only status</td></tr></tbody></table></div></article></main>
75
75
  <footer>EvalSutra AI · Created by Vatsal Shah · Apache-2.0 · Generated offline; no scripts, no trackers, no network calls.</footer></body></html>
@@ -54,7 +54,7 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
54
54
  <h2 id="extension-points">Extension points</h2>
55
55
  <p>Add an assertion type in <code>evaluate.mjs</code> (validate, evaluate, test, audit row). Add a reporter in <code>packages/reporters/</code>. Add a skill folder under <code>.agents/skills/</code> and run the sync (see <a href="12-contributor-handbook.html">Contributor handbook</a>). Planned adapters (OpenTelemetry, Promptfoo, Inspect AI, Harbor, MCP Inspector) are described in <a href="13-roadmap-and-status.html">Roadmap and status</a> and are <strong>not implemented</strong>.</p>
56
56
  <h2 id="deployment-topology">Deployment topology</h2>
57
- <p>A single Node process on the user&#39;s machine or CI runner. Distribution: GitHub source (<code>npx github:shahvatsalofficial/evalsutra</code>), and npm once the maintainer publishes. Nothing is hosted.</p>
57
+ <p>A single Node process on the user&#39;s machine or CI runner. Distribution: npm (<code>npx evalsutra</code>) and GitHub source (<code>npx github:shahvatsalofficial/evalsutra</code>). Nothing is hosted.</p>
58
58
  <h2 id="observability">Observability</h2>
59
59
  <p>Output is the report. Exit codes are the machine interface. There is no telemetry.</p></article></main>
60
60
  <footer>EvalSutra AI · Created by Vatsal Shah · Apache-2.0 · Generated offline; no scripts, no trackers, no network calls.</footer></body></html>
@@ -40,21 +40,27 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
40
40
  <header class="site"><div><strong>EvalSutra AI</strong><nav aria-label="Documentation"><a href="01-executive-summary.html">Executive summary</a><a href="02-functional-specification.html">Functional specification</a><a href="03-architecture.html">Architecture</a><a href="04-technical-reference.html" aria-current="page">Technical reference</a><a href="05-user-guide.html">User guide</a><a href="06-skills-catalog.html">Skills catalog</a><a href="07-security-and-legal.html">Security and legal</a><a href="index.html">All docs</a></nav></div></header>
41
41
  <main id="main"><article><h1 id="4-technical-reference">4. Technical reference</h1><nav class="toc" aria-label="On this page"><strong>On this page</strong><ul><li><a href="#requirements">Requirements</a></li><li><a href="#install-and-run">Install and run</a></li><li><a href="#exit-codes">Exit codes</a></li><li><a href="#commands">Commands</a></li><li><a href="#suite-format">Suite format</a></li><li><a href="#observations-format">Observations format</a></li><li><a href="#trace-format">Trace format</a></li><li><a href="#report-format-report-json-schemaversion-2">Report format (report.json, schemaVersion 2)</a></li><li><a href="#policy-file-gate-policy">Policy file (gate --policy)</a></li><li><a href="#use-in-ci">Use in CI</a></li><li><a href="#statistics">Statistics</a></li><li><a href="#security-helpers">Security helpers</a></li><li><a href="#known-limits">Known limits</a></li></ul></nav>
42
42
  <h2 id="requirements">Requirements</h2>
43
- <p>Node.js 20 or newer. Tested on Node 20, 22 and 24 on Ubuntu, Windows and macOS in GitHub Actions.</p>
43
+ <p>Node.js 22 or newer (Node 20 reached end of life in April 2026). Tested on Node 22, 24 and 26 on Ubuntu, Windows and macOS in GitHub Actions.</p>
44
44
  <h2 id="install-and-run">Install and run</h2>
45
- <pre><code># no install: run straight from GitHub
45
+ <pre><code># no install: run from npm
46
+ npx evalsutra doctor
47
+
48
+ # or straight from GitHub source
46
49
  npx github:shahvatsalofficial/evalsutra doctor
47
50
 
51
+ # or install globally (your choice)
52
+ npm install -g evalsutra
53
+
48
54
  # from a clone
49
55
  git clone https://github.com/shahvatsalofficial/evalsutra.git
50
56
  cd evalsutra
51
57
  npm test
52
58
  node packages/cli/evalsutra.mjs doctor</code></pre>
53
- <p>An npm package named <code>evalsutra</code> is configured but <strong>not yet published</strong>. Until it is, use the GitHub form above.</p>
59
+ <p>Published on npm as <a href="https://www.npmjs.com/package/evalsutra"><code>evalsutra</code></a> (Apache-2.0, zero dependencies). The GitHub form always tracks the latest commit.</p>
54
60
  <h2 id="exit-codes">Exit codes</h2>
55
61
  <div class="tablewrap"><table><thead><tr><th>Code</th><th>Meaning</th></tr></thead><tbody><tr><td>0</td><td>success, <code>GO_CANDIDATE</code>, compare <code>PASS</code>, audit <code>PASS</code></td></tr><tr><td>1</td><td>usage or runtime error (bad flag, unreadable file, refused overwrite)</td></tr><tr><td>2</td><td><code>NO_GO</code>, regression, trace failure, audit <code>FAIL</code></td></tr><tr><td>3</td><td><code>REVIEW</code>, compare <code>INCONCLUSIVE</code>, trace inconclusive</td></tr><tr><td>4</td><td>invalid evidence: bad suite, bad observations, capsule hash mismatch</td></tr></tbody></table></div>
56
62
  <h2 id="commands">Commands</h2>
57
- <div class="tablewrap"><table><thead><tr><th>Command</th><th>Purpose</th></tr></thead><tbody><tr><td><code>doctor</code></td><td>environment facts as JSON; states <code>liveAgentExecution: false</code></td></tr><tr><td><code>init [--dir d]</code></td><td>write starter <code>suite.json</code> and <code>observations.json</code>; refuses to overwrite</td></tr><tr><td><code>validate --suite s</code></td><td>check a suite; exit 4 on errors</td></tr><tr><td><code>run --suite s --observations o --out d [--runner replay] [--dry-run]</code></td><td>evaluate; write 4 report files</td></tr><tr><td><code>compare --baseline a.json --candidate b.json [--out f]</code></td><td>paired comparison</td></tr><tr><td><code>report --in report.json --format md|html|junit</code></td><td>re-render a report to stdout</td></tr><tr><td><code>gate --report r.json [--policy p.json]</code></td><td>apply a stricter policy</td></tr><tr><td><code>trace analyze --input t.json [--forbid a,b] [--allow a,b]</code></td><td>tool trace findings</td></tr><tr><td><code>repro export --suite s --observations o --report r --out d</code></td><td>build capsule</td></tr><tr><td><code>repro verify dir</code></td><td>check capsule hashes</td></tr><tr><td><code>audit [--out d]</code></td><td>self-audit; writes <code>audit-report.json/.md/.html</code></td></tr><tr><td><code>install --ide x --project p [--skills a,b] [--dry-run]</code></td><td>copy skills (x: claude, cursor, codex, gemini, copilot)</td></tr><tr><td><code>uninstall --ide x --project p [--dry-run]</code></td><td>remove unmodified EvalSutra files</td></tr><tr><td><code>list-skills</code></td><td>print the 18 skill names</td></tr><tr><td><code>matrix</code></td><td>host compatibility matrix: verified vs unknown</td></tr></tbody></table></div>
63
+ <div class="tablewrap"><table><thead><tr><th>Command</th><th>Purpose</th></tr></thead><tbody><tr><td><code>demo [--bad] [--out d]</code></td><td>run the bundled demo suite on bundled synthetic data from any folder; <code>--bad</code> gives exit 2</td></tr><tr><td><code>doctor</code></td><td>environment facts as JSON; states <code>liveAgentExecution: false</code></td></tr><tr><td><code>init [--dir d]</code></td><td>write starter <code>suite.json</code> and <code>observations.json</code>; refuses to overwrite</td></tr><tr><td><code>validate --suite s</code></td><td>check a suite; exit 4 on errors</td></tr><tr><td><code>run --suite s --observations o --out d [--runner replay] [--dry-run]</code></td><td>evaluate; write 4 report files</td></tr><tr><td><code>compare --baseline a.json --candidate b.json [--out f]</code></td><td>paired comparison</td></tr><tr><td><code>report --in report.json --format md|html|junit</code></td><td>re-render a report to stdout</td></tr><tr><td><code>gate --report r.json [--policy p.json]</code></td><td>apply a stricter policy</td></tr><tr><td><code>trace analyze --input t.json [--forbid a,b] [--allow a,b]</code></td><td>tool trace findings</td></tr><tr><td><code>repro export --suite s --observations o --report r --out d</code></td><td>build capsule</td></tr><tr><td><code>repro verify dir</code></td><td>check capsule hashes</td></tr><tr><td><code>audit [--out d]</code></td><td>self-audit; writes <code>audit-report.json/.md/.html</code></td></tr><tr><td><code>install --ide x --project p [--skills a,b] [--dry-run]</code></td><td>copy skills (x: claude, cursor, codex, gemini, copilot)</td></tr><tr><td><code>uninstall --ide x --project p [--dry-run]</code></td><td>remove unmodified EvalSutra files</td></tr><tr><td><code>list-skills</code></td><td>print the 18 skill names</td></tr><tr><td><code>matrix</code></td><td>host compatibility matrix: verified vs unknown</td></tr></tbody></table></div>
58
64
  <p>Every command accepts <code>--help</code>. <code>--runner</code> other than <code>replay</code> returns exit 1 with <code>not_supported</code>.</p>
59
65
  <h2 id="suite-format">Suite format</h2>
60
66
  <pre><code>{
@@ -89,8 +95,8 @@ node packages/cli/evalsutra.mjs doctor</code></pre>
89
95
  <h2 id="policy-file-gate-policy">Policy file (<code>gate --policy</code>)</h2>
90
96
  <pre><code>{ &quot;minPassRate&quot;: 0.9, &quot;minCoverage&quot;: 0.9, &quot;maxInconclusive&quot;: 0 }</code></pre>
91
97
  <h2 id="use-in-ci">Use in CI</h2>
92
- <pre><code>- run: npx github:shahvatsalofficial/evalsutra run --suite suites/mine.json --observations out/obs.json --out out/eval
93
- - run: npx github:shahvatsalofficial/evalsutra gate --report out/eval/report.json --policy policies/default.json
98
+ <pre><code>- run: npx evalsutra run --suite suites/mine.json --observations out/obs.json --out out/eval
99
+ - run: npx evalsutra gate --report out/eval/report.json --policy policies/default.json
94
100
  - uses: actions/upload-artifact@v4
95
101
  with: { name: evalsutra-report, path: out/eval }</code></pre>
96
102
  <p>A non-zero exit fails the job. Exit 3 (<code>REVIEW</code>) is also non-zero, so missing evidence blocks the build.</p>
@@ -40,12 +40,10 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
40
40
  <header class="site"><div><strong>EvalSutra AI</strong><nav aria-label="Documentation"><a href="01-executive-summary.html">Executive summary</a><a href="02-functional-specification.html">Functional specification</a><a href="03-architecture.html">Architecture</a><a href="04-technical-reference.html">Technical reference</a><a href="05-user-guide.html" aria-current="page">User guide</a><a href="06-skills-catalog.html">Skills catalog</a><a href="07-security-and-legal.html">Security and legal</a><a href="index.html">All docs</a></nav></div></header>
41
41
  <main id="main"><article><h1 id="5-user-guide">5. User guide</h1><nav class="toc" aria-label="On this page"><strong>On this page</strong><ul><li><a href="#1-five-minute-tour">1. Five-minute tour</a></li><li><a href="#2-evaluate-your-own-agent">2. Evaluate your own agent</a></li><li><a href="#3-read-the-result">3. Read the result</a></li><li><a href="#4-compare-versions">4. Compare versions</a></li><li><a href="#5-analyse-a-tool-trace">5. Analyse a tool trace</a></li><li><a href="#6-install-the-skills-in-your-editor">6. Install the skills in your editor</a></li><li><a href="#7-prove-the-tool-itself-works">7. Prove the tool itself works</a></li><li><a href="#8-use-in-ci">8. Use in CI</a></li><li><a href="#troubleshooting">Troubleshooting</a></li></ul></nav>
42
42
  <h2 id="1-five-minute-tour">1. Five-minute tour</h2>
43
- <pre><code>git clone https://github.com/shahvatsalofficial/evalsutra.git
44
- cd evalsutra
45
- node packages/cli/evalsutra.mjs run --suite suites/demo.json --observations examples/demo-observations.json --out artifacts/demo</code></pre>
46
- <p>You get <code>artifacts/demo/report.html</code> (open in a browser), <code>report.md</code>, <code>report.json</code>, <code>junit.xml</code>. The gate is <code>GO_CANDIDATE</code> because the sample evidence is clean.</p>
43
+ <pre><code>npx evalsutra demo</code></pre>
44
+ <p>You get <code>artifacts/demo/report.html</code> (in the folder where you ran it) (open in a browser), <code>report.md</code>, <code>report.json</code>, <code>junit.xml</code>. The gate is <code>GO_CANDIDATE</code> because the sample evidence is clean.</p>
47
45
  <p>Now the failing sample:</p>
48
- <pre><code>node packages/cli/evalsutra.mjs run --suite suites/demo.json --observations examples/demo-observations-bad.json --out artifacts/bad</code></pre>
46
+ <pre><code>npx evalsutra demo --bad</code></pre>
49
47
  <p>Exit code is 2 and the gate is <code>NO_GO</code>: the agent used a forbidden tool and leaked a canary value. One more case had no latency, so it is <code>inconclusive</code> rather than guessed.</p>
50
48
  <h2 id="2-evaluate-your-own-agent">2. Evaluate your own agent</h2>
51
49
  <ol><li>Run <code>evalsutra init --dir my-eval</code> to get a starter suite and observations.</li><li>Edit <code>suite.json</code>. Write cases <strong>before</strong> looking at results. Mark safety cases <code>critical</code>.</li><li>Record what your agent did for each case: the final text, the list of tool names it called, and the latency. Save as <code>observations.json</code>. Set <code>sourceType</code> honestly (<code>actual_run</code> if you ran it, <code>human_label</code> if a person judged it).</li><li><code>evalsutra validate --suite my-eval/suite.json</code></li><li><code>evalsutra run --suite my-eval/suite.json --observations my-eval/observations.json --out my-eval/out</code></li><li>Open <code>my-eval/out/report.html</code>.</li></ol>
@@ -70,5 +68,5 @@ evalsutra install --ide claude --project /path/to/project</code></pre>
70
68
  <h2 id="8-use-in-ci">8. Use in CI</h2>
71
69
  <p>See <a href="04-technical-reference.html#use-in-ci">Technical reference</a>.</p>
72
70
  <h2 id="troubleshooting">Troubleshooting</h2>
73
- <div class="tablewrap"><table><thead><tr><th>Symptom</th><th>Cause</th><th>Fix</th></tr></thead><tbody><tr><td><code>Invalid evidence: ...</code> exit 4</td><td>suite or observations malformed</td><td>run <code>validate</code>, read each message</td></tr><tr><td><code>Refusing to overwrite</code></td><td>file or skill already exists</td><td>choose another folder or remove the old one</td></tr><tr><td><code>Unsupported --ide</code></td><td>typo</td><td>use claude, cursor, codex, gemini or copilot</td></tr><tr><td><code>runner &quot;x&quot; is not_supported</code></td><td>only replay exists</td><td>use <code>--runner replay</code></td></tr><tr><td>Gate stays <code>REVIEW</code></td><td>an inconclusive case</td><td>add the missing observation field</td></tr><tr><td><code>npx</code> cannot find command</td><td>package not on npm yet</td><td>use <code>npx github:shahvatsalofficial/evalsutra</code></td></tr></tbody></table></div></article></main>
71
+ <div class="tablewrap"><table><thead><tr><th>Symptom</th><th>Cause</th><th>Fix</th></tr></thead><tbody><tr><td><code>Invalid evidence: ...</code> exit 4</td><td>suite or observations malformed</td><td>run <code>validate</code>, read each message</td></tr><tr><td><code>Refusing to overwrite</code></td><td>file or skill already exists</td><td>choose another folder or remove the old one</td></tr><tr><td><code>Unsupported --ide</code></td><td>typo</td><td>use claude, cursor, codex, gemini or copilot</td></tr><tr><td><code>runner &quot;x&quot; is not_supported</code></td><td>only replay exists</td><td>use <code>--runner replay</code></td></tr><tr><td>Gate stays <code>REVIEW</code></td><td>an inconclusive case</td><td>add the missing observation field</td></tr><tr><td><code>npx</code> cannot find command</td><td>package not on npm yet</td><td>use <code>npx evalsutra</code></td></tr></tbody></table></div></article></main>
74
72
  <footer>EvalSutra AI · Created by Vatsal Shah · Apache-2.0 · Generated offline; no scripts, no trackers, no network calls.</footer></body></html>
@@ -47,7 +47,7 @@ node scripts/build-evidence.mjs # regenerate the committed proof</cod
47
47
  <p>Uses the built-in <code>node:test</code> runner. No test framework is installed.</p>
48
48
  <h2 id="latest-recorded-results">Latest recorded results</h2>
49
49
  <p>The numbers are generated, not typed. See <a href="../evidence/README.html">evidence/README.md</a> for the table, <a href="../evidence/test-run.txt">test-run.txt</a> for the full output and <a href="../evidence/test-summary.json">test-summary.json</a> for machine-readable counts.</p>
50
- <p>Operating-system coverage comes from GitHub Actions: Ubuntu, Windows, macOS × Node 20, 22, 24 (9 jobs). Check the live status at <a href="https://github.com/shahvatsalofficial/evalsutra/actions">Actions</a>. A badge is only trustworthy because it links to those real runs.</p>
50
+ <p>Operating-system coverage comes from GitHub Actions: Ubuntu, Windows, macOS × Node 22, 24, 26 (9 jobs). Check the live status at <a href="https://github.com/shahvatsalofficial/evalsutra/actions">Actions</a>. A badge is only trustworthy because it links to those real runs.</p>
51
51
  <h2 id="what-the-tests-cover">What the tests cover</h2>
52
52
  <div class="tablewrap"><table><thead><tr><th>Area</th><th>Examples</th></tr></thead><tbody><tr><td>Assertions</td><td>every type has pass, fail and missing-evidence cases; unsafe and invalid regex; invalid JSON; array instead of object</td></tr><tr><td>Evidence rules</td><td>model-judge cannot pass; skipped is not pass; empty observations; inherited property ids</td></tr><tr><td>Scoring</td><td>coverage and pass rate separate; critical failure forces <code>NO_GO</code></td></tr><tr><td>Statistics</td><td>Wilson interval (including zero trials), pass@k edge cases and invalid input</td></tr><tr><td>Compare</td><td>regression gives <code>FAIL</code>; too few pairs or mismatched cases gives <code>INCONCLUSIVE</code></td></tr><tr><td>Trace</td><td>loops, forbidden tool, unsupported success claim, empty trace</td></tr><tr><td>Reporters</td><td>HTML and XML escaping of hostile names</td></tr><tr><td>CLI</td><td>exit codes 0, 1, 2, 3, 4; <code>--dry-run</code> writes nothing; unknown runner rejected; missing required flag</td></tr><tr><td>Repro capsule</td><td>export, verify, and detection of a modified file</td></tr><tr><td>Installer</td><td>dry-run, no-clobber, uninstall keeps edited files, symlink escape refused (POSIX)</td></tr><tr><td>Skills</td><td>18 present, valid frontmatter, canonical equals plugin copy</td></tr><tr><td>Audit</td><td>passes on a clean tree; <strong>fails</strong> when a token is planted or a skill removed</td></tr><tr><td>Docs</td><td>HTML matches Markdown; internal links resolve</td></tr></tbody></table></div>
53
53
  <h2 id="sample-evaluation-synthetic">Sample evaluation (synthetic)</h2>
@@ -40,21 +40,22 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
40
40
  <header class="site"><div><strong>EvalSutra AI</strong><nav aria-label="Documentation"><a href="01-executive-summary.html">Executive summary</a><a href="02-functional-specification.html">Functional specification</a><a href="03-architecture.html">Architecture</a><a href="04-technical-reference.html">Technical reference</a><a href="05-user-guide.html">User guide</a><a href="06-skills-catalog.html">Skills catalog</a><a href="07-security-and-legal.html">Security and legal</a><a href="index.html">All docs</a></nav></div></header>
41
41
  <main id="main"><article><h1 id="10-ci-cd-and-release">10. CI/CD and release</h1><nav class="toc" aria-label="On this page"><strong>On this page</strong><ul><li><a href="#pipeline">Pipeline</a></li><li><a href="#lessons-recorded">Lessons recorded</a></li><li><a href="#release-checklist">Release checklist</a></li><li><a href="#npm-publish-maintainer-action">npm publish (maintainer action)</a></li><li><a href="#versioning">Versioning</a></li><li><a href="#hardening-backlog">Hardening backlog</a></li></ul></nav>
42
42
  <h2 id="pipeline">Pipeline</h2>
43
- <p>File: <code>.github/workflows/ci.yml</code>. Triggers: push, pull request. Permissions: <code>contents: read</code> only.</p>
44
- <div class="tablewrap"><table><thead><tr><th>Job</th><th>Matrix</th><th>Steps</th></tr></thead><tbody><tr><td><code>tests</code></td><td>Ubuntu, Windows, macOS × Node 20, 22, 24</td><td><code>npm test</code>, <code>npm run validate</code>, <code>npm run doctor</code>, <code>audit</code>, docs freshness check</td></tr></tbody></table></div>
43
+ <p>Files: <code>.github/workflows/ci.yml</code> (tests) and <code>publish.yml</code> (npm release). Triggers: push and pull request for CI; a published GitHub release for npm. CI permissions: <code>contents: read</code> only. Actions are pinned to full commit SHAs (checkout v7.0.1, setup-node v7.1.0, upload-artifact v7.0.2); Dependabot proposes weekly updates.</p>
44
+ <div class="tablewrap"><table><thead><tr><th>Job</th><th>Matrix</th><th>Steps</th></tr></thead><tbody><tr><td><code>tests</code></td><td>Ubuntu, Windows, macOS × Node 22, 24, 26</td><td><code>npm test</code>, <code>npm run validate</code>, <code>npm run doctor</code>, <code>audit</code>, docs freshness check</td></tr></tbody></table></div>
45
45
  <p>No secrets are used. No network is needed by the tests. A failing job blocks nothing automatically unless branch protection requires it; enable branch protection with the <code>tests</code> checks required.</p>
46
46
  <h2 id="lessons-recorded">Lessons recorded</h2>
47
- <div class="tablewrap"><table><thead><tr><th>Incident</th><th>Cause</th><th>Fix</th></tr></thead><tbody><tr><td>Windows + Node 20 job failed with <code>Could not find ...tests\*.test.mjs</code></td><td>PowerShell does not expand globs and Node 20 does not either</td><td><code>npm test</code> now runs <code>node --test</code>, which discovers files itself</td></tr></tbody></table></div>
47
+ <div class="tablewrap"><table><thead><tr><th>Incident</th><th>Cause</th><th>Fix</th></tr></thead><tbody><tr><td>0.3.0 npm package: README demo failed outside a clone; audit L4 failed</td><td>demo used repo-relative paths; CONTRIBUTING.md and CODE_OF_CONDUCT.md not in <code>files</code></td><td>added <code>demo</code> command; added files; tests assert both</td></tr><tr><td>Windows + Node 20 job failed with <code>Could not find ...tests\*.test.mjs</code></td><td>PowerShell does not expand globs and Node 20 does not either</td><td><code>npm test</code> now runs <code>node --test</code>, which discovers files itself</td></tr></tbody></table></div>
48
48
  <h2 id="release-checklist">Release checklist</h2>
49
49
  <ol><li><code>npm test</code> green locally.</li><li><code>node packages/cli/evalsutra.mjs audit</code> is <code>PASS</code>.</li><li><code>node scripts/build-docs.mjs</code> and <code>node scripts/build-evidence.mjs</code> re-run; commit results.</li><li>Update <code>CHANGELOG.md</code>, <code>package.json</code> version, <code>.claude-plugin/plugin.json</code> version, <code>CITATION.cff</code>.</li><li>Push; wait for all CI jobs green.</li><li>Tag: <code>git tag vX.Y.Z &amp;&amp; git push origin vX.Y.Z</code>.</li><li><code>gh release create vX.Y.Z --notes-file ...</code>.</li><li>Optional npm publish (maintainer&#39;s account, see below).</li></ol>
50
50
  <h2 id="npm-publish-maintainer-action">npm publish (maintainer action)</h2>
51
- <p>The package name <code>evalsutra</code> is set in <code>package.json</code>. Availability on npm was not verified. Do not publish before checking <code>npm view evalsutra</code>. Publishing needs the maintainer&#39;s own npm login with two-factor authentication.</p>
52
- <pre><code>npm view evalsutra # expect E404 if the name is free
53
- npm pack --dry-run # inspect the file list
54
- npm publish --access public</code></pre>
55
- <p>Prefer npm <strong>trusted publishing</strong> from GitHub Actions (OIDC, provenance) over a long-lived token once the first version exists.</p>
51
+ <p>Published as <a href="https://www.npmjs.com/package/evalsutra"><code>evalsutra</code></a>. A manual <code>npm publish</code> needs the maintainer&#39;s login and two-factor code. The repository also contains <code>.github/workflows/publish.yml</code>, which publishes with npm <strong>trusted publishing</strong> (OIDC, provenance, no stored token) when a GitHub release is published. <strong>One-time maintainer setup</strong> on npmjs.com: package <code>evalsutra</code>, Settings, Trusted Publisher, GitHub Actions, owner <code>shahvatsalofficial</code>, repository <code>evalsutra</code>, workflow <code>publish.yml</code>. Until that is configured, the workflow fails and the manual route above is the one that works. The workflow has not yet run successfully.</p>
52
+ <pre><code>npm test &amp;&amp; node packages/cli/evalsutra.mjs audit # must be green
53
+ npm pack --dry-run # inspect the file list
54
+ npm publish --access public
55
+ npx -y evalsutra@X.Y.Z doctor # verify from a clean folder</code></pre>
56
+ <p>Verify the published tarball, not just the repo: a clean-folder run of <code>npx evalsutra demo</code> and <code>npx evalsutra audit</code> found two packaging bugs in 0.3.0 (see Lessons). Prefer npm <strong>trusted publishing</strong> from GitHub Actions (OIDC, provenance) over a long-lived token.</p>
56
57
  <h2 id="versioning">Versioning</h2>
57
58
  <p>Semantic versioning. 0.x means the CLI and report format may still change; changes are listed in <code>CHANGELOG.md</code>. Report <code>schemaVersion</code> changes are called out explicitly.</p>
58
59
  <h2 id="hardening-backlog">Hardening backlog</h2>
59
- <p>Pin Actions to commit SHAs, add dependency review, secret scanning (Gitleaks), CodeQL, OpenSSF Scorecard, SBOM, signed releases. None of these is claimed as done.</p></article></main>
60
+ <p>Done: Actions pinned to SHAs, Dependabot for Actions, provenance publish workflow (untested). Still open: dependency review, secret scanning (Gitleaks), CodeQL, OpenSSF Scorecard, SBOM, signed releases. None of the open items is claimed as done.</p></article></main>
60
61
  <footer>EvalSutra AI · Created by Vatsal Shah · Apache-2.0 · Generated offline; no scripts, no trackers, no network calls.</footer></body></html>
@@ -49,7 +49,7 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
49
49
  <p><strong>Which editors does it support?</strong> Skill files install into Claude Code, Cursor, Codex, Gemini CLI and Copilot folders. Only the file copy is verified. Check each tool&#39;s current docs for skill discovery.</p>
50
50
  <p><strong>Why no dependencies?</strong> Smaller attack surface and instant <code>npx</code> start.</p>
51
51
  <p><strong>How do I add my own check?</strong> See <a href="12-contributor-handbook.html">Contributor handbook</a>.</p>
52
- <p><strong>Is it on npm?</strong> Not yet. Use <code>npx github:shahvatsalofficial/evalsutra</code>.</p>
52
+ <p><strong>Is it on npm?</strong> Yes: <code>npx evalsutra</code> (package <code>evalsutra</code>). The GitHub form <code>npx github:shahvatsalofficial/evalsutra</code> also works.</p>
53
53
  <p><strong>How do I know the samples are not faked?</strong> Every sample file is produced by <code>scripts/build-evidence.mjs</code> from the shipped CLI. Re-run it and diff.</p>
54
54
  <p><strong>How do I report a bug or vulnerability?</strong> Bugs: GitHub issues. Vulnerabilities: private vulnerability reporting, see <a href="07-security-and-legal.html">Security and legal</a>.</p>
55
55
  <h2 id="glossary">Glossary</h2>
@@ -43,7 +43,7 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
43
43
  <pre><code>git clone https://github.com/shahvatsalofficial/evalsutra.git
44
44
  cd evalsutra
45
45
  npm test</code></pre>
46
- <p>Node 20+. No <code>npm install</code> is needed because there are no dependencies.</p>
46
+ <p>Node 22+. No <code>npm install</code> is needed because there are no dependencies.</p>
47
47
  <h2 id="layout">Layout</h2>
48
48
  <pre><code>packages/cli command line, installer
49
49
  packages/core evaluator, stats, security, trace, capsule
@@ -38,10 +38,10 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
38
38
  @media (prefers-reduced-motion:reduce){html{scroll-behavior:auto}}
39
39
  </style></head><body><a class="skip" href="#main">Skip to content</a>
40
40
  <header class="site"><div><strong>EvalSutra AI</strong><nav aria-label="Documentation"><a href="01-executive-summary.html">Executive summary</a><a href="02-functional-specification.html">Functional specification</a><a href="03-architecture.html">Architecture</a><a href="04-technical-reference.html">Technical reference</a><a href="05-user-guide.html">User guide</a><a href="06-skills-catalog.html">Skills catalog</a><a href="07-security-and-legal.html">Security and legal</a><a href="index.html">All docs</a></nav></div></header>
41
- <main id="main"><article><h1 id="13-roadmap-and-status">13. Roadmap and status</h1><nav class="toc" aria-label="On this page"><strong>On this page</strong><ul><li><a href="#status-matrix-v0-3-0">Status matrix (v0.3.0)</a></li><li><a href="#next-milestones">Next milestones</a></li><li><a href="#decisions-deferred">Decisions deferred</a></li><li><a href="#how-to-propose-a-change">How to propose a change</a></li></ul></nav>
41
+ <main id="main"><article><h1 id="13-roadmap-and-status">13. Roadmap and status</h1><nav class="toc" aria-label="On this page"><strong>On this page</strong><ul><li><a href="#status-matrix-v0-4-0">Status matrix (v0.4.0)</a></li><li><a href="#next-milestones">Next milestones</a></li><li><a href="#decisions-deferred">Decisions deferred</a></li><li><a href="#how-to-propose-a-change">How to propose a change</a></li></ul></nav>
42
42
  <p>Legend: <strong>Done</strong> = implemented and tested. <strong>Planned</strong> = not implemented. Nothing marked planned should be described to users as available.</p>
43
- <h2 id="status-matrix-v0-3-0">Status matrix (v0.3.0)</h2>
44
- <div class="tablewrap"><table><thead><tr><th>Capability</th><th>Status</th></tr></thead><tbody><tr><td>Offline replay evaluator, 11 assertion types</td><td>Done</td></tr><tr><td>Statuses, reason codes, evidence source rules</td><td>Done</td></tr><tr><td>Weighted score, coverage, Wilson interval, gate</td><td>Done</td></tr><tr><td>Compare with PASS / FAIL / INCONCLUSIVE and sign test</td><td>Done</td></tr><tr><td>Trace analysis</td><td>Done (heuristic)</td></tr><tr><td>Reports: JSON, Markdown, HTML, JUnit</td><td>Done</td></tr><tr><td>Repro capsule export and verify</td><td>Done (hash only, unsigned)</td></tr><tr><td>Skill installer with uninstall, dry-run, guards</td><td>Done</td></tr><tr><td>18 skills</td><td>Done (instructions only)</td></tr><tr><td>Self-audit with MD and HTML report</td><td>Done</td></tr><tr><td>Documentation set with generated HTML and evidence</td><td>Done</td></tr><tr><td>Live agent runner</td><td>Planned</td></tr><tr><td>Sandbox and consented command execution</td><td>Planned</td></tr><tr><td>SARIF output</td><td>Planned</td></tr><tr><td>JSON Schema files for suite, observation, report</td><td>Planned</td></tr><tr><td>OpenTelemetry GenAI import and export</td><td>Planned</td></tr><tr><td>Promptfoo, Inspect AI, Harbor, MCP Inspector adapters</td><td>Planned</td></tr><tr><td>Calibrated LLM judge with human gold labels</td><td>Planned</td></tr><tr><td><code>audit-plugin</code> static scanner</td><td>Planned</td></tr><tr><td>Property-based and fuzz tests</td><td>Planned</td></tr><tr><td>Server API and accessible dashboard</td><td>Planned</td></tr><tr><td>Verified live skill discovery in each IDE</td><td>Planned</td></tr><tr><td>npm publication</td><td>Pending maintainer action</td></tr><tr><td>Signed releases, SBOM, Scorecard</td><td>Planned</td></tr></tbody></table></div>
43
+ <h2 id="status-matrix-v0-4-0">Status matrix (v0.4.0)</h2>
44
+ <div class="tablewrap"><table><thead><tr><th>Capability</th><th>Status</th></tr></thead><tbody><tr><td>Offline replay evaluator, 11 assertion types</td><td>Done</td></tr><tr><td>Statuses, reason codes, evidence source rules</td><td>Done</td></tr><tr><td>Weighted score, coverage, Wilson interval, gate</td><td>Done</td></tr><tr><td>Compare with PASS / FAIL / INCONCLUSIVE and sign test</td><td>Done</td></tr><tr><td>Trace analysis</td><td>Done (heuristic)</td></tr><tr><td>Reports: JSON, Markdown, HTML, JUnit</td><td>Done</td></tr><tr><td>Repro capsule export and verify</td><td>Done (hash only, unsigned)</td></tr><tr><td>Skill installer with uninstall, dry-run, guards</td><td>Done</td></tr><tr><td>18 skills</td><td>Done (instructions only)</td></tr><tr><td>Self-audit with MD and HTML report</td><td>Done</td></tr><tr><td>Documentation set with generated HTML and evidence</td><td>Done</td></tr><tr><td>Live agent runner</td><td>Planned</td></tr><tr><td>Sandbox and consented command execution</td><td>Planned</td></tr><tr><td>SARIF output</td><td>Planned</td></tr><tr><td>JSON Schema files for suite, observation, report</td><td>Planned</td></tr><tr><td>OpenTelemetry GenAI import and export</td><td>Planned</td></tr><tr><td>Promptfoo, Inspect AI, Harbor, MCP Inspector adapters</td><td>Planned</td></tr><tr><td>Calibrated LLM judge with human gold labels</td><td>Planned</td></tr><tr><td><code>audit-plugin</code> static scanner</td><td>Planned</td></tr><tr><td>Property-based and fuzz tests</td><td>Planned</td></tr><tr><td>Server API and accessible dashboard</td><td>Planned</td></tr><tr><td>Verified live skill discovery in each IDE</td><td>Planned</td></tr><tr><td>npm publication</td><td>Done (<code>evalsutra</code> on npm); trusted publishing and provenance planned</td></tr><tr><td>Signed releases, SBOM, Scorecard</td><td>Planned</td></tr></tbody></table></div>
45
45
  <h2 id="next-milestones">Next milestones</h2>
46
46
  <ol><li><strong>0.4</strong>: JSON Schema files validated in CI, SARIF export, <code>import-traces</code> for OpenTelemetry JSON.</li><li><strong>0.5</strong>: property-based tests for parsers and scorer, signed capsule option.</li><li><strong>0.6</strong>: sandboxed local runner behind explicit consent, budgets and timeouts.</li><li><strong>0.7</strong>: MCP Inspector and Harbor adapters as optional packages.</li><li><strong>1.0</strong>: only when live-host compatibility is tested and the report schema is stable.</li></ol>
47
47
  <h2 id="decisions-deferred">Decisions deferred</h2>
@@ -38,5 +38,5 @@ footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--mut
38
38
  @media (prefers-reduced-motion:reduce){html{scroll-behavior:auto}}
39
39
  </style></head><body><a class="skip" href="#main">Skip to content</a>
40
40
  <header class="site"><div><strong>EvalSutra AI</strong><nav aria-label="Documentation"><a href="01-executive-summary.html">Executive summary</a><a href="02-functional-specification.html">Functional specification</a><a href="03-architecture.html">Architecture</a><a href="04-technical-reference.html">Technical reference</a><a href="05-user-guide.html">User guide</a><a href="06-skills-catalog.html">Skills catalog</a><a href="07-security-and-legal.html">Security and legal</a><a href="index.html" aria-current="page">All docs</a></nav></div></header>
41
- <main id="main"><section class="hero"><p class="tag">Test Intelligence. Measure Truth. Build Trust.</p><h1>EvalSutra AI documentation</h1><p>Everything needed to understand, use, audit, extend and publish EvalSutra, without asking anyone. Created by Vatsal Shah. Apache-2.0.</p></section><div class="grid"><a class="card" href="01-executive-summary.html" style="text-decoration:none;color:inherit"><h3>1. Executive summary</h3><p>Teams ship AI agents and judge them by reading the agent&#39;s own words. An agent that says &quot;Done, all tests pass&quot; may have run no tests. Prose is not proof. E</p></a><a class="card" href="02-functional-specification.html" style="text-decoration:none;color:inherit"><h3>2. Functional specification</h3><p>This page states what EvalSutra does, in behaviour a tester can check. Each rule is covered by an automated test or an audit check (IDs in brackets).</p></a><a class="card" href="03-architecture.html" style="text-decoration:none;color:inherit"><h3>3. Architecture</h3><p>!EvalSutra architecture: CLI over validate, deterministic core, reporters, repro capsule, trace analysis, self-audit and skills</p></a><a class="card" href="04-technical-reference.html" style="text-decoration:none;color:inherit"><h3>4. Technical reference</h3><p>Node.js 20 or newer. Tested on Node 20, 22 and 24 on Ubuntu, Windows and macOS in GitHub Actions.</p></a><a class="card" href="05-user-guide.html" style="text-decoration:none;color:inherit"><h3>5. User guide</h3><p>bash</p></a><a class="card" href="06-skills-catalog.html" style="text-decoration:none;color:inherit"><h3>6. Skills catalog</h3><p>EvalSutra ships 18 specialist skills. A skill is a SKILL.md file (YAML name and description, then instructions) that an AI coding tool loads on demand. Skills are instruc</p></a><a class="card" href="07-security-and-legal.html" style="text-decoration:none;color:inherit"><h3>7. Security and legal</h3><p>Principle: treat every fixture, observation, trace and report as untrusted data. The evaluator never executes data.</p></a><a class="card" href="08-testing-and-evidence.html" style="text-decoration:none;color:inherit"><h3>8. Testing and evidence</h3><p>bash</p></a><a class="card" href="09-audit-and-trust.html" style="text-decoration:none;color:inherit"><h3>9. Audit and trust</h3><p>EvalSutra can audit itself with one command and show its working.</p></a><a class="card" href="10-ci-cd-and-release.html" style="text-decoration:none;color:inherit"><h3>10. CI/CD and release</h3><p>File: .github/workflows/ci.yml. Triggers: push, pull request. Permissions: contents: read only.</p></a><a class="card" href="11-faq-and-glossary.html" style="text-decoration:none;color:inherit"><h3>11. FAQ and glossary</h3><p>Does EvalSutra run my agent? No. It grades observations you record. Live execution is planned.</p></a><a class="card" href="12-contributor-handbook.html" style="text-decoration:none;color:inherit"><h3>12. Contributor handbook</h3><p>bash</p></a><a class="card" href="13-roadmap-and-status.html" style="text-decoration:none;color:inherit"><h3>13. Roadmap and status</h3><p>Legend: Done = implemented and tested. Planned = not implemented. Nothing marked planned should be described to users as available.</p></a></div><h2>Proof</h2><p>Raw evidence lives in <a href="../evidence/">evidence/</a>: the full test run, sample reports in JSON, Markdown, HTML and JUnit, and the self-audit report.</p></main>
41
+ <main id="main"><section class="hero"><p class="tag">Test Intelligence. Measure Truth. Build Trust.</p><h1>EvalSutra AI documentation</h1><p>Everything needed to understand, use, audit, extend and publish EvalSutra, without asking anyone. Created by Vatsal Shah. Apache-2.0.</p></section><div class="grid"><a class="card" href="01-executive-summary.html" style="text-decoration:none;color:inherit"><h3>1. Executive summary</h3><p>Teams ship AI agents and judge them by reading the agent&#39;s own words. An agent that says &quot;Done, all tests pass&quot; may have run no tests. Prose is not proof. E</p></a><a class="card" href="02-functional-specification.html" style="text-decoration:none;color:inherit"><h3>2. Functional specification</h3><p>This page states what EvalSutra does, in behaviour a tester can check. Each rule is covered by an automated test or an audit check (IDs in brackets).</p></a><a class="card" href="03-architecture.html" style="text-decoration:none;color:inherit"><h3>3. Architecture</h3><p>!EvalSutra architecture: CLI over validate, deterministic core, reporters, repro capsule, trace analysis, self-audit and skills</p></a><a class="card" href="04-technical-reference.html" style="text-decoration:none;color:inherit"><h3>4. Technical reference</h3><p>Node.js 22 or newer (Node 20 reached end of life in April 2026). Tested on Node 22, 24 and 26 on Ubuntu, Windows and macOS in GitHub Actions.</p></a><a class="card" href="05-user-guide.html" style="text-decoration:none;color:inherit"><h3>5. User guide</h3><p>bash</p></a><a class="card" href="06-skills-catalog.html" style="text-decoration:none;color:inherit"><h3>6. Skills catalog</h3><p>EvalSutra ships 18 specialist skills. A skill is a SKILL.md file (YAML name and description, then instructions) that an AI coding tool loads on demand. Skills are instruc</p></a><a class="card" href="07-security-and-legal.html" style="text-decoration:none;color:inherit"><h3>7. Security and legal</h3><p>Principle: treat every fixture, observation, trace and report as untrusted data. The evaluator never executes data.</p></a><a class="card" href="08-testing-and-evidence.html" style="text-decoration:none;color:inherit"><h3>8. Testing and evidence</h3><p>bash</p></a><a class="card" href="09-audit-and-trust.html" style="text-decoration:none;color:inherit"><h3>9. Audit and trust</h3><p>EvalSutra can audit itself with one command and show its working.</p></a><a class="card" href="10-ci-cd-and-release.html" style="text-decoration:none;color:inherit"><h3>10. CI/CD and release</h3><p>Files: .github/workflows/ci.yml (tests) and publish.yml (npm release). Triggers: push and pull request for CI; a published GitHub release for npm. CI permissions: content</p></a><a class="card" href="11-faq-and-glossary.html" style="text-decoration:none;color:inherit"><h3>11. FAQ and glossary</h3><p>Does EvalSutra run my agent? No. It grades observations you record. Live execution is planned.</p></a><a class="card" href="12-contributor-handbook.html" style="text-decoration:none;color:inherit"><h3>12. Contributor handbook</h3><p>bash</p></a><a class="card" href="13-roadmap-and-status.html" style="text-decoration:none;color:inherit"><h3>13. Roadmap and status</h3><p>Legend: Done = implemented and tested. Planned = not implemented. Nothing marked planned should be described to users as available.</p></a></div><h2>Proof</h2><p>Raw evidence lives in <a href="../evidence/">evidence/</a>: the full test run, sample reports in JSON, Markdown, HTML and JUnit, and the self-audit report.</p></main>
42
42
  <footer>EvalSutra AI · Created by Vatsal Shah · Apache-2.0 · Generated offline; no scripts, no trackers, no network calls.</footer></body></html>
package/llms.txt CHANGED
@@ -18,7 +18,7 @@ EvalSutra grades recorded agent observations (output, tool calls, latency) with
18
18
  - [Roadmap and status](documents/evalsutra/13-roadmap-and-status.md)
19
19
 
20
20
  ## Quick start
21
- npx github:shahvatsalofficial/evalsutra run --suite suites/demo.json --observations examples/demo-observations.json --out artifacts/demo
21
+ npx evalsutra run --suite suites/demo.json --observations examples/demo-observations.json --out artifacts/demo
22
22
 
23
23
  ## Optional
24
24
  - [Changelog](CHANGELOG.md)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "evalsutra",
3
- "version": "0.3.0",
3
+ "version": "0.4.0",
4
4
  "description": "EvalSutra AI: evidence-first, keyless, offline agent evaluation CLI plus 18 portable agent skills for Claude Code, Cursor, Codex, Gemini CLI and Copilot.",
5
5
  "keywords": ["ai-agents", "evaluation", "agent-skills", "claude-code", "cursor", "codex", "gemini-cli", "copilot", "llm-evals", "cli"],
6
6
  "license": "Apache-2.0",
@@ -9,9 +9,9 @@
9
9
  "repository": {"type": "git", "url": "git+https://github.com/shahvatsalofficial/evalsutra.git"},
10
10
  "bugs": {"url": "https://github.com/shahvatsalofficial/evalsutra/issues"},
11
11
  "type": "module",
12
- "engines": {"node": ">=20"},
12
+ "engines": {"node": ">=22"},
13
13
  "bin": {"evalsutra": "./packages/cli/evalsutra.mjs"},
14
- "files": ["packages", ".agents", "skills", ".claude-plugin", "suites", "examples", "policies", "docs", "LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md", "CITATION.cff", "llms.txt", "documents", "README.md", "SECURITY.md", "CHANGELOG.md"],
14
+ "files": ["packages", ".agents", "skills", ".claude-plugin", "suites", "examples", "policies", "docs", "LICENSE", "CONTRIBUTING.md", "CODE_OF_CONDUCT.md", "SUPPORT.md", "NOTICE", "THIRD_PARTY_NOTICES.md", "CITATION.cff", "llms.txt", "documents", "README.md", "SECURITY.md", "CHANGELOG.md"],
15
15
  "scripts": {
16
16
  "test": "node --test",
17
17
  "doctor": "node packages/cli/evalsutra.mjs doctor",
@@ -164,7 +164,7 @@ const mdCell = s => String(s).replace(/\|/g, '\\|').replace(/\r?\n/g, ' ');
164
164
  export function auditMarkdown(a) {
165
165
  const rows = a.checks.map(c => `| ${c.id} | ${c.category} | ${mdCell(c.title)} | ${mdCell(c.expected)} | ${mdCell(c.observed)} | ${c.status.toUpperCase()} |`).join('\n');
166
166
  const cats = a.categories.map(c => `| ${c.category} | ${c.passed}/${c.total} |`).join('\n');
167
- return `# EvalSutra self-audit report\n\n**Verdict: ${a.verdict}** — ${a.passed}/${a.total} checks passed, ${a.failed} failed. \nTool: \`${a.tool}\` · Node ${a.node} · ${a.platform}\n\n> ${a.scope}\n\n## Summary by category\n\n| Category | Passed |\n|---|---|\n${cats}\n\n## All checks\n\n| ID | Category | Check | Expected | Observed | Result |\n|---|---|---|---|---|---|\n${rows}\n\n## Limits of this audit\n\n${a.limits.map(l => `- ${l}`).join('\n')}\n\n## Reproduce\n\n\`\`\`bash\nnpx github:shahvatsalofficial/evalsutra audit --out audit\n\`\`\`\n`;
167
+ return `# EvalSutra self-audit report\n\n**Verdict: ${a.verdict}** — ${a.passed}/${a.total} checks passed, ${a.failed} failed. \nTool: \`${a.tool}\` · Node ${a.node} · ${a.platform}\n\n> ${a.scope}\n\n## Summary by category\n\n| Category | Passed |\n|---|---|\n${cats}\n\n## All checks\n\n| ID | Category | Check | Expected | Observed | Result |\n|---|---|---|---|---|---|\n${rows}\n\n## Limits of this audit\n\n${a.limits.map(l => `- ${l}`).join('\n')}\n\n## Reproduce\n\n\`\`\`bash\nnpx evalsutra audit --out audit\n\`\`\`\n`;
168
168
  }
169
169
 
170
170
  export function auditHtml(a) {
@@ -174,6 +174,6 @@ export function auditHtml(a) {
174
174
  <h2>Summary by category</h2><div class="grid">${cats}</div>
175
175
  <h2>All checks</h2><div class="tablewrap"><table><caption>Every check with expected and observed value. Result is text plus symbol.</caption><thead><tr><th>ID</th><th>Category</th><th>Check</th><th>Expected</th><th>Observed</th><th>Result</th></tr></thead><tbody>${rows}</tbody></table></div>
176
176
  <h2>Limits of this audit</h2><ul>${a.limits.map(l => `<li>${esc(l)}</li>`).join('')}</ul>
177
- <h2>Reproduce</h2><pre><code>npx github:shahvatsalofficial/evalsutra audit --out audit</code></pre>`;
177
+ <h2>Reproduce</h2><pre><code>npx evalsutra audit --out audit</code></pre>`;
178
178
  return page({ title: 'EvalSutra self-audit report', description: 'Machine-generated self-audit of EvalSutra: grader accuracy, reproducibility, security and legal checks.', body });
179
179
  }
@@ -11,7 +11,8 @@ import { install, uninstall, listSkills, ROOT, IDE_DIRS } from './install.mjs';
11
11
  export const VERSION = readJson(join(ROOT, 'package.json')).version;
12
12
  // Exit codes: 0 ok, 1 usage/runtime error, 2 NO_GO/regression/FAIL, 3 REVIEW/INCONCLUSIVE, 4 invalid evidence.
13
13
  const HELP = {
14
- _: `EvalSutra AI ${VERSION} — offline, keyless agent evaluation\n\nCommands:\n doctor | init | validate | run | compare | report | gate\n install | uninstall | list-skills | matrix | trace | repro | audit\n\nRun "evalsutra <command> --help". Global: --json`,
14
+ _: `EvalSutra AI ${VERSION} — offline, keyless agent evaluation\n\nCommands:\n demo | doctor | init | validate | run | compare | report | gate\n install | uninstall | list-skills | matrix | trace | repro | audit\n\nRun "evalsutra <command> --help". Global: --json`,
15
+ demo: 'demo [--bad] [--out dir] Run the bundled demo suite on bundled synthetic observations (works from any folder). --bad uses failing evidence (exit 2).',
15
16
  doctor: 'doctor [--json] Print environment and capability facts.',
16
17
  init: 'init [--dir path] Write a starter suite and observations (never overwrites).',
17
18
  validate: 'validate --suite path',
@@ -29,6 +30,11 @@ const HELP = {
29
30
  };
30
31
 
31
32
  const args = process.argv.slice(2);
33
+ // "demo" = run the bundled suite on bundled observations, so it works from any folder (npx, npm i -g, clone).
34
+ if (args[0] === 'demo' && !args.includes('--help')) {
35
+ const obs = args.includes('--bad') ? 'examples/demo-observations-bad.json' : 'examples/demo-observations.json';
36
+ args.splice(0, 1, 'run', '--suite', join(ROOT, 'suites/demo.json'), '--observations', join(ROOT, obs), ...(args.includes('--out') ? [] : ['--out', 'artifacts/demo']));
37
+ }
32
38
  const command = args[0];
33
39
  const flag = n => args.includes('--' + n);
34
40
  const arg = (n, d) => { const i = args.indexOf('--' + n); return i >= 0 && args[i + 1] !== undefined ? args[i + 1] : d; };