memento-mori-jester 0.1.52 → 0.1.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +1 -0
- package/ROADMAP.md +3 -1
- package/docs/MAINTAINER_TRIAGE.md +5 -2
- package/docs/PRODUCTION_READINESS.md +2 -0
- package/docs/RELEASE.md +2 -0
- package/docs/RELEASE_NOTES_v0.1.53.md +32 -0
- package/docs/RELEASE_NOTES_v0.1.54.md +39 -0
- package/examples/fixtures/README.md +4 -0
- package/package.json +3 -2
- package/scripts/check-fixtures.mjs +2 -14
- package/scripts/check-production-readiness.mjs +17 -0
- package/scripts/report-fixtures.mjs +258 -0
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,17 @@ All notable changes to Memento Mori Jester are tracked here.
|
|
|
4
4
|
|
|
5
5
|
## Unreleased
|
|
6
6
|
|
|
7
|
+
## 0.1.54
|
|
8
|
+
|
|
9
|
+
- Added `npm run fixtures:report`, a local fixture coverage report for rule, preset, review-kind, and verdict coverage.
|
|
10
|
+
- The report highlights rules without pass-case coverage, thin rule coverage, preset/kind gaps, and quiet pass fixtures, with text and `--json` output.
|
|
11
|
+
- Wired fixture coverage reporting into `npm test` and production-readiness checks so coverage gaps stay visible during maintenance.
|
|
12
|
+
|
|
13
|
+
## 0.1.53
|
|
14
|
+
|
|
15
|
+
- Made `npm run fixtures:check` self-contained so it works from the published npm package, where `src/` files are intentionally not shipped.
|
|
16
|
+
- Added a production-readiness guard to prevent the fixture validator from depending on source-only files.
|
|
17
|
+
|
|
7
18
|
## 0.1.52
|
|
8
19
|
|
|
9
20
|
- Added `npm run fixtures:check`, a local fixture authoring validator for duplicate IDs, weak metadata, unsafe-looking content, duplicate content, and explicit expected/absent rule intent.
|
package/README.md
CHANGED
|
@@ -501,6 +501,7 @@ Use the false-positive template for noisy cautions or blocks. Include `jester su
|
|
|
501
501
|
|
|
502
502
|
Maintainers can use [docs/MAINTAINER_TRIAGE.md](docs/MAINTAINER_TRIAGE.md) to turn useful false-positive reports into redacted fixtures.
|
|
503
503
|
Run `npm run fixtures:check` before merging fixture changes; it catches duplicate IDs, missing rule metadata, weak descriptions, unsafe-looking content, and duplicate content.
|
|
504
|
+
Run `npm run fixtures:report` to see fixture coverage by rule, preset, kind, and verdict before choosing the next pass-case fixture.
|
|
504
505
|
|
|
505
506
|
For vulnerabilities, private code exposure, or credential-handling concerns, follow [SECURITY.md](SECURITY.md) instead of opening a public issue with sensitive details.
|
|
506
507
|
|
package/ROADMAP.md
CHANGED
|
@@ -6,6 +6,8 @@ Memento Mori Jester is usable today as a CLI, MCP server, GitHub Action, and git
|
|
|
6
6
|
|
|
7
7
|
## Recently Shipped
|
|
8
8
|
|
|
9
|
+
- Fixture coverage report generator in v0.1.54 for rule, preset, review-kind, verdict, and pass-case gaps.
|
|
10
|
+
- Published-package fixture validator fix in v0.1.53 so `npm run fixtures:check` works outside a source checkout.
|
|
9
11
|
- Fixture authoring validator in v0.1.52 for duplicate IDs, missing expected/absent rule intent, weak metadata, unsafe-looking content, and duplicate content.
|
|
10
12
|
- Maintainer triage guide in v0.1.51 for turning useful false-positive reports into redacted fixture coverage.
|
|
11
13
|
- Security policy and GitHub issue templates in v0.1.50 for bug reports, false positives, feature requests, and vulnerability intake.
|
|
@@ -42,7 +44,7 @@ Memento Mori Jester is usable today as a CLI, MCP server, GitHub Action, and git
|
|
|
42
44
|
## Product Ideas
|
|
43
45
|
|
|
44
46
|
- Add more framework-specific false-positive examples from real reports so tuning guidance keeps getting sharper.
|
|
45
|
-
- Add
|
|
47
|
+
- Add the first targeted pass-case fixture batch from the fixture coverage report.
|
|
46
48
|
|
|
47
49
|
## Quality And Safety
|
|
48
50
|
|
|
@@ -75,14 +75,17 @@ Avoid fixtures that:
|
|
|
75
75
|
```powershell
|
|
76
76
|
npm.cmd test
|
|
77
77
|
npm.cmd run fixtures:check
|
|
78
|
+
npm.cmd run fixtures:report
|
|
79
|
+
npm.cmd run fixtures:report -- --json
|
|
78
80
|
node .\dist\cli.js tune <rule-id>
|
|
79
81
|
node .\dist\cli.js tune <rule-id> --json
|
|
80
82
|
node .\dist\cli.js tune coverage
|
|
81
83
|
```
|
|
82
84
|
|
|
83
85
|
5. Fix any duplicate IDs, missing expected rule metadata, weak descriptions, unsafe content, or duplicate content reported by `fixtures:check`.
|
|
84
|
-
6.
|
|
85
|
-
7.
|
|
86
|
+
6. Use `fixtures:report` to check whether the change improves pass-case, preset, kind, or verdict coverage.
|
|
87
|
+
7. Check whether support/confidence changed in the expected direction.
|
|
88
|
+
8. If the fixture changes verdict behavior, mention the exact rule impact in `CHANGELOG.md`.
|
|
86
89
|
|
|
87
90
|
## When To Change A Rule
|
|
88
91
|
|
|
@@ -53,6 +53,7 @@ This checklist defines what "production grade" means for Memento Mori Jester rig
|
|
|
53
53
|
- `SECURITY.md` routes vulnerability reports away from public issues and asks for redacted diagnostics.
|
|
54
54
|
- `docs/MAINTAINER_TRIAGE.md` explains how to turn useful false-positive reports into fixture coverage before changing rule logic.
|
|
55
55
|
- `npm run fixtures:check` validates fixture IDs, metadata, unsafe-looking content, duplicate content, and explicit expected/absent rule intent.
|
|
56
|
+
- `npm run fixtures:report` shows fixture coverage by rule, preset, kind, and verdict so maintainers can pick the next pass-case target.
|
|
56
57
|
- npm publish has a manual workflow fallback, but the normal release path is tag-driven trusted publishing.
|
|
57
58
|
|
|
58
59
|
## Static Guard
|
|
@@ -67,6 +68,7 @@ This checklist defines what "production grade" means for Memento Mori Jester rig
|
|
|
67
68
|
- `SECURITY.md` and GitHub issue templates exist and ask for the right diagnostics.
|
|
68
69
|
- maintainer triage docs exist and link noisy-rule reports back to fixture coverage.
|
|
69
70
|
- fixture authoring checks are wired into `npm test`.
|
|
71
|
+
- fixture coverage reports are wired into `npm test`.
|
|
70
72
|
|
|
71
73
|
`npm test` runs this check after the TypeScript build and unit tests.
|
|
72
74
|
|
package/docs/RELEASE.md
CHANGED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# v0.1.53 Release Notes
|
|
2
|
+
|
|
3
|
+
This patch fixes the newly added fixture authoring validator so it works from the published npm package, not only from a source checkout.
|
|
4
|
+
|
|
5
|
+
## What Changed
|
|
6
|
+
|
|
7
|
+
- Made `scripts/check-fixtures.mjs` self-contained by removing its source-file reads.
|
|
8
|
+
- Added a production-readiness guard that prevents `fixtures:check` from depending on `src/config.ts` or `src/types.ts`.
|
|
9
|
+
|
|
10
|
+
## Behavior Notes
|
|
11
|
+
|
|
12
|
+
- No CLI, MCP, config, rule, playground, GitHub Action runtime, or release automation behavior changed.
|
|
13
|
+
- Review fixture expectations remain unchanged.
|
|
14
|
+
|
|
15
|
+
## Release Validation
|
|
16
|
+
|
|
17
|
+
```powershell
|
|
18
|
+
npm.cmd test
|
|
19
|
+
npm.cmd run fixtures:check
|
|
20
|
+
npm.cmd run production:check
|
|
21
|
+
npm.cmd run demo:svg:check
|
|
22
|
+
npm.cmd run pack:dry
|
|
23
|
+
git diff --check
|
|
24
|
+
node .\dist\cli.js doctor --json
|
|
25
|
+
git diff | node .\dist\cli.js diff --fail-on block --subject "v0.1.53 published fixture validator fix"
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Post-publish smoke:
|
|
29
|
+
|
|
30
|
+
```powershell
|
|
31
|
+
npm.cmd exec --yes --package memento-mori-jester@latest -- npm run fixtures:check --prefix <published-package-path>
|
|
32
|
+
```
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# v0.1.54 Release Notes
|
|
2
|
+
|
|
3
|
+
This release adds a fixture coverage report generator so maintainers can see which rules, presets, review kinds, and verdicts need stronger fixture coverage.
|
|
4
|
+
|
|
5
|
+
## What Changed
|
|
6
|
+
|
|
7
|
+
- Added `scripts/report-fixtures.mjs`.
|
|
8
|
+
- Added `npm run fixtures:report`.
|
|
9
|
+
- Added `npm run fixtures:report -- --json` for stable structured output.
|
|
10
|
+
- Wired fixture reporting into `npm test`.
|
|
11
|
+
- The report shows:
|
|
12
|
+
- total fixtures, weighted fixtures, and edge-case fixtures,
|
|
13
|
+
- coverage by verdict, review kind, and preset,
|
|
14
|
+
- rule coverage from `expectedRuleIds`,
|
|
15
|
+
- rules without pass-case coverage,
|
|
16
|
+
- thin rule coverage,
|
|
17
|
+
- preset/kind gaps,
|
|
18
|
+
- quiet pass fixtures.
|
|
19
|
+
- Updated maintainer triage docs, fixture docs, release docs, roadmap, changelog, and production readiness checks.
|
|
20
|
+
|
|
21
|
+
## Behavior Notes
|
|
22
|
+
|
|
23
|
+
- No CLI, MCP, config, rule, playground, GitHub Action runtime, or release automation behavior changed.
|
|
24
|
+
- Existing review fixture expectations remain unchanged.
|
|
25
|
+
|
|
26
|
+
## Release Validation
|
|
27
|
+
|
|
28
|
+
```powershell
|
|
29
|
+
npm.cmd test
|
|
30
|
+
npm.cmd run fixtures:check
|
|
31
|
+
npm.cmd run fixtures:report
|
|
32
|
+
npm.cmd run fixtures:report -- --json
|
|
33
|
+
npm.cmd run production:check
|
|
34
|
+
npm.cmd run demo:svg:check
|
|
35
|
+
npm.cmd run pack:dry
|
|
36
|
+
git diff --check
|
|
37
|
+
node .\dist\cli.js doctor --json
|
|
38
|
+
git diff | node .\dist\cli.js diff --fail-on block --subject "v0.1.54 fixture coverage report"
|
|
39
|
+
```
|
|
@@ -20,6 +20,8 @@ Maintainer triage guidance lives in [docs/MAINTAINER_TRIAGE.md](../../docs/MAINT
|
|
|
20
20
|
```powershell
|
|
21
21
|
npm.cmd test
|
|
22
22
|
npm.cmd run fixtures:check
|
|
23
|
+
npm.cmd run fixtures:report
|
|
24
|
+
npm.cmd run fixtures:report -- --json
|
|
23
25
|
```
|
|
24
26
|
|
|
25
27
|
For one-off manual review, paste a fixture `content` value into:
|
|
@@ -42,3 +44,5 @@ Use the smallest redacted example that still reproduces the behavior. A good fix
|
|
|
42
44
|
Do not add secrets, private code, customer data, complete logs, or machine-specific paths. If a false-positive report is safe but broad, add a passing fixture before loosening a rule.
|
|
43
45
|
|
|
44
46
|
`npm run fixtures:check` validates duplicate IDs, missing expected rule metadata, weak descriptions, unsafe-looking fixture content, and duplicate content before the fixture suite becomes tuning evidence.
|
|
47
|
+
|
|
48
|
+
`npm run fixtures:report` summarizes coverage by rule, preset, review kind, and verdict. Use it to find rules without pass-case coverage, thin rule coverage, preset/kind gaps, and quiet pass fixtures.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "memento-mori-jester",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.54",
|
|
4
4
|
"description": "A local court-jester sidecar for AI coding agents: review plans, commands, diffs, and final claims before they get too pleased with themselves.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"repository": {
|
|
@@ -40,11 +40,12 @@
|
|
|
40
40
|
"build": "tsc -p tsconfig.json",
|
|
41
41
|
"start": "node dist/server.js",
|
|
42
42
|
"start:mcp": "node dist/server.js",
|
|
43
|
-
"test": "npm run build && node scripts/run-tests.mjs && npm run fixtures:check && npm run production:check",
|
|
43
|
+
"test": "npm run build && node scripts/run-tests.mjs && npm run fixtures:check && npm run fixtures:report && npm run production:check",
|
|
44
44
|
"doctor": "node dist/cli.js doctor",
|
|
45
45
|
"demo:svg": "node scripts/render-demo-svg.mjs",
|
|
46
46
|
"demo:svg:check": "node scripts/render-demo-svg.mjs --check",
|
|
47
47
|
"fixtures:check": "node scripts/check-fixtures.mjs",
|
|
48
|
+
"fixtures:report": "node scripts/report-fixtures.mjs",
|
|
48
49
|
"production:check": "node scripts/check-production-readiness.mjs",
|
|
49
50
|
"pack:dry": "npm pack --dry-run",
|
|
50
51
|
"prepare": "npm run build",
|
|
@@ -10,20 +10,8 @@ function read(path) {
|
|
|
10
10
|
return readFileSync(join(root, path), "utf8");
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const match = source.match(new RegExp(`export const ${constName} = \\[([^\\]]+)\\] as const`));
|
|
16
|
-
|
|
17
|
-
if (!match) {
|
|
18
|
-
failures.push(`Could not read ${constName} from ${path}.`);
|
|
19
|
-
return [];
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
return [...match[1].matchAll(/"([^"]+)"/g)].map((entry) => entry[1]);
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
const allowedPresets = new Set(readConstStringArray("src/config.ts", "configPresetNames"));
|
|
26
|
-
const allowedKinds = new Set(readConstStringArray("src/types.ts", "reviewKinds"));
|
|
13
|
+
const allowedPresets = new Set(["default", "node", "python", "web", "api", "infra", "ai", "security"]);
|
|
14
|
+
const allowedKinds = new Set(["plan", "command", "diff", "final"]);
|
|
27
15
|
const allowedVerdicts = new Set(["pass", "caution", "block"]);
|
|
28
16
|
const unsafeContentPatterns = [
|
|
29
17
|
{ name: "private key block", pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ },
|
|
@@ -26,6 +26,13 @@ function requireText(path, pattern, description) {
|
|
|
26
26
|
}
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
function forbidText(path, pattern, description) {
|
|
30
|
+
const content = read(path);
|
|
31
|
+
if (pattern.test(content)) {
|
|
32
|
+
failures.push(`${path} should not include ${description}.`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
29
36
|
function requirePackageFile(packageJson, value) {
|
|
30
37
|
if (!Array.isArray(packageJson.files) || !packageJson.files.includes(value)) {
|
|
31
38
|
failures.push(`package.json files should include ${value}.`);
|
|
@@ -58,6 +65,7 @@ for (const path of [
|
|
|
58
65
|
`docs/RELEASE_NOTES_${tag}.md`,
|
|
59
66
|
"action.yml",
|
|
60
67
|
"scripts/check-fixtures.mjs",
|
|
68
|
+
"scripts/report-fixtures.mjs",
|
|
61
69
|
".github/ISSUE_TEMPLATE/bug_report.yml",
|
|
62
70
|
".github/ISSUE_TEMPLATE/false_positive.yml",
|
|
63
71
|
".github/ISSUE_TEMPLATE/feature_request.yml",
|
|
@@ -85,6 +93,7 @@ requireText("README.md", /SECURITY\.md/, "security policy link");
|
|
|
85
93
|
requireText("README.md", /false-positive/i, "false-positive support guidance");
|
|
86
94
|
requireText("README.md", /MAINTAINER_TRIAGE\.md/, "maintainer triage guide link");
|
|
87
95
|
requireText("README.md", /fixtures:check/, "fixture authoring check guidance");
|
|
96
|
+
requireText("README.md", /fixtures:report/, "fixture coverage report guidance");
|
|
88
97
|
requireText("README.md", /License: PolyForm Noncommercial/, "the noncommercial license badge");
|
|
89
98
|
requireText("docs/PRODUCTION_READINESS.md", /npm package/i, "npm package readiness");
|
|
90
99
|
requireText("docs/PRODUCTION_READINESS.md", /GitHub Action/i, "GitHub Action readiness");
|
|
@@ -96,6 +105,7 @@ requireText("docs/PRODUCTION_READINESS.md", /SECURITY\.md/, "security policy rea
|
|
|
96
105
|
requireText("docs/PRODUCTION_READINESS.md", /issue templates/i, "issue template readiness");
|
|
97
106
|
requireText("docs/PRODUCTION_READINESS.md", /MAINTAINER_TRIAGE\.md/, "maintainer triage readiness");
|
|
98
107
|
requireText("docs/PRODUCTION_READINESS.md", /fixtures:check/, "fixture authoring check readiness");
|
|
108
|
+
requireText("docs/PRODUCTION_READINESS.md", /fixtures:report/, "fixture coverage report readiness");
|
|
99
109
|
requireText("docs/CLI.md", /jester doctor --json/, "doctor JSON CLI docs");
|
|
100
110
|
requireText("docs/MAINTAINER_TRIAGE.md", /doctor --json/, "doctor JSON triage prompt");
|
|
101
111
|
requireText("docs/MAINTAINER_TRIAGE.md", /tune <rule-id> --json/, "tune JSON triage prompt");
|
|
@@ -105,10 +115,17 @@ requireText("docs/MAINTAINER_TRIAGE.md", /absentRuleIds/, "fixture absent rule g
|
|
|
105
115
|
requireText("examples/fixtures/README.md", /MAINTAINER_TRIAGE\.md/, "maintainer triage link");
|
|
106
116
|
requireText("examples/fixtures/README.md", /Adding A Fixture From A Report/, "fixture report conversion guidance");
|
|
107
117
|
requireText("examples/fixtures/README.md", /fixtures:check/, "fixture authoring check guidance");
|
|
118
|
+
requireText("examples/fixtures/README.md", /fixtures:report/, "fixture coverage report guidance");
|
|
108
119
|
requireText("scripts/check-fixtures.mjs", /duplicated/, "duplicate fixture id check");
|
|
109
120
|
requireText("scripts/check-fixtures.mjs", /unsafeContentPatterns/, "unsafe fixture content checks");
|
|
121
|
+
forbidText("scripts/check-fixtures.mjs", /src\/config\.ts|src\/types\.ts/, "source-only fixture validator dependencies");
|
|
122
|
+
requireText("scripts/report-fixtures.mjs", /rulesWithoutPassCases/, "rules without pass-case coverage report");
|
|
123
|
+
requireText("scripts/report-fixtures.mjs", /presetKindGaps/, "preset and kind gap report");
|
|
124
|
+
forbidText("scripts/report-fixtures.mjs", /src\/config\.ts|src\/types\.ts/, "source-only fixture report dependencies");
|
|
110
125
|
requireText("package.json", /"fixtures:check": "node scripts\/check-fixtures\.mjs"/, "fixture authoring check script");
|
|
126
|
+
requireText("package.json", /"fixtures:report": "node scripts\/report-fixtures\.mjs"/, "fixture coverage report script");
|
|
111
127
|
requireText("package.json", /npm run fixtures:check/, "fixture authoring check in npm test");
|
|
128
|
+
requireText("package.json", /npm run fixtures:report/, "fixture coverage report in npm test");
|
|
112
129
|
requireText("SECURITY.md", /doctor --json/, "doctor JSON redaction guidance");
|
|
113
130
|
requireText("SECURITY.md", /security\/advisories\/new/, "private vulnerability report link");
|
|
114
131
|
requireText(".github/ISSUE_TEMPLATE/bug_report.yml", /doctor --json/, "doctor JSON support prompt");
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
|
|
5
|
+
const root = process.cwd();
|
|
6
|
+
const fixturePath = "examples/fixtures/preset-review-cases.json";
|
|
7
|
+
const allowedPresets = ["default", "node", "python", "web", "api", "infra", "ai", "security"];
|
|
8
|
+
const allowedKinds = ["plan", "command", "diff", "final"];
|
|
9
|
+
const allowedVerdicts = ["pass", "caution", "block"];
|
|
10
|
+
const sampleLimit = 3;
|
|
11
|
+
|
|
12
|
+
const args = new Set(process.argv.slice(2));
|
|
13
|
+
const json = args.has("--json");
|
|
14
|
+
|
|
15
|
+
function read(path) {
|
|
16
|
+
return readFileSync(join(root, path), "utf8");
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
let fixtures;
|
|
20
|
+
try {
|
|
21
|
+
fixtures = JSON.parse(read(fixturePath));
|
|
22
|
+
} catch (error) {
|
|
23
|
+
process.stderr.write(`Could not parse ${fixturePath}: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
24
|
+
process.exit(1);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if (!Array.isArray(fixtures)) {
|
|
28
|
+
process.stderr.write(`${fixturePath} should contain a JSON array.\n`);
|
|
29
|
+
process.exit(1);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const report = buildFixtureReport(fixtures);
|
|
33
|
+
|
|
34
|
+
if (json) {
|
|
35
|
+
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
36
|
+
} else {
|
|
37
|
+
process.stdout.write(renderFixtureReport(report));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function buildFixtureReport(rawFixtures) {
|
|
41
|
+
const byPreset = zeroCounts(allowedPresets);
|
|
42
|
+
const byKind = zeroCounts(allowedKinds);
|
|
43
|
+
const byVerdict = zeroCounts(allowedVerdicts);
|
|
44
|
+
const rules = new Map();
|
|
45
|
+
const quietPassFixtures = [];
|
|
46
|
+
|
|
47
|
+
let totalWeight = 0;
|
|
48
|
+
let edgeCaseFixtures = 0;
|
|
49
|
+
|
|
50
|
+
for (const fixture of rawFixtures) {
|
|
51
|
+
const preset = typeof fixture.preset === "string" ? fixture.preset : "unknown";
|
|
52
|
+
const kind = typeof fixture.kind === "string" ? fixture.kind : "unknown";
|
|
53
|
+
const verdict = typeof fixture.expectedVerdict === "string" ? fixture.expectedVerdict : "unknown";
|
|
54
|
+
const expectedRuleIds = Array.isArray(fixture.expectedRuleIds) ? fixture.expectedRuleIds : [];
|
|
55
|
+
const weight = fixtureWeight(fixture.weight);
|
|
56
|
+
const edgeCase = fixture.edgeCase === true;
|
|
57
|
+
const sample = {
|
|
58
|
+
id: String(fixture.id ?? "unknown"),
|
|
59
|
+
description: String(fixture.description ?? ""),
|
|
60
|
+
preset,
|
|
61
|
+
kind,
|
|
62
|
+
verdict
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
byPreset[preset] = (byPreset[preset] ?? 0) + 1;
|
|
66
|
+
byKind[kind] = (byKind[kind] ?? 0) + 1;
|
|
67
|
+
byVerdict[verdict] = (byVerdict[verdict] ?? 0) + 1;
|
|
68
|
+
totalWeight += weight;
|
|
69
|
+
if (edgeCase) {
|
|
70
|
+
edgeCaseFixtures += 1;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if (verdict === "pass" && expectedRuleIds.length === 0) {
|
|
74
|
+
quietPassFixtures.push(sample);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
for (const ruleId of expectedRuleIds) {
|
|
78
|
+
const entry = rules.get(ruleId) ?? createRuleEntry(ruleId);
|
|
79
|
+
entry.total += 1;
|
|
80
|
+
entry.weight += weight;
|
|
81
|
+
entry.verdicts[verdict] = (entry.verdicts[verdict] ?? 0) + 1;
|
|
82
|
+
entry.kinds[kind] = (entry.kinds[kind] ?? 0) + 1;
|
|
83
|
+
entry.presets[preset] = (entry.presets[preset] ?? 0) + 1;
|
|
84
|
+
if (edgeCase) {
|
|
85
|
+
entry.edgeCases += 1;
|
|
86
|
+
}
|
|
87
|
+
entry.samples.push(sample);
|
|
88
|
+
rules.set(ruleId, entry);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const ruleSummaries = [...rules.values()]
|
|
93
|
+
.map((entry) => ({
|
|
94
|
+
ruleId: entry.ruleId,
|
|
95
|
+
total: entry.total,
|
|
96
|
+
weight: entry.weight,
|
|
97
|
+
passCases: entry.verdicts.pass ?? 0,
|
|
98
|
+
cautionCases: entry.verdicts.caution ?? 0,
|
|
99
|
+
blockCases: entry.verdicts.block ?? 0,
|
|
100
|
+
edgeCases: entry.edgeCases,
|
|
101
|
+
verdicts: orderedCounts(entry.verdicts, allowedVerdicts),
|
|
102
|
+
kinds: orderedCounts(entry.kinds, allowedKinds),
|
|
103
|
+
presets: orderedCounts(entry.presets, allowedPresets),
|
|
104
|
+
samples: entry.samples
|
|
105
|
+
.slice()
|
|
106
|
+
.sort((a, b) => a.id.localeCompare(b.id))
|
|
107
|
+
.slice(0, sampleLimit)
|
|
108
|
+
}))
|
|
109
|
+
.sort((a, b) => a.ruleId.localeCompare(b.ruleId));
|
|
110
|
+
|
|
111
|
+
const gaps = {
|
|
112
|
+
rulesWithoutPassCases: ruleSummaries
|
|
113
|
+
.filter((entry) => entry.passCases === 0)
|
|
114
|
+
.sort((a, b) => b.total - a.total || a.ruleId.localeCompare(b.ruleId))
|
|
115
|
+
.map(ruleGapSummary),
|
|
116
|
+
thinRuleCoverage: ruleSummaries
|
|
117
|
+
.filter((entry) => entry.total < 2)
|
|
118
|
+
.sort((a, b) => a.total - b.total || a.ruleId.localeCompare(b.ruleId))
|
|
119
|
+
.map(ruleGapSummary),
|
|
120
|
+
presetKindGaps: presetKindGaps(rawFixtures),
|
|
121
|
+
quietPassFixtures: quietPassFixtures
|
|
122
|
+
.slice()
|
|
123
|
+
.sort((a, b) => a.id.localeCompare(b.id))
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
return {
|
|
127
|
+
totalFixtures: rawFixtures.length,
|
|
128
|
+
totalWeight,
|
|
129
|
+
edgeCaseFixtures,
|
|
130
|
+
byVerdict: orderedCounts(byVerdict, allowedVerdicts),
|
|
131
|
+
byKind: orderedCounts(byKind, allowedKinds),
|
|
132
|
+
byPreset: orderedCounts(byPreset, allowedPresets),
|
|
133
|
+
rules: ruleSummaries,
|
|
134
|
+
gaps
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function renderFixtureReport(report) {
|
|
139
|
+
const lines = [
|
|
140
|
+
"Fixture coverage report",
|
|
141
|
+
"",
|
|
142
|
+
`Fixtures: ${report.totalFixtures}`,
|
|
143
|
+
`Weighted fixtures: ${report.totalWeight}`,
|
|
144
|
+
`Edge-case fixtures: ${report.edgeCaseFixtures}`,
|
|
145
|
+
`Rules covered by expectedRuleIds: ${report.rules.length}`,
|
|
146
|
+
"",
|
|
147
|
+
`By verdict: ${formatCounts(report.byVerdict)}`,
|
|
148
|
+
`By kind: ${formatCounts(report.byKind)}`,
|
|
149
|
+
`By preset: ${formatCounts(report.byPreset)}`,
|
|
150
|
+
"",
|
|
151
|
+
"Rules without pass-case coverage:"
|
|
152
|
+
];
|
|
153
|
+
|
|
154
|
+
lines.push(...formatRuleGaps(report.gaps.rulesWithoutPassCases));
|
|
155
|
+
lines.push("", "Thin rule coverage:");
|
|
156
|
+
lines.push(...formatRuleGaps(report.gaps.thinRuleCoverage));
|
|
157
|
+
lines.push("", "Preset/kind gaps:");
|
|
158
|
+
lines.push(...formatPresetKindGaps(report.gaps.presetKindGaps));
|
|
159
|
+
lines.push("", "Quiet pass fixtures:");
|
|
160
|
+
lines.push(...formatFixtureSamples(report.gaps.quietPassFixtures));
|
|
161
|
+
lines.push(
|
|
162
|
+
"",
|
|
163
|
+
"Next:",
|
|
164
|
+
" npm run fixtures:check",
|
|
165
|
+
" npm run fixtures:report -- --json",
|
|
166
|
+
" node .\\dist\\cli.js tune coverage"
|
|
167
|
+
);
|
|
168
|
+
|
|
169
|
+
return `${lines.join("\n")}\n`;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function createRuleEntry(ruleId) {
|
|
173
|
+
return {
|
|
174
|
+
ruleId,
|
|
175
|
+
total: 0,
|
|
176
|
+
weight: 0,
|
|
177
|
+
edgeCases: 0,
|
|
178
|
+
verdicts: zeroCounts(allowedVerdicts),
|
|
179
|
+
kinds: zeroCounts(allowedKinds),
|
|
180
|
+
presets: zeroCounts(allowedPresets),
|
|
181
|
+
samples: []
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function fixtureWeight(rawWeight) {
|
|
186
|
+
if (typeof rawWeight === "number" && rawWeight > 0 && Number.isFinite(rawWeight)) {
|
|
187
|
+
return Math.max(1, Math.min(3, Math.round(rawWeight)));
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
return 1;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function zeroCounts(keys) {
|
|
194
|
+
return Object.fromEntries(keys.map((key) => [key, 0]));
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function orderedCounts(counts, keys) {
|
|
198
|
+
const ordered = zeroCounts(keys);
|
|
199
|
+
for (const [key, value] of Object.entries(counts)) {
|
|
200
|
+
ordered[key] = value;
|
|
201
|
+
}
|
|
202
|
+
return ordered;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function presetKindGaps(fixtures) {
|
|
206
|
+
const seen = new Set(fixtures.map((fixture) => `${fixture.preset}:${fixture.kind}`));
|
|
207
|
+
return allowedPresets
|
|
208
|
+
.map((preset) => ({
|
|
209
|
+
preset,
|
|
210
|
+
missingKinds: allowedKinds.filter((kind) => !seen.has(`${preset}:${kind}`))
|
|
211
|
+
}))
|
|
212
|
+
.filter((entry) => entry.missingKinds.length > 0);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function ruleGapSummary(entry) {
|
|
216
|
+
return {
|
|
217
|
+
ruleId: entry.ruleId,
|
|
218
|
+
total: entry.total,
|
|
219
|
+
passCases: entry.passCases,
|
|
220
|
+
cautionCases: entry.cautionCases,
|
|
221
|
+
blockCases: entry.blockCases,
|
|
222
|
+
samples: entry.samples
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function formatCounts(counts) {
|
|
227
|
+
return Object.entries(counts)
|
|
228
|
+
.map(([key, value]) => `${key} ${value}`)
|
|
229
|
+
.join(", ");
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function formatRuleGaps(entries) {
|
|
233
|
+
if (entries.length === 0) {
|
|
234
|
+
return ["- none"];
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
return entries
|
|
238
|
+
.slice(0, 12)
|
|
239
|
+
.map((entry) => `- ${entry.ruleId}: ${entry.total} fixture(s), pass ${entry.passCases}, caution ${entry.cautionCases}, block ${entry.blockCases}`);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function formatPresetKindGaps(entries) {
|
|
243
|
+
if (entries.length === 0) {
|
|
244
|
+
return ["- none"];
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
return entries.map((entry) => `- ${entry.preset}: ${entry.missingKinds.join(", ")}`);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function formatFixtureSamples(entries) {
|
|
251
|
+
if (entries.length === 0) {
|
|
252
|
+
return ["- none"];
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
return entries
|
|
256
|
+
.slice(0, 8)
|
|
257
|
+
.map((entry) => `- ${entry.id}: ${entry.description}`);
|
|
258
|
+
}
|