@event4u/agent-config 7.4.0 → 7.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +4 -1
- package/CHANGELOG.md +48 -0
- package/README.md +1 -1
- package/dist/agent-src/commands/review-changes.md +14 -7
- package/dist/agent-src/rules/artifact-engagement-recording.md +9 -0
- package/dist/agent-src/rules/domain-safety-pii.md +4 -3
- package/dist/agent-src/rules/icon-consistency.md +1 -0
- package/dist/agent-src/rules/legal-safety-floor.md +2 -2
- package/dist/agent-src/rules/untrusted-input-defense.md +8 -5
- package/dist/agent-src/skills/adr-create/SKILL.md +5 -1
- package/dist/agent-src/skills/adversarial-review/SKILL.md +1 -1
- package/dist/agent-src/skills/blade-ui/SKILL.md +4 -0
- package/dist/agent-src/skills/brand-to-tokens/SKILL.md +2 -1
- package/dist/agent-src/skills/code-review/SKILL.md +4 -0
- package/dist/agent-src/skills/context-authoring/SKILL.md +1 -1
- package/dist/agent-src/skills/design-intelligence/SKILL.md +40 -0
- package/dist/agent-src/skills/design-review/SKILL.md +23 -11
- package/dist/agent-src/skills/design-system-capture/SKILL.md +41 -0
- package/dist/agent-src/skills/design-system-capture/reference/design-system-json.md +62 -0
- package/dist/agent-src/skills/design-tokens/SKILL.md +1 -0
- package/dist/agent-src/skills/existing-ui-audit/SKILL.md +6 -0
- package/dist/agent-src/skills/flux/SKILL.md +4 -0
- package/dist/agent-src/skills/iconography/SKILL.md +1 -0
- package/dist/agent-src/skills/image-editing/evals/triggers.json +13 -13
- package/dist/agent-src/skills/judge-artifact-completeness/SKILL.md +136 -0
- package/dist/agent-src/skills/judge-artifact-completeness/calibration/fixtures.json +149 -0
- package/dist/agent-src/skills/judge-artifact-completeness/evals/triggers.json +17 -0
- package/dist/agent-src/skills/judge-artifact-completeness/rubrics/architecture-score.json +73 -0
- package/dist/agent-src/skills/judge-artifact-completeness/rubrics/pr-review-score.json +65 -0
- package/dist/agent-src/skills/judge-artifact-completeness/rubrics/roadmap-score.json +73 -0
- package/dist/agent-src/skills/judge-artifact-completeness/rubrics/ticket-quality-score.json +73 -0
- package/dist/agent-src/skills/judge-injection-defense/SKILL.md +99 -0
- package/dist/agent-src/skills/judge-injection-defense/calibration/fixtures.json +24 -0
- package/dist/agent-src/skills/judge-injection-defense/evals/triggers.json +17 -0
- package/dist/agent-src/skills/judge-injection-defense/fixtures/behavioral-fixtures.json +63 -0
- package/dist/agent-src/skills/judge-injection-defense/fixtures/coverage-report.md +53 -0
- package/dist/agent-src/skills/judge-injection-defense/fixtures/perturbation-taxonomy.json +110 -0
- package/dist/agent-src/skills/judge-synthesis/SKILL.md +134 -0
- package/dist/agent-src/skills/judge-synthesis/evals/triggers.json +17 -0
- package/dist/agent-src/skills/logo-generation/SKILL.md +1 -1
- package/dist/agent-src/skills/markitdown/SKILL.md +1 -1
- package/dist/agent-src/skills/react-shadcn-ui/SKILL.md +18 -0
- package/dist/agent-src/skills/react-shadcn-ui/reference/registry.md +59 -0
- package/dist/agent-src/skills/refine-ticket/SKILL.md +5 -1
- package/dist/agent-src/skills/release-comms/SKILL.md +1 -1
- package/dist/agent-src/skills/roadmap-management/SKILL.md +4 -0
- package/dist/agent-src/skills/roadmap-writing/SKILL.md +1 -1
- package/dist/agent-src/skills/script-writing/SKILL.md +1 -1
- package/dist/agent-src/skills/sql-writing/SKILL.md +1 -1
- package/dist/agent-src/skills/systematic-debugging/SKILL.md +1 -1
- package/dist/agent-src/skills/tailwind-engineer/SKILL.md +4 -0
- package/dist/agent-src/skills/testing-anti-patterns/SKILL.md +1 -1
- package/dist/agent-src/skills/token-optimizer/SKILL.md +1 -1
- package/dist/agent-src/skills/typography-system/SKILL.md +2 -1
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +107 -41
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +3 -3
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +8 -5
- package/dist/discovery/trust-report.md +2 -2
- package/dist/discovery/workspaces.json +7 -4
- package/dist/mcp/registry-manifest.json +3 -3
- package/dist/server/schemas/settings.js +3 -0
- package/dist/server/schemas/settings.js.map +1 -1
- package/docs/architecture.md +2 -2
- package/docs/catalog.md +28 -20
- package/docs/command-flows.md +1 -1
- package/docs/guidelines/design-antipatterns.md +26 -0
- package/docs/guidelines/design-canon.md +62 -0
- package/package.json +1 -1
- package/src/config/agent-settings.template.yml +12 -0
- package/src/scripts/design_slop_rules.test.ts +193 -0
- package/src/scripts/design_slop_rules.ts +502 -0
- package/src/scripts/external_sources_denylist.json +7 -1
- package/src/scripts/hook_manifest.yaml +14 -3
- package/src/scripts/hooks/design_slop_hook.ts +181 -0
- package/src/scripts/lint_agent_security.ts +1 -0
- package/src/scripts/lint_confusables.ts +197 -0
- package/src/scripts/lint_design_slop.ts +245 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
---
|
|
2
|
+
model_tier: high
|
|
3
|
+
name: judge-artifact-completeness
|
|
4
|
+
description: "Use when scoring a roadmap, PR review, ADR, or ticket for completeness — risk, tests, migration, maintainability. Dispatched by /refine-ticket, /adr-create, /review-changes; never auto-gates."
|
|
5
|
+
domain: quality
|
|
6
|
+
workspaces:
|
|
7
|
+
- agent-config-maintainer
|
|
8
|
+
packs:
|
|
9
|
+
- meta
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
# judge-artifact-completeness
|
|
13
|
+
|
|
14
|
+
> Judge specialized in **artifact completeness**. Score non-code deliverables — roadmap, PR review, ADR, ticket — against a rubric and surface missing/partial dimensions. Does **not** judge code quality, bugs, or security. Never auto-gates: score + gaps → human decides.
|
|
15
|
+
|
|
16
|
+
## When to use
|
|
17
|
+
|
|
18
|
+
* Roadmap produced — score completeness against AC, risk, migration surface.
|
|
19
|
+
* PR review complete — check evidence quality + test coverage.
|
|
20
|
+
* ADR drafted — alternatives / consequences / reversibility completeness pass.
|
|
21
|
+
* Ticket exits refinement — DoR readiness check.
|
|
22
|
+
* `/refine-ticket`, `/adr-create`, `/roadmap:create`, `/review-changes` surface score as optional output pass.
|
|
23
|
+
|
|
24
|
+
Do NOT use when:
|
|
25
|
+
|
|
26
|
+
* Code quality / naming / DRY → [`judge-code-quality`](../judge-code-quality/SKILL.md)
|
|
27
|
+
* Functional bug → [`judge-bug-hunter`](../judge-bug-hunter/SKILL.md)
|
|
28
|
+
* Missing test files → [`judge-test-coverage`](../judge-test-coverage/SKILL.md)
|
|
29
|
+
* Security issue → [`judge-security-auditor`](../judge-security-auditor/SKILL.md)
|
|
30
|
+
|
|
31
|
+
## Procedure
|
|
32
|
+
|
|
33
|
+
### 1. Identify artifact type
|
|
34
|
+
|
|
35
|
+
| Artifact | Rubric schema |
|
|
36
|
+
|---|---|
|
|
37
|
+
| Roadmap / plan | `rubrics/roadmap-score.json` |
|
|
38
|
+
| PR review / code-review comment | `rubrics/pr-review-score.json` |
|
|
39
|
+
| ADR / architecture decision | `rubrics/architecture-score.json` |
|
|
40
|
+
| Jira / Linear ticket | `rubrics/ticket-quality-score.json` |
|
|
41
|
+
|
|
42
|
+
Ambiguous type → one question before scoring.
|
|
43
|
+
|
|
44
|
+
### 2. Score each dimension
|
|
45
|
+
|
|
46
|
+
* **0** — absent. Criterion not addressed.
|
|
47
|
+
* **1** — partial. Mentioned but too vague to be actionable (e.g., "risks exist" without naming one).
|
|
48
|
+
* **weight** — fully present. Criterion met concretely and traceably.
|
|
49
|
+
|
|
50
|
+
Use only `criterion` field. Do not penalise for style or length. Short artifact covering all dimensions scores same as long one — completeness ≠ verbosity.
|
|
51
|
+
|
|
52
|
+
Mark **N/A** (full credit) only when rubric schema explicitly allows it.
|
|
53
|
+
|
|
54
|
+
### 3. Compute verdict
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
total_earned = sum of all dimension scores (N/A = weight)
|
|
58
|
+
total_possible = sum of all dimension weights
|
|
59
|
+
ratio = total_earned / total_possible
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
| Ratio | Verdict |
|
|
63
|
+
|---|---|
|
|
64
|
+
| ≥ 0.80 | `complete` |
|
|
65
|
+
| ≥ 0.50 | `partial` |
|
|
66
|
+
| < 0.50 | `incomplete` |
|
|
67
|
+
|
|
68
|
+
### 4. Surface gaps
|
|
69
|
+
|
|
70
|
+
Top 1–3 gaps — dimensions with score = 0 or 1, highest weight. Gap entry names dimension + specific unmet criterion.
|
|
71
|
+
|
|
72
|
+
## Validation
|
|
73
|
+
|
|
74
|
+
Before finalising:
|
|
75
|
+
|
|
76
|
+
1. Every scored dimension maps to rubric schema field.
|
|
77
|
+
2. No dimension penalised for length or word count.
|
|
78
|
+
3. N/A credit granted only where schema allows.
|
|
79
|
+
4. Verdict follows ratio thresholds, not intuition.
|
|
80
|
+
5. Top gaps = highest-weight missing — not every minor gap.
|
|
81
|
+
|
|
82
|
+
## Output format
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
Judge: judge-artifact-completeness
|
|
86
|
+
Type: roadmap | pr-review | architecture | ticket
|
|
87
|
+
Target: <one-line artifact description>
|
|
88
|
+
Verdict: complete | partial | incomplete
|
|
89
|
+
Score: <earned>/<possible> (<pct>%)
|
|
90
|
+
|
|
91
|
+
Dimensions:
|
|
92
|
+
✅ <dimension-name> (<earned>/<weight>) — <one-line note, or "meets criterion">
|
|
93
|
+
⚠️ <dimension-name> (<earned>/<weight>) — PARTIAL: <specific gap>
|
|
94
|
+
❌ <dimension-name> (0/<weight>) — MISSING: <what would satisfy the criterion>
|
|
95
|
+
|
|
96
|
+
Top gaps:
|
|
97
|
+
1. <highest-weight missing dimension>: <concrete action to close the gap>
|
|
98
|
+
2. ...
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Required fields (ordered):
|
|
102
|
+
|
|
103
|
+
1. **Judge**, **Type**, **Target** — identification
|
|
104
|
+
2. **Verdict** — `complete`, `partial`, or `incomplete`
|
|
105
|
+
3. **Score** — raw earned/possible and percentage
|
|
106
|
+
4. **Dimensions** — one line per dimension with emoji + score + note
|
|
107
|
+
5. **Top gaps** — highest-weight missing with concrete close action
|
|
108
|
+
|
|
109
|
+
Output surfaced as recommendation. Human decides whether to act on gaps.
|
|
110
|
+
|
|
111
|
+
## Gotcha
|
|
112
|
+
|
|
113
|
+
* **Length ≠ completeness** — terse complete roadmap scores same as long one.
|
|
114
|
+
* **N/A abuse** — `migration_effort` is only N/A when no public-interface change. Mark 1 (partial) when unsure.
|
|
115
|
+
* **Partial credit creep** — "mentioned but vague" = partial (1), not full credit. "Risks exist" without naming one = partial.
|
|
116
|
+
* **Verdict as gate** — recommendation, never blocker. Surface it; human decides.
|
|
117
|
+
|
|
118
|
+
## Do NOT
|
|
119
|
+
|
|
120
|
+
* NEVER penalise artifact for being short or concise
|
|
121
|
+
* NEVER grant full credit to vague mention — that is partial (1)
|
|
122
|
+
* NEVER auto-reject or auto-approve based on verdict alone
|
|
123
|
+
* NEVER score code quality, correctness, or security — out of scope
|
|
124
|
+
* NEVER invent dimensions not in rubric schema
|
|
125
|
+
|
|
126
|
+
## Calibration
|
|
127
|
+
|
|
128
|
+
Fixtures: `calibration/fixtures.json`. Rubric designed monotone: removing a fully-present dimension lowers score by at least `dimension.weight`. No single dimension dominates (max weight = 3; total possible = 12–15 per rubric).
|
|
129
|
+
|
|
130
|
+
Anti-length: fixtures include `SHORT_COMPLETE` (short, all dimensions) and `LONG_INCOMPLETE` (verbose, missing high-weight dimensions). Correct judge scores `SHORT_COMPLETE` > `LONG_INCOMPLETE`.
|
|
131
|
+
|
|
132
|
+
## References
|
|
133
|
+
|
|
134
|
+
- Sibling judges: [`judge-code-quality`](../judge-code-quality/SKILL.md), [`judge-bug-hunter`](../judge-bug-hunter/SKILL.md), [`judge-security-auditor`](../judge-security-auditor/SKILL.md), [`judge-test-coverage`](../judge-test-coverage/SKILL.md).
|
|
135
|
+
- Dispatchers: [`/refine-ticket`](../../commands/refine-ticket.md), [`/adr-create`](../../commands/adr-create.md), [`/review-changes`](../../commands/review-changes.md).
|
|
136
|
+
- Rubric schemas: `rubrics/roadmap-score.json`, `rubrics/pr-review-score.json`, `rubrics/architecture-score.json`, `rubrics/ticket-quality-score.json`.
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
{
|
|
2
|
+
"description": "Calibration fixtures for judge-artifact-completeness. Used to verify (a) monotonicity — removing a dimension lowers the score by exactly its weight; (b) strict tier ordering — EXCELLENT > GOOD > FAIR > POOR > TERRIBLE; (c) anti-length — SHORT_COMPLETE scores higher than LONG_INCOMPLETE.",
|
|
3
|
+
"rubric": "roadmap-score",
|
|
4
|
+
"gold_tiers": {
|
|
5
|
+
"scale": "0–1 mapped to earned/possible ratio",
|
|
6
|
+
"EXCELLENT": { "expected_ratio_min": 0.90, "expected_verdict": "complete" },
|
|
7
|
+
"GOOD": { "expected_ratio_min": 0.70, "expected_verdict": "partial" },
|
|
8
|
+
"FAIR": { "expected_ratio_min": 0.50, "expected_verdict": "partial" },
|
|
9
|
+
"POOR": { "expected_ratio_min": 0.20, "expected_verdict": "incomplete" },
|
|
10
|
+
"TERRIBLE": { "expected_ratio_min": 0.0, "expected_verdict": "incomplete" }
|
|
11
|
+
},
|
|
12
|
+
"anti_length_assertion": {
|
|
13
|
+
"description": "SHORT_COMPLETE must score strictly above LONG_INCOMPLETE regardless of word count.",
|
|
14
|
+
"rule": "score(SHORT_COMPLETE) > score(LONG_INCOMPLETE)"
|
|
15
|
+
},
|
|
16
|
+
"fixtures": [
|
|
17
|
+
{
|
|
18
|
+
"id": "EXCELLENT",
|
|
19
|
+
"label": "All dimensions fully present, terse",
|
|
20
|
+
"artifact": {
|
|
21
|
+
"goal": "Ship a typed parser for the config format by 2026-Q3.",
|
|
22
|
+
"acceptance_criteria": "task test exits 0 on the full corpus; zero regressions in existing consumers; coverage ≥ 90% per codecov.",
|
|
23
|
+
"phases": [
|
|
24
|
+
{
|
|
25
|
+
"name": "Phase 1 — Lexer",
|
|
26
|
+
"deliverable": "A tokeniser with 100% token-type coverage.",
|
|
27
|
+
"verification": "Pest suite: 40 fixtures, all green."
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"name": "Phase 2 — Parser",
|
|
31
|
+
"deliverable": "A recursive-descent parser emitting a typed AST.",
|
|
32
|
+
"verification": "Integration test: parse all fixtures; compare AST to golden JSON."
|
|
33
|
+
}
|
|
34
|
+
],
|
|
35
|
+
"risk": "Ambiguous grammar rule for nested keys — spike required in Phase 1 before committing to the AST shape.",
|
|
36
|
+
"migration": "Existing consumers use the old string-based API. A shim is provided in Phase 2; shim deprecated in Phase 3.",
|
|
37
|
+
"owner": "matze4u; quarterly review each January.",
|
|
38
|
+
"do_not_build": "No serialiser (inverse direction). No streaming parser. No error recovery beyond fail-fast."
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"id": "GOOD",
|
|
43
|
+
"label": "Acceptance criteria and most dimensions present; migration vague",
|
|
44
|
+
"artifact": {
|
|
45
|
+
"goal": "Ship a typed parser for the config format.",
|
|
46
|
+
"acceptance_criteria": "All existing tests pass after migration.",
|
|
47
|
+
"phases": [
|
|
48
|
+
{ "name": "Phase 1 — Lexer", "deliverable": "Tokeniser." },
|
|
49
|
+
{ "name": "Phase 2 — Parser", "deliverable": "AST." }
|
|
50
|
+
],
|
|
51
|
+
"risk": "Grammar ambiguity may require a rework.",
|
|
52
|
+
"migration": "Migration needed for consumers.",
|
|
53
|
+
"owner": "matze4u",
|
|
54
|
+
"do_not_build": "No serialiser."
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"id": "FAIR",
|
|
59
|
+
"label": "Acceptance criteria and do-not-build missing; phases underdefined",
|
|
60
|
+
"artifact": {
|
|
61
|
+
"goal": "Build a config parser.",
|
|
62
|
+
"phases": [
|
|
63
|
+
{ "name": "Phase 1", "deliverable": "Lexer." },
|
|
64
|
+
{ "name": "Phase 2", "deliverable": "Parser." }
|
|
65
|
+
],
|
|
66
|
+
"risk": "Grammar issues may arise.",
|
|
67
|
+
"owner": "matze4u"
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"id": "POOR",
|
|
72
|
+
"label": "Only a goal and phase list; no cross-cutting dimensions",
|
|
73
|
+
"artifact": {
|
|
74
|
+
"goal": "Build a config parser.",
|
|
75
|
+
"phases": [
|
|
76
|
+
{ "name": "Phase 1", "deliverable": "Lexer." },
|
|
77
|
+
{ "name": "Phase 2", "deliverable": "Parser." }
|
|
78
|
+
]
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"id": "TERRIBLE",
|
|
83
|
+
"label": "Barely any content",
|
|
84
|
+
"artifact": {
|
|
85
|
+
"goal": "Improve parsing."
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
"id": "SHORT_COMPLETE",
|
|
90
|
+
"label": "Anti-length: short artifact, all dimensions covered",
|
|
91
|
+
"word_count_approx": 80,
|
|
92
|
+
"dimensions_present": ["acceptance_criteria_fit", "phase_decomposition", "risk", "tests_per_step", "migration_effort", "maintainability_owner", "do_not_build_clarity"],
|
|
93
|
+
"artifact": {
|
|
94
|
+
"goal": "Ship a typed config parser.",
|
|
95
|
+
"acceptance_criteria": "task test green; ≥90% coverage.",
|
|
96
|
+
"phases": [{ "name": "Phase 1", "deliverable": "Lexer + parser.", "verification": "Pest suite, 40 fixtures green." }],
|
|
97
|
+
"risk": "Grammar ambiguity — spike in Phase 1.",
|
|
98
|
+
"migration": "Shim provided; old API deprecated in Phase 2.",
|
|
99
|
+
"owner": "matze4u",
|
|
100
|
+
"do_not_build": "No serialiser."
|
|
101
|
+
}
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
"id": "LONG_INCOMPLETE",
|
|
105
|
+
"label": "Anti-length: verbose artifact, missing risk + tests + migration",
|
|
106
|
+
"word_count_approx": 400,
|
|
107
|
+
"dimensions_present": ["phase_decomposition"],
|
|
108
|
+
"artifact": {
|
|
109
|
+
"goal": "We need to significantly overhaul our configuration parsing layer to support the new typed AST format that the team has been discussing over the last several sprints. This will involve extensive rework of the tokenisation logic as well as the construction of a proper recursive-descent parser that can handle all the edge cases we have identified in our current configuration files.",
|
|
110
|
+
"phases": [
|
|
111
|
+
{
|
|
112
|
+
"name": "Phase 1 — Lexer Overhaul",
|
|
113
|
+
"deliverable": "A comprehensive rewrite of the tokeniser component to support the full range of token types that we expect to encounter in our configuration files. The tokeniser will be responsible for converting raw input text into a structured stream of tokens that can be consumed by the parser in Phase 2."
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"name": "Phase 2 — Parser Implementation",
|
|
117
|
+
"deliverable": "A recursive-descent parser that consumes the token stream produced by the Phase 1 tokeniser and constructs a fully typed abstract syntax tree. The parser will handle nested structures, arrays, inline tables, and all other grammar constructs."
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
"name": "Phase 3 — Consumer Migration",
|
|
121
|
+
"deliverable": "Update all downstream consumers to use the new typed API. This will require changes across multiple modules and may involve coordination with other teams."
|
|
122
|
+
}
|
|
123
|
+
],
|
|
124
|
+
"owner": "The team."
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
],
|
|
128
|
+
"dimension_ablation": {
|
|
129
|
+
"description": "Monotonicity test: take EXCELLENT, remove one dimension at a time, verify score drops by exactly that dimension's weight.",
|
|
130
|
+
"base_fixture": "EXCELLENT",
|
|
131
|
+
"ablations": [
|
|
132
|
+
{ "remove_dimension": "acceptance_criteria_fit", "expected_score_drop": 3 },
|
|
133
|
+
{ "remove_dimension": "phase_decomposition", "expected_score_drop": 2 },
|
|
134
|
+
{ "remove_dimension": "risk", "expected_score_drop": 2 },
|
|
135
|
+
{ "remove_dimension": "tests_per_step", "expected_score_drop": 2 },
|
|
136
|
+
{ "remove_dimension": "migration_effort", "expected_score_drop": 2 },
|
|
137
|
+
{ "remove_dimension": "maintainability_owner", "expected_score_drop": 2 },
|
|
138
|
+
{ "remove_dimension": "do_not_build_clarity", "expected_score_drop": 1 }
|
|
139
|
+
],
|
|
140
|
+
"dominance_check": {
|
|
141
|
+
"description": "No single dimension should exceed 25% of total possible score.",
|
|
142
|
+
"max_weight": 3,
|
|
143
|
+
"total_possible": 14,
|
|
144
|
+
"max_allowed_share": 0.25,
|
|
145
|
+
"actual_max_share": 0.214,
|
|
146
|
+
"pass": true
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill": "judge-artifact-completeness",
|
|
3
|
+
"last_eval": "2026-06-27",
|
|
4
|
+
"description": "5 should-trigger + 5 should-not-trigger. Should-trigger covers scoring roadmaps, PR reviews, ADRs, and tickets for completeness (risk, tests, migration, AC, DoR). Should-not covers near-miss neighbours: code-quality review (judge-code-quality), bug hunting (judge-bug-hunter), test-file coverage (judge-test-coverage), security audit (judge-security-auditor), and creating an ADR from scratch (adr-create).",
|
|
5
|
+
"queries": [
|
|
6
|
+
{ "q": "score this roadmap for completeness — does it cover risk, tests, and migration?", "trigger": true },
|
|
7
|
+
{ "q": "does this PR review have enough evidence? is each finding traceable to a diff line?", "trigger": true },
|
|
8
|
+
{ "q": "check if this ticket meets DoR — are the acceptance criteria verifiable?", "trigger": true },
|
|
9
|
+
{ "q": "is this ADR complete? does it name alternatives and state consequences?", "trigger": true },
|
|
10
|
+
{ "q": "what dimensions are missing from this roadmap — risk, maintainability, migration?", "trigger": true },
|
|
11
|
+
{ "q": "review the naming and DRY violations in this diff", "trigger": false, "note": "code quality → judge-code-quality" },
|
|
12
|
+
{ "q": "find the bugs introduced by this change", "trigger": false, "note": "bug hunting → judge-bug-hunter" },
|
|
13
|
+
{ "q": "does this PR add enough test coverage for the new behaviour?", "trigger": false, "note": "test-file coverage → judge-test-coverage" },
|
|
14
|
+
{ "q": "are there security issues in this authentication middleware?", "trigger": false, "note": "security → judge-security-auditor" },
|
|
15
|
+
{ "q": "create an ADR for switching from REST to GraphQL", "trigger": false, "note": "ADR creation → adr-create, not scoring" }
|
|
16
|
+
]
|
|
17
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
{
|
|
2
|
+
"rubric": "architecture-score",
|
|
3
|
+
"version": "1.0",
|
|
4
|
+
"artifact_type": "architecture",
|
|
5
|
+
"description": "Scores an ADR or architecture decision for completeness: decision clarity, alternatives, consequences, and the four cross-cutting dimensions.",
|
|
6
|
+
"anchors": ["adr-create", "decision-record"],
|
|
7
|
+
"scoring": {
|
|
8
|
+
"scale": "0 = absent, 1 = partial (mentioned but vague), weight = fully present and concrete",
|
|
9
|
+
"verdict_thresholds": {
|
|
10
|
+
"complete": 0.80,
|
|
11
|
+
"partial": 0.50,
|
|
12
|
+
"incomplete": 0.0
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"dimensions": [
|
|
16
|
+
{
|
|
17
|
+
"id": "decision_clarity",
|
|
18
|
+
"name": "Decision clarity",
|
|
19
|
+
"cross_cutting": false,
|
|
20
|
+
"weight": 3,
|
|
21
|
+
"criterion": "The chosen option is stated unambiguously in one sentence. The scope of the decision is bounded — it is clear what is inside vs. outside the decision. Status (Accepted / Proposed / Superseded) is set.",
|
|
22
|
+
"gap_signal": "The decision section is prose about the problem rather than a declarative statement of the choice; or the status field is missing."
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "alternatives_considered",
|
|
26
|
+
"name": "Alternatives considered",
|
|
27
|
+
"cross_cutting": false,
|
|
28
|
+
"weight": 2,
|
|
29
|
+
"criterion": "At least two alternatives are named with a one-sentence rationale for why each was not chosen. 'We didn't consider other options' fails. A genuinely obvious single-choice with no real alternatives earns partial credit with an explanation.",
|
|
30
|
+
"gap_signal": "No alternatives section; or alternatives are listed without rationale for rejection."
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "consequences",
|
|
34
|
+
"name": "Consequences",
|
|
35
|
+
"cross_cutting": false,
|
|
36
|
+
"weight": 2,
|
|
37
|
+
"criterion": "Both positive and negative consequences of the decision are named. Positive-only consequences lists signal motivated reasoning and earn partial credit.",
|
|
38
|
+
"gap_signal": "Only upsides described; negative consequences, limitations, or future complexity are absent."
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"id": "risk",
|
|
42
|
+
"name": "Risk coverage",
|
|
43
|
+
"cross_cutting": true,
|
|
44
|
+
"weight": 2,
|
|
45
|
+
"criterion": "Risks and the conditions under which the decision should be revisited are named. A 'revisit if' clause or 'this breaks down when' note earns full credit.",
|
|
46
|
+
"gap_signal": "No risk named; no statement of when the decision would need to be reversed or superseded."
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"id": "reversibility_migration",
|
|
50
|
+
"name": "Reversibility and migration",
|
|
51
|
+
"cross_cutting": true,
|
|
52
|
+
"weight": 2,
|
|
53
|
+
"criterion": "If the decision is hard to reverse (data schema, public contract, naming), the cost of reversal is stated. Easy-to-reverse decisions may state 'low reversal cost' as N/A with full credit.",
|
|
54
|
+
"gap_signal": "An irreversible or costly-to-reverse decision does not state the reversal cost or migration path."
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"id": "maintainability",
|
|
58
|
+
"name": "Maintainability and ownership",
|
|
59
|
+
"cross_cutting": true,
|
|
60
|
+
"weight": 1,
|
|
61
|
+
"criterion": "A maintainer or team is named. Long-lived architectural commitments state a review trigger (e.g., 'revisit when performance exceeds X').",
|
|
62
|
+
"gap_signal": "No owner named; no trigger for revisiting a long-lived commitment."
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "adr_format_completeness",
|
|
66
|
+
"name": "ADR format completeness",
|
|
67
|
+
"cross_cutting": false,
|
|
68
|
+
"weight": 1,
|
|
69
|
+
"criterion": "Standard ADR sections are present: Status, Context, Decision, Consequences, Alternatives. An index entry exists (regenerated by `adr-create`). Superseded ADRs link to their successor.",
|
|
70
|
+
"gap_signal": "Missing standard sections; or no index entry; or superseded record does not link to successor."
|
|
71
|
+
}
|
|
72
|
+
]
|
|
73
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"rubric": "pr-review-score",
|
|
3
|
+
"version": "1.0",
|
|
4
|
+
"artifact_type": "pr-review",
|
|
5
|
+
"description": "Scores a PR review for completeness: evidence quality, actionability, and the four cross-cutting dimensions (risk, tests, migration, maintainability).",
|
|
6
|
+
"anchors": ["code-review", "requesting-code-review", "receiving-code-review"],
|
|
7
|
+
"scoring": {
|
|
8
|
+
"scale": "0 = absent, 1 = partial (mentioned but vague), weight = fully present and concrete",
|
|
9
|
+
"verdict_thresholds": {
|
|
10
|
+
"complete": 0.80,
|
|
11
|
+
"partial": 0.50,
|
|
12
|
+
"incomplete": 0.0
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"dimensions": [
|
|
16
|
+
{
|
|
17
|
+
"id": "evidence_fit",
|
|
18
|
+
"name": "Evidence fit",
|
|
19
|
+
"cross_cutting": false,
|
|
20
|
+
"weight": 3,
|
|
21
|
+
"criterion": "Every finding cites a specific file:line or diff hunk. Claims rest on observable evidence in the diff, not on assumptions about the author's intent. 'This looks wrong' without a reference fails.",
|
|
22
|
+
"gap_signal": "Findings reference vague areas ('the API layer', 'this part') rather than specific locations; or claims are asserted without citing a diff line."
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "concrete_fixes",
|
|
26
|
+
"name": "Concrete fixes",
|
|
27
|
+
"cross_cutting": false,
|
|
28
|
+
"weight": 2,
|
|
29
|
+
"criterion": "Each finding proposes a concrete action — a renamed variable, a specific refactor, a test case that would catch the bug. 'Make it cleaner' or 'consider improving this' are not actionable.",
|
|
30
|
+
"gap_signal": "Findings are observations without prescribed remedies; or remedies are directional ('improve X') rather than concrete ('rename X to Y')."
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "risk_blast_radius",
|
|
34
|
+
"name": "Risk and blast radius",
|
|
35
|
+
"cross_cutting": true,
|
|
36
|
+
"weight": 2,
|
|
37
|
+
"criterion": "The review calls out high-risk hunks: areas touching auth, tenancy, billing, public API contracts, or data migrations. Blast radius is named if a change could affect more callers than the diff shows.",
|
|
38
|
+
"gap_signal": "Security or tenancy paths in the diff are not flagged; a breaking API change is reviewed as if no callers exist outside the diff."
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"id": "test_coverage",
|
|
42
|
+
"name": "Test coverage of the change",
|
|
43
|
+
"cross_cutting": true,
|
|
44
|
+
"weight": 2,
|
|
45
|
+
"criterion": "The review notes whether the diff adds tests for new behavior or regression coverage for a fix. Missing tests for testable paths are flagged as findings, not left implicit.",
|
|
46
|
+
"gap_signal": "New or changed behavior has no test; the review does not flag this."
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"id": "migration_rollback",
|
|
50
|
+
"name": "Migration and rollback",
|
|
51
|
+
"cross_cutting": true,
|
|
52
|
+
"weight": 2,
|
|
53
|
+
"criterion": "If the diff includes a schema migration, config change, or breaking API change, the review confirms a rollback or migration path exists. If none is needed, this is N/A (full credit).",
|
|
54
|
+
"gap_signal": "An irreversible schema migration or breaking API change ships without a rollback note or migration guide."
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"id": "maintainability",
|
|
58
|
+
"name": "Maintainability signal",
|
|
59
|
+
"cross_cutting": true,
|
|
60
|
+
"weight": 1,
|
|
61
|
+
"criterion": "The review flags clarity and naming issues that will burden future readers — confusing abstractions, missing context, or patterns inconsistent with the module's conventions.",
|
|
62
|
+
"gap_signal": "Reviewable naming or abstraction issues are present in the diff but not mentioned."
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
{
|
|
2
|
+
"rubric": "roadmap-score",
|
|
3
|
+
"version": "1.0",
|
|
4
|
+
"artifact_type": "roadmap",
|
|
5
|
+
"description": "Scores a roadmap for completeness: acceptance criteria, phase decomposition, and the four cross-cutting dimensions (risk, tests, migration, maintainability).",
|
|
6
|
+
"anchors": ["roadmap-management", "roadmap-writing"],
|
|
7
|
+
"scoring": {
|
|
8
|
+
"scale": "0 = absent, 1 = partial (mentioned but vague), weight = fully present and concrete",
|
|
9
|
+
"verdict_thresholds": {
|
|
10
|
+
"complete": 0.80,
|
|
11
|
+
"partial": 0.50,
|
|
12
|
+
"incomplete": 0.0
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"dimensions": [
|
|
16
|
+
{
|
|
17
|
+
"id": "acceptance_criteria_fit",
|
|
18
|
+
"name": "Acceptance criteria fit",
|
|
19
|
+
"cross_cutting": false,
|
|
20
|
+
"weight": 3,
|
|
21
|
+
"criterion": "Every phase or the roadmap as a whole has measurable acceptance criteria traceable to the goal statement. Criteria must be verifiable — a pass/fail test or observable outcome, not 'implement X'.",
|
|
22
|
+
"gap_signal": "No acceptance criteria section, or criteria are vague task descriptions rather than measurable outcomes."
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "phase_decomposition",
|
|
26
|
+
"name": "Phase decomposition",
|
|
27
|
+
"cross_cutting": false,
|
|
28
|
+
"weight": 2,
|
|
29
|
+
"criterion": "Work is decomposed into phases with clear scope boundaries. Each phase has a defined deliverable. The ordering is justified (dependency order or logical progression).",
|
|
30
|
+
"gap_signal": "A flat list of steps with no phase grouping, or phases whose ordering is arbitrary with no stated rationale."
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "risk",
|
|
34
|
+
"name": "Risk coverage",
|
|
35
|
+
"cross_cutting": true,
|
|
36
|
+
"weight": 2,
|
|
37
|
+
"criterion": "Risks, blockers, and open dependencies are explicitly named — not buried in step prose. At least one 'what could go wrong' or 'blocking on' note per phase is expected.",
|
|
38
|
+
"gap_signal": "No risks mentioned, or only implicit risks (e.g., 'verify the linter works' without stating what the failure mode is)."
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"id": "tests_per_step",
|
|
42
|
+
"name": "Verification per step",
|
|
43
|
+
"cross_cutting": true,
|
|
44
|
+
"weight": 2,
|
|
45
|
+
"criterion": "Each non-trivial step states how its completion is verified — a concrete command, observable output, or pass/fail criterion. 'verify by inspection' is partial credit.",
|
|
46
|
+
"gap_signal": "Steps have no verification clause, or verification is implied rather than stated."
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"id": "migration_effort",
|
|
50
|
+
"name": "Migration / breaking-change coverage",
|
|
51
|
+
"cross_cutting": true,
|
|
52
|
+
"weight": 2,
|
|
53
|
+
"criterion": "If the roadmap changes a public interface, contract, or schema, migration steps for consumers are addressed. If there is no migration surface, this dimension is marked N/A (full credit).",
|
|
54
|
+
"gap_signal": "A breaking change is introduced but no migration path, deprecation window, or consumer-impact note exists."
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"id": "maintainability_owner",
|
|
58
|
+
"name": "Maintainability and ownership",
|
|
59
|
+
"cross_cutting": true,
|
|
60
|
+
"weight": 2,
|
|
61
|
+
"criterion": "The roadmap names who owns each phase or the whole deliverable. Long-lived artifacts (skills, rules, linters) state a refresh or review cadence. One-shot scripts that are deleted on completion are exempt.",
|
|
62
|
+
"gap_signal": "No owner named; no refresh cadence for long-lived output; no mention of how the result is kept current."
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "do_not_build_clarity",
|
|
66
|
+
"name": "Do-not-build clarity",
|
|
67
|
+
"cross_cutting": false,
|
|
68
|
+
"weight": 1,
|
|
69
|
+
"criterion": "Explicitly out-of-scope items are named so implementers do not over-build. A dedicated 'Do-not-build' or 'Out of scope' section earns full credit; absence is only acceptable when the roadmap is trivially narrow.",
|
|
70
|
+
"gap_signal": "No out-of-scope signal; a reader could reasonably extend the scope in several directions without guidance."
|
|
71
|
+
}
|
|
72
|
+
]
|
|
73
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
{
|
|
2
|
+
"rubric": "ticket-quality-score",
|
|
3
|
+
"version": "1.0",
|
|
4
|
+
"artifact_type": "ticket",
|
|
5
|
+
"description": "Scores a Jira/Linear ticket for completeness against Definition of Ready: acceptance criteria, estimate, and the four cross-cutting dimensions.",
|
|
6
|
+
"anchors": ["refine-ticket", "estimate-ticket", "emit-tickets"],
|
|
7
|
+
"scoring": {
|
|
8
|
+
"scale": "0 = absent, 1 = partial (mentioned but vague), weight = fully present and concrete",
|
|
9
|
+
"verdict_thresholds": {
|
|
10
|
+
"complete": 0.80,
|
|
11
|
+
"partial": 0.50,
|
|
12
|
+
"incomplete": 0.0
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"dimensions": [
|
|
16
|
+
{
|
|
17
|
+
"id": "acceptance_criteria",
|
|
18
|
+
"name": "Acceptance criteria",
|
|
19
|
+
"cross_cutting": false,
|
|
20
|
+
"weight": 3,
|
|
21
|
+
"criterion": "Acceptance criteria are written as verifiable Given-When-Then or numbered checklist items, not as task descriptions. Each criterion is independently testable. An implementer can write a test before touching code.",
|
|
22
|
+
"gap_signal": "AC section missing; or AC items are task descriptions ('implement the API') rather than verifiable outcomes ('returns 200 with a body matching the schema')."
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "dor_readiness",
|
|
26
|
+
"name": "Definition of Ready",
|
|
27
|
+
"cross_cutting": false,
|
|
28
|
+
"weight": 2,
|
|
29
|
+
"criterion": "The ticket passes Definition of Ready: scope is bounded, context sufficient to start without a separate discovery session, and no external blockers are unresolved.",
|
|
30
|
+
"gap_signal": "Scope is unbounded ('improve the search'); key context is missing ('see Confluence') without a direct link; blockers are noted but unresolved."
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "estimate_sizing",
|
|
34
|
+
"name": "Estimate and sizing",
|
|
35
|
+
"cross_cutting": false,
|
|
36
|
+
"weight": 1,
|
|
37
|
+
"criterion": "A story-point or T-shirt-size estimate is present. The estimate is consistent with the scope: a 1-pointer has trivially bounded scope; an 8-pointer has a split plan.",
|
|
38
|
+
"gap_signal": "No estimate; or estimate is wildly inconsistent with the visible scope."
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"id": "risk",
|
|
42
|
+
"name": "Risk coverage",
|
|
43
|
+
"cross_cutting": true,
|
|
44
|
+
"weight": 2,
|
|
45
|
+
"criterion": "Top risks are listed (e.g., auth surface, data migration, external dependency). At least one mitigation or 'watch out for' note is present. Low-risk tickets may state 'no identified risks' with full credit.",
|
|
46
|
+
"gap_signal": "Ticket touches auth, tenancy, or data migration but no risks are listed."
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"id": "test_plan",
|
|
50
|
+
"name": "Test plan",
|
|
51
|
+
"cross_cutting": true,
|
|
52
|
+
"weight": 2,
|
|
53
|
+
"criterion": "The ticket states what kind of tests will cover it — unit, integration, E2E, or 'covered by AC'. Edge cases and failure paths are named. 'TBD' fails this dimension.",
|
|
54
|
+
"gap_signal": "No test plan; or test plan is 'write tests' without specifying scope or edge cases."
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"id": "dependencies",
|
|
58
|
+
"name": "Dependencies",
|
|
59
|
+
"cross_cutting": false,
|
|
60
|
+
"weight": 1,
|
|
61
|
+
"criterion": "Upstream and downstream dependencies are named (tickets, services, teams). Blocked-by links exist in the tracker. 'No dependencies' is acceptable with full credit when true.",
|
|
62
|
+
"gap_signal": "References to other work exist in the description text but are not linked in the tracker; or known blockers are undeclared."
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "maintainability",
|
|
66
|
+
"name": "Maintainability considerations",
|
|
67
|
+
"cross_cutting": true,
|
|
68
|
+
"weight": 1,
|
|
69
|
+
"criterion": "If the ticket introduces long-lived output (a new module, a schema change, a new public endpoint), ownership and refresh cadence are named. Short-lived or purely additive work is exempt (full credit).",
|
|
70
|
+
"gap_signal": "A new long-lived module or schema is introduced with no named owner or documentation note."
|
|
71
|
+
}
|
|
72
|
+
]
|
|
73
|
+
}
|