@event4u/agent-config 7.3.0 → 7.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +4 -1
- package/CHANGELOG.md +81 -0
- package/README.md +2 -2
- package/dist/agent-src/commands/fix/pr-comments.md +42 -13
- package/dist/agent-src/commands/review-changes.md +14 -7
- package/dist/agent-src/rules/artifact-engagement-recording.md +9 -0
- package/dist/agent-src/rules/cli-output-handling.md +18 -0
- package/dist/agent-src/rules/domain-safety-pii.md +4 -3
- package/dist/agent-src/rules/icon-consistency.md +1 -0
- package/dist/agent-src/rules/legal-safety-floor.md +2 -2
- package/dist/agent-src/rules/untrusted-input-defense.md +8 -5
- package/dist/agent-src/skills/adr-create/SKILL.md +5 -1
- package/dist/agent-src/skills/adversarial-review/SKILL.md +1 -1
- package/dist/agent-src/skills/blade-ui/SKILL.md +4 -0
- package/dist/agent-src/skills/brand-to-tokens/SKILL.md +2 -1
- package/dist/agent-src/skills/code-review/SKILL.md +4 -0
- package/dist/agent-src/skills/context-authoring/SKILL.md +1 -1
- package/dist/agent-src/skills/design-intelligence/SKILL.md +40 -0
- package/dist/agent-src/skills/design-review/SKILL.md +23 -11
- package/dist/agent-src/skills/design-system-capture/SKILL.md +41 -0
- package/dist/agent-src/skills/design-system-capture/reference/design-system-json.md +62 -0
- package/dist/agent-src/skills/design-tokens/SKILL.md +1 -0
- package/dist/agent-src/skills/existing-ui-audit/SKILL.md +6 -0
- package/dist/agent-src/skills/flux/SKILL.md +4 -0
- package/dist/agent-src/skills/iconography/SKILL.md +1 -0
- package/dist/agent-src/skills/image-editing/evals/triggers.json +13 -13
- package/dist/agent-src/skills/judge-artifact-completeness/SKILL.md +136 -0
- package/dist/agent-src/skills/judge-artifact-completeness/calibration/fixtures.json +149 -0
- package/dist/agent-src/skills/judge-artifact-completeness/evals/triggers.json +17 -0
- package/dist/agent-src/skills/judge-artifact-completeness/rubrics/architecture-score.json +73 -0
- package/dist/agent-src/skills/judge-artifact-completeness/rubrics/pr-review-score.json +65 -0
- package/dist/agent-src/skills/judge-artifact-completeness/rubrics/roadmap-score.json +73 -0
- package/dist/agent-src/skills/judge-artifact-completeness/rubrics/ticket-quality-score.json +73 -0
- package/dist/agent-src/skills/judge-injection-defense/SKILL.md +99 -0
- package/dist/agent-src/skills/judge-injection-defense/calibration/fixtures.json +24 -0
- package/dist/agent-src/skills/judge-injection-defense/evals/triggers.json +17 -0
- package/dist/agent-src/skills/judge-injection-defense/fixtures/behavioral-fixtures.json +63 -0
- package/dist/agent-src/skills/judge-injection-defense/fixtures/coverage-report.md +53 -0
- package/dist/agent-src/skills/judge-injection-defense/fixtures/perturbation-taxonomy.json +110 -0
- package/dist/agent-src/skills/judge-synthesis/SKILL.md +134 -0
- package/dist/agent-src/skills/judge-synthesis/evals/triggers.json +17 -0
- package/dist/agent-src/skills/logo-generation/SKILL.md +1 -1
- package/dist/agent-src/skills/markitdown/SKILL.md +1 -1
- package/dist/agent-src/skills/react-shadcn-ui/SKILL.md +18 -0
- package/dist/agent-src/skills/react-shadcn-ui/reference/registry.md +59 -0
- package/dist/agent-src/skills/refine-ticket/SKILL.md +5 -1
- package/dist/agent-src/skills/release-comms/SKILL.md +1 -1
- package/dist/agent-src/skills/roadmap-management/SKILL.md +4 -0
- package/dist/agent-src/skills/roadmap-writing/SKILL.md +1 -1
- package/dist/agent-src/skills/script-writing/SKILL.md +1 -1
- package/dist/agent-src/skills/sql-writing/SKILL.md +1 -1
- package/dist/agent-src/skills/systematic-debugging/SKILL.md +1 -1
- package/dist/agent-src/skills/tailwind-engineer/SKILL.md +4 -0
- package/dist/agent-src/skills/testing-anti-patterns/SKILL.md +1 -1
- package/dist/agent-src/skills/token-optimizer/SKILL.md +1 -1
- package/dist/agent-src/skills/typography-system/SKILL.md +2 -1
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/cli/registry.js +1 -0
- package/dist/cli/registry.js.map +1 -1
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +109 -43
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +3 -3
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +8 -5
- package/dist/discovery/trust-report.md +2 -2
- package/dist/discovery/workspaces.json +7 -4
- package/dist/mcp/registry-manifest.json +3 -3
- package/dist/router.json +1 -1
- package/dist/server/schemas/settings.js +6 -0
- package/dist/server/schemas/settings.js.map +1 -1
- package/docs/architecture.md +2 -2
- package/docs/catalog.md +28 -20
- package/docs/command-flows.md +1 -1
- package/docs/enforcement-by-host.md +9 -0
- package/docs/governance-advantage.md +72 -0
- package/docs/guidelines/design-antipatterns.md +26 -0
- package/docs/guidelines/design-canon.md +62 -0
- package/package.json +2 -1
- package/src/config/agent-settings.template.yml +21 -0
- package/src/scripts/_cli/cmd_analyze_session.ts +347 -0
- package/src/scripts/_dispatch.bash +8 -0
- package/src/scripts/_lib/token_count.ts +110 -35
- package/src/scripts/audit_initial_context.ts +37 -12
- package/src/scripts/bench_quality_run.ts +253 -0
- package/src/scripts/bench_rule_load_latency.ts +178 -0
- package/src/scripts/check_kernel_prefix_stability.ts +0 -0
- package/src/scripts/check_quality_regression.ts +244 -0
- package/src/scripts/check_token_quality_golden.ts +253 -0
- package/src/scripts/check_token_regression.ts +266 -0
- package/src/scripts/design_slop_rules.test.ts +193 -0
- package/src/scripts/design_slop_rules.ts +502 -0
- package/src/scripts/external_sources_denylist.json +7 -1
- package/src/scripts/hook_manifest.yaml +26 -3
- package/src/scripts/hooks/design_slop_hook.ts +181 -0
- package/src/scripts/hooks/rtk_wrap_hook.ts +228 -0
- package/src/scripts/lint_agent_security.ts +1 -0
- package/src/scripts/lint_confusables.ts +197 -0
- package/src/scripts/lint_design_slop.ts +245 -0
- package/src/scripts/probe_host_compliance.ts +145 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# `design-system.json` — import contract
|
|
2
|
+
|
|
3
|
+
> Lazy-loaded by `design-system-capture` ONLY when importing an extracted design
|
|
4
|
+
> system. Not in the always-on skill body. The **consumer-side contract**: any
|
|
5
|
+
> external static-extraction tool (URL / git repo / local dir) emits this shape;
|
|
6
|
+
> the skill READS it to seed/merge `DESIGN.md`. We own the contract, not the
|
|
7
|
+
> crawler (council 2026-06-28).
|
|
8
|
+
|
|
9
|
+
## Shape
|
|
10
|
+
|
|
11
|
+
```json
|
|
12
|
+
{
|
|
13
|
+
"source": { "kind": "url|repo|dir", "ref": "https://example.com", "captured_at": "<ISO-8601>" },
|
|
14
|
+
"colors": {
|
|
15
|
+
"light": { "<role>": "<value>" },
|
|
16
|
+
"dark": { "<role>": "<value>" }
|
|
17
|
+
},
|
|
18
|
+
"typography": {
|
|
19
|
+
"families": [{ "role": "body|display|mono", "name": "<font>", "bundled_local": false }],
|
|
20
|
+
"scale": [{ "step": "xs|sm|base|lg|…", "size": "<rem/px>", "lineHeight": "<n>" }]
|
|
21
|
+
},
|
|
22
|
+
"spacing": { "base": "<px>", "scale": ["<px>", "…"] },
|
|
23
|
+
"radius": { "<role>": "<px>" },
|
|
24
|
+
"shadow": { "<role>": "<box-shadow>" },
|
|
25
|
+
"motion": { "durations": { "<role>": "<ms>" }, "easings": { "<role>": "<cubic-bezier>" },
|
|
26
|
+
"_meta": { "detected_libs": ["gsap", "lottie"] } },
|
|
27
|
+
"components": [{ "name": "<Name>", "observed": { "classes": ["…"], "props": ["…"] } }]
|
|
28
|
+
}
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Field rules
|
|
32
|
+
|
|
33
|
+
- **Map to DTCG where it maps cleanly.** `colors` / `typography` / `spacing` /
|
|
34
|
+
`radius` / `shadow` = the DTCG `.tokens.json` shape `design-tokens` /
|
|
35
|
+
`brand-to-tokens` already author — reuse it; never a parallel token format.
|
|
36
|
+
- **Mark extraction-only metadata.** Observation, not a token decision, lives
|
|
37
|
+
under `_meta` (`motion._meta.detected_libs`, `components[].observed`). Informs
|
|
38
|
+
the human; never becomes a token.
|
|
39
|
+
- **`source` is mandatory** — `kind` + `ref` + `captured_at`. No provenance →
|
|
40
|
+
reject (can't confirm what you can't trace).
|
|
41
|
+
- **`bundled_local`** is a flag, not an instruction: the package never
|
|
42
|
+
downloads/bundles fonts (out of scope); it records that the source did.
|
|
43
|
+
|
|
44
|
+
## Trust posture (mandatory)
|
|
45
|
+
|
|
46
|
+
An import is **observed, not authoritative** (mirrors `source-discovery`:
|
|
47
|
+
evidence vs. authoritative). It seeds `DESIGN.md` as a **proposal the human
|
|
48
|
+
confirms per field** — never a silent write. A field conflicting with a
|
|
49
|
+
registered brand value (confirmed `.tokens.json` / brand token) is **flagged,
|
|
50
|
+
never auto-applied** (`brand-source-of-truth`). Precedence: registered brand
|
|
51
|
+
tokens > confirmed `DESIGN.md` > imported observation.
|
|
52
|
+
|
|
53
|
+
## Two sources, one shape
|
|
54
|
+
|
|
55
|
+
- **External target** (a site/repo you don't own) → external standalone tool
|
|
56
|
+
emits `design-system.json`; hand it to `design-system-capture`.
|
|
57
|
+
- **Current repo** → prefer [`existing-ui-audit`](../../existing-ui-audit/SKILL.md):
|
|
58
|
+
it inventories the codebase and emits the same shape, so the import path is
|
|
59
|
+
identical either way.
|
|
60
|
+
|
|
61
|
+
Package **owns this contract**; does **not** ship the crawler, the Playwright
|
|
62
|
+
runtime, or a font-bundler.
|
|
@@ -129,6 +129,7 @@ four-operation split assigns to *rules/linters*, not the corpus).
|
|
|
129
129
|
|
|
130
130
|
## See also
|
|
131
131
|
|
|
132
|
+
- [`design-canon.md`](../../../docs/guidelines/design-canon.md) § Colour references — a11y-contrast + culturally-specific palettes + named-system token models.
|
|
132
133
|
- [`design-intelligence`](../design-intelligence/SKILL.md) — grounded value selection.
|
|
133
134
|
- [`tailwind-engineer`](../tailwind-engineer/SKILL.md) — utility-discipline consumer.
|
|
134
135
|
- [`react-shadcn-ui`](../react-shadcn-ui/SKILL.md) — shadcn token conventions.
|
|
@@ -199,6 +199,12 @@ violations will surface as new findings on every run.
|
|
|
199
199
|
7. **`state.ui_audit.greenfield`** — boolean; when true, `greenfield_decision` MUST also be set before the dispatcher advances
|
|
200
200
|
8. **`state.ui_audit.a11y_baseline`** *(optional)* — array of `{rule, selector, severity?}` entries documenting pre-existing a11y violations the review gate should treat as informational. Omit the key entirely when no baseline applies; do not write `[]` for "I checked and there are none" — that disables the gate's filter for every future run.
|
|
201
201
|
|
|
202
|
+
**Design-system export (current-repo source):** to seed `DESIGN.md` from *this*
|
|
203
|
+
repo, emit the inventory as a `design-system.json` artifact (the import contract
|
|
204
|
+
in [`design-system-capture`](../design-system-capture/reference/design-system-json.md))
|
|
205
|
+
and hand it to `design-system-capture` — same shape an external tool produces,
|
|
206
|
+
so the import path is uniform.
|
|
207
|
+
|
|
202
208
|
## Gotcha
|
|
203
209
|
|
|
204
210
|
- The model tends to skip the audit and start designing straight from the request — the dispatcher gate at `directives/ui/audit.py` enforces "no design without audit findings". Never treat this skill as optional for non-trivial UI.
|
|
@@ -112,6 +112,10 @@ current state on the next polish round.
|
|
|
112
112
|
- Flux has built-in validation display — don't add manual error rendering alongside it.
|
|
113
113
|
- Don't mix Flux with raw HTML form elements in the same form.
|
|
114
114
|
|
|
115
|
+
## Taste Dials
|
|
116
|
+
|
|
117
|
+
`DESIGN.md` `## Taste Dials` → honour: Variance → layout-family spread + asymmetry; Motion → animation budget + reduced-motion posture; Density → spacing scale + info-per-viewport. Absent → follow brief's inferred dials.
|
|
118
|
+
|
|
115
119
|
## Do NOT
|
|
116
120
|
|
|
117
121
|
- Do NOT use raw `<input>`, `<select>`, `<button>` when Flux equivalents exist.
|
|
@@ -82,6 +82,7 @@ execution:
|
|
|
82
82
|
|
|
83
83
|
## See also
|
|
84
84
|
|
|
85
|
+
- [`design-canon.md`](../../../docs/guidelines/design-canon.md) § Icon systems — prefer a named system's icon set (Material Symbols / SF Symbols / Fluent / Carbon) when that system is in play.
|
|
85
86
|
- [`design-tokens`](../design-tokens/SKILL.md) — token system the icon color/size variables should reference.
|
|
86
87
|
- [`typography-system`](../typography-system/SKILL.md) — sibling visual-consistency skill.
|
|
87
88
|
- [`fe-design`](../fe-design/SKILL.md) — broader frontend design discipline.
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
{
|
|
2
|
+
"skill": "image-editing",
|
|
2
3
|
"last_eval": "2026-06-16",
|
|
3
|
-
"
|
|
4
|
-
|
|
5
|
-
"
|
|
6
|
-
"
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
"
|
|
12
|
-
"
|
|
13
|
-
"
|
|
14
|
-
"
|
|
15
|
-
"do I need a rights check before using a celebrity's likeness in my image?"
|
|
4
|
+
"description": "5 should-trigger + 5 should-not-trigger. Should-trigger covers inpainting, background replacement, colour variation, object removal, and targeted render modification. Should-not covers near-miss neighbours: greenfield image generation (image-creator/image-generation), provider selection (image-provider-routing), prompt writing (image-creator), rights/licence check (media governance).",
|
|
5
|
+
"queries": [
|
|
6
|
+
{ "q": "edit the background of this product shot to a white studio setting", "trigger": true },
|
|
7
|
+
{ "q": "inpaint the logo out of the upper-right corner of this image", "trigger": true },
|
|
8
|
+
{ "q": "make a variation of this hero image in a warmer colour palette", "trigger": true },
|
|
9
|
+
{ "q": "remove the person standing on the left side of the photo", "trigger": true },
|
|
10
|
+
{ "q": "modify this existing render — I want the sky replaced with a sunset", "trigger": true },
|
|
11
|
+
{ "q": "generate a new product shot from scratch — I have no existing image", "trigger": false, "note": "greenfield generation → image-creator / image-generation" },
|
|
12
|
+
{ "q": "which image provider should I use for this job?", "trigger": false, "note": "provider routing → image-provider-routing" },
|
|
13
|
+
{ "q": "create a logo for my company", "trigger": false, "note": "logo creation → logo-generation" },
|
|
14
|
+
{ "q": "how do I write a good prompt for Ideogram?", "trigger": false, "note": "prompt engineering → image-creator" },
|
|
15
|
+
{ "q": "do I need a rights check before using a celebrity's likeness in my image?", "trigger": false, "note": "likeness / rights → media governance rules" }
|
|
16
16
|
]
|
|
17
17
|
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
---
|
|
2
|
+
model_tier: high
|
|
3
|
+
name: judge-artifact-completeness
|
|
4
|
+
description: "Use when scoring a roadmap, PR review, ADR, or ticket for completeness — risk, tests, migration, maintainability. Dispatched by /refine-ticket, /adr-create, /review-changes; never auto-gates."
|
|
5
|
+
domain: quality
|
|
6
|
+
workspaces:
|
|
7
|
+
- agent-config-maintainer
|
|
8
|
+
packs:
|
|
9
|
+
- meta
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
# judge-artifact-completeness
|
|
13
|
+
|
|
14
|
+
> Judge specialized in **artifact completeness**. Score non-code deliverables — roadmap, PR review, ADR, ticket — against a rubric and surface missing/partial dimensions. Does **not** judge code quality, bugs, or security. Never auto-gates: score + gaps → human decides.
|
|
15
|
+
|
|
16
|
+
## When to use
|
|
17
|
+
|
|
18
|
+
* Roadmap produced — score completeness against AC, risk, migration surface.
|
|
19
|
+
* PR review complete — check evidence quality + test coverage.
|
|
20
|
+
* ADR drafted — alternatives / consequences / reversibility completeness pass.
|
|
21
|
+
* Ticket exits refinement — DoR readiness check.
|
|
22
|
+
* `/refine-ticket`, `/adr-create`, `/roadmap:create`, `/review-changes` surface score as optional output pass.
|
|
23
|
+
|
|
24
|
+
Do NOT use when:
|
|
25
|
+
|
|
26
|
+
* Code quality / naming / DRY → [`judge-code-quality`](../judge-code-quality/SKILL.md)
|
|
27
|
+
* Functional bug → [`judge-bug-hunter`](../judge-bug-hunter/SKILL.md)
|
|
28
|
+
* Missing test files → [`judge-test-coverage`](../judge-test-coverage/SKILL.md)
|
|
29
|
+
* Security issue → [`judge-security-auditor`](../judge-security-auditor/SKILL.md)
|
|
30
|
+
|
|
31
|
+
## Procedure
|
|
32
|
+
|
|
33
|
+
### 1. Identify artifact type
|
|
34
|
+
|
|
35
|
+
| Artifact | Rubric schema |
|
|
36
|
+
|---|---|
|
|
37
|
+
| Roadmap / plan | `rubrics/roadmap-score.json` |
|
|
38
|
+
| PR review / code-review comment | `rubrics/pr-review-score.json` |
|
|
39
|
+
| ADR / architecture decision | `rubrics/architecture-score.json` |
|
|
40
|
+
| Jira / Linear ticket | `rubrics/ticket-quality-score.json` |
|
|
41
|
+
|
|
42
|
+
Ambiguous type → one question before scoring.
|
|
43
|
+
|
|
44
|
+
### 2. Score each dimension
|
|
45
|
+
|
|
46
|
+
* **0** — absent. Criterion not addressed.
|
|
47
|
+
* **1** — partial. Mentioned but too vague to be actionable (e.g., "risks exist" without naming one).
|
|
48
|
+
* **weight** — fully present. Criterion met concretely and traceably.
|
|
49
|
+
|
|
50
|
+
Use only `criterion` field. Do not penalise for style or length. Short artifact covering all dimensions scores same as long one — completeness ≠ verbosity.
|
|
51
|
+
|
|
52
|
+
Mark **N/A** (full credit) only when rubric schema explicitly allows it.
|
|
53
|
+
|
|
54
|
+
### 3. Compute verdict
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
total_earned = sum of all dimension scores (N/A = weight)
|
|
58
|
+
total_possible = sum of all dimension weights
|
|
59
|
+
ratio = total_earned / total_possible
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
| Ratio | Verdict |
|
|
63
|
+
|---|---|
|
|
64
|
+
| ≥ 0.80 | `complete` |
|
|
65
|
+
| ≥ 0.50 | `partial` |
|
|
66
|
+
| < 0.50 | `incomplete` |
|
|
67
|
+
|
|
68
|
+
### 4. Surface gaps
|
|
69
|
+
|
|
70
|
+
Top 1–3 gaps — dimensions with score = 0 or 1, highest weight. Gap entry names dimension + specific unmet criterion.
|
|
71
|
+
|
|
72
|
+
## Validation
|
|
73
|
+
|
|
74
|
+
Before finalising:
|
|
75
|
+
|
|
76
|
+
1. Every scored dimension maps to rubric schema field.
|
|
77
|
+
2. No dimension penalised for length or word count.
|
|
78
|
+
3. N/A credit granted only where schema allows.
|
|
79
|
+
4. Verdict follows ratio thresholds, not intuition.
|
|
80
|
+
5. Top gaps = highest-weight missing — not every minor gap.
|
|
81
|
+
|
|
82
|
+
## Output format
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
Judge: judge-artifact-completeness
|
|
86
|
+
Type: roadmap | pr-review | architecture | ticket
|
|
87
|
+
Target: <one-line artifact description>
|
|
88
|
+
Verdict: complete | partial | incomplete
|
|
89
|
+
Score: <earned>/<possible> (<pct>%)
|
|
90
|
+
|
|
91
|
+
Dimensions:
|
|
92
|
+
✅ <dimension-name> (<earned>/<weight>) — <one-line note, or "meets criterion">
|
|
93
|
+
⚠️ <dimension-name> (<earned>/<weight>) — PARTIAL: <specific gap>
|
|
94
|
+
❌ <dimension-name> (0/<weight>) — MISSING: <what would satisfy the criterion>
|
|
95
|
+
|
|
96
|
+
Top gaps:
|
|
97
|
+
1. <highest-weight missing dimension>: <concrete action to close the gap>
|
|
98
|
+
2. ...
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Required fields (ordered):
|
|
102
|
+
|
|
103
|
+
1. **Judge**, **Type**, **Target** — identification
|
|
104
|
+
2. **Verdict** — `complete`, `partial`, or `incomplete`
|
|
105
|
+
3. **Score** — raw earned/possible and percentage
|
|
106
|
+
4. **Dimensions** — one line per dimension with emoji + score + note
|
|
107
|
+
5. **Top gaps** — highest-weight missing with concrete close action
|
|
108
|
+
|
|
109
|
+
Output surfaced as recommendation. Human decides whether to act on gaps.
|
|
110
|
+
|
|
111
|
+
## Gotcha
|
|
112
|
+
|
|
113
|
+
* **Length ≠ completeness** — terse complete roadmap scores same as long one.
|
|
114
|
+
* **N/A abuse** — `migration_effort` is only N/A when no public-interface change. Mark 1 (partial) when unsure.
|
|
115
|
+
* **Partial credit creep** — "mentioned but vague" = partial (1), not full credit. "Risks exist" without naming one = partial.
|
|
116
|
+
* **Verdict as gate** — recommendation, never blocker. Surface it; human decides.
|
|
117
|
+
|
|
118
|
+
## Do NOT
|
|
119
|
+
|
|
120
|
+
* NEVER penalise artifact for being short or concise
|
|
121
|
+
* NEVER grant full credit to vague mention — that is partial (1)
|
|
122
|
+
* NEVER auto-reject or auto-approve based on verdict alone
|
|
123
|
+
* NEVER score code quality, correctness, or security — out of scope
|
|
124
|
+
* NEVER invent dimensions not in rubric schema
|
|
125
|
+
|
|
126
|
+
## Calibration
|
|
127
|
+
|
|
128
|
+
Fixtures: `calibration/fixtures.json`. Rubric designed monotone: removing a fully-present dimension lowers score by at least `dimension.weight`. No single dimension dominates (max weight = 3; total possible = 12–15 per rubric).
|
|
129
|
+
|
|
130
|
+
Anti-length: fixtures include `SHORT_COMPLETE` (short, all dimensions) and `LONG_INCOMPLETE` (verbose, missing high-weight dimensions). Correct judge scores `SHORT_COMPLETE` > `LONG_INCOMPLETE`.
|
|
131
|
+
|
|
132
|
+
## References
|
|
133
|
+
|
|
134
|
+
- Sibling judges: [`judge-code-quality`](../judge-code-quality/SKILL.md), [`judge-bug-hunter`](../judge-bug-hunter/SKILL.md), [`judge-security-auditor`](../judge-security-auditor/SKILL.md), [`judge-test-coverage`](../judge-test-coverage/SKILL.md).
|
|
135
|
+
- Dispatchers: [`/refine-ticket`](../../commands/refine-ticket.md), [`/adr-create`](../../commands/adr-create.md), [`/review-changes`](../../commands/review-changes.md).
|
|
136
|
+
- Rubric schemas: `rubrics/roadmap-score.json`, `rubrics/pr-review-score.json`, `rubrics/architecture-score.json`, `rubrics/ticket-quality-score.json`.
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
{
|
|
2
|
+
"description": "Calibration fixtures for judge-artifact-completeness. Used to verify (a) monotonicity — removing a dimension lowers the score by exactly its weight; (b) strict tier ordering — EXCELLENT > GOOD > FAIR > POOR > TERRIBLE; (c) anti-length — SHORT_COMPLETE scores higher than LONG_INCOMPLETE.",
|
|
3
|
+
"rubric": "roadmap-score",
|
|
4
|
+
"gold_tiers": {
|
|
5
|
+
"scale": "0–1 mapped to earned/possible ratio",
|
|
6
|
+
"EXCELLENT": { "expected_ratio_min": 0.90, "expected_verdict": "complete" },
|
|
7
|
+
"GOOD": { "expected_ratio_min": 0.70, "expected_verdict": "partial" },
|
|
8
|
+
"FAIR": { "expected_ratio_min": 0.50, "expected_verdict": "partial" },
|
|
9
|
+
"POOR": { "expected_ratio_min": 0.20, "expected_verdict": "incomplete" },
|
|
10
|
+
"TERRIBLE": { "expected_ratio_min": 0.0, "expected_verdict": "incomplete" }
|
|
11
|
+
},
|
|
12
|
+
"anti_length_assertion": {
|
|
13
|
+
"description": "SHORT_COMPLETE must score strictly above LONG_INCOMPLETE regardless of word count.",
|
|
14
|
+
"rule": "score(SHORT_COMPLETE) > score(LONG_INCOMPLETE)"
|
|
15
|
+
},
|
|
16
|
+
"fixtures": [
|
|
17
|
+
{
|
|
18
|
+
"id": "EXCELLENT",
|
|
19
|
+
"label": "All dimensions fully present, terse",
|
|
20
|
+
"artifact": {
|
|
21
|
+
"goal": "Ship a typed parser for the config format by 2026-Q3.",
|
|
22
|
+
"acceptance_criteria": "task test exits 0 on the full corpus; zero regressions in existing consumers; coverage ≥ 90% per codecov.",
|
|
23
|
+
"phases": [
|
|
24
|
+
{
|
|
25
|
+
"name": "Phase 1 — Lexer",
|
|
26
|
+
"deliverable": "A tokeniser with 100% token-type coverage.",
|
|
27
|
+
"verification": "Pest suite: 40 fixtures, all green."
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"name": "Phase 2 — Parser",
|
|
31
|
+
"deliverable": "A recursive-descent parser emitting a typed AST.",
|
|
32
|
+
"verification": "Integration test: parse all fixtures; compare AST to golden JSON."
|
|
33
|
+
}
|
|
34
|
+
],
|
|
35
|
+
"risk": "Ambiguous grammar rule for nested keys — spike required in Phase 1 before committing to the AST shape.",
|
|
36
|
+
"migration": "Existing consumers use the old string-based API. A shim is provided in Phase 2; shim deprecated in Phase 3.",
|
|
37
|
+
"owner": "matze4u; quarterly review each January.",
|
|
38
|
+
"do_not_build": "No serialiser (inverse direction). No streaming parser. No error recovery beyond fail-fast."
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"id": "GOOD",
|
|
43
|
+
"label": "Acceptance criteria and most dimensions present; migration vague",
|
|
44
|
+
"artifact": {
|
|
45
|
+
"goal": "Ship a typed parser for the config format.",
|
|
46
|
+
"acceptance_criteria": "All existing tests pass after migration.",
|
|
47
|
+
"phases": [
|
|
48
|
+
{ "name": "Phase 1 — Lexer", "deliverable": "Tokeniser." },
|
|
49
|
+
{ "name": "Phase 2 — Parser", "deliverable": "AST." }
|
|
50
|
+
],
|
|
51
|
+
"risk": "Grammar ambiguity may require a rework.",
|
|
52
|
+
"migration": "Migration needed for consumers.",
|
|
53
|
+
"owner": "matze4u",
|
|
54
|
+
"do_not_build": "No serialiser."
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"id": "FAIR",
|
|
59
|
+
"label": "Acceptance criteria and do-not-build missing; phases underdefined",
|
|
60
|
+
"artifact": {
|
|
61
|
+
"goal": "Build a config parser.",
|
|
62
|
+
"phases": [
|
|
63
|
+
{ "name": "Phase 1", "deliverable": "Lexer." },
|
|
64
|
+
{ "name": "Phase 2", "deliverable": "Parser." }
|
|
65
|
+
],
|
|
66
|
+
"risk": "Grammar issues may arise.",
|
|
67
|
+
"owner": "matze4u"
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"id": "POOR",
|
|
72
|
+
"label": "Only a goal and phase list; no cross-cutting dimensions",
|
|
73
|
+
"artifact": {
|
|
74
|
+
"goal": "Build a config parser.",
|
|
75
|
+
"phases": [
|
|
76
|
+
{ "name": "Phase 1", "deliverable": "Lexer." },
|
|
77
|
+
{ "name": "Phase 2", "deliverable": "Parser." }
|
|
78
|
+
]
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"id": "TERRIBLE",
|
|
83
|
+
"label": "Barely any content",
|
|
84
|
+
"artifact": {
|
|
85
|
+
"goal": "Improve parsing."
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
"id": "SHORT_COMPLETE",
|
|
90
|
+
"label": "Anti-length: short artifact, all dimensions covered",
|
|
91
|
+
"word_count_approx": 80,
|
|
92
|
+
"dimensions_present": ["acceptance_criteria_fit", "phase_decomposition", "risk", "tests_per_step", "migration_effort", "maintainability_owner", "do_not_build_clarity"],
|
|
93
|
+
"artifact": {
|
|
94
|
+
"goal": "Ship a typed config parser.",
|
|
95
|
+
"acceptance_criteria": "task test green; ≥90% coverage.",
|
|
96
|
+
"phases": [{ "name": "Phase 1", "deliverable": "Lexer + parser.", "verification": "Pest suite, 40 fixtures green." }],
|
|
97
|
+
"risk": "Grammar ambiguity — spike in Phase 1.",
|
|
98
|
+
"migration": "Shim provided; old API deprecated in Phase 2.",
|
|
99
|
+
"owner": "matze4u",
|
|
100
|
+
"do_not_build": "No serialiser."
|
|
101
|
+
}
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
"id": "LONG_INCOMPLETE",
|
|
105
|
+
"label": "Anti-length: verbose artifact, missing risk + tests + migration",
|
|
106
|
+
"word_count_approx": 400,
|
|
107
|
+
"dimensions_present": ["phase_decomposition"],
|
|
108
|
+
"artifact": {
|
|
109
|
+
"goal": "We need to significantly overhaul our configuration parsing layer to support the new typed AST format that the team has been discussing over the last several sprints. This will involve extensive rework of the tokenisation logic as well as the construction of a proper recursive-descent parser that can handle all the edge cases we have identified in our current configuration files.",
|
|
110
|
+
"phases": [
|
|
111
|
+
{
|
|
112
|
+
"name": "Phase 1 — Lexer Overhaul",
|
|
113
|
+
"deliverable": "A comprehensive rewrite of the tokeniser component to support the full range of token types that we expect to encounter in our configuration files. The tokeniser will be responsible for converting raw input text into a structured stream of tokens that can be consumed by the parser in Phase 2."
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"name": "Phase 2 — Parser Implementation",
|
|
117
|
+
"deliverable": "A recursive-descent parser that consumes the token stream produced by the Phase 1 tokeniser and constructs a fully typed abstract syntax tree. The parser will handle nested structures, arrays, inline tables, and all other grammar constructs."
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
"name": "Phase 3 — Consumer Migration",
|
|
121
|
+
"deliverable": "Update all downstream consumers to use the new typed API. This will require changes across multiple modules and may involve coordination with other teams."
|
|
122
|
+
}
|
|
123
|
+
],
|
|
124
|
+
"owner": "The team."
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
],
|
|
128
|
+
"dimension_ablation": {
|
|
129
|
+
"description": "Monotonicity test: take EXCELLENT, remove one dimension at a time, verify score drops by exactly that dimension's weight.",
|
|
130
|
+
"base_fixture": "EXCELLENT",
|
|
131
|
+
"ablations": [
|
|
132
|
+
{ "remove_dimension": "acceptance_criteria_fit", "expected_score_drop": 3 },
|
|
133
|
+
{ "remove_dimension": "phase_decomposition", "expected_score_drop": 2 },
|
|
134
|
+
{ "remove_dimension": "risk", "expected_score_drop": 2 },
|
|
135
|
+
{ "remove_dimension": "tests_per_step", "expected_score_drop": 2 },
|
|
136
|
+
{ "remove_dimension": "migration_effort", "expected_score_drop": 2 },
|
|
137
|
+
{ "remove_dimension": "maintainability_owner", "expected_score_drop": 2 },
|
|
138
|
+
{ "remove_dimension": "do_not_build_clarity", "expected_score_drop": 1 }
|
|
139
|
+
],
|
|
140
|
+
"dominance_check": {
|
|
141
|
+
"description": "No single dimension should exceed 25% of total possible score.",
|
|
142
|
+
"max_weight": 3,
|
|
143
|
+
"total_possible": 14,
|
|
144
|
+
"max_allowed_share": 0.25,
|
|
145
|
+
"actual_max_share": 0.214,
|
|
146
|
+
"pass": true
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill": "judge-artifact-completeness",
|
|
3
|
+
"last_eval": "2026-06-27",
|
|
4
|
+
"description": "5 should-trigger + 5 should-not-trigger. Should-trigger covers scoring roadmaps, PR reviews, ADRs, and tickets for completeness (risk, tests, migration, AC, DoR). Should-not covers near-miss neighbours: code-quality review (judge-code-quality), bug hunting (judge-bug-hunter), test-file coverage (judge-test-coverage), security audit (judge-security-auditor), and creating an ADR from scratch (adr-create).",
|
|
5
|
+
"queries": [
|
|
6
|
+
{ "q": "score this roadmap for completeness — does it cover risk, tests, and migration?", "trigger": true },
|
|
7
|
+
{ "q": "does this PR review have enough evidence? is each finding traceable to a diff line?", "trigger": true },
|
|
8
|
+
{ "q": "check if this ticket meets DoR — are the acceptance criteria verifiable?", "trigger": true },
|
|
9
|
+
{ "q": "is this ADR complete? does it name alternatives and state consequences?", "trigger": true },
|
|
10
|
+
{ "q": "what dimensions are missing from this roadmap — risk, maintainability, migration?", "trigger": true },
|
|
11
|
+
{ "q": "review the naming and DRY violations in this diff", "trigger": false, "note": "code quality → judge-code-quality" },
|
|
12
|
+
{ "q": "find the bugs introduced by this change", "trigger": false, "note": "bug hunting → judge-bug-hunter" },
|
|
13
|
+
{ "q": "does this PR add enough test coverage for the new behaviour?", "trigger": false, "note": "test-file coverage → judge-test-coverage" },
|
|
14
|
+
{ "q": "are there security issues in this authentication middleware?", "trigger": false, "note": "security → judge-security-auditor" },
|
|
15
|
+
{ "q": "create an ADR for switching from REST to GraphQL", "trigger": false, "note": "ADR creation → adr-create, not scoring" }
|
|
16
|
+
]
|
|
17
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
{
|
|
2
|
+
"rubric": "architecture-score",
|
|
3
|
+
"version": "1.0",
|
|
4
|
+
"artifact_type": "architecture",
|
|
5
|
+
"description": "Scores an ADR or architecture decision for completeness: decision clarity, alternatives, consequences, and the four cross-cutting dimensions.",
|
|
6
|
+
"anchors": ["adr-create", "decision-record"],
|
|
7
|
+
"scoring": {
|
|
8
|
+
"scale": "0 = absent, 1 = partial (mentioned but vague), weight = fully present and concrete",
|
|
9
|
+
"verdict_thresholds": {
|
|
10
|
+
"complete": 0.80,
|
|
11
|
+
"partial": 0.50,
|
|
12
|
+
"incomplete": 0.0
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"dimensions": [
|
|
16
|
+
{
|
|
17
|
+
"id": "decision_clarity",
|
|
18
|
+
"name": "Decision clarity",
|
|
19
|
+
"cross_cutting": false,
|
|
20
|
+
"weight": 3,
|
|
21
|
+
"criterion": "The chosen option is stated unambiguously in one sentence. The scope of the decision is bounded — it is clear what is inside vs. outside the decision. Status (Accepted / Proposed / Superseded) is set.",
|
|
22
|
+
"gap_signal": "The decision section is prose about the problem rather than a declarative statement of the choice; or the status field is missing."
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "alternatives_considered",
|
|
26
|
+
"name": "Alternatives considered",
|
|
27
|
+
"cross_cutting": false,
|
|
28
|
+
"weight": 2,
|
|
29
|
+
"criterion": "At least two alternatives are named with a one-sentence rationale for why each was not chosen. 'We didn't consider other options' fails. A genuinely obvious single-choice with no real alternatives earns partial credit with an explanation.",
|
|
30
|
+
"gap_signal": "No alternatives section; or alternatives are listed without rationale for rejection."
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "consequences",
|
|
34
|
+
"name": "Consequences",
|
|
35
|
+
"cross_cutting": false,
|
|
36
|
+
"weight": 2,
|
|
37
|
+
"criterion": "Both positive and negative consequences of the decision are named. Positive-only consequences lists signal motivated reasoning and earn partial credit.",
|
|
38
|
+
"gap_signal": "Only upsides described; negative consequences, limitations, or future complexity are absent."
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"id": "risk",
|
|
42
|
+
"name": "Risk coverage",
|
|
43
|
+
"cross_cutting": true,
|
|
44
|
+
"weight": 2,
|
|
45
|
+
"criterion": "Risks and the conditions under which the decision should be revisited are named. A 'revisit if' clause or 'this breaks down when' note earns full credit.",
|
|
46
|
+
"gap_signal": "No risk named; no statement of when the decision would need to be reversed or superseded."
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"id": "reversibility_migration",
|
|
50
|
+
"name": "Reversibility and migration",
|
|
51
|
+
"cross_cutting": true,
|
|
52
|
+
"weight": 2,
|
|
53
|
+
"criterion": "If the decision is hard to reverse (data schema, public contract, naming), the cost of reversal is stated. Easy-to-reverse decisions may state 'low reversal cost' as N/A with full credit.",
|
|
54
|
+
"gap_signal": "An irreversible or costly-to-reverse decision does not state the reversal cost or migration path."
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"id": "maintainability",
|
|
58
|
+
"name": "Maintainability and ownership",
|
|
59
|
+
"cross_cutting": true,
|
|
60
|
+
"weight": 1,
|
|
61
|
+
"criterion": "A maintainer or team is named. Long-lived architectural commitments state a review trigger (e.g., 'revisit when performance exceeds X').",
|
|
62
|
+
"gap_signal": "No owner named; no trigger for revisiting a long-lived commitment."
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "adr_format_completeness",
|
|
66
|
+
"name": "ADR format completeness",
|
|
67
|
+
"cross_cutting": false,
|
|
68
|
+
"weight": 1,
|
|
69
|
+
"criterion": "Standard ADR sections are present: Status, Context, Decision, Consequences, Alternatives. An index entry exists (regenerated by `adr-create`). Superseded ADRs link to their successor.",
|
|
70
|
+
"gap_signal": "Missing standard sections; or no index entry; or superseded record does not link to successor."
|
|
71
|
+
}
|
|
72
|
+
]
|
|
73
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"rubric": "pr-review-score",
|
|
3
|
+
"version": "1.0",
|
|
4
|
+
"artifact_type": "pr-review",
|
|
5
|
+
"description": "Scores a PR review for completeness: evidence quality, actionability, and the four cross-cutting dimensions (risk, tests, migration, maintainability).",
|
|
6
|
+
"anchors": ["code-review", "requesting-code-review", "receiving-code-review"],
|
|
7
|
+
"scoring": {
|
|
8
|
+
"scale": "0 = absent, 1 = partial (mentioned but vague), weight = fully present and concrete",
|
|
9
|
+
"verdict_thresholds": {
|
|
10
|
+
"complete": 0.80,
|
|
11
|
+
"partial": 0.50,
|
|
12
|
+
"incomplete": 0.0
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"dimensions": [
|
|
16
|
+
{
|
|
17
|
+
"id": "evidence_fit",
|
|
18
|
+
"name": "Evidence fit",
|
|
19
|
+
"cross_cutting": false,
|
|
20
|
+
"weight": 3,
|
|
21
|
+
"criterion": "Every finding cites a specific file:line or diff hunk. Claims rest on observable evidence in the diff, not on assumptions about the author's intent. 'This looks wrong' without a reference fails.",
|
|
22
|
+
"gap_signal": "Findings reference vague areas ('the API layer', 'this part') rather than specific locations; or claims are asserted without citing a diff line."
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "concrete_fixes",
|
|
26
|
+
"name": "Concrete fixes",
|
|
27
|
+
"cross_cutting": false,
|
|
28
|
+
"weight": 2,
|
|
29
|
+
"criterion": "Each finding proposes a concrete action — a renamed variable, a specific refactor, a test case that would catch the bug. 'Make it cleaner' or 'consider improving this' are not actionable.",
|
|
30
|
+
"gap_signal": "Findings are observations without prescribed remedies; or remedies are directional ('improve X') rather than concrete ('rename X to Y')."
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "risk_blast_radius",
|
|
34
|
+
"name": "Risk and blast radius",
|
|
35
|
+
"cross_cutting": true,
|
|
36
|
+
"weight": 2,
|
|
37
|
+
"criterion": "The review calls out high-risk hunks: areas touching auth, tenancy, billing, public API contracts, or data migrations. Blast radius is named if a change could affect more callers than the diff shows.",
|
|
38
|
+
"gap_signal": "Security or tenancy paths in the diff are not flagged; a breaking API change is reviewed as if no callers exist outside the diff."
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"id": "test_coverage",
|
|
42
|
+
"name": "Test coverage of the change",
|
|
43
|
+
"cross_cutting": true,
|
|
44
|
+
"weight": 2,
|
|
45
|
+
"criterion": "The review notes whether the diff adds tests for new behavior or regression coverage for a fix. Missing tests for testable paths are flagged as findings, not left implicit.",
|
|
46
|
+
"gap_signal": "New or changed behavior has no test; the review does not flag this."
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"id": "migration_rollback",
|
|
50
|
+
"name": "Migration and rollback",
|
|
51
|
+
"cross_cutting": true,
|
|
52
|
+
"weight": 2,
|
|
53
|
+
"criterion": "If the diff includes a schema migration, config change, or breaking API change, the review confirms a rollback or migration path exists. If none is needed, this is N/A (full credit).",
|
|
54
|
+
"gap_signal": "An irreversible schema migration or breaking API change ships without a rollback note or migration guide."
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"id": "maintainability",
|
|
58
|
+
"name": "Maintainability signal",
|
|
59
|
+
"cross_cutting": true,
|
|
60
|
+
"weight": 1,
|
|
61
|
+
"criterion": "The review flags clarity and naming issues that will burden future readers — confusing abstractions, missing context, or patterns inconsistent with the module's conventions.",
|
|
62
|
+
"gap_signal": "Reviewable naming or abstraction issues are present in the diff but not mentioned."
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
}
|