@adia-ai/adia-ui-forge 0.8.8 → 0.8.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +22 -0
- package/package.json +1 -1
- package/skills/adia-a2ui/evals/evals.json +23 -0
- package/skills/adia-a2ui/evals/routing-corpus.json +130 -29
- package/skills/adia-author/evals/evals.json +22 -0
- package/skills/adia-author/evals/routing-corpus.json +155 -34
- package/skills/adia-deploy/evals/evals.json +22 -0
- package/skills/adia-deploy/evals/routing-corpus.json +125 -28
- package/skills/adia-dogfood/evals/evals.json +20 -0
- package/skills/adia-dogfood/evals/routing-corpus.json +110 -25
- package/skills/adia-gen-review/evals/evals.json +20 -0
- package/skills/adia-gen-review/evals/routing-corpus.json +110 -25
- package/skills/adia-llm-internals/evals/evals.json +22 -0
- package/skills/adia-llm-internals/evals/routing-corpus.json +125 -28
- package/skills/adia-release/evals/routing-corpus.json +115 -26
- package/skills/adia-release/scripts/pr-bridge.mjs +41 -4
- package/skills/adia-release/scripts/release-pack.mjs +68 -4
- package/skills/adia-site-docs/evals/evals.json +20 -0
- package/skills/adia-site-docs/evals/routing-corpus.json +130 -29
- package/skills/adia-ssr/evals/evals.json +20 -0
- package/skills/adia-ssr/evals/routing-corpus.json +110 -25
|
@@ -1,37 +1,134 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"name": "adia-deploy routing accuracy corpus",
|
|
4
|
+
"version": "2.1.0",
|
|
5
|
+
"purpose": "Routing-eval corpus for adia-deploy. Each phrase declares the skill (expected), a forbidden skill (expected_not, for phrases the source data only ever asserted as \"not this skill\"), or neither. Scored by scripts/skills/run-skill-evals.mjs (TF-IDF token overlap over per-skill description+triggers).",
|
|
6
|
+
"scoring_notes": "Heuristic signal, not ground truth. Treat misroutes as a prompt to tighten the skill description, never as a reason to keyword-stuff it. Real harness routing is LLM-driven.",
|
|
7
|
+
"scope": "adia-deploy routing \u2014 does this phrase activate adia-deploy?",
|
|
8
|
+
"phrases": [
|
|
9
|
+
{
|
|
10
|
+
"id": "adia-deploy-pos-01",
|
|
11
|
+
"phrase": "deploy the latest build to ui-kit.exe.xyz",
|
|
12
|
+
"expected": "adia-deploy"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"id": "adia-deploy-pos-02",
|
|
16
|
+
"phrase": "push the dist folder to the VM",
|
|
17
|
+
"expected": "adia-deploy"
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"id": "adia-deploy-pos-03",
|
|
21
|
+
"phrase": "diagnose why the site returns Port 8000 unbound",
|
|
22
|
+
"expected": "adia-deploy"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "adia-deploy-pos-04",
|
|
26
|
+
"phrase": "the site is throwing a 502, check the VM service",
|
|
27
|
+
"expected": "adia-deploy"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"id": "adia-deploy-pos-05",
|
|
31
|
+
"phrase": "restart the adia-ui service on the exe.dev VM",
|
|
32
|
+
"expected": "adia-deploy"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"id": "adia-deploy-pos-06",
|
|
36
|
+
"phrase": "provision a fresh exe.dev VM for a new host",
|
|
37
|
+
"expected": "adia-deploy"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"id": "adia-deploy-pos-07",
|
|
41
|
+
"phrase": "rotate the API keys on the production VM",
|
|
42
|
+
"expected": "adia-deploy"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "adia-deploy-pos-08",
|
|
46
|
+
"phrase": "push a site-v* tag to deploy the current build",
|
|
47
|
+
"expected": "adia-deploy"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"id": "adia-deploy-pos-09",
|
|
51
|
+
"phrase": "the site looks stale, new fixtures are 404ing",
|
|
52
|
+
"expected": "adia-deploy"
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "adia-deploy-pos-10",
|
|
56
|
+
"phrase": "check the dry-run delete summary before the deploy job runs",
|
|
57
|
+
"expected": "adia-deploy"
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"id": "adia-deploy-pos-11",
|
|
61
|
+
"phrase": "did we deploy the site after the last lockstep cut, or is it behind npm",
|
|
62
|
+
"expected": "adia-deploy"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "adia-deploy-pos-12",
|
|
66
|
+
"phrase": "roll back the last site deploy, the new build is broken",
|
|
67
|
+
"expected": "adia-deploy"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"id": "adia-deploy-neg-01",
|
|
71
|
+
"phrase": "cut a release of the next version and publish the npm packages",
|
|
72
|
+
"expected_not": "adia-deploy"
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"id": "adia-deploy-neg-02",
|
|
76
|
+
"phrase": "verify that this consumer app screen renders correctly before shipping",
|
|
77
|
+
"expected_not": "adia-deploy"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"id": "adia-deploy-neg-03",
|
|
81
|
+
"phrase": "add a new prop to button-ui",
|
|
82
|
+
"expected_not": "adia-deploy"
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"id": "adia-deploy-neg-04",
|
|
86
|
+
"phrase": "run a dogfood sweep on the demo pages",
|
|
87
|
+
"expected_not": "adia-deploy"
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"id": "adia-deploy-neg-05",
|
|
91
|
+
"phrase": "tune the zettel retrieval coverage, eval dropped to 38%",
|
|
92
|
+
"expected_not": "adia-deploy"
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"id": "adia-deploy-neg-06",
|
|
96
|
+
"phrase": "publish the npm packages for the next lockstep cut",
|
|
97
|
+
"expected_not": "adia-deploy"
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"id": "adia-deploy-neg-07",
|
|
101
|
+
"phrase": "author the release notes and the migration guide",
|
|
102
|
+
"expected_not": "adia-deploy"
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"id": "adia-deploy-neg-08",
|
|
106
|
+
"phrase": "why does accordion-ui crash under SSR in Astro",
|
|
107
|
+
"expected_not": "adia-deploy"
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
"id": "adia-deploy-neg-09",
|
|
111
|
+
"phrase": "fix the streaming adapter bug in the llm package",
|
|
112
|
+
"expected_not": "adia-deploy"
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
"id": "adia-deploy-neg-10",
|
|
116
|
+
"phrase": "review the getting-started docs page for consistency",
|
|
117
|
+
"expected_not": "adia-deploy"
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
"id": "adia-deploy-neg-11",
|
|
121
|
+
"phrase": "score the gen-ui gallery outputs for this cycle",
|
|
122
|
+
"expected_not": "adia-deploy"
|
|
123
|
+
}
|
|
15
124
|
],
|
|
16
|
-
"
|
|
17
|
-
"cut a release of the next version and publish the npm packages",
|
|
18
|
-
"verify that this consumer app screen renders correctly before shipping",
|
|
19
|
-
"add a new prop to button-ui",
|
|
20
|
-
"run a dogfood sweep on the demo pages",
|
|
21
|
-
"tune the zettel retrieval coverage, eval dropped to 38%",
|
|
22
|
-
"publish the npm packages for the next lockstep cut",
|
|
23
|
-
"author the release notes and the migration guide",
|
|
24
|
-
"why does accordion-ui crash under SSR in Astro",
|
|
25
|
-
"fix the streaming adapter bug in the llm package",
|
|
26
|
-
"review the getting-started docs page for consistency",
|
|
27
|
-
"score the gen-ui gallery outputs for this cycle"
|
|
28
|
-
],
|
|
29
|
-
"_measured": {
|
|
125
|
+
"_measured_historical": {
|
|
30
126
|
"as_of": "2026-07-18",
|
|
31
127
|
"scorer": "routing_eval.py (nonoun-plugins/forge)",
|
|
32
128
|
"f1": 0.917,
|
|
33
129
|
"precision": 0.917,
|
|
34
130
|
"recall": 0.917,
|
|
35
|
-
"exit_code": 0
|
|
131
|
+
"exit_code": 0,
|
|
132
|
+
"note": "measured by the external routing_eval.py (nonoun-plugins/forge) against the pre-conversion positives/negatives shape; historical record only, not regenerated by run-skill-evals.mjs."
|
|
36
133
|
}
|
|
37
134
|
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill": "adia-dogfood",
|
|
3
|
+
"note": "Trigger-routing suite (gh#369). expect=trigger: the skill should fire on the prompt in a fresh session. expect=no-trigger: a near-miss owned elsewhere (owner named per case) or generic knowledge with no AdiaUI skill involved. Cases derived from evals/routing-corpus.json's phrases array (gh#355 vetted corpus) plus the SKILL.md description's own trigger/NOT-fence language.",
|
|
4
|
+
"cases": [
|
|
5
|
+
{ "id": "t01", "prompt": "Run a dogfood sweep before we cut the release", "expect": "trigger" },
|
|
6
|
+
{ "id": "t02", "prompt": "Find broken demos across the component site", "expect": "trigger" },
|
|
7
|
+
{ "id": "t03", "prompt": "Audit native primitive leaks in the admin app", "expect": "trigger" },
|
|
8
|
+
{ "id": "t04", "prompt": "Check for attr-quote typos that broke the rendered HTML", "expect": "trigger" },
|
|
9
|
+
{ "id": "t05", "prompt": "Audit admin-shell composition for missing canonical parts", "expect": "trigger" },
|
|
10
|
+
{ "id": "t06", "prompt": "Sweep card anatomy for a collapsed header wrapper", "expect": "trigger" },
|
|
11
|
+
{ "id": "t07", "prompt": "Run npm run dogfood:status and post the findings in the PR description", "expect": "trigger" },
|
|
12
|
+
{ "id": "t08", "prompt": "Find token drift and contrast collapse across the components after the refactor", "expect": "trigger" },
|
|
13
|
+
{ "id": "n01", "prompt": "Score the gen-ui gallery output quality against the rubric", "expect": "no-trigger", "owner": "adia-gen-review" },
|
|
14
|
+
{ "id": "n02", "prompt": "Add a new primitive component to packages/web-components", "expect": "no-trigger", "owner": "adia-author" },
|
|
15
|
+
{ "id": "n03", "prompt": "Cut the next release and publish the lockstep packages", "expect": "no-trigger", "owner": "adia-release" },
|
|
16
|
+
{ "id": "n04", "prompt": "Deploy the latest build to ui-kit.exe.xyz", "expect": "no-trigger", "owner": "adia-deploy" },
|
|
17
|
+
{ "id": "n05", "prompt": "Zettel coverage dropped on the nightly eval, find out why", "expect": "no-trigger", "owner": "adia-a2ui" },
|
|
18
|
+
{ "id": "n06", "prompt": "Compose the billing settings screen from the catalog", "expect": "no-trigger", "owner": "adia-compose" }
|
|
19
|
+
]
|
|
20
|
+
}
|
|
@@ -1,34 +1,119 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"name": "adia-dogfood routing accuracy corpus",
|
|
4
|
+
"version": "2.1.0",
|
|
5
|
+
"purpose": "Routing-eval corpus for adia-dogfood. Each phrase declares the skill (expected), a forbidden skill (expected_not, for phrases the source data only ever asserted as \"not this skill\"), or neither. Scored by scripts/skills/run-skill-evals.mjs (TF-IDF token overlap over per-skill description+triggers).",
|
|
6
|
+
"scoring_notes": "Heuristic signal, not ground truth. Treat misroutes as a prompt to tighten the skill description, never as a reason to keyword-stuff it. Real harness routing is LLM-driven.",
|
|
7
|
+
"scope": "adia-dogfood routing \u2014 does this phrase activate adia-dogfood?",
|
|
8
|
+
"phrases": [
|
|
9
|
+
{
|
|
10
|
+
"id": "adia-dogfood-pos-01",
|
|
11
|
+
"phrase": "run a dogfood sweep before we cut the release",
|
|
12
|
+
"expected": "adia-dogfood"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"id": "adia-dogfood-pos-02",
|
|
16
|
+
"phrase": "find broken demos across the component site",
|
|
17
|
+
"expected": "adia-dogfood"
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"id": "adia-dogfood-pos-03",
|
|
21
|
+
"phrase": "audit native primitive leaks in the admin app",
|
|
22
|
+
"expected": "adia-dogfood"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "adia-dogfood-pos-04",
|
|
26
|
+
"phrase": "check for attr-quote typos that broke the rendered HTML",
|
|
27
|
+
"expected": "adia-dogfood"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"id": "adia-dogfood-pos-05",
|
|
31
|
+
"phrase": "run the app-shell QA sweep after the apps/ structural refactor",
|
|
32
|
+
"expected": "adia-dogfood"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"id": "adia-dogfood-pos-06",
|
|
36
|
+
"phrase": "audit admin-shell composition for missing canonical parts",
|
|
37
|
+
"expected": "adia-dogfood"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"id": "adia-dogfood-pos-07",
|
|
41
|
+
"phrase": "sweep card anatomy for a collapsed header wrapper",
|
|
42
|
+
"expected": "adia-dogfood"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "adia-dogfood-pos-08",
|
|
46
|
+
"phrase": "run npm run dogfood:status and post the findings in the PR description",
|
|
47
|
+
"expected": "adia-dogfood"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"id": "adia-dogfood-pos-09",
|
|
51
|
+
"phrase": "is this surface clean before we merge",
|
|
52
|
+
"expected": "adia-dogfood"
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "adia-dogfood-pos-10",
|
|
56
|
+
"phrase": "find token drift and contrast collapse across the components",
|
|
57
|
+
"expected": "adia-dogfood"
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"id": "adia-dogfood-pos-11",
|
|
61
|
+
"phrase": "audit the demo pages for broken components after the token refactor",
|
|
62
|
+
"expected": "adia-dogfood"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "adia-dogfood-pos-12",
|
|
66
|
+
"phrase": "check for unregistered custom elements on the component pages",
|
|
67
|
+
"expected": "adia-dogfood"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"id": "adia-dogfood-neg-01",
|
|
71
|
+
"phrase": "score the gen-ui gallery output quality against the rubric",
|
|
72
|
+
"expected_not": "adia-dogfood"
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"id": "adia-dogfood-neg-02",
|
|
76
|
+
"phrase": "review the rubric score for this generated screen and root-cause the gap",
|
|
77
|
+
"expected_not": "adia-dogfood"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"id": "adia-dogfood-neg-03",
|
|
81
|
+
"phrase": "add a new primitive component to packages/web-components",
|
|
82
|
+
"expected_not": "adia-dogfood"
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"id": "adia-dogfood-neg-04",
|
|
86
|
+
"phrase": "cut the next release and publish the lockstep packages",
|
|
87
|
+
"expected_not": "adia-dogfood"
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"id": "adia-dogfood-neg-05",
|
|
91
|
+
"phrase": "deploy the latest build to ui-kit.exe.xyz",
|
|
92
|
+
"expected": "adia-deploy"
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"id": "adia-dogfood-neg-06",
|
|
96
|
+
"phrase": "zettel coverage dropped on the nightly eval, find out why",
|
|
97
|
+
"expected_not": "adia-dogfood"
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"id": "adia-dogfood-neg-07",
|
|
101
|
+
"phrase": "fix the openai stopReason mapping in the llm client",
|
|
102
|
+
"expected_not": "adia-dogfood"
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"id": "adia-dogfood-neg-08",
|
|
106
|
+
"phrase": "compose the billing settings screen from the catalog",
|
|
107
|
+
"expected_not": "adia-dogfood"
|
|
108
|
+
}
|
|
15
109
|
],
|
|
16
|
-
"
|
|
17
|
-
"score the gen-ui gallery output quality against the rubric",
|
|
18
|
-
"review the rubric score for this generated screen and root-cause the gap",
|
|
19
|
-
"add a new primitive component to packages/web-components",
|
|
20
|
-
"cut the next release and publish the lockstep packages",
|
|
21
|
-
"deploy the latest build to ui-kit.exe.xyz",
|
|
22
|
-
"zettel coverage dropped on the nightly eval, find out why",
|
|
23
|
-
"fix the openai stopReason mapping in the llm client",
|
|
24
|
-
"compose the billing settings screen from the catalog"
|
|
25
|
-
],
|
|
26
|
-
"_measured": {
|
|
110
|
+
"_measured_historical": {
|
|
27
111
|
"as_of": "2026-07-18",
|
|
28
112
|
"scorer": "routing_eval.py (nonoun-plugins/forge)",
|
|
29
113
|
"f1": 0.857,
|
|
30
114
|
"precision": 1.0,
|
|
31
115
|
"recall": 0.75,
|
|
32
|
-
"exit_code": 0
|
|
116
|
+
"exit_code": 0,
|
|
117
|
+
"note": "measured by the external routing_eval.py (nonoun-plugins/forge) against the pre-conversion positives/negatives shape; historical record only, not regenerated by run-skill-evals.mjs."
|
|
33
118
|
}
|
|
34
119
|
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill": "adia-gen-review",
|
|
3
|
+
"note": "Trigger-routing suite (gh#369). expect=trigger: the skill should fire on the prompt in a fresh session. expect=no-trigger: a near-miss owned elsewhere (owner named per case) or generic knowledge with no AdiaUI skill involved. Cases derived from evals/routing-corpus.json's phrases array (gh#355 vetted corpus) plus the SKILL.md description's own trigger/NOT-fence language.",
|
|
4
|
+
"cases": [
|
|
5
|
+
{ "id": "t01", "prompt": "Score the gen-ui gallery outputs against the rubric", "expect": "trigger" },
|
|
6
|
+
{ "id": "t02", "prompt": "Run a /gen-review cycle over the gallery", "expect": "trigger" },
|
|
7
|
+
{ "id": "t03", "prompt": "Decompose the rendered canvas and score the gap against the ideal composition", "expect": "trigger" },
|
|
8
|
+
{ "id": "t04", "prompt": "Check for overflow elements in the generated screens", "expect": "trigger" },
|
|
9
|
+
{ "id": "t05", "prompt": "Root-cause why this gen-ui prompt's render score is low", "expect": "trigger" },
|
|
10
|
+
{ "id": "t06", "prompt": "Run the exit gate check on the review ledger", "expect": "trigger" },
|
|
11
|
+
{ "id": "t07", "prompt": "Write a fix plan for the failing prompts in the gallery", "expect": "trigger" },
|
|
12
|
+
{ "id": "t08", "prompt": "Is the gen-ui gallery review cycle complete yet?", "expect": "trigger" },
|
|
13
|
+
{ "id": "n01", "prompt": "Run a dogfood sweep on the demo pages before we cut", "expect": "no-trigger", "owner": "adia-dogfood" },
|
|
14
|
+
{ "id": "n02", "prompt": "Fix the chunk corpus so this component stops rendering wrong", "expect": "no-trigger", "owner": "adia-a2ui" },
|
|
15
|
+
{ "id": "n03", "prompt": "Add a new primitive to the component catalog", "expect": "no-trigger", "owner": "adia-author" },
|
|
16
|
+
{ "id": "n04", "prompt": "Audit the admin-shell composition for missing canonical parts", "expect": "no-trigger", "owner": "adia-dogfood" },
|
|
17
|
+
{ "id": "n05", "prompt": "[scheduled] cut v0.9.3 overnight", "expect": "no-trigger", "owner": "adia-release" },
|
|
18
|
+
{ "id": "n06", "prompt": "The gemini adapter is dropping usage tokens on streamed responses", "expect": "no-trigger", "owner": "adia-llm-internals" }
|
|
19
|
+
]
|
|
20
|
+
}
|
|
@@ -1,34 +1,119 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"name": "adia-gen-review routing accuracy corpus",
|
|
4
|
+
"version": "2.1.0",
|
|
5
|
+
"purpose": "Routing-eval corpus for adia-gen-review. Each phrase declares the skill (expected), a forbidden skill (expected_not, for phrases the source data only ever asserted as \"not this skill\"), or neither. Scored by scripts/skills/run-skill-evals.mjs (TF-IDF token overlap over per-skill description+triggers).",
|
|
6
|
+
"scoring_notes": "Heuristic signal, not ground truth. Treat misroutes as a prompt to tighten the skill description, never as a reason to keyword-stuff it. Real harness routing is LLM-driven.",
|
|
7
|
+
"scope": "adia-gen-review routing \u2014 does this phrase activate adia-gen-review?",
|
|
8
|
+
"phrases": [
|
|
9
|
+
{
|
|
10
|
+
"id": "adia-gen-review-pos-01",
|
|
11
|
+
"phrase": "score the gen-ui gallery outputs against the rubric",
|
|
12
|
+
"expected": "adia-gen-review"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"id": "adia-gen-review-pos-02",
|
|
16
|
+
"phrase": "run a gen-review cycle over the gallery",
|
|
17
|
+
"expected": "adia-gen-review"
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"id": "adia-gen-review-pos-03",
|
|
21
|
+
"phrase": "review the gen-ui outputs for quality",
|
|
22
|
+
"expected": "adia-gen-review"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "adia-gen-review-pos-04",
|
|
26
|
+
"phrase": "decompose the rendered canvas and score the gap against the ideal composition",
|
|
27
|
+
"expected": "adia-gen-review"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"id": "adia-gen-review-pos-05",
|
|
31
|
+
"phrase": "check for overflow elements in the generated screens",
|
|
32
|
+
"expected": "adia-gen-review"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"id": "adia-gen-review-pos-06",
|
|
36
|
+
"phrase": "root-cause why this gen-ui prompt's render score is low",
|
|
37
|
+
"expected": "adia-gen-review"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"id": "adia-gen-review-pos-07",
|
|
41
|
+
"phrase": "run the exit gate check on the review ledger",
|
|
42
|
+
"expected": "adia-gen-review"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "adia-gen-review-pos-08",
|
|
46
|
+
"phrase": "classify the cause code for this bad gen-ui render",
|
|
47
|
+
"expected": "adia-gen-review"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"id": "adia-gen-review-pos-09",
|
|
51
|
+
"phrase": "write a fix plan for the failing prompts in the gallery",
|
|
52
|
+
"expected": "adia-gen-review"
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "adia-gen-review-pos-10",
|
|
56
|
+
"phrase": "score this prompt against its ideal composition and report the delta",
|
|
57
|
+
"expected": "adia-gen-review"
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"id": "adia-gen-review-pos-11",
|
|
61
|
+
"phrase": "check the cosmetic findings in the gallery screenshots",
|
|
62
|
+
"expected": "adia-gen-review"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "adia-gen-review-pos-12",
|
|
66
|
+
"phrase": "is the gen-ui gallery review cycle complete yet",
|
|
67
|
+
"expected": "adia-gen-review"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"id": "adia-gen-review-neg-01",
|
|
71
|
+
"phrase": "run a dogfood sweep on the demo pages before we cut",
|
|
72
|
+
"expected_not": "adia-gen-review"
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"id": "adia-gen-review-neg-02",
|
|
76
|
+
"phrase": "find native primitive leaks across the app shells",
|
|
77
|
+
"expected_not": "adia-gen-review"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"id": "adia-gen-review-neg-03",
|
|
81
|
+
"phrase": "fix the chunk corpus so this component stops rendering wrong",
|
|
82
|
+
"expected_not": "adia-gen-review"
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"id": "adia-gen-review-neg-04",
|
|
86
|
+
"phrase": "add a new primitive to the component catalog",
|
|
87
|
+
"expected_not": "adia-gen-review"
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"id": "adia-gen-review-neg-05",
|
|
91
|
+
"phrase": "audit the admin-shell composition for missing canonical parts",
|
|
92
|
+
"expected_not": "adia-gen-review"
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"id": "adia-gen-review-neg-06",
|
|
96
|
+
"phrase": "[scheduled] cut v0.9.3 overnight",
|
|
97
|
+
"expected_not": "adia-gen-review"
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"id": "adia-gen-review-neg-07",
|
|
101
|
+
"phrase": "the gemini adapter is dropping usage tokens on streamed responses",
|
|
102
|
+
"expected": "adia-llm-internals"
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"id": "adia-gen-review-neg-08",
|
|
106
|
+
"phrase": "deploy the latest build to ui-kit.exe.xyz",
|
|
107
|
+
"expected": "adia-deploy"
|
|
108
|
+
}
|
|
15
109
|
],
|
|
16
|
-
"
|
|
17
|
-
"run a dogfood sweep on the demo pages before we cut",
|
|
18
|
-
"find native primitive leaks across the app shells",
|
|
19
|
-
"fix the chunk corpus so this component stops rendering wrong",
|
|
20
|
-
"add a new primitive to the component catalog",
|
|
21
|
-
"audit the admin-shell composition for missing canonical parts",
|
|
22
|
-
"[scheduled] cut v0.9.3 overnight",
|
|
23
|
-
"the gemini adapter is dropping usage tokens on streamed responses",
|
|
24
|
-
"deploy the latest build to ui-kit.exe.xyz"
|
|
25
|
-
],
|
|
26
|
-
"_measured": {
|
|
110
|
+
"_measured_historical": {
|
|
27
111
|
"as_of": "2026-07-18",
|
|
28
112
|
"scorer": "routing_eval.py (nonoun-plugins/forge)",
|
|
29
113
|
"f1": 0.857,
|
|
30
114
|
"precision": 1.0,
|
|
31
115
|
"recall": 0.75,
|
|
32
|
-
"exit_code": 0
|
|
116
|
+
"exit_code": 0,
|
|
117
|
+
"note": "measured by the external routing_eval.py (nonoun-plugins/forge) against the pre-conversion positives/negatives shape; historical record only, not regenerated by run-skill-evals.mjs."
|
|
33
118
|
}
|
|
34
119
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill": "adia-llm-internals",
|
|
3
|
+
"note": "Trigger-routing suite (gh#369). expect=trigger: the skill should fire on the prompt in a fresh session. expect=no-trigger: a near-miss owned elsewhere (owner named per case) or generic knowledge with no AdiaUI skill involved. Cases derived from evals/routing-corpus.json's phrases array (gh#355 vetted corpus) plus the SKILL.md description's own trigger/NOT-fence language.",
|
|
4
|
+
"cases": [
|
|
5
|
+
{ "id": "t01", "prompt": "The gemini adapter is dropping usage tokens on streamed responses", "expect": "trigger" },
|
|
6
|
+
{ "id": "t02", "prompt": "stopReason from openai never reaches the client as MAX_TOKENS", "expect": "trigger" },
|
|
7
|
+
{ "id": "t03", "prompt": "The SSE parser chokes when a chunk splits mid-line", "expect": "trigger" },
|
|
8
|
+
{ "id": "t04", "prompt": "The passthrough proxy is returning 401 for anthropic requests in production", "expect": "trigger" },
|
|
9
|
+
{ "id": "t05", "prompt": "Add a new model to the MODELS registry and set it as DEFAULT_MODEL", "expect": "trigger" },
|
|
10
|
+
{ "id": "t06", "prompt": "createAdapter() is leaking a real API key to the browser on a production host", "expect": "trigger" },
|
|
11
|
+
{ "id": "t07", "prompt": "Add support for a new OpenAI-compatible gateway provider adapter", "expect": "trigger" },
|
|
12
|
+
{ "id": "t08", "prompt": "streamChat() never emits a terminal done chunk", "expect": "trigger" },
|
|
13
|
+
{ "id": "t09", "prompt": "The StreamChunk union is missing a variant for tool_use deltas", "expect": "trigger" },
|
|
14
|
+
{ "id": "n01", "prompt": "Wire the chat client into our admin app's chat-shell", "expect": "no-trigger", "owner": "adia-llm" },
|
|
15
|
+
{ "id": "n02", "prompt": "Cut a release and bump @adia-ai/llm's version with the rest of the lockstep packages", "expect": "no-trigger", "owner": "adia-release" },
|
|
16
|
+
{ "id": "n03", "prompt": "Explain how Server-Sent Events work in general", "expect": "no-trigger" },
|
|
17
|
+
{ "id": "n04", "prompt": "The chat-shell message list isn't rendering streamed replies", "expect": "no-trigger", "owner": "adia-llm" },
|
|
18
|
+
{ "id": "n05", "prompt": "Tune STRONG_MATCH and lift the eval fails in the compose strategies", "expect": "no-trigger", "owner": "adia-a2ui" },
|
|
19
|
+
{ "id": "n06", "prompt": "Deploy the ui-kit dist to exe.dev and restart the service", "expect": "no-trigger", "owner": "adia-deploy" },
|
|
20
|
+
{ "id": "n07", "prompt": "Run a dogfood sweep to find broken demos", "expect": "no-trigger", "owner": "adia-dogfood" }
|
|
21
|
+
]
|
|
22
|
+
}
|