@adia-ai/adia-ui-forge 0.8.9 → 0.8.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +11 -0
- package/package.json +1 -1
- package/skills/adia-a2ui/evals/evals.json +23 -0
- package/skills/adia-a2ui/evals/routing-corpus.json +130 -29
- package/skills/adia-author/evals/evals.json +22 -0
- package/skills/adia-author/evals/routing-corpus.json +155 -34
- package/skills/adia-deploy/evals/evals.json +22 -0
- package/skills/adia-deploy/evals/routing-corpus.json +125 -28
- package/skills/adia-dogfood/evals/evals.json +20 -0
- package/skills/adia-dogfood/evals/routing-corpus.json +110 -25
- package/skills/adia-gen-review/evals/evals.json +20 -0
- package/skills/adia-gen-review/evals/routing-corpus.json +110 -25
- package/skills/adia-llm-internals/evals/evals.json +22 -0
- package/skills/adia-llm-internals/evals/routing-corpus.json +125 -28
- package/skills/adia-release/evals/routing-corpus.json +115 -26
- package/skills/adia-site-docs/evals/evals.json +20 -0
- package/skills/adia-site-docs/evals/routing-corpus.json +130 -29
- package/skills/adia-ssr/evals/evals.json +20 -0
- package/skills/adia-ssr/evals/routing-corpus.json +110 -25
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill": "adia-dogfood",
|
|
3
|
+
"note": "Trigger-routing suite (gh#369). expect=trigger: the skill should fire on the prompt in a fresh session. expect=no-trigger: a near-miss owned elsewhere (owner named per case) or generic knowledge with no AdiaUI skill involved. Cases derived from evals/routing-corpus.json's phrases array (gh#355 vetted corpus) plus the SKILL.md description's own trigger/NOT-fence language.",
|
|
4
|
+
"cases": [
|
|
5
|
+
{ "id": "t01", "prompt": "Run a dogfood sweep before we cut the release", "expect": "trigger" },
|
|
6
|
+
{ "id": "t02", "prompt": "Find broken demos across the component site", "expect": "trigger" },
|
|
7
|
+
{ "id": "t03", "prompt": "Audit native primitive leaks in the admin app", "expect": "trigger" },
|
|
8
|
+
{ "id": "t04", "prompt": "Check for attr-quote typos that broke the rendered HTML", "expect": "trigger" },
|
|
9
|
+
{ "id": "t05", "prompt": "Audit admin-shell composition for missing canonical parts", "expect": "trigger" },
|
|
10
|
+
{ "id": "t06", "prompt": "Sweep card anatomy for a collapsed header wrapper", "expect": "trigger" },
|
|
11
|
+
{ "id": "t07", "prompt": "Run npm run dogfood:status and post the findings in the PR description", "expect": "trigger" },
|
|
12
|
+
{ "id": "t08", "prompt": "Find token drift and contrast collapse across the components after the refactor", "expect": "trigger" },
|
|
13
|
+
{ "id": "n01", "prompt": "Score the gen-ui gallery output quality against the rubric", "expect": "no-trigger", "owner": "adia-gen-review" },
|
|
14
|
+
{ "id": "n02", "prompt": "Add a new primitive component to packages/web-components", "expect": "no-trigger", "owner": "adia-author" },
|
|
15
|
+
{ "id": "n03", "prompt": "Cut the next release and publish the lockstep packages", "expect": "no-trigger", "owner": "adia-release" },
|
|
16
|
+
{ "id": "n04", "prompt": "Deploy the latest build to ui-kit.exe.xyz", "expect": "no-trigger", "owner": "adia-deploy" },
|
|
17
|
+
{ "id": "n05", "prompt": "Zettel coverage dropped on the nightly eval, find out why", "expect": "no-trigger", "owner": "adia-a2ui" },
|
|
18
|
+
{ "id": "n06", "prompt": "Compose the billing settings screen from the catalog", "expect": "no-trigger", "owner": "adia-compose" }
|
|
19
|
+
]
|
|
20
|
+
}
|
|
@@ -1,34 +1,119 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"name": "adia-dogfood routing accuracy corpus",
|
|
4
|
+
"version": "2.1.0",
|
|
5
|
+
"purpose": "Routing-eval corpus for adia-dogfood. Each phrase declares the skill (expected), a forbidden skill (expected_not, for phrases the source data only ever asserted as \"not this skill\"), or neither. Scored by scripts/skills/run-skill-evals.mjs (TF-IDF token overlap over per-skill description+triggers).",
|
|
6
|
+
"scoring_notes": "Heuristic signal, not ground truth. Treat misroutes as a prompt to tighten the skill description, never as a reason to keyword-stuff it. Real harness routing is LLM-driven.",
|
|
7
|
+
"scope": "adia-dogfood routing \u2014 does this phrase activate adia-dogfood?",
|
|
8
|
+
"phrases": [
|
|
9
|
+
{
|
|
10
|
+
"id": "adia-dogfood-pos-01",
|
|
11
|
+
"phrase": "run a dogfood sweep before we cut the release",
|
|
12
|
+
"expected": "adia-dogfood"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"id": "adia-dogfood-pos-02",
|
|
16
|
+
"phrase": "find broken demos across the component site",
|
|
17
|
+
"expected": "adia-dogfood"
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"id": "adia-dogfood-pos-03",
|
|
21
|
+
"phrase": "audit native primitive leaks in the admin app",
|
|
22
|
+
"expected": "adia-dogfood"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "adia-dogfood-pos-04",
|
|
26
|
+
"phrase": "check for attr-quote typos that broke the rendered HTML",
|
|
27
|
+
"expected": "adia-dogfood"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"id": "adia-dogfood-pos-05",
|
|
31
|
+
"phrase": "run the app-shell QA sweep after the apps/ structural refactor",
|
|
32
|
+
"expected": "adia-dogfood"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"id": "adia-dogfood-pos-06",
|
|
36
|
+
"phrase": "audit admin-shell composition for missing canonical parts",
|
|
37
|
+
"expected": "adia-dogfood"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"id": "adia-dogfood-pos-07",
|
|
41
|
+
"phrase": "sweep card anatomy for a collapsed header wrapper",
|
|
42
|
+
"expected": "adia-dogfood"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "adia-dogfood-pos-08",
|
|
46
|
+
"phrase": "run npm run dogfood:status and post the findings in the PR description",
|
|
47
|
+
"expected": "adia-dogfood"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"id": "adia-dogfood-pos-09",
|
|
51
|
+
"phrase": "is this surface clean before we merge",
|
|
52
|
+
"expected": "adia-dogfood"
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "adia-dogfood-pos-10",
|
|
56
|
+
"phrase": "find token drift and contrast collapse across the components",
|
|
57
|
+
"expected": "adia-dogfood"
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"id": "adia-dogfood-pos-11",
|
|
61
|
+
"phrase": "audit the demo pages for broken components after the token refactor",
|
|
62
|
+
"expected": "adia-dogfood"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "adia-dogfood-pos-12",
|
|
66
|
+
"phrase": "check for unregistered custom elements on the component pages",
|
|
67
|
+
"expected": "adia-dogfood"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"id": "adia-dogfood-neg-01",
|
|
71
|
+
"phrase": "score the gen-ui gallery output quality against the rubric",
|
|
72
|
+
"expected_not": "adia-dogfood"
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"id": "adia-dogfood-neg-02",
|
|
76
|
+
"phrase": "review the rubric score for this generated screen and root-cause the gap",
|
|
77
|
+
"expected_not": "adia-dogfood"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"id": "adia-dogfood-neg-03",
|
|
81
|
+
"phrase": "add a new primitive component to packages/web-components",
|
|
82
|
+
"expected_not": "adia-dogfood"
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"id": "adia-dogfood-neg-04",
|
|
86
|
+
"phrase": "cut the next release and publish the lockstep packages",
|
|
87
|
+
"expected_not": "adia-dogfood"
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"id": "adia-dogfood-neg-05",
|
|
91
|
+
"phrase": "deploy the latest build to ui-kit.exe.xyz",
|
|
92
|
+
"expected": "adia-deploy"
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"id": "adia-dogfood-neg-06",
|
|
96
|
+
"phrase": "zettel coverage dropped on the nightly eval, find out why",
|
|
97
|
+
"expected_not": "adia-dogfood"
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"id": "adia-dogfood-neg-07",
|
|
101
|
+
"phrase": "fix the openai stopReason mapping in the llm client",
|
|
102
|
+
"expected_not": "adia-dogfood"
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"id": "adia-dogfood-neg-08",
|
|
106
|
+
"phrase": "compose the billing settings screen from the catalog",
|
|
107
|
+
"expected_not": "adia-dogfood"
|
|
108
|
+
}
|
|
15
109
|
],
|
|
16
|
-
"
|
|
17
|
-
"score the gen-ui gallery output quality against the rubric",
|
|
18
|
-
"review the rubric score for this generated screen and root-cause the gap",
|
|
19
|
-
"add a new primitive component to packages/web-components",
|
|
20
|
-
"cut the next release and publish the lockstep packages",
|
|
21
|
-
"deploy the latest build to ui-kit.exe.xyz",
|
|
22
|
-
"zettel coverage dropped on the nightly eval, find out why",
|
|
23
|
-
"fix the openai stopReason mapping in the llm client",
|
|
24
|
-
"compose the billing settings screen from the catalog"
|
|
25
|
-
],
|
|
26
|
-
"_measured": {
|
|
110
|
+
"_measured_historical": {
|
|
27
111
|
"as_of": "2026-07-18",
|
|
28
112
|
"scorer": "routing_eval.py (nonoun-plugins/forge)",
|
|
29
113
|
"f1": 0.857,
|
|
30
114
|
"precision": 1.0,
|
|
31
115
|
"recall": 0.75,
|
|
32
|
-
"exit_code": 0
|
|
116
|
+
"exit_code": 0,
|
|
117
|
+
"note": "measured by the external routing_eval.py (nonoun-plugins/forge) against the pre-conversion positives/negatives shape; historical record only, not regenerated by run-skill-evals.mjs."
|
|
33
118
|
}
|
|
34
119
|
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill": "adia-gen-review",
|
|
3
|
+
"note": "Trigger-routing suite (gh#369). expect=trigger: the skill should fire on the prompt in a fresh session. expect=no-trigger: a near-miss owned elsewhere (owner named per case) or generic knowledge with no AdiaUI skill involved. Cases derived from evals/routing-corpus.json's phrases array (gh#355 vetted corpus) plus the SKILL.md description's own trigger/NOT-fence language.",
|
|
4
|
+
"cases": [
|
|
5
|
+
{ "id": "t01", "prompt": "Score the gen-ui gallery outputs against the rubric", "expect": "trigger" },
|
|
6
|
+
{ "id": "t02", "prompt": "Run a /gen-review cycle over the gallery", "expect": "trigger" },
|
|
7
|
+
{ "id": "t03", "prompt": "Decompose the rendered canvas and score the gap against the ideal composition", "expect": "trigger" },
|
|
8
|
+
{ "id": "t04", "prompt": "Check for overflow elements in the generated screens", "expect": "trigger" },
|
|
9
|
+
{ "id": "t05", "prompt": "Root-cause why this gen-ui prompt's render score is low", "expect": "trigger" },
|
|
10
|
+
{ "id": "t06", "prompt": "Run the exit gate check on the review ledger", "expect": "trigger" },
|
|
11
|
+
{ "id": "t07", "prompt": "Write a fix plan for the failing prompts in the gallery", "expect": "trigger" },
|
|
12
|
+
{ "id": "t08", "prompt": "Is the gen-ui gallery review cycle complete yet?", "expect": "trigger" },
|
|
13
|
+
{ "id": "n01", "prompt": "Run a dogfood sweep on the demo pages before we cut", "expect": "no-trigger", "owner": "adia-dogfood" },
|
|
14
|
+
{ "id": "n02", "prompt": "Fix the chunk corpus so this component stops rendering wrong", "expect": "no-trigger", "owner": "adia-a2ui" },
|
|
15
|
+
{ "id": "n03", "prompt": "Add a new primitive to the component catalog", "expect": "no-trigger", "owner": "adia-author" },
|
|
16
|
+
{ "id": "n04", "prompt": "Audit the admin-shell composition for missing canonical parts", "expect": "no-trigger", "owner": "adia-dogfood" },
|
|
17
|
+
{ "id": "n05", "prompt": "[scheduled] cut v0.9.3 overnight", "expect": "no-trigger", "owner": "adia-release" },
|
|
18
|
+
{ "id": "n06", "prompt": "The gemini adapter is dropping usage tokens on streamed responses", "expect": "no-trigger", "owner": "adia-llm-internals" }
|
|
19
|
+
]
|
|
20
|
+
}
|
|
@@ -1,34 +1,119 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"name": "adia-gen-review routing accuracy corpus",
|
|
4
|
+
"version": "2.1.0",
|
|
5
|
+
"purpose": "Routing-eval corpus for adia-gen-review. Each phrase declares the skill (expected), a forbidden skill (expected_not, for phrases the source data only ever asserted as \"not this skill\"), or neither. Scored by scripts/skills/run-skill-evals.mjs (TF-IDF token overlap over per-skill description+triggers).",
|
|
6
|
+
"scoring_notes": "Heuristic signal, not ground truth. Treat misroutes as a prompt to tighten the skill description, never as a reason to keyword-stuff it. Real harness routing is LLM-driven.",
|
|
7
|
+
"scope": "adia-gen-review routing \u2014 does this phrase activate adia-gen-review?",
|
|
8
|
+
"phrases": [
|
|
9
|
+
{
|
|
10
|
+
"id": "adia-gen-review-pos-01",
|
|
11
|
+
"phrase": "score the gen-ui gallery outputs against the rubric",
|
|
12
|
+
"expected": "adia-gen-review"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"id": "adia-gen-review-pos-02",
|
|
16
|
+
"phrase": "run a gen-review cycle over the gallery",
|
|
17
|
+
"expected": "adia-gen-review"
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"id": "adia-gen-review-pos-03",
|
|
21
|
+
"phrase": "review the gen-ui outputs for quality",
|
|
22
|
+
"expected": "adia-gen-review"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "adia-gen-review-pos-04",
|
|
26
|
+
"phrase": "decompose the rendered canvas and score the gap against the ideal composition",
|
|
27
|
+
"expected": "adia-gen-review"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"id": "adia-gen-review-pos-05",
|
|
31
|
+
"phrase": "check for overflow elements in the generated screens",
|
|
32
|
+
"expected": "adia-gen-review"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"id": "adia-gen-review-pos-06",
|
|
36
|
+
"phrase": "root-cause why this gen-ui prompt's render score is low",
|
|
37
|
+
"expected": "adia-gen-review"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"id": "adia-gen-review-pos-07",
|
|
41
|
+
"phrase": "run the exit gate check on the review ledger",
|
|
42
|
+
"expected": "adia-gen-review"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "adia-gen-review-pos-08",
|
|
46
|
+
"phrase": "classify the cause code for this bad gen-ui render",
|
|
47
|
+
"expected": "adia-gen-review"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"id": "adia-gen-review-pos-09",
|
|
51
|
+
"phrase": "write a fix plan for the failing prompts in the gallery",
|
|
52
|
+
"expected": "adia-gen-review"
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "adia-gen-review-pos-10",
|
|
56
|
+
"phrase": "score this prompt against its ideal composition and report the delta",
|
|
57
|
+
"expected": "adia-gen-review"
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"id": "adia-gen-review-pos-11",
|
|
61
|
+
"phrase": "check the cosmetic findings in the gallery screenshots",
|
|
62
|
+
"expected": "adia-gen-review"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "adia-gen-review-pos-12",
|
|
66
|
+
"phrase": "is the gen-ui gallery review cycle complete yet",
|
|
67
|
+
"expected": "adia-gen-review"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"id": "adia-gen-review-neg-01",
|
|
71
|
+
"phrase": "run a dogfood sweep on the demo pages before we cut",
|
|
72
|
+
"expected_not": "adia-gen-review"
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"id": "adia-gen-review-neg-02",
|
|
76
|
+
"phrase": "find native primitive leaks across the app shells",
|
|
77
|
+
"expected_not": "adia-gen-review"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"id": "adia-gen-review-neg-03",
|
|
81
|
+
"phrase": "fix the chunk corpus so this component stops rendering wrong",
|
|
82
|
+
"expected_not": "adia-gen-review"
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"id": "adia-gen-review-neg-04",
|
|
86
|
+
"phrase": "add a new primitive to the component catalog",
|
|
87
|
+
"expected_not": "adia-gen-review"
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"id": "adia-gen-review-neg-05",
|
|
91
|
+
"phrase": "audit the admin-shell composition for missing canonical parts",
|
|
92
|
+
"expected_not": "adia-gen-review"
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"id": "adia-gen-review-neg-06",
|
|
96
|
+
"phrase": "[scheduled] cut v0.9.3 overnight",
|
|
97
|
+
"expected_not": "adia-gen-review"
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"id": "adia-gen-review-neg-07",
|
|
101
|
+
"phrase": "the gemini adapter is dropping usage tokens on streamed responses",
|
|
102
|
+
"expected": "adia-llm-internals"
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"id": "adia-gen-review-neg-08",
|
|
106
|
+
"phrase": "deploy the latest build to ui-kit.exe.xyz",
|
|
107
|
+
"expected": "adia-deploy"
|
|
108
|
+
}
|
|
15
109
|
],
|
|
16
|
-
"
|
|
17
|
-
"run a dogfood sweep on the demo pages before we cut",
|
|
18
|
-
"find native primitive leaks across the app shells",
|
|
19
|
-
"fix the chunk corpus so this component stops rendering wrong",
|
|
20
|
-
"add a new primitive to the component catalog",
|
|
21
|
-
"audit the admin-shell composition for missing canonical parts",
|
|
22
|
-
"[scheduled] cut v0.9.3 overnight",
|
|
23
|
-
"the gemini adapter is dropping usage tokens on streamed responses",
|
|
24
|
-
"deploy the latest build to ui-kit.exe.xyz"
|
|
25
|
-
],
|
|
26
|
-
"_measured": {
|
|
110
|
+
"_measured_historical": {
|
|
27
111
|
"as_of": "2026-07-18",
|
|
28
112
|
"scorer": "routing_eval.py (nonoun-plugins/forge)",
|
|
29
113
|
"f1": 0.857,
|
|
30
114
|
"precision": 1.0,
|
|
31
115
|
"recall": 0.75,
|
|
32
|
-
"exit_code": 0
|
|
116
|
+
"exit_code": 0,
|
|
117
|
+
"note": "measured by the external routing_eval.py (nonoun-plugins/forge) against the pre-conversion positives/negatives shape; historical record only, not regenerated by run-skill-evals.mjs."
|
|
33
118
|
}
|
|
34
119
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill": "adia-llm-internals",
|
|
3
|
+
"note": "Trigger-routing suite (gh#369). expect=trigger: the skill should fire on the prompt in a fresh session. expect=no-trigger: a near-miss owned elsewhere (owner named per case) or generic knowledge with no AdiaUI skill involved. Cases derived from evals/routing-corpus.json's phrases array (gh#355 vetted corpus) plus the SKILL.md description's own trigger/NOT-fence language.",
|
|
4
|
+
"cases": [
|
|
5
|
+
{ "id": "t01", "prompt": "The gemini adapter is dropping usage tokens on streamed responses", "expect": "trigger" },
|
|
6
|
+
{ "id": "t02", "prompt": "stopReason from openai never reaches the client as MAX_TOKENS", "expect": "trigger" },
|
|
7
|
+
{ "id": "t03", "prompt": "The SSE parser chokes when a chunk splits mid-line", "expect": "trigger" },
|
|
8
|
+
{ "id": "t04", "prompt": "The passthrough proxy is returning 401 for anthropic requests in production", "expect": "trigger" },
|
|
9
|
+
{ "id": "t05", "prompt": "Add a new model to the MODELS registry and set it as DEFAULT_MODEL", "expect": "trigger" },
|
|
10
|
+
{ "id": "t06", "prompt": "createAdapter() is leaking a real API key to the browser on a production host", "expect": "trigger" },
|
|
11
|
+
{ "id": "t07", "prompt": "Add support for a new OpenAI-compatible gateway provider adapter", "expect": "trigger" },
|
|
12
|
+
{ "id": "t08", "prompt": "streamChat() never emits a terminal done chunk", "expect": "trigger" },
|
|
13
|
+
{ "id": "t09", "prompt": "The StreamChunk union is missing a variant for tool_use deltas", "expect": "trigger" },
|
|
14
|
+
{ "id": "n01", "prompt": "Wire the chat client into our admin app's chat-shell", "expect": "no-trigger", "owner": "adia-llm" },
|
|
15
|
+
{ "id": "n02", "prompt": "Cut a release and bump @adia-ai/llm's version with the rest of the lockstep packages", "expect": "no-trigger", "owner": "adia-release" },
|
|
16
|
+
{ "id": "n03", "prompt": "Explain how Server-Sent Events work in general", "expect": "no-trigger" },
|
|
17
|
+
{ "id": "n04", "prompt": "The chat-shell message list isn't rendering streamed replies", "expect": "no-trigger", "owner": "adia-llm" },
|
|
18
|
+
{ "id": "n05", "prompt": "Tune STRONG_MATCH and lift the eval fails in the compose strategies", "expect": "no-trigger", "owner": "adia-a2ui" },
|
|
19
|
+
{ "id": "n06", "prompt": "Deploy the ui-kit dist to exe.dev and restart the service", "expect": "no-trigger", "owner": "adia-deploy" },
|
|
20
|
+
{ "id": "n07", "prompt": "Run a dogfood sweep to find broken demos", "expect": "no-trigger", "owner": "adia-dogfood" }
|
|
21
|
+
]
|
|
22
|
+
}
|
|
@@ -1,37 +1,134 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"name": "adia-llm-internals routing accuracy corpus",
|
|
4
|
+
"version": "2.1.0",
|
|
5
|
+
"purpose": "Routing-eval corpus for adia-llm-internals. Each phrase declares the skill (expected), a forbidden skill (expected_not, for phrases the source data only ever asserted as \"not this skill\"), or neither. Scored by scripts/skills/run-skill-evals.mjs (TF-IDF token overlap over per-skill description+triggers).",
|
|
6
|
+
"scoring_notes": "Heuristic signal, not ground truth. Treat misroutes as a prompt to tighten the skill description, never as a reason to keyword-stuff it. Real harness routing is LLM-driven.",
|
|
7
|
+
"scope": "adia-llm-internals routing \u2014 does this phrase activate adia-llm-internals?",
|
|
8
|
+
"phrases": [
|
|
9
|
+
{
|
|
10
|
+
"id": "adia-llm-internals-pos-01",
|
|
11
|
+
"phrase": "the gemini adapter is dropping usage tokens on streamed responses",
|
|
12
|
+
"expected": "adia-llm-internals"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"id": "adia-llm-internals-pos-02",
|
|
16
|
+
"phrase": "stopReason from openai never reaches the client as MAX_TOKENS",
|
|
17
|
+
"expected": "adia-llm-internals"
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"id": "adia-llm-internals-pos-03",
|
|
21
|
+
"phrase": "the SSE parser chokes when a chunk splits mid-line",
|
|
22
|
+
"expected": "adia-llm-internals"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "adia-llm-internals-pos-04",
|
|
26
|
+
"phrase": "the passthrough proxy is returning 401 for anthropic requests in production",
|
|
27
|
+
"expected": "adia-llm-internals"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"id": "adia-llm-internals-pos-05",
|
|
31
|
+
"phrase": "add a new model to the MODELS registry and set it as DEFAULT_MODEL",
|
|
32
|
+
"expected": "adia-llm-internals"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"id": "adia-llm-internals-pos-06",
|
|
36
|
+
"phrase": "createAdapter() is leaking a real API key to the browser on a production host",
|
|
37
|
+
"expected": "adia-llm-internals"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"id": "adia-llm-internals-pos-07",
|
|
41
|
+
"phrase": "add support for a new OpenAI-compatible gateway provider adapter",
|
|
42
|
+
"expected": "adia-llm-internals"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "adia-llm-internals-pos-08",
|
|
46
|
+
"phrase": "streamChat() never emits a terminal done chunk",
|
|
47
|
+
"expected": "adia-llm-internals"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"id": "adia-llm-internals-pos-09",
|
|
51
|
+
"phrase": "detectProvider isn't resolving the new model id to the right adapter",
|
|
52
|
+
"expected": "adia-llm-internals"
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "adia-llm-internals-pos-10",
|
|
56
|
+
"phrase": "the anthropic adapter's buildRequest() diverges from the passthrough proxy path",
|
|
57
|
+
"expected": "adia-llm-internals"
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"id": "adia-llm-internals-pos-11",
|
|
61
|
+
"phrase": "the StreamChunk union is missing a variant for tool_use deltas",
|
|
62
|
+
"expected": "adia-llm-internals"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "adia-llm-internals-pos-12",
|
|
66
|
+
"phrase": "the stub LLM adapter isn't returning parseable A2UI JSON in dev mode",
|
|
67
|
+
"expected": "adia-llm-internals"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"id": "adia-llm-internals-neg-01",
|
|
71
|
+
"phrase": "wire the chat client into our admin app's chat-shell",
|
|
72
|
+
"expected_not": "adia-llm-internals"
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"id": "adia-llm-internals-neg-02",
|
|
76
|
+
"phrase": "why did the composer emit composition-synthesized instead of composition-match",
|
|
77
|
+
"expected_not": "adia-llm-internals"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"id": "adia-llm-internals-neg-03",
|
|
81
|
+
"phrase": "cut a release and bump @adia-ai/llm's version with the rest of the lockstep packages",
|
|
82
|
+
"expected_not": "adia-llm-internals"
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"id": "adia-llm-internals-neg-04",
|
|
86
|
+
"phrase": "add a new prop to the chat-input-ui primitive",
|
|
87
|
+
"expected_not": "adia-llm-internals"
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"id": "adia-llm-internals-neg-05",
|
|
91
|
+
"phrase": "explain how Server-Sent Events work in general",
|
|
92
|
+
"expected_not": "adia-llm-internals"
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"id": "adia-llm-internals-neg-06",
|
|
96
|
+
"phrase": "add a chat box to the admin app",
|
|
97
|
+
"expected_not": "adia-llm-internals"
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"id": "adia-llm-internals-neg-07",
|
|
101
|
+
"phrase": "the chat-shell message list isn't rendering streamed replies",
|
|
102
|
+
"expected_not": "adia-llm-internals"
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"id": "adia-llm-internals-neg-08",
|
|
106
|
+
"phrase": "tune STRONG_MATCH and lift the eval fails in the compose strategies",
|
|
107
|
+
"expected_not": "adia-llm-internals"
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
"id": "adia-llm-internals-neg-09",
|
|
111
|
+
"phrase": "deploy the ui-kit dist to exe.dev and restart the service",
|
|
112
|
+
"expected_not": "adia-llm-internals"
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
"id": "adia-llm-internals-neg-10",
|
|
116
|
+
"phrase": "run a dogfood sweep to find broken demos",
|
|
117
|
+
"expected_not": "adia-llm-internals"
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
"id": "adia-llm-internals-neg-11",
|
|
121
|
+
"phrase": "score the gen-ui gallery outputs against the exit gate",
|
|
122
|
+
"expected_not": "adia-llm-internals"
|
|
123
|
+
}
|
|
15
124
|
],
|
|
16
|
-
"
|
|
17
|
-
"wire the chat client into our admin app's chat-shell",
|
|
18
|
-
"why did the composer emit composition-synthesized instead of composition-match",
|
|
19
|
-
"cut a release and bump @adia-ai/llm's version with the rest of the lockstep packages",
|
|
20
|
-
"add a new prop to the chat-input-ui primitive",
|
|
21
|
-
"explain how Server-Sent Events work in general",
|
|
22
|
-
"add a chat box to the admin app",
|
|
23
|
-
"the chat-shell message list isn't rendering streamed replies",
|
|
24
|
-
"tune STRONG_MATCH and lift the eval fails in the compose strategies",
|
|
25
|
-
"deploy the ui-kit dist to exe.dev and restart the service",
|
|
26
|
-
"run a dogfood sweep to find broken demos",
|
|
27
|
-
"score the gen-ui gallery outputs against the exit gate"
|
|
28
|
-
],
|
|
29
|
-
"_measured": {
|
|
125
|
+
"_measured_historical": {
|
|
30
126
|
"as_of": "2026-07-18",
|
|
31
127
|
"scorer": "routing_eval.py (nonoun-plugins/forge)",
|
|
32
128
|
"f1": 0.857,
|
|
33
129
|
"precision": 1.0,
|
|
34
130
|
"recall": 0.75,
|
|
35
|
-
"exit_code": 0
|
|
131
|
+
"exit_code": 0,
|
|
132
|
+
"note": "measured by the external routing_eval.py (nonoun-plugins/forge) against the pre-conversion positives/negatives shape; historical record only, not regenerated by run-skill-evals.mjs."
|
|
36
133
|
}
|
|
37
134
|
}
|