@adia-ai/adia-ui-forge 0.8.9 → 0.8.10

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "adia-forge",
-  "version": "0.8.9",
+  "version": "0.8.10",
   "description": "Maintain the adia-ui (@adia-ai) framework itself \u2014 author primitives and shells, run the A2UI generation pipeline and its corpus, review gen-UI quality, sweep QA, cut releases, deploy. The maintainer counterpart to adia-factory (the consumer/app-author plugin).",
   "author": {
     "name": "Kim",

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog — adia-forge
 
+## [0.8.10] — 2026-07-20
+
+### Added
+- **`evals/evals.json` behavioral test-case files for 8 skills** (gh#369) — the file `release_gate.py`'s G7 coverage check actually reads (a schema distinct from `evals/routing-corpus.json`). Derived from each skill's already-vetted `routing-corpus.json` phrases, with real judgment applied — the forge side required genuine authoring since its corpus phrases are terse fragments, not natural-language prompts (independently reviewed, sampled).
+
+### Fixed
+- **`evals/routing-corpus.json` unified onto one schema across all skills** (gh#355) — converged forge's `{positives,negatives}` shape and factory's `{phrases}` shape onto the `{phrases:[...]}` form `scripts/skills/run-skill-evals.mjs` actually reads. `run-skill-evals.mjs` extended to aggregate the global corpus with every per-skill corpus (per-skill wins on id collision) instead of reading only the 97-phrase global file; added a `selftest` subcommand. Does NOT touch `routing_eval.py`/`release_gate.py` — those live in an externally-installed marketplace plugin, out of this repo's reach; gh#369 is the separate, real fix for G7's actual coverage check.
+
+### Maintenance
+- **`.claude-plugin/plugin.json` version bump** — moves in lockstep with package.json (the `/plugin update` cache key).
+
 ## [0.8.9] — 2026-07-19
 
 ### Fixed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adia-ai/adia-ui-forge",
-  "version": "0.8.9",
+  "version": "0.8.10",
   "description": "Maintain the adia-ui (@adia-ai) framework itself \u2014 author primitives and shells, run the A2UI generation pipeline and its corpus, review gen-UI quality, sweep QA, cut releases, deploy. The maintainer counterpart to adia-factory (the consumer/app-author plugin).",
   "keywords": [
     "adia-ui",

package/skills/adia-a2ui/evals/evals.json

@@ -0,0 +1,23 @@
+{
+  "skill": "adia-a2ui",
+  "note": "Trigger-routing suite (gh#369). expect=trigger: the skill should fire on the prompt in a fresh session. expect=no-trigger: a near-miss owned elsewhere (owner named per case) or generic knowledge with no AdiaUI skill involved. Cases derived from evals/routing-corpus.json's phrases array (gh#355 vetted corpus) plus the SKILL.md description's own trigger/NOT-fence language.",
+  "cases": [
+    { "id": "t01", "prompt": "Validate this A2UI document against the schema", "expect": "trigger" },
+    { "id": "t02", "prompt": "Harvest a chunk from this HTML demo for the corpus", "expect": "trigger" },
+    { "id": "t03", "prompt": "Zettel coverage dropped after the last corpus change, find out why", "expect": "trigger" },
+    { "id": "t04", "prompt": "Tune the STRONG_MATCH threshold for the retrieval scorer", "expect": "trigger" },
+    { "id": "t05", "prompt": "Run check_anti_patterns on this rendered gallery HTML", "expect": "trigger" },
+    { "id": "t06", "prompt": "Add a new MCP tool to the a2ui server for refine_composition", "expect": "trigger" },
+    { "id": "t07", "prompt": "Diagnose this eval gap — coverage regressed and I need the root cause", "expect": "trigger" },
+    { "id": "t08", "prompt": "Lift a semantic fail that's scoring under sixty in the nightly eval", "expect": "trigger" },
+    { "id": "t09", "prompt": "Why did the composer emit composition-match instead of composition-synthesized for this prompt?", "expect": "trigger" },
+    { "id": "t10", "prompt": "This content shape can't be expressed by any component's A2UI contract — audit the catalog", "expect": "trigger" },
+    { "id": "n01", "prompt": "Build a settings page from existing primitives for our app", "expect": "no-trigger", "owner": "adia-compose" },
+    { "id": "n02", "prompt": "Add a slot to button-ui and update its yaml contract", "expect": "no-trigger", "owner": "adia-author" },
+    { "id": "n03", "prompt": "Score the generated UI quality in apps/genui against the rubric", "expect": "no-trigger", "owner": "adia-gen-review" },
+    { "id": "n04", "prompt": "Run a broad visual QA sweep across site and playgrounds before we cut", "expect": "no-trigger", "owner": "adia-dogfood" },
+    { "id": "n05", "prompt": "Fix the @adia-ai/llm SSE adapter so it emits a terminal done chunk", "expect": "no-trigger", "owner": "adia-llm-internals" },
+    { "id": "n06", "prompt": "Mount gen-root and wire data resolvers for end users at runtime", "expect": "no-trigger", "owner": "adia-genui" },
+    { "id": "n07", "prompt": "Edit the getting-started docs page on the site for consistency", "expect": "no-trigger", "owner": "adia-site-docs" }
+  ]
+}

package/skills/adia-a2ui/evals/routing-corpus.json

@@ -1,38 +1,139 @@
 {
-  "positives": [
-    "validate this A2UI document",
-    "harvest a chunk from this HTML demo",
-    "zettel coverage dropped after the last change",
-    "should this repeated subtree become its own chunk",
-    "tune the STRONG_MATCH threshold",
-    "run check_anti_patterns on this rendered HTML",
-    "refine composition with an OAuth row",
-    "MRR dropped in the chunk retrieval eval",
-    "add an MCP tool to the a2ui server",
-    "audit the training data corpus for drift",
-    "diagnose this eval gap and regression",
-    "run the MCP pipeline: generate_ui then validate_schema",
-    "lift a semantic fail under sixty",
-    "why did the composer emit composition-match instead of composition-synthesized"
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "name": "adia-a2ui routing accuracy corpus",
+  "version": "2.1.0",
+  "purpose": "Routing-eval corpus for adia-a2ui. Each phrase declares the skill (expected), a forbidden skill (expected_not, for phrases the source data only ever asserted as \"not this skill\"), or neither. Scored by scripts/skills/run-skill-evals.mjs (TF-IDF token overlap over per-skill description+triggers).",
+  "scoring_notes": "Heuristic signal, not ground truth. Treat misroutes as a prompt to tighten the skill description, never as a reason to keyword-stuff it. Real harness routing is LLM-driven.",
+  "scope": "adia-a2ui routing \u2014 does this phrase activate adia-a2ui?",
+  "phrases": [
+    {
+      "id": "adia-a2ui-pos-01",
+      "phrase": "validate this A2UI document",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-02",
+      "phrase": "harvest a chunk from this HTML demo",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-03",
+      "phrase": "zettel coverage dropped after the last change",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-04",
+      "phrase": "should this repeated subtree become its own chunk",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-05",
+      "phrase": "tune the STRONG_MATCH threshold",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-06",
+      "phrase": "run check_anti_patterns on this rendered HTML",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-07",
+      "phrase": "refine composition with an OAuth row",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-08",
+      "phrase": "MRR dropped in the chunk retrieval eval",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-09",
+      "phrase": "add an MCP tool to the a2ui server",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-10",
+      "phrase": "audit the training data corpus for drift",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-11",
+      "phrase": "diagnose this eval gap and regression",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-12",
+      "phrase": "run the MCP pipeline: generate_ui then validate_schema",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-13",
+      "phrase": "lift a semantic fail under sixty",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-pos-14",
+      "phrase": "why did the composer emit composition-match instead of composition-synthesized",
+      "expected": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-neg-01",
+      "phrase": "build a settings page from existing primitives",
+      "expected_not": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-neg-02",
+      "phrase": "add a slot to button-ui and update its yaml contract",
+      "expected_not": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-neg-03",
+      "phrase": "cut a release and bump the version tag",
+      "expected_not": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-neg-04",
+      "phrase": "score the generated UI quality in apps/genui",
+      "expected_not": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-neg-05",
+      "phrase": "run a broad visual QA sweep across site and playgrounds",
+      "expected_not": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-neg-06",
+      "phrase": "fix the @adia-ai/llm SSE adapter",
+      "expected_not": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-neg-07",
+      "phrase": "explain how React state management works",
+      "expected_not": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-neg-08",
+      "phrase": "compose a settings screen from catalog primitives",
+      "expected_not": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-neg-09",
+      "phrase": "mount gen-root and wire data resolvers for end users at runtime",
+      "expected_not": "adia-a2ui"
+    },
+    {
+      "id": "adia-a2ui-neg-10",
+      "phrase": "edit the getting-started docs page on the site",
+      "expected_not": "adia-a2ui"
+    }
   ],
-  "negatives": [
-    "build a settings page from existing primitives",
-    "add a slot to button-ui and update its yaml contract",
-    "cut a release and bump the version tag",
-    "score the generated UI quality in apps/genui",
-    "run a broad visual QA sweep across site and playgrounds",
-    "fix the @adia-ai/llm SSE adapter",
-    "explain how React state management works",
-    "compose a settings screen from catalog primitives",
-    "mount gen-root and wire data resolvers for end users at runtime",
-    "edit the getting-started docs page on the site"
-  ],
-  "_measured": {
+  "_measured_historical": {
     "as_of": "2026-07-18",
     "scorer": "routing_eval.py (nonoun-plugins/forge)",
     "f1": 0.783,
     "precision": 1.0,
     "recall": 0.643,
-    "exit_code": 0
+    "exit_code": 0,
+    "note": "measured by the external routing_eval.py (nonoun-plugins/forge) against the pre-conversion positives/negatives shape; historical record only, not regenerated by run-skill-evals.mjs."
   }
 }

package/skills/adia-author/evals/evals.json

@@ -0,0 +1,22 @@
+{
+  "skill": "adia-author",
+  "note": "Trigger-routing suite (gh#369). expect=trigger: the skill should fire on the prompt in a fresh session. expect=no-trigger: a near-miss owned elsewhere (owner named per case) or generic knowledge with no AdiaUI skill involved. Cases derived from evals/routing-corpus.json's phrases array (gh#355 vetted corpus) plus the SKILL.md description's own trigger/NOT-fence language.",
+  "cases": [
+    { "id": "t01", "prompt": "Add a new <toggle-ui> primitive component from scratch", "expect": "trigger" },
+    { "id": "t02", "prompt": "Add a new prop to table-ui for sticky headers", "expect": "trigger" },
+    { "id": "t03", "prompt": "Fix the CSS in card-ui to handle the new bleed variant", "expect": "trigger" },
+    { "id": "t04", "prompt": "Update the input-ui yaml to add a placeholder attribute", "expect": "trigger" },
+    { "id": "t05", "prompt": "Build the chat-shell composition with a new sidebar slot", "expect": "trigger" },
+    { "id": "t06", "prompt": "This billing-summary block keeps repeating across apps — extract it to a shared web-modules module", "expect": "trigger" },
+    { "id": "t07", "prompt": "Audit the four-axis contract for table-ui — token usage and lifecycle symmetry", "expect": "trigger" },
+    { "id": "t08", "prompt": "Author a demo for the billing-overview composite, an examples.html", "expect": "trigger" },
+    { "id": "t09", "prompt": "Add a pane behaviour to admin-shell per the bespoke shell-tier convention", "expect": "trigger" },
+    { "id": "n01", "prompt": "Compose this Figma mock into AdiaUI components for our consumer app", "expect": "no-trigger", "owner": "adia-compose" },
+    { "id": "n02", "prompt": "Cut a release of the next version and publish the npm packages", "expect": "no-trigger", "owner": "adia-release" },
+    { "id": "n03", "prompt": "Tune the zettel composition coverage, the eval is at 38%", "expect": "no-trigger", "owner": "adia-a2ui" },
+    { "id": "n04", "prompt": "Deploy the API service to the hosting platform", "expect": "no-trigger", "owner": "adia-deploy" },
+    { "id": "n05", "prompt": "The gemini adapter is dropping usage tokens on streamed responses", "expect": "no-trigger", "owner": "adia-llm-internals" },
+    { "id": "n06", "prompt": "Extend the LLM bridge in @adia-ai/llm to add a new provider", "expect": "no-trigger", "owner": "adia-llm-internals" },
+    { "id": "n07", "prompt": "Explain how React hooks work in general", "expect": "no-trigger" }
+  ]
+}

package/skills/adia-author/evals/routing-corpus.json

@@ -1,43 +1,164 @@
 {
-  "positives": [
-    "add a new button-ui primitive component",
-    "build a new <toggle-ui> component from scratch",
-    "create a new primitive web component in packages/web-components",
-    "add a new prop to table-ui",
-    "fix the CSS in card-ui to handle the new bleed variant",
-    "update the input-ui yaml to add a placeholder attribute",
-    "build the chat-shell composition with the new sidebar slot",
-    "author a new web-module shell at packages/web-modules/editor/editor-shell",
-    "add a pane behaviour to admin-shell per the bespoke shell-tier convention",
-    "promote this inline content to a shared web-modules module",
-    "this content keeps repeating across apps, extract to a shared module",
-    "lift this billing-summary block from apps/saas to web-modules",
-    "audit the four-axis contract for table-ui",
-    "check the token usage in card-ui for drift",
-    "lifecycle audit on dropdown-ui \u2014 symmetric connected/disconnected?",
-    "author a demo for the billing-overview composite",
-    "compose the examples.html for dashboard-layout",
-    "write the demo for <integrations-page-ui> in packages/web-modules/settings"
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "name": "adia-author routing accuracy corpus",
+  "version": "2.1.0",
+  "purpose": "Routing-eval corpus for adia-author. Each phrase declares the skill (expected), a forbidden skill (expected_not, for phrases the source data only ever asserted as \"not this skill\"), or neither. Scored by scripts/skills/run-skill-evals.mjs (TF-IDF token overlap over per-skill description+triggers).",
+  "scoring_notes": "Heuristic signal, not ground truth. Treat misroutes as a prompt to tighten the skill description, never as a reason to keyword-stuff it. Real harness routing is LLM-driven.",
+  "scope": "adia-author routing \u2014 does this phrase activate adia-author?",
+  "phrases": [
+    {
+      "id": "adia-author-pos-01",
+      "phrase": "add a new button-ui primitive component",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-02",
+      "phrase": "build a new <toggle-ui> component from scratch",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-03",
+      "phrase": "create a new primitive web component in packages/web-components",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-04",
+      "phrase": "add a new prop to table-ui",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-05",
+      "phrase": "fix the CSS in card-ui to handle the new bleed variant",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-06",
+      "phrase": "update the input-ui yaml to add a placeholder attribute",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-07",
+      "phrase": "build the chat-shell composition with the new sidebar slot",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-08",
+      "phrase": "author a new web-module shell at packages/web-modules/editor/editor-shell",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-09",
+      "phrase": "add a pane behaviour to admin-shell per the bespoke shell-tier convention",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-10",
+      "phrase": "promote this inline content to a shared web-modules module",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-11",
+      "phrase": "this content keeps repeating across apps, extract to a shared module",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-12",
+      "phrase": "lift this billing-summary block from apps/saas to web-modules",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-13",
+      "phrase": "audit the four-axis contract for table-ui",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-14",
+      "phrase": "check the token usage in card-ui for drift",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-15",
+      "phrase": "lifecycle audit on dropdown-ui \u2014 symmetric connected/disconnected?",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-16",
+      "phrase": "author a demo for the billing-overview composite",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-17",
+      "phrase": "compose the examples.html for dashboard-layout",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-pos-18",
+      "phrase": "write the demo for <integrations-page-ui> in packages/web-modules/settings",
+      "expected": "adia-author"
+    },
+    {
+      "id": "adia-author-neg-01",
+      "phrase": "compose this Figma mock into AdiaUI components for our consumer app",
+      "expected_not": "adia-author"
+    },
+    {
+      "id": "adia-author-neg-02",
+      "phrase": "cut a release of the next version and publish the npm packages",
+      "expected_not": "adia-author"
+    },
+    {
+      "id": "adia-author-neg-03",
+      "phrase": "tune the zettel composition coverage; eval is at 38%",
+      "expected_not": "adia-author"
+    },
+    {
+      "id": "adia-author-neg-04",
+      "phrase": "build a user profile card with avatar and edit actions",
+      "expected_not": "adia-author"
+    },
+    {
+      "id": "adia-author-neg-05",
+      "phrase": "deploy the api service to the hosting platform",
+      "expected_not": "adia-author"
+    },
+    {
+      "id": "adia-author-neg-06",
+      "phrase": "explain how React hooks work",
+      "expected_not": "adia-author"
+    },
+    {
+      "id": "adia-author-neg-07",
+      "phrase": "the gemini adapter is dropping usage tokens on streamed responses",
+      "expected": "adia-llm-internals"
+    },
+    {
+      "id": "adia-author-neg-08",
+      "phrase": "fix the openai stopReason mapping in the llm client",
+      "expected_not": "adia-author"
+    },
+    {
+      "id": "adia-author-neg-09",
+      "phrase": "extend the LLM bridge to add a new provider",
+      "expected_not": "adia-author"
+    },
+    {
+      "id": "adia-author-neg-10",
+      "phrase": "modify createAdapter in @adia-ai/llm to add chunk-shape support",
+      "expected_not": "adia-author"
+    },
+    {
+      "id": "adia-author-neg-11",
+      "phrase": "set up the LLM-bridge surface for a new tool-call provider",
+      "expected_not": "adia-author"
+    }
   ],
-  "negatives": [
-    "compose this Figma mock into AdiaUI components for our consumer app",
-    "cut a release of the next version and publish the npm packages",
-    "tune the zettel composition coverage; eval is at 38%",
-    "build a user profile card with avatar and edit actions",
-    "deploy the api service to the hosting platform",
-    "explain how React hooks work",
-    "the gemini adapter is dropping usage tokens on streamed responses",
-    "fix the openai stopReason mapping in the llm client",
-    "extend the LLM bridge to add a new provider",
-    "modify createAdapter in @adia-ai/llm to add chunk-shape support",
-    "set up the LLM-bridge surface for a new tool-call provider"
-  ],
-  "_measured": {
+  "_measured_historical": {
     "as_of": "2026-07-18",
     "scorer": "routing_eval.py (nonoun-plugins/forge)",
     "f1": 0.971,
     "precision": 1.0,
     "recall": 0.944,
-    "exit_code": 0
+    "exit_code": 0,
+    "note": "measured by the external routing_eval.py (nonoun-plugins/forge) against the pre-conversion positives/negatives shape; historical record only, not regenerated by run-skill-evals.mjs."
   }
 }

package/skills/adia-deploy/evals/evals.json

@@ -0,0 +1,22 @@
+{
+  "skill": "adia-deploy",
+  "note": "Trigger-routing suite (gh#369). expect=trigger: the skill should fire on the prompt in a fresh session. expect=no-trigger: a near-miss owned elsewhere (owner named per case) or generic knowledge with no AdiaUI skill involved. Cases derived from evals/routing-corpus.json's phrases array (gh#355 vetted corpus) plus the SKILL.md description's own trigger/NOT-fence language.",
+  "cases": [
+    { "id": "t01", "prompt": "Deploy the latest build to ui-kit.exe.xyz", "expect": "trigger" },
+    { "id": "t02", "prompt": "Diagnose why the site returns Port 8000 unbound", "expect": "trigger" },
+    { "id": "t03", "prompt": "The site is throwing a 502, check the VM service", "expect": "trigger" },
+    { "id": "t04", "prompt": "Restart the adia-ui service on the exe.dev VM", "expect": "trigger" },
+    { "id": "t05", "prompt": "Provision a fresh exe.dev VM for a new host", "expect": "trigger" },
+    { "id": "t06", "prompt": "Rotate the API keys on the production VM", "expect": "trigger" },
+    { "id": "t07", "prompt": "Push a site-v* tag to deploy the current build, review the dry-run delete summary first", "expect": "trigger" },
+    { "id": "t08", "prompt": "Did we deploy the site after the last lockstep cut, or is it behind npm?", "expect": "trigger" },
+    { "id": "t09", "prompt": "Roll back the last site deploy, the new build is broken", "expect": "trigger" },
+    { "id": "n01", "prompt": "Cut a release of the next version and publish the npm packages", "expect": "no-trigger", "owner": "adia-release" },
+    { "id": "n02", "prompt": "Verify that this consumer app screen renders correctly before shipping", "expect": "no-trigger", "owner": "adia-verify" },
+    { "id": "n03", "prompt": "Add a new prop to button-ui", "expect": "no-trigger", "owner": "adia-author" },
+    { "id": "n04", "prompt": "Run a dogfood sweep on the demo pages", "expect": "no-trigger", "owner": "adia-dogfood" },
+    { "id": "n05", "prompt": "Tune the zettel retrieval coverage, the eval dropped to 38%", "expect": "no-trigger", "owner": "adia-a2ui" },
+    { "id": "n06", "prompt": "Why does accordion-ui crash under SSR in Astro?", "expect": "no-trigger", "owner": "adia-ssr" },
+    { "id": "n07", "prompt": "Fix the streaming adapter bug in the @adia-ai/llm package", "expect": "no-trigger", "owner": "adia-llm-internals" }
+  ]
+}

package/skills/adia-deploy/evals/routing-corpus.json

@@ -1,37 +1,134 @@
 {
-  "positives": [
-    "deploy the latest build to ui-kit.exe.xyz",
-    "push the dist folder to the VM",
-    "diagnose why the site returns Port 8000 unbound",
-    "the site is throwing a 502, check the VM service",
-    "restart the adia-ui service on the exe.dev VM",
-    "provision a fresh exe.dev VM for a new host",
-    "rotate the API keys on the production VM",
-    "push a site-v* tag to deploy the current build",
-    "the site looks stale, new fixtures are 404ing",
-    "check the dry-run delete summary before the deploy job runs",
-    "did we deploy the site after the last lockstep cut, or is it behind npm",
-    "roll back the last site deploy, the new build is broken"
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "name": "adia-deploy routing accuracy corpus",
+  "version": "2.1.0",
+  "purpose": "Routing-eval corpus for adia-deploy. Each phrase declares the skill (expected), a forbidden skill (expected_not, for phrases the source data only ever asserted as \"not this skill\"), or neither. Scored by scripts/skills/run-skill-evals.mjs (TF-IDF token overlap over per-skill description+triggers).",
+  "scoring_notes": "Heuristic signal, not ground truth. Treat misroutes as a prompt to tighten the skill description, never as a reason to keyword-stuff it. Real harness routing is LLM-driven.",
+  "scope": "adia-deploy routing \u2014 does this phrase activate adia-deploy?",
+  "phrases": [
+    {
+      "id": "adia-deploy-pos-01",
+      "phrase": "deploy the latest build to ui-kit.exe.xyz",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-pos-02",
+      "phrase": "push the dist folder to the VM",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-pos-03",
+      "phrase": "diagnose why the site returns Port 8000 unbound",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-pos-04",
+      "phrase": "the site is throwing a 502, check the VM service",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-pos-05",
+      "phrase": "restart the adia-ui service on the exe.dev VM",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-pos-06",
+      "phrase": "provision a fresh exe.dev VM for a new host",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-pos-07",
+      "phrase": "rotate the API keys on the production VM",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-pos-08",
+      "phrase": "push a site-v* tag to deploy the current build",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-pos-09",
+      "phrase": "the site looks stale, new fixtures are 404ing",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-pos-10",
+      "phrase": "check the dry-run delete summary before the deploy job runs",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-pos-11",
+      "phrase": "did we deploy the site after the last lockstep cut, or is it behind npm",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-pos-12",
+      "phrase": "roll back the last site deploy, the new build is broken",
+      "expected": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-neg-01",
+      "phrase": "cut a release of the next version and publish the npm packages",
+      "expected_not": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-neg-02",
+      "phrase": "verify that this consumer app screen renders correctly before shipping",
+      "expected_not": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-neg-03",
+      "phrase": "add a new prop to button-ui",
+      "expected_not": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-neg-04",
+      "phrase": "run a dogfood sweep on the demo pages",
+      "expected_not": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-neg-05",
+      "phrase": "tune the zettel retrieval coverage, eval dropped to 38%",
+      "expected_not": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-neg-06",
+      "phrase": "publish the npm packages for the next lockstep cut",
+      "expected_not": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-neg-07",
+      "phrase": "author the release notes and the migration guide",
+      "expected_not": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-neg-08",
+      "phrase": "why does accordion-ui crash under SSR in Astro",
+      "expected_not": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-neg-09",
+      "phrase": "fix the streaming adapter bug in the llm package",
+      "expected_not": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-neg-10",
+      "phrase": "review the getting-started docs page for consistency",
+      "expected_not": "adia-deploy"
+    },
+    {
+      "id": "adia-deploy-neg-11",
+      "phrase": "score the gen-ui gallery outputs for this cycle",
+      "expected_not": "adia-deploy"
+    }
   ],
-  "negatives": [
-    "cut a release of the next version and publish the npm packages",
-    "verify that this consumer app screen renders correctly before shipping",
-    "add a new prop to button-ui",
-    "run a dogfood sweep on the demo pages",
-    "tune the zettel retrieval coverage, eval dropped to 38%",
-    "publish the npm packages for the next lockstep cut",
-    "author the release notes and the migration guide",
-    "why does accordion-ui crash under SSR in Astro",
-    "fix the streaming adapter bug in the llm package",
-    "review the getting-started docs page for consistency",
-    "score the gen-ui gallery outputs for this cycle"
-  ],
-  "_measured": {
+  "_measured_historical": {
     "as_of": "2026-07-18",
     "scorer": "routing_eval.py (nonoun-plugins/forge)",
     "f1": 0.917,
     "precision": 0.917,
     "recall": 0.917,
-    "exit_code": 0
+    "exit_code": 0,
+    "note": "measured by the external routing_eval.py (nonoun-plugins/forge) against the pre-conversion positives/negatives shape; historical record only, not regenerated by run-skill-evals.mjs."
   }
 }