@adia-ai/adia-ui-factory 0.8.6 → 0.8.8

package/skills/adia-patterns/evals/routing-corpus.json

@@ -0,0 +1,166 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "name": "adia-patterns routing accuracy corpus",
+  "version": "1.0.0",
+  "purpose": "Routing-eval corpus for adia-patterns (adia-ui-factory plugin). Each phrase declares whether adia-patterns SHOULD be the routing target. Scored by the estate's TF-IDF routing-eval runner against the skill's description (heuristic token overlap).",
+  "scoring_notes": "Heuristic signal, not ground truth. Treat misroutes as a prompt to tighten the skill description, never as a reason to keyword-stuff it. Real harness routing is LLM-driven.",
+  "license": "internal",
+  "scope": "adia-patterns routing — does this phrase activate adia-patterns?",
+
+  "minimums_per_spec": {
+    "trigger_phrases": 14,
+    "adversarial_phrases": 6,
+    "task_shapes_covered": "find-pattern, find-template, pre-compose-check, flow-screen-lookup, inventory-question, reuse-existing",
+    "adversarial_fraction": "30%"
+  },
+
+  "phrases": [
+    {
+      "id": "patterns-find-01",
+      "phrase": "is there an existing pattern for a kanban board?",
+      "should_route_to_patterns": true,
+      "expected_shape": "find-pattern",
+      "rationale": "The exact 'is there an existing pattern for X' trigger phrase named in the description, asked against a named pattern (kanban-board is in the 45-pattern index)."
+    },
+    {
+      "id": "patterns-find-02",
+      "phrase": "do we already have a pre-assembled surface for an audit log?",
+      "should_route_to_patterns": true,
+      "expected_shape": "find-pattern",
+      "rationale": "'pre-assembled surface' is the description's own framing for a pattern; audit-log is a named entry in the index."
+    },
+    {
+      "id": "patterns-find-03",
+      "phrase": "what patterns exist for permissions and role management?",
+      "should_route_to_patterns": true,
+      "expected_shape": "inventory-question",
+      "rationale": "A category-scoped inventory ask ('what patterns exist for X') against the permissions-access category, not a request to build anything yet."
+    },
+    {
+      "id": "patterns-find-04",
+      "phrase": "find me a template for a billing settings page",
+      "should_route_to_patterns": true,
+      "expected_shape": "find-template",
+      "rationale": "Names 'template' directly against the settings-page category the index covers (Billing sub-page)."
+    },
+    {
+      "id": "patterns-find-05",
+      "phrase": "is there a ready-made sign-in with MFA screen I can start from?",
+      "should_route_to_patterns": true,
+      "expected_shape": "flow-screen-lookup",
+      "rationale": "A specific auth-flow screen lookup ('ready-made... start from') matching the auth-flow template category (MFA screen)."
+    },
+    {
+      "id": "patterns-find-06",
+      "phrase": "start from an existing pattern instead of building this from scratch",
+      "should_route_to_patterns": true,
+      "expected_shape": "reuse-existing",
+      "rationale": "The description's core reuse-before-compose framing, stated generically with no pattern named yet — still adia-patterns' job to find the candidate."
+    },
+    {
+      "id": "patterns-find-07",
+      "phrase": "which template screens cover onboarding checklists?",
+      "should_route_to_patterns": true,
+      "expected_shape": "inventory-question",
+      "rationale": "Category-scoped inventory ask against the onboarding-flow template category."
+    },
+    {
+      "id": "patterns-find-08",
+      "phrase": "before I compose this dashboard, check if a pattern already covers it",
+      "should_route_to_patterns": true,
+      "expected_shape": "pre-compose-check",
+      "rationale": "The literal 'use BEFORE composing any screen from primitives' trigger condition named in the description, against the dashboard template category."
+    },
+    {
+      "id": "patterns-find-09",
+      "phrase": "reuse one of our pre-assembled registration flow screens",
+      "should_route_to_patterns": true,
+      "expected_shape": "reuse-existing",
+      "rationale": "Explicit reuse ask naming 'pre-assembled' and the registration-flow template category."
+    },
+    {
+      "id": "patterns-find-10",
+      "phrase": "does the catalog have an assembled chart-in-card surface?",
+      "should_route_to_patterns": true,
+      "expected_shape": "find-pattern",
+      "rationale": "Names a specific pattern (chart-in-card) in the data-viz category with the 'assembled surface' framing."
+    },
+    {
+      "id": "patterns-find-11",
+      "phrase": "list the existing agent tool-call and agent memory patterns",
+      "should_route_to_patterns": true,
+      "expected_shape": "inventory-question",
+      "rationale": "A listing ask against two named agent-ops category entries (agent-tool-call, agent-memory)."
+    },
+    {
+      "id": "patterns-find-12",
+      "phrase": "what's in the pattern and template index?",
+      "should_route_to_patterns": true,
+      "expected_shape": "inventory-question",
+      "rationale": "Names the index itself directly — the skill's own artifact."
+    },
+    {
+      "id": "patterns-find-13",
+      "phrase": "match this mockup against our existing patterns before building",
+      "should_route_to_patterns": true,
+      "expected_shape": "pre-compose-check",
+      "rationale": "A pre-compose reuse check triggered by a mockup input, before any primitive is chosen."
+    },
+    {
+      "id": "patterns-find-14",
+      "phrase": "is there a pre-built forgot-password / reset flow to copy?",
+      "should_route_to_patterns": true,
+      "expected_shape": "flow-screen-lookup",
+      "rationale": "Names a specific auth-flow template screen (Forgot Password / Reset Password) with a copy-source ask."
+    },
+    {
+      "id": "patterns-adv-01",
+      "phrase": "compose a brand-new screen from catalog primitives",
+      "should_route_to_patterns": false,
+      "expected_alternative": "adia-compose",
+      "rationale": "Adversarial — explicitly a fresh composition with no reuse framing; the description's own NOT-fence points this at adia-compose."
+    },
+    {
+      "id": "patterns-adv-02",
+      "phrase": "give me a sidebar plus topbar admin shell layout",
+      "should_route_to_patterns": false,
+      "expected_alternative": "adia-shells",
+      "rationale": "Adversarial — page-chrome selection is adia-shells' fence, not a pre-assembled content pattern, even though 'admin' vocabulary overlaps the index's shell-chrome category."
+    },
+    {
+      "id": "patterns-adv-03",
+      "phrase": "author a new pattern page for the framework docs",
+      "should_route_to_patterns": false,
+      "expected_alternative": "adia-author (adia-ui-forge)",
+      "rationale": "Adversarial — authoring a NEW pattern in the monorepo is forge-side framework work, not factory-side index lookup, despite the shared word 'pattern'."
+    },
+    {
+      "id": "patterns-adv-04",
+      "phrase": "tune the gen-ui chunk corpus retrieval for search_patterns",
+      "should_route_to_patterns": false,
+      "expected_alternative": "adia-a2ui (adia-ui-forge)",
+      "rationale": "Adversarial — a different corpus (the a2ui chunk corpus + its search_patterns MCP tool) that this skill's own SKILL.md explicitly distinguishes itself from."
+    },
+    {
+      "id": "patterns-adv-05",
+      "phrase": "wire the SSR host for this composed screen",
+      "should_route_to_patterns": false,
+      "expected_alternative": "adia-host",
+      "rationale": "Adversarial — host/SSR wiring is explicitly carved out to adia-host; no index lookup is implied."
+    },
+    {
+      "id": "patterns-adv-06",
+      "phrase": "generate UI at runtime from a user prompt in the app",
+      "should_route_to_patterns": false,
+      "expected_alternative": "adia-genui",
+      "rationale": "Adversarial — runtime generative UI is adia-genui's territory; the index is a build-time reuse aid, not a runtime generation surface."
+    }
+  ],
+
+  "evaluator_notes": {
+    "current_state": "20 cases (14 trigger + 6 adversarial). 6 task shapes across the 14 trigger cases: find-pattern (3), find-template (1), inventory-question (4), flow-screen-lookup (2), pre-compose-check (2), reuse-existing (2).",
+    "promotion_criteria": "Promote to a CI gate (warn → hard-fail) when F1 >= 0.85 on this corpus across 3+ consecutive runs without description edits.",
+    "review_cadence": "Re-run on every adia-patterns description edit. Add 1-2 new cases per checkpoint reflecting newly-observed routing failure modes.",
+    "known_limitations": "The a2ui-corpus fence (patterns-adv-04) and the adia-author fence (patterns-adv-03) both share the literal word 'pattern' with this skill's name — the heuristic TF-IDF scorer may not separate them as cleanly as an LLM router would."
+  }
+}