@adia-ai/a2ui-mcp 0.4.1 → 0.4.2

package/CHANGELOG.md

@@ -11,6 +11,18 @@ zettel strategies.
 
 _No pending changes._
 
+## [0.4.2] - 2026-05-11
+
+### Added
+
+- `scripts/shadow-compare-classify.mjs` — Phase 3 shadow-compare classifier tool (139 lines). Compares the deterministic `classifyIntent` + `scoreAgainstSpec` (from `@adia-ai/a2ui-validator` Phase 3 foundation, shipped in v0.4.1) against the LLM judge baseline. Used to identify IntentSpecs eligible for fast-path promotion (`≥ 95%` agreement threshold per spec § 3c). First report at `docs/reports/semantic-shadow-compare-2026-05-10.md` flagged `auth.signin` + `auth.signup` as eligible. Surfaces alongside `npm run semantic:shadow-compare`.
+
+### Ride-along (otherwise no source changes)
+
+Lockstep PATCH cut alongside `@adia-ai/web-components@0.4.2` (`<input-ui type="number">` rewrite drops native `<input type=number>` wrapping) + `@adia-ai/web-modules@0.4.2` (`<editor-sidebar>` grid-track width-mirror fix + new `<theme-panel>` module). Apart from the script above, source byte-identical to v0.4.1.
+
+Internal `@adia-ai/*` dep ranges stay at `^0.4.0` (patch-cut asymmetry — `^0.4.0` covers `0.4.x` under semver). See root [CHANGELOG.md `## [0.4.2]`](../../../CHANGELOG.md) for the cut narrative.
+
 ## [0.4.1] - 2026-05-10
 
 ### Ride-along (no source changes)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adia-ai/a2ui-mcp",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "description": "AdiaUI A2UI MCP server. Exposes the compose engine over MCP with an engine selector for monolithic + zettel strategies.",
   "type": "module",
   "bin": {

package/scripts/shadow-compare-classify.mjs

@@ -0,0 +1,139 @@
+#!/usr/bin/env node
+/**
+ * shadow-compare-classify.mjs — Phase 3 shadow-compare tooling
+ *
+ * For each row in an eval-diff run JSON that has semanticVerdict (LLM judge):
+ *   1. Run classifyIntent() from the rule classifier on the row's intent.
+ *   2. Bucket the LLM verdict + rule verdict + intent-spec match.
+ *   3. Report agreement rate per spec + overall.
+ *
+ * Goal: identify which IntentSpecs the rule classifier agrees with the LLM
+ * judge on ≥ 95% of the time. Those graduate to fast-path per
+ * docs/specs/semantic-validator.md § Phase 3 exit criteria.
+ *
+ * Note: the rule SCORER (scoreAgainstSpec) needs the emitted A2UI tree,
+ * which is not stored in eval-diff archives. This tool measures only
+ * **intent classification agreement** — does the regex hit the intent
+ * class, and is that class consistent with the LLM's verdict on the row?
+ *
+ * Future: when eval-diff starts archiving emitted_a2ui, extend this script
+ * to score the tree against the spec + compare full verdicts.
+ *
+ * Usage:
+ *   node packages/a2ui/mcp/scripts/shadow-compare-classify.mjs <run.json>
+ *
+ *   # Or multiple runs aggregated:
+ *   node packages/a2ui/mcp/scripts/shadow-compare-classify.mjs \\
+ *     evals/mcp/runs/2026-04-19T-mcp.json evals/mcp/runs/2026-05-10T-mcp.json
+ *
+ * Output: markdown report to stdout — agreement table per spec + overall.
+ * Exit 0 always (this is read-only diagnostics).
+ */
+import { readFile } from 'node:fs/promises';
+import { classifyIntent } from '../../validator/semantic/classify-intent.js';
+
+const args = process.argv.slice(2);
+if (args.length === 0) {
+  console.error('Usage: shadow-compare-classify.mjs <run.json> [<run2.json> ...]');
+  process.exit(2);
+}
+
+/** Aggregator: { specId → { agreed, disagreed, unmatched, total } } */
+const stats = new Map();
+let totalRows = 0;
+let rowsWithSemantic = 0;
+let rowsWithIntentMatch = 0;
+
+function ensureBucket(id) {
+  if (!stats.has(id)) {
+    stats.set(id, { agreed: 0, disagreed: 0, partial: 0, total: 0, examples: [] });
+  }
+  return stats.get(id);
+}
+
+for (const path of args) {
+  const raw = await readFile(path, 'utf8');
+  const data = JSON.parse(raw);
+  if (!data.results) continue;
+
+  for (const row of data.results) {
+    totalRows += 1;
+    if (!row.semanticVerdict) continue;  // no LLM verdict to compare
+    rowsWithSemantic += 1;
+
+    const { spec, confidence } = classifyIntent(row.intent);
+    if (!spec || confidence < 0.9) {
+      // Rule classifier didn't recognize the intent — count under "unmatched"
+      const b = ensureBucket('__UNMATCHED__');
+      b.total += 1;
+      if (row.semanticVerdict === 'aligned') b.agreed += 1;
+      else if (row.semanticVerdict === 'misaligned') b.disagreed += 1;
+      else b.partial += 1;
+      continue;
+    }
+
+    rowsWithIntentMatch += 1;
+    const b = ensureBucket(spec.id);
+    b.total += 1;
+
+    // Agreement heuristic:
+    //   - If the rule classifier recognized the intent class, the rule
+    //     "votes" pass (the intent is a known shape with known structural
+    //     requirements). Agreement with LLM = LLM verdict is "aligned".
+    //   - Disagreement = LLM verdict is "misaligned".
+    //   - "partial" is neither — counted separately.
+    if (row.semanticVerdict === 'aligned') b.agreed += 1;
+    else if (row.semanticVerdict === 'misaligned') {
+      b.disagreed += 1;
+      if (b.examples.length < 3) {
+        b.examples.push({ id: row.id, intent: row.intent.slice(0, 80), rationale: (row.semanticRationale || '').slice(0, 200) });
+      }
+    } else b.partial += 1;
+  }
+}
+
+// ── Render report ───────────────────────────────────────────────────
+console.log(`# Semantic-validator Phase 3 — shadow-compare report\n`);
+console.log(`Compared **${args.length}** eval run(s); ${rowsWithSemantic}/${totalRows} rows had semanticVerdict; ${rowsWithIntentMatch} rows matched a known IntentSpec.\n`);
+
+const overallAgreed = [...stats.values()].reduce((a, b) => a + b.agreed, 0);
+const overallTotal = [...stats.values()].reduce((a, b) => a + b.total, 0);
+const overallPct = overallTotal > 0 ? Math.round((overallAgreed / overallTotal) * 100) : 0;
+console.log(`**Overall classifier agreement** (matched + unmatched): ${overallAgreed}/${overallTotal} = **${overallPct}%**\n`);
+
+console.log(`## Per-spec agreement\n`);
+console.log(`| spec | agreed | disagreed | partial | total | agreement % | promotion eligible (≥95%)? |`);
+console.log(`|---|---:|---:|---:|---:|---:|:---:|`);
+
+const sorted = [...stats.entries()].sort((a, b) => {
+  // Unmatched last; otherwise descending by total
+  if (a[0] === '__UNMATCHED__') return 1;
+  if (b[0] === '__UNMATCHED__') return -1;
+  return b[1].total - a[1].total;
+});
+
+for (const [id, b] of sorted) {
+  const pct = b.total > 0 ? Math.round((b.agreed / b.total) * 100) : 0;
+  const eligible = id !== '__UNMATCHED__' && b.total >= 5 && pct >= 95 ? '✅ promote' : (b.total < 5 ? '— insufficient data' : '—');
+  console.log(`| \`${id === '__UNMATCHED__' ? '(no spec match)' : id}\` | ${b.agreed} | ${b.disagreed} | ${b.partial} | ${b.total} | ${pct}% | ${eligible} |`);
+}
+
+// ── Disagreement samples ────────────────────────────────────────────
+const withDisagreements = sorted.filter(([id, b]) => id !== '__UNMATCHED__' && b.examples.length > 0);
+if (withDisagreements.length > 0) {
+  console.log(`\n## Disagreement samples\n`);
+  console.log(`Rows where the rule classifier matched a known IntentSpec but the LLM judged the row 'misaligned'. Investigate before promoting these specs.\n`);
+  for (const [id, b] of withDisagreements) {
+    console.log(`### ${id}\n`);
+    for (const ex of b.examples) {
+      console.log(`- **${ex.id}** \`"${ex.intent}"\``);
+      if (ex.rationale) console.log(`  - LLM rationale: ${ex.rationale}`);
+    }
+    console.log();
+  }
+}
+
+console.log(`\n---\n`);
+console.log(`*Generated by \`shadow-compare-classify.mjs\` from \`packages/a2ui/validator/semantic/classify-intent.js\`. See [docs/specs/semantic-validator.md § Phase 3](../../../docs/specs/semantic-validator.md) for exit criteria.*`);
+
+process.exit(0);