@adia-ai/a2ui-validator 0.0.1

package/semantic/judge.js

@@ -0,0 +1,180 @@
+/**
+ * LLM judge for semantic A2UI validation (Phase 1: Anthropic only).
+ *
+ * Implements the rubric from docs/specs/semantic-validator.md §4 (recommendation
+ * is pure 3a LLM-judge as first phase). Provider-agnostic shape per §5.3;
+ * only Anthropic wired here.
+ *
+ * Contract:
+ *   callJudge({ intent, componentSummary, rubricVersion }) →
+ *     { raw, parsed: SemanticVerdict, usage }
+ */
+
+const RUBRIC_VERSION = 'v1';
+
+const RUBRIC_TEXT = `You are a senior UI reviewer scoring whether an emitted UI actually
+satisfies a user's intent. You are NOT scoring visual polish or copy.
+You are scoring intent/output alignment on three axes.
+
+Score each axis 0-100. Cite evidence from the component summary.
+
+Axes:
+  1. dominantPattern: does the top-level pattern match what the intent asked
+     for? (form | data-display | navigation | marketing | feedback | settings
+     | chat | media | dashboard | detail | list | auth | other). A pricing
+     card in response to "signup form" is a dominant-pattern mismatch — score
+     very low (<15) regardless of filler components.
+  2. requiredCapabilities: does the output contain the specific controls /
+     affordances the intent requires? (e.g. "login form" requires Input(email
+     or username) + Input(password) + Button(submit); "users table" requires
+     a Table or tabular Grid with rows). List what was expected and what was
+     missing.
+  3. forbiddenNoise: are there prominent components that are off-topic for
+     this intent? (e.g. pricing tiers in a signup form, avatars in a data
+     table). 100 = nothing off-topic; drop for each noisy element.
+
+Overall verdict:
+  aligned     — all three axes ≥ 75
+  partial     — one axis < 60 OR overall score 40-74
+  misaligned  — dominantPattern < 40 OR two axes < 60
+  off-topic   — dominantPattern < 15
+
+Final score = round(0.5*dominantPattern + 0.35*requiredCapabilities + 0.15*forbiddenNoise).
+
+Return ONLY valid JSON, no prose, no markdown fences:
+{
+  "axes": {
+    "dominantPattern":      { "expected": "<string>", "observed": "<string>", "score": <0-100>, "evidence": "<short string>" },
+    "requiredCapabilities": { "expected": ["..."], "missing": ["..."], "score": <0-100> },
+    "forbiddenNoise":       { "observed": ["..."], "score": <0-100> }
+  },
+  "verdict": "aligned" | "partial" | "misaligned" | "off-topic",
+  "score": <0-100>,
+  "rationale": "<<= 240 chars>",
+  "evidence": ["<component id or short quote>", ...]
+}`;
+
+export function getRubric() {
+  return { version: RUBRIC_VERSION, text: RUBRIC_TEXT };
+}
+
+/**
+ * Summarize emitted A2UI messages into a compact human-readable string
+ * (§5.4). Handles both monolithic (messages with .components array) and
+ * zettel (flat beginComponent/endComponent stream) shapes.
+ */
+export function summarizeA2UI(messages) {
+  const msgs = Array.isArray(messages) ? messages : [];
+  const counts = new Map();
+  const samples = new Map(); // component -> [labels]
+  let rootType = null;
+  let totalComponents = 0;
+
+  const record = (type, props) => {
+    if (!type) return;
+    totalComponents += 1;
+    counts.set(type, (counts.get(type) || 0) + 1);
+    if (!rootType) rootType = type;
+    const label = props?.label || props?.text || props?.title || props?.placeholder;
+    if (label && typeof label === 'string') {
+      const arr = samples.get(type) || [];
+      if (arr.length < 3) arr.push(label.slice(0, 32));
+      samples.set(type, arr);
+    }
+  };
+
+  for (const m of msgs) {
+    if (Array.isArray(m?.components)) {
+      for (const c of m.components) record(c.component, c.props || c);
+    } else if (m?.componentType && m?.messageType !== 'endComponent') {
+      record(m.componentType, m.props || m);
+    }
+  }
+
+  const parts = [];
+  for (const [type, n] of [...counts.entries()].sort((a, b) => b[1] - a[1])) {
+    const s = samples.get(type);
+    const lbl = s && s.length ? ` (labels: ${s.map((x) => JSON.stringify(x)).join(', ')})` : '';
+    parts.push(`${type}×${n}${lbl}`);
+  }
+  return `Root: ${rootType || 'none'}\nTotal components: ${totalComponents}\nBreakdown: ${parts.join('; ') || '(empty)'}`;
+}
+
+function parseJSONLoose(raw) {
+  if (!raw) throw new Error('empty judge response');
+  let text = String(raw).trim();
+  // Strip markdown fences if present.
+  const fence = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/);
+  if (fence) text = fence[1].trim();
+  // Grab the first top-level { ... } block.
+  const first = text.indexOf('{');
+  const last = text.lastIndexOf('}');
+  if (first < 0 || last < 0 || last <= first) throw new Error('no JSON object in judge response');
+  const slice = text.slice(first, last + 1);
+  return JSON.parse(slice);
+}
+
+/**
+ * Call the LLM judge. Anthropic-only in Phase 1.
+ * Returns { raw, parsed, usage, latencyMs, provider, model }.
+ */
+export async function callJudge({
+  intent,
+  componentSummary,
+  rubricVersion = RUBRIC_VERSION,
+  timeoutMs = 15000,
+  model = 'claude-haiku-4-5-20251001',
+} = {}) {
+  const key = process.env.ANTHROPIC_API_KEY;
+  if (!key) throw new Error('ANTHROPIC_API_KEY missing — cannot call semantic judge');
+
+  const rubric = RUBRIC_TEXT;
+  const userBlock = `Intent: ${intent}\n\nEmitted UI summary:\n${componentSummary}\n\nRespond with JSON only.`;
+
+  const ac = new AbortController();
+  const timer = setTimeout(() => ac.abort(), timeoutMs);
+  const started = Date.now();
+  let res;
+  try {
+    res = await fetch('https://api.anthropic.com/v1/messages', {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        'x-api-key': key,
+        'anthropic-version': '2023-06-01',
+      },
+      body: JSON.stringify({
+        model,
+        max_tokens: 1024,
+        temperature: 0,
+        system: rubric,
+        messages: [{ role: 'user', content: userBlock }],
+      }),
+      signal: ac.signal,
+    });
+  } finally {
+    clearTimeout(timer);
+  }
+
+  const latencyMs = Date.now() - started;
+  if (!res.ok) {
+    const body = await res.text().catch(() => '');
+    throw new Error(`Anthropic judge ${res.status}: ${body.slice(0, 200)}`);
+  }
+  const data = await res.json();
+  const raw = data.content?.[0]?.text || '';
+  const parsed = parseJSONLoose(raw);
+
+  return {
+    raw,
+    parsed,
+    usage: {
+      inputTokens: data.usage?.input_tokens ?? 0,
+      outputTokens: data.usage?.output_tokens ?? 0,
+    },
+    latencyMs,
+    provider: 'anthropic',
+    model,
+    rubricVersion,
+  };
+}