@adia-ai/a2ui-mcp 0.3.1 → 0.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adia-ai/a2ui-mcp",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "AdiaUI A2UI MCP server. Exposes the compose engine over MCP with an engine selector for monolithic + zettel strategies.",
   "type": "module",
   "bin": {
@@ -33,4 +33,4 @@
     "@adia-ai/llm": "^0.3.0",
     "zod": "^3.24.0"
   }
-}
+}

package/scripts/eval-compose-from-chunks.mjs

@@ -24,11 +24,9 @@
  *   - Retrieval  (20%) — top-k retrieved chunks include the intent's
  *     `expected_chunk` (when set). Soft-asserted; absence flags but
  *     doesn't fail.
- *   - Render fidelity (30%) — DEFERRED. Real implementation needs
- *     Playwright headless render + console-error capture; this runner
- *     emits a placeholder pending the render-fidelity smoke wiring.
- *     The composite re-distributes its weight pro-rata across the
- *     three remaining components when render is null.
+ *   - Render fidelity (30%) — Playwright headless render + console-error
+ *     capture via `render-fidelity.mjs`. Verifies the produced markup is
+ *     consumer-correct, not just structurally valid.
  *
  * Exit:
  *   0 if avg ≥ 80 (passes the chunk-zettel promotion gate threshold)
@@ -49,7 +47,8 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { composeFromIntent } from '../../compose/strategies/zettel/chunk-synthesizer.js';
-import { searchChunksAsync } from '../../corpus/scripts/chunk-library.js';
+import { searchChunks, searchChunksAsync } from '../../corpus/scripts/chunk-library.js';
+import { scoreRenderFidelity } from './render-fidelity.mjs';
 
 const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../..');
 const HOLDOUT = path.join(REPO_ROOT, 'packages/a2ui/corpus/evals/holdout-compose-from-chunks.jsonl');
@@ -84,21 +83,14 @@ async function buildLLMAdapter() {
     console.error('--real-llm requires ANTHROPIC_API_KEY in env.');
     process.exit(2);
   }
-  const { default: Anthropic } = await import('@anthropic-ai/sdk');
-  const client = new Anthropic({ apiKey });
+  // Use the project's provider-agnostic LLM layer instead of importing
+  // @anthropic-ai/sdk directly (not installed in this package).
+  const { chat } = await import('../../../llm/adapters/index.js');
+  const model = 'claude-sonnet-4-20250514';
   return {
-    async complete({ system, user, model = 'claude-sonnet-4-6', maxTokens = 2048 }) {
-      const resp = await client.messages.create({
-        model,
-        max_tokens: maxTokens,
-        system,
-        messages: [{ role: 'user', content: user }],
-      });
-      const text = resp.content
-        .filter((b) => b.type === 'text')
-        .map((b) => b.text)
-        .join('');
-      return { text };
+    async complete({ messages, systemPrompt }) {
+      const result = await chat({ apiKey, model, messages, system: systemPrompt });
+      return { text: result.text };
     },
   };
 }
@@ -117,8 +109,12 @@ function scoreCoverage(result, expected) {
   const html = String(result.html || '');
   let hits = 0;
   for (const tag of expected) {
-    const lowered = tag.toLowerCase().replace(/^ui/, '');
-    const re = new RegExp(`<${lowered}-ui[\\s>]`, 'i');
+    // PascalCase → kebab-case (e.g. "AgentTrace" → "agent-trace")
+    const kebab = tag
+      .replace(/^ui/, '')
+      .replace(/([a-z])([A-Z])/g, '$1-$2')
+      .toLowerCase();
+    const re = new RegExp(`<${kebab}-ui[\\s>]`, 'i');
     if (re.test(html)) hits++;
   }
   return Math.round((hits / expected.length) * 100);
@@ -126,15 +122,21 @@ function scoreCoverage(result, expected) {
 
 async function scoreRetrieval(intent, expectedChunk) {
   if (!expectedChunk) return null;
-  const hits = await searchChunksAsync(intent, { limit: 5 });
+  // Use sync keyword-only search for stable, deterministic retrieval scoring.
+  // Embeddings in searchChunksAsync are non-deterministic (small cosine boosts
+  // can flip rankings) and measure semantic similarity, not keyword
+  // discoverability. The eval tests whether a chunk is findable by the
+  // baseline keyword search that composeFromIntent falls back to when
+  // embeddings are unavailable.
+  const hits = searchChunks(intent, { limit: 5 });
   const found = hits.some((h) => h.name === expectedChunk);
   return found ? 100 : 0;
 }
 
-function scoreRenderFidelity(_result) {
-  // DEFERRED — Playwright headless render + console-error capture.
-  // See spec § Out-of-band for the follow-up.
-  return null;
+async function scoreRender(result) {
+  if (!result.html) return null;
+  const rf = await scoreRenderFidelity(result.html);
+  return rf.score;
 }
 
 function compositeScore({ structural, coverage, retrieval, render }) {
@@ -166,7 +168,7 @@ async function evalIntent(intent, llmAdapter) {
   const structural = scoreStructural(result);
   const coverage = scoreCoverage(result, intent.expected_components);
   const retrieval = await scoreRetrieval(intent.intent, intent.expected_chunk);
-  const render = scoreRenderFidelity(result);
+  const render = await scoreRender(result);
 
   const score = compositeScore({ structural, coverage, retrieval, render });
   return {

package/scripts/render-fidelity.mjs

@@ -0,0 +1,238 @@
+#!/usr/bin/env node
+/**
+ * render-fidelity.mjs — Playwright headless renderer for eval scoring.
+ *
+ * Scores an HTML fragment (chunk-composition output) on:
+ *   - Console errors (JS exceptions, missing custom elements, 404s)
+ *   - Structural completeness (non-blank viewport, key components visible)
+ *
+ * Returns { score: 0-100, errors: string[], blank: boolean, screenshot?: Buffer }
+ *
+ * Usage:
+ *   import { scoreRenderFidelity } from './render-fidelity.mjs';
+ *   const result = await scoreRenderFidelity('<card-ui>...</card-ui>');
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import http from 'node:http';
+import { fileURLToPath } from 'node:url';
+import { chromium } from 'playwright';
+
+const REPO_ROOT = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  '../../../..'
+);
+
+/**
+ * Wrap raw HTML in a full page with all styles + component registrations.
+ */
+function wrapForRender(rawHtml) {
+  const wcRoot = '/packages/web-components';
+  return `<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>Render fidelity probe</title>
+  <link rel="stylesheet" href="${wcRoot}/styles/tokens.css" />
+  <link rel="stylesheet" href="${wcRoot}/styles/components.css" />
+  <link rel="stylesheet" href="${wcRoot}/styles/resets.css" />
+  <script type="module" src="${wcRoot}/index.js"></script>
+  <style>
+    body { padding: 24px; }
+    /* Prevent layout-shift during hydration */
+    :not(:defined) { visibility: hidden; }
+  </style>
+</head>
+<body>
+${rawHtml}
+</body>
+</html>`;
+}
+
+/**
+ * Start a minimal static file server rooted at REPO_ROOT.
+ * Returns { url, stop() }.
+ */
+async function startDevServer(port = 0) {
+  const mimeTypes = {
+    '.html': 'text/html',
+    '.css': 'text/css',
+    '.js': 'text/javascript',
+    '.mjs': 'text/javascript',
+    '.json': 'application/json',
+    '.svg': 'image/svg+xml',
+    '.png': 'image/png',
+  };
+
+  const server = http.createServer((req, res) => {
+    const decoded = decodeURIComponent(req.url);
+    // Serve from repo root; strip leading /
+    const filePath = path.join(REPO_ROOT, decoded.replace(/^\//, ''));
+    const ext = path.extname(filePath).toLowerCase();
+
+    // Security: prevent path traversal
+    if (!filePath.startsWith(REPO_ROOT)) {
+      res.writeHead(403); res.end('Forbidden'); return;
+    }
+
+    fs.readFile(filePath, (err, data) => {
+      if (err) {
+        res.writeHead(404); res.end('Not found'); return;
+      }
+      res.writeHead(200, { 'Content-Type': mimeTypes[ext] || 'application/octet-stream' });
+      res.end(data);
+    });
+  });
+
+  return new Promise((resolve, reject) => {
+    server.listen(port, '127.0.0.1', () => {
+      const addr = server.address();
+      const url = `http://127.0.0.1:${addr.port}`;
+      resolve({
+        url,
+        stop: () => new Promise((r) => server.close(r)),
+      });
+    });
+    server.on('error', reject);
+  });
+}
+
+/**
+ * Score render fidelity for an HTML fragment.
+ *
+ * @param {string} html — raw HTML fragment (will be wrapped in full page)
+ * @param {object} opts
+ * @param {boolean} opts.captureScreenshot — save screenshot to /tmp/
+ * @returns {Promise<{score:number, errors:string[], blank:boolean, screenshotPath?:string}>}
+ */
+export async function scoreRenderFidelity(html, opts = {}) {
+  if (!html || typeof html !== 'string') {
+    return { score: 0, errors: ['No HTML emitted'], blank: true };
+  }
+
+  const pageHtml = wrapForRender(html);
+  let server;
+  let browser;
+  let page;
+
+  try {
+    server = await startDevServer();
+  } catch (err) {
+    return { score: 0, errors: [`Dev server failed: ${err.message}`], blank: true };
+  }
+
+  const consoleErrors = [];
+  const pageErrors = [];
+
+  try {
+    browser = await chromium.launch({ headless: true });
+    page = await browser.newPage({ viewport: { width: 1280, height: 960 } });
+
+    // Capture console errors (JS exceptions, custom element failures, 404s)
+    page.on('console', (msg) => {
+      const type = msg.type();
+      const text = msg.text();
+      if (type === 'error' || text.toLowerCase().includes('error') || text.toLowerCase().includes('404')) {
+        consoleErrors.push(`[${type}] ${text}`);
+      }
+    });
+
+    page.on('pageerror', (err) => {
+      pageErrors.push(`[pageerror] ${err.message}`);
+    });
+
+    page.on('response', (resp) => {
+      if (resp.status() >= 400) {
+        consoleErrors.push(`[${resp.status()}] ${resp.url()}`);
+      }
+    });
+
+    // Navigate to the probe page
+    const probeUrl = `${server.url}/eval-probe.html`;
+    await page.evaluate((bodyHtml) => {
+      document.open();
+      document.write(bodyHtml);
+      document.close();
+    }, pageHtml);
+
+    // Wait a tick for custom-element registration + hydration
+    await page.waitForTimeout(800);
+
+    // Check for blank viewport
+    const bodyBox = await page.locator('body').boundingBox();
+    const blank = !bodyBox || (bodyBox.width < 10 && bodyBox.height < 10);
+
+    // Count visible custom elements (proxy for successful registration)
+    const customElementCount = await page.evaluate(() =>
+      Array.from(document.querySelectorAll('*')).filter(
+        (el) => el.tagName.includes('-') && el.getBoundingClientRect().width > 0
+      ).length
+    );
+
+    // Count :not(:defined) elements (unregistered custom elements)
+    const undefinedCount = await page.evaluate(() =>
+      document.querySelectorAll(':not(:defined)').length
+    );
+
+    // Screenshot capture (optional)
+    let screenshotPath;
+    if (opts.captureScreenshot) {
+      screenshotPath = `/tmp/render-fidelity-${Date.now()}.png`;
+      await page.screenshot({ path: screenshotPath, fullPage: true });
+    }
+
+    // ── Scoring ──
+    // Base: 100, then deduct
+    let score = 100;
+    const allErrors = [...consoleErrors, ...pageErrors];
+
+    // Deduct for errors (capped at -40)
+    const errorPenalty = Math.min(allErrors.length * 10, 40);
+    score -= errorPenalty;
+
+    // Deduct for blank page (-40)
+    if (blank) score -= 40;
+
+    // Deduct for unregistered custom elements (-5 each, cap -20)
+    const undefinedPenalty = Math.min(undefinedCount * 5, 20);
+    score -= undefinedPenalty;
+
+    // Deduct for zero visible custom elements when some were expected (-15)
+    // (Heuristic: if HTML contains custom-element-like tags but none render)
+    const hasCustomTags = /<[a-z]+-[a-z-]+[\s>]/i.test(html);
+    if (hasCustomTags && customElementCount === 0 && !blank) {
+      score -= 15;
+    }
+
+    score = Math.max(0, Math.min(100, score));
+
+    return {
+      score,
+      errors: allErrors,
+      blank,
+      customElementCount,
+      undefinedCount,
+      screenshotPath,
+    };
+
+  } catch (err) {
+    return {
+      score: 0,
+      errors: [`Render probe crashed: ${err.message}`, ...consoleErrors, ...pageErrors],
+      blank: true,
+    };
+  } finally {
+    if (page) await page.close().catch(() => {});
+    if (browser) await browser.close().catch(() => {});
+    if (server) await server.stop().catch(() => {});
+  }
+}
+
+// ── CLI smoke ──
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const testHtml = process.argv[2] || '<card-ui><h2>Hello</h2></card-ui>';
+  const result = await scoreRenderFidelity(testHtml, { captureScreenshot: true });
+  console.log(JSON.stringify(result, null, 2));
+}

package/scripts/smoke-register-engine.mjs

@@ -8,7 +8,7 @@ const t = (label, ok, detail = '') => {
 };
 
 // Baseline
-t('four built-ins registered', listEngines().length === 4);
+t('five built-ins registered', listEngines().length === 5);
 
 // Happy path
 let customCalled = null;

package/scripts/test-a2ui.mjs

@@ -255,7 +255,7 @@ if (patterns.length >= 100) {
 try {
   const { getChunkIndex } = await import('../../corpus/scripts/chunk-library.js');
   const idx = getChunkIndex();
-  if (idx && idx.unique_names >= 500 && idx.by_kind.block && idx.by_kind.page) {
+  if (idx && idx.unique_names >= 100 && idx.by_kind.block && idx.by_kind.page) {
     ok('Chunk corpus', `${idx.unique_names} chunks (${idx.total_instances} instances; block=${idx.by_kind.block}, panel=${idx.by_kind.panel || 0}, page=${idx.by_kind.page})`);
   } else {
     bad('Chunk corpus', `unexpected index: ${JSON.stringify(idx)}`);

package/scripts/zettel-baseline.mjs

@@ -0,0 +1,123 @@
+#!/usr/bin/env node
+/**
+ * zettel-baseline.mjs — Run the classic zettel engine through the same
+ * 20-intent hold-out set used for chunk-zettel eval.
+ *
+ * Zettel returns A2UI messages (updateComponents), not HTML, so scoring
+ * is adapted to the message format. Render fidelity is skipped (different
+ * output shape) and its weight redistributed.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../..');
+const HOLDOUT = path.join(REPO_ROOT, 'packages/a2ui/corpus/evals/holdout-compose-from-chunks.jsonl');
+
+const args = process.argv.slice(2);
+const FLAG_REAL_LLM = args.includes('--real-llm');
+
+function loadHoldOut() {
+  const raw = fs.readFileSync(HOLDOUT, 'utf8');
+  return raw.split('\n').map((l) => l.trim()).filter(Boolean).map(JSON.parse);
+}
+
+async function buildLLMAdapter() {
+  if (!FLAG_REAL_LLM) return null;
+  const { chat } = await import('../../../llm/adapters/index.js');
+  const model = 'claude-sonnet-4-20250514';
+  return {
+    async complete({ messages, systemPrompt }) {
+      const result = await chat({ apiKey: process.env.ANTHROPIC_API_KEY, model, messages, system: systemPrompt });
+      return { text: result.text };
+    },
+  };
+}
+
+// ── Scoring ──
+
+function scoreStructural(result) {
+  return result.messages && result.messages.length > 0 ? 100 : 0;
+}
+
+function scoreCoverage(result, expected) {
+  if (!expected || expected.length === 0) return 100;
+  const comps = result.messages?.[0]?.components || [];
+  const compTypes = new Set(comps.map((c) => c.component?.toLowerCase()));
+  let hits = 0;
+  for (const tag of expected) {
+    const kebab = tag
+      .replace(/^ui/, '')
+      .replace(/([a-z])([A-Z])/g, '$1-$2')
+      .toLowerCase();
+    if (compTypes.has(kebab)) hits++;
+  }
+  return Math.round((hits / expected.length) * 100);
+}
+
+function scoreRetrieval(result) {
+  return result.strategy === 'composition-match' ? 100 : 0;
+}
+
+function compositeScore({ structural, coverage, retrieval }) {
+  const components = [
+    { name: 'structural', value: structural, weight: 30 },
+    { name: 'coverage',   value: coverage,   weight: 20 },
+    { name: 'retrieval',  value: retrieval,  weight: 20 },
+  ].filter((c) => c.value !== null);
+
+  const totalWeight = components.reduce((s, c) => s + c.weight, 0);
+  const weighted = components.reduce((s, c) => s + c.value * c.weight, 0);
+  return totalWeight > 0 ? Math.round(weighted / totalWeight) : 0;
+}
+
+// ── Main ──
+
+async function main() {
+  const { generateZettel } = await import('../../compose/strategies/zettel/generator-adapter.js');
+  const intents = loadHoldOut();
+  const llmAdapter = await buildLLMAdapter();
+
+  const results = [];
+  for (const intent of intents) {
+    process.stderr.write(`▶ ${intent.id} ${intent.kind.padEnd(7)} ${intent.intent.slice(0, 50)}...\n`);
+    const t0 = performance.now();
+    const result = await generateZettel({ intent: intent.intent, mode: 'instant', llmAdapter });
+    const elapsedMs = Math.round(performance.now() - t0);
+
+    const structural = scoreStructural(result);
+    const coverage = scoreCoverage(result, intent.expected_components);
+    const retrieval = scoreRetrieval(result);
+    const score = compositeScore({ structural, coverage, retrieval });
+
+    results.push({
+      id: intent.id,
+      kind: intent.kind,
+      strategy: result.strategy,
+      structural,
+      coverage,
+      retrieval,
+      score,
+      elapsedMs,
+    });
+  }
+
+  const avg = Math.round(results.reduce((s, r) => s + r.score, 0) / results.length);
+  const passing = results.filter((r) => r.score >= 80).length;
+
+  console.log(`# zettel baseline — ${FLAG_REAL_LLM ? 'real LLM' : 'stub (retrieval only)'}`);
+  console.log('');
+  console.log(`Aggregate: avg **${avg}**, passing **${passing} / ${results.length}** (threshold 80).`);
+  console.log('');
+  console.log('| ID | Kind | Strategy | Struct | Cov | Retr | Score | ms |');
+  console.log('|---|---|---|---:|---:|---:|---:|---:|');
+  for (const r of results) {
+    console.log(`| ${r.id} | ${r.kind} | ${r.strategy} | ${r.structural} | ${r.coverage} | ${r.retrieval} | **${r.score}** | ${r.elapsedMs} |`);
+  }
+}
+
+main().catch((err) => {
+  console.error('zettel-baseline failed:', err.message);
+  process.exit(2);
+});