@adia-ai/a2ui-validator 0.0.1

package/CHANGELOG.md

@@ -0,0 +1,36 @@
+# Changelog — @adia-ai/a2ui-validator
+
+Follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and
+[Semantic Versioning](https://semver.org/).
+
+## [Unreleased]
+
+_Nothing yet._
+
+---
+
+## [0.0.1] - 2026-04-24
+
+First public release. Extracted from
+[`@adia-ai/a2ui-compose`](../compose/) during the 2026-04-24
+consolidation.
+
+### Included
+
+- **Structural validator** (`validator.js`) — JSON Schema validation
+  against the A2UI protocol schema via `ajv` + `ajv-formats`. Returns
+  `{ valid, errors }`.
+- **Catalog validator** (`catalog-validator.js`) — semantic checks:
+  component-exists, props-match-YAML, references-resolve. Consumed by
+  the compose engine's post-generation validator pass and by CI
+  drift gates.
+- **Semantic validator** (`semantic/`) — optional LLM-judged rubric
+  scoring in shadow mode. Disk-cached by content hash; gated behind
+  `--semantic` flag in `eval-diff.mjs`. See
+  [`docs/specs/semantic-validator.md`](https://github.com/adiahealth/gen-ui-kit/blob/main/docs/specs/semantic-validator.md).
+
+### Dependencies
+
+- `@adia-ai/a2ui-utils` — A2UI registry + runtime primitives.
+- `ajv` ^8 — JSON Schema validator.
+- `ajv-formats` ^3 — standard format validators (uri, date, etc.).

package/README.md

@@ -0,0 +1,54 @@
+# `@adia-ai/a2ui-validator`
+
+JSON Schema structural validation + catalog-aware semantic validation
+for A2UI (Agent-to-UI) protocol messages. Extracted from
+[`@adia-ai/a2ui-compose`](../compose/) so non-compose tooling (tests,
+MCP validator tools, CI gates) can depend on validation without
+pulling the full generator graph.
+
+## Install
+
+```bash
+npm install @adia-ai/a2ui-validator
+```
+
+## Usage
+
+```js
+import { validateSchema } from '@adia-ai/a2ui-validator';
+
+const messages = [/* A2UI protocol messages */];
+const result = validateSchema(messages);
+if (!result.valid) {
+  console.error(result.errors);
+}
+```
+
+Catalog-aware validation (component exists + props match YAML):
+
+```js
+import { validateAgainstCatalog } from '@adia-ai/a2ui-validator/catalog';
+
+const result = validateAgainstCatalog(messages, catalog);
+```
+
+## What's here
+
+- **Structural validator** — JSON Schema validation against the A2UI
+  protocol schema (message shape, required fields, enum constraints).
+- **Catalog validator** — semantic checks: does this component exist in
+  the catalog? Do its props match the YAML contract? Are references
+  resolvable?
+- **Semantic validator** (optional, shadow-mode) — LLM-judged output
+  quality against a rubric; cached on disk by content hash.
+
+## Runtime
+
+- `ajv` + `ajv-formats` for structural validation.
+- `@adia-ai/a2ui-utils` for the registry shape.
+
+## Links
+
+- Repo: [`adiahealth/gen-ui-kit`](https://github.com/adiahealth/gen-ui-kit)
+- Spec: [`docs/specs/semantic-validator.md`](https://github.com/adiahealth/gen-ui-kit/blob/main/docs/specs/semantic-validator.md)
+- CHANGELOG: [`CHANGELOG.md`](./CHANGELOG.md)

package/catalog-validator.js

@@ -0,0 +1,162 @@
+/**
+ * Catalog-backed structural validator — checks that every emitted A2UI
+ * component conforms to its catalog schema (allOf: ComponentCommon +
+ * CatalogComponentCommon + component-specific props, with
+ * unevaluatedProperties: false so unknown props are rejected).
+ *
+ * The catalog at a2ui/corpus/catalog-a2ui_0_9.json is a freestanding
+ * JSON Schema draft-2020-12 document produced by scripts/build-components.mjs.
+ * At first use we compile one validator per component type and cache them,
+ * then dispatch per-component by the `component` discriminator.
+ *
+ * Fast path when the catalog is missing (e.g. browser bundle without assets):
+ * return { valid: true, skipped: true } so the scoring validator still runs.
+ */
+
+import Ajv2020 from 'ajv/dist/2020.js';
+import addFormats from 'ajv-formats';
+
+const IS_NODE = typeof process !== 'undefined' && !!process.versions?.node;
+
+let _catalog = null;
+let _ajv = null;
+let _validators = null;           // Map<componentName, compiledValidator>
+let _loadPromise = null;
+
+async function _loadCatalog() {
+  if (_catalog) return _catalog;
+  if (_loadPromise) return _loadPromise;
+  _loadPromise = (async () => {
+    try {
+      if (IS_NODE) {
+        const fs = await import(/* @vite-ignore */ 'node:fs/promises');
+        const path = await import(/* @vite-ignore */ 'node:path');
+        const url = await import(/* @vite-ignore */ 'node:url');
+        const here = path.dirname(url.fileURLToPath(import.meta.url));
+        // a2ui/validator → a2ui/corpus/catalog-a2ui_0_9.json
+        const p = path.resolve(here, '../corpus/catalog-a2ui_0_9.json');
+        const raw = await fs.readFile(p, 'utf8');
+        _catalog = JSON.parse(raw);
+      } else {
+        // Browser: Vite glob-resolves the JSON at build time.
+        const mod = await import('../corpus/catalog-a2ui_0_9.json',
+          { with: { type: 'json' } }).catch(() => null);
+        _catalog = mod?.default ?? null;
+      }
+    } catch {
+      _catalog = null;  // missing catalog → validator no-ops; caller sees skipped: true
+    }
+    return _catalog;
+  })();
+  return _loadPromise;
+}
+
+function _getAjv() {
+  if (_ajv) return _ajv;
+  _ajv = new Ajv2020({
+    strict: false,          // our schemas are authored, not hardened; silence warnings
+    allErrors: true,         // return every violation, not just the first
+    verbose: false,
+    allowUnionTypes: true,
+    validateFormats: false,  // formats cost compilation time and aren't strictly needed
+  });
+  addFormats(_ajv);
+  return _ajv;
+}
+
+async function _getValidatorFor(componentName) {
+  if (_validators && _validators.has(componentName)) return _validators.get(componentName);
+  const catalog = await _loadCatalog();
+  if (!catalog) return null;
+
+  if (!_validators) {
+    _validators = new Map();
+    const ajv = _getAjv();
+    // Register the catalog as a single addressable document; sub-schemas are
+    // retrieved by JSON Pointer via getSchema(). Intra-catalog `$ref`s (#/$defs/X)
+    // resolve naturally because the catalog IS the root.
+    const catalogId = catalog.$id || 'adiaui-catalog';
+    try {
+      ajv.addSchema(catalog, catalogId);
+    } catch (e) {
+      console.warn(`[catalog-validator] failed to register catalog: ${e.message}`);
+      return null;
+    }
+
+    for (const name of Object.keys(catalog.components || {})) {
+      try {
+        // Get a compiled validator for the sub-schema at #/components/<name>.
+        // AJV resolves it against the already-registered catalog.
+        const validate = ajv.getSchema(`${catalogId}#/components/${name}`);
+        if (validate) _validators.set(name, validate);
+      } catch (e) {
+        console.warn(`[catalog-validator] skip ${name}: ${e.message}`);
+      }
+    }
+  }
+  return _validators.get(componentName) || null;
+}
+
+/**
+ * Validate one A2UI component object against its catalog schema.
+ *
+ * @param {object} component — A2UI component ({ id, component, ...props })
+ * @returns {Promise<{ valid: boolean, errors?: string[], skipped?: boolean }>}
+ */
+export async function validateComponent(component) {
+  if (!component || typeof component !== 'object') {
+    return { valid: false, errors: ['not an object'] };
+  }
+  const typeName = component.component;
+  if (!typeName) return { valid: false, errors: ['missing `component` discriminator'] };
+
+  const validate = await _getValidatorFor(typeName);
+  if (!validate) {
+    // Catalog missing OR component type not in catalog → structural-only fallback.
+    return { valid: true, skipped: true };
+  }
+
+  const ok = validate(component);
+  if (ok) return { valid: true };
+
+  const errors = (validate.errors || []).map(e => {
+    const path = e.instancePath || '/';
+    return `${path} ${e.message}${e.params && Object.keys(e.params).length ? ` (${JSON.stringify(e.params)})` : ''}`;
+  });
+  return { valid: false, errors };
+}
+
+/**
+ * Validate an array of A2UI messages ({ type: 'updateComponents', components: [...] }).
+ * Flattens all updateComponents.components[] across messages and runs per-component validation.
+ *
+ * @returns {Promise<{ valid: boolean, totalChecked: number, failures: Array<{ id, component, errors }> }>}
+ */
+export async function validateMessages(messages) {
+  const failures = [];
+  let totalChecked = 0;
+
+  for (const msg of messages || []) {
+    if (msg?.type !== 'updateComponents') continue;
+    for (const comp of msg.components || []) {
+      totalChecked++;
+      const { valid, errors, skipped } = await validateComponent(comp);
+      if (skipped) continue;
+      if (!valid) {
+        failures.push({
+          id: comp.id || '(no id)',
+          component: comp.component || '(no component)',
+          errors: errors || ['unknown validation failure'],
+        });
+      }
+    }
+  }
+
+  return { valid: failures.length === 0, totalChecked, failures };
+}
+
+/** Diagnostics: has the catalog loaded successfully? */
+export async function isCatalogLoaded() {
+  await _loadCatalog();
+  return _catalog !== null;
+}

package/index.js

@@ -0,0 +1,11 @@
+/**
+ * @adia-ai/a2ui-validator — barrel.
+ *
+ * Schema + semantic validation for A2UI protocol messages. Use
+ * `validateSchema` for structural checks (shape, types, required
+ * fields); use `validateMessages` for catalog-aware checks (component
+ * exists, props match the YAML contract).
+ */
+
+export * from './validator.js';
+export * from './catalog-validator.js';

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@adia-ai/a2ui-validator",
+  "version": "0.0.1",
+  "description": "AdiaUI A2UI validator — JSON Schema structural validation plus catalog-aware semantic validation (component exists, props match YAML). Split out from the compose engine so non-compose tooling (tests, MCP validator tools, CI gates) can depend on validation without pulling the whole generator graph.",
+  "type": "module",
+  "main": "./index.js",
+  "exports": {
+    ".":           "./index.js",
+    "./validator": "./validator.js",
+    "./catalog":   "./catalog-validator.js"
+  },
+  "files": [
+    "index.js",
+    "validator.js",
+    "catalog-validator.js",
+    "semantic/",
+    "README.md",
+    "CHANGELOG.md"
+  ],
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/adiahealth/gen-ui-kit.git",
+    "directory": "packages/a2ui/validator"
+  },
+  "dependencies": {
+    "@adia-ai/a2ui-utils": "^0.0.2",
+    "ajv": "^8.0.0",
+    "ajv-formats": "^3.0.0"
+  }
+}

package/semantic/cache.js

@@ -0,0 +1,54 @@
+/**
+ * Content-hash cache for semantic verdicts.
+ *
+ * Key: (rubricVersion, intent, a2ui-hash).
+ * Store: one JSON file per key under evals/mcp/runs/.semantic-cache/.
+ *
+ * Phase 1 scope: read/write; no eviction. Bump rubricVersion to invalidate.
+ */
+import { createHash } from 'node:crypto';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const repoRoot = join(__dirname, '..', '..', '..', '..');
+const DEFAULT_DIR = join(repoRoot, 'evals', 'mcp', 'runs', '.semantic-cache');
+
+export function hashA2UI(messages) {
+  const h = createHash('sha256');
+  h.update(JSON.stringify(messages ?? []));
+  return h.digest('hex').slice(0, 16);
+}
+
+export function cacheKey({ rubricVersion, intent, messages }) {
+  const h = createHash('sha256');
+  h.update(String(rubricVersion || 'v1'));
+  h.update('\x1f');
+  h.update(String(intent || ''));
+  h.update('\x1f');
+  h.update(hashA2UI(messages));
+  return h.digest('hex').slice(0, 32);
+}
+
+export function makeCache({ dir = DEFAULT_DIR } = {}) {
+  return {
+    dir,
+    async get(key) {
+      const p = join(dir, `${key}.json`);
+      if (!existsSync(p)) return null;
+      try {
+        const raw = await readFile(p, 'utf8');
+        return JSON.parse(raw);
+      } catch {
+        return null;
+      }
+    },
+    async set(key, verdict) {
+      await mkdir(dir, { recursive: true });
+      const p = join(dir, `${key}.json`);
+      await writeFile(p, JSON.stringify(verdict, null, 2));
+    },
+  };
+}

package/semantic/index.js

@@ -0,0 +1,163 @@
+/**
+ * Semantic validator — public API (Phase 1: LLM-only, shadow mode).
+ *
+ * See docs/specs/semantic-validator.md §5.2 for the SemanticVerdict shape.
+ * Phase 1: LLM-judge only. Hybrid rule path is Phase 3.
+ *
+ * Usage:
+ *   import { validateSemantics } from 'packages/a2ui/validator/semantic/index.js';
+ *   const v = await validateSemantics({ intent, messages }, { cache: true });
+ */
+import { cacheKey, makeCache } from './cache.js';
+import { callJudge, getRubric, summarizeA2UI } from './judge.js';
+
+function clamp(n, lo = 0, hi = 100) {
+  const x = Number.isFinite(n) ? n : 0;
+  return Math.max(lo, Math.min(hi, Math.round(x)));
+}
+
+function shapeVerdict(parsed, { rubricVersion, cost }) {
+  const axes = parsed?.axes || {};
+  const dp = axes.dominantPattern || {};
+  const rc = axes.requiredCapabilities || {};
+  const fn = axes.forbiddenNoise || {};
+
+  const dpScore = clamp(dp.score);
+  const rcScore = clamp(rc.score);
+  const fnScore = clamp(fn.score);
+
+  // Trust the judge's score if sane; otherwise recompute from axes.
+  let score = parsed?.score;
+  if (!Number.isFinite(score) || score < 0 || score > 100) {
+    score = Math.round(0.5 * dpScore + 0.35 * rcScore + 0.15 * fnScore);
+  } else {
+    score = clamp(score);
+  }
+
+  const verdictIn = String(parsed?.verdict || '').toLowerCase();
+  const verdict = ['aligned', 'partial', 'misaligned', 'off-topic'].includes(verdictIn)
+    ? verdictIn
+    : (() => {
+        if (dpScore < 15) return 'off-topic';
+        if (dpScore < 40 || [dpScore, rcScore, fnScore].filter((s) => s < 60).length >= 2) return 'misaligned';
+        if (score >= 75 && dpScore >= 75 && rcScore >= 75 && fnScore >= 75) return 'aligned';
+        return 'partial';
+      })();
+
+  return {
+    score,
+    verdict,
+    path: 'llm-only',
+    axes: {
+      dominantPattern: {
+        expected: String(dp.expected || ''),
+        observed: String(dp.observed || ''),
+        score: dpScore,
+      },
+      requiredCapabilities: {
+        expected: Array.isArray(rc.expected) ? rc.expected.map(String) : [],
+        missing: Array.isArray(rc.missing) ? rc.missing.map(String) : [],
+        score: rcScore,
+      },
+      forbiddenNoise: {
+        observed: Array.isArray(fn.observed) ? fn.observed.map(String) : [],
+        score: fnScore,
+      },
+    },
+    rationale: String(parsed?.rationale || '').slice(0, 240),
+    evidence: Array.isArray(parsed?.evidence) ? parsed.evidence.map(String).slice(0, 10) : [],
+    cost,
+    rubricVersion,
+  };
+}
+
+/**
+ * validateSemantics — Phase 1 implementation (LLM-only + cache).
+ */
+export async function validateSemantics(input, opts = {}) {
+  const { intent, messages } = input || {};
+  const {
+    cache = true,
+    timeoutMs = 15000,
+    model,
+    rubricVersion = getRubric().version,
+  } = opts;
+
+  if (!intent || !Array.isArray(messages) || messages.length === 0) {
+    return {
+      score: 0,
+      verdict: 'off-topic',
+      path: 'llm-only',
+      axes: {
+        dominantPattern: { expected: '', observed: 'none', score: 0 },
+        requiredCapabilities: { expected: [], missing: [], score: 0 },
+        forbiddenNoise: { observed: [], score: 100 },
+      },
+      rationale: 'No messages emitted; nothing to score.',
+      evidence: [],
+      cost: { cached: false, latencyMs: 0 },
+      rubricVersion,
+      error: 'no-messages',
+    };
+  }
+
+  const key = cacheKey({ rubricVersion, intent, messages });
+  const store = cache ? makeCache() : null;
+
+  if (store) {
+    const hit = await store.get(key);
+    if (hit) {
+      return { ...hit, cost: { ...(hit.cost || {}), cached: true } };
+    }
+  }
+
+  const summary = summarizeA2UI(messages);
+  let judgeRes;
+  try {
+    judgeRes = await callJudge({
+      intent,
+      componentSummary: summary,
+      rubricVersion,
+      timeoutMs,
+      ...(model ? { model } : {}),
+    });
+  } catch (err) {
+    return {
+      score: 0,
+      verdict: 'partial',
+      path: 'llm-only',
+      axes: {
+        dominantPattern: { expected: '', observed: '', score: 0 },
+        requiredCapabilities: { expected: [], missing: [], score: 0 },
+        forbiddenNoise: { observed: [], score: 0 },
+      },
+      rationale: `Judge error: ${err.message || String(err)}`.slice(0, 240),
+      evidence: [],
+      cost: { cached: false, latencyMs: 0 },
+      rubricVersion,
+      error: 'judge-failed',
+    };
+  }
+
+  const cost = {
+    provider: judgeRes.provider,
+    model: judgeRes.model,
+    inputTokens: judgeRes.usage.inputTokens,
+    outputTokens: judgeRes.usage.outputTokens,
+    cached: false,
+    latencyMs: judgeRes.latencyMs,
+  };
+
+  const verdict = shapeVerdict(judgeRes.parsed, { rubricVersion, cost });
+
+  if (store) {
+    try {
+      await store.set(key, verdict);
+    } catch {
+      /* cache write failure is non-fatal */
+    }
+  }
+  return verdict;
+}
+
+export { getRubric, summarizeA2UI };