@adia-ai/a2ui-mcp 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -0
- package/README.md +154 -0
- package/package.json +35 -0
- package/scripts/dogfood-test.mjs +107 -0
- package/scripts/eval-diff.mjs +282 -0
- package/scripts/eval-fix.mjs +446 -0
- package/scripts/generate.mjs +189 -0
- package/scripts/multi-turn-test.mjs +247 -0
- package/scripts/smoke-engine-registry.mjs +43 -0
- package/scripts/smoke-merged.mjs +50 -0
- package/scripts/smoke-register-engine.mjs +51 -0
- package/scripts/smoke-searchable-select.mjs +39 -0
- package/scripts/smoke-synthesis.mjs +59 -0
- package/scripts/smoke-zettel.mjs +37 -0
- package/scripts/test-a2ui.mjs +269 -0
- package/scripts/test-evals.mjs +238 -0
- package/scripts/visual-validate.mjs +158 -0
- package/server.js +573 -0
|
@@ -0,0 +1,446 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* eval-fix.mjs — Recursive improvement loop for A2UI generation quality.
|
|
5
|
+
*
|
|
6
|
+
* Runs evals, diagnoses failures, traces root causes upstream, and applies fixes.
|
|
7
|
+
*
|
|
8
|
+
* The loop:
|
|
9
|
+
* 1. Run eval suite → collect scores + failures
|
|
10
|
+
* 2. For each failure, diagnose: pattern issue? template issue? training gap? validator bug?
|
|
11
|
+
* 3. Generate a fix plan (which file, what change)
|
|
12
|
+
* 4. Apply fixes (patch pattern JSON, update training manifest, etc.)
|
|
13
|
+
* 5. Re-run evals to verify the fix worked
|
|
14
|
+
* 6. Repeat until all pass or max iterations reached
|
|
15
|
+
*
|
|
16
|
+
* Diagnosis categories (in priority order):
|
|
17
|
+
* ANTI_PATTERN → pattern template has bare Text (no variant), unslotted header children,
|
|
18
|
+
* or buttons without text. Fix: patch the pattern JSON.
|
|
19
|
+
* INTENT_MISS → generated output missing required components. Fix: check if the
|
|
20
|
+
* matched pattern has those components; if not, update the pattern or
|
|
21
|
+
* add the component to the template.
|
|
22
|
+
* CARD_MODEL → Card children don't follow Header/Section/Footer structure.
|
|
23
|
+
* Fix: restructure the pattern template.
|
|
24
|
+
* STRUCTURAL → validation score below threshold. Fix: usually a template syntax
|
|
25
|
+
* issue (missing root, dangling children, duplicate IDs).
|
|
26
|
+
* COVERAGE → uses forbidden component types. Fix: swap components in template.
|
|
27
|
+
*
|
|
28
|
+
* Usage:
|
|
29
|
+
* node packages/a2ui/mcp/scripts/eval-fix.mjs # diagnose + report (dry run)
|
|
30
|
+
* node packages/a2ui/mcp/scripts/eval-fix.mjs --apply # diagnose + apply fixes + re-verify
|
|
31
|
+
* node packages/a2ui/mcp/scripts/eval-fix.mjs --max-iter=3 # limit fix iterations (default: 3)
|
|
32
|
+
* node packages/a2ui/mcp/scripts/eval-fix.mjs --verbose # show detailed diagnosis
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import '../../../../scripts/load-env.mjs';
|
|
36
|
+
import { readFile, writeFile } from 'node:fs/promises';
|
|
37
|
+
import { dirname, join } from 'node:path';
|
|
38
|
+
import { fileURLToPath } from 'node:url';
|
|
39
|
+
|
|
40
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
41
|
+
const REPO_ROOT = join(__dirname, '..', '..', '..', '..');
|
|
42
|
+
const EVALS_PATH = join(REPO_ROOT, '.claude', 'skills', 'adia-ui-kit', 'evals', 'evals.json');
|
|
43
|
+
const PATTERNS_DIR = join(REPO_ROOT, 'packages', 'web-components', 'catalog', 'patterns');
|
|
44
|
+
|
|
45
|
+
const args = new Set(process.argv.slice(2));
|
|
46
|
+
const APPLY = args.has('--apply');
|
|
47
|
+
const VERBOSE = args.has('--verbose');
|
|
48
|
+
const TICKETS = args.has('--tickets');
|
|
49
|
+
const MAX_ITER = parseInt([...args].find(a => a.startsWith('--max-iter='))?.split('=')[1] || '3');
|
|
50
|
+
|
|
51
|
+
// ── Load modules ──
|
|
52
|
+
|
|
53
|
+
const { generateUI } = await import('../../compose/engine/generator.js');
|
|
54
|
+
const { validateSchema } = await import('../../validator/validator.js');
|
|
55
|
+
const { getPattern, searchPatterns } = await import('../../retrieval/pattern-library.js');
|
|
56
|
+
const { createTicket, formatTicket, formatTicketList } = await import('../../../../.tickets/tickets.js');
|
|
57
|
+
|
|
58
|
+
// ── Scoring (same as test-evals.mjs) ──
|
|
59
|
+
|
|
60
|
+
function scoreAntiPatterns(components) {
|
|
61
|
+
const violations = [];
|
|
62
|
+
for (const c of components) {
|
|
63
|
+
if (c.component === 'Text' && !c.variant) {
|
|
64
|
+
violations.push({ id: c.id, issue: 'text-no-variant', message: `Text "${c.id}" has no variant attribute` });
|
|
65
|
+
}
|
|
66
|
+
const parent = components.find(p => p.children?.includes(c.id));
|
|
67
|
+
if (parent?.component === 'Header' && !c.slot && c.component === 'Text') {
|
|
68
|
+
violations.push({ id: c.id, issue: 'header-child-no-slot', message: `Text "${c.id}" in Header without slot attribute` });
|
|
69
|
+
}
|
|
70
|
+
if (c.component === 'Button' && !c.text && !c.icon) {
|
|
71
|
+
violations.push({ id: c.id, issue: 'button-no-label', message: `Button "${c.id}" has no text or icon` });
|
|
72
|
+
}
|
|
73
|
+
// Text used for clickable actions (should be Button)
|
|
74
|
+
if (c.component === 'Text' && c.textContent) {
|
|
75
|
+
const actionWords = /\b(view|contact|learn more|retry|download|sign|log ?in|log ?out|submit|cancel|delete|remove|edit|save|close|open|click|tap|go to|visit)\b/i;
|
|
76
|
+
if (actionWords.test(c.textContent) && !c.slot) {
|
|
77
|
+
violations.push({ id: c.id, issue: 'text-as-action', message: `Text "${c.id}" looks like an action ("${c.textContent.slice(0, 30)}") — should be a Button` });
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return violations;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function scoreIntent(components, evalCase) {
|
|
85
|
+
const required = evalCase.required_components || [];
|
|
86
|
+
const present = new Set(components.map(c => c.component));
|
|
87
|
+
const missing = required.filter(r => !present.has(r));
|
|
88
|
+
return { required, present: [...present], missing };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function scoreCardModel(components) {
|
|
92
|
+
const issues = [];
|
|
93
|
+
const cards = components.filter(c => c.component === 'Card');
|
|
94
|
+
for (const card of cards) {
|
|
95
|
+
const childIds = card.children || [];
|
|
96
|
+
const children = childIds.map(id => components.find(c => c.id === id)).filter(Boolean);
|
|
97
|
+
const types = children.map(c => c.component);
|
|
98
|
+
if (!types.includes('Header') && !types.includes('Section')) {
|
|
99
|
+
issues.push({ cardId: card.id, issue: 'no-header-or-section', children: types });
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return issues;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// ── Diagnosis ──
|
|
106
|
+
|
|
107
|
+
function diagnose(evalCase, result) {
|
|
108
|
+
const components = result.messages?.[0]?.components || [];
|
|
109
|
+
// Find matched pattern by running the same search the generator uses
|
|
110
|
+
const searchResults = searchPatterns(evalCase.prompt);
|
|
111
|
+
const matched = searchResults[0]?.name || null;
|
|
112
|
+
const findings = [];
|
|
113
|
+
|
|
114
|
+
// 1. Anti-pattern violations
|
|
115
|
+
const antiViolations = scoreAntiPatterns(components);
|
|
116
|
+
if (antiViolations.length > 0) {
|
|
117
|
+
findings.push({
|
|
118
|
+
category: 'ANTI_PATTERN',
|
|
119
|
+
severity: 'high',
|
|
120
|
+
matched_pattern: matched,
|
|
121
|
+
violations: antiViolations,
|
|
122
|
+
fix: matched
|
|
123
|
+
? `Patch pattern "${matched}": add variant to Text nodes, add slot to Header children`
|
|
124
|
+
: 'No pattern matched — anti-patterns come from fallback generation',
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// 2. Intent alignment
|
|
129
|
+
const intent = scoreIntent(components, evalCase);
|
|
130
|
+
if (intent.missing.length > 0) {
|
|
131
|
+
findings.push({
|
|
132
|
+
category: 'INTENT_MISS',
|
|
133
|
+
severity: 'medium',
|
|
134
|
+
matched_pattern: matched,
|
|
135
|
+
missing_components: intent.missing,
|
|
136
|
+
present_components: intent.present,
|
|
137
|
+
fix: matched
|
|
138
|
+
? `Pattern "${matched}" is missing: ${intent.missing.join(', ')}. Add these components to the pattern template.`
|
|
139
|
+
: `No pattern matched. Consider creating a pattern for intent: "${evalCase.prompt.slice(0, 60)}"`,
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// 3. Card model
|
|
144
|
+
const cardIssues = scoreCardModel(components);
|
|
145
|
+
if (cardIssues.length > 0) {
|
|
146
|
+
findings.push({
|
|
147
|
+
category: 'CARD_MODEL',
|
|
148
|
+
severity: 'medium',
|
|
149
|
+
matched_pattern: matched,
|
|
150
|
+
issues: cardIssues,
|
|
151
|
+
fix: matched
|
|
152
|
+
? `Pattern "${matched}" has Card nodes without Header/Section children`
|
|
153
|
+
: 'Card model violation in generated output',
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// 4. Structural
|
|
158
|
+
const validation = validateSchema(result.messages);
|
|
159
|
+
const failedChecks = (validation.checks || []).filter(c => !c.passed);
|
|
160
|
+
if (failedChecks.length > 0) {
|
|
161
|
+
findings.push({
|
|
162
|
+
category: 'STRUCTURAL',
|
|
163
|
+
severity: failedChecks.some(c => c.hardFail) ? 'critical' : 'low',
|
|
164
|
+
matched_pattern: matched,
|
|
165
|
+
failed_checks: failedChecks.map(c => ({ name: c.name, message: c.message })),
|
|
166
|
+
fix: matched
|
|
167
|
+
? `Pattern "${matched}" fails validation: ${failedChecks.map(c => c.name).join(', ')}`
|
|
168
|
+
: 'Structural issues in generated output',
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return { evalId: evalCase.id, prompt: evalCase.prompt, matched_pattern: matched, findings };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// ── Fix application ──
|
|
176
|
+
|
|
177
|
+
async function applyFix(diagnosis) {
|
|
178
|
+
const fixes = [];
|
|
179
|
+
|
|
180
|
+
for (const finding of diagnosis.findings) {
|
|
181
|
+
const patternName = finding.matched_pattern;
|
|
182
|
+
if (!patternName) continue; // Can't fix if no pattern matched
|
|
183
|
+
|
|
184
|
+
// Find the pattern JSON file
|
|
185
|
+
const pattern = getPattern(patternName);
|
|
186
|
+
if (!pattern) continue;
|
|
187
|
+
|
|
188
|
+
const domain = pattern.domain || 'layout';
|
|
189
|
+
const filePath = join(PATTERNS_DIR, domain, `${patternName}.json`);
|
|
190
|
+
|
|
191
|
+
let patternJson;
|
|
192
|
+
try {
|
|
193
|
+
patternJson = JSON.parse(await readFile(filePath, 'utf8'));
|
|
194
|
+
} catch {
|
|
195
|
+
console.log(` ⚠ Cannot read ${filePath} — skipping fix`);
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
let modified = false;
|
|
200
|
+
|
|
201
|
+
if (finding.category === 'ANTI_PATTERN') {
|
|
202
|
+
for (const v of finding.violations) {
|
|
203
|
+
const node = patternJson.template.find(c => c.id === v.id);
|
|
204
|
+
if (!node) continue;
|
|
205
|
+
|
|
206
|
+
if (v.issue === 'text-no-variant' && node.component === 'Text') {
|
|
207
|
+
node.variant = node.variant || 'body';
|
|
208
|
+
modified = true;
|
|
209
|
+
fixes.push(` + Added variant="body" to ${node.id} in ${patternName}`);
|
|
210
|
+
}
|
|
211
|
+
if (v.issue === 'header-child-no-slot' && node.component === 'Text') {
|
|
212
|
+
// First text child = heading, second = description
|
|
213
|
+
const headerNode = patternJson.template.find(c => c.children?.includes(node.id) && c.component === 'Header');
|
|
214
|
+
if (headerNode) {
|
|
215
|
+
const childTexts = headerNode.children
|
|
216
|
+
.map(id => patternJson.template.find(c => c.id === id))
|
|
217
|
+
.filter(c => c?.component === 'Text');
|
|
218
|
+
const idx = childTexts.indexOf(node);
|
|
219
|
+
node.slot = idx === 0 ? 'heading' : 'description';
|
|
220
|
+
modified = true;
|
|
221
|
+
fixes.push(` + Added slot="${node.slot}" to ${node.id} in ${patternName}`);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
if (v.issue === 'button-no-label' && node.component === 'Button') {
|
|
225
|
+
node.text = node.text || 'Action';
|
|
226
|
+
modified = true;
|
|
227
|
+
fixes.push(` + Added text="Action" to ${node.id} in ${patternName}`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
if (finding.category === 'INTENT_MISS' && finding.missing_components.length > 0) {
|
|
233
|
+
const template = patternJson.template;
|
|
234
|
+
const root = template.find(c => c.id === 'root');
|
|
235
|
+
|
|
236
|
+
for (const comp of finding.missing_components) {
|
|
237
|
+
// Auto-fix: add Footer with Button if pattern is missing both
|
|
238
|
+
if (comp === 'Footer' && finding.missing_components.includes('Button')) {
|
|
239
|
+
if (root && !template.some(c => c.component === 'Footer')) {
|
|
240
|
+
const ftrId = 'ftr';
|
|
241
|
+
const btnId = 'action-btn';
|
|
242
|
+
template.push({ id: ftrId, component: 'Footer', children: [btnId] });
|
|
243
|
+
template.push({ id: btnId, component: 'Button', text: 'Save', variant: 'primary', slot: 'action' });
|
|
244
|
+
if (root.children && !root.children.includes(ftrId)) {
|
|
245
|
+
root.children.push(ftrId);
|
|
246
|
+
}
|
|
247
|
+
// Also add to components list
|
|
248
|
+
if (!patternJson.components.includes('Footer')) patternJson.components.push('Footer');
|
|
249
|
+
if (!patternJson.components.includes('Button')) patternJson.components.push('Button');
|
|
250
|
+
modified = true;
|
|
251
|
+
fixes.push(` + Added Footer with Button to ${patternName}`);
|
|
252
|
+
}
|
|
253
|
+
continue; // Skip individual Button fix since Footer handled it
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// Auto-fix: add Button to Footer if Footer exists but Button doesn't
|
|
257
|
+
if (comp === 'Button' && !finding.missing_components.includes('Footer')) {
|
|
258
|
+
const footer = template.find(c => c.component === 'Footer');
|
|
259
|
+
if (footer) {
|
|
260
|
+
const btnId = 'save-btn';
|
|
261
|
+
template.push({ id: btnId, component: 'Button', text: 'Save', variant: 'primary', slot: 'action' });
|
|
262
|
+
if (footer.children) footer.children.push(btnId);
|
|
263
|
+
else footer.children = [btnId];
|
|
264
|
+
if (!patternJson.components.includes('Button')) patternJson.components.push('Button');
|
|
265
|
+
modified = true;
|
|
266
|
+
fixes.push(` + Added Button to existing Footer in ${patternName}`);
|
|
267
|
+
} else {
|
|
268
|
+
// No footer — add one
|
|
269
|
+
const ftrId = 'ftr';
|
|
270
|
+
const btnId = 'save-btn';
|
|
271
|
+
template.push({ id: ftrId, component: 'Footer', children: [btnId] });
|
|
272
|
+
template.push({ id: btnId, component: 'Button', text: 'Save', variant: 'primary', slot: 'action' });
|
|
273
|
+
if (root?.children) root.children.push(ftrId);
|
|
274
|
+
if (!patternJson.components.includes('Footer')) patternJson.components.push('Footer');
|
|
275
|
+
if (!patternJson.components.includes('Button')) patternJson.components.push('Button');
|
|
276
|
+
modified = true;
|
|
277
|
+
fixes.push(` + Added Footer + Button to ${patternName}`);
|
|
278
|
+
}
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// Auto-fix: wrap root in Column if Column is missing
|
|
283
|
+
if (comp === 'Column' && root && root.component !== 'Column') {
|
|
284
|
+
// If root is Card, check if Section children could be wrapped
|
|
285
|
+
const section = template.find(c => c.component === 'Section' && root.children?.includes(c.id));
|
|
286
|
+
if (section && section.children?.length > 0) {
|
|
287
|
+
// Wrap section children in a Column
|
|
288
|
+
const colId = 'col-wrap';
|
|
289
|
+
const originalChildren = [...section.children];
|
|
290
|
+
template.push({ id: colId, component: 'Column', children: originalChildren, gap: '4' });
|
|
291
|
+
section.children = [colId];
|
|
292
|
+
if (!patternJson.components.includes('Column')) patternJson.components.push('Column');
|
|
293
|
+
modified = true;
|
|
294
|
+
fixes.push(` + Wrapped Section children in Column for ${patternName}`);
|
|
295
|
+
} else {
|
|
296
|
+
fixes.push(` ⚠ Pattern "${patternName}" needs ${comp} — manual review required`);
|
|
297
|
+
}
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
fixes.push(` ⚠ Pattern "${patternName}" needs ${comp} — manual review required`);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
if (modified) {
|
|
306
|
+
patternJson.version = (patternJson.version || 1) + 1;
|
|
307
|
+
await writeFile(filePath, JSON.stringify(patternJson, null, 2) + '\n');
|
|
308
|
+
fixes.push(` ✓ Wrote ${filePath}`);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
return fixes;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// ── Main loop ──
|
|
316
|
+
|
|
317
|
+
async function runEvalLoop() {
|
|
318
|
+
const evalsData = JSON.parse(await readFile(EVALS_PATH, 'utf8'));
|
|
319
|
+
const evalCases = evalsData.evals;
|
|
320
|
+
|
|
321
|
+
for (let iter = 0; iter < MAX_ITER; iter++) {
|
|
322
|
+
console.log(`\n${'═'.repeat(60)}`);
|
|
323
|
+
console.log(` Iteration ${iter + 1}/${MAX_ITER}`);
|
|
324
|
+
console.log('═'.repeat(60));
|
|
325
|
+
|
|
326
|
+
// Run all evals
|
|
327
|
+
const diagnoses = [];
|
|
328
|
+
let allClean = true;
|
|
329
|
+
|
|
330
|
+
for (const evalCase of evalCases) {
|
|
331
|
+
const result = await generateUI({ intent: evalCase.prompt, mode: evalCase.mode || 'instant' });
|
|
332
|
+
const diag = diagnose(evalCase, result);
|
|
333
|
+
|
|
334
|
+
if (diag.findings.length > 0) {
|
|
335
|
+
allClean = false;
|
|
336
|
+
diagnoses.push(diag);
|
|
337
|
+
|
|
338
|
+
const validation = validateSchema(result.messages);
|
|
339
|
+
console.log(`\n ✗ #${evalCase.id} [${validation.score}] ${evalCase.prompt.slice(0, 55)}...`);
|
|
340
|
+
if (diag.matched_pattern) console.log(` Matched: ${diag.matched_pattern}`);
|
|
341
|
+
for (const f of diag.findings) {
|
|
342
|
+
console.log(` ${f.category} (${f.severity}): ${f.fix}`);
|
|
343
|
+
if (VERBOSE && f.violations) {
|
|
344
|
+
for (const v of f.violations) console.log(` - ${v.message}`);
|
|
345
|
+
}
|
|
346
|
+
if (VERBOSE && f.missing_components) {
|
|
347
|
+
console.log(` Missing: ${f.missing_components.join(', ')}`);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
} else {
|
|
351
|
+
const validation = validateSchema(result.messages);
|
|
352
|
+
console.log(` ✓ #${evalCase.id} [${validation.score}] ${evalCase.prompt.slice(0, 55)}...`);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
if (allClean) {
|
|
357
|
+
console.log('\n ✓ All evals clean — no issues found.');
|
|
358
|
+
break;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
if (!APPLY && !TICKETS) {
|
|
362
|
+
console.log(`\n ${diagnoses.length} eval(s) with issues. Run with --apply to fix or --tickets to create tickets.`);
|
|
363
|
+
break;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// Create tickets for all findings (when --tickets flag is set, or for unfixable issues during --apply)
|
|
367
|
+
if (TICKETS) {
|
|
368
|
+
console.log('\n── Creating tickets ──');
|
|
369
|
+
for (const diag of diagnoses) {
|
|
370
|
+
for (const f of diag.findings) {
|
|
371
|
+
const ticket = await createTicket({
|
|
372
|
+
type: f.category === 'INTENT_MISS' ? 'improvement' : 'bug',
|
|
373
|
+
title: `Eval #${diag.evalId}: ${f.category} in ${f.matched_pattern || 'generated output'}`,
|
|
374
|
+
source: 'eval-fix',
|
|
375
|
+
severity: f.severity,
|
|
376
|
+
category: f.category === 'ANTI_PATTERN' || f.category === 'INTENT_MISS' ? 'pattern'
|
|
377
|
+
: f.category === 'CARD_MODEL' ? 'pattern'
|
|
378
|
+
: f.category === 'STRUCTURAL' ? 'validator' : 'generator',
|
|
379
|
+
target: f.matched_pattern ? `patterns/${getPattern(f.matched_pattern)?.domain || 'layout'}/${f.matched_pattern}.json` : null,
|
|
380
|
+
description: f.fix,
|
|
381
|
+
evidence: {
|
|
382
|
+
evalId: diag.evalId,
|
|
383
|
+
prompt: diag.prompt.slice(0, 100),
|
|
384
|
+
matched_pattern: f.matched_pattern,
|
|
385
|
+
...(f.violations ? { violations: f.violations.map(v => v.message) } : {}),
|
|
386
|
+
...(f.missing_components ? { missing_components: f.missing_components } : {}),
|
|
387
|
+
...(f.issues ? { card_issues: f.issues } : {}),
|
|
388
|
+
...(f.failed_checks ? { failed_checks: f.failed_checks.map(c => c.name) } : {}),
|
|
389
|
+
},
|
|
390
|
+
suggested_fix: f.fix,
|
|
391
|
+
});
|
|
392
|
+
console.log(` 📋 ${ticket.id}`);
|
|
393
|
+
console.log(` ${ticket.title}`);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
if (!APPLY) break;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
if (!APPLY) break;
|
|
400
|
+
|
|
401
|
+
// Apply fixes
|
|
402
|
+
console.log('\n── Applying fixes ──');
|
|
403
|
+
let totalFixes = 0;
|
|
404
|
+
for (const diag of diagnoses) {
|
|
405
|
+
const fixes = await applyFix(diag);
|
|
406
|
+
for (const f of fixes) {
|
|
407
|
+
console.log(f);
|
|
408
|
+
totalFixes++;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
if (totalFixes === 0) {
|
|
413
|
+
console.log(' No auto-fixable issues found. Remaining issues need manual review.');
|
|
414
|
+
break;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
console.log(`\n Applied ${totalFixes} fixes. Re-running evals...`);
|
|
418
|
+
|
|
419
|
+
// Reload pattern library to pick up changes
|
|
420
|
+
const { loadCorpus } = await import('../../retrieval/pattern-library.js');
|
|
421
|
+
// Force reload by resetting the loaded state
|
|
422
|
+
// (The pattern files were rewritten, but the in-memory Map still has old data)
|
|
423
|
+
// We need to re-import fresh — but ESM caching prevents that.
|
|
424
|
+
// Instead, directly re-read the patched files and update the Map.
|
|
425
|
+
for (const diag of diagnoses) {
|
|
426
|
+
if (!diag.matched_pattern) continue;
|
|
427
|
+
const pattern = getPattern(diag.matched_pattern);
|
|
428
|
+
if (!pattern) continue;
|
|
429
|
+
const domain = pattern.domain || 'layout';
|
|
430
|
+
const filePath = join(PATTERNS_DIR, domain, `${diag.matched_pattern}.json`);
|
|
431
|
+
try {
|
|
432
|
+
const fresh = JSON.parse(await readFile(filePath, 'utf8'));
|
|
433
|
+
// Normalize tags
|
|
434
|
+
if (fresh.tags && !Array.isArray(fresh.tags)) {
|
|
435
|
+
fresh.tagsMeta = fresh.tags;
|
|
436
|
+
fresh.tags = fresh.keywords || [];
|
|
437
|
+
}
|
|
438
|
+
// Update in-memory
|
|
439
|
+
const { registerPattern } = await import('../../retrieval/pattern-library.js');
|
|
440
|
+
registerPattern(fresh, { replace: true });
|
|
441
|
+
} catch {}
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
runEvalLoop().catch(console.error);
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* generate.mjs — Generate A2UI components from the command line.
|
|
5
|
+
*
|
|
6
|
+
* Quick way to test generation without MCP or browser.
|
|
7
|
+
*
|
|
8
|
+
* Usage:
|
|
9
|
+
* node packages/a2ui/mcp/scripts/generate.mjs "login form" # instant mode
|
|
10
|
+
* node packages/a2ui/mcp/scripts/generate.mjs --thinking "settings page" # thinking mode (LLM)
|
|
11
|
+
* node packages/a2ui/mcp/scripts/generate.mjs --stream "dashboard" # streaming mode
|
|
12
|
+
* node packages/a2ui/mcp/scripts/generate.mjs --json "pricing table" # output raw JSON
|
|
13
|
+
* node packages/a2ui/mcp/scripts/generate.mjs --html "user profile" # output as HTML
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import '../../../../scripts/load-env.mjs';
|
|
17
|
+
|
|
18
|
+
const argv = process.argv.slice(2);
|
|
19
|
+
const flags = new Set();
|
|
20
|
+
const positional = [];
|
|
21
|
+
|
|
22
|
+
for (const arg of argv) {
|
|
23
|
+
if (arg.startsWith('--')) flags.add(arg);
|
|
24
|
+
else positional.push(arg);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const intent = positional.join(' ');
|
|
28
|
+
if (!intent) {
|
|
29
|
+
console.error('Usage: node packages/a2ui/mcp/scripts/generate.mjs [--thinking|--stream|--json|--html] "intent"');
|
|
30
|
+
process.exit(1);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const mode = flags.has('--thinking') ? 'thinking' : flags.has('--pro') ? 'pro' : flags.has('--stream') ? 'stream' : 'instant';
|
|
34
|
+
const JSON_OUT = flags.has('--json');
|
|
35
|
+
const HTML_OUT = flags.has('--html');
|
|
36
|
+
|
|
37
|
+
const { generateUI, generateUIStream } = await import('../../compose/engine/generator.js');
|
|
38
|
+
|
|
39
|
+
console.error(`Mode: ${mode} | Intent: "${intent}"`);
|
|
40
|
+
console.error('─'.repeat(50));
|
|
41
|
+
|
|
42
|
+
if (mode === 'stream') {
|
|
43
|
+
// Stream mode — show progressive updates
|
|
44
|
+
let finalResult;
|
|
45
|
+
for await (const event of generateUIStream({ intent })) {
|
|
46
|
+
if (event.type === 'status') {
|
|
47
|
+
console.error(` [${event.stage}] ${event.message}`);
|
|
48
|
+
} else if (event.type === 'clarify') {
|
|
49
|
+
console.error(` [clarify] ${event.questions.join('; ')}`);
|
|
50
|
+
} else if (event.type === 'partial') {
|
|
51
|
+
const count = event.messages?.[0]?.components?.length || 0;
|
|
52
|
+
console.error(` [partial] ${count} components so far...`);
|
|
53
|
+
} else if (event.type === 'complete') {
|
|
54
|
+
finalResult = event;
|
|
55
|
+
} else if (event.type === 'error') {
|
|
56
|
+
console.error(` [error] ${event.error}`);
|
|
57
|
+
process.exit(1);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (finalResult) printResult(finalResult);
|
|
61
|
+
} else {
|
|
62
|
+
const start = Date.now();
|
|
63
|
+
const result = await generateUI({ intent, mode });
|
|
64
|
+
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
|
65
|
+
console.error(` Generated in ${elapsed}s`);
|
|
66
|
+
printResult(result);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function printResult(result) {
|
|
70
|
+
const comps = result.messages?.[0]?.components || [];
|
|
71
|
+
|
|
72
|
+
if (JSON_OUT) {
|
|
73
|
+
console.log(JSON.stringify(result.messages, null, 2));
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (HTML_OUT) {
|
|
78
|
+
console.log(componentsToHTML(comps));
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Default: summary view
|
|
83
|
+
console.error(` Components: ${comps.length}`);
|
|
84
|
+
console.error(` Source: ${result.source || (result.messages?.[0]?.components ? 'pattern' : 'fallback')}`);
|
|
85
|
+
if (result.suggestions?.length) {
|
|
86
|
+
console.error(` Suggestions: ${result.suggestions.join('; ')}`);
|
|
87
|
+
}
|
|
88
|
+
console.error('');
|
|
89
|
+
console.log('Component tree:');
|
|
90
|
+
printTree(comps);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function printTree(comps) {
|
|
94
|
+
const map = new Map(comps.map(c => [c.id, c]));
|
|
95
|
+
const roots = comps.filter(c => !comps.some(p => p.children?.includes(c.id)));
|
|
96
|
+
|
|
97
|
+
function walk(id, depth = 0) {
|
|
98
|
+
const c = map.get(id);
|
|
99
|
+
if (!c) return;
|
|
100
|
+
const indent = ' '.repeat(depth);
|
|
101
|
+
const props = [];
|
|
102
|
+
if (c.variant) props.push(`variant="${c.variant}"`);
|
|
103
|
+
if (c.textContent) props.push(`"${c.textContent.slice(0, 40)}${c.textContent.length > 40 ? '...' : ''}"`);
|
|
104
|
+
if (c.slot) props.push(`slot="${c.slot}"`);
|
|
105
|
+
if (c.text) props.push(`text="${c.text}"`);
|
|
106
|
+
if (c.label) props.push(`label="${c.label}"`);
|
|
107
|
+
if (c.icon) props.push(`icon="${c.icon}"`);
|
|
108
|
+
if (c.name) props.push(`name="${c.name}"`);
|
|
109
|
+
console.log(`${indent}${c.component} (${c.id})${props.length ? ' ' + props.join(' ') : ''}`);
|
|
110
|
+
if (c.children) {
|
|
111
|
+
for (const childId of c.children) walk(childId, depth + 1);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
for (const root of roots) walk(root.id);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function componentsToHTML(comps) {
|
|
119
|
+
const TAG_MAP = {
|
|
120
|
+
// Layout
|
|
121
|
+
Card: 'card-ui', Column: 'col-ui', Row: 'row-ui', Grid: 'grid-ui', Stack: 'stack-ui',
|
|
122
|
+
Header: 'header', Footer: 'footer', Section: 'section',
|
|
123
|
+
Pane: 'pane-ui', Block: 'block-ui',
|
|
124
|
+
// Controls
|
|
125
|
+
Button: 'button-ui', Input: 'input-ui', Select: 'select-ui',
|
|
126
|
+
CheckBox: 'check-ui', Switch: 'switch-ui', Toggle: 'switch-ui',
|
|
127
|
+
Slider: 'slider-ui', Radio: 'radio-ui', Range: 'range-ui',
|
|
128
|
+
TextArea: 'textarea-ui', Upload: 'upload-ui',
|
|
129
|
+
OtpInput: 'otp-input-ui', ColorPicker: 'color-picker-ui',
|
|
130
|
+
CalendarPicker: 'calendar-picker-ui',
|
|
131
|
+
// Display
|
|
132
|
+
Text: null, Icon: 'icon-ui', Badge: 'badge-ui', Avatar: 'avatar-ui',
|
|
133
|
+
AvatarGroup: 'avatar-group-ui', Tag: 'tag-ui', Kbd: 'kbd-ui',
|
|
134
|
+
Image: 'image-ui', Code: 'code-ui', Embed: 'embed-ui',
|
|
135
|
+
// Feedback
|
|
136
|
+
Alert: 'alert-ui', Progress: 'progress-ui', Skeleton: 'skeleton-ui', Toast: 'toast-ui',
|
|
137
|
+
// Data
|
|
138
|
+
Stat: 'stat-ui', Table: 'table-ui', Chart: 'chart-ui',
|
|
139
|
+
List: 'list-ui', Pagination: 'pagination-ui',
|
|
140
|
+
// Navigation
|
|
141
|
+
Tabs: 'tabs-ui', Tab: 'tab-ui', Nav: 'nav-n',
|
|
142
|
+
Breadcrumb: 'breadcrumb-ui', SegmentedControl: 'segmented-ui', Segment: 'segment-ui',
|
|
143
|
+
// Overlay
|
|
144
|
+
Modal: 'modal-ui', Drawer: 'drawer-ui', Popover: 'popover-ui',
|
|
145
|
+
Tooltip: 'tooltip-ui', Menu: 'menu-ui', Command: 'command-ui',
|
|
146
|
+
// Structure
|
|
147
|
+
Toolbar: 'toolbar-ui', Divider: 'divider-ui', EmptyState: 'empty-state-ui',
|
|
148
|
+
Accordion: 'accordion-ui', AccordionItem: 'accordion-item-ui',
|
|
149
|
+
Timeline: 'timeline-ui', TimelineItem: 'timeline-item-ui',
|
|
150
|
+
Tree: 'tree-ui',
|
|
151
|
+
};
|
|
152
|
+
const map = new Map(comps.map(c => [c.id, c]));
|
|
153
|
+
const roots = comps.filter(c => !comps.some(p => p.children?.includes(c.id)));
|
|
154
|
+
|
|
155
|
+
function render(id, depth = 0) {
|
|
156
|
+
const c = map.get(id);
|
|
157
|
+
if (!c) return '';
|
|
158
|
+
const indent = ' '.repeat(depth);
|
|
159
|
+
|
|
160
|
+
// Text → native element based on variant
|
|
161
|
+
if (c.component === 'Text') {
|
|
162
|
+
const isHeading = c.variant?.match(/^h[1-6]$/);
|
|
163
|
+
const tag = isHeading ? c.variant : c.variant === 'body' ? 'p' : c.variant === 'caption' ? 'small' : 'span';
|
|
164
|
+
const attrs = [];
|
|
165
|
+
// variant is always required on headings; on span/small it provides visual style
|
|
166
|
+
if (c.variant && c.variant !== 'body') attrs.push(`variant="${c.variant}"`);
|
|
167
|
+
if (c.slot) attrs.push(`slot="${c.slot}"`);
|
|
168
|
+
if (c.color) attrs.push(`color="${c.color}"`);
|
|
169
|
+
const attrStr = attrs.length ? ' ' + attrs.join(' ') : '';
|
|
170
|
+
return `${indent}<${tag}${attrStr} nomargin>${c.textContent || ''}</${tag}>`;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const tag = TAG_MAP[c.component] || c.component.toLowerCase() + '-n';
|
|
174
|
+
const skip = new Set(['id', 'component', 'children', 'textContent']);
|
|
175
|
+
const attrs = Object.entries(c)
|
|
176
|
+
.filter(([k]) => !skip.has(k))
|
|
177
|
+
.map(([k, v]) => v === true ? k : `${k}="${v}"`)
|
|
178
|
+
.join(' ');
|
|
179
|
+
const attrStr = attrs ? ' ' + attrs : '';
|
|
180
|
+
|
|
181
|
+
if (!c.children?.length) {
|
|
182
|
+
return `${indent}<${tag}${attrStr}></${tag}>`;
|
|
183
|
+
}
|
|
184
|
+
const children = c.children.map(cid => render(cid, depth + 1)).join('\n');
|
|
185
|
+
return `${indent}<${tag}${attrStr}>\n${children}\n${indent}</${tag}>`;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
return roots.map(r => render(r.id)).join('\n');
|
|
189
|
+
}
|