kui-vue 6.0.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AI.en-US.md +21 -1
- package/AI.md +37 -1
- package/ai/behaviors.json +82 -0
- package/ai/evals/README.md +59 -0
- package/ai/evals/cases.json +91 -24
- package/ai/evals/config.json +4 -0
- package/ai/evals/evaluate.mjs +267 -0
- package/ai/evals/run.mjs +118 -0
- package/ai/kui-components.json +7894 -955
- package/ai/kui-components.schema.json +67 -0
- package/ai/mcp.mjs +210 -166
- package/ai/skills/kui-vue/SKILL.md +2 -1
- package/ai/skills/kui-vue/references/mcp.md +12 -12
- package/ai/templates/form.vue +49 -0
- package/ai/templates/index.json +23 -0
- package/ai/templates/modal-editor.vue +59 -0
- package/ai/templates/table.vue +63 -0
- package/ai/validate.mjs +225 -0
- package/dist/index.js +2 -2
- package/es/index.js +2180 -2179
- package/es/locale/de.js +1 -1
- package/es/locale/el.js +1 -1
- package/es/locale/en.js +1 -1
- package/es/locale/fr.js +1 -1
- package/es/locale/it.js +1 -1
- package/es/locale/ja.js +1 -1
- package/es/locale/ko.js +1 -1
- package/es/locale/ru.js +1 -1
- package/es/locale/th.js +1 -1
- package/es/locale/uk.js +1 -1
- package/es/locale/vi.js +1 -1
- package/es/locale/zh-CN.js +1 -1
- package/es/locale/zh-TW.js +1 -1
- package/lib/index.cjs +2 -2
- package/lib/locale/de.cjs +1 -1
- package/lib/locale/el.cjs +1 -1
- package/lib/locale/en.cjs +1 -1
- package/lib/locale/fr.cjs +1 -1
- package/lib/locale/it.cjs +1 -1
- package/lib/locale/ja.cjs +1 -1
- package/lib/locale/ko.cjs +1 -1
- package/lib/locale/ru.cjs +1 -1
- package/lib/locale/th.cjs +1 -1
- package/lib/locale/uk.cjs +1 -1
- package/lib/locale/vi.cjs +1 -1
- package/lib/locale/zh-CN.cjs +1 -1
- package/lib/locale/zh-TW.cjs +1 -1
- package/package.json +37 -31
- package/style/base.css +1 -1
- package/style/components.css +2 -2
- package/style/index.css +2 -2
- package/style/motion.css +1 -1
- package/style/theme.css +2 -2
- package/types/affix/index.d.ts +5 -5
- package/types/alert/index.d.ts +9 -7
- package/types/anchor/anchor-link.d.ts +4 -4
- package/types/anchor/anchor.d.ts +6 -6
- package/types/anchor/context.d.ts +1 -1
- package/types/anchor/index.d.ts +4 -4
- package/types/auto-complete/index.d.ts +7 -11
- package/types/avatar/avatar-group.d.ts +5 -5
- package/types/avatar/avatar.d.ts +6 -6
- package/types/avatar/context.d.ts +1 -1
- package/types/avatar/index.d.ts +4 -4
- package/types/back-top/index.d.ts +5 -5
- package/types/badge/index.d.ts +6 -6
- package/types/base/transition.d.ts +1 -1
- package/types/breadcrumb/breadcrumb-item.d.ts +6 -6
- package/types/breadcrumb/breadcrumb.d.ts +4 -4
- package/types/breadcrumb/index.d.ts +4 -4
- package/types/button/button-group.d.ts +5 -5
- package/types/button/button.d.ts +4 -4
- package/types/button/index.d.ts +4 -4
- package/types/calendar/index.d.ts +8 -5
- package/types/card/card-meta.d.ts +4 -4
- package/types/card/index.d.ts +8 -8
- package/types/carousel/carousel-item.d.ts +1 -1
- package/types/carousel/carousel.d.ts +20 -7
- package/types/carousel/index.d.ts +3 -3
- package/types/cascader/index.d.ts +41 -32
- package/types/cascader/types.d.ts +7 -3
- package/types/check-card/check-card-group.d.ts +10 -7
- package/types/check-card/check-card.d.ts +21 -8
- package/types/check-card/context.d.ts +3 -3
- package/types/check-card/index.d.ts +5 -5
- package/types/check-card/types.d.ts +2 -2
- package/types/checkbox/checkbox-group.d.ts +20 -7
- package/types/checkbox/checkbox.d.ts +4 -3
- package/types/checkbox/index.d.ts +5 -5
- package/types/collapse/collapse-panel.d.ts +6 -6
- package/types/collapse/collapse.d.ts +7 -7
- package/types/collapse/context.d.ts +1 -1
- package/types/collapse/index.d.ts +4 -4
- package/types/color-picker/alpha.d.ts +10 -6
- package/types/color-picker/hue.d.ts +7 -4
- package/types/color-picker/index.d.ts +34 -20
- package/types/color-picker/mode.d.ts +9 -9
- package/types/color-picker/paint.d.ts +9 -6
- package/types/color-picker/presets.d.ts +7 -7
- package/types/color-picker/types.d.ts +1 -0
- package/types/components.d.ts +166 -161
- package/types/config/context.d.ts +2 -2
- package/types/config/index.d.ts +7 -7
- package/types/config/popup.d.ts +1 -1
- package/types/const/types.d.ts +2 -2
- package/types/date-picker/index.d.ts +49 -19
- package/types/descriptions/descriptions-item.d.ts +5 -5
- package/types/descriptions/descriptions.d.ts +5 -5
- package/types/descriptions/index.d.ts +4 -2
- package/types/directives/outsiteclick.d.ts +1 -1
- package/types/directives/resize.d.ts +1 -1
- package/types/directives/scroll.d.ts +1 -1
- package/types/divider/index.d.ts +5 -5
- package/types/drawer/index.d.ts +24 -11
- package/types/dropdown/dropdown-button.d.ts +7 -7
- package/types/dropdown/dropdown.d.ts +12 -16
- package/types/dropdown/index.d.ts +4 -2
- package/types/dropdown/trigger.d.ts +6 -6
- package/types/empty/index.d.ts +1 -1
- package/types/feature-card/index.d.ts +7 -7
- package/types/feedback-panel/feedback-panel.d.ts +7 -7
- package/types/feedback-panel/index.d.ts +3 -3
- package/types/flame-wrap/index.d.ts +9 -9
- package/types/flex/index.d.ts +7 -7
- package/types/flex/types.d.ts +1 -1
- package/types/form/context.d.ts +3 -3
- package/types/form/form-item.d.ts +6 -6
- package/types/form/form.d.ts +9 -13
- package/types/form/index.d.ts +6 -6
- package/types/global.d.ts +2 -0
- package/types/grid/grid-item.d.ts +2 -2
- package/types/grid/grid.d.ts +3 -3
- package/types/grid/index.d.ts +5 -5
- package/types/grid/useBreakpoint.d.ts +1 -1
- package/types/icon/index.d.ts +8 -8
- package/types/image/context.d.ts +2 -2
- package/types/image/group.d.ts +4 -4
- package/types/image/image.d.ts +7 -7
- package/types/image/index.d.ts +5 -5
- package/types/image/instance.d.ts +11 -4
- package/types/image/preview.d.ts +6 -6
- package/types/index.d.cts +2 -2
- package/types/index.d.ts +2 -2
- package/types/input/index.d.ts +6 -6
- package/types/input/input-box.d.ts +6 -6
- package/types/input/input-group.d.ts +5 -5
- package/types/input/input.d.ts +27 -10
- package/types/input/textarea.d.ts +17 -8
- package/types/input-number/index.d.ts +10 -7
- package/types/input-otp/index.d.ts +3 -3
- package/types/input-otp/input-otp.d.ts +21 -8
- package/types/input-tag/index.d.ts +6 -6
- package/types/kanban/index.d.ts +5 -5
- package/types/layout/index.d.ts +17 -17
- package/types/list-panel/index.d.ts +7 -7
- package/types/mentions/index.d.ts +7 -7
- package/types/menu/index.d.ts +6 -6
- package/types/menu/menu-context.d.ts +4 -2
- package/types/menu/menu-divider.d.ts +1 -1
- package/types/menu/menu-group.d.ts +4 -4
- package/types/menu/menu-item.d.ts +10 -6
- package/types/menu/menu.d.ts +24 -17
- package/types/menu/recursive-menu.d.ts +5 -5
- package/types/menu/sub-menu.d.ts +6 -6
- package/types/menu/types.d.ts +2 -2
- package/types/message/index.d.ts +3 -3
- package/types/modal/index.d.ts +15 -10
- package/types/modal/modal.d.ts +25 -97
- package/types/modal/toast.d.ts +11 -10
- package/types/notice/container.d.ts +3 -3
- package/types/notice/content.d.ts +10 -77
- package/types/notice/index.d.ts +3 -3
- package/types/notice/instance.d.ts +2 -2
- package/types/page/index.d.ts +6 -6
- package/types/page-header/index.d.ts +4 -4
- package/types/popconfirm/index.d.ts +8 -45
- package/types/poptip/index.d.ts +9 -47
- package/types/popup/index.d.ts +251 -0
- package/types/popup/layers.d.ts +9 -0
- package/types/popup/types.d.ts +15 -0
- package/types/progress/index.d.ts +6 -6
- package/types/qrcode/index.d.ts +7 -7
- package/types/radio/index.d.ts +8 -7
- package/types/radio/radio-button.d.ts +8 -8
- package/types/radio/radio-group.d.ts +4 -3
- package/types/radio/radio.d.ts +7 -7
- package/types/radio/types.d.ts +1 -1
- package/types/rate/index.d.ts +7 -7
- package/types/rate/star.d.ts +7 -7
- package/types/result/index.d.ts +3 -3
- package/types/result/result.d.ts +7 -7
- package/types/ripple/index.d.ts +6 -6
- package/types/row-col/col.d.ts +4 -4
- package/types/row-col/index.d.ts +4 -4
- package/types/row-col/row.d.ts +4 -4
- package/types/segmented/index.d.ts +6 -4
- package/types/select/index.d.ts +5 -5
- package/types/select/option.d.ts +13 -14
- package/types/select/select.d.ts +10 -6
- package/types/select/types.d.ts +1 -1
- package/types/skeleton/index.d.ts +10 -10
- package/types/skeleton/skeleton-avatar.d.ts +13 -13
- package/types/skeleton/skeleton-button.d.ts +15 -15
- package/types/skeleton/skeleton-image.d.ts +11 -11
- package/types/skeleton/skeleton-text.d.ts +11 -11
- package/types/skeleton/skeleton.d.ts +15 -15
- package/types/skeleton/types.d.ts +2 -2
- package/types/skeleton/use-skeleton-loading.d.ts +1 -1
- package/types/slider/index.d.ts +8 -3
- package/types/slider/thumb.d.ts +7 -7
- package/types/space/index.d.ts +4 -4
- package/types/spin/index.d.ts +18 -5
- package/types/splitter/index.d.ts +2 -2
- package/types/splitter/splitter-panel.d.ts +4 -4
- package/types/splitter/splitter.d.ts +6 -6
- package/types/stat-card/countup.d.ts +4 -4
- package/types/stat-card/index.d.ts +5 -5
- package/types/stat-card/rollup.d.ts +6 -4
- package/types/stat-card/stat-card.d.ts +7 -7
- package/types/stat-card/stat-number.d.ts +19 -6
- package/types/stat-card/types.d.ts +1 -1
- package/types/stat-card/utils/countup.d.ts +1 -1
- package/types/stat-card/utils/odometer.d.ts +8 -5
- package/types/steps/index.d.ts +8 -8
- package/types/switch/index.d.ts +3 -2
- package/types/table/column-setting.d.ts +7 -7
- package/types/table/index.d.ts +13 -26
- package/types/table/types.d.ts +2 -2
- package/types/table/utils.d.ts +1 -1
- package/types/tabs/index.d.ts +5 -4
- package/types/tabs/tab-panel.d.ts +6 -6
- package/types/tabs/tabs.d.ts +6 -6
- package/types/tabs/types.d.ts +1 -1
- package/types/tag/index.d.ts +9 -7
- package/types/time-line/index.d.ts +5 -5
- package/types/time-line/timeline-item.d.ts +5 -5
- package/types/time-line/timeline.d.ts +6 -6
- package/types/tooltip/index.d.ts +21 -35
- package/types/tour/index.d.ts +34 -23
- package/types/transfer/index.d.ts +18 -5
- package/types/tree/context.d.ts +1 -1
- package/types/tree/index.d.ts +13 -10
- package/types/tree/types.d.ts +1 -1
- package/types/tree/utils.d.ts +1 -1
- package/types/tree-select/index.d.ts +14 -7
- package/types/typography/index.d.ts +26 -17
- package/types/upload/file-list.d.ts +8 -8
- package/types/upload/index.d.ts +9 -22
- package/types/upload/picture-sort.d.ts +2 -0
- package/types/upload/selector.d.ts +9 -9
- package/types/upload/types.d.ts +1 -1
- package/types/utils/checked.d.ts +1 -1
- package/types/utils/model-value.d.ts +10 -0
- package/types/utils/number.d.ts +1 -1
- package/types/utils/placement.d.ts +1 -1
- package/types/utils/popup.d.ts +11 -0
- package/types/utils/selection-tags.d.ts +18 -0
- package/types/utils/vnode.d.ts +3 -3
- package/types/utils/vue.d.ts +7 -1
- package/types/virtual-list/index.d.ts +6 -6
- package/types/watermark/index.d.ts +8 -8
- package/vetur/attributes.json +1717 -781
- package/vetur/tags.json +359 -159
- package/vetur/web-types.json +7563 -6197
- package/types/directives/transfer.d.ts +0 -2
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { spawnSync } from "node:child_process";
|
|
4
|
+
import { createRequire } from "node:module";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
import ts from "typescript";
|
|
7
|
+
import { validateUsage } from "../validate.mjs";
|
|
8
|
+
|
|
9
|
+
export const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
10
|
+
export const config = JSON.parse(fs.readFileSync(path.join(root, "ai/evals/config.json"), "utf8"));
|
|
11
|
+
const require = createRequire(import.meta.url);
|
|
12
|
+
export const metadata = JSON.parse(
|
|
13
|
+
fs.readFileSync(path.join(root, "ai/kui-components.json"), "utf8"),
|
|
14
|
+
);
|
|
15
|
+
export function loadCases() {
|
|
16
|
+
const cases = JSON.parse(fs.readFileSync(path.join(root, "ai/evals/cases.json"), "utf8"));
|
|
17
|
+
const names = new Set();
|
|
18
|
+
for (const item of cases) {
|
|
19
|
+
if (!/^[a-z][a-z0-9-]+$/.test(item.name) || names.has(item.name))
|
|
20
|
+
throw Error(`Invalid/duplicate case: ${item.name}`);
|
|
21
|
+
names.add(item.name);
|
|
22
|
+
if (!item.prompt?.trim() || !item.components?.length)
|
|
23
|
+
throw Error(`Incomplete case: ${item.name}`);
|
|
24
|
+
for (const name of item.components) {
|
|
25
|
+
if (!metadata.components.some((component) => component.name === name))
|
|
26
|
+
throw Error(`Unknown component: ${name}`);
|
|
27
|
+
}
|
|
28
|
+
if (item.sourceFile)
|
|
29
|
+
item.source = fs.readFileSync(path.resolve(root, "ai/evals", item.sourceFile), "utf8");
|
|
30
|
+
if (!item.source?.trim()) throw Error(`Missing reference: ${item.name}`);
|
|
31
|
+
}
|
|
32
|
+
if (cases.length < 20) throw Error("Expected at least 20 reference cases");
|
|
33
|
+
return cases;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function inspectSource(source, item) {
|
|
37
|
+
const issues = [];
|
|
38
|
+
const tags = new Set();
|
|
39
|
+
const imports = new Set();
|
|
40
|
+
const nodes = [];
|
|
41
|
+
const canonical = (name) => name.replace(/-/g, "").toLowerCase();
|
|
42
|
+
const literal = (expression) => {
|
|
43
|
+
const ast = ts.createSourceFile(
|
|
44
|
+
"value.ts",
|
|
45
|
+
`const value = ${expression};`,
|
|
46
|
+
ts.ScriptTarget.Latest,
|
|
47
|
+
true,
|
|
48
|
+
);
|
|
49
|
+
const value = ast.statements[0]?.declarationList?.declarations[0]?.initializer;
|
|
50
|
+
if (!value) return undefined;
|
|
51
|
+
if (ts.isStringLiteral(value)) return value.text;
|
|
52
|
+
if (ts.isNumericLiteral(value)) return Number(value.text);
|
|
53
|
+
if (value.kind === ts.SyntaxKind.TrueKeyword) return true;
|
|
54
|
+
if (value.kind === ts.SyntaxKind.FalseKeyword) return false;
|
|
55
|
+
return undefined;
|
|
56
|
+
};
|
|
57
|
+
const collectImports = (script) => {
|
|
58
|
+
const ast = ts.createSourceFile(
|
|
59
|
+
"imports.tsx",
|
|
60
|
+
script,
|
|
61
|
+
ts.ScriptTarget.Latest,
|
|
62
|
+
true,
|
|
63
|
+
ts.ScriptKind.TSX,
|
|
64
|
+
);
|
|
65
|
+
for (const node of ast.statements) {
|
|
66
|
+
if (
|
|
67
|
+
!ts.isImportDeclaration(node) ||
|
|
68
|
+
!ts.isStringLiteral(node.moduleSpecifier) ||
|
|
69
|
+
node.moduleSpecifier.text !== config.library
|
|
70
|
+
)
|
|
71
|
+
continue;
|
|
72
|
+
const bindings = node.importClause?.namedBindings;
|
|
73
|
+
if (bindings && ts.isNamedImports(bindings)) {
|
|
74
|
+
for (const element of bindings.elements) {
|
|
75
|
+
if (
|
|
76
|
+
!element.isTypeOnly &&
|
|
77
|
+
!node.importClause.isTypeOnly &&
|
|
78
|
+
(!element.propertyName || element.propertyName.text === element.name.text)
|
|
79
|
+
)
|
|
80
|
+
imports.add(element.name.text);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
let template = source;
|
|
86
|
+
if (typeof source !== "string" || !source.trim())
|
|
87
|
+
return { issues: ["Missing generated source"], skipped: [] };
|
|
88
|
+
if (/@ts-(?:nocheck|ignore|expect-error)/.test(source))
|
|
89
|
+
issues.push("Type-check suppression is not allowed");
|
|
90
|
+
try {
|
|
91
|
+
if (config.framework === "vue") {
|
|
92
|
+
const { parse, compileScript, compileTemplate } = require("@vue/compiler-sfc");
|
|
93
|
+
const { baseParse, NodeTypes } = require("@vue/compiler-dom");
|
|
94
|
+
const { descriptor, errors } = parse(source, { filename: `${item.name}.vue` });
|
|
95
|
+
issues.push(...errors.map(String));
|
|
96
|
+
if (!descriptor.scriptSetup || descriptor.scriptSetup.lang !== "ts")
|
|
97
|
+
issues.push('Use a complete SFC with <script setup lang="ts"> and local imports');
|
|
98
|
+
collectImports(descriptor.scriptSetup?.content ?? "");
|
|
99
|
+
if (descriptor.scriptSetup) compileScript(descriptor, { id: item.name });
|
|
100
|
+
if (!descriptor.template) issues.push("Missing template");
|
|
101
|
+
template = descriptor.template?.content ?? "";
|
|
102
|
+
const compiled = compileTemplate({
|
|
103
|
+
source: template,
|
|
104
|
+
id: item.name,
|
|
105
|
+
filename: `${item.name}.vue`,
|
|
106
|
+
});
|
|
107
|
+
issues.push(...compiled.errors.map(String));
|
|
108
|
+
const visit = (node) => {
|
|
109
|
+
if (node.type === NodeTypes.ELEMENT) {
|
|
110
|
+
tags.add(node.tag);
|
|
111
|
+
const props = {};
|
|
112
|
+
for (const attr of node.props) {
|
|
113
|
+
if (attr.type === NodeTypes.ATTRIBUTE)
|
|
114
|
+
props[canonical(attr.name)] = attr.value?.content ?? true;
|
|
115
|
+
else if (attr.name === "bind" && attr.arg?.isStatic)
|
|
116
|
+
props[canonical(attr.arg.content)] = literal(attr.exp?.content ?? "");
|
|
117
|
+
}
|
|
118
|
+
nodes.push({ tag: node.tag, props });
|
|
119
|
+
}
|
|
120
|
+
node.children?.forEach(visit);
|
|
121
|
+
};
|
|
122
|
+
visit(baseParse(template));
|
|
123
|
+
} else {
|
|
124
|
+
collectImports(source);
|
|
125
|
+
const ast = ts.createSourceFile(
|
|
126
|
+
`${item.name}.tsx`,
|
|
127
|
+
source,
|
|
128
|
+
ts.ScriptTarget.Latest,
|
|
129
|
+
true,
|
|
130
|
+
ts.ScriptKind.TSX,
|
|
131
|
+
);
|
|
132
|
+
const visit = (node) => {
|
|
133
|
+
if (ts.isJsxOpeningElement(node) || ts.isJsxSelfClosingElement(node)) {
|
|
134
|
+
const tag = node.tagName.getText(ast);
|
|
135
|
+
tags.add(tag);
|
|
136
|
+
const props = {};
|
|
137
|
+
for (const attr of node.attributes.properties) {
|
|
138
|
+
if (!ts.isJsxAttribute(attr)) continue;
|
|
139
|
+
const init = attr.initializer;
|
|
140
|
+
props[canonical(attr.name.getText(ast))] = !init
|
|
141
|
+
? true
|
|
142
|
+
: ts.isStringLiteral(init)
|
|
143
|
+
? init.text
|
|
144
|
+
: literal(init.expression?.getText(ast) ?? "");
|
|
145
|
+
}
|
|
146
|
+
nodes.push({ tag, props });
|
|
147
|
+
}
|
|
148
|
+
ts.forEachChild(node, visit);
|
|
149
|
+
};
|
|
150
|
+
visit(ast);
|
|
151
|
+
if (
|
|
152
|
+
!ast.statements.some(
|
|
153
|
+
(node) =>
|
|
154
|
+
ts.isExportAssignment(node) ||
|
|
155
|
+
node.modifiers?.some((m) => m.kind === ts.SyntaxKind.DefaultKeyword),
|
|
156
|
+
)
|
|
157
|
+
)
|
|
158
|
+
issues.push("Missing default component export");
|
|
159
|
+
}
|
|
160
|
+
for (const name of item.components)
|
|
161
|
+
if (!tags.has(name)) issues.push(`Missing required component: ${name} (use canonical names)`);
|
|
162
|
+
for (const name of item.components)
|
|
163
|
+
if (!imports.has(name))
|
|
164
|
+
issues.push(`Missing canonical import from ${config.library}: ${name}`);
|
|
165
|
+
for (const expected of item.expectProps ?? []) {
|
|
166
|
+
if (
|
|
167
|
+
!nodes.some(
|
|
168
|
+
(node) =>
|
|
169
|
+
node.tag === expected.component &&
|
|
170
|
+
node.props[canonical(expected.prop)] === expected.value,
|
|
171
|
+
)
|
|
172
|
+
)
|
|
173
|
+
issues.push(
|
|
174
|
+
`Missing scenario requirement: ${expected.component}.${expected.prop}=${JSON.stringify(expected.value)} (use a literal prop)`,
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
const staticResult = validateUsage(template, metadata);
|
|
178
|
+
issues.push(...staticResult.issues.map((issue) => JSON.stringify(issue)));
|
|
179
|
+
return { issues, skipped: staticResult.skipped ?? [] };
|
|
180
|
+
} catch (error) {
|
|
181
|
+
issues.push(String(error));
|
|
182
|
+
return { issues, skipped: [] };
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Generated source is only parsed/typechecked, never imported or executed.
|
|
187
|
+
export function evaluate(cases, sources) {
|
|
188
|
+
const directory = fs.mkdtempSync(path.join(root, ".ai-eval-"));
|
|
189
|
+
const results = cases.map((item) => ({
|
|
190
|
+
name: item.name,
|
|
191
|
+
...inspectSource(sources[item.name], item),
|
|
192
|
+
typeErrors: [],
|
|
193
|
+
}));
|
|
194
|
+
try {
|
|
195
|
+
const ext = config.framework === "vue" ? "vue" : "tsx";
|
|
196
|
+
for (const item of cases)
|
|
197
|
+
fs.writeFileSync(path.join(directory, `${item.name}.${ext}`), sources[item.name] || "");
|
|
198
|
+
fs.writeFileSync(
|
|
199
|
+
path.join(directory, "tsconfig.json"),
|
|
200
|
+
JSON.stringify({
|
|
201
|
+
extends: "../tsconfig.app.json",
|
|
202
|
+
compilerOptions: {
|
|
203
|
+
composite: false,
|
|
204
|
+
incremental: false,
|
|
205
|
+
declaration: true,
|
|
206
|
+
emitDeclarationOnly: false,
|
|
207
|
+
noEmit: true,
|
|
208
|
+
noUnusedLocals: false,
|
|
209
|
+
noUnusedParameters: false,
|
|
210
|
+
},
|
|
211
|
+
include: [`./*.${ext}`, "../src/**/*.d.ts"],
|
|
212
|
+
exclude: [],
|
|
213
|
+
}),
|
|
214
|
+
);
|
|
215
|
+
const compiler = config.framework === "vue" ? "vue-tsc/bin/vue-tsc.js" : "typescript/bin/tsc";
|
|
216
|
+
const checked = spawnSync(
|
|
217
|
+
process.execPath,
|
|
218
|
+
[
|
|
219
|
+
"--max-old-space-size=2048",
|
|
220
|
+
require.resolve(compiler),
|
|
221
|
+
"-p",
|
|
222
|
+
path.join(directory, "tsconfig.json"),
|
|
223
|
+
"--pretty",
|
|
224
|
+
"false",
|
|
225
|
+
],
|
|
226
|
+
{
|
|
227
|
+
cwd: root,
|
|
228
|
+
encoding: "utf8",
|
|
229
|
+
timeout: 180000,
|
|
230
|
+
maxBuffer: 8 * 1024 * 1024,
|
|
231
|
+
},
|
|
232
|
+
);
|
|
233
|
+
if (checked.status !== 0) {
|
|
234
|
+
const output = [checked.stdout, checked.stderr, checked.error?.message]
|
|
235
|
+
.filter(Boolean)
|
|
236
|
+
.join("\n");
|
|
237
|
+
const globalErrors = [];
|
|
238
|
+
let diagnosticTarget = globalErrors;
|
|
239
|
+
for (const line of output.split("\n").filter(Boolean)) {
|
|
240
|
+
const match = line.match(/\.ai-eval-[^/\\]+[/\\]([a-z0-9-]+)\.(?:tsx|vue)\(/);
|
|
241
|
+
const result = match && results.find((item) => item.name === match[1]);
|
|
242
|
+
if (result) diagnosticTarget = result.typeErrors;
|
|
243
|
+
else if (!/^\s/.test(line)) diagnosticTarget = globalErrors;
|
|
244
|
+
diagnosticTarget.push(line.replaceAll(directory, "<eval>"));
|
|
245
|
+
}
|
|
246
|
+
if (!output.trim())
|
|
247
|
+
globalErrors.push(`Typecheck failed: status=${checked.status}, signal=${checked.signal}`);
|
|
248
|
+
if (globalErrors.length) results.forEach((item) => item.typeErrors.push(...globalErrors));
|
|
249
|
+
}
|
|
250
|
+
for (const result of results)
|
|
251
|
+
result.passed = !result.issues.length && !result.typeErrors.length;
|
|
252
|
+
return {
|
|
253
|
+
library: config.library,
|
|
254
|
+
libraryVersion: JSON.parse(fs.readFileSync(path.join(root, "package.json"), "utf8")).version,
|
|
255
|
+
evaluatedAt: new Date().toISOString(),
|
|
256
|
+
criteria:
|
|
257
|
+
"Component usage, static API validation and strict TypeScript checks. Not a runtime or semantic correctness score.",
|
|
258
|
+
runtimeEvaluated: false,
|
|
259
|
+
total: results.length,
|
|
260
|
+
passed: results.filter((item) => item.passed).length,
|
|
261
|
+
passRate: results.filter((item) => item.passed).length / results.length,
|
|
262
|
+
results,
|
|
263
|
+
};
|
|
264
|
+
} finally {
|
|
265
|
+
fs.rmSync(directory, { recursive: true, force: true });
|
|
266
|
+
}
|
|
267
|
+
}
|
package/ai/evals/run.mjs
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { spawnSync } from "node:child_process";
|
|
5
|
+
import { config, evaluate, loadCases, metadata, root } from "./evaluate.mjs";
|
|
6
|
+
|
|
7
|
+
const args = process.argv.slice(2);
|
|
8
|
+
const options = {};
|
|
9
|
+
for (let i = 0; i < args.length; i++) {
|
|
10
|
+
const key = args[i];
|
|
11
|
+
if (key === "--reference") options.reference = true;
|
|
12
|
+
else if (
|
|
13
|
+
["--model", "--generator", "--responses", "--out", "--case", "--export-prompts"].includes(
|
|
14
|
+
key,
|
|
15
|
+
) &&
|
|
16
|
+
args[i + 1]
|
|
17
|
+
)
|
|
18
|
+
options[key.slice(2)] = args[++i];
|
|
19
|
+
else throw Error(`Unknown or incomplete argument: ${key}`);
|
|
20
|
+
}
|
|
21
|
+
const modes = [
|
|
22
|
+
options.reference,
|
|
23
|
+
options.generator,
|
|
24
|
+
options.responses,
|
|
25
|
+
options["export-prompts"],
|
|
26
|
+
].filter(Boolean);
|
|
27
|
+
if (modes.length !== 1)
|
|
28
|
+
throw Error(
|
|
29
|
+
"Choose exactly one: --reference, --generator executable, --responses file.json, --export-prompts file.json",
|
|
30
|
+
);
|
|
31
|
+
let cases = loadCases();
|
|
32
|
+
if (options.case) {
|
|
33
|
+
cases = cases.filter((item) => item.name === options.case);
|
|
34
|
+
if (!cases.length) throw Error(`Unknown case: ${options.case}`);
|
|
35
|
+
}
|
|
36
|
+
const skill = fs.readFileSync(path.join(root, `ai/skills/${config.library}/SKILL.md`), "utf8");
|
|
37
|
+
const prompts = cases.map((item) => ({
|
|
38
|
+
name: item.name,
|
|
39
|
+
model: options.model,
|
|
40
|
+
prompt: item.prompt,
|
|
41
|
+
expectProps: item.expectProps ?? [],
|
|
42
|
+
instructions: `${skill}\nReturn only a complete ${config.framework === "vue" ? 'Vue SFC with <script setup lang="ts">, local imports and template' : "TSX module with local imports and a default component export"}. Use canonical component names: ${item.components.join(", ")}. Use literal values for expectProps requirements. Do not use type-check suppression or external relative imports.`,
|
|
43
|
+
api: metadata.components.filter((component) => item.components.includes(component.name)),
|
|
44
|
+
}));
|
|
45
|
+
const write = (file, value) => {
|
|
46
|
+
fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true });
|
|
47
|
+
fs.writeFileSync(file, JSON.stringify(value, null, 2) + "\n");
|
|
48
|
+
};
|
|
49
|
+
if (options["export-prompts"]) {
|
|
50
|
+
write(options["export-prompts"], prompts);
|
|
51
|
+
console.log(`Exported ${prompts.length} prompts without reference answers.`);
|
|
52
|
+
} else {
|
|
53
|
+
if (!options.reference && !options.model)
|
|
54
|
+
throw Error("Model evaluations require --model (record the actual model/version)");
|
|
55
|
+
const started = Date.now();
|
|
56
|
+
const sources = {};
|
|
57
|
+
const generation = {};
|
|
58
|
+
if (options.reference)
|
|
59
|
+
cases.forEach((item) => {
|
|
60
|
+
sources[item.name] = item.source;
|
|
61
|
+
});
|
|
62
|
+
else if (options.responses) {
|
|
63
|
+
const data = JSON.parse(fs.readFileSync(options.responses, "utf8"));
|
|
64
|
+
if (!data || Array.isArray(data) || typeof data !== "object")
|
|
65
|
+
throw Error("Responses must be an object mapping case names to source strings");
|
|
66
|
+
for (const item of cases) {
|
|
67
|
+
if (typeof data[item.name] !== "string")
|
|
68
|
+
throw Error(`Missing source string for ${item.name}`);
|
|
69
|
+
sources[item.name] = data[item.name];
|
|
70
|
+
}
|
|
71
|
+
} else {
|
|
72
|
+
for (const prompt of prompts) {
|
|
73
|
+
const before = Date.now();
|
|
74
|
+
// A trusted, user-supplied executable receives JSON on stdin; stdout is code.
|
|
75
|
+
// No shell interpolation and no model-generated commands are executed.
|
|
76
|
+
const response = spawnSync(options.generator, [], {
|
|
77
|
+
input: JSON.stringify(prompt),
|
|
78
|
+
encoding: "utf8",
|
|
79
|
+
timeout: 180000,
|
|
80
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
81
|
+
});
|
|
82
|
+
generation[prompt.name] = {
|
|
83
|
+
durationMs: Date.now() - before,
|
|
84
|
+
exitCode: response.status,
|
|
85
|
+
failed: response.status !== 0,
|
|
86
|
+
signal: response.signal,
|
|
87
|
+
};
|
|
88
|
+
sources[prompt.name] =
|
|
89
|
+
response.status === 0
|
|
90
|
+
? response.stdout.trim().replace(/^```(?:tsx|typescript|vue)?\s*\n([\s\S]*?)\n```$/, "$1")
|
|
91
|
+
: "";
|
|
92
|
+
console.log(`${prompt.name}: ${response.status === 0 ? "generated" : "generation failed"}`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const report = {
|
|
96
|
+
mode: options.reference ? "reference" : "model",
|
|
97
|
+
model: options.model ?? null,
|
|
98
|
+
...evaluate(cases, sources),
|
|
99
|
+
contextMode: "skill-and-api-injection",
|
|
100
|
+
suiteHash: createHash("sha256").update(JSON.stringify(cases)).digest("hex"),
|
|
101
|
+
contextHash: createHash("sha256").update(JSON.stringify(prompts)).digest("hex"),
|
|
102
|
+
generation,
|
|
103
|
+
durationMs: Date.now() - started,
|
|
104
|
+
};
|
|
105
|
+
const output = path.resolve(
|
|
106
|
+
options.out ??
|
|
107
|
+
path.join(root, ".ai-eval-results", options.reference ? "reference.json" : "model.json"),
|
|
108
|
+
);
|
|
109
|
+
write(output, { ...report, sources });
|
|
110
|
+
console.log(
|
|
111
|
+
`${config.library}: ${report.passed}/${report.total} passed static/API/type gates (${(report.passRate * 100).toFixed(1)}%). Report: ${output}`,
|
|
112
|
+
);
|
|
113
|
+
if (report.passed !== report.total) {
|
|
114
|
+
for (const item of report.results.filter((result) => !result.passed))
|
|
115
|
+
console.error(item.name, [...item.issues, ...item.typeErrors].join("\n"));
|
|
116
|
+
process.exitCode = 1;
|
|
117
|
+
}
|
|
118
|
+
}
|