@urbicon-ui/mcp-server 6.24.0 → 6.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -3
- package/dist/data/catalog-loader.d.ts +0 -37
- package/dist/data/catalog-loader.d.ts.map +0 -1
- package/dist/data/catalog-loader.js +0 -15
- package/dist/data/catalog-loader.js.map +0 -1
- package/dist/data/component-loader.d.ts +0 -2
- package/dist/data/component-loader.d.ts.map +0 -1
- package/dist/data/component-loader.js +0 -17
- package/dist/data/component-loader.js.map +0 -1
- package/dist/data/recipe-loader.d.ts +0 -4
- package/dist/data/recipe-loader.d.ts.map +0 -1
- package/dist/data/recipe-loader.js +0 -102
- package/dist/data/recipe-loader.js.map +0 -1
- package/dist/data/template-loader.d.ts +0 -8
- package/dist/data/template-loader.d.ts.map +0 -1
- package/dist/data/template-loader.js +0 -33
- package/dist/data/template-loader.js.map +0 -1
- package/dist/index.d.ts +0 -3
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -57
- package/dist/index.js.map +0 -1
- package/dist/resources/catalog.d.ts +0 -3
- package/dist/resources/catalog.d.ts.map +0 -1
- package/dist/resources/catalog.js +0 -20
- package/dist/resources/catalog.js.map +0 -1
- package/dist/resources/component.d.ts +0 -3
- package/dist/resources/component.d.ts.map +0 -1
- package/dist/resources/component.js +0 -29
- package/dist/resources/component.js.map +0 -1
- package/dist/resources/guides.d.ts +0 -3
- package/dist/resources/guides.d.ts.map +0 -1
- package/dist/resources/guides.js +0 -36
- package/dist/resources/guides.js.map +0 -1
- package/dist/server.d.ts +0 -3
- package/dist/server.d.ts.map +0 -1
- package/dist/server.js.map +0 -1
- package/dist/tools/find-components.d.ts +0 -3
- package/dist/tools/find-components.d.ts.map +0 -1
- package/dist/tools/find-components.js +0 -21
- package/dist/tools/find-components.js.map +0 -1
- package/dist/tools/get-recipe.d.ts +0 -3
- package/dist/tools/get-recipe.d.ts.map +0 -1
- package/dist/tools/get-recipe.js +0 -48
- package/dist/tools/get-recipe.js.map +0 -1
- package/dist/tools/suggest-implementation.d.ts +0 -3
- package/dist/tools/suggest-implementation.d.ts.map +0 -1
- package/dist/tools/suggest-implementation.js +0 -178
- package/dist/tools/suggest-implementation.js.map +0 -1
- package/dist/transports/http.d.ts +0 -2
- package/dist/transports/http.d.ts.map +0 -1
- package/dist/transports/http.js +0 -77
- package/dist/transports/http.js.map +0 -1
- package/dist/transports/stdio.d.ts +0 -3
- package/dist/transports/stdio.d.ts.map +0 -1
- package/dist/transports/stdio.js +0 -6
- package/dist/transports/stdio.js.map +0 -1
- package/dist/tsconfig.tsbuildinfo +0 -1
- package/dist/utils/format-catalog.d.ts +0 -7
- package/dist/utils/format-catalog.d.ts.map +0 -1
- package/dist/utils/format-catalog.js +0 -93
- package/dist/utils/format-catalog.js.map +0 -1
- package/dist/utils/paths.d.ts +0 -7
- package/dist/utils/paths.d.ts.map +0 -1
- package/dist/utils/paths.js +0 -23
- package/dist/utils/paths.js.map +0 -1
- package/dist/utils/search.d.ts +0 -3
- package/dist/utils/search.d.ts.map +0 -1
- package/dist/utils/search.js +0 -44
- package/dist/utils/search.js.map +0 -1
- package/src/data/catalog-loader.test.ts +0 -42
- package/src/data/design-system-loader.test.ts +0 -82
- package/src/data/icon-loader.test.ts +0 -30
- package/src/eval/briefs.ts +0 -104
- package/src/eval/eval.test.ts +0 -106
- package/src/eval/index.ts +0 -11
- package/src/eval/score.ts +0 -128
- package/src/prompts/design-prompts.test.ts +0 -79
- package/src/server.test.ts +0 -78
- package/src/utils/format-catalog.test.ts +0 -144
- package/tsconfig.json +0 -27
- package/vitest.config.ts +0 -15
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"format-catalog.js","sourceRoot":"","sources":["../../src/utils/format-catalog.ts"],"names":[],"mappings":"AAEA,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,cAAc;IACd,aAAa;IACb,YAAY;IACZ,UAAU;IACV,wBAAwB;IACxB,SAAS;IACT,iBAAiB;IACjB,gBAAgB;CACjB,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;AAE5F,MAAM,UAAU,GAA2B;IACzC,MAAM,EAAE,SAAS;IACjB,IAAI,EAAE,OAAO;IACb,MAAM,EAAE,qBAAqB;IAC7B,QAAQ,EAAE,mBAAmB;IAC7B,UAAU,EAAE,YAAY;IACxB,OAAO,EAAE,oBAAoB;IAC7B,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,UAAU,wBAAwB,CACtC,UAAmC;IAEnC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,UAAmC,EACnC,OAAsD;IAEtD,IAAI,QAAQ,GAAG,wBAAwB,CAAC,UAAU,CAAC,CAAC;IAEpD,IAAI,OAAO,EAAE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACjE,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;IACrF,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmC,CAAC;IAC3D,MAAM,SAAS,GAA4B,EAAE,CAAC;IAE9C,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,UAAU,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;gBAAE,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CAAC,UAAU,CAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,IAAI,EAAE,GAAG,6BAA6B,CAAC;IACvC,EAAE,IAAI,KAAK,QAAQ,CAAC,MAAM,0BAA0B,CAAC;IACrD,EAAE,IAAI,2EAA2E,CAAC;IAClF,EAAE;QACA,4GAA4G,CAAC;IAE/G,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAE3C,EAAE,IAAI,MAAM,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC;QAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,EAAE,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QACD,EAAE,IAAI,IAAI,CAAC;IACb,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,EAAE,IAAI,YAAY,CAAC;QACnB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,EAAE,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QACD,EAAE,IAAI,IAAI,CAAC;IACb,CAAC;IAED,IAAI,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,EAAE,IAAI,cAAc,CAAC;QACrB,EAAE,IAAI,oFAAoF,CAAC;QAC3F,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,EAAE,IAAI,OAAO,MAAM,CAAC,EAAE,QAAQ,MAAM,CAAC,WAAW,KAAK,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QACzF,CAAC;QACD,EAAE,IAAI,IAAI,CAAC;IACb,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAgB;IACxC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAClC,OAAO,CACL,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC;QACxE,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CACvE,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,IAA2B;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ;SAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;SAC1C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;SAC9C,IAAI,CAAC,KAAK,CAAC,CAAC;IAEf,MAAM,OAAO,GACX,IAAI,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAE9F,IAAI,IAAI,GAAG,OAAO,IAAI,CAAC,IAAI,QAAQ,IAAI,CAAC,WAAW,EAAE,CAAC;IACtD,IAAI,QAAQ;QAAE,IAAI,IAAI,MAAM,QAAQ,EAAE,CAAC;IACvC,IAAI,IAAI,OAAO,CAAC;IAChB,IAAI,IAAI,IAAI,CAAC;IAEb,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/dist/utils/paths.d.ts
DELETED
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
export declare function getDataDir(): string;
|
|
2
|
-
export declare function getTemplateDir(): string;
|
|
3
|
-
export declare function getRecipeDir(): string;
|
|
4
|
-
export declare function getTemplatePath(): string;
|
|
5
|
-
export declare function getCatalogPath(): string;
|
|
6
|
-
export declare function getComponentLlmPath(group: string, slug: string): string;
|
|
7
|
-
//# sourceMappingURL=paths.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"paths.d.ts","sourceRoot":"","sources":["../../src/utils/paths.ts"],"names":[],"mappings":"AAMA,wBAAgB,UAAU,IAAI,MAAM,CAEnC;AAED,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAED,wBAAgB,eAAe,IAAI,MAAM,CAExC;AAED,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAEvE"}
|
package/dist/utils/paths.js
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { dirname, resolve } from 'node:path';
|
|
2
|
-
import { fileURLToPath } from 'node:url';
|
|
3
|
-
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
4
|
-
const packageRoot = resolve(__dirname, '..', '..');
|
|
5
|
-
export function getDataDir() {
|
|
6
|
-
return process.env.DATA_DIR ?? resolve(packageRoot, '..', '..', 'apps', 'docs', 'static');
|
|
7
|
-
}
|
|
8
|
-
export function getTemplateDir() {
|
|
9
|
-
return resolve(packageRoot, '..', '..', 'docs');
|
|
10
|
-
}
|
|
11
|
-
export function getRecipeDir() {
|
|
12
|
-
return resolve(packageRoot, '..', '..', 'apps', 'docs', 'src', 'routes', 'recipes');
|
|
13
|
-
}
|
|
14
|
-
export function getTemplatePath() {
|
|
15
|
-
return resolve(getTemplateDir(), 'llms-full-template.md');
|
|
16
|
-
}
|
|
17
|
-
export function getCatalogPath() {
|
|
18
|
-
return resolve(getDataDir(), 'mcp', 'component-catalog.json');
|
|
19
|
-
}
|
|
20
|
-
export function getComponentLlmPath(group, slug) {
|
|
21
|
-
return resolve(getDataDir(), group, slug, 'llm.txt');
|
|
22
|
-
}
|
|
23
|
-
//# sourceMappingURL=paths.js.map
|
package/dist/utils/paths.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"paths.js","sourceRoot":"","sources":["../../src/utils/paths.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEnD,MAAM,UAAU,UAAU;IACxB,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AAC5F,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;AACtF,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,OAAO,CAAC,cAAc,EAAE,EAAE,uBAAuB,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,OAAO,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,wBAAwB,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,IAAY;IAC7D,OAAO,OAAO,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;AACvD,CAAC"}
|
package/dist/utils/search.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/utils/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAOvE,wBAAgB,eAAe,CAC7B,UAAU,EAAE,qBAAqB,EAAE,EACnC,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,KAAK,SAAI,GACR,qBAAqB,EAAE,CAiDzB"}
|
package/dist/utils/search.js
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
export function matchComponents(components, query, tags, limit = 5) {
|
|
2
|
-
const keywords = query
|
|
3
|
-
.toLowerCase()
|
|
4
|
-
.split(/[\s,\-_]+/)
|
|
5
|
-
.filter((w) => w.length > 1);
|
|
6
|
-
const scored = components.map((entry) => {
|
|
7
|
-
let score = 0;
|
|
8
|
-
const nameLower = entry.name.toLowerCase();
|
|
9
|
-
const slugLower = entry.slug.toLowerCase();
|
|
10
|
-
const descLower = entry.description.toLowerCase();
|
|
11
|
-
for (const kw of keywords) {
|
|
12
|
-
if (nameLower === kw || slugLower === kw) {
|
|
13
|
-
score += 10;
|
|
14
|
-
}
|
|
15
|
-
else if (nameLower.includes(kw) || slugLower.includes(kw)) {
|
|
16
|
-
score += 7;
|
|
17
|
-
}
|
|
18
|
-
if (entry.tags.some((t) => t.toLowerCase() === kw)) {
|
|
19
|
-
score += 5;
|
|
20
|
-
}
|
|
21
|
-
if (descLower.includes(kw)) {
|
|
22
|
-
score += 3;
|
|
23
|
-
}
|
|
24
|
-
if (entry.keyProps.some((p) => p.toLowerCase().includes(kw))) {
|
|
25
|
-
score += 1;
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
if (tags && tags.length > 0) {
|
|
29
|
-
const entryTags = entry.tags.map((t) => t.toLowerCase());
|
|
30
|
-
for (const tag of tags) {
|
|
31
|
-
if (entryTags.includes(tag.toLowerCase())) {
|
|
32
|
-
score += 5;
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
return { entry, score };
|
|
37
|
-
});
|
|
38
|
-
return scored
|
|
39
|
-
.filter((s) => s.score > 0)
|
|
40
|
-
.sort((a, b) => b.score - a.score)
|
|
41
|
-
.slice(0, limit)
|
|
42
|
-
.map((s) => s.entry);
|
|
43
|
-
}
|
|
44
|
-
//# sourceMappingURL=search.js.map
|
package/dist/utils/search.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/utils/search.ts"],"names":[],"mappings":"AAOA,MAAM,UAAU,eAAe,CAC7B,UAAmC,EACnC,KAAa,EACb,IAAe,EACf,KAAK,GAAG,CAAC;IAET,MAAM,QAAQ,GAAG,KAAK;SACnB,WAAW,EAAE;SACb,KAAK,CAAC,WAAW,CAAC;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/B,MAAM,MAAM,GAAkB,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACrD,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QAElD,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,IAAI,SAAS,KAAK,EAAE,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;gBACzC,KAAK,IAAI,EAAE,CAAC;YACd,CAAC;iBAAM,IAAI,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC5D,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;YAED,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;gBACnD,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;YAED,IAAI,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC3B,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;YAED,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC7D,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;QACH,CAAC;QAED,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;oBAC1C,KAAK,IAAI,CAAC,CAAC;gBACb,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM;SACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AACzB,CAAC"}
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
import { existsSync } from 'node:fs';
|
|
2
|
-
import { getCatalogPath } from '@urbicon-ui/design-content';
|
|
3
|
-
import { describe, expect, it } from 'vitest';
|
|
4
|
-
import { getCachedCatalog, loadCatalog } from './catalog-loader.js';
|
|
5
|
-
|
|
6
|
-
// Integration test: exercises the real catalog JSON produced by docs-gen.
|
|
7
|
-
// If the catalog does not exist (e.g. in a fresh checkout before the docs
|
|
8
|
-
// pipeline has run), the test is skipped rather than failing.
|
|
9
|
-
const catalogPath = getCatalogPath();
|
|
10
|
-
const catalogAvailable = existsSync(catalogPath);
|
|
11
|
-
|
|
12
|
-
describe.skipIf(!catalogAvailable)('catalog-loader integration', () => {
|
|
13
|
-
it('loads the catalog from disk', async () => {
|
|
14
|
-
const catalog = await loadCatalog();
|
|
15
|
-
expect(catalog).toHaveProperty('components');
|
|
16
|
-
expect(catalog).toHaveProperty('recipes');
|
|
17
|
-
expect(catalog).toHaveProperty('tags');
|
|
18
|
-
expect(Array.isArray(catalog.components)).toBe(true);
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
it('every component has the required shape', async () => {
|
|
22
|
-
const catalog = await loadCatalog();
|
|
23
|
-
for (const component of catalog.components) {
|
|
24
|
-
expect(component.name).toBeTruthy();
|
|
25
|
-
expect(component.slug).toBeTruthy();
|
|
26
|
-
expect(component.package).toMatch(/^@urbicon-ui\//);
|
|
27
|
-
expect(['primitives', 'components', 'core', 'auth']).toContain(component.group);
|
|
28
|
-
expect(Array.isArray(component.tags)).toBe(true);
|
|
29
|
-
expect(Array.isArray(component.keyProps)).toBe(true);
|
|
30
|
-
expect(Array.isArray(component.slots)).toBe(true);
|
|
31
|
-
}
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
it('caches the catalog between invocations', async () => {
|
|
35
|
-
await loadCatalog();
|
|
36
|
-
const cached = getCachedCatalog();
|
|
37
|
-
expect(cached).not.toBeNull();
|
|
38
|
-
|
|
39
|
-
const second = await loadCatalog();
|
|
40
|
-
expect(second).toBe(cached);
|
|
41
|
-
});
|
|
42
|
-
});
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import { existsSync } from 'node:fs';
|
|
2
|
-
import { resolve } from 'node:path';
|
|
3
|
-
import { getDesignSystemDir } from '@urbicon-ui/design-content';
|
|
4
|
-
import { describe, expect, it } from 'vitest';
|
|
5
|
-
import {
|
|
6
|
-
extractPrincipleSection,
|
|
7
|
-
getPatternByName,
|
|
8
|
-
loadPatterns,
|
|
9
|
-
loadPrinciples
|
|
10
|
-
} from './design-system-loader.js';
|
|
11
|
-
|
|
12
|
-
const dsDir = getDesignSystemDir();
|
|
13
|
-
const principlesAvailable = existsSync(resolve(dsDir, 'principles.md'));
|
|
14
|
-
const patternsAvailable = existsSync(resolve(dsDir, 'patterns'));
|
|
15
|
-
|
|
16
|
-
describe.skipIf(!principlesAvailable)('loadPrinciples', () => {
|
|
17
|
-
it('loads principles as a non-empty string', async () => {
|
|
18
|
-
const principles = await loadPrinciples();
|
|
19
|
-
expect(principles.length).toBeGreaterThan(0);
|
|
20
|
-
});
|
|
21
|
-
|
|
22
|
-
it('contains expected topic headings', async () => {
|
|
23
|
-
const principles = await loadPrinciples();
|
|
24
|
-
expect(principles).toContain('## Visual Hierarchy');
|
|
25
|
-
expect(principles).toContain('## Theming');
|
|
26
|
-
expect(principles).toContain('## Component Selection');
|
|
27
|
-
});
|
|
28
|
-
});
|
|
29
|
-
|
|
30
|
-
describe.skipIf(!principlesAvailable)('extractPrincipleSection', () => {
|
|
31
|
-
it('extracts a specific topic section', async () => {
|
|
32
|
-
const principles = await loadPrinciples();
|
|
33
|
-
const section = extractPrincipleSection(principles, 'theming');
|
|
34
|
-
expect(section).not.toBeNull();
|
|
35
|
-
expect(section).toContain('## Theming');
|
|
36
|
-
expect(section).not.toContain('## Visual Hierarchy');
|
|
37
|
-
});
|
|
38
|
-
|
|
39
|
-
it('returns null for unknown topics', async () => {
|
|
40
|
-
const principles = await loadPrinciples();
|
|
41
|
-
// @ts-expect-error testing invalid input
|
|
42
|
-
const section = extractPrincipleSection(principles, 'nonexistent');
|
|
43
|
-
expect(section).toBeNull();
|
|
44
|
-
});
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
describe.skipIf(!patternsAvailable)('loadPatterns', () => {
|
|
48
|
-
it('loads at least one pattern', async () => {
|
|
49
|
-
const patterns = await loadPatterns();
|
|
50
|
-
expect(patterns.length).toBeGreaterThan(0);
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
it('each pattern has required fields', async () => {
|
|
54
|
-
const patterns = await loadPatterns();
|
|
55
|
-
for (const p of patterns) {
|
|
56
|
-
expect(p.name).toBeTruthy();
|
|
57
|
-
expect(p.title).toBeTruthy();
|
|
58
|
-
expect(p.description).toBeTruthy();
|
|
59
|
-
expect(p.content.length).toBeGreaterThan(0);
|
|
60
|
-
}
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
it('patterns are sorted by name', async () => {
|
|
64
|
-
const patterns = await loadPatterns();
|
|
65
|
-
const names = patterns.map((p) => p.name);
|
|
66
|
-
expect(names).toEqual([...names].sort());
|
|
67
|
-
});
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
describe.skipIf(!patternsAvailable)('getPatternByName', () => {
|
|
71
|
-
it('finds an existing pattern', async () => {
|
|
72
|
-
const pattern = await getPatternByName('settings-page');
|
|
73
|
-
expect(pattern).not.toBeNull();
|
|
74
|
-
expect(pattern?.name).toBe('settings-page');
|
|
75
|
-
expect(pattern?.title).toBe('Settings Page');
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
it('returns null for unknown patterns', async () => {
|
|
79
|
-
const pattern = await getPatternByName('nonexistent');
|
|
80
|
-
expect(pattern).toBeNull();
|
|
81
|
-
});
|
|
82
|
-
});
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import { existsSync } from 'node:fs';
|
|
2
|
-
import { getIconsPath } from '@urbicon-ui/design-content';
|
|
3
|
-
import { describe, expect, it } from 'vitest';
|
|
4
|
-
import { loadIcons } from './icon-loader.js';
|
|
5
|
-
|
|
6
|
-
// Integration test: exercises the real icons.json from the content bundle. The
|
|
7
|
-
// registry-parsing logic itself is unit-tested in docs-gen (icons.test.ts); here we
|
|
8
|
-
// only assert the loader reads + shapes the bundled JSON. Skipped on a fresh checkout
|
|
9
|
-
// before the bundle has been generated.
|
|
10
|
-
const iconsAvailable = existsSync(getIconsPath());
|
|
11
|
-
|
|
12
|
-
describe.skipIf(!iconsAvailable)('icon-loader integration', () => {
|
|
13
|
-
it('loads icons with the expected shape', async () => {
|
|
14
|
-
const icons = await loadIcons();
|
|
15
|
-
expect(icons.length).toBeGreaterThan(0);
|
|
16
|
-
for (const icon of icons) {
|
|
17
|
-
expect(icon.name).toBeTruthy();
|
|
18
|
-
expect(icon.componentName).toMatch(/Icon$/);
|
|
19
|
-
expect(typeof icon.label).toBe('string');
|
|
20
|
-
expect(Array.isArray(icon.categories)).toBe(true);
|
|
21
|
-
expect(Array.isArray(icon.keywords)).toBe(true);
|
|
22
|
-
}
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
it('caches between invocations', async () => {
|
|
26
|
-
const first = await loadIcons();
|
|
27
|
-
const second = await loadIcons();
|
|
28
|
-
expect(second).toBe(first);
|
|
29
|
-
});
|
|
30
|
-
});
|
package/src/eval/briefs.ts
DELETED
|
@@ -1,104 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The eval-suite design briefs — the standing benchmark for "does a change to the
|
|
3
|
-
* design layers actually improve generation, or just lengthen the context?"
|
|
4
|
-
* (docs/internal/DESIGN-MCP.md, cross-cutting eval). Systematises earlier one-off
|
|
5
|
-
* design-quality comparisons into a repeatable set.
|
|
6
|
-
*
|
|
7
|
-
* Each brief is generated under two conditions (baseline vs. design-MCP loop),
|
|
8
|
-
* then scored by `scoreImplementation` (deterministic linter) and the judge
|
|
9
|
-
* rubric (get_design_principles(as="rubric")). The brief spread covers the page
|
|
10
|
-
* archetypes the composition patterns target, plus a couple of component-dense
|
|
11
|
-
* surfaces where token hallucination tends to spike.
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
export interface EvalBrief {
|
|
15
|
-
id: string;
|
|
16
|
-
title: string;
|
|
17
|
-
/** The prompt handed to the generating instance. */
|
|
18
|
-
prompt: string;
|
|
19
|
-
/** Suggested composition pattern, when one fits (for the design-MCP condition). */
|
|
20
|
-
pattern?: string;
|
|
21
|
-
tags: string[];
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
export const EVAL_BRIEFS: readonly EvalBrief[] = [
|
|
25
|
-
{
|
|
26
|
-
id: 'ops-dashboard',
|
|
27
|
-
title: 'Ops Dashboard',
|
|
28
|
-
prompt:
|
|
29
|
-
'Build an operations dashboard for a SaaS product: a row of KPI tiles (users, revenue, error rate, latency), a primary chart area, a recent-activity feed, and a quick-actions panel. Some metrics are healthy, some are degraded.',
|
|
30
|
-
pattern: 'dashboard',
|
|
31
|
-
tags: ['dashboard', 'data-dense', 'status']
|
|
32
|
-
},
|
|
33
|
-
{
|
|
34
|
-
id: 'account-settings',
|
|
35
|
-
title: 'Account Settings',
|
|
36
|
-
prompt:
|
|
37
|
-
'Build an account settings page with sections for Profile, Notifications, and Security. Booleans use toggles, enums use selects, and there is a destructive "Delete account" action separated at the bottom.',
|
|
38
|
-
pattern: 'settings-page',
|
|
39
|
-
tags: ['settings', 'form', 'destructive']
|
|
40
|
-
},
|
|
41
|
-
{
|
|
42
|
-
id: 'signup-wizard',
|
|
43
|
-
title: 'Multi-Step Signup',
|
|
44
|
-
prompt:
|
|
45
|
-
'Build a three-step signup wizard (account → organisation → review) with inline field validation, a progress indicator, and a sticky continue/back action bar.',
|
|
46
|
-
pattern: 'form-page',
|
|
47
|
-
tags: ['form', 'wizard', 'validation']
|
|
48
|
-
},
|
|
49
|
-
{
|
|
50
|
-
id: 'pricing',
|
|
51
|
-
title: 'Pricing Page',
|
|
52
|
-
prompt:
|
|
53
|
-
'Build a pricing page with three tiers (Starter, Pro, Enterprise), one highlighted as recommended, each with a feature list and a call-to-action. Include a monthly/yearly toggle.',
|
|
54
|
-
tags: ['marketing', 'cards', 'cta']
|
|
55
|
-
},
|
|
56
|
-
{
|
|
57
|
-
id: 'users-table',
|
|
58
|
-
title: 'Users Admin Table',
|
|
59
|
-
prompt:
|
|
60
|
-
'Build a users administration screen: a searchable, filterable table of users with avatar, name, role badge, status, and per-row actions (edit, suspend). Include a header with a search field and a "Add user" button.',
|
|
61
|
-
tags: ['data-table', 'admin', 'data-dense']
|
|
62
|
-
},
|
|
63
|
-
{
|
|
64
|
-
id: 'empty-inbox',
|
|
65
|
-
title: 'Empty Inbox State',
|
|
66
|
-
prompt:
|
|
67
|
-
'Build an empty state for an inbox that has no messages yet: an illustration or icon, a friendly headline, a one-line explanation, and a primary action to compose the first message.',
|
|
68
|
-
tags: ['empty-state', 'feedback']
|
|
69
|
-
},
|
|
70
|
-
{
|
|
71
|
-
id: 'login',
|
|
72
|
-
title: 'Login Page',
|
|
73
|
-
prompt:
|
|
74
|
-
'Build a login page with email and password fields, a "forgot password" link, a primary sign-in button, and a secondary "Sign in with a passkey" option. Show inline validation on the fields.',
|
|
75
|
-
pattern: 'form-page',
|
|
76
|
-
tags: ['auth', 'form']
|
|
77
|
-
},
|
|
78
|
-
{
|
|
79
|
-
id: 'profile-card',
|
|
80
|
-
title: 'Profile Card',
|
|
81
|
-
prompt:
|
|
82
|
-
'Build a user profile card: avatar, name, role, a short bio, three stat figures (followers, following, posts), and follow / message actions.',
|
|
83
|
-
tags: ['identity', 'card']
|
|
84
|
-
},
|
|
85
|
-
{
|
|
86
|
-
id: 'checkout-summary',
|
|
87
|
-
title: 'Checkout Summary',
|
|
88
|
-
prompt:
|
|
89
|
-
'Build a checkout order summary: a list of line items with quantity and price, a promo-code field, a subtotal/tax/total breakdown, and a sticky "Pay now" button. One item is on sale.',
|
|
90
|
-
tags: ['commerce', 'data-dense', 'sticky']
|
|
91
|
-
},
|
|
92
|
-
{
|
|
93
|
-
id: 'notification-center',
|
|
94
|
-
title: 'Notification Center',
|
|
95
|
-
prompt:
|
|
96
|
-
'Build a notification center panel: notifications grouped by Today / Earlier, each with an icon by type (mention, system, billing), a timestamp, read/unread state, and a dismiss control. Include a "Mark all as read" action.',
|
|
97
|
-
pattern: 'tab-navigation',
|
|
98
|
-
tags: ['feedback', 'list', 'state-driven']
|
|
99
|
-
}
|
|
100
|
-
];
|
|
101
|
-
|
|
102
|
-
export function getBriefById(id: string): EvalBrief | undefined {
|
|
103
|
-
return EVAL_BRIEFS.find((b) => b.id === id);
|
|
104
|
-
}
|
package/src/eval/eval.test.ts
DELETED
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
import { RUBRIC_CRITERIA } from '@urbicon-ui/design-engine/rubric';
|
|
2
|
-
import { describe, expect, it } from 'vitest';
|
|
3
|
-
import { EVAL_BRIEFS, getBriefById } from './briefs.js';
|
|
4
|
-
import type { EvalEntry } from './score.js';
|
|
5
|
-
import { aggregateRubric, formatAbReport, scoreImplementation } from './score.js';
|
|
6
|
-
|
|
7
|
-
describe('eval briefs', () => {
|
|
8
|
-
it('provides ~10 briefs with unique ids', () => {
|
|
9
|
-
expect(EVAL_BRIEFS.length).toBeGreaterThanOrEqual(10);
|
|
10
|
-
expect(new Set(EVAL_BRIEFS.map((b) => b.id)).size).toBe(EVAL_BRIEFS.length);
|
|
11
|
-
});
|
|
12
|
-
it('only references real composition patterns', () => {
|
|
13
|
-
const known = new Set([
|
|
14
|
-
'dashboard',
|
|
15
|
-
'form-page',
|
|
16
|
-
'settings-page',
|
|
17
|
-
'tab-navigation',
|
|
18
|
-
'onboarding-guide'
|
|
19
|
-
]);
|
|
20
|
-
for (const b of EVAL_BRIEFS) {
|
|
21
|
-
if (b.pattern) expect(known, `${b.id} → ${b.pattern}`).toContain(b.pattern);
|
|
22
|
-
}
|
|
23
|
-
});
|
|
24
|
-
it('looks up by id', () => {
|
|
25
|
-
expect(getBriefById('ops-dashboard')?.title).toBe('Ops Dashboard');
|
|
26
|
-
expect(getBriefById('nope')).toBeUndefined();
|
|
27
|
-
});
|
|
28
|
-
});
|
|
29
|
-
|
|
30
|
-
describe('scoreImplementation', () => {
|
|
31
|
-
it('scores clean code 100/100 with no findings', () => {
|
|
32
|
-
const s = scoreImplementation('<div class="bg-surface-base text-text-primary">ok</div>');
|
|
33
|
-
expect(s.correctness).toBe(100);
|
|
34
|
-
expect(s.slop).toBe(100);
|
|
35
|
-
expect(s.errors).toBe(0);
|
|
36
|
-
});
|
|
37
|
-
it('penalises hallucinated tokens and raw colours on the correctness axis', () => {
|
|
38
|
-
const s = scoreImplementation('<div class="bg-blue-500 text-status-bad">x</div>');
|
|
39
|
-
expect(s.correctness).toBeLessThan(100);
|
|
40
|
-
expect(s.errors + s.warnings).toBeGreaterThan(0);
|
|
41
|
-
});
|
|
42
|
-
});
|
|
43
|
-
|
|
44
|
-
describe('aggregateRubric', () => {
|
|
45
|
-
const full = Object.fromEntries(RUBRIC_CRITERIA.map((c) => [c.id, 4]));
|
|
46
|
-
|
|
47
|
-
it('sums a complete 1–5 score set', () => {
|
|
48
|
-
expect(aggregateRubric(full)).toBe(RUBRIC_CRITERIA.length * 4);
|
|
49
|
-
});
|
|
50
|
-
it('throws on a missing, out-of-range, or non-finite criterion', () => {
|
|
51
|
-
expect(() => aggregateRubric({ ...full, correctness: 7 })).toThrow();
|
|
52
|
-
expect(() => aggregateRubric({ ...full, correctness: NaN })).toThrow();
|
|
53
|
-
const { correctness, ...missing } = full;
|
|
54
|
-
void correctness;
|
|
55
|
-
expect(() => aggregateRubric(missing)).toThrow();
|
|
56
|
-
});
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
describe('formatAbReport edge cases', () => {
|
|
60
|
-
it('reports a clean-baseline error delta as "—", not a false percentage', () => {
|
|
61
|
-
const entries: EvalEntry[] = [
|
|
62
|
-
{
|
|
63
|
-
briefId: 'x',
|
|
64
|
-
condition: 'baseline',
|
|
65
|
-
score: { linter: { correctness: 100, slop: 100, errors: 0, warnings: 0, infos: 0 } }
|
|
66
|
-
},
|
|
67
|
-
{
|
|
68
|
-
briefId: 'x',
|
|
69
|
-
condition: 'design-mcp',
|
|
70
|
-
score: { linter: { correctness: 100, slop: 100, errors: 0, warnings: 0, infos: 0 } }
|
|
71
|
-
}
|
|
72
|
-
];
|
|
73
|
-
const report = formatAbReport(entries, 'baseline', 'design-mcp');
|
|
74
|
-
expect(report).not.toContain('-100.0%');
|
|
75
|
-
});
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
describe('formatAbReport', () => {
|
|
79
|
-
const entries: EvalEntry[] = [
|
|
80
|
-
{
|
|
81
|
-
briefId: 'a',
|
|
82
|
-
condition: 'baseline',
|
|
83
|
-
score: {
|
|
84
|
-
linter: { correctness: 70, slop: 80, errors: 1, warnings: 2, infos: 0 },
|
|
85
|
-
rubricTotal: 22
|
|
86
|
-
}
|
|
87
|
-
},
|
|
88
|
-
{
|
|
89
|
-
briefId: 'a',
|
|
90
|
-
condition: 'design-mcp',
|
|
91
|
-
score: {
|
|
92
|
-
linter: { correctness: 95, slop: 90, errors: 0, warnings: 0, infos: 1 },
|
|
93
|
-
rubricTotal: 31
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
];
|
|
97
|
-
const report = formatAbReport(entries, 'baseline', 'design-mcp');
|
|
98
|
-
|
|
99
|
-
it('renders per-brief and aggregate sections with a delta', () => {
|
|
100
|
-
expect(report).toContain('## Per-brief');
|
|
101
|
-
expect(report).toContain('## Aggregate');
|
|
102
|
-
expect(report).toContain('baseline');
|
|
103
|
-
expect(report).toContain('design-mcp');
|
|
104
|
-
expect(report).toContain('%'); // a percentage delta is present
|
|
105
|
-
});
|
|
106
|
-
});
|
package/src/eval/index.ts
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Public API of the eval-suite (docs/internal/DESIGN-MCP.md, cross-cutting eval). The
|
|
3
|
-
* briefs and the deterministic scorer are reusable across runs; the LLM
|
|
4
|
-
* generation and rubric judging are injected by whoever drives a run (an agent
|
|
5
|
-
* harness this round, an API loop later) so the suite stays automation-friendly.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
export type { EvalBrief } from './briefs.js';
|
|
9
|
-
export { EVAL_BRIEFS, getBriefById } from './briefs.js';
|
|
10
|
-
export type { EvalEntry, ImplementationScore, LinterScore } from './score.js';
|
|
11
|
-
export { aggregateRubric, formatAbReport, scoreImplementation } from './score.js';
|
package/src/eval/score.ts
DELETED
|
@@ -1,128 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Scoring + A/B reporting for the eval-suite. The linter half is deterministic
|
|
3
|
-
* (no LLM): `scoreImplementation` runs the design-linter and returns the score
|
|
4
|
-
* and finding counts. The rubric half is judge-supplied (1–5 per criterion);
|
|
5
|
-
* `aggregateRubric` validates and sums it. `formatAbReport` renders the
|
|
6
|
-
* baseline-vs-treatment comparison in the shape of experiment/round-2's summary
|
|
7
|
-
* tables, so a new run is directly comparable to that +33.8% baseline.
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
import { lintDesign } from '@urbicon-ui/design-engine/linter';
|
|
11
|
-
import { MAX_RUBRIC_SCORE, RUBRIC_CRITERIA } from '@urbicon-ui/design-engine/rubric';
|
|
12
|
-
|
|
13
|
-
export interface LinterScore {
|
|
14
|
-
/**
|
|
15
|
-
* Stage-1 correctness axis, 0–100 (deterministic defects only). The stable A/B
|
|
16
|
-
* headline metric: unaffected by the slop-floor heuristics, so it stays directly
|
|
17
|
-
* comparable to the pre-slop-floor baseline (the +33.8% measurement).
|
|
18
|
-
*/
|
|
19
|
-
correctness: number;
|
|
20
|
-
/** Stage-2 slop-floor axis, 0–100 (system-agnostic "looks generic" heuristics). */
|
|
21
|
-
slop: number;
|
|
22
|
-
errors: number;
|
|
23
|
-
warnings: number;
|
|
24
|
-
infos: number;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export interface ImplementationScore {
|
|
28
|
-
linter: LinterScore;
|
|
29
|
-
/** Judge rubric total /MAX_RUBRIC_SCORE, when a rubric pass was run. */
|
|
30
|
-
rubricTotal?: number;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
export interface EvalEntry {
|
|
34
|
-
briefId: string;
|
|
35
|
-
/** e.g. "baseline" or "design-mcp". */
|
|
36
|
-
condition: string;
|
|
37
|
-
score: ImplementationScore;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
/** Deterministic linter score for one generated implementation. */
|
|
41
|
-
export function scoreImplementation(code: string): LinterScore {
|
|
42
|
-
const r = lintDesign(code);
|
|
43
|
-
return {
|
|
44
|
-
correctness: r.scores.correctness,
|
|
45
|
-
slop: r.scores.slop,
|
|
46
|
-
errors: r.counts.error,
|
|
47
|
-
warnings: r.counts.warning,
|
|
48
|
-
infos: r.counts.info
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
/** Validate + sum a judge's per-criterion 1–5 scores into a /40 total. Missing/out-of-range entries throw. */
|
|
53
|
-
export function aggregateRubric(perCriterion: Record<string, number>): number {
|
|
54
|
-
let total = 0;
|
|
55
|
-
for (const c of RUBRIC_CRITERIA) {
|
|
56
|
-
const v = perCriterion[c.id];
|
|
57
|
-
if (typeof v !== 'number' || !Number.isFinite(v) || v < 1 || v > 5) {
|
|
58
|
-
throw new Error(`Rubric score for "${c.id}" must be a finite 1–5, got ${v}`);
|
|
59
|
-
}
|
|
60
|
-
total += v;
|
|
61
|
-
}
|
|
62
|
-
return total;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
function mean(xs: number[]): number {
|
|
66
|
-
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
function pct(from: number, to: number): string {
|
|
70
|
-
if (from === 0) return '—';
|
|
71
|
-
const change = ((to - from) / from) * 100;
|
|
72
|
-
return `${change >= 0 ? '+' : ''}${change.toFixed(1)}%`;
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
/**
|
|
76
|
-
* Render a baseline-vs-treatment A/B report. `baseline` and `treatment` are the
|
|
77
|
-
* condition labels present in `entries`. Per-brief rows + an aggregate summary.
|
|
78
|
-
*/
|
|
79
|
-
export function formatAbReport(entries: EvalEntry[], baseline: string, treatment: string): string {
|
|
80
|
-
const briefIds = [...new Set(entries.map((e) => e.briefId))];
|
|
81
|
-
const pick = (briefId: string, condition: string) =>
|
|
82
|
-
entries.find((e) => e.briefId === briefId && e.condition === condition)?.score;
|
|
83
|
-
|
|
84
|
-
let md = `# Eval A/B — ${baseline} vs ${treatment}\n\n`;
|
|
85
|
-
md += '## Per-brief\n\n';
|
|
86
|
-
md += `| Brief | ${baseline} correctness | ${treatment} correctness | ${baseline} rubric | ${treatment} rubric |\n`;
|
|
87
|
-
md += '|---|---|---|---|---|\n';
|
|
88
|
-
|
|
89
|
-
const baseLint: number[] = [];
|
|
90
|
-
const treatLint: number[] = [];
|
|
91
|
-
const baseSlop: number[] = [];
|
|
92
|
-
const treatSlop: number[] = [];
|
|
93
|
-
const baseRub: number[] = [];
|
|
94
|
-
const treatRub: number[] = [];
|
|
95
|
-
|
|
96
|
-
for (const id of briefIds) {
|
|
97
|
-
const b = pick(id, baseline);
|
|
98
|
-
const t = pick(id, treatment);
|
|
99
|
-
if (b) {
|
|
100
|
-
baseLint.push(b.linter.correctness);
|
|
101
|
-
baseSlop.push(b.linter.slop);
|
|
102
|
-
}
|
|
103
|
-
if (t) {
|
|
104
|
-
treatLint.push(t.linter.correctness);
|
|
105
|
-
treatSlop.push(t.linter.slop);
|
|
106
|
-
}
|
|
107
|
-
if (b?.rubricTotal !== undefined) baseRub.push(b.rubricTotal);
|
|
108
|
-
if (t?.rubricTotal !== undefined) treatRub.push(t.rubricTotal);
|
|
109
|
-
const rub = (s?: ImplementationScore) =>
|
|
110
|
-
s?.rubricTotal !== undefined ? `${s.rubricTotal}/${MAX_RUBRIC_SCORE}` : '—';
|
|
111
|
-
md += `| ${id} | ${b?.linter.correctness ?? '—'} | ${t?.linter.correctness ?? '—'} | ${rub(b)} | ${rub(t)} |\n`;
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
md += '\n## Aggregate\n\n';
|
|
115
|
-
md += `| Metric | ${baseline} | ${treatment} | Δ |\n|---|---|---|---|\n`;
|
|
116
|
-
md += `| Mean correctness | ${mean(baseLint).toFixed(1)} | ${mean(treatLint).toFixed(1)} | ${pct(mean(baseLint), mean(treatLint))} |\n`;
|
|
117
|
-
md += `| Mean slop-floor | ${mean(baseSlop).toFixed(1)} | ${mean(treatSlop).toFixed(1)} | ${pct(mean(baseSlop), mean(treatSlop))} |\n`;
|
|
118
|
-
if (baseRub.length && treatRub.length) {
|
|
119
|
-
md += `| Mean rubric /${MAX_RUBRIC_SCORE} | ${mean(baseRub).toFixed(1)} | ${mean(treatRub).toFixed(1)} | ${pct(mean(baseRub), mean(treatRub))} |\n`;
|
|
120
|
-
}
|
|
121
|
-
const totalErr = (cond: string) =>
|
|
122
|
-
entries
|
|
123
|
-
.filter((e) => e.condition === cond)
|
|
124
|
-
.reduce((a, e) => a + e.score.linter.errors + e.score.linter.warnings, 0);
|
|
125
|
-
md += `| Total linter errors+warnings | ${totalErr(baseline)} | ${totalErr(treatment)} | ${pct(totalErr(baseline), totalErr(treatment))} |\n`;
|
|
126
|
-
|
|
127
|
-
return md;
|
|
128
|
-
}
|