@superblocksteam/vite-plugin-file-sync 2.0.18-next.9 → 2.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-service/evals/helpers/critical-scorers.d.ts +10 -0
- package/dist/ai-service/evals/helpers/critical-scorers.d.ts.map +1 -0
- package/dist/ai-service/evals/helpers/critical-scorers.js +331 -0
- package/dist/ai-service/evals/helpers/critical-scorers.js.map +1 -0
- package/dist/ai-service/evals/helpers/eval-scoring.d.ts +53 -8
- package/dist/ai-service/evals/helpers/eval-scoring.d.ts.map +1 -1
- package/dist/ai-service/evals/helpers/eval-scoring.js +151 -408
- package/dist/ai-service/evals/helpers/eval-scoring.js.map +1 -1
- package/dist/ai-service/evals/helpers/index.d.ts +14 -2
- package/dist/ai-service/evals/helpers/index.d.ts.map +1 -1
- package/dist/ai-service/evals/helpers/index.js +39 -20
- package/dist/ai-service/evals/helpers/index.js.map +1 -1
- package/dist/ai-service/evals/helpers/non-critical-scorers.d.ts +6 -0
- package/dist/ai-service/evals/helpers/non-critical-scorers.d.ts.map +1 -0
- package/dist/ai-service/evals/helpers/non-critical-scorers.js +187 -0
- package/dist/ai-service/evals/helpers/non-critical-scorers.js.map +1 -0
- package/dist/ai-service/evals/helpers/scorer-types.d.ts +39 -0
- package/dist/ai-service/evals/helpers/scorer-types.d.ts.map +1 -0
- package/dist/ai-service/evals/helpers/scorer-types.js +19 -0
- package/dist/ai-service/evals/helpers/scorer-types.js.map +1 -0
- package/dist/ai-service/index.d.ts.map +1 -1
- package/dist/ai-service/index.js +8 -3
- package/dist/ai-service/index.js.map +1 -1
- package/dist/ai-service/llm/impl/anthropic.js +3 -3
- package/dist/ai-service/llm/impl/anthropic.js.map +1 -1
- package/dist/ai-service/llm/provider.js +3 -3
- package/dist/ai-service/llm/provider.js.map +1 -1
- package/dist/ai-service/llm/types.d.ts +1 -1
- package/dist/ai-service/llm/types.d.ts.map +1 -1
- package/dist/ai-service/llm/utils.js +6 -6
- package/dist/ai-service/llm/utils.js.map +1 -1
- package/dist/ai-service/prompt-builder-service/builders/code-generation.d.ts.map +1 -1
- package/dist/ai-service/prompt-builder-service/builders/code-generation.js +62 -87
- package/dist/ai-service/prompt-builder-service/builders/code-generation.js.map +1 -1
- package/dist/ai-service/prompt-builder-service/builders/specific-edits.d.ts +13 -0
- package/dist/ai-service/prompt-builder-service/builders/specific-edits.d.ts.map +1 -0
- package/dist/ai-service/prompt-builder-service/builders/specific-edits.js +71 -0
- package/dist/ai-service/prompt-builder-service/builders/specific-edits.js.map +1 -0
- package/dist/ai-service/prompt-builder-service/classifiers/prompt-interpret-task.d.ts +0 -1
- package/dist/ai-service/prompt-builder-service/classifiers/prompt-interpret-task.d.ts.map +1 -1
- package/dist/ai-service/prompt-builder-service/classifiers/prompt-interpret-task.js +4 -13
- package/dist/ai-service/prompt-builder-service/classifiers/prompt-interpret-task.js.map +1 -1
- package/dist/ai-service/prompt-builder-service/fragment-generators/base-fragment.d.ts +1 -1
- package/dist/ai-service/prompt-builder-service/fragment-generators/base-fragment.d.ts.map +1 -1
- package/dist/ai-service/prompt-builder-service/fragment-generators/base-fragment.js +2 -2
- package/dist/ai-service/prompt-builder-service/fragment-generators/base-fragment.js.map +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/ButtonPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/CheckboxPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/ColumnPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/ContainerPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/DatePickerPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/DropdownPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/IconPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/ImagePropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/InputPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/ModalPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/PagePropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/SectionPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/SlideoutPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/SwitchPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/TablePropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-components/TextPropsDocs.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-typedefs/Dim.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-typedefs/EventFlow.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/library-typedefs/TextStyleWithVariant.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/full-examples.d.ts +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/full-examples.d.ts.map +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/full-examples.js +2 -2
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/full-examples.js.map +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/index.d.ts +1 -0
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/index.d.ts.map +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/index.js +1 -0
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/index.js.map +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-api.d.ts +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-api.d.ts.map +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-api.js +2 -2
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-api.js.map +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-components-rules.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-custom-components.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-data-filtering.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-event-flow.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-forms.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-layouts.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-page.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-rbac.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-routes.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-state.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/superblocks-theming.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/system-base.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/system-incremental.js +1 -1
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/system-specific-edit.d.ts +2 -0
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/system-specific-edit.d.ts.map +1 -0
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/system-specific-edit.js +6 -0
- package/dist/ai-service/prompt-builder-service/static-fragments/platform-parts/system-specific-edit.js.map +1 -0
- package/dist/ai-service/prompt-builder-service/types.d.ts +3 -3
- package/dist/ai-service/prompt-builder-service/types.js +3 -3
- package/dist/ai-service/prompt-builder-service/types.js.map +1 -1
- package/dist/ai-service/state-machine/handlers/agent-planning.d.ts.map +1 -1
- package/dist/ai-service/state-machine/handlers/agent-planning.js +8 -6
- package/dist/ai-service/state-machine/handlers/agent-planning.js.map +1 -1
- package/dist/ai-service/state-machine/handlers/post-processing.js +5 -5
- package/dist/ai-service/state-machine/handlers/post-processing.js.map +1 -1
- package/dist/ai-service/test-utils/app-generation-mocks/orders-app.d.ts +1 -1
- package/dist/ai-service/test-utils/app-generation-mocks/orders-app.d.ts.map +1 -1
- package/dist/ai-service/test-utils/app-generation-mocks/orders-app.js +80 -38
- package/dist/ai-service/test-utils/app-generation-mocks/orders-app.js.map +1 -1
- package/dist/ai-service/test-utils/app-generation-mocks/smoketest.d.ts +1 -1
- package/dist/ai-service/test-utils/app-generation-mocks/smoketest.d.ts.map +1 -1
- package/dist/ai-service/test-utils/app-generation-mocks/smoketest.js +9 -3
- package/dist/ai-service/test-utils/app-generation-mocks/smoketest.js.map +1 -1
- package/dist/components-manager.d.ts +1 -1
- package/dist/components-manager.d.ts.map +1 -1
- package/dist/components-manager.js +81 -18
- package/dist/components-manager.js.map +1 -1
- package/dist/file-sync-vite-plugin.d.ts.map +1 -1
- package/dist/file-sync-vite-plugin.js +15 -15
- package/dist/file-sync-vite-plugin.js.map +1 -1
- package/dist/file-system-manager.d.ts.map +1 -1
- package/dist/file-system-manager.js +2 -1
- package/dist/file-system-manager.js.map +1 -1
- package/dist/refactor/javascript.d.ts.map +1 -1
- package/dist/refactor/javascript.js +19 -2
- package/dist/refactor/javascript.js.map +1 -1
- package/dist/rename-manager.d.ts +2 -1
- package/dist/rename-manager.d.ts.map +1 -1
- package/dist/rename-manager.js +8 -6
- package/dist/rename-manager.js.map +1 -1
- package/dist/sb-scope-manager.d.ts.map +1 -1
- package/dist/sb-scope-manager.js +19 -0
- package/dist/sb-scope-manager.js.map +1 -1
- package/dist/source-tracker.d.ts +2 -1
- package/dist/source-tracker.d.ts.map +1 -1
- package/dist/source-tracker.js +2 -1
- package/dist/source-tracker.js.map +1 -1
- package/dist/virtual-modules.d.ts +5 -0
- package/dist/virtual-modules.d.ts.map +1 -0
- package/dist/virtual-modules.js +5 -0
- package/dist/virtual-modules.js.map +1 -0
- package/package.json +6 -6
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { ScorerContext, ScorerResult } from "./scorer-types.js";
|
|
2
|
+
export declare function scoreExpectedPageCount(context: ScorerContext): ScorerResult;
|
|
3
|
+
export declare function scoreExpectedScopeCount(context: ScorerContext): ScorerResult;
|
|
4
|
+
export declare function scoreUniqueApiNames(context: ScorerContext): ScorerResult;
|
|
5
|
+
export declare function scoreApisInCorrectLocation(context: ScorerContext): ScorerResult;
|
|
6
|
+
export declare function scoreExpectedCustomComponentCount(context: ScorerContext): ScorerResult;
|
|
7
|
+
export declare function scoreRequiredUiComponentsInPage(context: ScorerContext): ScorerResult;
|
|
8
|
+
export declare function scoreRequiredUiComponentProperties(context: ScorerContext): ScorerResult;
|
|
9
|
+
export declare function scoreExpectedComponentArtifacts(context: ScorerContext): ScorerResult;
|
|
10
|
+
//# sourceMappingURL=critical-scorers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"critical-scorers.d.ts","sourceRoot":"","sources":["../../../../src/ai-service/evals/helpers/critical-scorers.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAKrE,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CA0B3E;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CA0B5E;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAcxE;AAED,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,aAAa,GACrB,YAAY,CAqCd;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,aAAa,GACrB,YAAY,CAuBd;AAED,wBAAgB,+BAA+B,CAC7C,OAAO,EAAE,aAAa,GACrB,YAAY,CA0Cd;AAiBD,wBAAgB,kCAAkC,CAChD,OAAO,EAAE,aAAa,GACrB,YAAY,CA8Ed;AAED,wBAAgB,+BAA+B,CAC7C,OAAO,EAAE,aAAa,GACrB,YAAY,CA0Gd"}
|
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
import { expect } from "vitest";
|
|
2
|
+
import { findComponentAtPath, findComponentsInPage, parseScopeEntityNames, } from "../content-matchers/index.js";
|
|
3
|
+
import { SCORING_CONFIG } from "./scorer-types.js";
|
|
4
|
+
// Critical scorer functions - these result in -100 point penalties when they fail
|
|
5
|
+
// ------------------------
|
|
6
|
+
export function scoreExpectedPageCount(context) {
|
|
7
|
+
const expectedPageCount = context.config.expectedPageCount !== undefined
|
|
8
|
+
? context.config.expectedPageCount
|
|
9
|
+
: 1;
|
|
10
|
+
const passed = context.pages.length === expectedPageCount &&
|
|
11
|
+
(expectedPageCount === 0
|
|
12
|
+
? true
|
|
13
|
+
: expectedPageCount > 1
|
|
14
|
+
? context.pages.every((page, i) => page.params.pageName === `Page${i + 1}`)
|
|
15
|
+
: context.pages[0]?.params.pageName === "Page1");
|
|
16
|
+
return {
|
|
17
|
+
checkName: "hasExpectedPageCount",
|
|
18
|
+
passed,
|
|
19
|
+
scoreDelta: passed ? 0 : SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
20
|
+
isCritical: true,
|
|
21
|
+
metadata: {
|
|
22
|
+
expectedPageCount,
|
|
23
|
+
actualPageCount: context.pages.length,
|
|
24
|
+
pageNames: context.pages.map((p) => p.params.pageName),
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
export function scoreExpectedScopeCount(context) {
|
|
29
|
+
const expectedScopeCount = context.config.expectedScopeCount !== undefined
|
|
30
|
+
? context.config.expectedScopeCount
|
|
31
|
+
: 1;
|
|
32
|
+
const passed = context.scopes.length === expectedScopeCount &&
|
|
33
|
+
(expectedScopeCount === 0
|
|
34
|
+
? true
|
|
35
|
+
: expectedScopeCount > 1
|
|
36
|
+
? context.scopes.every((scope, i) => scope.params.pageName === `Page${i + 1}`)
|
|
37
|
+
: context.scopes[0]?.params.pageName === "Page1");
|
|
38
|
+
return {
|
|
39
|
+
checkName: "hasExpectedScopeCount",
|
|
40
|
+
passed,
|
|
41
|
+
scoreDelta: passed ? 0 : SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
42
|
+
isCritical: true,
|
|
43
|
+
metadata: {
|
|
44
|
+
expectedScopeCount,
|
|
45
|
+
actualScopeCount: context.scopes.length,
|
|
46
|
+
scopeNames: context.scopes.map((s) => s.params.pageName),
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
export function scoreUniqueApiNames(context) {
|
|
51
|
+
const passed = new Set(context.apiNames).size === context.apiNames.length;
|
|
52
|
+
return {
|
|
53
|
+
checkName: "hasUniqueApiNames",
|
|
54
|
+
passed,
|
|
55
|
+
scoreDelta: passed ? 0 : SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
56
|
+
isCritical: true,
|
|
57
|
+
metadata: {
|
|
58
|
+
apiNames: context.apiNames,
|
|
59
|
+
uniqueCount: new Set(context.apiNames).size,
|
|
60
|
+
totalCount: context.apiNames.length,
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
export function scoreApisInCorrectLocation(context) {
|
|
65
|
+
// Check if all APIs are in Page1
|
|
66
|
+
for (const api of context.apis) {
|
|
67
|
+
if (api.params.pageName !== "Page1") {
|
|
68
|
+
return {
|
|
69
|
+
checkName: "hasApisInCorrectLocation",
|
|
70
|
+
passed: false,
|
|
71
|
+
scoreDelta: SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
72
|
+
isCritical: true,
|
|
73
|
+
metadata: { incorrectApiLocation: api.params.pageName },
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
// Check if all API names are in the scope
|
|
78
|
+
if (context.scopes[0]) {
|
|
79
|
+
const scopeEntityNames = parseScopeEntityNames(context.scopes[0]);
|
|
80
|
+
for (const apiName of context.apiNames) {
|
|
81
|
+
if (!scopeEntityNames.includes(apiName)) {
|
|
82
|
+
return {
|
|
83
|
+
checkName: "hasApisInCorrectLocation",
|
|
84
|
+
passed: false,
|
|
85
|
+
scoreDelta: SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
86
|
+
isCritical: true,
|
|
87
|
+
metadata: { missingApiInScope: apiName, scopeEntityNames },
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return {
|
|
93
|
+
checkName: "hasApisInCorrectLocation",
|
|
94
|
+
passed: true,
|
|
95
|
+
scoreDelta: 0,
|
|
96
|
+
isCritical: true,
|
|
97
|
+
metadata: { allApisInCorrectLocation: true },
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
export function scoreExpectedCustomComponentCount(context) {
|
|
101
|
+
const expectedCount = context.config.expectedCustomComponentCount;
|
|
102
|
+
let passed = false;
|
|
103
|
+
if (typeof expectedCount === "number") {
|
|
104
|
+
passed = context.customComponents.length === expectedCount;
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
const [min, max] = expectedCount;
|
|
108
|
+
passed =
|
|
109
|
+
context.customComponents.length >= min &&
|
|
110
|
+
context.customComponents.length <= max;
|
|
111
|
+
}
|
|
112
|
+
return {
|
|
113
|
+
checkName: "hasExpectedCustomComponentCount",
|
|
114
|
+
passed,
|
|
115
|
+
scoreDelta: passed ? 0 : SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
116
|
+
isCritical: true,
|
|
117
|
+
metadata: {
|
|
118
|
+
expectedCount,
|
|
119
|
+
actualCount: context.customComponents.length,
|
|
120
|
+
},
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
export function scoreRequiredUiComponentsInPage(context) {
|
|
124
|
+
for (const uiComponent of context.config.requiredUiComponents ?? []) {
|
|
125
|
+
const foundComponents = findComponentsInPage(context.pageContent, uiComponent.type);
|
|
126
|
+
let componentCheckPassed = false;
|
|
127
|
+
if (typeof uiComponent.expectedCount === "number") {
|
|
128
|
+
if (uiComponent.expectedCount === 0) {
|
|
129
|
+
componentCheckPassed = foundComponents.length === 0;
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
componentCheckPassed = foundComponents.length > 0;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
const [min, max] = uiComponent.expectedCount;
|
|
137
|
+
componentCheckPassed =
|
|
138
|
+
foundComponents.length >= min && foundComponents.length <= max;
|
|
139
|
+
}
|
|
140
|
+
if (!componentCheckPassed) {
|
|
141
|
+
return {
|
|
142
|
+
checkName: "hasExpectedUiComponentsInPage",
|
|
143
|
+
passed: false,
|
|
144
|
+
scoreDelta: SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
145
|
+
isCritical: true,
|
|
146
|
+
metadata: {
|
|
147
|
+
failedComponent: uiComponent.type,
|
|
148
|
+
expectedCount: uiComponent.expectedCount,
|
|
149
|
+
foundCount: foundComponents.length,
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
checkName: "hasExpectedUiComponentsInPage",
|
|
156
|
+
passed: true,
|
|
157
|
+
scoreDelta: 0,
|
|
158
|
+
isCritical: true,
|
|
159
|
+
metadata: { allRequiredComponentsFound: true },
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
function getComponentForPropertyCheck(foundComponents, config) {
|
|
163
|
+
if (config.path) {
|
|
164
|
+
return findComponentAtPath(foundComponents, config.path);
|
|
165
|
+
}
|
|
166
|
+
const index = config.index ?? 0;
|
|
167
|
+
return foundComponents[index];
|
|
168
|
+
}
|
|
169
|
+
export function scoreRequiredUiComponentProperties(context) {
|
|
170
|
+
for (const propertyCheck of context.config.requiredUiComponentProperties ??
|
|
171
|
+
[]) {
|
|
172
|
+
const foundComponents = findComponentsInPage(context.pageContent, propertyCheck.type);
|
|
173
|
+
const component = getComponentForPropertyCheck(foundComponents, propertyCheck);
|
|
174
|
+
if (!component) {
|
|
175
|
+
return {
|
|
176
|
+
checkName: "hasExpectedUiComponentProperties",
|
|
177
|
+
passed: false,
|
|
178
|
+
scoreDelta: SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
179
|
+
isCritical: true,
|
|
180
|
+
metadata: {
|
|
181
|
+
failedPropertyCheck: propertyCheck.type,
|
|
182
|
+
reason: "componentNotFound",
|
|
183
|
+
},
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
// Check expected properties
|
|
187
|
+
if (propertyCheck.expectedProperties) {
|
|
188
|
+
for (const [key, value] of Object.entries(propertyCheck.expectedProperties)) {
|
|
189
|
+
try {
|
|
190
|
+
expect(component.props).toHaveProperty(key, value);
|
|
191
|
+
}
|
|
192
|
+
catch {
|
|
193
|
+
return {
|
|
194
|
+
checkName: "hasExpectedUiComponentProperties",
|
|
195
|
+
passed: false,
|
|
196
|
+
scoreDelta: SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
197
|
+
isCritical: true,
|
|
198
|
+
metadata: {
|
|
199
|
+
failedPropertyCheck: propertyCheck.type,
|
|
200
|
+
reason: "propertyMismatch",
|
|
201
|
+
property: key,
|
|
202
|
+
expected: value,
|
|
203
|
+
actual: component.props[key],
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
// Check expected undefined properties
|
|
210
|
+
if (propertyCheck.expectedUndefinedProperties) {
|
|
211
|
+
for (const key of propertyCheck.expectedUndefinedProperties) {
|
|
212
|
+
if (component.props[key] !== undefined) {
|
|
213
|
+
return {
|
|
214
|
+
checkName: "hasExpectedUiComponentProperties",
|
|
215
|
+
passed: false,
|
|
216
|
+
scoreDelta: SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
217
|
+
isCritical: true,
|
|
218
|
+
metadata: {
|
|
219
|
+
failedPropertyCheck: propertyCheck.type,
|
|
220
|
+
reason: "propertyNotUndefined",
|
|
221
|
+
property: key,
|
|
222
|
+
actual: component.props[key],
|
|
223
|
+
},
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return {
|
|
230
|
+
checkName: "hasExpectedUiComponentProperties",
|
|
231
|
+
passed: true,
|
|
232
|
+
scoreDelta: 0,
|
|
233
|
+
isCritical: true,
|
|
234
|
+
metadata: { allRequiredPropertiesValid: true },
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
export function scoreExpectedComponentArtifacts(context) {
|
|
238
|
+
for (const expectedArtifact of context.config.expectedComponentArtifacts ??
|
|
239
|
+
[]) {
|
|
240
|
+
const matchedArtifact = context.components.find((component) => component.filePath.includes(expectedArtifact.filePath));
|
|
241
|
+
if (!matchedArtifact) {
|
|
242
|
+
return {
|
|
243
|
+
checkName: "hasExpectedComponentArtifacts",
|
|
244
|
+
passed: false,
|
|
245
|
+
scoreDelta: SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
246
|
+
isCritical: true,
|
|
247
|
+
metadata: {
|
|
248
|
+
missingArtifact: expectedArtifact.filePath,
|
|
249
|
+
},
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
// Check if the expected tag name is found in the component
|
|
253
|
+
const foundComponent = findComponentsInPage(matchedArtifact.content, expectedArtifact.tagName)[0];
|
|
254
|
+
if (!foundComponent) {
|
|
255
|
+
return {
|
|
256
|
+
checkName: "hasExpectedComponentArtifacts",
|
|
257
|
+
passed: false,
|
|
258
|
+
scoreDelta: SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
259
|
+
isCritical: true,
|
|
260
|
+
metadata: {
|
|
261
|
+
artifactPath: expectedArtifact.filePath,
|
|
262
|
+
missingTagName: expectedArtifact.tagName,
|
|
263
|
+
},
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
// Check bind name if specified
|
|
267
|
+
if (expectedArtifact.bindName) {
|
|
268
|
+
if (foundComponent.props.bind !== `{${expectedArtifact.bindName}}`) {
|
|
269
|
+
return {
|
|
270
|
+
checkName: "hasExpectedComponentArtifacts",
|
|
271
|
+
passed: false,
|
|
272
|
+
scoreDelta: SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
273
|
+
isCritical: true,
|
|
274
|
+
metadata: {
|
|
275
|
+
artifactPath: expectedArtifact.filePath,
|
|
276
|
+
expectedBindName: expectedArtifact.bindName,
|
|
277
|
+
actualBindName: foundComponent.props.bindName,
|
|
278
|
+
},
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
// Check expected properties
|
|
283
|
+
if (expectedArtifact.expectedProperties) {
|
|
284
|
+
for (const [key, value] of Object.entries(expectedArtifact.expectedProperties)) {
|
|
285
|
+
try {
|
|
286
|
+
expect(foundComponent.props).toHaveProperty(key, value);
|
|
287
|
+
}
|
|
288
|
+
catch {
|
|
289
|
+
return {
|
|
290
|
+
checkName: "hasExpectedComponentArtifacts",
|
|
291
|
+
passed: false,
|
|
292
|
+
scoreDelta: SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
293
|
+
isCritical: true,
|
|
294
|
+
metadata: {
|
|
295
|
+
artifactPath: expectedArtifact.filePath,
|
|
296
|
+
propertyMismatch: key,
|
|
297
|
+
expected: value,
|
|
298
|
+
actual: foundComponent.props[key],
|
|
299
|
+
},
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
// Check expected undefined properties
|
|
305
|
+
if (expectedArtifact.expectedUndefinedProperties) {
|
|
306
|
+
for (const key of expectedArtifact.expectedUndefinedProperties) {
|
|
307
|
+
if (foundComponent.props[key] !== undefined) {
|
|
308
|
+
return {
|
|
309
|
+
checkName: "hasExpectedComponentArtifacts",
|
|
310
|
+
passed: false,
|
|
311
|
+
scoreDelta: SCORING_CONFIG.CRITICAL_FAILURE_PENALTY,
|
|
312
|
+
isCritical: true,
|
|
313
|
+
metadata: {
|
|
314
|
+
artifactPath: expectedArtifact.filePath,
|
|
315
|
+
propertyNotUndefined: key,
|
|
316
|
+
actual: foundComponent.props[key],
|
|
317
|
+
},
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
return {
|
|
324
|
+
checkName: "hasExpectedComponentArtifacts",
|
|
325
|
+
passed: true,
|
|
326
|
+
scoreDelta: 0,
|
|
327
|
+
isCritical: true,
|
|
328
|
+
metadata: { allExpectedArtifactsFound: true },
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
//# sourceMappingURL=critical-scorers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"critical-scorers.js","sourceRoot":"","sources":["../../../../src/ai-service/evals/helpers/critical-scorers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,qBAAqB,GACtB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGnD,kFAAkF;AAClF,2BAA2B;AAE3B,MAAM,UAAU,sBAAsB,CAAC,OAAsB;IAC3D,MAAM,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,iBAAiB,KAAK,SAAS;QAC5C,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,iBAAiB;QAClC,CAAC,CAAC,CAAC,CAAC;IACR,MAAM,MAAM,GACV,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,iBAAiB;QAC1C,CAAC,iBAAiB,KAAK,CAAC;YACtB,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,iBAAiB,GAAG,CAAC;gBACrB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CACjB,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,OAAO,CAAC,GAAG,CAAC,EAAE,CACrD;gBACH,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC;IAEvD,OAAO;QACL,SAAS,EAAE,sBAAsB;QACjC,MAAM;QACN,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,wBAAwB;QAChE,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE;YACR,iBAAiB;YACjB,eAAe,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM;YACrC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;SACvD;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,OAAsB;IAC5D,MAAM,kBAAkB,GACtB,OAAO,CAAC,MAAM,CAAC,kBAAkB,KAAK,SAAS;QAC7C,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB;QACnC,CAAC,CAAC,CAAC,CAAC;IACR,MAAM,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,kBAAkB;QAC5C,CAAC,kBAAkB,KAAK,CAAC;YACvB,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,kBAAkB,GAAG,CAAC;gBACtB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,KAAK,OAAO,CAAC,GAAG,CAAC,EAAE,CACvD;gBACH,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC;IAExD,OAAO;QACL,SAAS,EAAE,uBAAuB;QAClC,MAAM;QACN,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,wBAAwB;QAChE,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE;YACR,kBAAkB;YAClB,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;YACvC,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;SACzD;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAAsB;IACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAE1E,OAAO;QACL,SAAS,EAAE,mBAAmB;QAC9B,MAAM;QACN,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,wBAAwB;QAChE,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE;YACR,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,WAAW,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI;YAC3C,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;SACpC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,OAAsB;IAEtB,iCAAiC;IACjC,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACpC,OAAO;gBACL,SAAS,EAAE,0BAA0B;gBACrC,MAAM,EAAE,KAAK;gBACb,UAAU,EAAE,cAAc,CAAC,wBAAwB;gBACnD,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,EAAE,oBAAoB,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE;aACxD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,0CAA0C;IAC1C,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtB,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACxC,OAAO;oBACL,SAAS,EAAE,0BAA0B;oBACrC,MAAM,EAAE,KAAK;oBACb,UAAU,EAAE,cAAc,CAAC,wBAAwB;oBACnD,UAAU,EAAE,IAAI;oBAChB,QAAQ,EAAE,EAAE,iBAAiB,EAAE,OAAO,EAAE,gBAAgB,EAAE;iBAC3D,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,SAAS,EAAE,0BAA0B;QACrC,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,EAAE,wBAAwB,EAAE,IAAI,EAAE;KAC7C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAC/C,OAAsB;IAEtB,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,4BAA4B,CAAC;IAClE,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,MAAM,KAAK,aAAa,CAAC;IAC7D,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,aAAa,CAAC;QACjC,MAAM;YACJ,OAAO,CAAC,gBAAgB,CAAC,MAAM,IAAI,GAAG;gBACtC,OAAO,CAAC,gBAAgB,CAAC,MAAM,IAAI,GAAG,CAAC;IAC3C,CAAC;IAED,OAAO;QACL,SAAS,EAAE,iCAAiC;QAC5C,MAAM;QACN,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,wBAAwB;QAChE,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE;YACR,aAAa;YACb,WAAW,EAAE,OAAO,CAAC,gBAAgB,CAAC,MAAM;SAC7C;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,OAAsB;IAEtB,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC,oBAAoB,IAAI,EAAE,EAAE,CAAC;QACpE,MAAM,eAAe,GAAG,oBAAoB,CAC1C,OAAO,CAAC,WAAW,EACnB,WAAW,CAAC,IAAI,CACjB,CAAC;QACF,IAAI,oBAAoB,GAAG,KAAK,CAAC;QAEjC,IAAI,OAAO,WAAW,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;YAClD,IAAI,WAAW,CAAC,aAAa,KAAK,CAAC,EAAE,CAAC;gBACpC,oBAAoB,GAAG,eAAe,CAAC,MAAM,KAAK,CAAC,CAAC;YACtD,CAAC;iBAAM,CAAC;gBACN,oBAAoB,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,WAAW,CAAC,aAAa,CAAC;YAC7C,oBAAoB;gBAClB,eAAe,CAAC,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,MAAM,IAAI,GAAG,CAAC;QACnE,CAAC;QAED,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC1B,OAAO;gBACL,SAAS,EAAE,+BAA+B;gBAC1C,MAAM,EAAE,KAAK;gBACb,UAAU,EAAE,cAAc,CAAC,wBAAwB;gBACnD,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE;oBACR,eAAe,EAAE,WAAW,CAAC,IAAI;oBACjC,aAAa,EAAE,WAAW,CAAC,aAAa;oBACxC,UAAU,EAAE,eAAe,CAAC,MAAM;iBACnC;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,SAAS,EAAE,+BAA+B;QAC1C,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,EAAE,0BAA0B,EAAE,IAAI,EAAE;KAC/C,CAAC;AACJ,CAAC;AAED,SAAS,4BAA4B,CACnC,eAAwD,EACxD,MAGC;IAED,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,mBAAmB,CAAC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;IAChC,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kCAAkC,CAChD,OAAsB;IAEtB,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,6BAA6B;QACtE,EAAE,EAAE,CAAC;QACL,MAAM,eAAe,GAAG,oBAAoB,CAC1C,OAAO,CAAC,WAAW,EACnB,aAAa,CAAC,IAAI,CACnB,CAAC;QACF,MAAM,SAAS,GAAG,4BAA4B,CAC5C,eAAe,EACf,aAAa,CACd,CAAC;QAEF,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO;gBACL,SAAS,EAAE,kCAAkC;gBAC7C,MAAM,EAAE,KAAK;gBACb,UAAU,EAAE,cAAc,CAAC,wBAAwB;gBACnD,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE;oBACR,mBAAmB,EAAE,aAAa,CAAC,IAAI;oBACvC,MAAM,EAAE,mBAAmB;iBAC5B;aACF,CAAC;QACJ,CAAC;QAED,4BAA4B;QAC5B,IAAI,aAAa,CAAC,kBAAkB,EAAE,CAAC;YACrC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CACvC,aAAa,CAAC,kBAAkB,CACjC,EAAE,CAAC;gBACF,IAAI,CAAC;oBACH,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBACrD,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;wBACL,SAAS,EAAE,kCAAkC;wBAC7C,MAAM,EAAE,KAAK;wBACb,UAAU,EAAE,cAAc,CAAC,wBAAwB;wBACnD,UAAU,EAAE,IAAI;wBAChB,QAAQ,EAAE;4BACR,mBAAmB,EAAE,aAAa,CAAC,IAAI;4BACvC,MAAM,EAAE,kBAAkB;4BAC1B,QAAQ,EAAE,GAAG;4BACb,QAAQ,EAAE,KAAK;4BACf,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC;yBAC7B;qBACF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,IAAI,aAAa,CAAC,2BAA2B,EAAE,CAAC;YAC9C,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,2BAA2B,EAAE,CAAC;gBAC5D,IAAI,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;oBACvC,OAAO;wBACL,SAAS,EAAE,kCAAkC;wBAC7C,MAAM,EAAE,KAAK;wBACb,UAAU,EAAE,cAAc,CAAC,wBAAwB;wBACnD,UAAU,EAAE,IAAI;wBAChB,QAAQ,EAAE;4BACR,mBAAmB,EAAE,aAAa,CAAC,IAAI;4BACvC,MAAM,EAAE,sBAAsB;4BAC9B,QAAQ,EAAE,GAAG;4BACb,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC;yBAC7B;qBACF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,SAAS,EAAE,kCAAkC;QAC7C,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,EAAE,0BAA0B,EAAE,IAAI,EAAE;KAC/C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,OAAsB;IAEtB,KAAK,MAAM,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,0BAA0B;QACtE,EAAE,EAAE,CAAC;QACL,MAAM,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAC5D,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CACvD,CAAC;QAEF,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO;gBACL,SAAS,EAAE,+BAA+B;gBAC1C,MAAM,EAAE,KAAK;gBACb,UAAU,EAAE,cAAc,CAAC,wBAAwB;gBACnD,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE;oBACR,eAAe,EAAE,gBAAgB,CAAC,QAAQ;iBAC3C;aACF,CAAC;QACJ,CAAC;QAED,2DAA2D;QAC3D,MAAM,cAAc,GAAG,oBAAoB,CACzC,eAAe,CAAC,OAAO,EACvB,gBAAgB,CAAC,OAAO,CACzB,CAAC,CAAC,CAAC,CAAC;QAEL,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,OAAO;gBACL,SAAS,EAAE,+BAA+B;gBAC1C,MAAM,EAAE,KAAK;gBACb,UAAU,EAAE,cAAc,CAAC,wBAAwB;gBACnD,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE;oBACR,YAAY,EAAE,gBAAgB,CAAC,QAAQ;oBACvC,cAAc,EAAE,gBAAgB,CAAC,OAAO;iBACzC;aACF,CAAC;QACJ,CAAC;QAED,+BAA+B;QAC/B,IAAI,gBAAgB,CAAC,QAAQ,EAAE,CAAC;YAC9B,IAAI,cAAc,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,gBAAgB,CAAC,QAAQ,GAAG,EAAE,CAAC;gBACnE,OAAO;oBACL,SAAS,EAAE,+BAA+B;oBAC1C,MAAM,EAAE,KAAK;oBACb,UAAU,EAAE,cAAc,CAAC,wBAAwB;oBACnD,UAAU,EAAE,IAAI;oBAChB,QAAQ,EAAE;wBACR,YAAY,EAAE,gBAAgB,CAAC,QAAQ;wBACvC,gBAAgB,EAAE,gBAAgB,CAAC,QAAQ;wBAC3C,cAAc,EAAE,cAAc,CAAC,KAAK,CAAC,QAAQ;qBAC9C;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,4BAA4B;QAC5B,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CACvC,gBAAgB,CAAC,kBAAkB,CACpC,EAAE,CAAC;gBACF,IAAI,CAAC;oBACH,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC1D,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;wBACL,SAAS,EAAE,+BAA+B;wBAC1C,MAAM,EAAE,KAAK;wBACb,UAAU,EAAE,cAAc,CAAC,wBAAwB;wBACnD,UAAU,EAAE,IAAI;wBAChB,QAAQ,EAAE;4BACR,YAAY,EAAE,gBAAgB,CAAC,QAAQ;4BACvC,gBAAgB,EAAE,GAAG;4BACrB,QAAQ,EAAE,KAAK;4BACf,MAAM,EAAE,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC;yBAClC;qBACF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,IAAI,gBAAgB,CAAC,2BAA2B,EAAE,CAAC;YACjD,KAAK,MAAM,GAAG,IAAI,gBAAgB,CAAC,2BAA2B,EAAE,CAAC;gBAC/D,IAAI,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;oBAC5C,OAAO;wBACL,SAAS,EAAE,+BAA+B;wBAC1C,MAAM,EAAE,KAAK;wBACb,UAAU,EAAE,cAAc,CAAC,wBAAwB;wBACnD,UAAU,EAAE,IAAI;wBAChB,QAAQ,EAAE;4BACR,YAAY,EAAE,gBAAgB,CAAC,QAAQ;4BACvC,oBAAoB,EAAE,GAAG;4BACzB,MAAM,EAAE,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC;yBAClC;qBACF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,SAAS,EAAE,+BAA+B;QAC1C,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,EAAE,yBAAyB,EAAE,IAAI,EAAE;KAC9C,CAAC;AACJ,CAAC"}
|
|
@@ -20,10 +20,20 @@ export interface EvalScoreConfig {
|
|
|
20
20
|
expectedScopeCount?: number;
|
|
21
21
|
expectedApiCountRange: number | [number, number];
|
|
22
22
|
expectedCustomComponentCount: number | [number, number];
|
|
23
|
+
requiredUiComponents?: Array<{
|
|
24
|
+
type: string;
|
|
25
|
+
expectedCount: number | [number, number];
|
|
26
|
+
}>;
|
|
23
27
|
expectedUiComponents?: Array<{
|
|
24
28
|
type: string;
|
|
25
29
|
expectedCount: number | [number, number];
|
|
26
|
-
|
|
30
|
+
}>;
|
|
31
|
+
requiredUiComponentProperties?: Array<{
|
|
32
|
+
type: string;
|
|
33
|
+
path?: string[];
|
|
34
|
+
index?: number;
|
|
35
|
+
expectedProperties?: Record<string, any>;
|
|
36
|
+
expectedUndefinedProperties?: string[];
|
|
27
37
|
}>;
|
|
28
38
|
expectedUiComponentProperties?: Array<{
|
|
29
39
|
type: string;
|
|
@@ -40,20 +50,55 @@ export interface EvalScoreConfig {
|
|
|
40
50
|
expectedUndefinedProperties?: string[];
|
|
41
51
|
}>;
|
|
42
52
|
}
|
|
43
|
-
export declare function calculateEvalScore({ artifacts, getTempDir, config,
|
|
53
|
+
export declare function calculateEvalScore({ artifacts, getTempDir, config, classification, }: {
|
|
44
54
|
artifacts: ReturnType<typeof extractArtifacts>;
|
|
45
55
|
getTempDir: () => string;
|
|
46
56
|
config: EvalScoreConfig;
|
|
47
|
-
|
|
48
|
-
inputData: any;
|
|
49
|
-
outputData: any;
|
|
50
|
-
metrics: any;
|
|
51
|
-
metadata: any;
|
|
52
|
-
};
|
|
57
|
+
classification?: string;
|
|
53
58
|
}): Promise<{
|
|
54
59
|
evalScore: number;
|
|
55
60
|
evalPassed: boolean;
|
|
56
61
|
evalChecks: EvalChecks;
|
|
62
|
+
deductionBreakdown: {
|
|
63
|
+
criticalFailures: Array<{
|
|
64
|
+
check: string;
|
|
65
|
+
deduction: number;
|
|
66
|
+
}>;
|
|
67
|
+
nonCriticalDeductions: {
|
|
68
|
+
lintErrors: {
|
|
69
|
+
count: number;
|
|
70
|
+
deduction: number;
|
|
71
|
+
};
|
|
72
|
+
lintWarnings: {
|
|
73
|
+
count: number;
|
|
74
|
+
deduction: number;
|
|
75
|
+
};
|
|
76
|
+
typeErrors: {
|
|
77
|
+
count: number;
|
|
78
|
+
deduction: number;
|
|
79
|
+
};
|
|
80
|
+
apiIssues: {
|
|
81
|
+
count: number;
|
|
82
|
+
deduction: number;
|
|
83
|
+
};
|
|
84
|
+
componentIssues: {
|
|
85
|
+
count: number;
|
|
86
|
+
deduction: number;
|
|
87
|
+
};
|
|
88
|
+
propertyIssues: {
|
|
89
|
+
count: number;
|
|
90
|
+
deduction: number;
|
|
91
|
+
};
|
|
92
|
+
bindEntityIssues: {
|
|
93
|
+
count: number;
|
|
94
|
+
deduction: number;
|
|
95
|
+
};
|
|
96
|
+
frontendEntityIssues: {
|
|
97
|
+
count: number;
|
|
98
|
+
deduction: number;
|
|
99
|
+
};
|
|
100
|
+
};
|
|
101
|
+
};
|
|
57
102
|
softScore: number;
|
|
58
103
|
criticalChecksPassed: boolean;
|
|
59
104
|
lintResult: LintResult;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-scoring.d.ts","sourceRoot":"","sources":["../../../../src/ai-service/evals/helpers/eval-scoring.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"eval-scoring.d.ts","sourceRoot":"","sources":["../../../../src/ai-service/evals/helpers/eval-scoring.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAgC,KAAK,UAAU,EAAE,MAAM,YAAY,CAAC;AAC3E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAMnD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAI5C,MAAM,WAAW,UAAU;IAEzB,oBAAoB,EAAE,OAAO,CAAC;IAC9B,qBAAqB,EAAE,OAAO,CAAC;IAC/B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,wBAAwB,EAAE,OAAO,CAAC;IAClC,+BAA+B,EAAE,OAAO,CAAC;IACzC,6BAA6B,EAAE,OAAO,CAAC;IACvC,gCAAgC,EAAE,OAAO,CAAC;IAC1C,6BAA6B,EAAE,OAAO,CAAC;IACvC,aAAa,EAAE,OAAO,CAAC;IAGvB,qBAAqB,EAAE,OAAO,CAAC;IAC/B,oCAAoC,EAAE,OAAO,CAAC;CAC/C;AAED,MAAM,WAAW,eAAe;IAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,qBAAqB,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjD,4BAA4B,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGxD,oBAAoB,CAAC,EAAE,KAAK,CAAC;QAC3B,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC1C,CAAC,CAAC;IAGH,oBAAoB,CAAC,EAAE,KAAK,CAAC;QAC3B,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC1C,CAAC,CAAC;IAGH,6BAA6B,CAAC,EAAE,KAAK,CAAC;QACpC,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACzC,2BAA2B,CAAC,EAAE,MAAM,EAAE,CAAC;KACxC,CAAC,CAAC;IAGH,6BAA6B,CAAC,EAAE,KAAK,CAAC;QACpC,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACzC,2BAA2B,CAAC,EAAE,MAAM,EAAE,CAAC;KACxC,CAAC,CAAC;IAEH,0BAA0B,CAAC,EAAE,KAAK,CAAC;QACjC,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACzC,2BAA2B,CAAC,EAAE,MAAM,EAAE,CAAC;KACxC,CAAC,CAAC;CACJ;AAyED,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,UAAU,EACV,MAAM,EACN,cAAc,GACf,EAAE;IACD,SAAS,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC;IAC/C,UAAU,EAAE,MAAM,MAAM,CAAC;IACzB,MAAM,EAAE,eAAe,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,GAAG,OAAO,CAAC;IACV,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,kBAAkB,EAAE;QAClB,gBAAgB,EAAE,KAAK,CAAC;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,SAAS,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC9D,qBAAqB,EAAE;YACrB,UAAU,EAAE;gBAAE,KAAK,EAAE,MAAM,CAAC;gBAAC,SAAS,EAAE,MAAM,CAAA;aAAE,CAAC;YACjD,YAAY,EAAE;gBAAE,KAAK,EAAE,MAAM,CAAC;gBAAC,SAAS,EAAE,MAAM,CAAA;aAAE,CAAC;YACnD,UAAU,EAAE;gBAAE,KAAK,EAAE,MAAM,CAAC;gBAAC,SAAS,EAAE,MAAM,CAAA;aAAE,CAAC;YACjD,SAAS,EAAE;gBAAE,KAAK,EAAE,MAAM,CAAC;gBAAC,SAAS,EAAE,MAAM,CAAA;aAAE,CAAC;YAChD,eAAe,EAAE;gBAAE,KAAK,EAAE,MAAM,CAAC;gBAAC,SAAS,EAAE,MAAM,CAAA;aAAE,CAAC;YACtD,cAAc,EAAE;gBAAE,KAAK,EAAE,MAAM,CAAC;gBAAC,SAAS,EAAE,MAAM,CAAA;aAAE,CAAC;YACrD,gBAAgB,EAAE;gBAAE,KAAK,EAAE,MAAM,CAAC;gBAAC,SAAS,EAAE,MAAM,CAAA;aAAE,CAAC;YACvD,oBAAoB,EAAE;gBAAE,KAAK,EAAE,MAAM,CAAC;gBAAC,SAAS,EAAE,MAAM,CAAA;aAAE,CAAC;SAC5D,CAAC;KACH,CAAC;IAEF,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,OAAO,CAAC;IAC9B,UAAU,EAAE,UAAU,CAAC;IACvB,cAAc,EAAE;QACd,MAAM,EAAE,eAAe,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5C,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5C,IAAI,EAAE,eAAe,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;QAC7C,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,gBAAgB,EAAE,eAAe,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAC1D,UAAU,EAAE,eAAe,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;KACrD,CAAC;CACH,CAAC,CAuJD"}
|