@agentv/core 4.15.8-next.1 → 4.15.9-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/evaluation/validation/index.cjs +295 -50
- package/dist/evaluation/validation/index.cjs.map +1 -1
- package/dist/evaluation/validation/index.d.cts +29 -2
- package/dist/evaluation/validation/index.d.ts +29 -2
- package/dist/evaluation/validation/index.js +276 -33
- package/dist/evaluation/validation/index.js.map +1 -1
- package/dist/index.cjs +4 -2
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Validation result types for AgentV file validation.
|
|
3
3
|
*/
|
|
4
|
-
type FileType = 'eval' | 'targets' | 'config' | 'unknown';
|
|
4
|
+
type FileType = 'eval' | 'targets' | 'config' | 'cases' | 'unknown';
|
|
5
5
|
type ValidationSeverity = 'error' | 'warning';
|
|
6
6
|
interface ValidationError {
|
|
7
7
|
readonly severity: ValidationSeverity;
|
|
@@ -44,6 +44,15 @@ declare function getExpectedSchema(fileType: FileType): string | undefined;
|
|
|
44
44
|
*/
|
|
45
45
|
declare function validateEvalFile(filePath: string): Promise<ValidationResult>;
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Validate a cases file — a YAML file whose root is an array of test case objects.
|
|
49
|
+
*
|
|
50
|
+
* Cases files are referenced from eval files via `tests: path/to/cases.yaml` or
|
|
51
|
+
* `file://cases/accuracy.yaml` entries in the tests array. Each item must have
|
|
52
|
+
* at least an `id` (non-empty string) and an `input` (string or array).
|
|
53
|
+
*/
|
|
54
|
+
declare function validateCasesFile(filePath: string): Promise<ValidationResult>;
|
|
55
|
+
|
|
47
56
|
/**
|
|
48
57
|
* Validate a targets file (agentv-targets-v2.1 schema).
|
|
49
58
|
*/
|
|
@@ -61,4 +70,22 @@ declare function validateConfigFile(filePath: string): Promise<ValidationResult>
|
|
|
61
70
|
*/
|
|
62
71
|
declare function validateFileReferences(evalFilePath: string): Promise<readonly ValidationError[]>;
|
|
63
72
|
|
|
64
|
-
|
|
73
|
+
/**
|
|
74
|
+
* Validate that workspace template and hook script paths in eval files exist.
|
|
75
|
+
*
|
|
76
|
+
* Catches two classes of errors that surface as `setup_error` at runtime but are
|
|
77
|
+
* detectable statically:
|
|
78
|
+
*
|
|
79
|
+
* 1. `workspace.template` — the template path must exist.
|
|
80
|
+
* 2. `workspace.hooks.*.command` — script arguments that look like relative file
|
|
81
|
+
* paths (start with `./`/`../` or carry a script extension) must resolve to
|
|
82
|
+
* existing files using the same cwd precedence the runtime uses:
|
|
83
|
+
* `hook.cwd ?? workspaceFileDir ?? evalDir`
|
|
84
|
+
*
|
|
85
|
+
* When `workspace` is a string path to an external file, that file is also read
|
|
86
|
+
* and its template/hook paths are checked relative to the workspace file's directory.
|
|
87
|
+
* (The workspace file's existence itself is already checked by eval-validator.)
|
|
88
|
+
*/
|
|
89
|
+
declare function validateWorkspacePaths(evalFilePath: string): Promise<readonly ValidationError[]>;
|
|
90
|
+
|
|
91
|
+
export { type FileType, type ValidationError, type ValidationResult, type ValidationSeverity, type ValidationSummary, detectFileType, getExpectedSchema, isValidSchema, validateCasesFile, validateConfigFile, validateEvalFile, validateFileReferences, validateTargetsFile, validateWorkspacePaths };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Validation result types for AgentV file validation.
|
|
3
3
|
*/
|
|
4
|
-
type FileType = 'eval' | 'targets' | 'config' | 'unknown';
|
|
4
|
+
type FileType = 'eval' | 'targets' | 'config' | 'cases' | 'unknown';
|
|
5
5
|
type ValidationSeverity = 'error' | 'warning';
|
|
6
6
|
interface ValidationError {
|
|
7
7
|
readonly severity: ValidationSeverity;
|
|
@@ -44,6 +44,15 @@ declare function getExpectedSchema(fileType: FileType): string | undefined;
|
|
|
44
44
|
*/
|
|
45
45
|
declare function validateEvalFile(filePath: string): Promise<ValidationResult>;
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Validate a cases file — a YAML file whose root is an array of test case objects.
|
|
49
|
+
*
|
|
50
|
+
* Cases files are referenced from eval files via `tests: path/to/cases.yaml` or
|
|
51
|
+
* `file://cases/accuracy.yaml` entries in the tests array. Each item must have
|
|
52
|
+
* at least an `id` (non-empty string) and an `input` (string or array).
|
|
53
|
+
*/
|
|
54
|
+
declare function validateCasesFile(filePath: string): Promise<ValidationResult>;
|
|
55
|
+
|
|
47
56
|
/**
|
|
48
57
|
* Validate a targets file (agentv-targets-v2.1 schema).
|
|
49
58
|
*/
|
|
@@ -61,4 +70,22 @@ declare function validateConfigFile(filePath: string): Promise<ValidationResult>
|
|
|
61
70
|
*/
|
|
62
71
|
declare function validateFileReferences(evalFilePath: string): Promise<readonly ValidationError[]>;
|
|
63
72
|
|
|
64
|
-
|
|
73
|
+
/**
|
|
74
|
+
* Validate that workspace template and hook script paths in eval files exist.
|
|
75
|
+
*
|
|
76
|
+
* Catches two classes of errors that surface as `setup_error` at runtime but are
|
|
77
|
+
* detectable statically:
|
|
78
|
+
*
|
|
79
|
+
* 1. `workspace.template` — the template path must exist.
|
|
80
|
+
* 2. `workspace.hooks.*.command` — script arguments that look like relative file
|
|
81
|
+
* paths (start with `./`/`../` or carry a script extension) must resolve to
|
|
82
|
+
* existing files using the same cwd precedence the runtime uses:
|
|
83
|
+
* `hook.cwd ?? workspaceFileDir ?? evalDir`
|
|
84
|
+
*
|
|
85
|
+
* When `workspace` is a string path to an external file, that file is also read
|
|
86
|
+
* and its template/hook paths are checked relative to the workspace file's directory.
|
|
87
|
+
* (The workspace file's existence itself is already checked by eval-validator.)
|
|
88
|
+
*/
|
|
89
|
+
declare function validateWorkspacePaths(evalFilePath: string): Promise<readonly ValidationError[]>;
|
|
90
|
+
|
|
91
|
+
export { type FileType, type ValidationError, type ValidationResult, type ValidationSeverity, type ValidationSummary, detectFileType, getExpectedSchema, isValidSchema, validateCasesFile, validateConfigFile, validateEvalFile, validateFileReferences, validateTargetsFile, validateWorkspacePaths };
|
|
@@ -23,6 +23,9 @@ async function detectFileType(filePath) {
|
|
|
23
23
|
try {
|
|
24
24
|
const content = await readFile(filePath, "utf8");
|
|
25
25
|
const parsed = parse(content);
|
|
26
|
+
if (Array.isArray(parsed)) {
|
|
27
|
+
return "cases";
|
|
28
|
+
}
|
|
26
29
|
if (typeof parsed !== "object" || parsed === null) {
|
|
27
30
|
return inferFileTypeFromPath(filePath);
|
|
28
31
|
}
|
|
@@ -56,7 +59,11 @@ function inferFileTypeFromPath(filePath) {
|
|
|
56
59
|
return "targets";
|
|
57
60
|
}
|
|
58
61
|
}
|
|
59
|
-
|
|
62
|
+
const lower = basename.toLowerCase();
|
|
63
|
+
if (lower.endsWith(".eval.yaml") || lower.endsWith(".eval.yml")) {
|
|
64
|
+
return "eval";
|
|
65
|
+
}
|
|
66
|
+
return "unknown";
|
|
60
67
|
}
|
|
61
68
|
function isValidSchema(schema) {
|
|
62
69
|
return schema === SCHEMA_EVAL_V2 || schema === SCHEMA_TARGETS_V2 || schema === SCHEMA_CONFIG_V2;
|
|
@@ -75,7 +82,7 @@ function getExpectedSchema(fileType) {
|
|
|
75
82
|
}
|
|
76
83
|
|
|
77
84
|
// src/evaluation/validation/eval-validator.ts
|
|
78
|
-
import { readFile as readFile2 } from "node:fs/promises";
|
|
85
|
+
import { readFile as readFile2, readdir } from "node:fs/promises";
|
|
79
86
|
import path2 from "node:path";
|
|
80
87
|
import { parse as parse2 } from "yaml";
|
|
81
88
|
var ASSERTION_TYPES_WITH_STRING_VALUE = /* @__PURE__ */ new Set([
|
|
@@ -97,6 +104,7 @@ var KNOWN_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
|
|
|
97
104
|
"$schema",
|
|
98
105
|
"name",
|
|
99
106
|
"description",
|
|
107
|
+
"category",
|
|
100
108
|
"version",
|
|
101
109
|
"author",
|
|
102
110
|
"tags",
|
|
@@ -105,13 +113,19 @@ var KNOWN_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
|
|
|
105
113
|
"input",
|
|
106
114
|
"input_files",
|
|
107
115
|
"tests",
|
|
108
|
-
"eval_cases",
|
|
109
116
|
"target",
|
|
110
117
|
"execution",
|
|
111
118
|
"assertions",
|
|
112
119
|
"evaluators",
|
|
120
|
+
"preprocessors",
|
|
113
121
|
"workspace"
|
|
114
122
|
]);
|
|
123
|
+
var DEPRECATED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Map([
|
|
124
|
+
["eval_cases", "'eval_cases' is deprecated. Use 'tests' instead."],
|
|
125
|
+
["evalcases", "'evalcases' is deprecated. Use 'tests' instead."],
|
|
126
|
+
["evaluator", "'evaluator' is deprecated. Use 'assertions' instead."],
|
|
127
|
+
["assert", "'assert' is deprecated. Use 'assertions' instead."]
|
|
128
|
+
]);
|
|
115
129
|
var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
|
|
116
130
|
"id",
|
|
117
131
|
"criteria",
|
|
@@ -120,12 +134,12 @@ var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
|
|
|
120
134
|
"expected_output",
|
|
121
135
|
"assertions",
|
|
122
136
|
"evaluators",
|
|
137
|
+
"rubrics",
|
|
123
138
|
"execution",
|
|
124
139
|
"workspace",
|
|
125
140
|
"metadata",
|
|
126
141
|
"conversation_id",
|
|
127
142
|
"suite",
|
|
128
|
-
"note",
|
|
129
143
|
"depends_on",
|
|
130
144
|
"on_dependency_failure",
|
|
131
145
|
"mode",
|
|
@@ -134,13 +148,48 @@ var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
|
|
|
134
148
|
"on_turn_failure",
|
|
135
149
|
"window_size"
|
|
136
150
|
]);
|
|
151
|
+
var DEPRECATED_TEST_FIELDS = /* @__PURE__ */ new Map([
|
|
152
|
+
["evaluator", "'evaluator' is deprecated. Use 'assertions' instead."],
|
|
153
|
+
["assert", "'assert' is deprecated. Use 'assertions' instead."],
|
|
154
|
+
["expected_outcome", "'expected_outcome' is deprecated. Use 'criteria' instead."]
|
|
155
|
+
]);
|
|
137
156
|
var NAME_PATTERN = /^[a-z0-9-]+$/;
|
|
157
|
+
var ASSERTION_SCRIPT_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".js", ".mts", ".mjs", ".cts", ".cjs"]);
|
|
158
|
+
var customAssertionCache = /* @__PURE__ */ new Map();
|
|
159
|
+
function discoverCustomAssertionTypes(baseDir) {
|
|
160
|
+
const resolved = path2.resolve(baseDir);
|
|
161
|
+
const cached = customAssertionCache.get(resolved);
|
|
162
|
+
if (cached) return cached;
|
|
163
|
+
const promise = (async () => {
|
|
164
|
+
const types = /* @__PURE__ */ new Set();
|
|
165
|
+
let dir = resolved;
|
|
166
|
+
const root = path2.parse(dir).root;
|
|
167
|
+
while (dir !== root) {
|
|
168
|
+
const assertionsDir = path2.join(dir, ".agentv", "assertions");
|
|
169
|
+
try {
|
|
170
|
+
const entries = await readdir(assertionsDir, { withFileTypes: true });
|
|
171
|
+
for (const entry of entries) {
|
|
172
|
+
if (!entry.isFile()) continue;
|
|
173
|
+
const ext = path2.extname(entry.name).toLowerCase();
|
|
174
|
+
if (!ASSERTION_SCRIPT_EXTENSIONS.has(ext)) continue;
|
|
175
|
+
types.add(entry.name.slice(0, -ext.length));
|
|
176
|
+
}
|
|
177
|
+
} catch {
|
|
178
|
+
}
|
|
179
|
+
dir = path2.dirname(dir);
|
|
180
|
+
}
|
|
181
|
+
return types;
|
|
182
|
+
})();
|
|
183
|
+
customAssertionCache.set(resolved, promise);
|
|
184
|
+
return promise;
|
|
185
|
+
}
|
|
138
186
|
function isObject(value) {
|
|
139
187
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
140
188
|
}
|
|
141
189
|
async function validateEvalFile(filePath) {
|
|
142
190
|
const errors = [];
|
|
143
191
|
const absolutePath = path2.resolve(filePath);
|
|
192
|
+
const customAssertionTypes = await discoverCustomAssertionTypes(path2.dirname(absolutePath));
|
|
144
193
|
let parsed;
|
|
145
194
|
try {
|
|
146
195
|
const content = await readFile2(absolutePath, "utf8");
|
|
@@ -173,7 +222,15 @@ async function validateEvalFile(filePath) {
|
|
|
173
222
|
}
|
|
174
223
|
validateMetadata(parsed, absolutePath, errors);
|
|
175
224
|
for (const key of Object.keys(parsed)) {
|
|
176
|
-
|
|
225
|
+
const deprecationMessage = DEPRECATED_TOP_LEVEL_FIELDS.get(key);
|
|
226
|
+
if (deprecationMessage) {
|
|
227
|
+
errors.push({
|
|
228
|
+
severity: "warning",
|
|
229
|
+
filePath: absolutePath,
|
|
230
|
+
location: key,
|
|
231
|
+
message: deprecationMessage
|
|
232
|
+
});
|
|
233
|
+
} else if (!KNOWN_TOP_LEVEL_FIELDS.has(key)) {
|
|
177
234
|
errors.push({
|
|
178
235
|
severity: "warning",
|
|
179
236
|
filePath: absolutePath,
|
|
@@ -271,7 +328,15 @@ async function validateEvalFile(filePath) {
|
|
|
271
328
|
continue;
|
|
272
329
|
}
|
|
273
330
|
for (const key of Object.keys(evalCase)) {
|
|
274
|
-
|
|
331
|
+
const deprecationMessage = DEPRECATED_TEST_FIELDS.get(key);
|
|
332
|
+
if (deprecationMessage) {
|
|
333
|
+
errors.push({
|
|
334
|
+
severity: "warning",
|
|
335
|
+
filePath: absolutePath,
|
|
336
|
+
location: `${location}.${key}`,
|
|
337
|
+
message: deprecationMessage
|
|
338
|
+
});
|
|
339
|
+
} else if (!KNOWN_TEST_FIELDS.has(key)) {
|
|
275
340
|
errors.push({
|
|
276
341
|
severity: "warning",
|
|
277
342
|
filePath: absolutePath,
|
|
@@ -343,7 +408,7 @@ async function validateEvalFile(filePath) {
|
|
|
343
408
|
}
|
|
344
409
|
const assertField = evalCase.assertions;
|
|
345
410
|
if (assertField !== void 0) {
|
|
346
|
-
validateAssertArray(assertField, location, absolutePath, errors);
|
|
411
|
+
validateAssertArray(assertField, location, absolutePath, errors, customAssertionTypes);
|
|
347
412
|
}
|
|
348
413
|
validateConversationMode(evalCase, location, absolutePath, errors);
|
|
349
414
|
await validateWorkspaceConfig(
|
|
@@ -554,7 +619,7 @@ function validateTestsStringPath(testsPath, filePath, errors) {
|
|
|
554
619
|
});
|
|
555
620
|
}
|
|
556
621
|
}
|
|
557
|
-
function validateAssertArray(assertField, parentLocation, filePath, errors) {
|
|
622
|
+
function validateAssertArray(assertField, parentLocation, filePath, errors, customAssertionTypes = /* @__PURE__ */ new Set()) {
|
|
558
623
|
if (!Array.isArray(assertField)) {
|
|
559
624
|
errors.push({
|
|
560
625
|
severity: "warning",
|
|
@@ -602,7 +667,7 @@ function validateAssertArray(assertField, parentLocation, filePath, errors) {
|
|
|
602
667
|
continue;
|
|
603
668
|
}
|
|
604
669
|
const typeValue = rawTypeValue.replace(/_/g, "-");
|
|
605
|
-
if (!isEvaluatorKind(typeValue)) {
|
|
670
|
+
if (!isEvaluatorKind(typeValue) && !customAssertionTypes.has(typeValue)) {
|
|
606
671
|
errors.push({
|
|
607
672
|
severity: "warning",
|
|
608
673
|
filePath,
|
|
@@ -769,13 +834,89 @@ function validateConversationMode(evalCase, location, filePath, errors) {
|
|
|
769
834
|
}
|
|
770
835
|
}
|
|
771
836
|
|
|
772
|
-
// src/evaluation/validation/
|
|
837
|
+
// src/evaluation/validation/cases-validator.ts
|
|
773
838
|
import { readFile as readFile3 } from "node:fs/promises";
|
|
774
839
|
import path3 from "node:path";
|
|
775
840
|
import { parse as parse3 } from "yaml";
|
|
776
841
|
function isObject2(value) {
|
|
777
842
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
778
843
|
}
|
|
844
|
+
async function validateCasesFile(filePath) {
|
|
845
|
+
const errors = [];
|
|
846
|
+
const absolutePath = path3.resolve(filePath);
|
|
847
|
+
let parsed;
|
|
848
|
+
try {
|
|
849
|
+
const content = await readFile3(absolutePath, "utf8");
|
|
850
|
+
parsed = parse3(content);
|
|
851
|
+
} catch (error) {
|
|
852
|
+
errors.push({
|
|
853
|
+
severity: "error",
|
|
854
|
+
filePath: absolutePath,
|
|
855
|
+
message: `Failed to parse YAML: ${error.message}`
|
|
856
|
+
});
|
|
857
|
+
return { valid: false, filePath: absolutePath, fileType: "cases", errors };
|
|
858
|
+
}
|
|
859
|
+
if (!Array.isArray(parsed)) {
|
|
860
|
+
errors.push({
|
|
861
|
+
severity: "error",
|
|
862
|
+
filePath: absolutePath,
|
|
863
|
+
message: "Cases file must contain a YAML array of test case objects"
|
|
864
|
+
});
|
|
865
|
+
return { valid: false, filePath: absolutePath, fileType: "cases", errors };
|
|
866
|
+
}
|
|
867
|
+
for (let i = 0; i < parsed.length; i++) {
|
|
868
|
+
const item = parsed[i];
|
|
869
|
+
const location = `[${i}]`;
|
|
870
|
+
if (!isObject2(item)) {
|
|
871
|
+
errors.push({
|
|
872
|
+
severity: "error",
|
|
873
|
+
filePath: absolutePath,
|
|
874
|
+
location,
|
|
875
|
+
message: "Each test case must be an object"
|
|
876
|
+
});
|
|
877
|
+
continue;
|
|
878
|
+
}
|
|
879
|
+
const id = item.id;
|
|
880
|
+
if (typeof id !== "string" || id.trim().length === 0) {
|
|
881
|
+
errors.push({
|
|
882
|
+
severity: "error",
|
|
883
|
+
filePath: absolutePath,
|
|
884
|
+
location: `${location}.id`,
|
|
885
|
+
message: "Missing or invalid 'id' field (must be a non-empty string)"
|
|
886
|
+
});
|
|
887
|
+
}
|
|
888
|
+
const input = item.input;
|
|
889
|
+
if (input === void 0) {
|
|
890
|
+
errors.push({
|
|
891
|
+
severity: "error",
|
|
892
|
+
filePath: absolutePath,
|
|
893
|
+
location: `${location}.input`,
|
|
894
|
+
message: "Missing 'input' field (must be a string or array of messages)"
|
|
895
|
+
});
|
|
896
|
+
} else if (typeof input !== "string" && !Array.isArray(input)) {
|
|
897
|
+
errors.push({
|
|
898
|
+
severity: "error",
|
|
899
|
+
filePath: absolutePath,
|
|
900
|
+
location: `${location}.input`,
|
|
901
|
+
message: "Invalid 'input' field (must be a string or array of messages)"
|
|
902
|
+
});
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
return {
|
|
906
|
+
valid: errors.filter((e) => e.severity === "error").length === 0,
|
|
907
|
+
filePath: absolutePath,
|
|
908
|
+
fileType: "cases",
|
|
909
|
+
errors
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
// src/evaluation/validation/targets-validator.ts
|
|
914
|
+
import { readFile as readFile4 } from "node:fs/promises";
|
|
915
|
+
import path4 from "node:path";
|
|
916
|
+
import { parse as parse4 } from "yaml";
|
|
917
|
+
function isObject3(value) {
|
|
918
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
919
|
+
}
|
|
779
920
|
var COMMON_SETTINGS = new Set(COMMON_TARGET_SETTINGS);
|
|
780
921
|
var RETRY_SETTINGS = /* @__PURE__ */ new Set([
|
|
781
922
|
"max_retries",
|
|
@@ -995,11 +1136,11 @@ function validateUnknownSettings(target, provider, absolutePath, location, error
|
|
|
995
1136
|
}
|
|
996
1137
|
async function validateTargetsFile(filePath) {
|
|
997
1138
|
const errors = [];
|
|
998
|
-
const absolutePath =
|
|
1139
|
+
const absolutePath = path4.resolve(filePath);
|
|
999
1140
|
let parsed;
|
|
1000
1141
|
try {
|
|
1001
|
-
const content = await
|
|
1002
|
-
parsed =
|
|
1142
|
+
const content = await readFile4(absolutePath, "utf8");
|
|
1143
|
+
parsed = parse4(content);
|
|
1003
1144
|
} catch (error) {
|
|
1004
1145
|
errors.push({
|
|
1005
1146
|
severity: "error",
|
|
@@ -1031,7 +1172,7 @@ async function validateTargetsFile(filePath) {
|
|
|
1031
1172
|
}
|
|
1032
1173
|
}
|
|
1033
1174
|
function validateCliHealthcheck(healthcheck, absolutePath2, location, errors2) {
|
|
1034
|
-
if (!
|
|
1175
|
+
if (!isObject3(healthcheck)) {
|
|
1035
1176
|
errors2.push({
|
|
1036
1177
|
severity: "error",
|
|
1037
1178
|
filePath: absolutePath2,
|
|
@@ -1106,7 +1247,7 @@ async function validateTargetsFile(filePath) {
|
|
|
1106
1247
|
}
|
|
1107
1248
|
return result;
|
|
1108
1249
|
}
|
|
1109
|
-
if (!
|
|
1250
|
+
if (!isObject3(parsed)) {
|
|
1110
1251
|
errors.push({
|
|
1111
1252
|
severity: "error",
|
|
1112
1253
|
filePath: absolutePath,
|
|
@@ -1138,7 +1279,7 @@ async function validateTargetsFile(filePath) {
|
|
|
1138
1279
|
for (let i = 0; i < targets.length; i++) {
|
|
1139
1280
|
const target = targets[i];
|
|
1140
1281
|
const location = `targets[${i}]`;
|
|
1141
|
-
if (!
|
|
1282
|
+
if (!isObject3(target)) {
|
|
1142
1283
|
errors.push({
|
|
1143
1284
|
severity: "error",
|
|
1144
1285
|
filePath: absolutePath,
|
|
@@ -1212,13 +1353,13 @@ async function validateTargetsFile(filePath) {
|
|
|
1212
1353
|
}
|
|
1213
1354
|
|
|
1214
1355
|
// src/evaluation/validation/config-validator.ts
|
|
1215
|
-
import { readFile as
|
|
1216
|
-
import { parse as
|
|
1356
|
+
import { readFile as readFile5 } from "node:fs/promises";
|
|
1357
|
+
import { parse as parse5 } from "yaml";
|
|
1217
1358
|
async function validateConfigFile(filePath) {
|
|
1218
1359
|
const errors = [];
|
|
1219
1360
|
try {
|
|
1220
|
-
const content = await
|
|
1221
|
-
const parsed =
|
|
1361
|
+
const content = await readFile5(filePath, "utf8");
|
|
1362
|
+
const parsed = parse5(content);
|
|
1222
1363
|
if (typeof parsed !== "object" || parsed === null) {
|
|
1223
1364
|
errors.push({
|
|
1224
1365
|
severity: "error",
|
|
@@ -1354,15 +1495,15 @@ async function validateConfigFile(filePath) {
|
|
|
1354
1495
|
}
|
|
1355
1496
|
|
|
1356
1497
|
// src/evaluation/validation/file-reference-validator.ts
|
|
1357
|
-
import { readFile as
|
|
1358
|
-
import
|
|
1359
|
-
import { parse as
|
|
1360
|
-
function
|
|
1498
|
+
import { readFile as readFile6 } from "node:fs/promises";
|
|
1499
|
+
import path5 from "node:path";
|
|
1500
|
+
import { parse as parse6 } from "yaml";
|
|
1501
|
+
function isObject4(value) {
|
|
1361
1502
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1362
1503
|
}
|
|
1363
1504
|
async function validateFileReferences(evalFilePath) {
|
|
1364
1505
|
const errors = [];
|
|
1365
|
-
const absolutePath =
|
|
1506
|
+
const absolutePath = path5.resolve(evalFilePath);
|
|
1366
1507
|
const gitRoot = await findGitRoot(absolutePath);
|
|
1367
1508
|
if (!gitRoot) {
|
|
1368
1509
|
errors.push({
|
|
@@ -1375,12 +1516,12 @@ async function validateFileReferences(evalFilePath) {
|
|
|
1375
1516
|
const searchRoots = buildSearchRoots(absolutePath, gitRoot);
|
|
1376
1517
|
let parsed;
|
|
1377
1518
|
try {
|
|
1378
|
-
const content = await
|
|
1379
|
-
parsed =
|
|
1519
|
+
const content = await readFile6(absolutePath, "utf8");
|
|
1520
|
+
parsed = parse6(content);
|
|
1380
1521
|
} catch {
|
|
1381
1522
|
return errors;
|
|
1382
1523
|
}
|
|
1383
|
-
if (!
|
|
1524
|
+
if (!isObject4(parsed)) {
|
|
1384
1525
|
return errors;
|
|
1385
1526
|
}
|
|
1386
1527
|
let cases = parsed.tests;
|
|
@@ -1395,7 +1536,7 @@ async function validateFileReferences(evalFilePath) {
|
|
|
1395
1536
|
}
|
|
1396
1537
|
for (let i = 0; i < cases.length; i++) {
|
|
1397
1538
|
const evalCase = cases[i];
|
|
1398
|
-
if (!
|
|
1539
|
+
if (!isObject4(evalCase)) {
|
|
1399
1540
|
continue;
|
|
1400
1541
|
}
|
|
1401
1542
|
const inputField = evalCase.input;
|
|
@@ -1424,7 +1565,7 @@ async function validateFileReferences(evalFilePath) {
|
|
|
1424
1565
|
async function validateMessagesFileRefs(messages, location, searchRoots, filePath, errors) {
|
|
1425
1566
|
for (let i = 0; i < messages.length; i++) {
|
|
1426
1567
|
const message = messages[i];
|
|
1427
|
-
if (!
|
|
1568
|
+
if (!isObject4(message)) {
|
|
1428
1569
|
continue;
|
|
1429
1570
|
}
|
|
1430
1571
|
const content = message.content;
|
|
@@ -1436,7 +1577,7 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
|
|
|
1436
1577
|
}
|
|
1437
1578
|
for (let j = 0; j < content.length; j++) {
|
|
1438
1579
|
const contentItem = content[j];
|
|
1439
|
-
if (!
|
|
1580
|
+
if (!isObject4(contentItem)) {
|
|
1440
1581
|
continue;
|
|
1441
1582
|
}
|
|
1442
1583
|
const type = contentItem.type;
|
|
@@ -1463,7 +1604,7 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
|
|
|
1463
1604
|
});
|
|
1464
1605
|
} else {
|
|
1465
1606
|
try {
|
|
1466
|
-
const fileContent = await
|
|
1607
|
+
const fileContent = await readFile6(resolvedPath, "utf8");
|
|
1467
1608
|
if (fileContent.trim().length === 0) {
|
|
1468
1609
|
errors.push({
|
|
1469
1610
|
severity: "warning",
|
|
@@ -1484,13 +1625,115 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
|
|
|
1484
1625
|
}
|
|
1485
1626
|
}
|
|
1486
1627
|
}
|
|
1628
|
+
|
|
1629
|
+
// src/evaluation/validation/workspace-path-validator.ts
|
|
1630
|
+
import { access, readFile as readFile7 } from "node:fs/promises";
|
|
1631
|
+
import path6 from "node:path";
|
|
1632
|
+
import { parse as parse7 } from "yaml";
|
|
1633
|
+
function isObject5(value) {
|
|
1634
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1635
|
+
}
|
|
1636
|
+
async function validateWorkspacePaths(evalFilePath) {
|
|
1637
|
+
const errors = [];
|
|
1638
|
+
const absolutePath = path6.resolve(evalFilePath);
|
|
1639
|
+
const evalDir = path6.dirname(absolutePath);
|
|
1640
|
+
let parsed;
|
|
1641
|
+
try {
|
|
1642
|
+
const content = await readFile7(absolutePath, "utf8");
|
|
1643
|
+
parsed = parse7(content);
|
|
1644
|
+
} catch {
|
|
1645
|
+
return errors;
|
|
1646
|
+
}
|
|
1647
|
+
if (!isObject5(parsed)) return errors;
|
|
1648
|
+
const workspaceRaw = parsed.workspace;
|
|
1649
|
+
if (workspaceRaw === void 0 || workspaceRaw === null) return errors;
|
|
1650
|
+
if (typeof workspaceRaw === "string") {
|
|
1651
|
+
const workspaceFilePath = path6.resolve(evalDir, workspaceRaw);
|
|
1652
|
+
try {
|
|
1653
|
+
const wsContent = await readFile7(workspaceFilePath, "utf8");
|
|
1654
|
+
const wsParsed = parse7(wsContent);
|
|
1655
|
+
if (isObject5(wsParsed)) {
|
|
1656
|
+
const wsDir = path6.dirname(workspaceFilePath);
|
|
1657
|
+
await validateWorkspaceObject(wsParsed, wsDir, absolutePath, "workspace", errors);
|
|
1658
|
+
}
|
|
1659
|
+
} catch {
|
|
1660
|
+
}
|
|
1661
|
+
} else if (isObject5(workspaceRaw)) {
|
|
1662
|
+
await validateWorkspaceObject(workspaceRaw, evalDir, absolutePath, "workspace", errors);
|
|
1663
|
+
}
|
|
1664
|
+
return errors;
|
|
1665
|
+
}
|
|
1666
|
+
async function validateWorkspaceObject(obj, baseDir, evalFilePath, location, errors) {
|
|
1667
|
+
const template = obj.template;
|
|
1668
|
+
if (typeof template === "string") {
|
|
1669
|
+
const templatePath = path6.isAbsolute(template) ? template : path6.resolve(baseDir, template);
|
|
1670
|
+
if (!await fileExists(templatePath)) {
|
|
1671
|
+
errors.push({
|
|
1672
|
+
severity: "error",
|
|
1673
|
+
filePath: evalFilePath,
|
|
1674
|
+
location: `${location}.template`,
|
|
1675
|
+
message: `Template path not found: ${template} (resolved to ${templatePath})`
|
|
1676
|
+
});
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
const hooks = obj.hooks;
|
|
1680
|
+
if (!isObject5(hooks)) return;
|
|
1681
|
+
for (const hookName of ["before_all", "before_each", "after_each", "after_all"]) {
|
|
1682
|
+
const hook = hooks[hookName];
|
|
1683
|
+
if (!isObject5(hook)) continue;
|
|
1684
|
+
const hookCwdRaw = typeof hook.cwd === "string" ? hook.cwd : void 0;
|
|
1685
|
+
const hookCwd = hookCwdRaw ? path6.isAbsolute(hookCwdRaw) ? hookCwdRaw : path6.resolve(baseDir, hookCwdRaw) : baseDir;
|
|
1686
|
+
const command = hook.command ?? hook.script;
|
|
1687
|
+
if (!Array.isArray(command)) continue;
|
|
1688
|
+
for (let i = 0; i < command.length; i++) {
|
|
1689
|
+
const arg = command[i];
|
|
1690
|
+
if (typeof arg !== "string") continue;
|
|
1691
|
+
if (!looksLikeFilePath(arg)) continue;
|
|
1692
|
+
const resolved = path6.isAbsolute(arg) ? arg : path6.resolve(hookCwd, arg);
|
|
1693
|
+
if (!await fileExists(resolved)) {
|
|
1694
|
+
errors.push({
|
|
1695
|
+
severity: "error",
|
|
1696
|
+
filePath: evalFilePath,
|
|
1697
|
+
location: `${location}.hooks.${hookName}.command[${i}]`,
|
|
1698
|
+
message: `Hook script not found: ${arg} (resolved to ${resolved})`
|
|
1699
|
+
});
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
function looksLikeFilePath(arg) {
|
|
1705
|
+
if (arg.startsWith("./") || arg.startsWith("../")) return true;
|
|
1706
|
+
const scriptExtensions = [
|
|
1707
|
+
".mjs",
|
|
1708
|
+
".cjs",
|
|
1709
|
+
".js",
|
|
1710
|
+
".ts",
|
|
1711
|
+
".sh",
|
|
1712
|
+
".bash",
|
|
1713
|
+
".zsh",
|
|
1714
|
+
".py",
|
|
1715
|
+
".rb",
|
|
1716
|
+
".pl"
|
|
1717
|
+
];
|
|
1718
|
+
return scriptExtensions.some((ext) => arg.endsWith(ext));
|
|
1719
|
+
}
|
|
1720
|
+
async function fileExists(filePath) {
|
|
1721
|
+
try {
|
|
1722
|
+
await access(filePath);
|
|
1723
|
+
return true;
|
|
1724
|
+
} catch {
|
|
1725
|
+
return false;
|
|
1726
|
+
}
|
|
1727
|
+
}
|
|
1487
1728
|
export {
|
|
1488
1729
|
detectFileType,
|
|
1489
1730
|
getExpectedSchema,
|
|
1490
1731
|
isValidSchema,
|
|
1732
|
+
validateCasesFile,
|
|
1491
1733
|
validateConfigFile,
|
|
1492
1734
|
validateEvalFile,
|
|
1493
1735
|
validateFileReferences,
|
|
1494
|
-
validateTargetsFile
|
|
1736
|
+
validateTargetsFile,
|
|
1737
|
+
validateWorkspacePaths
|
|
1495
1738
|
};
|
|
1496
1739
|
//# sourceMappingURL=index.js.map
|