@agentv/core 4.15.8-next.1 → 4.15.9-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Validation result types for AgentV file validation.
3
3
  */
4
- type FileType = 'eval' | 'targets' | 'config' | 'unknown';
4
+ type FileType = 'eval' | 'targets' | 'config' | 'cases' | 'unknown';
5
5
  type ValidationSeverity = 'error' | 'warning';
6
6
  interface ValidationError {
7
7
  readonly severity: ValidationSeverity;
@@ -44,6 +44,15 @@ declare function getExpectedSchema(fileType: FileType): string | undefined;
44
44
  */
45
45
  declare function validateEvalFile(filePath: string): Promise<ValidationResult>;
46
46
 
47
+ /**
48
+ * Validate a cases file — a YAML file whose root is an array of test case objects.
49
+ *
50
+ * Cases files are referenced from eval files via `tests: path/to/cases.yaml` or
51
+ * `file://cases/accuracy.yaml` entries in the tests array. Each item must have
52
+ * at least an `id` (non-empty string) and an `input` (string or array).
53
+ */
54
+ declare function validateCasesFile(filePath: string): Promise<ValidationResult>;
55
+
47
56
  /**
48
57
  * Validate a targets file (agentv-targets-v2.1 schema).
49
58
  */
@@ -61,4 +70,22 @@ declare function validateConfigFile(filePath: string): Promise<ValidationResult>
61
70
  */
62
71
  declare function validateFileReferences(evalFilePath: string): Promise<readonly ValidationError[]>;
63
72
 
64
- export { type FileType, type ValidationError, type ValidationResult, type ValidationSeverity, type ValidationSummary, detectFileType, getExpectedSchema, isValidSchema, validateConfigFile, validateEvalFile, validateFileReferences, validateTargetsFile };
73
+ /**
74
+ * Validate that workspace template and hook script paths in eval files exist.
75
+ *
76
+ * Catches two classes of errors that surface as `setup_error` at runtime but are
77
+ * detectable statically:
78
+ *
79
+ * 1. `workspace.template` — the template path must exist.
80
+ * 2. `workspace.hooks.*.command` — script arguments that look like relative file
81
+ * paths (start with `./`/`../` or carry a script extension) must resolve to
82
+ * existing files using the same cwd precedence the runtime uses:
83
+ * `hook.cwd ?? workspaceFileDir ?? evalDir`
84
+ *
85
+ * When `workspace` is a string path to an external file, that file is also read
86
+ * and its template/hook paths are checked relative to the workspace file's directory.
87
+ * (The workspace file's existence itself is already checked by eval-validator.)
88
+ */
89
+ declare function validateWorkspacePaths(evalFilePath: string): Promise<readonly ValidationError[]>;
90
+
91
+ export { type FileType, type ValidationError, type ValidationResult, type ValidationSeverity, type ValidationSummary, detectFileType, getExpectedSchema, isValidSchema, validateCasesFile, validateConfigFile, validateEvalFile, validateFileReferences, validateTargetsFile, validateWorkspacePaths };
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Validation result types for AgentV file validation.
3
3
  */
4
- type FileType = 'eval' | 'targets' | 'config' | 'unknown';
4
+ type FileType = 'eval' | 'targets' | 'config' | 'cases' | 'unknown';
5
5
  type ValidationSeverity = 'error' | 'warning';
6
6
  interface ValidationError {
7
7
  readonly severity: ValidationSeverity;
@@ -44,6 +44,15 @@ declare function getExpectedSchema(fileType: FileType): string | undefined;
44
44
  */
45
45
  declare function validateEvalFile(filePath: string): Promise<ValidationResult>;
46
46
 
47
+ /**
48
+ * Validate a cases file — a YAML file whose root is an array of test case objects.
49
+ *
50
+ * Cases files are referenced from eval files via `tests: path/to/cases.yaml` or
51
+ * `file://cases/accuracy.yaml` entries in the tests array. Each item must have
52
+ * at least an `id` (non-empty string) and an `input` (string or array).
53
+ */
54
+ declare function validateCasesFile(filePath: string): Promise<ValidationResult>;
55
+
47
56
  /**
48
57
  * Validate a targets file (agentv-targets-v2.1 schema).
49
58
  */
@@ -61,4 +70,22 @@ declare function validateConfigFile(filePath: string): Promise<ValidationResult>
61
70
  */
62
71
  declare function validateFileReferences(evalFilePath: string): Promise<readonly ValidationError[]>;
63
72
 
64
- export { type FileType, type ValidationError, type ValidationResult, type ValidationSeverity, type ValidationSummary, detectFileType, getExpectedSchema, isValidSchema, validateConfigFile, validateEvalFile, validateFileReferences, validateTargetsFile };
73
+ /**
74
+ * Validate that workspace template and hook script paths in eval files exist.
75
+ *
76
+ * Catches two classes of errors that surface as `setup_error` at runtime but are
77
+ * detectable statically:
78
+ *
79
+ * 1. `workspace.template` — the template path must exist.
80
+ * 2. `workspace.hooks.*.command` — script arguments that look like relative file
81
+ * paths (start with `./`/`../` or carry a script extension) must resolve to
82
+ * existing files using the same cwd precedence the runtime uses:
83
+ * `hook.cwd ?? workspaceFileDir ?? evalDir`
84
+ *
85
+ * When `workspace` is a string path to an external file, that file is also read
86
+ * and its template/hook paths are checked relative to the workspace file's directory.
87
+ * (The workspace file's existence itself is already checked by eval-validator.)
88
+ */
89
+ declare function validateWorkspacePaths(evalFilePath: string): Promise<readonly ValidationError[]>;
90
+
91
+ export { type FileType, type ValidationError, type ValidationResult, type ValidationSeverity, type ValidationSummary, detectFileType, getExpectedSchema, isValidSchema, validateCasesFile, validateConfigFile, validateEvalFile, validateFileReferences, validateTargetsFile, validateWorkspacePaths };
@@ -23,6 +23,9 @@ async function detectFileType(filePath) {
23
23
  try {
24
24
  const content = await readFile(filePath, "utf8");
25
25
  const parsed = parse(content);
26
+ if (Array.isArray(parsed)) {
27
+ return "cases";
28
+ }
26
29
  if (typeof parsed !== "object" || parsed === null) {
27
30
  return inferFileTypeFromPath(filePath);
28
31
  }
@@ -56,7 +59,11 @@ function inferFileTypeFromPath(filePath) {
56
59
  return "targets";
57
60
  }
58
61
  }
59
- return "eval";
62
+ const lower = basename.toLowerCase();
63
+ if (lower.endsWith(".eval.yaml") || lower.endsWith(".eval.yml")) {
64
+ return "eval";
65
+ }
66
+ return "unknown";
60
67
  }
61
68
  function isValidSchema(schema) {
62
69
  return schema === SCHEMA_EVAL_V2 || schema === SCHEMA_TARGETS_V2 || schema === SCHEMA_CONFIG_V2;
@@ -75,7 +82,7 @@ function getExpectedSchema(fileType) {
75
82
  }
76
83
 
77
84
  // src/evaluation/validation/eval-validator.ts
78
- import { readFile as readFile2 } from "node:fs/promises";
85
+ import { readFile as readFile2, readdir } from "node:fs/promises";
79
86
  import path2 from "node:path";
80
87
  import { parse as parse2 } from "yaml";
81
88
  var ASSERTION_TYPES_WITH_STRING_VALUE = /* @__PURE__ */ new Set([
@@ -97,6 +104,7 @@ var KNOWN_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
97
104
  "$schema",
98
105
  "name",
99
106
  "description",
107
+ "category",
100
108
  "version",
101
109
  "author",
102
110
  "tags",
@@ -105,13 +113,19 @@ var KNOWN_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
105
113
  "input",
106
114
  "input_files",
107
115
  "tests",
108
- "eval_cases",
109
116
  "target",
110
117
  "execution",
111
118
  "assertions",
112
119
  "evaluators",
120
+ "preprocessors",
113
121
  "workspace"
114
122
  ]);
123
+ var DEPRECATED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Map([
124
+ ["eval_cases", "'eval_cases' is deprecated. Use 'tests' instead."],
125
+ ["evalcases", "'evalcases' is deprecated. Use 'tests' instead."],
126
+ ["evaluator", "'evaluator' is deprecated. Use 'assertions' instead."],
127
+ ["assert", "'assert' is deprecated. Use 'assertions' instead."]
128
+ ]);
115
129
  var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
116
130
  "id",
117
131
  "criteria",
@@ -120,12 +134,12 @@ var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
120
134
  "expected_output",
121
135
  "assertions",
122
136
  "evaluators",
137
+ "rubrics",
123
138
  "execution",
124
139
  "workspace",
125
140
  "metadata",
126
141
  "conversation_id",
127
142
  "suite",
128
- "note",
129
143
  "depends_on",
130
144
  "on_dependency_failure",
131
145
  "mode",
@@ -134,13 +148,48 @@ var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
134
148
  "on_turn_failure",
135
149
  "window_size"
136
150
  ]);
151
+ var DEPRECATED_TEST_FIELDS = /* @__PURE__ */ new Map([
152
+ ["evaluator", "'evaluator' is deprecated. Use 'assertions' instead."],
153
+ ["assert", "'assert' is deprecated. Use 'assertions' instead."],
154
+ ["expected_outcome", "'expected_outcome' is deprecated. Use 'criteria' instead."]
155
+ ]);
137
156
  var NAME_PATTERN = /^[a-z0-9-]+$/;
157
+ var ASSERTION_SCRIPT_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".js", ".mts", ".mjs", ".cts", ".cjs"]);
158
+ var customAssertionCache = /* @__PURE__ */ new Map();
159
+ function discoverCustomAssertionTypes(baseDir) {
160
+ const resolved = path2.resolve(baseDir);
161
+ const cached = customAssertionCache.get(resolved);
162
+ if (cached) return cached;
163
+ const promise = (async () => {
164
+ const types = /* @__PURE__ */ new Set();
165
+ let dir = resolved;
166
+ const root = path2.parse(dir).root;
167
+ while (dir !== root) {
168
+ const assertionsDir = path2.join(dir, ".agentv", "assertions");
169
+ try {
170
+ const entries = await readdir(assertionsDir, { withFileTypes: true });
171
+ for (const entry of entries) {
172
+ if (!entry.isFile()) continue;
173
+ const ext = path2.extname(entry.name).toLowerCase();
174
+ if (!ASSERTION_SCRIPT_EXTENSIONS.has(ext)) continue;
175
+ types.add(entry.name.slice(0, -ext.length));
176
+ }
177
+ } catch {
178
+ }
179
+ dir = path2.dirname(dir);
180
+ }
181
+ return types;
182
+ })();
183
+ customAssertionCache.set(resolved, promise);
184
+ return promise;
185
+ }
138
186
  function isObject(value) {
139
187
  return typeof value === "object" && value !== null && !Array.isArray(value);
140
188
  }
141
189
  async function validateEvalFile(filePath) {
142
190
  const errors = [];
143
191
  const absolutePath = path2.resolve(filePath);
192
+ const customAssertionTypes = await discoverCustomAssertionTypes(path2.dirname(absolutePath));
144
193
  let parsed;
145
194
  try {
146
195
  const content = await readFile2(absolutePath, "utf8");
@@ -173,7 +222,15 @@ async function validateEvalFile(filePath) {
173
222
  }
174
223
  validateMetadata(parsed, absolutePath, errors);
175
224
  for (const key of Object.keys(parsed)) {
176
- if (!KNOWN_TOP_LEVEL_FIELDS.has(key)) {
225
+ const deprecationMessage = DEPRECATED_TOP_LEVEL_FIELDS.get(key);
226
+ if (deprecationMessage) {
227
+ errors.push({
228
+ severity: "warning",
229
+ filePath: absolutePath,
230
+ location: key,
231
+ message: deprecationMessage
232
+ });
233
+ } else if (!KNOWN_TOP_LEVEL_FIELDS.has(key)) {
177
234
  errors.push({
178
235
  severity: "warning",
179
236
  filePath: absolutePath,
@@ -271,7 +328,15 @@ async function validateEvalFile(filePath) {
271
328
  continue;
272
329
  }
273
330
  for (const key of Object.keys(evalCase)) {
274
- if (!KNOWN_TEST_FIELDS.has(key)) {
331
+ const deprecationMessage = DEPRECATED_TEST_FIELDS.get(key);
332
+ if (deprecationMessage) {
333
+ errors.push({
334
+ severity: "warning",
335
+ filePath: absolutePath,
336
+ location: `${location}.${key}`,
337
+ message: deprecationMessage
338
+ });
339
+ } else if (!KNOWN_TEST_FIELDS.has(key)) {
275
340
  errors.push({
276
341
  severity: "warning",
277
342
  filePath: absolutePath,
@@ -343,7 +408,7 @@ async function validateEvalFile(filePath) {
343
408
  }
344
409
  const assertField = evalCase.assertions;
345
410
  if (assertField !== void 0) {
346
- validateAssertArray(assertField, location, absolutePath, errors);
411
+ validateAssertArray(assertField, location, absolutePath, errors, customAssertionTypes);
347
412
  }
348
413
  validateConversationMode(evalCase, location, absolutePath, errors);
349
414
  await validateWorkspaceConfig(
@@ -554,7 +619,7 @@ function validateTestsStringPath(testsPath, filePath, errors) {
554
619
  });
555
620
  }
556
621
  }
557
- function validateAssertArray(assertField, parentLocation, filePath, errors) {
622
+ function validateAssertArray(assertField, parentLocation, filePath, errors, customAssertionTypes = /* @__PURE__ */ new Set()) {
558
623
  if (!Array.isArray(assertField)) {
559
624
  errors.push({
560
625
  severity: "warning",
@@ -602,7 +667,7 @@ function validateAssertArray(assertField, parentLocation, filePath, errors) {
602
667
  continue;
603
668
  }
604
669
  const typeValue = rawTypeValue.replace(/_/g, "-");
605
- if (!isEvaluatorKind(typeValue)) {
670
+ if (!isEvaluatorKind(typeValue) && !customAssertionTypes.has(typeValue)) {
606
671
  errors.push({
607
672
  severity: "warning",
608
673
  filePath,
@@ -769,13 +834,89 @@ function validateConversationMode(evalCase, location, filePath, errors) {
769
834
  }
770
835
  }
771
836
 
772
- // src/evaluation/validation/targets-validator.ts
837
+ // src/evaluation/validation/cases-validator.ts
773
838
  import { readFile as readFile3 } from "node:fs/promises";
774
839
  import path3 from "node:path";
775
840
  import { parse as parse3 } from "yaml";
776
841
  function isObject2(value) {
777
842
  return typeof value === "object" && value !== null && !Array.isArray(value);
778
843
  }
844
+ async function validateCasesFile(filePath) {
845
+ const errors = [];
846
+ const absolutePath = path3.resolve(filePath);
847
+ let parsed;
848
+ try {
849
+ const content = await readFile3(absolutePath, "utf8");
850
+ parsed = parse3(content);
851
+ } catch (error) {
852
+ errors.push({
853
+ severity: "error",
854
+ filePath: absolutePath,
855
+ message: `Failed to parse YAML: ${error.message}`
856
+ });
857
+ return { valid: false, filePath: absolutePath, fileType: "cases", errors };
858
+ }
859
+ if (!Array.isArray(parsed)) {
860
+ errors.push({
861
+ severity: "error",
862
+ filePath: absolutePath,
863
+ message: "Cases file must contain a YAML array of test case objects"
864
+ });
865
+ return { valid: false, filePath: absolutePath, fileType: "cases", errors };
866
+ }
867
+ for (let i = 0; i < parsed.length; i++) {
868
+ const item = parsed[i];
869
+ const location = `[${i}]`;
870
+ if (!isObject2(item)) {
871
+ errors.push({
872
+ severity: "error",
873
+ filePath: absolutePath,
874
+ location,
875
+ message: "Each test case must be an object"
876
+ });
877
+ continue;
878
+ }
879
+ const id = item.id;
880
+ if (typeof id !== "string" || id.trim().length === 0) {
881
+ errors.push({
882
+ severity: "error",
883
+ filePath: absolutePath,
884
+ location: `${location}.id`,
885
+ message: "Missing or invalid 'id' field (must be a non-empty string)"
886
+ });
887
+ }
888
+ const input = item.input;
889
+ if (input === void 0) {
890
+ errors.push({
891
+ severity: "error",
892
+ filePath: absolutePath,
893
+ location: `${location}.input`,
894
+ message: "Missing 'input' field (must be a string or array of messages)"
895
+ });
896
+ } else if (typeof input !== "string" && !Array.isArray(input)) {
897
+ errors.push({
898
+ severity: "error",
899
+ filePath: absolutePath,
900
+ location: `${location}.input`,
901
+ message: "Invalid 'input' field (must be a string or array of messages)"
902
+ });
903
+ }
904
+ }
905
+ return {
906
+ valid: errors.filter((e) => e.severity === "error").length === 0,
907
+ filePath: absolutePath,
908
+ fileType: "cases",
909
+ errors
910
+ };
911
+ }
912
+
913
+ // src/evaluation/validation/targets-validator.ts
914
+ import { readFile as readFile4 } from "node:fs/promises";
915
+ import path4 from "node:path";
916
+ import { parse as parse4 } from "yaml";
917
+ function isObject3(value) {
918
+ return typeof value === "object" && value !== null && !Array.isArray(value);
919
+ }
779
920
  var COMMON_SETTINGS = new Set(COMMON_TARGET_SETTINGS);
780
921
  var RETRY_SETTINGS = /* @__PURE__ */ new Set([
781
922
  "max_retries",
@@ -995,11 +1136,11 @@ function validateUnknownSettings(target, provider, absolutePath, location, error
995
1136
  }
996
1137
  async function validateTargetsFile(filePath) {
997
1138
  const errors = [];
998
- const absolutePath = path3.resolve(filePath);
1139
+ const absolutePath = path4.resolve(filePath);
999
1140
  let parsed;
1000
1141
  try {
1001
- const content = await readFile3(absolutePath, "utf8");
1002
- parsed = parse3(content);
1142
+ const content = await readFile4(absolutePath, "utf8");
1143
+ parsed = parse4(content);
1003
1144
  } catch (error) {
1004
1145
  errors.push({
1005
1146
  severity: "error",
@@ -1031,7 +1172,7 @@ async function validateTargetsFile(filePath) {
1031
1172
  }
1032
1173
  }
1033
1174
  function validateCliHealthcheck(healthcheck, absolutePath2, location, errors2) {
1034
- if (!isObject2(healthcheck)) {
1175
+ if (!isObject3(healthcheck)) {
1035
1176
  errors2.push({
1036
1177
  severity: "error",
1037
1178
  filePath: absolutePath2,
@@ -1106,7 +1247,7 @@ async function validateTargetsFile(filePath) {
1106
1247
  }
1107
1248
  return result;
1108
1249
  }
1109
- if (!isObject2(parsed)) {
1250
+ if (!isObject3(parsed)) {
1110
1251
  errors.push({
1111
1252
  severity: "error",
1112
1253
  filePath: absolutePath,
@@ -1138,7 +1279,7 @@ async function validateTargetsFile(filePath) {
1138
1279
  for (let i = 0; i < targets.length; i++) {
1139
1280
  const target = targets[i];
1140
1281
  const location = `targets[${i}]`;
1141
- if (!isObject2(target)) {
1282
+ if (!isObject3(target)) {
1142
1283
  errors.push({
1143
1284
  severity: "error",
1144
1285
  filePath: absolutePath,
@@ -1212,13 +1353,13 @@ async function validateTargetsFile(filePath) {
1212
1353
  }
1213
1354
 
1214
1355
  // src/evaluation/validation/config-validator.ts
1215
- import { readFile as readFile4 } from "node:fs/promises";
1216
- import { parse as parse4 } from "yaml";
1356
+ import { readFile as readFile5 } from "node:fs/promises";
1357
+ import { parse as parse5 } from "yaml";
1217
1358
  async function validateConfigFile(filePath) {
1218
1359
  const errors = [];
1219
1360
  try {
1220
- const content = await readFile4(filePath, "utf8");
1221
- const parsed = parse4(content);
1361
+ const content = await readFile5(filePath, "utf8");
1362
+ const parsed = parse5(content);
1222
1363
  if (typeof parsed !== "object" || parsed === null) {
1223
1364
  errors.push({
1224
1365
  severity: "error",
@@ -1354,15 +1495,15 @@ async function validateConfigFile(filePath) {
1354
1495
  }
1355
1496
 
1356
1497
  // src/evaluation/validation/file-reference-validator.ts
1357
- import { readFile as readFile5 } from "node:fs/promises";
1358
- import path4 from "node:path";
1359
- import { parse as parse5 } from "yaml";
1360
- function isObject3(value) {
1498
+ import { readFile as readFile6 } from "node:fs/promises";
1499
+ import path5 from "node:path";
1500
+ import { parse as parse6 } from "yaml";
1501
+ function isObject4(value) {
1361
1502
  return typeof value === "object" && value !== null && !Array.isArray(value);
1362
1503
  }
1363
1504
  async function validateFileReferences(evalFilePath) {
1364
1505
  const errors = [];
1365
- const absolutePath = path4.resolve(evalFilePath);
1506
+ const absolutePath = path5.resolve(evalFilePath);
1366
1507
  const gitRoot = await findGitRoot(absolutePath);
1367
1508
  if (!gitRoot) {
1368
1509
  errors.push({
@@ -1375,12 +1516,12 @@ async function validateFileReferences(evalFilePath) {
1375
1516
  const searchRoots = buildSearchRoots(absolutePath, gitRoot);
1376
1517
  let parsed;
1377
1518
  try {
1378
- const content = await readFile5(absolutePath, "utf8");
1379
- parsed = parse5(content);
1519
+ const content = await readFile6(absolutePath, "utf8");
1520
+ parsed = parse6(content);
1380
1521
  } catch {
1381
1522
  return errors;
1382
1523
  }
1383
- if (!isObject3(parsed)) {
1524
+ if (!isObject4(parsed)) {
1384
1525
  return errors;
1385
1526
  }
1386
1527
  let cases = parsed.tests;
@@ -1395,7 +1536,7 @@ async function validateFileReferences(evalFilePath) {
1395
1536
  }
1396
1537
  for (let i = 0; i < cases.length; i++) {
1397
1538
  const evalCase = cases[i];
1398
- if (!isObject3(evalCase)) {
1539
+ if (!isObject4(evalCase)) {
1399
1540
  continue;
1400
1541
  }
1401
1542
  const inputField = evalCase.input;
@@ -1424,7 +1565,7 @@ async function validateFileReferences(evalFilePath) {
1424
1565
  async function validateMessagesFileRefs(messages, location, searchRoots, filePath, errors) {
1425
1566
  for (let i = 0; i < messages.length; i++) {
1426
1567
  const message = messages[i];
1427
- if (!isObject3(message)) {
1568
+ if (!isObject4(message)) {
1428
1569
  continue;
1429
1570
  }
1430
1571
  const content = message.content;
@@ -1436,7 +1577,7 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
1436
1577
  }
1437
1578
  for (let j = 0; j < content.length; j++) {
1438
1579
  const contentItem = content[j];
1439
- if (!isObject3(contentItem)) {
1580
+ if (!isObject4(contentItem)) {
1440
1581
  continue;
1441
1582
  }
1442
1583
  const type = contentItem.type;
@@ -1463,7 +1604,7 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
1463
1604
  });
1464
1605
  } else {
1465
1606
  try {
1466
- const fileContent = await readFile5(resolvedPath, "utf8");
1607
+ const fileContent = await readFile6(resolvedPath, "utf8");
1467
1608
  if (fileContent.trim().length === 0) {
1468
1609
  errors.push({
1469
1610
  severity: "warning",
@@ -1484,13 +1625,115 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
1484
1625
  }
1485
1626
  }
1486
1627
  }
1628
+
1629
+ // src/evaluation/validation/workspace-path-validator.ts
1630
+ import { access, readFile as readFile7 } from "node:fs/promises";
1631
+ import path6 from "node:path";
1632
+ import { parse as parse7 } from "yaml";
1633
+ function isObject5(value) {
1634
+ return typeof value === "object" && value !== null && !Array.isArray(value);
1635
+ }
1636
+ async function validateWorkspacePaths(evalFilePath) {
1637
+ const errors = [];
1638
+ const absolutePath = path6.resolve(evalFilePath);
1639
+ const evalDir = path6.dirname(absolutePath);
1640
+ let parsed;
1641
+ try {
1642
+ const content = await readFile7(absolutePath, "utf8");
1643
+ parsed = parse7(content);
1644
+ } catch {
1645
+ return errors;
1646
+ }
1647
+ if (!isObject5(parsed)) return errors;
1648
+ const workspaceRaw = parsed.workspace;
1649
+ if (workspaceRaw === void 0 || workspaceRaw === null) return errors;
1650
+ if (typeof workspaceRaw === "string") {
1651
+ const workspaceFilePath = path6.resolve(evalDir, workspaceRaw);
1652
+ try {
1653
+ const wsContent = await readFile7(workspaceFilePath, "utf8");
1654
+ const wsParsed = parse7(wsContent);
1655
+ if (isObject5(wsParsed)) {
1656
+ const wsDir = path6.dirname(workspaceFilePath);
1657
+ await validateWorkspaceObject(wsParsed, wsDir, absolutePath, "workspace", errors);
1658
+ }
1659
+ } catch {
1660
+ }
1661
+ } else if (isObject5(workspaceRaw)) {
1662
+ await validateWorkspaceObject(workspaceRaw, evalDir, absolutePath, "workspace", errors);
1663
+ }
1664
+ return errors;
1665
+ }
1666
+ async function validateWorkspaceObject(obj, baseDir, evalFilePath, location, errors) {
1667
+ const template = obj.template;
1668
+ if (typeof template === "string") {
1669
+ const templatePath = path6.isAbsolute(template) ? template : path6.resolve(baseDir, template);
1670
+ if (!await fileExists(templatePath)) {
1671
+ errors.push({
1672
+ severity: "error",
1673
+ filePath: evalFilePath,
1674
+ location: `${location}.template`,
1675
+ message: `Template path not found: ${template} (resolved to ${templatePath})`
1676
+ });
1677
+ }
1678
+ }
1679
+ const hooks = obj.hooks;
1680
+ if (!isObject5(hooks)) return;
1681
+ for (const hookName of ["before_all", "before_each", "after_each", "after_all"]) {
1682
+ const hook = hooks[hookName];
1683
+ if (!isObject5(hook)) continue;
1684
+ const hookCwdRaw = typeof hook.cwd === "string" ? hook.cwd : void 0;
1685
+ const hookCwd = hookCwdRaw ? path6.isAbsolute(hookCwdRaw) ? hookCwdRaw : path6.resolve(baseDir, hookCwdRaw) : baseDir;
1686
+ const command = hook.command ?? hook.script;
1687
+ if (!Array.isArray(command)) continue;
1688
+ for (let i = 0; i < command.length; i++) {
1689
+ const arg = command[i];
1690
+ if (typeof arg !== "string") continue;
1691
+ if (!looksLikeFilePath(arg)) continue;
1692
+ const resolved = path6.isAbsolute(arg) ? arg : path6.resolve(hookCwd, arg);
1693
+ if (!await fileExists(resolved)) {
1694
+ errors.push({
1695
+ severity: "error",
1696
+ filePath: evalFilePath,
1697
+ location: `${location}.hooks.${hookName}.command[${i}]`,
1698
+ message: `Hook script not found: ${arg} (resolved to ${resolved})`
1699
+ });
1700
+ }
1701
+ }
1702
+ }
1703
+ }
1704
+ function looksLikeFilePath(arg) {
1705
+ if (arg.startsWith("./") || arg.startsWith("../")) return true;
1706
+ const scriptExtensions = [
1707
+ ".mjs",
1708
+ ".cjs",
1709
+ ".js",
1710
+ ".ts",
1711
+ ".sh",
1712
+ ".bash",
1713
+ ".zsh",
1714
+ ".py",
1715
+ ".rb",
1716
+ ".pl"
1717
+ ];
1718
+ return scriptExtensions.some((ext) => arg.endsWith(ext));
1719
+ }
1720
+ async function fileExists(filePath) {
1721
+ try {
1722
+ await access(filePath);
1723
+ return true;
1724
+ } catch {
1725
+ return false;
1726
+ }
1727
+ }
1487
1728
  export {
1488
1729
  detectFileType,
1489
1730
  getExpectedSchema,
1490
1731
  isValidSchema,
1732
+ validateCasesFile,
1491
1733
  validateConfigFile,
1492
1734
  validateEvalFile,
1493
1735
  validateFileReferences,
1494
- validateTargetsFile
1736
+ validateTargetsFile,
1737
+ validateWorkspacePaths
1495
1738
  };
1496
1739
  //# sourceMappingURL=index.js.map