@agentv/core 1.2.0 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. package/dist/{chunk-V3JCB3HI.js → chunk-4A6L2F6L.js} +11 -5
  2. package/dist/chunk-4A6L2F6L.js.map +1 -0
  3. package/dist/evaluation/validation/index.cjs +12 -0
  4. package/dist/evaluation/validation/index.cjs.map +1 -1
  5. package/dist/evaluation/validation/index.js +13 -1
  6. package/dist/evaluation/validation/index.js.map +1 -1
  7. package/dist/index.cjs +176 -8
  8. package/dist/index.cjs.map +1 -1
  9. package/dist/index.d.cts +1 -1
  10. package/dist/index.d.ts +1 -1
  11. package/dist/index.js +167 -5
  12. package/dist/index.js.map +1 -1
  13. package/package.json +1 -1
  14. package/dist/chunk-V3JCB3HI.js.map +0 -1
package/dist/evaluation/validation/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../src/evaluation/validation/file-type.ts","../../../src/evaluation/validation/eval-validator.ts","../../../src/evaluation/validation/targets-validator.ts","../../../src/evaluation/validation/config-validator.ts","../../../src/evaluation/validation/file-reference-validator.ts"],"sourcesContent":["import { readFile } from 'node:fs/promises';\nimport path from 'node:path';\nimport { parse } from 'yaml';\n\nimport type { FileType } from './types.js';\n\nconst SCHEMA_EVAL_V2 = 'agentv-eval-v2';\nconst SCHEMA_TARGETS_V2 = 'agentv-targets-v2.2';\nconst SCHEMA_CONFIG_V2 = 'agentv-config-v2';\n\n/**\n * Detect file type by reading $schema field from YAML file.\n * If $schema is missing, infers type from filename/path:\n * - config.yaml under .agentv folder → 'config'\n * - targets.yaml under .agentv folder → 'targets'\n * - All other YAML files → 'eval' (default)\n */\nexport async function detectFileType(filePath: string): Promise<FileType> {\n try {\n const content = await readFile(filePath, 'utf8');\n const parsed = parse(content) as unknown;\n\n if (typeof parsed !== 'object' || parsed === null) {\n return inferFileTypeFromPath(filePath);\n }\n\n const record = parsed as Record<string, unknown>;\n const schema = record.$schema;\n\n if (typeof schema !== 'string') {\n // No $schema field - infer from path\n return inferFileTypeFromPath(filePath);\n }\n\n switch (schema) {\n case SCHEMA_EVAL_V2:\n return 'eval';\n case SCHEMA_TARGETS_V2:\n return 'targets';\n case SCHEMA_CONFIG_V2:\n return 'config';\n default:\n // Unknown schema - infer from path\n return inferFileTypeFromPath(filePath);\n }\n } catch {\n return inferFileTypeFromPath(filePath);\n }\n}\n\n/**\n * Infer file type from filename and directory path.\n */\nfunction inferFileTypeFromPath(filePath: string): FileType {\n const normalized = path.normalize(filePath).replace(/\\\\/g, '/');\n const basename = path.basename(filePath);\n\n // Check if file is under .agentv folder\n if (normalized.includes('/.agentv/')) {\n if (basename === 'config.yaml' || basename === 'config.yml') {\n return 'config';\n }\n if (basename === 'targets.yaml' || basename === 'targets.yml') {\n return 'targets';\n }\n }\n\n // Default to eval file\n return 'eval';\n}\n\n/**\n * Check if a schema value is a valid AgentV schema identifier.\n */\nexport function isValidSchema(schema: unknown): boolean {\n return schema === SCHEMA_EVAL_V2 || schema === SCHEMA_TARGETS_V2 || schema === SCHEMA_CONFIG_V2;\n}\n\n/**\n * Get the expected schema for a file type.\n */\nexport function getExpectedSchema(fileType: FileType): string | undefined {\n switch (fileType) {\n case 'eval':\n return SCHEMA_EVAL_V2;\n case 'targets':\n return SCHEMA_TARGETS_V2;\n case 'config':\n return SCHEMA_CONFIG_V2;\n default:\n return undefined;\n }\n}\n","import { readFile } from 'node:fs/promises';\r\nimport path from 'node:path';\r\nimport { parse } from 'yaml';\r\n\r\nimport type { ValidationError, ValidationResult } from './types.js';\r\n\r\ntype JsonValue = string | number | boolean | null | JsonObject | JsonArray;\r\ntype JsonObject = { readonly [key: string]: JsonValue };\r\ntype JsonArray = readonly JsonValue[];\r\n\r\nfunction isObject(value: unknown): value is JsonObject {\r\n return typeof value === 'object' && value !== null && !Array.isArray(value);\r\n}\r\n\r\n/**\r\n * Validate an eval file (agentv-eval-v2 schema).\r\n */\r\nexport async function validateEvalFile(filePath: string): Promise<ValidationResult> {\r\n const errors: ValidationError[] = [];\r\n const absolutePath = path.resolve(filePath);\r\n\r\n let parsed: unknown;\r\n try {\r\n const content = await readFile(absolutePath, 'utf8');\r\n parsed = parse(content);\r\n } catch (error) {\r\n errors.push({\r\n severity: 'error',\r\n filePath: absolutePath,\r\n message: `Failed to parse YAML: ${(error as Error).message}`,\r\n });\r\n return {\r\n valid: false,\r\n filePath: absolutePath,\r\n fileType: 'eval',\r\n errors,\r\n };\r\n }\r\n\r\n if (!isObject(parsed)) {\r\n errors.push({\r\n severity: 'error',\r\n filePath: absolutePath,\r\n message: 'File must contain a YAML object',\r\n });\r\n return {\r\n valid: false,\r\n filePath: absolutePath,\r\n fileType: 'eval',\r\n errors,\r\n };\r\n }\r\n\r\n // Validate evalcases array\r\n const evalcases = parsed.evalcases;\r\n if (!Array.isArray(evalcases)) {\r\n errors.push({\r\n severity: 'error',\r\n filePath: absolutePath,\r\n location: 'evalcases',\r\n message: \"Missing or invalid 'evalcases' field (must be an array)\",\r\n });\r\n return {\r\n valid: errors.length === 0,\r\n filePath: absolutePath,\r\n fileType: 'eval',\r\n errors,\r\n };\r\n }\r\n\r\n // Validate each eval case\r\n for (let i = 0; i < evalcases.length; i++) {\r\n const evalCase = evalcases[i];\r\n const location = `evalcases[${i}]`;\r\n\r\n if (!isObject(evalCase)) {\r\n errors.push({\r\n severity: 'error',\r\n filePath: absolutePath,\r\n location,\r\n message: 'Eval case must be an object',\r\n });\r\n continue;\r\n }\r\n\r\n // Required fields: id, input_messages, expected_messages\r\n const id = evalCase.id;\r\n if (typeof id !== 'string' || id.trim().length === 0) {\r\n errors.push({\r\n severity: 'error',\r\n filePath: absolutePath,\r\n location: `${location}.id`,\r\n message: \"Missing or invalid 'id' field (must be a non-empty string)\",\r\n });\r\n }\r\n\r\n // Optional: expected_outcome or outcome for backward compatibility\r\n const expectedOutcome = evalCase.expected_outcome ?? evalCase.outcome;\r\n if (\r\n expectedOutcome !== undefined &&\r\n (typeof expectedOutcome !== 'string' || expectedOutcome.trim().length === 0)\r\n ) {\r\n errors.push({\r\n severity: 'error',\r\n filePath: absolutePath,\r\n location: `${location}.expected_outcome`,\r\n message:\r\n \"Invalid 'expected_outcome' or 'outcome' field (must be a non-empty string if provided)\",\r\n });\r\n }\r\n\r\n const inputMessages = evalCase.input_messages;\r\n if (!Array.isArray(inputMessages)) {\r\n errors.push({\r\n severity: 'error',\r\n filePath: absolutePath,\r\n location: `${location}.input_messages`,\r\n message: \"Missing or invalid 'input_messages' field (must be an array)\",\r\n });\r\n } else {\r\n validateMessages(inputMessages, `${location}.input_messages`, absolutePath, errors);\r\n }\r\n\r\n // expected_messages is optional - for outcome-only evaluation\r\n const expectedMessages = evalCase.expected_messages;\r\n if (expectedMessages !== undefined && !Array.isArray(expectedMessages)) {\r\n errors.push({\r\n severity: 'error',\r\n filePath: absolutePath,\r\n location: `${location}.expected_messages`,\r\n message: \"Invalid 'expected_messages' field (must be an array if provided)\",\r\n });\r\n } else if (Array.isArray(expectedMessages)) {\r\n validateMessages(expectedMessages, `${location}.expected_messages`, absolutePath, errors);\r\n }\r\n }\r\n\r\n return {\r\n valid: errors.length === 0,\r\n filePath: absolutePath,\r\n fileType: 'eval',\r\n errors,\r\n };\r\n}\r\n\r\nfunction validateMessages(\r\n messages: JsonArray,\r\n location: string,\r\n filePath: string,\r\n errors: ValidationError[],\r\n): void {\r\n for (let i = 0; i < messages.length; i++) {\r\n const message = messages[i];\r\n const msgLocation = `${location}[${i}]`;\r\n\r\n if (!isObject(message)) {\r\n errors.push({\r\n severity: 'error',\r\n filePath,\r\n location: msgLocation,\r\n message: 'Message must be an object',\r\n });\r\n continue;\r\n }\r\n\r\n // Validate role field\r\n const role = message.role;\r\n const validRoles = ['system', 'user', 'assistant'];\r\n if (!validRoles.includes(role as string)) {\r\n errors.push({\r\n severity: 'error',\r\n filePath,\r\n location: `${msgLocation}.role`,\r\n message: `Invalid role '${role}'. Must be one of: ${validRoles.join(', ')}`,\r\n });\r\n }\r\n\r\n // Validate content field (can be string or array)\r\n const content = message.content;\r\n if (typeof content === 'string') {\r\n validateContentForRoleMarkers(content, `${msgLocation}.content`, filePath, errors);\r\n } else if (Array.isArray(content)) {\r\n // Array content - validate each element\r\n for (let j = 0; j < content.length; j++) {\r\n const contentItem = content[j];\r\n const contentLocation = `${msgLocation}.content[${j}]`;\r\n\r\n if (typeof contentItem === 'string') {\r\n validateContentForRoleMarkers(contentItem, contentLocation, filePath, errors);\r\n } else if (isObject(contentItem)) {\r\n const type = contentItem.type;\r\n if (typeof type !== 'string') {\r\n errors.push({\r\n severity: 'error',\r\n filePath,\r\n location: `${contentLocation}.type`,\r\n message: \"Content object must have a 'type' field\",\r\n });\r\n }\r\n\r\n // For 'file' type, we'll validate existence later in file-reference-validator\r\n // For 'text' type, require 'value' field\r\n if (type === 'text') {\r\n const value = contentItem.value;\r\n if (typeof value !== 'string') {\r\n errors.push({\r\n severity: 'error',\r\n filePath,\r\n location: `${contentLocation}.value`,\r\n message: \"Content with type 'text' must have a 'value' field\",\r\n });\r\n } else {\r\n validateContentForRoleMarkers(value, `${contentLocation}.value`, filePath, errors);\r\n }\r\n }\r\n } else {\r\n errors.push({\r\n severity: 'error',\r\n filePath,\r\n location: contentLocation,\r\n message: 'Content array items must be strings or objects',\r\n });\r\n }\r\n }\r\n } else {\r\n errors.push({\r\n severity: 'error',\r\n filePath,\r\n location: `${msgLocation}.content`,\r\n message: \"Missing or invalid 'content' field (must be a string or array)\",\r\n });\r\n }\r\n }\r\n}\r\n\r\nfunction validateContentForRoleMarkers(\r\n content: string,\r\n location: string,\r\n filePath: string,\r\n errors: ValidationError[],\r\n): void {\r\n // Check for standard role markers that might confuse agentic providers\r\n const markers = ['@[System]:', '@[User]:', '@[Assistant]:', '@[Tool]:'];\r\n for (const marker of markers) {\r\n if (content.toLowerCase().includes(marker.toLowerCase())) {\r\n errors.push({\r\n severity: 'warning',\r\n filePath,\r\n location,\r\n message: `Content contains potential role marker '${marker}'. This may confuse agentic providers or cause prompt injection.`,\r\n });\r\n }\r\n }\r\n}\r\n","import { readFile } from 'node:fs/promises';\nimport path from 'node:path';\nimport { parse } from 'yaml';\n\nimport { CLI_PLACEHOLDERS } from '../providers/targets.js';\nimport { KNOWN_PROVIDERS, PROVIDER_ALIASES, TARGETS_SCHEMA_V2 } from '../providers/types.js';\nimport type { ValidationError, ValidationResult } from './types.js';\n\ntype JsonValue = string | number | boolean | null | JsonObject | JsonArray;\ntype JsonObject = { readonly [key: string]: JsonValue };\ntype JsonArray = readonly JsonValue[];\n\nfunction isObject(value: unknown): value is JsonObject {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n\n// Known settings properties for each provider type\nconst COMMON_SETTINGS = new Set(['provider_batching', 'providerBatching']);\n\nconst RETRY_SETTINGS = new Set([\n 'max_retries',\n 'maxRetries',\n 'retry_initial_delay_ms',\n 'retryInitialDelayMs',\n 'retry_max_delay_ms',\n 'retryMaxDelayMs',\n 'retry_backoff_factor',\n 'retryBackoffFactor',\n 'retry_status_codes',\n 'retryStatusCodes',\n]);\n\nconst AZURE_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n ...RETRY_SETTINGS,\n 'endpoint',\n 'resource',\n 'resourceName',\n 'api_key',\n 'apiKey',\n 'deployment',\n 'deploymentName',\n 'model',\n 'version',\n 'api_version',\n 'temperature',\n 'max_output_tokens',\n 'maxTokens',\n]);\n\nconst ANTHROPIC_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n ...RETRY_SETTINGS,\n 'api_key',\n 'apiKey',\n 'model',\n 'deployment',\n 'variant',\n 'temperature',\n 'max_output_tokens',\n 'maxTokens',\n 'thinking_budget',\n 'thinkingBudget',\n]);\n\nconst GEMINI_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n ...RETRY_SETTINGS,\n 'api_key',\n 'apiKey',\n 'model',\n 'deployment',\n 'variant',\n 'temperature',\n 'max_output_tokens',\n 'maxTokens',\n]);\n\nconst CODEX_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n 'executable',\n 'command',\n 'binary',\n 'args',\n 'arguments',\n 'cwd',\n 'timeout_seconds',\n 'timeoutSeconds',\n 'log_dir',\n 'logDir',\n 'log_directory',\n 'logDirectory',\n 'log_format',\n 'logFormat',\n 'log_output_format',\n 'logOutputFormat',\n]);\n\nconst VSCODE_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n 'workspace_template',\n 'workspaceTemplate',\n 'vscode_cmd',\n 'command',\n 'wait',\n 'dry_run',\n 'dryRun',\n 'subagent_root',\n 'subagentRoot',\n]);\n\nconst MOCK_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n 'response',\n 'delayMs',\n 'delayMinMs',\n 'delayMaxMs',\n 'trace', // For testing tool_trajectory evaluator\n]);\n\nconst CLI_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n 'command_template',\n 'commandTemplate',\n 'files_format',\n 'filesFormat',\n 'attachments_format',\n 'attachmentsFormat',\n 'cwd',\n 'env',\n 'timeout_seconds',\n 'timeoutSeconds',\n 'healthcheck',\n]);\n\nfunction getKnownSettings(provider: string): Set<string> | null {\n const normalizedProvider = provider.toLowerCase();\n switch (normalizedProvider) {\n case 'azure':\n case 'azure-openai':\n return AZURE_SETTINGS;\n case 'anthropic':\n return ANTHROPIC_SETTINGS;\n case 'gemini':\n case 'google':\n case 'google-gemini':\n return GEMINI_SETTINGS;\n case 'codex':\n case 'codex-cli':\n return CODEX_SETTINGS;\n case 'vscode':\n case 'vscode-insiders':\n return VSCODE_SETTINGS;\n case 'mock':\n return MOCK_SETTINGS;\n case 'cli':\n return CLI_SETTINGS;\n default:\n return null; // Unknown provider, can't validate settings\n }\n}\n\nfunction validateUnknownSettings(\n target: JsonObject,\n provider: string,\n absolutePath: string,\n location: string,\n errors: ValidationError[],\n): void {\n const knownSettings = getKnownSettings(provider);\n if (!knownSettings) {\n // Unknown provider, skip settings validation\n return;\n }\n\n // Known base target fields that aren't settings\n const baseFields = new Set(['name', 'provider', 'judge_target', 'workers', '$schema', 'targets']);\n\n for (const key of Object.keys(target)) {\n if (!baseFields.has(key) && !knownSettings.has(key)) {\n errors.push({\n severity: 'warning',\n filePath: absolutePath,\n location: `${location}.${key}`,\n message: `Unknown setting '${key}' for ${provider} provider. This property will be ignored.`,\n });\n }\n }\n}\n\n/**\n * Validate a targets file (agentv-targets-v2.1 schema).\n */\nexport async function validateTargetsFile(filePath: string): Promise<ValidationResult> {\n const errors: ValidationError[] = [];\n const absolutePath = path.resolve(filePath);\n\n let parsed: unknown;\n try {\n const content = await readFile(absolutePath, 'utf8');\n parsed = parse(content);\n } catch (error) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n message: `Failed to parse YAML: ${(error as Error).message}`,\n });\n return {\n valid: false,\n filePath: absolutePath,\n fileType: 'targets',\n errors,\n };\n }\n\n function validateCliSettings(\n target: JsonObject,\n absolutePath: string,\n location: string,\n errors: ValidationError[],\n ): void {\n const commandTemplate = target.command_template ?? target.commandTemplate;\n if (typeof commandTemplate !== 'string' || commandTemplate.trim().length === 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.commandTemplate`,\n message: \"CLI provider requires 'commandTemplate' as a non-empty string\",\n });\n } else {\n recordUnknownPlaceholders(\n commandTemplate,\n absolutePath,\n `${location}.commandTemplate`,\n errors,\n );\n }\n\n const attachmentsFormat = target.attachments_format ?? target.attachmentsFormat;\n if (attachmentsFormat !== undefined && typeof attachmentsFormat !== 'string') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.attachmentsFormat`,\n message: \"'attachmentsFormat' must be a string when provided\",\n });\n }\n\n const filesFormat = target.files_format ?? target.filesFormat;\n if (filesFormat !== undefined && typeof filesFormat !== 'string') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.filesFormat`,\n message: \"'filesFormat' must be a string when provided\",\n });\n }\n\n const cwd = target.cwd;\n if (cwd !== undefined && typeof cwd !== 'string') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.cwd`,\n message: \"'cwd' must be a string when provided\",\n });\n }\n\n const timeoutSeconds = target.timeout_seconds ?? target.timeoutSeconds;\n if (timeoutSeconds !== undefined) {\n const numericTimeout = Number(timeoutSeconds);\n if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.timeoutSeconds`,\n message: \"'timeoutSeconds' must be a positive number when provided\",\n });\n }\n }\n\n const healthcheck = target.healthcheck;\n if (healthcheck !== undefined) {\n validateCliHealthcheck(healthcheck, absolutePath, `${location}.healthcheck`, errors);\n }\n }\n\n function validateCliHealthcheck(\n healthcheck: unknown,\n absolutePath: string,\n location: string,\n errors: ValidationError[],\n ): void {\n if (!isObject(healthcheck)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location,\n message: \"'healthcheck' must be an object when provided\",\n });\n return;\n }\n\n const type = healthcheck.type;\n if (type !== 'http' && type !== 'command') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.type`,\n message: \"healthcheck.type must be either 'http' or 'command'\",\n });\n return;\n }\n\n const timeoutSeconds = healthcheck.timeout_seconds ?? healthcheck.timeoutSeconds;\n if (timeoutSeconds !== undefined) {\n const numericTimeout = Number(timeoutSeconds);\n if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.timeoutSeconds`,\n message: 'healthcheck.timeoutSeconds must be a positive number when provided',\n });\n }\n }\n\n if (type === 'http') {\n const url = healthcheck.url;\n if (typeof url !== 'string' || url.trim().length === 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.url`,\n message: 'healthcheck.url must be a non-empty string for http checks',\n });\n }\n return;\n }\n\n const commandTemplate = healthcheck.command_template ?? healthcheck.commandTemplate;\n if (typeof commandTemplate !== 'string' || commandTemplate.trim().length === 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.commandTemplate`,\n message: 'healthcheck.commandTemplate must be a non-empty string for command checks',\n });\n } else {\n recordUnknownPlaceholders(\n commandTemplate,\n absolutePath,\n `${location}.commandTemplate`,\n errors,\n );\n }\n\n const cwd = healthcheck.cwd;\n if (cwd !== undefined && typeof cwd !== 'string') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.cwd`,\n message: 'healthcheck.cwd must be a string when provided',\n });\n }\n }\n\n function recordUnknownPlaceholders(\n template: string,\n absolutePath: string,\n location: string,\n errors: ValidationError[],\n ): void {\n const placeholders = extractPlaceholders(template);\n for (const placeholder of placeholders) {\n if (!CLI_PLACEHOLDERS.has(placeholder)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location,\n message: `Unknown CLI placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS).join(', ')}`,\n });\n }\n }\n }\n\n function extractPlaceholders(template: string): string[] {\n const matches = template.matchAll(/\\{([A-Z_]+)\\}/g);\n const result: string[] = [];\n for (const match of matches) {\n const placeholder = match[1];\n if (placeholder) {\n result.push(placeholder);\n }\n }\n return result;\n }\n\n if (!isObject(parsed)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n message: 'File must contain a YAML object',\n });\n return {\n valid: false,\n filePath: absolutePath,\n fileType: 'targets',\n errors,\n };\n }\n\n // Validate targets array\n const targets = parsed.targets;\n if (!Array.isArray(targets)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: 'targets',\n message: \"Missing or invalid 'targets' field (must be an array)\",\n });\n return {\n valid: errors.length === 0,\n filePath: absolutePath,\n fileType: 'targets',\n errors,\n };\n }\n\n // Validate each target definition\n const knownProviders = [...KNOWN_PROVIDERS, ...PROVIDER_ALIASES];\n\n for (let i = 0; i < targets.length; i++) {\n const target = targets[i];\n const location = `targets[${i}]`;\n\n if (!isObject(target)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location,\n message: 'Target must be an object',\n });\n continue;\n }\n\n // Required field: name\n const name = target.name;\n if (typeof name !== 'string' || name.trim().length === 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.name`,\n message: \"Missing or invalid 'name' field (must be a non-empty string)\",\n });\n }\n\n // Required field: provider\n const provider = target.provider;\n const providerValue = typeof provider === 'string' ? provider.trim().toLowerCase() : undefined;\n if (typeof provider !== 'string' || provider.trim().length === 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.provider`,\n message: \"Missing or invalid 'provider' field (must be a non-empty string)\",\n });\n } else if (!knownProviders.includes(provider)) {\n // Warning for unknown providers (non-fatal)\n errors.push({\n severity: 'warning',\n filePath: absolutePath,\n location: `${location}.provider`,\n message: `Unknown provider '${provider}'. Known providers: ${knownProviders.join(', ')}`,\n });\n }\n\n // Validate CLI provider fields\n if (providerValue === 'cli') {\n validateCliSettings(target, absolutePath, location, errors);\n }\n\n // Check for unknown settings properties on target object\n if (typeof provider === 'string') {\n validateUnknownSettings(target, provider, absolutePath, location, errors);\n }\n\n // Optional field: judge_target (must be string if present)\n const judgeTarget = target.judge_target;\n if (judgeTarget !== undefined && typeof judgeTarget !== 'string') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.judge_target`,\n message: \"Invalid 'judge_target' field (must be a string)\",\n });\n }\n }\n\n return {\n valid: errors.filter((e) => e.severity === 'error').length === 0,\n filePath: absolutePath,\n fileType: 'targets',\n errors,\n };\n}\n","import { readFile } from 'node:fs/promises';\nimport { parse } from 'yaml';\n\nimport type { ValidationError, ValidationResult } from './types.js';\n\nconst SCHEMA_CONFIG_V2 = 'agentv-config-v2';\n\n/**\n * Validate a config.yaml file for schema compliance and structural correctness.\n */\nexport async function validateConfigFile(filePath: string): Promise<ValidationResult> {\n const errors: ValidationError[] = [];\n\n try {\n const content = await readFile(filePath, 'utf8');\n const parsed = parse(content) as unknown;\n\n // Check if parsed content is an object\n if (typeof parsed !== 'object' || parsed === null) {\n errors.push({\n severity: 'error',\n filePath,\n message: 'Config file must contain a valid YAML object',\n });\n return { valid: false, filePath, fileType: 'config', errors };\n }\n\n const config = parsed as Record<string, unknown>;\n\n // Validate $schema field (optional, but if present must be correct)\n const schema = config.$schema;\n if (schema !== undefined && schema !== SCHEMA_CONFIG_V2) {\n const message = `Invalid $schema value '${schema}'. Expected '${SCHEMA_CONFIG_V2}' or omit the field.`;\n errors.push({\n severity: 'error',\n filePath,\n location: '$schema',\n message,\n });\n }\n\n // Validate guideline_patterns if present\n const guidelinePatterns = config.guideline_patterns;\n if (guidelinePatterns !== undefined) {\n if (!Array.isArray(guidelinePatterns)) {\n errors.push({\n severity: 'error',\n filePath,\n location: 'guideline_patterns',\n message: \"Field 'guideline_patterns' must be an array\",\n });\n } else if (!guidelinePatterns.every((p) => typeof p === 'string')) {\n errors.push({\n severity: 'error',\n filePath,\n location: 'guideline_patterns',\n message: \"All entries in 'guideline_patterns' must be strings\",\n });\n } else if (guidelinePatterns.length === 0) {\n errors.push({\n severity: 'warning',\n filePath,\n location: 'guideline_patterns',\n message: \"Field 'guideline_patterns' is empty. Consider removing it or adding patterns.\",\n });\n }\n }\n\n // Check for unexpected fields\n const allowedFields = new Set(['$schema', 'guideline_patterns']);\n const unexpectedFields = Object.keys(config).filter((key) => !allowedFields.has(key));\n\n if (unexpectedFields.length > 0) {\n errors.push({\n severity: 'warning',\n filePath,\n message: `Unexpected fields: ${unexpectedFields.join(', ')}`,\n });\n }\n\n return {\n valid: errors.filter((e) => e.severity === 'error').length === 0,\n filePath,\n fileType: 'config',\n errors,\n };\n } catch (error) {\n errors.push({\n severity: 'error',\n filePath,\n message: `Failed to parse config file: ${(error as Error).message}`,\n });\n return { valid: false, filePath, fileType: 'config', errors };\n }\n}\n","import { readFile } from 'node:fs/promises';\nimport path from 'node:path';\nimport { parse } from 'yaml';\n\nimport { buildSearchRoots, findGitRoot, resolveFileReference } from '../file-utils.js';\nimport type { ValidationError } from './types.js';\n\ntype JsonValue = string | number | boolean | null | JsonObject | JsonArray;\ntype JsonObject = { readonly [key: string]: JsonValue };\ntype JsonArray = readonly JsonValue[];\n\nfunction isObject(value: unknown): value is JsonObject {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n\n/**\n * Validate that file references in eval file content exist.\n * Checks content blocks with type: \"file\" and validates the referenced file exists.\n * Also checks that referenced files are not empty.\n */\nexport async function validateFileReferences(\n evalFilePath: string,\n): Promise<readonly ValidationError[]> {\n const errors: ValidationError[] = [];\n const absolutePath = path.resolve(evalFilePath);\n\n // Find git root and build search roots (same as yaml-parser does at runtime)\n const gitRoot = await findGitRoot(absolutePath);\n if (!gitRoot) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n message: 'Cannot validate file references: git repository root not found',\n });\n return errors;\n }\n\n const searchRoots = buildSearchRoots(absolutePath, gitRoot);\n\n let parsed: unknown;\n try {\n const content = await readFile(absolutePath, 'utf8');\n parsed = parse(content);\n } catch {\n // Parse errors are already caught by eval-validator\n return errors;\n }\n\n if (!isObject(parsed)) {\n return errors;\n }\n\n const evalcases = parsed.evalcases;\n if (!Array.isArray(evalcases)) {\n return errors;\n }\n\n for (let i = 0; i < evalcases.length; i++) {\n const evalCase = evalcases[i];\n if (!isObject(evalCase)) {\n continue;\n }\n\n // Check input_messages\n const inputMessages = evalCase.input_messages;\n if (Array.isArray(inputMessages)) {\n await validateMessagesFileRefs(\n inputMessages,\n `evalcases[${i}].input_messages`,\n searchRoots,\n absolutePath,\n errors,\n );\n }\n\n // Check expected_messages\n const expectedMessages = evalCase.expected_messages;\n if (Array.isArray(expectedMessages)) {\n await validateMessagesFileRefs(\n expectedMessages,\n `evalcases[${i}].expected_messages`,\n searchRoots,\n absolutePath,\n errors,\n );\n }\n }\n\n return errors;\n}\n\nasync function validateMessagesFileRefs(\n messages: JsonArray,\n location: string,\n searchRoots: readonly string[],\n filePath: string,\n errors: ValidationError[],\n): Promise<void> {\n for (let i = 0; i < messages.length; i++) {\n const message = messages[i];\n if (!isObject(message)) {\n continue;\n }\n\n const content = message.content;\n if (typeof content === 'string') {\n continue;\n }\n\n if (!Array.isArray(content)) {\n continue;\n }\n\n for (let j = 0; j < content.length; j++) {\n const contentItem = content[j];\n if (!isObject(contentItem)) {\n continue;\n }\n\n const type = contentItem.type;\n if (type !== 'file') {\n continue;\n }\n\n const value = contentItem.value;\n if (typeof value !== 'string') {\n errors.push({\n severity: 'error',\n filePath,\n location: `${location}[${i}].content[${j}].value`,\n message: \"File reference must have a 'value' field with the file path\",\n });\n continue;\n }\n\n // Use the same file resolution logic as yaml-parser at runtime\n const { resolvedPath } = await resolveFileReference(value, searchRoots);\n\n if (!resolvedPath) {\n errors.push({\n severity: 'error',\n filePath,\n location: `${location}[${i}].content[${j}]`,\n message: `Referenced file not found: ${value}`,\n });\n } else {\n // Check that file is not empty\n try {\n const fileContent = await readFile(resolvedPath, 'utf8');\n if (fileContent.trim().length === 0) {\n errors.push({\n severity: 'warning',\n filePath,\n location: `${location}[${i}].content[${j}]`,\n message: `Referenced file is empty: ${value}`,\n });\n }\n } catch (error) {\n errors.push({\n severity: 'error',\n filePath,\n location: `${location}[${i}].content[${j}]`,\n message: `Cannot read referenced file: ${value} (${(error as Error).message})`,\n });\n }\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;AAAA,SAAS,gBAAgB;AACzB,OAAO,UAAU;AACjB,SAAS,aAAa;AAItB,IAAM,iBAAiB;AACvB,IAAM,oBAAoB;AAC1B,IAAM,mBAAmB;AASzB,eAAsB,eAAe,UAAqC;AACxE,MAAI;AACF,UAAM,UAAU,MAAM,SAAS,UAAU,MAAM;AAC/C,UAAM,SAAS,MAAM,OAAO;AAE5B,QAAI,OAAO,WAAW,YAAY,WAAW,MAAM;AACjD,aAAO,sBAAsB,QAAQ;AAAA,IACvC;AAEA,UAAM,SAAS;AACf,UAAM,SAAS,OAAO;AAEtB,QAAI,OAAO,WAAW,UAAU;AAE9B,aAAO,sBAAsB,QAAQ;AAAA,IACvC;AAEA,YAAQ,QAAQ;AAAA,MACd,KAAK;AACH,eAAO;AAAA,MACT,KAAK;AACH,eAAO;AAAA,MACT,KAAK;AACH,eAAO;AAAA,MACT;AAEE,eAAO,sBAAsB,QAAQ;AAAA,IACzC;AAAA,EACF,QAAQ;AACN,WAAO,sBAAsB,QAAQ;AAAA,EACvC;AACF;AAKA,SAAS,sBAAsB,UAA4B;AACzD,QAAM,aAAa,KAAK,UAAU,QAAQ,EAAE,QAAQ,OAAO,GAAG;AAC9D,QAAM,WAAW,KAAK,SAAS,QAAQ;AAGvC,MAAI,WAAW,SAAS,WAAW,GAAG;AACpC,QAAI,aAAa,iBAAiB,aAAa,cAAc;AAC3D,aAAO;AAAA,IACT;AACA,QAAI,aAAa,kBAAkB,aAAa,eAAe;AAC7D,aAAO;AAAA,IACT;AAAA,EACF;AAGA,SAAO;AACT;AAKO,SAAS,cAAc,QAA0B;AACtD,SAAO,WAAW,kBAAkB,WAAW,qBAAqB,WAAW;AACjF;AAKO,SAAS,kBAAkB,UAAwC;AACxE,UAAQ,UAAU;AAAA,IAChB,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT;AACE,aAAO;AAAA,EACX;AACF;;;AC5FA,SAAS,YAAAA,iBAAgB;AACzB,OAAOC,WAAU;AACjB,SAAS,SAAAC,cAAa;AAQtB,SAAS,SAAS,OAAqC;AACrD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAKA,eAAsB,iBAAiB,UAA6C;AAClF,QAAM,SAA4B,CAAC;AACnC,QAAM,eAAeD,MAAK,QAAQ,QAAQ;AAE1C,MAAI;AACJ,MAAI;AACF,UAAM,UAAU,MAAMD,UAAS,cAAc,MAAM;AACnD,aAASE,OAAM,OAAO;AAAA,EACxB,SAAS,OAAO;AACd,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS,yBAA0B,MAAgB,OAAO;AAAA,IAC5D,CAAC;AACD,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAEA,MAAI,CAAC,SAAS,MAAM,GAAG;AACrB,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AACD,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAGA,QAAM,YAAY,OAAO;AACzB,MAAI,CAAC,MAAM,QAAQ,SAAS,GAAG;AAC7B,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AACD,WAAO;AAAA,MACL,OAAO,OAAO,WAAW;AAAA,MACzB,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAGA,WAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,UAAM,WAAW,UAAU,CAAC;AAC5B,UAAM,WAAW,aAAa,CAAC;AAE/B,QAAI,CAAC,SAAS,QAAQ,GAAG;AACvB,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAGA,UAAM,KAAK,SAAS;AACpB,QAAI,OAAO,OAAO,YAAY,GAAG,KAAK,EAAE,WAAW,GAAG;AACpD,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAGA,UAAM,kBAAkB,SAAS,oBAAoB,SAAS;AAC9D,QACE,oBAAoB,WACnB,OAAO,oBAAoB,YAAY,gBAAgB,KAAK,EAAE,WAAW,IAC1E;AACA,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SACE;AAAA,MACJ,CAAC;AAAA,IACH;AAEA,UAAM,gBAAgB,SAAS;AAC/B,QAAI,CAAC,MAAM,QAAQ,aAAa,GAAG;AACjC,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,OAAO;AACL,uBAAiB,eAAe,GAAG,QAAQ,mBAAmB,cAAc,MAAM;AAAA,IACpF;AAGA,UAAM,mBAAmB,SAAS;AAClC,QAAI,qBAAqB,UAAa,CAAC,MAAM,QAAQ,gBAAgB,GAAG;AACtE,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,WAAW,MAAM,QAAQ,gBAAgB,GAAG;AAC1C,uBAAiB,kBAAkB,GAAG,QAAQ,sBAAsB,cAAc,MAAM;AAAA,IAC1F;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO,OAAO,WAAW;AAAA,IACzB,UAAU;AAAA,IACV,UAAU;AAAA,IACV;AAAA,EACF;AACF;AAEA,SAAS,iBACP,UACA,UACA,UACA,QACM;AACN,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,UAAM,UAAU,SAAS,CAAC;AAC1B,UAAM,cAAc,GAAG,QAAQ,IAAI,CAAC;AAEpC,QAAI,CAAC,SAAS,OAAO,GAAG;AACtB,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,UAAU;AAAA,QACV,SAAS;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAGA,UAAM,OAAO,QAAQ;AACrB,UAAM,aAAa,CAAC,UAAU,QAAQ,WAAW;AACjD,QAAI,CAAC,WAAW,SAAS,IAAc,GAAG;AACxC,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,UAAU,GAAG,WAAW;AAAA,QACxB,SAAS,iBAAiB,IAAI,sBAAsB,WAAW,KAAK,IAAI,CAAC;AAAA,MAC3E,CAAC;AAAA,IACH;AAGA,UAAM,UAAU,QAAQ;AACxB,QAAI,OAAO,YAAY,UAAU;AAC/B,oCAA8B,SAAS,GAAG,WAAW,YAAY,UAAU,MAAM;AAAA,IACnF,WAAW,MAAM,QAAQ,OAAO,GAAG;AAEjC,eAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,cAAM,cAAc,QAAQ,CAAC;AAC7B,cAAM,kBAAkB,GAAG,WAAW,YAAY,CAAC;AAEnD,YAAI,OAAO,gBAAgB,UAAU;AACnC,wCAA8B,aAAa,iBAAiB,UAAU,MAAM;AAAA,QAC9E,WAAW,SAAS,WAAW,GAAG;AAChC,gBAAM,OAAO,YAAY;AACzB,cAAI,OAAO,SAAS,UAAU;AAC5B,mBAAO,KAAK;AAAA,cACV,UAAU;AAAA,cACV;AAAA,cACA,UAAU,GAAG,eAAe;AAAA,cAC5B,SAAS;AAAA,YACX,CAAC;AAAA,UACH;AAIA,cAAI,SAAS,QAAQ;AACnB,kBAAM,QAAQ,YAAY;AAC1B,gBAAI,OAAO,UAAU,UAAU;AAC7B,qBAAO,KAAK;AAAA,gBACV,UAAU;AAAA,gBACV;AAAA,gBACA,UAAU,GAAG,eAAe;AAAA,gBAC5B,SAAS;AAAA,cACX,CAAC;AAAA,YACH,OAAO;AACL,4CAA8B,OAAO,GAAG,eAAe,UAAU,UAAU,MAAM;AAAA,YACnF;AAAA,UACF;AAAA,QACF,OAAO;AACL,iBAAO,KAAK;AAAA,YACV,UAAU;AAAA,YACV;AAAA,YACA,UAAU;AAAA,YACV,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,OAAO;AACL,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,UAAU,GAAG,WAAW;AAAA,QACxB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAEA,SAAS,8BACP,SACA,UACA,UACA,QACM;AAEN,QAAM,UAAU,CAAC,cAAc,YAAY,iBAAiB,UAAU;AACtE,aAAW,UAAU,SAAS;AAC5B,QAAI,QAAQ,YAAY,EAAE,SAAS,OAAO,YAAY,CAAC,GAAG;AACxD,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA;AAAA,QACA,SAAS,2CAA2C,MAAM;AAAA,MAC5D,CAAC;AAAA,IACH;AAAA,EACF;AACF;;;AC7PA,SAAS,YAAAC,iBAAgB;AACzB,OAAOC,WAAU;AACjB,SAAS,SAAAC,cAAa;AAUtB,SAASC,UAAS,OAAqC;AACrD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAGA,IAAM,kBAAkB,oBAAI,IAAI,CAAC,qBAAqB,kBAAkB,CAAC;AAEzE,IAAM,iBAAiB,oBAAI,IAAI;AAAA,EAC7B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,iBAAiB,oBAAI,IAAI;AAAA,EAC7B,GAAG;AAAA,EACH,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,qBAAqB,oBAAI,IAAI;AAAA,EACjC,GAAG;AAAA,EACH,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC9B,GAAG;AAAA,EACH,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,iBAAiB,oBAAI,IAAI;AAAA,EAC7B,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC9B,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,gBAAgB,oBAAI,IAAI;AAAA,EAC5B,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AACF,CAAC;AAED,IAAM,eAAe,oBAAI,IAAI;AAAA,EAC3B,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,SAAS,iBAAiB,UAAsC;AAC9D,QAAM,qBAAqB,SAAS,YAAY;AAChD,UAAQ,oBAAoB;AAAA,IAC1B,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT;AACE,aAAO;AAAA,EACX;AACF;AAEA,SAAS,wBACP,QACA,UACA,cACA,UACA,QACM;AACN,QAAM,gBAAgB,iBAAiB,QAAQ;AAC/C,MAAI,CAAC,eAAe;AAElB;AAAA,EACF;AAGA,QAAM,aAAa,oBAAI,IAAI,CAAC,QAAQ,YAAY,gBAAgB,WAAW,WAAW,SAAS,CAAC;AAEhG,aAAW,OAAO,OAAO,KAAK,MAAM,GAAG;AACrC,QAAI,CAAC,WAAW,IAAI,GAAG,KAAK,CAAC,cAAc,IAAI,GAAG,GAAG;AACnD,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ,IAAI,GAAG;AAAA,QAC5B,SAAS,oBAAoB,GAAG,SAAS,QAAQ;AAAA,MACnD,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAKA,eAAsB,oBAAoB,UAA6C;AACrF,QAAM,SAA4B,CAAC;AACnC,QAAM,eAAeC,MAAK,QAAQ,QAAQ;AAE1C,MAAI;AACJ,MAAI;AACF,UAAM,UAAU,MAAMC,UAAS,cAAc,MAAM;AACnD,aAASC,OAAM,OAAO;AAAA,EACxB,SAAS,OAAO;AACd,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS,yBAA0B,MAAgB,OAAO;AAAA,IAC5D,CAAC;AACD,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBACP,QACAC,eACA,UACAC,SACM;AACN,UAAM,kBAAkB,OAAO,oBAAoB,OAAO;AAC1D,QAAI,OAAO,oBAAoB,YAAY,gBAAgB,KAAK,EAAE,WAAW,GAAG;AAC9E,MAAAA,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,OAAO;AACL;AAAA,QACE;AAAA,QACAA;AAAA,QACA,GAAG,QAAQ;AAAA,QACXC;AAAA,MACF;AAAA,IACF;AAEA,UAAM,oBAAoB,OAAO,sBAAsB,OAAO;AAC9D,QAAI,sBAAsB,UAAa,OAAO,sBAAsB,UAAU;AAC5E,MAAAA,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAEA,UAAM,cAAc,OAAO,gBAAgB,OAAO;AAClD,QAAI,gBAAgB,UAAa,OAAO,gBAAgB,UAAU;AAChE,MAAAC,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAEA,UAAM,MAAM,OAAO;AACnB,QAAI,QAAQ,UAAa,OAAO,QAAQ,UAAU;AAChD,MAAAC,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAEA,UAAM,iBAAiB,OAAO,mBAAmB,OAAO;AACxD,QAAI,mBAAmB,QAAW;AAChC,YAAM,iBAAiB,OAAO,cAAc;AAC5C,UAAI,CAAC,OAAO,SAAS,cAAc,KAAK,kBAAkB,GAAG;AAC3D,QAAAC,QAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV,UAAUD;AAAA,UACV,UAAU,GAAG,QAAQ;AAAA,UACrB,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAAA,IACF;AAEA,UAAM,cAAc,OAAO;AAC3B,QAAI,gBAAgB,QAAW;AAC7B,6BAAuB,aAAaA,eAAc,GAAG,QAAQ,gBAAgBC,OAAM;AAAA,IACrF;AAAA,EACF;AAEA,WAAS,uBACP,aACAD,eACA,UACAC,SACM;AACN,QAAI,CAACL,UAAS,WAAW,GAAG;AAC1B,MAAAK,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,OAAO,YAAY;AACzB,QAAI,SAAS,UAAU,SAAS,WAAW;AACzC,MAAAC,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,iBAAiB,YAAY,mBAAmB,YAAY;AAClE,QAAI,mBAAmB,QAAW;AAChC,YAAM,iBAAiB,OAAO,cAAc;AAC5C,UAAI,CAAC,OAAO,SAAS,cAAc,KAAK,kBAAkB,GAAG;AAC3D,QAAAC,QAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV,UAAUD;AAAA,UACV,UAAU,GAAG,QAAQ;AAAA,UACrB,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAAA,IACF;AAEA,QAAI,SAAS,QAAQ;AACnB,YAAM,MAAM,YAAY;AACxB,UAAI,OAAO,QAAQ,YAAY,IAAI,KAAK,EAAE,WAAW,GAAG;AACtD,QAAAC,QAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV,UAAUD;AAAA,UACV,UAAU,GAAG,QAAQ;AAAA,UACrB,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AACA;AAAA,IACF;AAEA,UAAM,kBAAkB,YAAY,oBAAoB,YAAY;AACpE,QAAI,OAAO,oBAAoB,YAAY,gBAAgB,KAAK,EAAE,WAAW,GAAG;AAC9E,MAAAC,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,OAAO;AACL;AAAA,QACE;AAAA,QACAA;AAAA,QACA,GAAG,QAAQ;AAAA,QACXC;AAAA,MACF;AAAA,IACF;AAEA,UAAM,MAAM,YAAY;AACxB,QAAI,QAAQ,UAAa,OAAO,QAAQ,UAAU;AAChD,MAAAA,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAAA,EACF;AAEA,WAAS,0BACP,UACAA,eACA,UACAC,SACM;AACN,UAAM,eAAe,oBAAoB,QAAQ;AACjD,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,iBAAiB,IAAI,WAAW,GAAG;AACtC,QAAAA,QAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV,UAAUD;AAAA,UACV;AAAA,UACA,SAAS,6BAA6B,WAAW,+BAA+B,MAAM,KAAK,gBAAgB,EAAE,KAAK,IAAI,CAAC;AAAA,QACzH,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBAAoB,UAA4B;AACvD,UAAM,UAAU,SAAS,SAAS,gBAAgB;AAClD,UAAM,SAAmB,CAAC;AAC1B,eAAW,SAAS,SAAS;AAC3B,YAAM,cAAc,MAAM,CAAC;AAC3B,UAAI,aAAa;AACf,eAAO,KAAK,WAAW;AAAA,MACzB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,MAAI,CAACJ,UAAS,MAAM,GAAG;AACrB,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AACD,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAGA,QAAM,UAAU,OAAO;AACvB,MAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;AAC3B,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AACD,WAAO;AAAA,MACL,OAAO,OAAO,WAAW;AAAA,MACzB,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAGA,QAAM,iBAAiB,CAAC,GAAG,iBAAiB,GAAG,gBAAgB;AAE/D,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,UAAM,SAAS,QAAQ,CAAC;AACxB,UAAM,WAAW,WAAW,CAAC;AAE7B,QAAI,CAACA,UAAS,MAAM,GAAG;AACrB,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAGA,UAAM,OAAO,OAAO;AACpB,QAAI,OAAO,SAAS,YAAY,KAAK,KAAK,EAAE,WAAW,GAAG;AACxD,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAGA,UAAM,WAAW,OAAO;AACxB,UAAM,gBAAgB,OAAO,aAAa,WAAW,SAAS,KAAK,EAAE,YAAY,IAAI;AACrF,QAAI,OAAO,aAAa,YAAY,SAAS,KAAK,EAAE,WAAW,GAAG;AAChE,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,WAAW,CAAC,eAAe,SAAS,QAAQ,GAAG;AAE7C,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS,qBAAqB,QAAQ,uBAAuB,eAAe,KAAK,IAAI,CAAC;AAAA,MACxF,CAAC;AAAA,IACH;AAGA,QAAI,kBAAkB,OAAO;AAC3B,0BAAoB,QAAQ,cAAc,UAAU,MAAM;AAAA,IAC5D;AAGA,QAAI,OAAO,aAAa,UAAU;AAChC,8BAAwB,QAAQ,UAAU,cAAc,UAAU,MAAM;AAAA,IAC1E;AAGA,UAAM,cAAc,OAAO;AAC3B,QAAI,gBAAgB,UAAa,OAAO,gBAAgB,UAAU;AAChE,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO,OAAO,OAAO,CAAC,MAAM,EAAE,aAAa,OAAO,EAAE,WAAW;AAAA,IAC/D,UAAU;AAAA,IACV,UAAU;AAAA,IACV;AAAA,EACF;AACF;;;AC1fA,SAAS,YAAAM,iBAAgB;AACzB,SAAS,SAAAC,cAAa;AAItB,IAAMC,oBAAmB;AAKzB,eAAsB,mBAAmB,UAA6C;AACpF,QAAM,SAA4B,CAAC;AAEnC,MAAI;AACF,UAAM,UAAU,MAAMF,UAAS,UAAU,MAAM;AAC/C,UAAM,SAASC,OAAM,OAAO;AAG5B,QAAI,OAAO,WAAW,YAAY,WAAW,MAAM;AACjD,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,aAAO,EAAE,OAAO,OAAO,UAAU,UAAU,UAAU,OAAO;AAAA,IAC9D;AAEA,UAAM,SAAS;AAGf,UAAM,SAAS,OAAO;AACtB,QAAI,WAAW,UAAa,WAAWC,mBAAkB;AACvD,YAAM,UAAU,0BAA0B,MAAM,gBAAgBA,iBAAgB;AAChF,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,UAAU;AAAA,QACV;AAAA,MACF,CAAC;AAAA,IACH;AAGA,UAAM,oBAAoB,OAAO;AACjC,QAAI,sBAAsB,QAAW;AACnC,UAAI,CAAC,MAAM,QAAQ,iBAAiB,GAAG;AACrC,eAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV;AAAA,UACA,UAAU;AAAA,UACV,SAAS;AAAA,QACX,CAAC;AAAA,MACH,WAAW,CAAC,kBAAkB,MAAM,CAAC,MAAM,OAAO,MAAM,QAAQ,GAAG;AACjE,eAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV;AAAA,UACA,UAAU;AAAA,UACV,SAAS;AAAA,QACX,CAAC;AAAA,MACH,WAAW,kBAAkB,WAAW,GAAG;AACzC,eAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV;AAAA,UACA,UAAU;AAAA,UACV,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAAA,IACF;AAGA,UAAM,gBAAgB,oBAAI,IAAI,CAAC,WAAW,oBAAoB,CAAC;AAC/D,UAAM,mBAAmB,OAAO,KAAK,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,IAAI,GAAG,CAAC;AAEpF,QAAI,iBAAiB,SAAS,GAAG;AAC/B,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,SAAS,sBAAsB,iBAAiB,KAAK,IAAI,CAAC;AAAA,MAC5D,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,MACL,OAAO,OAAO,OAAO,CAAC,MAAM,EAAE,aAAa,OAAO,EAAE,WAAW;AAAA,MAC/D;AAAA,MACA,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF,SAAS,OAAO;AACd,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV;AAAA,MACA,SAAS,gCAAiC,MAAgB,OAAO;AAAA,IACnE,CAAC;AACD,WAAO,EAAE,OAAO,OAAO,UAAU,UAAU,UAAU,OAAO;AAAA,EAC9D;AACF;;;AC9FA,SAAS,YAAAC,iBAAgB;AACzB,OAAOC,WAAU;AACjB,SAAS,SAAAC,cAAa;AAStB,SAASC,UAAS,OAAqC;AACrD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAOA,eAAsB,uBACpB,cACqC;AACrC,QAAM,SAA4B,CAAC;AACnC,QAAM,eAAeC,MAAK,QAAQ,YAAY;AAG9C,QAAM,UAAU,MAAM,YAAY,YAAY;AAC9C,MAAI,CAAC,SAAS;AACZ,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AACD,WAAO;AAAA,EACT;AAEA,QAAM,cAAc,iBAAiB,cAAc,OAAO;AAE1D,MAAI;AACJ,MAAI;AACF,UAAM,UAAU,MAAMC,UAAS,cAAc,MAAM;AACnD,aAASC,OAAM,OAAO;AAAA,EACxB,QAAQ;AAEN,WAAO;AAAA,EACT;AAEA,MAAI,CAACH,UAAS,MAAM,GAAG;AACrB,WAAO;AAAA,EACT;AAEA,QAAM,YAAY,OAAO;AACzB,MAAI,CAAC,MAAM,QAAQ,SAAS,GAAG;AAC7B,WAAO;AAAA,EACT;AAEA,WAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,UAAM,WAAW,UAAU,CAAC;AAC5B,QAAI,CAACA,UAAS,QAAQ,GAAG;AACvB;AAAA,IACF;AAGA,UAAM,gBAAgB,SAAS;AAC/B,QAAI,MAAM,QAAQ,aAAa,GAAG;AAChC,YAAM;AAAA,QACJ;AAAA,QACA,aAAa,CAAC;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAGA,UAAM,mBAAmB,SAAS;AAClC,QAAI,MAAM,QAAQ,gBAAgB,GAAG;AACnC,YAAM;AAAA,QACJ;AAAA,QACA,aAAa,CAAC;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAEA,eAAe,yBACb,UACA,UACA,aACA,UACA,QACe;AACf,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,UAAM,UAAU,SAAS,CAAC;AAC1B,QAAI,CAACA,UAAS,OAAO,GAAG;AACtB;AAAA,IACF;AAEA,UAAM,UAAU,QAAQ;AACxB,QAAI,OAAO,YAAY,UAAU;AAC/B;AAAA,IACF;AAEA,QAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;AAC3B;AAAA,IACF;AAEA,aAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,YAAM,cAAc,QAAQ,CAAC;AAC7B,UAAI,CAACA,UAAS,WAAW,GAAG;AAC1B;AAAA,MACF;AAEA,YAAM,OAAO,YAAY;AACzB,UAAI,SAAS,QAAQ;AACnB;AAAA,MACF;AAEA,YAAM,QAAQ,YAAY;AAC1B,UAAI,OAAO,UAAU,UAAU;AAC7B,eAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV;AAAA,UACA,UAAU,GAAG,QAAQ,IAAI,CAAC,aAAa,CAAC;AAAA,UACxC,SAAS;AAAA,QACX,CAAC;AACD;AAAA,MACF;AAGA,YAAM,EAAE,aAAa,IAAI,MAAM,qBAAqB,OAAO,WAAW;AAEtE,UAAI,CAAC,cAAc;AACjB,eAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV;AAAA,UACA,UAAU,GAAG,QAAQ,IAAI,CAAC,aAAa,CAAC;AAAA,UACxC,SAAS,8BAA8B,KAAK;AAAA,QAC9C,CAAC;AAAA,MACH,OAAO;AAEL,YAAI;AACF,gBAAM,cAAc,MAAME,UAAS,cAAc,MAAM;AACvD,cAAI,YAAY,KAAK,EAAE,WAAW,GAAG;AACnC,mBAAO,KAAK;AAAA,cACV,UAAU;AAAA,cACV;AAAA,cACA,UAAU,GAAG,QAAQ,IAAI,CAAC,aAAa,CAAC;AAAA,cACxC,SAAS,6BAA6B,KAAK;AAAA,YAC7C,CAAC;AAAA,UACH;AAAA,QACF,SAAS,OAAO;AACd,iBAAO,KAAK;AAAA,YACV,UAAU;AAAA,YACV;AAAA,YACA,UAAU,GAAG,QAAQ,IAAI,CAAC,aAAa,CAAC;AAAA,YACxC,SAAS,gCAAgC,KAAK,KAAM,MAAgB,OAAO;AAAA,UAC7E,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;","names":["readFile","path","parse","readFile","path","parse","isObject","path","readFile","parse","absolutePath","errors","readFile","parse","SCHEMA_CONFIG_V2","readFile","path","parse","isObject","path","readFile","parse"]}
1
+ {"version":3,"sources":["../../../src/evaluation/validation/file-type.ts","../../../src/evaluation/validation/eval-validator.ts","../../../src/evaluation/validation/targets-validator.ts","../../../src/evaluation/validation/config-validator.ts","../../../src/evaluation/validation/file-reference-validator.ts"],"sourcesContent":["import { readFile } from 'node:fs/promises';\nimport path from 'node:path';\nimport { parse } from 'yaml';\n\nimport type { FileType } from './types.js';\n\nconst SCHEMA_EVAL_V2 = 'agentv-eval-v2';\nconst SCHEMA_TARGETS_V2 = 'agentv-targets-v2.2';\nconst SCHEMA_CONFIG_V2 = 'agentv-config-v2';\n\n/**\n * Detect file type by reading $schema field from YAML file.\n * If $schema is missing, infers type from filename/path:\n * - config.yaml under .agentv folder → 'config'\n * - targets.yaml under .agentv folder → 'targets'\n * - All other YAML files → 'eval' (default)\n */\nexport async function detectFileType(filePath: string): Promise<FileType> {\n try {\n const content = await readFile(filePath, 'utf8');\n const parsed = parse(content) as unknown;\n\n if (typeof parsed !== 'object' || parsed === null) {\n return inferFileTypeFromPath(filePath);\n }\n\n const record = parsed as Record<string, unknown>;\n const schema = record.$schema;\n\n if (typeof schema !== 'string') {\n // No $schema field - infer from path\n return inferFileTypeFromPath(filePath);\n }\n\n switch (schema) {\n case SCHEMA_EVAL_V2:\n return 'eval';\n case SCHEMA_TARGETS_V2:\n return 'targets';\n case SCHEMA_CONFIG_V2:\n return 'config';\n default:\n // Unknown schema - infer from path\n return inferFileTypeFromPath(filePath);\n }\n } catch {\n return inferFileTypeFromPath(filePath);\n }\n}\n\n/**\n * Infer file type from filename and directory path.\n */\nfunction inferFileTypeFromPath(filePath: string): FileType {\n const normalized = path.normalize(filePath).replace(/\\\\/g, '/');\n const basename = path.basename(filePath);\n\n // Check if file is under .agentv folder\n if (normalized.includes('/.agentv/')) {\n if (basename === 'config.yaml' || basename === 'config.yml') {\n return 'config';\n }\n if (basename === 'targets.yaml' || basename === 'targets.yml') {\n return 'targets';\n }\n }\n\n // Default to eval file\n return 'eval';\n}\n\n/**\n * Check if a schema value is a valid AgentV schema identifier.\n */\nexport function isValidSchema(schema: unknown): boolean {\n return schema === SCHEMA_EVAL_V2 || schema === SCHEMA_TARGETS_V2 || schema === SCHEMA_CONFIG_V2;\n}\n\n/**\n * Get the expected schema for a file type.\n */\nexport function getExpectedSchema(fileType: FileType): string | undefined {\n switch (fileType) {\n case 'eval':\n return SCHEMA_EVAL_V2;\n case 'targets':\n return SCHEMA_TARGETS_V2;\n case 'config':\n return SCHEMA_CONFIG_V2;\n default:\n return undefined;\n }\n}\n","import { readFile } from 'node:fs/promises';\nimport path from 'node:path';\nimport { parse } from 'yaml';\n\nimport type { ValidationError, ValidationResult } from './types.js';\n\ntype JsonValue = string | number | boolean | null | JsonObject | JsonArray;\ntype JsonObject = { readonly [key: string]: JsonValue };\ntype JsonArray = readonly JsonValue[];\n\nfunction isObject(value: unknown): value is JsonObject {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n\n/**\n * Validate an eval file (agentv-eval-v2 schema).\n */\nexport async function validateEvalFile(filePath: string): Promise<ValidationResult> {\n const errors: ValidationError[] = [];\n const absolutePath = path.resolve(filePath);\n\n let parsed: unknown;\n try {\n const content = await readFile(absolutePath, 'utf8');\n parsed = parse(content);\n } catch (error) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n message: `Failed to parse YAML: ${(error as Error).message}`,\n });\n return {\n valid: false,\n filePath: absolutePath,\n fileType: 'eval',\n errors,\n };\n }\n\n if (!isObject(parsed)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n message: 'File must contain a YAML object',\n });\n return {\n valid: false,\n filePath: absolutePath,\n fileType: 'eval',\n errors,\n };\n }\n\n // Validate evalcases array\n const evalcases = parsed.evalcases;\n if (!Array.isArray(evalcases)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: 'evalcases',\n message: \"Missing or invalid 'evalcases' field (must be an array)\",\n });\n return {\n valid: errors.length === 0,\n filePath: absolutePath,\n fileType: 'eval',\n errors,\n };\n }\n\n // Validate each eval case\n for (let i = 0; i < evalcases.length; i++) {\n const evalCase = evalcases[i];\n const location = `evalcases[${i}]`;\n\n if (!isObject(evalCase)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location,\n message: 'Eval case must be an object',\n });\n continue;\n }\n\n // Required fields: id, input_messages, expected_messages\n const id = evalCase.id;\n if (typeof id !== 'string' || id.trim().length === 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.id`,\n message: \"Missing or invalid 'id' field (must be a non-empty string)\",\n });\n }\n\n // Optional: expected_outcome or outcome for backward compatibility\n const expectedOutcome = evalCase.expected_outcome ?? evalCase.outcome;\n if (\n expectedOutcome !== undefined &&\n (typeof expectedOutcome !== 'string' || expectedOutcome.trim().length === 0)\n ) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.expected_outcome`,\n message:\n \"Invalid 'expected_outcome' or 'outcome' field (must be a non-empty string if provided)\",\n });\n }\n\n const inputMessages = evalCase.input_messages;\n if (!Array.isArray(inputMessages)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.input_messages`,\n message: \"Missing or invalid 'input_messages' field (must be an array)\",\n });\n } else {\n validateMessages(inputMessages, `${location}.input_messages`, absolutePath, errors);\n }\n\n // expected_messages is optional - for outcome-only evaluation\n const expectedMessages = evalCase.expected_messages;\n if (expectedMessages !== undefined && !Array.isArray(expectedMessages)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.expected_messages`,\n message: \"Invalid 'expected_messages' field (must be an array if provided)\",\n });\n } else if (Array.isArray(expectedMessages)) {\n validateMessages(expectedMessages, `${location}.expected_messages`, absolutePath, errors);\n }\n }\n\n return {\n valid: errors.length === 0,\n filePath: absolutePath,\n fileType: 'eval',\n errors,\n };\n}\n\nfunction validateMessages(\n messages: JsonArray,\n location: string,\n filePath: string,\n errors: ValidationError[],\n): void {\n for (let i = 0; i < messages.length; i++) {\n const message = messages[i];\n const msgLocation = `${location}[${i}]`;\n\n if (!isObject(message)) {\n errors.push({\n severity: 'error',\n filePath,\n location: msgLocation,\n message: 'Message must be an object',\n });\n continue;\n }\n\n // Validate role field\n const role = message.role;\n const validRoles = ['system', 'user', 'assistant'];\n if (!validRoles.includes(role as string)) {\n errors.push({\n severity: 'error',\n filePath,\n location: `${msgLocation}.role`,\n message: `Invalid role '${role}'. Must be one of: ${validRoles.join(', ')}`,\n });\n }\n\n // Validate content field (can be string or array)\n const content = message.content;\n if (typeof content === 'string') {\n validateContentForRoleMarkers(content, `${msgLocation}.content`, filePath, errors);\n } else if (Array.isArray(content)) {\n // Array content - validate each element\n for (let j = 0; j < content.length; j++) {\n const contentItem = content[j];\n const contentLocation = `${msgLocation}.content[${j}]`;\n\n if (typeof contentItem === 'string') {\n validateContentForRoleMarkers(contentItem, contentLocation, filePath, errors);\n } else if (isObject(contentItem)) {\n const type = contentItem.type;\n if (typeof type !== 'string') {\n errors.push({\n severity: 'error',\n filePath,\n location: `${contentLocation}.type`,\n message: \"Content object must have a 'type' field\",\n });\n }\n\n // For 'file' type, we'll validate existence later in file-reference-validator\n // For 'text' type, require 'value' field\n if (type === 'text') {\n const value = contentItem.value;\n if (typeof value !== 'string') {\n errors.push({\n severity: 'error',\n filePath,\n location: `${contentLocation}.value`,\n message: \"Content with type 'text' must have a 'value' field\",\n });\n } else {\n validateContentForRoleMarkers(value, `${contentLocation}.value`, filePath, errors);\n }\n }\n } else {\n errors.push({\n severity: 'error',\n filePath,\n location: contentLocation,\n message: 'Content array items must be strings or objects',\n });\n }\n }\n } else {\n errors.push({\n severity: 'error',\n filePath,\n location: `${msgLocation}.content`,\n message: \"Missing or invalid 'content' field (must be a string or array)\",\n });\n }\n }\n}\n\nfunction validateContentForRoleMarkers(\n content: string,\n location: string,\n filePath: string,\n errors: ValidationError[],\n): void {\n // Check for standard role markers that might confuse agentic providers\n const markers = ['@[System]:', '@[User]:', '@[Assistant]:', '@[Tool]:'];\n for (const marker of markers) {\n if (content.toLowerCase().includes(marker.toLowerCase())) {\n errors.push({\n severity: 'warning',\n filePath,\n location,\n message: `Content contains potential role marker '${marker}'. This may confuse agentic providers or cause prompt injection.`,\n });\n }\n }\n}\n","import { readFile } from 'node:fs/promises';\nimport path from 'node:path';\nimport { parse } from 'yaml';\n\nimport { CLI_PLACEHOLDERS } from '../providers/targets.js';\nimport { KNOWN_PROVIDERS, PROVIDER_ALIASES, TARGETS_SCHEMA_V2 } from '../providers/types.js';\nimport type { ValidationError, ValidationResult } from './types.js';\n\ntype JsonValue = string | number | boolean | null | JsonObject | JsonArray;\ntype JsonObject = { readonly [key: string]: JsonValue };\ntype JsonArray = readonly JsonValue[];\n\nfunction isObject(value: unknown): value is JsonObject {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n\n// Known settings properties for each provider type\nconst COMMON_SETTINGS = new Set(['provider_batching', 'providerBatching']);\n\nconst RETRY_SETTINGS = new Set([\n 'max_retries',\n 'maxRetries',\n 'retry_initial_delay_ms',\n 'retryInitialDelayMs',\n 'retry_max_delay_ms',\n 'retryMaxDelayMs',\n 'retry_backoff_factor',\n 'retryBackoffFactor',\n 'retry_status_codes',\n 'retryStatusCodes',\n]);\n\nconst AZURE_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n ...RETRY_SETTINGS,\n 'endpoint',\n 'resource',\n 'resourceName',\n 'api_key',\n 'apiKey',\n 'deployment',\n 'deploymentName',\n 'model',\n 'version',\n 'api_version',\n 'temperature',\n 'max_output_tokens',\n 'maxTokens',\n]);\n\nconst ANTHROPIC_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n ...RETRY_SETTINGS,\n 'api_key',\n 'apiKey',\n 'model',\n 'deployment',\n 'variant',\n 'temperature',\n 'max_output_tokens',\n 'maxTokens',\n 'thinking_budget',\n 'thinkingBudget',\n]);\n\nconst GEMINI_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n ...RETRY_SETTINGS,\n 'api_key',\n 'apiKey',\n 'model',\n 'deployment',\n 'variant',\n 'temperature',\n 'max_output_tokens',\n 'maxTokens',\n]);\n\nconst CODEX_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n 'executable',\n 'command',\n 'binary',\n 'args',\n 'arguments',\n 'cwd',\n 'timeout_seconds',\n 'timeoutSeconds',\n 'log_dir',\n 'logDir',\n 'log_directory',\n 'logDirectory',\n 'log_format',\n 'logFormat',\n 'log_output_format',\n 'logOutputFormat',\n]);\n\nconst VSCODE_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n 'workspace_template',\n 'workspaceTemplate',\n 'vscode_cmd',\n 'command',\n 'wait',\n 'dry_run',\n 'dryRun',\n 'subagent_root',\n 'subagentRoot',\n]);\n\nconst MOCK_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n 'response',\n 'delayMs',\n 'delayMinMs',\n 'delayMaxMs',\n 'trace', // For testing tool_trajectory evaluator\n]);\n\nconst CLI_SETTINGS = new Set([\n ...COMMON_SETTINGS,\n 'command_template',\n 'commandTemplate',\n 'verbose',\n 'cli_verbose',\n 'cliVerbose',\n 'files_format',\n 'filesFormat',\n 'attachments_format',\n 'attachmentsFormat',\n 'cwd',\n 'env',\n 'timeout_seconds',\n 'timeoutSeconds',\n 'healthcheck',\n]);\n\nfunction getKnownSettings(provider: string): Set<string> | null {\n const normalizedProvider = provider.toLowerCase();\n switch (normalizedProvider) {\n case 'azure':\n case 'azure-openai':\n return AZURE_SETTINGS;\n case 'anthropic':\n return ANTHROPIC_SETTINGS;\n case 'gemini':\n case 'google':\n case 'google-gemini':\n return GEMINI_SETTINGS;\n case 'codex':\n case 'codex-cli':\n return CODEX_SETTINGS;\n case 'vscode':\n case 'vscode-insiders':\n return VSCODE_SETTINGS;\n case 'mock':\n return MOCK_SETTINGS;\n case 'cli':\n return CLI_SETTINGS;\n default:\n return null; // Unknown provider, can't validate settings\n }\n}\n\nfunction validateUnknownSettings(\n target: JsonObject,\n provider: string,\n absolutePath: string,\n location: string,\n errors: ValidationError[],\n): void {\n const knownSettings = getKnownSettings(provider);\n if (!knownSettings) {\n // Unknown provider, skip settings validation\n return;\n }\n\n // Known base target fields that aren't settings\n const baseFields = new Set(['name', 'provider', 'judge_target', 'workers', '$schema', 'targets']);\n\n for (const key of Object.keys(target)) {\n if (!baseFields.has(key) && !knownSettings.has(key)) {\n errors.push({\n severity: 'warning',\n filePath: absolutePath,\n location: `${location}.${key}`,\n message: `Unknown setting '${key}' for ${provider} provider. This property will be ignored.`,\n });\n }\n }\n}\n\n/**\n * Validate a targets file (agentv-targets-v2.1 schema).\n */\nexport async function validateTargetsFile(filePath: string): Promise<ValidationResult> {\n const errors: ValidationError[] = [];\n const absolutePath = path.resolve(filePath);\n\n let parsed: unknown;\n try {\n const content = await readFile(absolutePath, 'utf8');\n parsed = parse(content);\n } catch (error) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n message: `Failed to parse YAML: ${(error as Error).message}`,\n });\n return {\n valid: false,\n filePath: absolutePath,\n fileType: 'targets',\n errors,\n };\n }\n\n function validateCliSettings(\n target: JsonObject,\n absolutePath: string,\n location: string,\n errors: ValidationError[],\n ): void {\n const commandTemplate = target.command_template ?? target.commandTemplate;\n if (typeof commandTemplate !== 'string' || commandTemplate.trim().length === 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.commandTemplate`,\n message: \"CLI provider requires 'commandTemplate' as a non-empty string\",\n });\n } else {\n recordUnknownPlaceholders(\n commandTemplate,\n absolutePath,\n `${location}.commandTemplate`,\n errors,\n );\n }\n\n const attachmentsFormat = target.attachments_format ?? target.attachmentsFormat;\n if (attachmentsFormat !== undefined && typeof attachmentsFormat !== 'string') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.attachmentsFormat`,\n message: \"'attachmentsFormat' must be a string when provided\",\n });\n }\n\n const filesFormat = target.files_format ?? target.filesFormat;\n if (filesFormat !== undefined && typeof filesFormat !== 'string') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.filesFormat`,\n message: \"'filesFormat' must be a string when provided\",\n });\n }\n\n const cwd = target.cwd;\n if (cwd !== undefined && typeof cwd !== 'string') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.cwd`,\n message: \"'cwd' must be a string when provided\",\n });\n }\n\n const timeoutSeconds = target.timeout_seconds ?? target.timeoutSeconds;\n if (timeoutSeconds !== undefined) {\n const numericTimeout = Number(timeoutSeconds);\n if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.timeoutSeconds`,\n message: \"'timeoutSeconds' must be a positive number when provided\",\n });\n }\n }\n\n const healthcheck = target.healthcheck;\n if (healthcheck !== undefined) {\n validateCliHealthcheck(healthcheck, absolutePath, `${location}.healthcheck`, errors);\n }\n\n const verbose = target.verbose ?? target.cli_verbose ?? target.cliVerbose;\n if (verbose !== undefined && typeof verbose !== 'boolean') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.verbose`,\n message: \"'verbose' must be a boolean when provided\",\n });\n }\n }\n\n function validateCliHealthcheck(\n healthcheck: unknown,\n absolutePath: string,\n location: string,\n errors: ValidationError[],\n ): void {\n if (!isObject(healthcheck)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location,\n message: \"'healthcheck' must be an object when provided\",\n });\n return;\n }\n\n const type = healthcheck.type;\n if (type !== 'http' && type !== 'command') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.type`,\n message: \"healthcheck.type must be either 'http' or 'command'\",\n });\n return;\n }\n\n const timeoutSeconds = healthcheck.timeout_seconds ?? healthcheck.timeoutSeconds;\n if (timeoutSeconds !== undefined) {\n const numericTimeout = Number(timeoutSeconds);\n if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.timeoutSeconds`,\n message: 'healthcheck.timeoutSeconds must be a positive number when provided',\n });\n }\n }\n\n if (type === 'http') {\n const url = healthcheck.url;\n if (typeof url !== 'string' || url.trim().length === 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.url`,\n message: 'healthcheck.url must be a non-empty string for http checks',\n });\n }\n return;\n }\n\n const commandTemplate = healthcheck.command_template ?? healthcheck.commandTemplate;\n if (typeof commandTemplate !== 'string' || commandTemplate.trim().length === 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.commandTemplate`,\n message: 'healthcheck.commandTemplate must be a non-empty string for command checks',\n });\n } else {\n recordUnknownPlaceholders(\n commandTemplate,\n absolutePath,\n `${location}.commandTemplate`,\n errors,\n );\n }\n\n const cwd = healthcheck.cwd;\n if (cwd !== undefined && typeof cwd !== 'string') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.cwd`,\n message: 'healthcheck.cwd must be a string when provided',\n });\n }\n }\n\n function recordUnknownPlaceholders(\n template: string,\n absolutePath: string,\n location: string,\n errors: ValidationError[],\n ): void {\n const placeholders = extractPlaceholders(template);\n for (const placeholder of placeholders) {\n if (!CLI_PLACEHOLDERS.has(placeholder)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location,\n message: `Unknown CLI placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS).join(', ')}`,\n });\n }\n }\n }\n\n function extractPlaceholders(template: string): string[] {\n const matches = template.matchAll(/\\{([A-Z_]+)\\}/g);\n const result: string[] = [];\n for (const match of matches) {\n const placeholder = match[1];\n if (placeholder) {\n result.push(placeholder);\n }\n }\n return result;\n }\n\n if (!isObject(parsed)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n message: 'File must contain a YAML object',\n });\n return {\n valid: false,\n filePath: absolutePath,\n fileType: 'targets',\n errors,\n };\n }\n\n // Validate targets array\n const targets = parsed.targets;\n if (!Array.isArray(targets)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: 'targets',\n message: \"Missing or invalid 'targets' field (must be an array)\",\n });\n return {\n valid: errors.length === 0,\n filePath: absolutePath,\n fileType: 'targets',\n errors,\n };\n }\n\n // Validate each target definition\n const knownProviders = [...KNOWN_PROVIDERS, ...PROVIDER_ALIASES];\n\n for (let i = 0; i < targets.length; i++) {\n const target = targets[i];\n const location = `targets[${i}]`;\n\n if (!isObject(target)) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location,\n message: 'Target must be an object',\n });\n continue;\n }\n\n // Required field: name\n const name = target.name;\n if (typeof name !== 'string' || name.trim().length === 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.name`,\n message: \"Missing or invalid 'name' field (must be a non-empty string)\",\n });\n }\n\n // Required field: provider\n const provider = target.provider;\n const providerValue = typeof provider === 'string' ? provider.trim().toLowerCase() : undefined;\n if (typeof provider !== 'string' || provider.trim().length === 0) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.provider`,\n message: \"Missing or invalid 'provider' field (must be a non-empty string)\",\n });\n } else if (!knownProviders.includes(provider)) {\n // Warning for unknown providers (non-fatal)\n errors.push({\n severity: 'warning',\n filePath: absolutePath,\n location: `${location}.provider`,\n message: `Unknown provider '${provider}'. Known providers: ${knownProviders.join(', ')}`,\n });\n }\n\n // Validate CLI provider fields\n if (providerValue === 'cli') {\n validateCliSettings(target, absolutePath, location, errors);\n }\n\n // Check for unknown settings properties on target object\n if (typeof provider === 'string') {\n validateUnknownSettings(target, provider, absolutePath, location, errors);\n }\n\n // Optional field: judge_target (must be string if present)\n const judgeTarget = target.judge_target;\n if (judgeTarget !== undefined && typeof judgeTarget !== 'string') {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n location: `${location}.judge_target`,\n message: \"Invalid 'judge_target' field (must be a string)\",\n });\n }\n }\n\n return {\n valid: errors.filter((e) => e.severity === 'error').length === 0,\n filePath: absolutePath,\n fileType: 'targets',\n errors,\n };\n}\n","import { readFile } from 'node:fs/promises';\nimport { parse } from 'yaml';\n\nimport type { ValidationError, ValidationResult } from './types.js';\n\nconst SCHEMA_CONFIG_V2 = 'agentv-config-v2';\n\n/**\n * Validate a config.yaml file for schema compliance and structural correctness.\n */\nexport async function validateConfigFile(filePath: string): Promise<ValidationResult> {\n const errors: ValidationError[] = [];\n\n try {\n const content = await readFile(filePath, 'utf8');\n const parsed = parse(content) as unknown;\n\n // Check if parsed content is an object\n if (typeof parsed !== 'object' || parsed === null) {\n errors.push({\n severity: 'error',\n filePath,\n message: 'Config file must contain a valid YAML object',\n });\n return { valid: false, filePath, fileType: 'config', errors };\n }\n\n const config = parsed as Record<string, unknown>;\n\n // Validate $schema field (optional, but if present must be correct)\n const schema = config.$schema;\n if (schema !== undefined && schema !== SCHEMA_CONFIG_V2) {\n const message = `Invalid $schema value '${schema}'. Expected '${SCHEMA_CONFIG_V2}' or omit the field.`;\n errors.push({\n severity: 'error',\n filePath,\n location: '$schema',\n message,\n });\n }\n\n // Validate guideline_patterns if present\n const guidelinePatterns = config.guideline_patterns;\n if (guidelinePatterns !== undefined) {\n if (!Array.isArray(guidelinePatterns)) {\n errors.push({\n severity: 'error',\n filePath,\n location: 'guideline_patterns',\n message: \"Field 'guideline_patterns' must be an array\",\n });\n } else if (!guidelinePatterns.every((p) => typeof p === 'string')) {\n errors.push({\n severity: 'error',\n filePath,\n location: 'guideline_patterns',\n message: \"All entries in 'guideline_patterns' must be strings\",\n });\n } else if (guidelinePatterns.length === 0) {\n errors.push({\n severity: 'warning',\n filePath,\n location: 'guideline_patterns',\n message: \"Field 'guideline_patterns' is empty. Consider removing it or adding patterns.\",\n });\n }\n }\n\n // Check for unexpected fields\n const allowedFields = new Set(['$schema', 'guideline_patterns']);\n const unexpectedFields = Object.keys(config).filter((key) => !allowedFields.has(key));\n\n if (unexpectedFields.length > 0) {\n errors.push({\n severity: 'warning',\n filePath,\n message: `Unexpected fields: ${unexpectedFields.join(', ')}`,\n });\n }\n\n return {\n valid: errors.filter((e) => e.severity === 'error').length === 0,\n filePath,\n fileType: 'config',\n errors,\n };\n } catch (error) {\n errors.push({\n severity: 'error',\n filePath,\n message: `Failed to parse config file: ${(error as Error).message}`,\n });\n return { valid: false, filePath, fileType: 'config', errors };\n }\n}\n","import { readFile } from 'node:fs/promises';\nimport path from 'node:path';\nimport { parse } from 'yaml';\n\nimport { buildSearchRoots, findGitRoot, resolveFileReference } from '../file-utils.js';\nimport type { ValidationError } from './types.js';\n\ntype JsonValue = string | number | boolean | null | JsonObject | JsonArray;\ntype JsonObject = { readonly [key: string]: JsonValue };\ntype JsonArray = readonly JsonValue[];\n\nfunction isObject(value: unknown): value is JsonObject {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n\n/**\n * Validate that file references in eval file content exist.\n * Checks content blocks with type: \"file\" and validates the referenced file exists.\n * Also checks that referenced files are not empty.\n */\nexport async function validateFileReferences(\n evalFilePath: string,\n): Promise<readonly ValidationError[]> {\n const errors: ValidationError[] = [];\n const absolutePath = path.resolve(evalFilePath);\n\n // Find git root and build search roots (same as yaml-parser does at runtime)\n const gitRoot = await findGitRoot(absolutePath);\n if (!gitRoot) {\n errors.push({\n severity: 'error',\n filePath: absolutePath,\n message: 'Cannot validate file references: git repository root not found',\n });\n return errors;\n }\n\n const searchRoots = buildSearchRoots(absolutePath, gitRoot);\n\n let parsed: unknown;\n try {\n const content = await readFile(absolutePath, 'utf8');\n parsed = parse(content);\n } catch {\n // Parse errors are already caught by eval-validator\n return errors;\n }\n\n if (!isObject(parsed)) {\n return errors;\n }\n\n const evalcases = parsed.evalcases;\n if (!Array.isArray(evalcases)) {\n return errors;\n }\n\n for (let i = 0; i < evalcases.length; i++) {\n const evalCase = evalcases[i];\n if (!isObject(evalCase)) {\n continue;\n }\n\n // Check input_messages\n const inputMessages = evalCase.input_messages;\n if (Array.isArray(inputMessages)) {\n await validateMessagesFileRefs(\n inputMessages,\n `evalcases[${i}].input_messages`,\n searchRoots,\n absolutePath,\n errors,\n );\n }\n\n // Check expected_messages\n const expectedMessages = evalCase.expected_messages;\n if (Array.isArray(expectedMessages)) {\n await validateMessagesFileRefs(\n expectedMessages,\n `evalcases[${i}].expected_messages`,\n searchRoots,\n absolutePath,\n errors,\n );\n }\n }\n\n return errors;\n}\n\nasync function validateMessagesFileRefs(\n messages: JsonArray,\n location: string,\n searchRoots: readonly string[],\n filePath: string,\n errors: ValidationError[],\n): Promise<void> {\n for (let i = 0; i < messages.length; i++) {\n const message = messages[i];\n if (!isObject(message)) {\n continue;\n }\n\n const content = message.content;\n if (typeof content === 'string') {\n continue;\n }\n\n if (!Array.isArray(content)) {\n continue;\n }\n\n for (let j = 0; j < content.length; j++) {\n const contentItem = content[j];\n if (!isObject(contentItem)) {\n continue;\n }\n\n const type = contentItem.type;\n if (type !== 'file') {\n continue;\n }\n\n const value = contentItem.value;\n if (typeof value !== 'string') {\n errors.push({\n severity: 'error',\n filePath,\n location: `${location}[${i}].content[${j}].value`,\n message: \"File reference must have a 'value' field with the file path\",\n });\n continue;\n }\n\n // Use the same file resolution logic as yaml-parser at runtime\n const { resolvedPath } = await resolveFileReference(value, searchRoots);\n\n if (!resolvedPath) {\n errors.push({\n severity: 'error',\n filePath,\n location: `${location}[${i}].content[${j}]`,\n message: `Referenced file not found: ${value}`,\n });\n } else {\n // Check that file is not empty\n try {\n const fileContent = await readFile(resolvedPath, 'utf8');\n if (fileContent.trim().length === 0) {\n errors.push({\n severity: 'warning',\n filePath,\n location: `${location}[${i}].content[${j}]`,\n message: `Referenced file is empty: ${value}`,\n });\n }\n } catch (error) {\n errors.push({\n severity: 'error',\n filePath,\n location: `${location}[${i}].content[${j}]`,\n message: `Cannot read referenced file: ${value} (${(error as Error).message})`,\n });\n }\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;AAAA,SAAS,gBAAgB;AACzB,OAAO,UAAU;AACjB,SAAS,aAAa;AAItB,IAAM,iBAAiB;AACvB,IAAM,oBAAoB;AAC1B,IAAM,mBAAmB;AASzB,eAAsB,eAAe,UAAqC;AACxE,MAAI;AACF,UAAM,UAAU,MAAM,SAAS,UAAU,MAAM;AAC/C,UAAM,SAAS,MAAM,OAAO;AAE5B,QAAI,OAAO,WAAW,YAAY,WAAW,MAAM;AACjD,aAAO,sBAAsB,QAAQ;AAAA,IACvC;AAEA,UAAM,SAAS;AACf,UAAM,SAAS,OAAO;AAEtB,QAAI,OAAO,WAAW,UAAU;AAE9B,aAAO,sBAAsB,QAAQ;AAAA,IACvC;AAEA,YAAQ,QAAQ;AAAA,MACd,KAAK;AACH,eAAO;AAAA,MACT,KAAK;AACH,eAAO;AAAA,MACT,KAAK;AACH,eAAO;AAAA,MACT;AAEE,eAAO,sBAAsB,QAAQ;AAAA,IACzC;AAAA,EACF,QAAQ;AACN,WAAO,sBAAsB,QAAQ;AAAA,EACvC;AACF;AAKA,SAAS,sBAAsB,UAA4B;AACzD,QAAM,aAAa,KAAK,UAAU,QAAQ,EAAE,QAAQ,OAAO,GAAG;AAC9D,QAAM,WAAW,KAAK,SAAS,QAAQ;AAGvC,MAAI,WAAW,SAAS,WAAW,GAAG;AACpC,QAAI,aAAa,iBAAiB,aAAa,cAAc;AAC3D,aAAO;AAAA,IACT;AACA,QAAI,aAAa,kBAAkB,aAAa,eAAe;AAC7D,aAAO;AAAA,IACT;AAAA,EACF;AAGA,SAAO;AACT;AAKO,SAAS,cAAc,QAA0B;AACtD,SAAO,WAAW,kBAAkB,WAAW,qBAAqB,WAAW;AACjF;AAKO,SAAS,kBAAkB,UAAwC;AACxE,UAAQ,UAAU;AAAA,IAChB,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT;AACE,aAAO;AAAA,EACX;AACF;;;AC5FA,SAAS,YAAAA,iBAAgB;AACzB,OAAOC,WAAU;AACjB,SAAS,SAAAC,cAAa;AAQtB,SAAS,SAAS,OAAqC;AACrD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAKA,eAAsB,iBAAiB,UAA6C;AAClF,QAAM,SAA4B,CAAC;AACnC,QAAM,eAAeD,MAAK,QAAQ,QAAQ;AAE1C,MAAI;AACJ,MAAI;AACF,UAAM,UAAU,MAAMD,UAAS,cAAc,MAAM;AACnD,aAASE,OAAM,OAAO;AAAA,EACxB,SAAS,OAAO;AACd,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS,yBAA0B,MAAgB,OAAO;AAAA,IAC5D,CAAC;AACD,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAEA,MAAI,CAAC,SAAS,MAAM,GAAG;AACrB,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AACD,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAGA,QAAM,YAAY,OAAO;AACzB,MAAI,CAAC,MAAM,QAAQ,SAAS,GAAG;AAC7B,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AACD,WAAO;AAAA,MACL,OAAO,OAAO,WAAW;AAAA,MACzB,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAGA,WAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,UAAM,WAAW,UAAU,CAAC;AAC5B,UAAM,WAAW,aAAa,CAAC;AAE/B,QAAI,CAAC,SAAS,QAAQ,GAAG;AACvB,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAGA,UAAM,KAAK,SAAS;AACpB,QAAI,OAAO,OAAO,YAAY,GAAG,KAAK,EAAE,WAAW,GAAG;AACpD,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAGA,UAAM,kBAAkB,SAAS,oBAAoB,SAAS;AAC9D,QACE,oBAAoB,WACnB,OAAO,oBAAoB,YAAY,gBAAgB,KAAK,EAAE,WAAW,IAC1E;AACA,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SACE;AAAA,MACJ,CAAC;AAAA,IACH;AAEA,UAAM,gBAAgB,SAAS;AAC/B,QAAI,CAAC,MAAM,QAAQ,aAAa,GAAG;AACjC,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,OAAO;AACL,uBAAiB,eAAe,GAAG,QAAQ,mBAAmB,cAAc,MAAM;AAAA,IACpF;AAGA,UAAM,mBAAmB,SAAS;AAClC,QAAI,qBAAqB,UAAa,CAAC,MAAM,QAAQ,gBAAgB,GAAG;AACtE,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,WAAW,MAAM,QAAQ,gBAAgB,GAAG;AAC1C,uBAAiB,kBAAkB,GAAG,QAAQ,sBAAsB,cAAc,MAAM;AAAA,IAC1F;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO,OAAO,WAAW;AAAA,IACzB,UAAU;AAAA,IACV,UAAU;AAAA,IACV;AAAA,EACF;AACF;AAEA,SAAS,iBACP,UACA,UACA,UACA,QACM;AACN,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,UAAM,UAAU,SAAS,CAAC;AAC1B,UAAM,cAAc,GAAG,QAAQ,IAAI,CAAC;AAEpC,QAAI,CAAC,SAAS,OAAO,GAAG;AACtB,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,UAAU;AAAA,QACV,SAAS;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAGA,UAAM,OAAO,QAAQ;AACrB,UAAM,aAAa,CAAC,UAAU,QAAQ,WAAW;AACjD,QAAI,CAAC,WAAW,SAAS,IAAc,GAAG;AACxC,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,UAAU,GAAG,WAAW;AAAA,QACxB,SAAS,iBAAiB,IAAI,sBAAsB,WAAW,KAAK,IAAI,CAAC;AAAA,MAC3E,CAAC;AAAA,IACH;AAGA,UAAM,UAAU,QAAQ;AACxB,QAAI,OAAO,YAAY,UAAU;AAC/B,oCAA8B,SAAS,GAAG,WAAW,YAAY,UAAU,MAAM;AAAA,IACnF,WAAW,MAAM,QAAQ,OAAO,GAAG;AAEjC,eAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,cAAM,cAAc,QAAQ,CAAC;AAC7B,cAAM,kBAAkB,GAAG,WAAW,YAAY,CAAC;AAEnD,YAAI,OAAO,gBAAgB,UAAU;AACnC,wCAA8B,aAAa,iBAAiB,UAAU,MAAM;AAAA,QAC9E,WAAW,SAAS,WAAW,GAAG;AAChC,gBAAM,OAAO,YAAY;AACzB,cAAI,OAAO,SAAS,UAAU;AAC5B,mBAAO,KAAK;AAAA,cACV,UAAU;AAAA,cACV;AAAA,cACA,UAAU,GAAG,eAAe;AAAA,cAC5B,SAAS;AAAA,YACX,CAAC;AAAA,UACH;AAIA,cAAI,SAAS,QAAQ;AACnB,kBAAM,QAAQ,YAAY;AAC1B,gBAAI,OAAO,UAAU,UAAU;AAC7B,qBAAO,KAAK;AAAA,gBACV,UAAU;AAAA,gBACV;AAAA,gBACA,UAAU,GAAG,eAAe;AAAA,gBAC5B,SAAS;AAAA,cACX,CAAC;AAAA,YACH,OAAO;AACL,4CAA8B,OAAO,GAAG,eAAe,UAAU,UAAU,MAAM;AAAA,YACnF;AAAA,UACF;AAAA,QACF,OAAO;AACL,iBAAO,KAAK;AAAA,YACV,UAAU;AAAA,YACV;AAAA,YACA,UAAU;AAAA,YACV,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,OAAO;AACL,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,UAAU,GAAG,WAAW;AAAA,QACxB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAEA,SAAS,8BACP,SACA,UACA,UACA,QACM;AAEN,QAAM,UAAU,CAAC,cAAc,YAAY,iBAAiB,UAAU;AACtE,aAAW,UAAU,SAAS;AAC5B,QAAI,QAAQ,YAAY,EAAE,SAAS,OAAO,YAAY,CAAC,GAAG;AACxD,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA;AAAA,QACA,SAAS,2CAA2C,MAAM;AAAA,MAC5D,CAAC;AAAA,IACH;AAAA,EACF;AACF;;;AC7PA,SAAS,YAAAC,iBAAgB;AACzB,OAAOC,WAAU;AACjB,SAAS,SAAAC,cAAa;AAUtB,SAASC,UAAS,OAAqC;AACrD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAGA,IAAM,kBAAkB,oBAAI,IAAI,CAAC,qBAAqB,kBAAkB,CAAC;AAEzE,IAAM,iBAAiB,oBAAI,IAAI;AAAA,EAC7B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,iBAAiB,oBAAI,IAAI;AAAA,EAC7B,GAAG;AAAA,EACH,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,qBAAqB,oBAAI,IAAI;AAAA,EACjC,GAAG;AAAA,EACH,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC9B,GAAG;AAAA,EACH,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,iBAAiB,oBAAI,IAAI;AAAA,EAC7B,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC9B,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,gBAAgB,oBAAI,IAAI;AAAA,EAC5B,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AACF,CAAC;AAED,IAAM,eAAe,oBAAI,IAAI;AAAA,EAC3B,GAAG;AAAA,EACH;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,SAAS,iBAAiB,UAAsC;AAC9D,QAAM,qBAAqB,SAAS,YAAY;AAChD,UAAQ,oBAAoB;AAAA,IAC1B,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT;AACE,aAAO;AAAA,EACX;AACF;AAEA,SAAS,wBACP,QACA,UACA,cACA,UACA,QACM;AACN,QAAM,gBAAgB,iBAAiB,QAAQ;AAC/C,MAAI,CAAC,eAAe;AAElB;AAAA,EACF;AAGA,QAAM,aAAa,oBAAI,IAAI,CAAC,QAAQ,YAAY,gBAAgB,WAAW,WAAW,SAAS,CAAC;AAEhG,aAAW,OAAO,OAAO,KAAK,MAAM,GAAG;AACrC,QAAI,CAAC,WAAW,IAAI,GAAG,KAAK,CAAC,cAAc,IAAI,GAAG,GAAG;AACnD,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ,IAAI,GAAG;AAAA,QAC5B,SAAS,oBAAoB,GAAG,SAAS,QAAQ;AAAA,MACnD,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAKA,eAAsB,oBAAoB,UAA6C;AACrF,QAAM,SAA4B,CAAC;AACnC,QAAM,eAAeC,MAAK,QAAQ,QAAQ;AAE1C,MAAI;AACJ,MAAI;AACF,UAAM,UAAU,MAAMC,UAAS,cAAc,MAAM;AACnD,aAASC,OAAM,OAAO;AAAA,EACxB,SAAS,OAAO;AACd,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS,yBAA0B,MAAgB,OAAO;AAAA,IAC5D,CAAC;AACD,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBACP,QACAC,eACA,UACAC,SACM;AACN,UAAM,kBAAkB,OAAO,oBAAoB,OAAO;AAC1D,QAAI,OAAO,oBAAoB,YAAY,gBAAgB,KAAK,EAAE,WAAW,GAAG;AAC9E,MAAAA,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,OAAO;AACL;AAAA,QACE;AAAA,QACAA;AAAA,QACA,GAAG,QAAQ;AAAA,QACXC;AAAA,MACF;AAAA,IACF;AAEA,UAAM,oBAAoB,OAAO,sBAAsB,OAAO;AAC9D,QAAI,sBAAsB,UAAa,OAAO,sBAAsB,UAAU;AAC5E,MAAAA,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAEA,UAAM,cAAc,OAAO,gBAAgB,OAAO;AAClD,QAAI,gBAAgB,UAAa,OAAO,gBAAgB,UAAU;AAChE,MAAAC,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAEA,UAAM,MAAM,OAAO;AACnB,QAAI,QAAQ,UAAa,OAAO,QAAQ,UAAU;AAChD,MAAAC,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAEA,UAAM,iBAAiB,OAAO,mBAAmB,OAAO;AACxD,QAAI,mBAAmB,QAAW;AAChC,YAAM,iBAAiB,OAAO,cAAc;AAC5C,UAAI,CAAC,OAAO,SAAS,cAAc,KAAK,kBAAkB,GAAG;AAC3D,QAAAC,QAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV,UAAUD;AAAA,UACV,UAAU,GAAG,QAAQ;AAAA,UACrB,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAAA,IACF;AAEA,UAAM,cAAc,OAAO;AAC3B,QAAI,gBAAgB,QAAW;AAC7B,6BAAuB,aAAaA,eAAc,GAAG,QAAQ,gBAAgBC,OAAM;AAAA,IACrF;AAEA,UAAM,UAAU,OAAO,WAAW,OAAO,eAAe,OAAO;AAC/D,QAAI,YAAY,UAAa,OAAO,YAAY,WAAW;AACzD,MAAAA,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAAA,EACF;AAEA,WAAS,uBACP,aACAA,eACA,UACAC,SACM;AACN,QAAI,CAACL,UAAS,WAAW,GAAG;AAC1B,MAAAK,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,OAAO,YAAY;AACzB,QAAI,SAAS,UAAU,SAAS,WAAW;AACzC,MAAAC,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,iBAAiB,YAAY,mBAAmB,YAAY;AAClE,QAAI,mBAAmB,QAAW;AAChC,YAAM,iBAAiB,OAAO,cAAc;AAC5C,UAAI,CAAC,OAAO,SAAS,cAAc,KAAK,kBAAkB,GAAG;AAC3D,QAAAC,QAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV,UAAUD;AAAA,UACV,UAAU,GAAG,QAAQ;AAAA,UACrB,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAAA,IACF;AAEA,QAAI,SAAS,QAAQ;AACnB,YAAM,MAAM,YAAY;AACxB,UAAI,OAAO,QAAQ,YAAY,IAAI,KAAK,EAAE,WAAW,GAAG;AACtD,QAAAC,QAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV,UAAUD;AAAA,UACV,UAAU,GAAG,QAAQ;AAAA,UACrB,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AACA;AAAA,IACF;AAEA,UAAM,kBAAkB,YAAY,oBAAoB,YAAY;AACpE,QAAI,OAAO,oBAAoB,YAAY,gBAAgB,KAAK,EAAE,WAAW,GAAG;AAC9E,MAAAC,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,OAAO;AACL;AAAA,QACE;AAAA,QACAA;AAAA,QACA,GAAG,QAAQ;AAAA,QACXC;AAAA,MACF;AAAA,IACF;AAEA,UAAM,MAAM,YAAY;AACxB,QAAI,QAAQ,UAAa,OAAO,QAAQ,UAAU;AAChD,MAAAA,QAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAUD;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAAA,EACF;AAEA,WAAS,0BACP,UACAA,eACA,UACAC,SACM;AACN,UAAM,eAAe,oBAAoB,QAAQ;AACjD,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,iBAAiB,IAAI,WAAW,GAAG;AACtC,QAAAA,QAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV,UAAUD;AAAA,UACV;AAAA,UACA,SAAS,6BAA6B,WAAW,+BAA+B,MAAM,KAAK,gBAAgB,EAAE,KAAK,IAAI,CAAC;AAAA,QACzH,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,WAAS,oBAAoB,UAA4B;AACvD,UAAM,UAAU,SAAS,SAAS,gBAAgB;AAClD,UAAM,SAAmB,CAAC;AAC1B,eAAW,SAAS,SAAS;AAC3B,YAAM,cAAc,MAAM,CAAC;AAC3B,UAAI,aAAa;AACf,eAAO,KAAK,WAAW;AAAA,MACzB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,MAAI,CAACJ,UAAS,MAAM,GAAG;AACrB,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AACD,WAAO;AAAA,MACL,OAAO;AAAA,MACP,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAGA,QAAM,UAAU,OAAO;AACvB,MAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;AAC3B,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AACD,WAAO;AAAA,MACL,OAAO,OAAO,WAAW;AAAA,MACzB,UAAU;AAAA,MACV,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAGA,QAAM,iBAAiB,CAAC,GAAG,iBAAiB,GAAG,gBAAgB;AAE/D,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,UAAM,SAAS,QAAQ,CAAC;AACxB,UAAM,WAAW,WAAW,CAAC;AAE7B,QAAI,CAACA,UAAS,MAAM,GAAG;AACrB,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAGA,UAAM,OAAO,OAAO;AACpB,QAAI,OAAO,SAAS,YAAY,KAAK,KAAK,EAAE,WAAW,GAAG;AACxD,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAGA,UAAM,WAAW,OAAO;AACxB,UAAM,gBAAgB,OAAO,aAAa,WAAW,SAAS,KAAK,EAAE,YAAY,IAAI;AACrF,QAAI,OAAO,aAAa,YAAY,SAAS,KAAK,EAAE,WAAW,GAAG;AAChE,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH,WAAW,CAAC,eAAe,SAAS,QAAQ,GAAG;AAE7C,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS,qBAAqB,QAAQ,uBAAuB,eAAe,KAAK,IAAI,CAAC;AAAA,MACxF,CAAC;AAAA,IACH;AAGA,QAAI,kBAAkB,OAAO;AAC3B,0BAAoB,QAAQ,cAAc,UAAU,MAAM;AAAA,IAC5D;AAGA,QAAI,OAAO,aAAa,UAAU;AAChC,8BAAwB,QAAQ,UAAU,cAAc,UAAU,MAAM;AAAA,IAC1E;AAGA,UAAM,cAAc,OAAO;AAC3B,QAAI,gBAAgB,UAAa,OAAO,gBAAgB,UAAU;AAChE,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV,UAAU;AAAA,QACV,UAAU,GAAG,QAAQ;AAAA,QACrB,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO,OAAO,OAAO,CAAC,MAAM,EAAE,aAAa,OAAO,EAAE,WAAW;AAAA,IAC/D,UAAU;AAAA,IACV,UAAU;AAAA,IACV;AAAA,EACF;AACF;;;ACvgBA,SAAS,YAAAM,iBAAgB;AACzB,SAAS,SAAAC,cAAa;AAItB,IAAMC,oBAAmB;AAKzB,eAAsB,mBAAmB,UAA6C;AACpF,QAAM,SAA4B,CAAC;AAEnC,MAAI;AACF,UAAM,UAAU,MAAMF,UAAS,UAAU,MAAM;AAC/C,UAAM,SAASC,OAAM,OAAO;AAG5B,QAAI,OAAO,WAAW,YAAY,WAAW,MAAM;AACjD,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,aAAO,EAAE,OAAO,OAAO,UAAU,UAAU,UAAU,OAAO;AAAA,IAC9D;AAEA,UAAM,SAAS;AAGf,UAAM,SAAS,OAAO;AACtB,QAAI,WAAW,UAAa,WAAWC,mBAAkB;AACvD,YAAM,UAAU,0BAA0B,MAAM,gBAAgBA,iBAAgB;AAChF,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,UAAU;AAAA,QACV;AAAA,MACF,CAAC;AAAA,IACH;AAGA,UAAM,oBAAoB,OAAO;AACjC,QAAI,sBAAsB,QAAW;AACnC,UAAI,CAAC,MAAM,QAAQ,iBAAiB,GAAG;AACrC,eAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV;AAAA,UACA,UAAU;AAAA,UACV,SAAS;AAAA,QACX,CAAC;AAAA,MACH,WAAW,CAAC,kBAAkB,MAAM,CAAC,MAAM,OAAO,MAAM,QAAQ,GAAG;AACjE,eAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV;AAAA,UACA,UAAU;AAAA,UACV,SAAS;AAAA,QACX,CAAC;AAAA,MACH,WAAW,kBAAkB,WAAW,GAAG;AACzC,eAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV;AAAA,UACA,UAAU;AAAA,UACV,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAAA,IACF;AAGA,UAAM,gBAAgB,oBAAI,IAAI,CAAC,WAAW,oBAAoB,CAAC;AAC/D,UAAM,mBAAmB,OAAO,KAAK,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,IAAI,GAAG,CAAC;AAEpF,QAAI,iBAAiB,SAAS,GAAG;AAC/B,aAAO,KAAK;AAAA,QACV,UAAU;AAAA,QACV;AAAA,QACA,SAAS,sBAAsB,iBAAiB,KAAK,IAAI,CAAC;AAAA,MAC5D,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,MACL,OAAO,OAAO,OAAO,CAAC,MAAM,EAAE,aAAa,OAAO,EAAE,WAAW;AAAA,MAC/D;AAAA,MACA,UAAU;AAAA,MACV;AAAA,IACF;AAAA,EACF,SAAS,OAAO;AACd,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV;AAAA,MACA,SAAS,gCAAiC,MAAgB,OAAO;AAAA,IACnE,CAAC;AACD,WAAO,EAAE,OAAO,OAAO,UAAU,UAAU,UAAU,OAAO;AAAA,EAC9D;AACF;;;AC9FA,SAAS,YAAAC,iBAAgB;AACzB,OAAOC,WAAU;AACjB,SAAS,SAAAC,cAAa;AAStB,SAASC,UAAS,OAAqC;AACrD,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAOA,eAAsB,uBACpB,cACqC;AACrC,QAAM,SAA4B,CAAC;AACnC,QAAM,eAAeC,MAAK,QAAQ,YAAY;AAG9C,QAAM,UAAU,MAAM,YAAY,YAAY;AAC9C,MAAI,CAAC,SAAS;AACZ,WAAO,KAAK;AAAA,MACV,UAAU;AAAA,MACV,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AACD,WAAO;AAAA,EACT;AAEA,QAAM,cAAc,iBAAiB,cAAc,OAAO;AAE1D,MAAI;AACJ,MAAI;AACF,UAAM,UAAU,MAAMC,UAAS,cAAc,MAAM;AACnD,aAASC,OAAM,OAAO;AAAA,EACxB,QAAQ;AAEN,WAAO;AAAA,EACT;AAEA,MAAI,CAACH,UAAS,MAAM,GAAG;AACrB,WAAO;AAAA,EACT;AAEA,QAAM,YAAY,OAAO;AACzB,MAAI,CAAC,MAAM,QAAQ,SAAS,GAAG;AAC7B,WAAO;AAAA,EACT;AAEA,WAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,UAAM,WAAW,UAAU,CAAC;AAC5B,QAAI,CAACA,UAAS,QAAQ,GAAG;AACvB;AAAA,IACF;AAGA,UAAM,gBAAgB,SAAS;AAC/B,QAAI,MAAM,QAAQ,aAAa,GAAG;AAChC,YAAM;AAAA,QACJ;AAAA,QACA,aAAa,CAAC;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAGA,UAAM,mBAAmB,SAAS;AAClC,QAAI,MAAM,QAAQ,gBAAgB,GAAG;AACnC,YAAM;AAAA,QACJ;AAAA,QACA,aAAa,CAAC;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAEA,eAAe,yBACb,UACA,UACA,aACA,UACA,QACe;AACf,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,UAAM,UAAU,SAAS,CAAC;AAC1B,QAAI,CAACA,UAAS,OAAO,GAAG;AACtB;AAAA,IACF;AAEA,UAAM,UAAU,QAAQ;AACxB,QAAI,OAAO,YAAY,UAAU;AAC/B;AAAA,IACF;AAEA,QAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;AAC3B;AAAA,IACF;AAEA,aAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,YAAM,cAAc,QAAQ,CAAC;AAC7B,UAAI,CAACA,UAAS,WAAW,GAAG;AAC1B;AAAA,MACF;AAEA,YAAM,OAAO,YAAY;AACzB,UAAI,SAAS,QAAQ;AACnB;AAAA,MACF;AAEA,YAAM,QAAQ,YAAY;AAC1B,UAAI,OAAO,UAAU,UAAU;AAC7B,eAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV;AAAA,UACA,UAAU,GAAG,QAAQ,IAAI,CAAC,aAAa,CAAC;AAAA,UACxC,SAAS;AAAA,QACX,CAAC;AACD;AAAA,MACF;AAGA,YAAM,EAAE,aAAa,IAAI,MAAM,qBAAqB,OAAO,WAAW;AAEtE,UAAI,CAAC,cAAc;AACjB,eAAO,KAAK;AAAA,UACV,UAAU;AAAA,UACV;AAAA,UACA,UAAU,GAAG,QAAQ,IAAI,CAAC,aAAa,CAAC;AAAA,UACxC,SAAS,8BAA8B,KAAK;AAAA,QAC9C,CAAC;AAAA,MACH,OAAO;AAEL,YAAI;AACF,gBAAM,cAAc,MAAME,UAAS,cAAc,MAAM;AACvD,cAAI,YAAY,KAAK,EAAE,WAAW,GAAG;AACnC,mBAAO,KAAK;AAAA,cACV,UAAU;AAAA,cACV;AAAA,cACA,UAAU,GAAG,QAAQ,IAAI,CAAC,aAAa,CAAC;AAAA,cACxC,SAAS,6BAA6B,KAAK;AAAA,YAC7C,CAAC;AAAA,UACH;AAAA,QACF,SAAS,OAAO;AACd,iBAAO,KAAK;AAAA,YACV,UAAU;AAAA,YACV;AAAA,YACA,UAAU,GAAG,QAAQ,IAAI,CAAC,aAAa,CAAC;AAAA,YACxC,SAAS,gCAAgC,KAAK,KAAM,MAAgB,OAAO;AAAA,UAC7E,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;","names":["readFile","path","parse","readFile","path","parse","isObject","path","readFile","parse","absolutePath","errors","readFile","parse","SCHEMA_CONFIG_V2","readFile","path","parse","isObject","path","readFile","parse"]}
package/dist/index.cjs CHANGED
@@ -835,6 +835,17 @@ async function processMessages(options) {
835
835
  }
836
836
  continue;
837
837
  }
838
+ if (isJsonObject(content)) {
839
+ const rendered = JSON.stringify(content, null, 2);
840
+ segments.push({ type: "text", value: rendered });
841
+ if (textParts) {
842
+ textParts.push(rendered);
843
+ }
844
+ continue;
845
+ }
846
+ if (!Array.isArray(content)) {
847
+ continue;
848
+ }
838
849
  for (const rawSegment of content) {
839
850
  if (!isJsonObject(rawSegment)) {
840
851
  continue;
@@ -1061,6 +1072,11 @@ async function buildPromptInputs(testCase, mode = "lm") {
1061
1072
  }
1062
1073
  }
1063
1074
  }
1075
+ } else if (isJsonObject(message.content)) {
1076
+ const rendered = JSON.stringify(message.content, null, 2);
1077
+ if (rendered.trim().length > 0) {
1078
+ messageSegments.push({ type: "text", value: rendered });
1079
+ }
1064
1080
  }
1065
1081
  segmentsByMessage.push(messageSegments);
1066
1082
  }
@@ -1909,7 +1925,7 @@ var CliProvider = class {
1909
1925
  id;
1910
1926
  kind = "cli";
1911
1927
  targetName;
1912
- supportsBatch = false;
1928
+ supportsBatch = true;
1913
1929
  config;
1914
1930
  runCommand;
1915
1931
  verbose;
@@ -1929,6 +1945,11 @@ var CliProvider = class {
1929
1945
  const outputFilePath = generateOutputFilePath(request.evalCaseId);
1930
1946
  const templateValues = buildTemplateValues(request, this.config, outputFilePath);
1931
1947
  const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
1948
+ if (this.verbose) {
1949
+ console.log(
1950
+ `[cli-provider:${this.targetName}] cwd=${this.config.cwd ?? ""} command=${renderedCommand}`
1951
+ );
1952
+ }
1932
1953
  const result = await this.runCommand(renderedCommand, {
1933
1954
  cwd: this.config.cwd,
1934
1955
  env: process.env,
@@ -1963,6 +1984,114 @@ var CliProvider = class {
1963
1984
  }
1964
1985
  };
1965
1986
  }
1987
+ async invokeBatch(requests) {
1988
+ if (requests.length === 0) {
1989
+ return [];
1990
+ }
1991
+ for (const request of requests) {
1992
+ if (request.signal?.aborted) {
1993
+ throw new Error("CLI provider batch request was aborted before execution");
1994
+ }
1995
+ }
1996
+ const controller = new AbortController();
1997
+ for (const request of requests) {
1998
+ request.signal?.addEventListener("abort", () => controller.abort(), { once: true });
1999
+ }
2000
+ await this.ensureHealthy(controller.signal);
2001
+ const outputFilePath = generateOutputFilePath("batch", ".jsonl");
2002
+ const batchInputFiles = [];
2003
+ for (const request of requests) {
2004
+ if (request.inputFiles && request.inputFiles.length > 0) {
2005
+ batchInputFiles.push(...request.inputFiles);
2006
+ }
2007
+ }
2008
+ const templateValues = buildTemplateValues(
2009
+ {
2010
+ question: "",
2011
+ guidelines: "",
2012
+ inputFiles: batchInputFiles,
2013
+ evalCaseId: "batch",
2014
+ attempt: 0
2015
+ },
2016
+ this.config,
2017
+ outputFilePath
2018
+ );
2019
+ const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
2020
+ if (this.verbose) {
2021
+ console.log(
2022
+ `[cli-provider:${this.targetName}] (batch size=${requests.length}) cwd=${this.config.cwd ?? ""} command=${renderedCommand}`
2023
+ );
2024
+ }
2025
+ const result = await this.runCommand(renderedCommand, {
2026
+ cwd: this.config.cwd,
2027
+ env: process.env,
2028
+ timeoutMs: this.config.timeoutMs,
2029
+ signal: controller.signal
2030
+ });
2031
+ if (result.failed || (result.exitCode ?? 0) !== 0) {
2032
+ if (controller.signal.aborted) {
2033
+ throw new Error("CLI provider request was aborted");
2034
+ }
2035
+ if (result.timedOut) {
2036
+ throw new Error(
2037
+ `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? void 0)}`
2038
+ );
2039
+ }
2040
+ const codeText = result.exitCode !== null ? result.exitCode : "unknown";
2041
+ const detail = result.stderr.trim() || result.stdout.trim();
2042
+ const message = detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`;
2043
+ throw new Error(message);
2044
+ }
2045
+ const responseContent = await this.readAndCleanupOutputFile(outputFilePath);
2046
+ const recordsById = this.parseJsonlBatchOutput(responseContent);
2047
+ const requestedIds = requests.map((request) => request.evalCaseId).filter((id) => typeof id === "string" && id.trim().length > 0);
2048
+ const missingIds = requestedIds.filter((id) => !recordsById.has(id));
2049
+ if (missingIds.length > 0) {
2050
+ throw new Error(`CLI batch output missing ids: ${missingIds.join(", ")}`);
2051
+ }
2052
+ const responses = requests.map((request) => {
2053
+ const evalCaseId = request.evalCaseId;
2054
+ if (!evalCaseId) {
2055
+ return {
2056
+ text: "",
2057
+ raw: {
2058
+ command: renderedCommand,
2059
+ stderr: result.stderr,
2060
+ exitCode: result.exitCode ?? 0,
2061
+ cwd: this.config.cwd,
2062
+ outputFile: outputFilePath
2063
+ }
2064
+ };
2065
+ }
2066
+ const parsed = recordsById.get(evalCaseId);
2067
+ if (!parsed) {
2068
+ return {
2069
+ text: "",
2070
+ raw: {
2071
+ command: renderedCommand,
2072
+ stderr: result.stderr,
2073
+ exitCode: result.exitCode ?? 0,
2074
+ cwd: this.config.cwd,
2075
+ outputFile: outputFilePath
2076
+ }
2077
+ };
2078
+ }
2079
+ return {
2080
+ text: parsed.text,
2081
+ trace: parsed.trace,
2082
+ traceRef: parsed.traceRef,
2083
+ raw: {
2084
+ command: renderedCommand,
2085
+ stderr: result.stderr,
2086
+ exitCode: result.exitCode ?? 0,
2087
+ cwd: this.config.cwd,
2088
+ outputFile: outputFilePath,
2089
+ recordId: evalCaseId
2090
+ }
2091
+ };
2092
+ });
2093
+ return responses;
2094
+ }
1966
2095
  /**
1967
2096
  * Parse output content from CLI.
1968
2097
  * If the content is valid JSON with a 'text' field, extract text and optional trace.
@@ -1988,6 +2117,38 @@ var CliProvider = class {
1988
2117
  const validEvents = trace.filter(isTraceEvent);
1989
2118
  return validEvents.length > 0 ? validEvents : void 0;
1990
2119
  }
2120
+ parseJsonlBatchOutput(content) {
2121
+ const records = /* @__PURE__ */ new Map();
2122
+ const lines = content.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
2123
+ for (const line of lines) {
2124
+ let parsed;
2125
+ try {
2126
+ parsed = JSON.parse(line);
2127
+ } catch (error) {
2128
+ const reason = error instanceof Error ? error.message : String(error);
2129
+ throw new Error(`CLI batch output contains invalid JSONL line: ${reason}`);
2130
+ }
2131
+ if (typeof parsed !== "object" || parsed === null) {
2132
+ throw new Error("CLI batch output JSONL line must be an object");
2133
+ }
2134
+ const obj = parsed;
2135
+ const id = typeof obj.id === "string" ? obj.id : void 0;
2136
+ if (!id || id.trim().length === 0) {
2137
+ throw new Error("CLI batch output JSONL line missing required string field: id");
2138
+ }
2139
+ if (records.has(id)) {
2140
+ throw new Error(`CLI batch output contains duplicate id: ${id}`);
2141
+ }
2142
+ const text = typeof obj.text === "string" ? obj.text : obj.text === void 0 ? "" : JSON.stringify(obj.text);
2143
+ const traceRef = typeof obj.traceRef === "string" ? obj.traceRef : typeof obj.trace_ref === "string" ? obj.trace_ref : void 0;
2144
+ records.set(id, {
2145
+ text,
2146
+ trace: this.parseTrace(obj.trace),
2147
+ traceRef
2148
+ });
2149
+ }
2150
+ return records;
2151
+ }
1991
2152
  async readAndCleanupOutputFile(filePath) {
1992
2153
  try {
1993
2154
  const content = await readTextFile(filePath);
@@ -2049,7 +2210,7 @@ var CliProvider = class {
2049
2210
  );
2050
2211
  if (this.verbose) {
2051
2212
  console.log(
2052
- `[cli-provider:${this.targetName}] (healthcheck) CLI_EVALS_DIR=${process.env.CLI_EVALS_DIR ?? ""} cwd=${healthcheck.cwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
2213
+ `[cli-provider:${this.targetName}] (healthcheck) cwd=${healthcheck.cwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
2053
2214
  );
2054
2215
  }
2055
2216
  const result = await this.runCommand(renderedCommand, {
@@ -2117,11 +2278,11 @@ function shellEscape(value) {
2117
2278
  }
2118
2279
  return `'${value.replace(/'/g, `'"'"'`)}'`;
2119
2280
  }
2120
- function generateOutputFilePath(evalCaseId) {
2281
+ function generateOutputFilePath(evalCaseId, extension = ".json") {
2121
2282
  const safeEvalId = evalCaseId || "unknown";
2122
2283
  const timestamp = Date.now();
2123
2284
  const random = Math.random().toString(36).substring(2, 9);
2124
- return import_node_path8.default.join(import_node_os.default.tmpdir(), `agentv-${safeEvalId}-${timestamp}-${random}.json`);
2285
+ return import_node_path8.default.join(import_node_os.default.tmpdir(), `agentv-${safeEvalId}-${timestamp}-${random}${extension}`);
2125
2286
  }
2126
2287
  function formatTimeoutSuffix(timeoutMs) {
2127
2288
  if (!timeoutMs || timeoutMs <= 0) {
@@ -3301,10 +3462,14 @@ function resolveCliConfig(target, env, evalFilePath) {
3301
3462
  const filesFormat = resolveOptionalLiteralString(
3302
3463
  target.files_format ?? target.filesFormat ?? target.attachments_format ?? target.attachmentsFormat
3303
3464
  );
3465
+ const verbose = resolveOptionalBoolean(target.verbose ?? target.cli_verbose ?? target.cliVerbose);
3304
3466
  let cwd = resolveOptionalString(target.cwd, env, `${target.name} working directory`, {
3305
3467
  allowLiteral: true,
3306
3468
  optionalEnv: true
3307
3469
  });
3470
+ if (cwd && evalFilePath && !import_node_path11.default.isAbsolute(cwd)) {
3471
+ cwd = import_node_path11.default.resolve(import_node_path11.default.dirname(import_node_path11.default.resolve(evalFilePath)), cwd);
3472
+ }
3308
3473
  if (!cwd && evalFilePath) {
3309
3474
  cwd = import_node_path11.default.dirname(import_node_path11.default.resolve(evalFilePath));
3310
3475
  }
@@ -3312,7 +3477,7 @@ function resolveCliConfig(target, env, evalFilePath) {
3312
3477
  target.timeout_seconds ?? target.timeoutSeconds,
3313
3478
  `${target.name} timeout`
3314
3479
  );
3315
- const healthcheck = resolveCliHealthcheck(target.healthcheck, env, target.name);
3480
+ const healthcheck = resolveCliHealthcheck(target.healthcheck, env, target.name, evalFilePath);
3316
3481
  const commandTemplate = resolveString(
3317
3482
  commandTemplateSource,
3318
3483
  env,
@@ -3325,7 +3490,8 @@ function resolveCliConfig(target, env, evalFilePath) {
3325
3490
  filesFormat,
3326
3491
  cwd,
3327
3492
  timeoutMs,
3328
- healthcheck
3493
+ healthcheck,
3494
+ verbose
3329
3495
  };
3330
3496
  }
3331
3497
  function resolveTimeoutMs(source, description) {
@@ -3338,7 +3504,7 @@ function resolveTimeoutMs(source, description) {
3338
3504
  }
3339
3505
  return Math.floor(seconds * 1e3);
3340
3506
  }
3341
- function resolveCliHealthcheck(source, env, targetName) {
3507
+ function resolveCliHealthcheck(source, env, targetName, evalFilePath) {
3342
3508
  if (source === void 0 || source === null) {
3343
3509
  return void 0;
3344
3510
  }
@@ -3371,11 +3537,12 @@ function resolveCliHealthcheck(source, env, targetName) {
3371
3537
  allowLiteral: true,
3372
3538
  optionalEnv: true
3373
3539
  });
3540
+ const resolvedCwd = cwd && evalFilePath && !import_node_path11.default.isAbsolute(cwd) ? import_node_path11.default.resolve(import_node_path11.default.dirname(import_node_path11.default.resolve(evalFilePath)), cwd) : cwd;
3374
3541
  return {
3375
3542
  type: "command",
3376
3543
  commandTemplate,
3377
3544
  timeoutMs,
3378
- cwd
3545
+ cwd: resolvedCwd
3379
3546
  };
3380
3547
  }
3381
3548
  throw new Error(`${targetName} healthcheck type must be 'http' or 'command'`);
@@ -4196,6 +4363,7 @@ var CodeEvaluator = class {
4196
4363
  {
4197
4364
  question: context.evalCase.question,
4198
4365
  expected_outcome: context.evalCase.expected_outcome,
4366
+ expected_messages: context.evalCase.expected_messages,
4199
4367
  reference_answer: context.evalCase.reference_answer,
4200
4368
  candidate_answer: context.candidate,
4201
4369
  guideline_files: context.evalCase.guideline_paths,