@lesliechan721/aw 0.1.0-beta.8 → 0.1.0-beta.9

package/dist/authoring/scripts/validate-plugin.mjs

@@ -18,7 +18,7 @@ const CLI_PACKAGE_ROOT = path.basename(authoringRuntimeRoot) === 'dist'
     : authoringRuntimeRoot;
 const errors = [];
 const warnings = [];
-const SOURCE_PLUGIN_MANIFEST_CONTRACT_VERSION = 8;
+const SOURCE_PLUGIN_MANIFEST_CONTRACT_VERSION = 9;
 let capabilityDiagnostic = null;
 const MAX_SKILL_NAME_LEN = 64;
 const MAX_SKILL_DESCRIPTION_LEN = 1024;
@@ -150,22 +150,9 @@ const ARTIFACT_PROJECT_MANAGED_KIND_BY_TARGET = {
     'codex.skill': 'skill',
 };
 const PLUGIN_NAME_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
-const PLUGIN_EVAL_SCHEMA_VERSION = 2;
-const PLUGIN_EVAL_CASE_SCHEMA_VERSION = 1;
-const PLUGIN_EVAL_SCOPES = new Set(['user', 'project']);
-const PLUGIN_EVAL_SPLITS = new Set(['train', 'validation', 'sanity', 'all']);
-const PLUGIN_EVAL_TRIGGER_CASE_SPLITS = new Set(['train', 'validation', 'sanity']);
-const PLUGIN_EVAL_COMMAND_SUPPORTS = new Set([
-    'split',
-    'runs',
-    'case',
-    'model',
-    'codexBin',
-    'timeout',
-    'allowEnvironmentSkips',
-]);
 let yamlParser = null;
 let pluginVersionCheckResult = null;
+let skillEvalManifestTools = null;
 function error(msg) {
     errors.push(msg);
 }
@@ -958,769 +945,19 @@ function validateNonEmptyString(value, fieldName, maxLength) {
     }
     return trimmed;
 }
-function isPathInsideOrEqual(root, candidate) {
-    const relative = path.relative(root, candidate);
-    return relative === '' || (relative !== '..' &&
-        !relative.startsWith(`..${path.sep}`) &&
-        !path.isAbsolute(relative));
-}
-function validateRealPathBoundary(boundary, candidate, fieldName, boundaryName) {
-    const canonicalBoundary = fs.realpathSync(boundary);
-    const canonicalCandidate = fs.realpathSync(candidate);
-    if (!isPathInsideOrEqual(canonicalBoundary, canonicalCandidate)) {
-        error(`${fieldName} must stay inside the ${boundaryName}`);
-        return false;
-    }
-    return true;
-}
-function validateEvalRelativeFile(evalDir, value, fieldName) {
-    const relativePath = validateNonEmptyString(value, fieldName);
-    if (!relativePath)
-        return null;
-    const normalized = stripDotSlash(relativePath).split('\\').join('/');
-    if (path.isAbsolute(normalized) || normalized.split('/').includes('..')) {
-        error(`${fieldName} must stay inside the Skill eval directory`);
-        return null;
-    }
-    const absolutePath = path.resolve(evalDir, normalized);
-    const relativeToRoot = path.relative(evalDir, absolutePath);
-    if (relativeToRoot.startsWith('..') || path.isAbsolute(relativeToRoot)) {
-        error(`${fieldName} must stay inside the Skill eval directory`);
-        return null;
-    }
-    if (!fs.existsSync(absolutePath) || !fs.statSync(absolutePath).isFile()) {
-        error(`${fieldName} does not exist: ${normalized}`);
-        return null;
-    }
-    if (!validateRealPathBoundary(evalDir, absolutePath, fieldName, 'Skill eval directory')) {
-        return null;
-    }
-    return absolutePath;
-}
-function validateEvalWorkspacePath(value, fieldName, options = {}) {
-    const raw = validateNonEmptyString(value, fieldName);
-    if (!raw)
-        return null;
-    const treePattern = options.allowTreePattern === true && raw.endsWith('/**');
-    const candidate = treePattern ? raw.slice(0, -3) : raw;
-    if (candidate === '' ||
-        candidate.includes('\0') ||
-        candidate.includes('\\') ||
-        candidate.includes('*') ||
-        candidate.includes('?') ||
-        candidate.includes('[') ||
-        candidate.includes(']') ||
-        path.posix.isAbsolute(candidate) ||
-        path.win32.isAbsolute(candidate) ||
-        path.posix.normalize(candidate) !== candidate ||
-        candidate.split('/').includes('..')) {
-        error(`${fieldName} must be a normalized POSIX path inside the eval workspace`);
-        return null;
-    }
-    const firstSegment = candidate.split('/')[0];
-    if (firstSegment === '.git' || firstSegment === '.agents' || firstSegment === '.codex') {
-        error(`${fieldName} must not target an eval control directory`);
-        return null;
-    }
-    return raw;
-}
-function validateEvalFixtureTree(evalRoot, target, fieldName) {
-    const relativePath = path.relative(evalRoot, target);
-    let currentPath = evalRoot;
-    for (const segment of relativePath.split(path.sep).filter(Boolean)) {
-        currentPath = path.join(currentPath, segment);
-        const stat = fs.lstatSync(currentPath);
-        if (stat.isSymbolicLink()) {
-            error(`${fieldName} must not contain symbolic links`);
-            return false;
-        }
-    }
-    if (!validateRealPathBoundary(evalRoot, target, fieldName, 'Eval fixture root')) {
-        return false;
-    }
-    const stat = fs.lstatSync(target);
-    if (stat.isFile())
-        return true;
-    if (!stat.isDirectory()) {
-        error(`${fieldName} must reference a regular file or directory fixture`);
-        return false;
-    }
-    return fs.readdirSync(target).every((entry) => validateEvalFixtureTree(evalRoot, path.join(target, entry), fieldName));
-}
-function evalWorkspaceTargetsOverlap(left, right) {
-    return left === right || left.startsWith(`${right}/`) || right.startsWith(`${left}/`);
-}
-function validateConversationEnvironment(value, fieldName, fixtureRoot, occupiedTargets = new Set()) {
-    if (value === undefined)
-        return [];
-    const environment = expectRecord(value, fieldName);
-    if (!environment)
-        return [];
-    validateAllowedKeys(environment, fieldName, ['pathPrepend']);
-    if (!Array.isArray(environment.pathPrepend) || environment.pathPrepend.length === 0) {
-        error(`${fieldName}.pathPrepend must be a non-empty array`);
-        return [];
-    }
-    const targets = environment.pathPrepend
-        .map((entry, index) => ({
-        index,
-        relativePath: validateEvalWorkspacePath(entry, `${fieldName}.pathPrepend[${index}]`, {
-            allowTreePattern: false,
-        }),
-    }))
-        .filter((entry) => entry.relativePath !== null);
-    if (new Set(targets.map((entry) => entry.relativePath)).size !== targets.length) {
-        error(`${fieldName}.pathPrepend must not contain duplicates`);
-    }
-    targets.forEach(({ index, relativePath }) => {
-        const target = path.resolve(fixtureRoot, relativePath);
-        if (!fs.existsSync(target)) {
-            error(`${fieldName}.pathPrepend[${index}] must reference an existing fixture`);
-            return;
-        }
-        validateEvalFixtureTree(fixtureRoot, target, `${fieldName}.pathPrepend[${index}]`);
-        if ([...occupiedTargets].some((occupiedTarget) => evalWorkspaceTargetsOverlap(relativePath, occupiedTarget))) {
-            error(`${fieldName}.pathPrepend[${index}] conflicts with another PATH fixture target`);
-            return;
-        }
-        occupiedTargets.add(relativePath);
-    });
-    return targets.map((entry) => entry.relativePath);
-}
-function evalWorkspacePathMatchesPattern(relativePath, pattern) {
-    if (pattern.endsWith('/**')) {
-        const prefix = pattern.slice(0, -3);
-        return relativePath === prefix || relativePath.startsWith(`${prefix}/`);
-    }
-    return relativePath === pattern;
-}
-function validateConversationCaseCorpus(evalDir, suite, fieldName, suitePathPrepend = []) {
-    const casesPath = validateEvalRelativeFile(evalDir, suite.cases, `${fieldName}.cases`);
-    if (!casesPath)
-        return;
-    let document;
-    try {
-        document = JSON.parse(fs.readFileSync(casesPath, 'utf8'));
-    }
-    catch (e) {
-        error(`${fieldName}.cases is not valid JSON: ${e.message}`);
-        return;
-    }
-    const corpus = expectRecord(document, `${fieldName}.cases document`);
-    if (!corpus)
-        return;
-    const skillName = validateNonEmptyString(corpus.skill_name, `${fieldName}.cases.skill_name`);
-    if (skillName && skillName !== suite.skill) {
-        error(`${fieldName}.cases.skill_name must match ${JSON.stringify(suite.skill)}`);
-    }
-    if (!Array.isArray(corpus.evals) || corpus.evals.length === 0) {
-        error(`${fieldName}.cases.evals must be a non-empty array`);
-        return;
-    }
-    const ids = new Set();
-    corpus.evals.forEach((value, caseIndex) => {
-        const caseField = `${fieldName}.cases.evals[${caseIndex}]`;
-        const entry = expectRecord(value, caseField);
-        if (!entry)
-            return;
-        const id = validateNonEmptyString(entry.id, `${caseField}.id`);
-        if (id && ids.has(id)) {
-            error(`${fieldName}.cases has duplicate case id ${JSON.stringify(id)}`);
-        }
-        else if (id) {
-            ids.add(id);
-        }
-        validateNonEmptyString(entry.prompt, `${caseField}.prompt`);
-        const sandbox = entry.sandbox === undefined
-            ? 'read-only'
-            : validateNonEmptyString(entry.sandbox, `${caseField}.sandbox`);
-        if (sandbox && sandbox !== 'read-only' && sandbox !== 'workspace-write') {
-            error(`${caseField}.sandbox must be "read-only" or "workspace-write"`);
-        }
-        if (!Array.isArray(entry.scripted_replies)) {
-            error(`${caseField}.scripted_replies must be an array`);
-        }
-        else {
-            entry.scripted_replies.forEach((reply, replyIndex) => {
-                const replyField = `${caseField}.scripted_replies[${replyIndex}]`;
-                if (typeof reply === 'string')
-                    return;
-                const replyEntry = expectRecord(reply, replyField);
-                if (!replyEntry)
-                    return;
-                if (!Array.isArray(replyEntry.answers_by_index) || replyEntry.answers_by_index.some((answer) => typeof answer !== 'string')) {
-                    error(`${replyField}.answers_by_index must be an array of strings`);
-                }
-                if (replyEntry.expected_question_count !== undefined &&
-                    (!Number.isInteger(replyEntry.expected_question_count) ||
-                        replyEntry.expected_question_count < 0)) {
-                    error(`${replyField}.expected_question_count must be a non-negative integer`);
-                }
-                validateNonEmptyString(replyEntry.unexpected_question_answer, `${replyField}.unexpected_question_answer`);
-            });
-        }
-        validateNonEmptyString(entry.expected_output, `${caseField}.expected_output`);
-        validateConversationEnvironment(entry.environment, `${caseField}.environment`, path.dirname(casesPath), new Set(suitePathPrepend));
-        if (entry.workspace_files !== undefined) {
-            const workspaceFiles = expectRecord(entry.workspace_files, `${caseField}.workspace_files`);
-            if (workspaceFiles) {
-                const workspaceFileEntries = Object.entries(workspaceFiles);
-                if (workspaceFileEntries.length === 0) {
-                    error(`${caseField}.workspace_files must contain at least one file`);
-                }
-                workspaceFileEntries.forEach(([relativePath, content]) => {
-                    validateEvalWorkspacePath(relativePath, `${caseField}.workspace_files.${relativePath}`, { allowTreePattern: false });
-                    validateNonEmptyString(content, `${caseField}.workspace_files.${relativePath}`);
-                });
-            }
-        }
-        const review = expectRecord(entry.review, `${caseField}.review`);
-        if (review) {
-            if (!Array.isArray(review.checks) || review.checks.length === 0 || review.checks.some((check) => typeof check !== 'string' || check.trim() === '')) {
-                error(`${caseField}.review.checks must be a non-empty array of strings`);
-            }
-            if (review.workspace_unchanged !== undefined &&
-                typeof review.workspace_unchanged !== 'boolean') {
-                error(`${caseField}.review.workspace_unchanged must be boolean`);
-            }
-            const allowedWorkspaceChanges = [];
-            if (review.allowed_workspace_changes !== undefined) {
-                if (!Array.isArray(review.allowed_workspace_changes) ||
-                    review.allowed_workspace_changes.length === 0) {
-                    error(`${caseField}.review.allowed_workspace_changes must be a non-empty array`);
-                }
-                else {
-                    review.allowed_workspace_changes.forEach((pathValue, pathIndex) => {
-                        const normalized = validateEvalWorkspacePath(pathValue, `${caseField}.review.allowed_workspace_changes[${pathIndex}]`, { allowTreePattern: true });
-                        if (normalized)
-                            allowedWorkspaceChanges.push(normalized);
-                    });
-                }
-            }
-            const artifactPaths = [];
-            if (review.artifacts !== undefined) {
-                if (!Array.isArray(review.artifacts) || review.artifacts.length === 0) {
-                    error(`${caseField}.review.artifacts must be a non-empty array`);
-                }
-                else {
-                    review.artifacts.forEach((artifactValue, artifactIndex) => {
-                        const artifactField = `${caseField}.review.artifacts[${artifactIndex}]`;
-                        const artifact = expectRecord(artifactValue, artifactField);
-                        if (!artifact)
-                            return;
-                        validateAllowedKeys(artifact, artifactField, ['path', 'exists', 'contains', 'not_contains', 'json']);
-                        const artifactPath = validateEvalWorkspacePath(artifact.path, `${artifactField}.path`);
-                        if (artifactPath)
-                            artifactPaths.push(artifactPath);
-                        if (artifact.exists !== undefined && typeof artifact.exists !== 'boolean') {
-                            error(`${artifactField}.exists must be boolean`);
-                        }
-                        for (const key of ['contains', 'not_contains']) {
-                            if (artifact[key] !== undefined &&
-                                (!Array.isArray(artifact[key]) || artifact[key].length === 0 || artifact[key].some((fragment) => typeof fragment !== 'string' || fragment.length === 0))) {
-                                error(`${artifactField}.${key} must be a non-empty array of strings`);
-                            }
-                        }
-                        if (artifact.json !== undefined && !isRecord(artifact.json)) {
-                            error(`${artifactField}.json must be an object`);
-                        }
-                    });
-                }
-            }
-            if (new Set(artifactPaths).size !== artifactPaths.length) {
-                error(`${caseField}.review.artifacts contains duplicate paths`);
-            }
-            if (sandbox === 'workspace-write') {
-                if (review.workspace_unchanged !== false) {
-                    error(`${caseField} workspace-write requires review.workspace_unchanged=false`);
-                }
-                if (allowedWorkspaceChanges.length === 0 || artifactPaths.length === 0) {
-                    error(`${caseField} workspace-write requires allowed changes and artifacts`);
-                }
-                const unmatchedArtifact = artifactPaths.find((artifactPath) => !allowedWorkspaceChanges.some((pattern) => evalWorkspacePathMatchesPattern(artifactPath, pattern)));
-                if (unmatchedArtifact) {
-                    error(`${caseField} artifact ${JSON.stringify(unmatchedArtifact)} is outside allowed changes`);
-                }
-            }
-        }
-    });
-}
-function validateTriggerCaseCorpus(evalDir, suite, fieldName) {
-    const casesPath = validateEvalRelativeFile(evalDir, suite.cases, `${fieldName}.cases`);
-    if (!casesPath)
-        return;
-    let document;
-    try {
-        document = JSON.parse(fs.readFileSync(casesPath, 'utf8'));
-    }
-    catch (e) {
-        error(`${fieldName}.cases is not valid JSON: ${e.message}`);
-        return;
-    }
-    const corpus = expectRecord(document, `${fieldName}.cases document`);
-    if (!corpus)
-        return;
-    if (corpus.schemaVersion !== PLUGIN_EVAL_CASE_SCHEMA_VERSION) {
-        error(`${fieldName}.cases.schemaVersion must be ${PLUGIN_EVAL_CASE_SCHEMA_VERSION}`);
-    }
-    if (corpus.skill !== suite.skill) {
-        error(`${fieldName}.cases.skill must match ${JSON.stringify(suite.skill)}`);
-    }
-    if (!Array.isArray(corpus.cases) || corpus.cases.length === 0) {
-        error(`${fieldName}.cases.cases must be a non-empty array`);
-        return;
-    }
-    const ids = new Set();
-    const seenSplits = new Set();
-    let hasWorkspaceFixture = false;
-    corpus.cases.forEach((value, caseIndex) => {
-        const caseField = `${fieldName}.cases.cases[${caseIndex}]`;
-        const entry = expectRecord(value, caseField);
-        if (!entry)
-            return;
-        const id = validateNonEmptyString(entry.id, `${caseField}.id`);
-        if (id && !/^[a-z0-9][a-z0-9_]*$/.test(id)) {
-            error(`${caseField}.id must use lowercase snake_case`);
-        }
-        else if (id && ids.has(id)) {
-            error(`${fieldName}.cases has duplicate case id ${JSON.stringify(id)}`);
-        }
-        else if (id) {
-            ids.add(id);
-        }
-        if (!PLUGIN_EVAL_TRIGGER_CASE_SPLITS.has(entry.split)) {
-            error(`${caseField}.split must be train, validation, or sanity`);
-        }
-        else {
-            seenSplits.add(entry.split);
-        }
-        validateNonEmptyString(entry.query, `${caseField}.query`);
-        const usesLegacyExpectation = entry.should_load_skill !== undefined;
-        const usesRouteExpectation = entry.expected_loaded_skills !== undefined;
-        if (usesLegacyExpectation === usesRouteExpectation) {
-            error(`${caseField} must declare exactly one of should_load_skill or expected_loaded_skills`);
-        }
-        else if (usesLegacyExpectation && typeof entry.should_load_skill !== 'boolean') {
-            error(`${caseField}.should_load_skill must be boolean`);
-        }
-        else if (usesRouteExpectation) {
-            if (!Array.isArray(entry.expected_loaded_skills)) {
-                error(`${caseField}.expected_loaded_skills must be an array`);
-            }
-            else {
-                const expectedSkills = entry.expected_loaded_skills;
-                expectedSkills.forEach((skillName, skillIndex) => {
-                    if (typeof skillName !== 'string' || skillName.trim() === '') {
-                        error(`${caseField}.expected_loaded_skills[${skillIndex}] must be a non-empty string`);
-                    }
-                    else if (!suite.skills.includes(skillName)) {
-                        error(`${caseField}.expected_loaded_skills[${skillIndex}] must name a participant Skill`);
-                    }
-                });
-                if (new Set(expectedSkills).size !== expectedSkills.length) {
-                    error(`${caseField}.expected_loaded_skills must not contain duplicates`);
-                }
-                if (suite.forbidMultipleSkillLoads === true && expectedSkills.length > 1) {
-                    error(`${caseField} expects multiple Skills but the suite forbids multiple Skill loads`);
-                }
-            }
-        }
-        if (suite.observeSpawnAgent === true) {
-            if (typeof entry.should_spawn_agent !== 'boolean') {
-                error(`${caseField}.should_spawn_agent is required when observeSpawnAgent=true`);
-            }
-        }
-        else if (entry.should_spawn_agent !== undefined) {
-            error(`${caseField}.should_spawn_agent requires observeSpawnAgent=true`);
-        }
-        if (entry.workspace_fixture !== undefined) {
-            hasWorkspaceFixture = true;
-            validateNonEmptyString(entry.workspace_fixture, `${caseField}.workspace_fixture`);
-        }
-        if (entry.use_project_rules !== undefined &&
-            typeof entry.use_project_rules !== 'boolean') {
-            error(`${caseField}.use_project_rules must be boolean`);
-        }
-        if (entry.tags !== undefined) {
-            if (!Array.isArray(entry.tags) || entry.tags.length === 0) {
-                error(`${caseField}.tags must be a non-empty array`);
-            }
-            else {
-                entry.tags.forEach((tag, tagIndex) => {
-                    validateNonEmptyString(tag, `${caseField}.tags[${tagIndex}]`);
-                });
-            }
-        }
-    });
-    if (hasWorkspaceFixture && suite.fixtureModule === undefined) {
-        error(`${fieldName}.fixtureModule is required when cases declare workspace_fixture`);
-    }
-    if (Array.isArray(suite.releaseSplits)) {
-        suite.releaseSplits.forEach((split) => {
-            if (split !== 'all' && !seenSplits.has(split)) {
-                error(`${fieldName}.cases has no cases for release split ${JSON.stringify(split)}`);
-            }
-        });
-    }
-}
-function resolveEvalProfilePlugin(pluginDir, ownerManifest, pluginName, fieldName) {
-    if (pluginName === ownerManifest.name) {
-        return { pluginDir, manifest: ownerManifest };
-    }
-    const companionDir = path.join(path.dirname(pluginDir), pluginName);
-    const manifestPath = path.join(companionDir, '.agents-plugin', 'plugin.json');
-    if (!fs.existsSync(manifestPath)) {
-        error(`${fieldName} must reference repo-local plugin "${pluginName}"`);
-        return null;
-    }
-    let manifest;
+async function loadSkillEvalManifestTools() {
+    if (skillEvalManifestTools)
+        return skillEvalManifestTools;
+    const modulePath = path.join(CLI_PACKAGE_ROOT, 'dist', 'src', 'eval', 'manifest-v1.js');
     try {
-        manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
-    }
-    catch (e) {
-        error(`${fieldName} companion manifest is not valid JSON: ${e.message}`);
-        return null;
+        skillEvalManifestTools = await import(pathToFileURL(modulePath).href);
+        return skillEvalManifestTools;
     }
-    if (!isRecord(manifest) || manifest.name !== pluginName) {
-        error(`${fieldName} companion source must declare plugin name "${pluginName}"`);
+    catch {
         return null;
     }
-    return { pluginDir: companionDir, manifest };
 }
-function validateOnePluginEvalManifest(pluginDir, pluginManifest, evalDir, expectedSkill) {
-    const evalManifestPath = path.join(evalDir, 'manifest.json');
-    const manifestField = path.relative(pluginDir, evalManifestPath).split(path.sep).join('/');
-    let document;
-    try {
-        document = JSON.parse(fs.readFileSync(evalManifestPath, 'utf8'));
-    }
-    catch (e) {
-        error(`${manifestField} is not valid JSON: ${e.message}`);
-        return;
-    }
-    const manifest = expectRecord(document, manifestField);
-    if (!manifest)
-        return;
-    validateAllowedKeys(manifest, manifestField, ['schemaVersion', 'plugin', 'skill', 'profiles', 'suites']);
-    if (manifest.schemaVersion !== PLUGIN_EVAL_SCHEMA_VERSION) {
-        error(`${manifestField}.schemaVersion must be ${PLUGIN_EVAL_SCHEMA_VERSION}`);
-    }
-    if (manifest.plugin !== pluginManifest.name) {
-        error(`${manifestField}.plugin must match plugin name ${JSON.stringify(pluginManifest.name)}`);
-    }
-    const skill = validateNonEmptyString(manifest.skill, `${manifestField}.skill`);
-    if (skill && skill !== expectedSkill) {
-        error(`${manifestField}.skill must match owning Skill ${JSON.stringify(expectedSkill)}`);
-    }
-    if (skill && !collectDeclaredSkillNames(pluginDir, pluginManifest).all.has(skill)) {
-        error(`${manifestField}.skill must reference a declared static or dynamic skill`);
-    }
-    if (!Array.isArray(manifest.profiles) || manifest.profiles.length === 0) {
-        error(`${manifestField}.profiles must be a non-empty array`);
-        return;
-    }
-    const profilesById = new Map();
-    manifest.profiles.forEach((value, index) => {
-        const fieldName = `${manifestField}.profiles[${index}]`;
-        const profile = expectRecord(value, fieldName);
-        if (!profile)
-            return;
-        validateAllowedKeys(profile, fieldName, ['id', 'scope', 'plugins', 'runModel', 'requiredForRelease']);
-        const id = validateNonEmptyString(profile.id, `${fieldName}.id`);
-        if (id && !PLUGIN_NAME_PATTERN.test(id)) {
-            error(`${fieldName}.id must use lowercase kebab-case`);
-        }
-        else if (id && profilesById.has(id)) {
-            error(`${manifestField} has duplicate profile id ${JSON.stringify(id)}`);
-        }
-        if (!PLUGIN_EVAL_SCOPES.has(profile.scope)) {
-            error(`${fieldName}.scope must be user or project`);
-        }
-        const skillOwners = new Map();
-        const installedPlugins = new Set();
-        if (!Array.isArray(profile.plugins) || profile.plugins.length === 0) {
-            error(`${fieldName}.plugins must be a non-empty array`);
-        }
-        else {
-            profile.plugins.forEach((value, pluginIndex) => {
-                const pluginField = `${fieldName}.plugins[${pluginIndex}]`;
-                const installation = expectRecord(value, pluginField);
-                if (!installation)
-                    return;
-                validateAllowedKeys(installation, pluginField, ['name', 'config']);
-                const pluginName = validateNonEmptyString(installation.name, `${pluginField}.name`);
-                if (!pluginName)
-                    return;
-                if (!PLUGIN_NAME_PATTERN.test(pluginName)) {
-                    error(`${pluginField}.name must use lowercase kebab-case`);
-                }
-                if (installedPlugins.has(pluginName)) {
-                    error(`${fieldName}.plugins has duplicate plugin ${JSON.stringify(pluginName)}`);
-                    return;
-                }
-                installedPlugins.add(pluginName);
-                if (installation.config !== undefined && !isRecord(installation.config)) {
-                    error(`${pluginField}.config must be an object`);
-                }
-                const source = resolveEvalProfilePlugin(pluginDir, pluginManifest, pluginName, `${pluginField}.name`);
-                if (!source)
-                    return;
-                for (const skillName of collectDeclaredSkillNames(source.pluginDir, source.manifest).all) {
-                    const existing = skillOwners.get(skillName);
-                    if (existing && existing !== pluginName) {
-                        error(`${fieldName} installs duplicate Skill key ${JSON.stringify(skillName)} ` +
-                            `from plugins ${JSON.stringify(existing)} and ${JSON.stringify(pluginName)}`);
-                    }
-                    else {
-                        skillOwners.set(skillName, pluginName);
-                    }
-                }
-            });
-        }
-        if (!installedPlugins.has(pluginManifest.name)) {
-            error(`${fieldName}.plugins must include owner plugin ${JSON.stringify(pluginManifest.name)}`);
-        }
-        if (typeof profile.runModel !== 'boolean') {
-            error(`${fieldName}.runModel must be boolean`);
-        }
-        if (typeof profile.requiredForRelease !== 'boolean') {
-            error(`${fieldName}.requiredForRelease must be boolean`);
-        }
-        if (profile.scope === 'user' && profile.runModel === true) {
-            error(`${fieldName}: eval schema v2 does not allow real model execution for user scope`);
-        }
-        if (id && PLUGIN_NAME_PATTERN.test(id) && !profilesById.has(id)) {
-            profilesById.set(id, { ...profile, skillOwners });
-        }
-    });
-    if (!manifest.profiles.some((profile) => isRecord(profile) && profile.scope === 'project' && profile.runModel === true)) {
-        error(`${manifestField} must define at least one project profile with runModel=true`);
-    }
-    if (!Array.isArray(manifest.suites) || manifest.suites.length === 0) {
-        error(`${manifestField}.suites must be a non-empty array`);
-        return;
-    }
-    const suiteIds = new Set();
-    manifest.suites.forEach((value, index) => {
-        const fieldName = `${manifestField}.suites[${index}]`;
-        const suite = expectRecord(value, fieldName);
-        if (!suite)
-            return;
-        const id = validateNonEmptyString(suite.id, `${fieldName}.id`);
-        if (id && !PLUGIN_NAME_PATTERN.test(id)) {
-            error(`${fieldName}.id must use lowercase kebab-case`);
-        }
-        else if (id && suiteIds.has(id)) {
-            error(`${manifestField} has duplicate suite id ${JSON.stringify(id)}`);
-        }
-        else if (id) {
-            suiteIds.add(id);
-        }
-        if (suite.kind !== 'command' && suite.kind !== 'conversation' && suite.kind !== 'trigger') {
-            error(`${fieldName}.kind must be command, conversation, or trigger`);
-        }
-        const suiteSkills = new Set();
-        if (!Array.isArray(suite.skills) || suite.skills.length === 0) {
-            error(`${fieldName}.skills must be a non-empty array`);
-        }
-        else {
-            suite.skills.forEach((skillName, skillIndex) => {
-                const skillField = `${fieldName}.skills[${skillIndex}]`;
-                const normalized = validateNonEmptyString(skillName, skillField);
-                if (!normalized)
-                    return;
-                if (!PLUGIN_NAME_PATTERN.test(normalized)) {
-                    error(`${skillField} must use lowercase kebab-case`);
-                }
-                else if (suiteSkills.has(normalized)) {
-                    error(`${fieldName}.skills has duplicate Skill ${JSON.stringify(normalized)}`);
-                }
-                else {
-                    suiteSkills.add(normalized);
-                }
-            });
-        }
-        if (skill && !suiteSkills.has(skill)) {
-            error(`${fieldName}.skills must include owner Skill ${JSON.stringify(skill)}`);
-        }
-        if (!Array.isArray(suite.profiles) || suite.profiles.length === 0) {
-            error(`${fieldName}.profiles must be a non-empty array`);
-        }
-        else {
-            suite.profiles.forEach((profileId, profileIndex) => {
-                const profileField = `${fieldName}.profiles[${profileIndex}]`;
-                if (typeof profileId !== 'string' || profileId.trim() === '') {
-                    error(`${profileField} must be a non-empty string`);
-                    return;
-                }
-                const profile = profilesById.get(profileId);
-                if (!profile) {
-                    error(`${fieldName} references unknown profile ${JSON.stringify(profileId)}`);
-                }
-                else if (profile.runModel !== true) {
-                    error(`${fieldName} cannot execute materialize-only profile ${JSON.stringify(profileId)}`);
-                }
-                if (profile) {
-                    for (const skillName of suiteSkills) {
-                        if (!profile.skillOwners.has(skillName)) {
-                            error(`${fieldName} requires Skill ${JSON.stringify(skillName)}, but profile ` +
-                                `${JSON.stringify(profileId)} does not install a plugin that declares it`);
-                        }
-                    }
-                }
-            });
-        }
-        if (!Number.isInteger(suite.defaultRuns) || suite.defaultRuns <= 0) {
-            error(`${fieldName}.defaultRuns must be a positive integer`);
-        }
-        if (!Number.isInteger(suite.perRunTimeoutMs) || suite.perRunTimeoutMs <= 0) {
-            error(`${fieldName}.perRunTimeoutMs must be a positive integer`);
-        }
-        if (typeof suite.requiredForRelease !== 'boolean') {
-            error(`${fieldName}.requiredForRelease must be boolean`);
-        }
-        if (!Array.isArray(suite.releaseSplits)) {
-            error(`${fieldName}.releaseSplits must be an array`);
-        }
-        else {
-            suite.releaseSplits.forEach((split, splitIndex) => {
-                if (!PLUGIN_EVAL_SPLITS.has(split)) {
-                    error(`${fieldName}.releaseSplits[${splitIndex}] is unsupported`);
-                }
-            });
-        }
-        if (suite.kind === 'command') {
-            validateAllowedKeys(suite, fieldName, [
-                'id', 'kind', 'skills', 'entry', 'cases', 'profiles', 'defaultRuns', 'releaseSplits',
-                'requiredForRelease', 'perRunTimeoutMs', 'suiteTimeoutMs', 'supports',
-            ]);
-            validateEvalRelativeFile(evalDir, suite.entry, `${fieldName}.entry`);
-            if (!Number.isInteger(suite.suiteTimeoutMs) || suite.suiteTimeoutMs <= 0) {
-                error(`${fieldName}.suiteTimeoutMs must be a positive integer`);
-            }
-            else if (suite.suiteTimeoutMs < suite.perRunTimeoutMs) {
-                error(`${fieldName}.suiteTimeoutMs must be greater than or equal to perRunTimeoutMs`);
-            }
-            if (!Array.isArray(suite.supports)) {
-                error(`${fieldName}.supports must be an array`);
-            }
-            else {
-                suite.supports.forEach((support, supportIndex) => {
-                    if (!PLUGIN_EVAL_COMMAND_SUPPORTS.has(support)) {
-                        error(`${fieldName}.supports[${supportIndex}] is unsupported`);
-                    }
-                });
-                if (Array.isArray(suite.releaseSplits) &&
-                    suite.releaseSplits.length > 0 &&
-                    !suite.supports.includes('split')) {
-                    error(`${fieldName} declares releaseSplits but does not support split`);
-                }
-                const supportsCase = suite.supports.includes('case');
-                if (supportsCase && suite.cases === undefined) {
-                    error(`${fieldName}.cases is required when supports includes case`);
-                }
-                else if (!supportsCase && suite.cases !== undefined) {
-                    error(`${fieldName}.cases requires supports to include case`);
-                }
-                if (supportsCase && suite.cases !== undefined) {
-                    const casesPath = validateEvalRelativeFile(evalDir, suite.cases, `${fieldName}.cases`);
-                    if (casesPath) {
-                        try {
-                            const casesDocument = JSON.parse(fs.readFileSync(casesPath, 'utf8'));
-                            if (!isRecord(casesDocument) || casesDocument.skill_name !== skill) {
-                                error(`${fieldName}.cases.skill_name must match ${JSON.stringify(skill)}`);
-                            }
-                            else if (!Array.isArray(casesDocument.evals) || casesDocument.evals.length === 0) {
-                                error(`${fieldName}.cases.evals must be a non-empty array`);
-                            }
-                            else {
-                                const caseIds = new Set();
-                                casesDocument.evals.forEach((entry, caseIndex) => {
-                                    const caseField = `${fieldName}.cases.evals[${caseIndex}]`;
-                                    if (!isRecord(entry) || typeof entry.id !== 'string' || !/^[a-z0-9_]+$/.test(entry.id)) {
-                                        error(`${caseField}.id must use lowercase snake_case`);
-                                    }
-                                    else if (caseIds.has(entry.id)) {
-                                        error(`${fieldName}.cases has duplicate case id ${JSON.stringify(entry.id)}`);
-                                    }
-                                    else {
-                                        caseIds.add(entry.id);
-                                    }
-                                });
-                            }
-                        }
-                        catch (cause) {
-                            error(`${fieldName}.cases is not valid JSON: ${cause.message}`);
-                        }
-                    }
-                }
-            }
-        }
-        else if (suite.kind === 'conversation' || suite.kind === 'trigger') {
-            const normalizedSuite = { ...suite, skill };
-            const isolation = suite.isolation ?? 'standard';
-            if (isolation !== 'standard' && isolation !== 'strict') {
-                error(`${fieldName}.isolation must be standard or strict`);
-            }
-            if (suite.kind === 'conversation') {
-                validateAllowedKeys(suite, fieldName, [
-                    'id', 'kind', 'skills', 'cases', 'profiles', 'defaultRuns', 'releaseSplits',
-                    'requiredForRelease', 'perRunTimeoutMs', 'isolation', 'environment',
-                ]);
-                if (Array.isArray(suite.releaseSplits) && suite.releaseSplits.length > 0) {
-                    error(`${fieldName} cannot declare releaseSplits because conversation cases do not support split filtering`);
-                }
-                const suitePathPrepend = validateConversationEnvironment(suite.environment, `${fieldName}.environment`, evalDir);
-                validateConversationCaseCorpus(evalDir, normalizedSuite, fieldName, suitePathPrepend);
-            }
-            else {
-                validateAllowedKeys(suite, fieldName, [
-                    'id', 'kind', 'skills', 'cases', 'fixtureModule', 'observeSpawnAgent', 'isolation',
-                    'strictNegativeSplits', 'minimumPassRate', 'forbidMultipleSkillLoads',
-                    'profiles', 'defaultRuns', 'releaseSplits',
-                    'requiredForRelease', 'perRunTimeoutMs',
-                ]);
-                if (typeof suite.observeSpawnAgent !== 'boolean') {
-                    error(`${fieldName}.observeSpawnAgent must be boolean`);
-                }
-                if (!Array.isArray(suite.strictNegativeSplits)) {
-                    error(`${fieldName}.strictNegativeSplits must be an array`);
-                }
-                else {
-                    suite.strictNegativeSplits.forEach((split, splitIndex) => {
-                        if (!PLUGIN_EVAL_TRIGGER_CASE_SPLITS.has(split)) {
-                            error(`${fieldName}.strictNegativeSplits[${splitIndex}] is unsupported`);
-                        }
-                    });
-                }
-                if (suite.minimumPassRate !== undefined &&
-                    (typeof suite.minimumPassRate !== 'number' ||
-                        !Number.isFinite(suite.minimumPassRate) ||
-                        suite.minimumPassRate <= 0 ||
-                        suite.minimumPassRate > 1)) {
-                    error(`${fieldName}.minimumPassRate must be greater than 0 and at most 1`);
-                }
-                if (suite.forbidMultipleSkillLoads !== undefined &&
-                    typeof suite.forbidMultipleSkillLoads !== 'boolean') {
-                    error(`${fieldName}.forbidMultipleSkillLoads must be boolean`);
-                }
-                if (suite.fixtureModule !== undefined) {
-                    const fixturePath = validateEvalRelativeFile(evalDir, suite.fixtureModule, `${fieldName}.fixtureModule`);
-                    if (fixturePath && path.extname(fixturePath) !== '.mjs') {
-                        error(`${fieldName}.fixtureModule must reference an .mjs file`);
-                    }
-                }
-                validateTriggerCaseCorpus(evalDir, normalizedSuite, fieldName);
-            }
-        }
-    });
-    if (skill
-        && collectDeclaredSkillNames(pluginDir, pluginManifest).dynamic.has(skill)
-        && !manifest.suites.some((suite) => isRecord(suite) && suite.requiredForRelease === true)) {
-        error(`${manifestField}.suites must include at least one release-required suite for dynamic Skill ${JSON.stringify(skill)}`);
-    }
-}
-function validatePluginEvalManifest(pluginDir, pluginManifest) {
+async function validatePluginEvalManifest(pluginDir, pluginManifest) {
     const legacyRoot = path.join(pluginDir, 'evals');
     if (fs.existsSync(legacyRoot)) {
         error('Plugin-root evals/ is legacy; put evals/ inside its owning static or dynamic Skill directory.');
@@ -1729,17 +966,33 @@ function validatePluginEvalManifest(pluginDir, pluginManifest) {
         .sort()
         .map((skill) => ({ skill, root: path.join(pluginDir, 'skills', skill) }));
     const seenSkills = new Set();
+    let manifestTools = null;
     for (const owner of skillRoots) {
         const evalDir = path.join(owner.root, 'evals');
         const manifestPath = path.join(evalDir, 'manifest.json');
         if (!fs.existsSync(manifestPath))
             continue;
+        const manifestField = path.relative(pluginDir, manifestPath).split(path.sep).join('/');
         if (seenSkills.has(owner.skill)) {
             error(`Skill ${JSON.stringify(owner.skill)} has more than one evals/manifest.json owner`);
             continue;
         }
         seenSkills.add(owner.skill);
-        validateOnePluginEvalManifest(pluginDir, pluginManifest, evalDir, owner.skill);
+        manifestTools ??= await loadSkillEvalManifestTools();
+        if (typeof manifestTools?.validateAwSkillEvalManifest !== 'function') {
+            error('Canonical AwSkillEvalManifest v1 validator is unavailable; run the current built aw CLI.');
+            continue;
+        }
+        const validation = await manifestTools.validateAwSkillEvalManifest({
+            plugin: {
+                rootDir: pluginDir,
+                manifestPath: path.join(pluginDir, '.agents-plugin', 'plugin.json'),
+                manifest: pluginManifest,
+            },
+            skill: owner.skill,
+        });
+        for (const message of validation.errors)
+            error(`${manifestField}: ${message}`);
     }
 }
 function countCharacters(value) {
@@ -4252,7 +3505,6 @@ async function validatePluginDir(pluginDir, options) {
             manifest: normalizedManifest,
             implementedCapabilities: cliManifestTools.implementedCapabilities,
         });
-        const blockingObservations = requirements.observations.filter((observation) => observation.blocking);
         if (requirements.unsupportedByCurrentAw.length > 0) {
             capabilityDiagnostic = {
                 errorCode: 'awCapabilityUnsupported',
@@ -4261,8 +3513,12 @@ async function validatePluginDir(pluginDir, options) {
             };
             error(`[awCapabilityUnsupported] missingCapabilities=${JSON.stringify(requirements.unsupportedByCurrentAw)}`);
         }
-        for (const observation of blockingObservations) {
-            error(`[awCapabilityScanBlocked] ${observation.path}: ${observation.message}`);
+        const satisfiedCapabilities = new Set([...requirements.declared, ...requirements.derived]);
+        const advisoryObservations = requirements.observations.filter((observation) => (observation.code !== 'aw-command-detected'
+            || observation.capability === undefined
+            || !satisfiedCapabilities.has(observation.capability)));
+        for (const observation of advisoryObservations) {
+            warn(`[awCapabilityObservation] ${observation.path}: ${observation.message}`);
         }
     }
     catch (e) {
@@ -4274,7 +3530,7 @@ async function validatePluginDir(pluginDir, options) {
     validateNoBuildOutputDirs(pluginDir);
     validateDeclaredNativeAssets(pluginDir, manifest);
     validateCanonicalSkillLayout(pluginDir, manifest);
-    validatePluginEvalManifest(pluginDir, manifest);
+    await validatePluginEvalManifest(pluginDir, manifest);
     const artifactRenderEntry = validateScriptsLightweight(pluginDir, manifest);
     const runtimeImports = await resolveAuthorRuntimeImports({
         pluginDir,