@post-print/agent-test 1.1.8 → 2.0.0

package/dist/validate-seeds.js

@@ -1,72 +0,0 @@
-import { resolve } from "node:path";
-import { createSealedWorkspace, defaultSealedOverlayPaths, isCallerHeadWorkspace, } from "@post-print/agent-harness";
-import { discoverSuites } from "./discover-suites.js";
-import { loadSuiteFile } from "./load-suite.js";
-import { seedScenarioWorktree } from "./scenario-seed.js";
-import { resolveScenarioWorkspaceRel } from "./validate-suite.js";
-/** Apply each seedPatch in a sealed workspace to verify hunks match the current baseline. */
-export async function validateSeedPatches(options) {
-    const suitesDir = resolve(options.cwd, options.suitesDir);
-    const suitePaths = await discoverSuites(suitesDir);
-    const filtered = options.filter
-        ? suitePaths.filter((path) => path.includes(`/${options.filter}/`) ||
-            path.endsWith(`/${options.filter}/scenarios.json`))
-        : suitePaths;
-    const issues = [];
-    let checked = 0;
-    for (const suitePath of filtered) {
-        const suite = await loadSuiteFile(suitePath, { rubricsDir: options.rubricsDir });
-        for (const scenario of suite.scenarios) {
-            if (!scenario.seedPatch) {
-                continue;
-            }
-            checked++;
-            try {
-                await validateOneSeed(options.cwd, scenario.seedPatch, {
-                    stageOnly: scenario.seedStageOnly,
-                    workspace: resolveScenarioWorkspaceRel(suite, scenario),
-                });
-            }
-            catch (error) {
-                issues.push({
-                    suite: suite.name,
-                    scenario: scenario.name,
-                    seedPatch: scenario.seedPatch,
-                    message: error instanceof Error ? error.message : String(error),
-                });
-            }
-        }
-    }
-    return { ok: issues.length === 0, issues, checked };
-}
-async function validateOneSeed(repoRoot, seedPatch, options) {
-    let cleanup;
-    let applyPath;
-    try {
-        const sealed = await createSealedWorkspace({
-            callerCwd: repoRoot,
-            workspace: options.workspace,
-            overlayPaths: isCallerHeadWorkspace(options.workspace)
-                ? defaultSealedOverlayPaths()
-                : undefined,
-        });
-        cleanup = sealed.cleanup;
-        applyPath = sealed.path;
-        await seedScenarioWorktree(repoRoot, applyPath, seedPatch, {
-            stageOnly: options.stageOnly,
-        });
-    }
-    finally {
-        if (cleanup) {
-            await cleanup().catch(() => undefined);
-        }
-    }
-}
-export function formatSeedValidationReport(report) {
-    if (report.ok) {
-        return `Validated ${report.checked} seed patch(es): OK`;
-    }
-    const lines = report.issues.map((issue) => `  - ${issue.suite}/${issue.scenario} (${issue.seedPatch}): ${issue.message}`);
-    return [`Seed validation failed (${report.issues.length} issue(s)):`, ...lines].join("\n");
-}
-//# sourceMappingURL=validate-seeds.js.map

package/dist/validate-seeds.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"validate-seeds.js","sourceRoot":"","sources":["../src/validate-seeds.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EACN,qBAAqB,EACrB,yBAAyB,EACzB,qBAAqB,GACrB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAC;AAelE,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAKzC;IACA,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM;QAC9B,CAAC,CAAC,UAAU,CAAC,MAAM,CACjB,CAAC,IAAI,EAAE,EAAE,CACR,IAAI,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YACpC,IAAI,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,MAAM,iBAAiB,CAAC,CACnD;QACF,CAAC,CAAC,UAAU,CAAC;IAEd,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,SAAS,IAAI,QAAQ,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;QACjF,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACxC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;gBACzB,SAAS;YACV,CAAC;YACD,OAAO,EAAE,CAAC;YACV,IAAI,CAAC;gBACJ,MAAM,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,SAAS,EAAE;oBACtD,SAAS,EAAE,QAAQ,CAAC,aAAa;oBACjC,SAAS,EAAE,2BAA2B,CAAC,KAAK,EAAE,QAAQ,CAAC;iBACvD,CAAC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,MAAM,CAAC,IAAI,CAAC;oBACX,KAAK,EAAE,KAAK,CAAC,IAAI;oBACjB,QAAQ,EAAE,QAAQ,CAAC,IAAI;oBACvB,SAAS,EAAE,QAAQ,CAAC,SAAS;oBAC7B,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC/D,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AACrD,CAAC;AAED,KAAK,UAAU,eAAe,CAC7B,QAAgB,EAChB,SAAiB,EACjB,OAAoD;IAEpD,IAAI,OAA0C,CAAC;IAC/C,IAAI,SAAiB,CAAC;IACtB,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC;YAC1C,SAAS,EAAE,QAAQ;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,YAAY,EAAE,qBAAqB,CAAC,OAAO,CAAC,SAAS,CAAC;gBACrD,CAAC,CAAC,yBAAyB,EAAE;gBAC7B,CAAC,CAAC,SAAS;SACZ,CAAC,CAAC;QACH,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QACzB,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC;QACxB,MAAM,oBAAoB,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE;YAC1D,SAAS,EAAE,OAAO,CAAC,SAAS;SAC5B,CAAC,CAAC;IACJ,CAAC;YAAS,CAAC;QACV,IAAI,OAAO,EAAE,CAAC;YACb,MAAM,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,MAA4B;IACtE,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,aAAa,MAAM,CAAC,OAAO,qBAAqB,CAAC;IACzD,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAC9B,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,SAAS,MAAM,KAAK,CAAC,OAAO,EAAE,CACxF,CAAC;IACF,OAAO,CAAC,2BAA2B,MAAM,CAAC,MAAM,CAAC,MAAM,aAAa,EAAE,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5F,CAAC"}

package/dist/validate-suite.d.ts

@@ -1,24 +0,0 @@
-import type { AgentScenario, AgentSuiteFile } from "./types.js";
-export interface SuiteValidationIssue {
-    suitePath: string;
-    scenario?: string;
-    field: string;
-    message: string;
-}
-export interface SuiteValidationReport {
-    ok: boolean;
-    issues: SuiteValidationIssue[];
-    suitesChecked: number;
-    scenariosChecked: number;
-}
-/** Semantic validation beyond structural loadSuiteFile checks. */
-export declare function validateSuiteFile(suitePath: string, suite: AgentSuiteFile): SuiteValidationIssue[];
-export declare function validateSuitePaths(suitePaths: string[], options?: {
-    validatePaths?: boolean;
-    repoRoot?: string;
-    rubricsDir?: string;
-}): Promise<SuiteValidationReport>;
-export declare function formatValidationReport(report: SuiteValidationReport): string;
-/** Scenario workspace wins over suite defaults. Caller HEAD when omit or `"."`. */
-export declare function resolveScenarioWorkspaceRel(suite: Pick<AgentSuiteFile, "defaults">, scenario: Pick<AgentScenario, "workspace">): string | undefined;
-//# sourceMappingURL=validate-suite.d.ts.map

package/dist/validate-suite.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"validate-suite.d.ts","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EACX,aAAa,EACb,cAAc,EAKd,MAAM,YAAY,CAAC;AAiEpB,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACrC,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CACzB;AAspBD,kEAAkE;AAClE,wBAAgB,iBAAiB,CAChC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,cAAc,GACnB,oBAAoB,EAAE,CAsCxB;AAED,wBAAsB,kBAAkB,CACvC,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,CAAC,EAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3E,OAAO,CAAC,qBAAqB,CAAC,CAyHhC;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,qBAAqB,GAAG,MAAM,CAY5E;AAED,mFAAmF;AACnF,wBAAgB,2BAA2B,CAC1C,KAAK,EAAE,IAAI,CAAC,cAAc,EAAE,UAAU,CAAC,EACvC,QAAQ,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,GACxC,MAAM,GAAG,SAAS,CAIpB"}

package/dist/validate-suite.js

@@ -1,542 +0,0 @@
-import { access, stat } from "node:fs/promises";
-import { resolve } from "node:path";
-import { isHostSlug, isKnownAgentHost, isRepoRelativeSkillPath, parseScenarioWorkspace, skillManifestRelPath, skillPathsFromSetting, } from "@post-print/agent-harness";
-import { applyCompareArm, parseCompareArmId, resolveCompareArms } from "./compare-scenario.js";
-import { loadSuiteFile } from "./load-suite.js";
-const REPLAY_DEPRECATION = "Replay-based testing is deprecated and no longer supported; use Cursor, Claude, or OpenAI.";
-function isKnownHostId(host) {
-    return isKnownAgentHost(host);
-}
-function unknownHostMessage(host) {
-    if (host === "replay") {
-        return REPLAY_DEPRECATION;
-    }
-    if (typeof host === "string" && isHostSlug(host)) {
-        return `unknown host "${host}". Register it with registerHostAdapter() or --adapter`;
-    }
-    return `host must be a lowercase slug (cursor|claude|openai or a registered adapter), got ${JSON.stringify(host)}`;
-}
-const VALID_PROFILES = new Set(["shared", "cursor", "claude", "skeleton"]);
-const VALID_CONTEXT_MODES = new Set(["host-native", "harness-preamble"]);
-function injectsSkillCatalog(value) {
-    if (value === undefined || value === "none")
-        return false;
-    if (Array.isArray(value))
-        return value.length > 0;
-    return value.mode !== "none";
-}
-function isValidSkillSetting(value) {
-    if (value === "none") {
-        return true;
-    }
-    if (Array.isArray(value)) {
-        return (value.length > 0 &&
-            value.every((item) => typeof item === "string" && isRepoRelativeSkillPath(item)));
-    }
-    if (!value || typeof value !== "object") {
-        return false;
-    }
-    const record = value;
-    if (record.mode !== undefined &&
-        record.mode !== "none" &&
-        record.mode !== "catalog" &&
-        record.mode !== "full") {
-        return false;
-    }
-    if (record.mode === "none") {
-        return true;
-    }
-    return (Array.isArray(record.include) &&
-        record.include.length > 0 &&
-        record.include.every((item) => typeof item === "string" && isRepoRelativeSkillPath(item)));
-}
-const VALID_TIERS = new Set(["low", "medium", "high"]);
-const VALID_REVIEW_DEPTHS = new Set([
-    "quick",
-    "standard",
-    "thorough",
-    "full",
-]);
-function pushIssue(issues, suitePath, field, message, scenario) {
-    issues.push({ suitePath, scenario, field, message });
-}
-function validateRubric(issues, suitePath, scenarioName, rubric, fieldPrefix = "rubric") {
-    if (rubric.tier !== undefined && !VALID_TIERS.has(rubric.tier)) {
-        pushIssue(issues, suitePath, `${fieldPrefix}.tier`, `tier must be low|medium|high, got ${JSON.stringify(rubric.tier)}`, scenarioName);
-    }
-    if (rubric.reviewDepth !== undefined && !VALID_REVIEW_DEPTHS.has(rubric.reviewDepth)) {
-        pushIssue(issues, suitePath, `${fieldPrefix}.reviewDepth`, `reviewDepth must be quick|standard|thorough|full, got ${JSON.stringify(rubric.reviewDepth)}`, scenarioName);
-    }
-    for (const key of [
-        "must",
-        "mustNot",
-        "mustRun",
-        "mustRunSuccessfully",
-        "allowedCommands",
-        "mustCallTool",
-        "mustCallToolsInOrder",
-        "mustNotCallTool",
-        "mustReadPath",
-        "mustNotReadPath",
-    ]) {
-        const value = rubric[key];
-        if (value !== undefined && !Array.isArray(value)) {
-            pushIssue(issues, suitePath, `${fieldPrefix}.${key}`, `${key} must be an array of strings`, scenarioName);
-        }
-    }
-    for (const key of ["mustInvokeSkill", "mustNotInvokeSkill"]) {
-        const value = rubric[key];
-        if (value !== undefined && !Array.isArray(value)) {
-            pushIssue(issues, suitePath, `${fieldPrefix}.${key}`, `${key} must be an array of skill folder names`, scenarioName);
-        }
-    }
-    if (rubric.judge !== undefined) {
-        if (!Array.isArray(rubric.judge)) {
-            pushIssue(issues, suitePath, `${fieldPrefix}.judge`, "judge must be an array of strings or { question } objects", scenarioName);
-        }
-        else {
-            for (const [index, item] of rubric.judge.entries()) {
-                if (typeof item === "string") {
-                    continue;
-                }
-                if (typeof item !== "object" || item === null || typeof item.question !== "string") {
-                    pushIssue(issues, suitePath, `${fieldPrefix}.judge[${index}]`, "judge item must be a string or { question: string }", scenarioName);
-                }
-            }
-        }
-    }
-}
-function validateOptionalDescription(issues, suitePath, field, value, scenarioName) {
-    if (value === undefined) {
-        return;
-    }
-    if (typeof value !== "string" || !value.trim()) {
-        pushIssue(issues, suitePath, field, "description must be a non-empty string", scenarioName);
-    }
-}
-function validateScenario(issues, suitePath, scenario) {
-    validateOptionalDescription(issues, suitePath, "description", scenario.description, scenario.name);
-    if (scenario.host !== undefined && !isKnownHostId(scenario.host)) {
-        pushIssue(issues, suitePath, "host", unknownHostMessage(scenario.host), scenario.name);
-    }
-    if (scenario.contextMode !== undefined && !VALID_CONTEXT_MODES.has(scenario.contextMode)) {
-        pushIssue(issues, suitePath, "contextMode", `contextMode must be host-native|harness-preamble, got ${JSON.stringify(scenario.contextMode)}`, scenario.name);
-    }
-    if ("replayTrace" in scenario) {
-        pushIssue(issues, suitePath, "replayTrace", REPLAY_DEPRECATION, scenario.name);
-    }
-    if (scenario.profile !== undefined && !VALID_PROFILES.has(scenario.profile)) {
-        pushIssue(issues, suitePath, "profile", `profile must be shared|cursor|claude|skeleton, got ${JSON.stringify(scenario.profile)}`, scenario.name);
-    }
-    if (scenario.contextSources !== undefined && !Array.isArray(scenario.contextSources)) {
-        pushIssue(issues, suitePath, "contextSources", "contextSources must be an array of strings", scenario.name);
-    }
-    if (scenario.skills !== undefined && !isValidSkillSetting(scenario.skills)) {
-        pushIssue(issues, suitePath, "skills", `skills must be "none" or repo-relative SKILL.md / skill-folder paths, got ${JSON.stringify(scenario.skills)}`, scenario.name);
-    }
-    if (scenario.workspace !== undefined) {
-        const parsed = parseScenarioWorkspace(scenario.workspace);
-        if (!parsed.ok) {
-            pushIssue(issues, suitePath, "workspace", parsed.message, scenario.name);
-        }
-    }
-    if (scenario.allowUserSkills !== undefined && typeof scenario.allowUserSkills !== "boolean") {
-        pushIssue(issues, suitePath, "allowUserSkills", `allowUserSkills must be a boolean, got ${JSON.stringify(scenario.allowUserSkills)}`, scenario.name);
-    }
-    if (scenario.networkAccess !== undefined && typeof scenario.networkAccess !== "boolean") {
-        pushIssue(issues, suitePath, "networkAccess", `networkAccess must be a boolean, got ${JSON.stringify(scenario.networkAccess)}`, scenario.name);
-    }
-    if (scenario.seedStageOnly && !scenario.seedPatch) {
-        pushIssue(issues, suitePath, "seedStageOnly", "seedStageOnly requires seedPatch", scenario.name);
-    }
-    validateCompare(issues, suitePath, scenario);
-    validateRubric(issues, suitePath, scenario.name, scenario.rubric);
-}
-function validateCompareArmFields(issues, suitePath, scenarioName, fieldPrefix, arm) {
-    if (arm.label !== undefined && (typeof arm.label !== "string" || !arm.label.trim())) {
-        pushIssue(issues, suitePath, fieldPrefix, "label must be a non-empty string", scenarioName);
-    }
-    if (typeof arm.description !== "string" || !arm.description.trim()) {
-        pushIssue(issues, suitePath, `${fieldPrefix}.description`, "description is required", scenarioName);
-    }
-    if (arm.prompt !== undefined && typeof arm.prompt !== "string") {
-        pushIssue(issues, suitePath, `${fieldPrefix}.prompt`, "prompt must be a string", scenarioName);
-    }
-    if (arm.host !== undefined && !isKnownHostId(arm.host)) {
-        pushIssue(issues, suitePath, `${fieldPrefix}.host`, unknownHostMessage(arm.host), scenarioName);
-    }
-    if (arm.contextMode !== undefined && !VALID_CONTEXT_MODES.has(arm.contextMode)) {
-        pushIssue(issues, suitePath, `${fieldPrefix}.contextMode`, `contextMode must be host-native|harness-preamble, got ${JSON.stringify(arm.contextMode)}`, scenarioName);
-    }
-    if (arm.workspace !== undefined) {
-        const parsed = parseScenarioWorkspace(arm.workspace);
-        if (!parsed.ok) {
-            pushIssue(issues, suitePath, `${fieldPrefix}.workspace`, parsed.message, scenarioName);
-        }
-    }
-    if (arm.allowUserSkills !== undefined && typeof arm.allowUserSkills !== "boolean") {
-        pushIssue(issues, suitePath, `${fieldPrefix}.allowUserSkills`, `allowUserSkills must be a boolean, got ${JSON.stringify(arm.allowUserSkills)}`, scenarioName);
-    }
-    if (arm.seedStageOnly && !arm.seedPatch) {
-        pushIssue(issues, suitePath, `${fieldPrefix}.seedStageOnly`, "seedStageOnly requires seedPatch", scenarioName);
-    }
-    if (arm.skills !== undefined && !isValidSkillSetting(arm.skills)) {
-        pushIssue(issues, suitePath, `${fieldPrefix}.skills`, `skills must be "none" or repo-relative SKILL.md / skill-folder paths, got ${JSON.stringify(arm.skills)}`, scenarioName);
-    }
-    if (arm.rubric !== undefined) {
-        if (typeof arm.rubric !== "object" || arm.rubric === null) {
-            pushIssue(issues, suitePath, `${fieldPrefix}.rubric`, "rubric must be an object", scenarioName);
-        }
-        else if (arm.rubric.judge !== undefined) {
-            pushIssue(issues, suitePath, `${fieldPrefix}.rubric.judge`, "arm rubric cannot include judge. Put judge on the scenario", scenarioName);
-        }
-        else {
-            validateRubric(issues, suitePath, scenarioName, arm.rubric, `${fieldPrefix}.rubric`);
-        }
-    }
-}
-function validateCompare(issues, suitePath, scenario) {
-    if (scenario.compare === undefined) {
-        return;
-    }
-    if (typeof scenario.compare !== "object" || scenario.compare === null) {
-        pushIssue(issues, suitePath, "compare", "compare must be an object with a and b, or arms", scenario.name);
-        return;
-    }
-    const compare = scenario.compare;
-    const hasLegacyArms = compare.a !== undefined || compare.b !== undefined;
-    const hasNamedArms = compare.arms !== undefined;
-    if (hasLegacyArms && hasNamedArms) {
-        pushIssue(issues, suitePath, "compare", "compare must use a and b, or arms, not both", scenario.name);
-        return;
-    }
-    if (hasNamedArms) {
-        validateNamedCompareArms(issues, suitePath, scenario.name, compare);
-    }
-    else {
-        if (typeof compare.a !== "object" || compare.a === null) {
-            pushIssue(issues, suitePath, "compare.a", "compare.a must be an object", scenario.name);
-        }
-        else {
-            validateCompareArmFields(issues, suitePath, scenario.name, "compare.a", compare.a);
-        }
-        if (typeof compare.b !== "object" || compare.b === null) {
-            pushIssue(issues, suitePath, "compare.b", "compare.b must be an object", scenario.name);
-        }
-        else {
-            validateCompareArmFields(issues, suitePath, scenario.name, "compare.b", compare.b);
-        }
-    }
-    const armIds = resolveCompareArms(compare)
-        .map((entry) => entry.id)
-        .filter(Boolean);
-    const raw = compare;
-    for (const removed of ["faster", "cheaper"]) {
-        if (removed in raw)
-            pushIssue(issues, suitePath, `compare.${removed}`, `compare.${removed} was removed. Use compare.gates`, scenario.name);
-    }
-    validateCompareGates(issues, suitePath, scenario.name, compare.gates, armIds);
-    if (compare.judgeMetrics !== undefined) {
-        if (!Array.isArray(compare.judgeMetrics) || compare.judgeMetrics.length === 0) {
-            pushIssue(issues, suitePath, "compare.judgeMetrics", "judgeMetrics must be a non-empty array", scenario.name);
-        }
-        else {
-            const ids = new Set();
-            for (const [index, metric] of compare.judgeMetrics.entries()) {
-                if (!metric ||
-                    typeof metric.id !== "string" ||
-                    !metric.id.trim() ||
-                    typeof metric.question !== "string" ||
-                    !metric.question.trim())
-                    pushIssue(issues, suitePath, `compare.judgeMetrics[${index}]`, "judge metric needs an id and a question", scenario.name);
-                else if (ids.has(metric.id))
-                    pushIssue(issues, suitePath, `compare.judgeMetrics[${index}].id`, "judge metric id must be unique", scenario.name);
-                else
-                    ids.add(metric.id);
-            }
-        }
-    }
-    const judgeMetricIds = new Set(compare.judgeMetrics?.map((metric) => metric.id) ?? []);
-    for (const [index, gate] of (compare.gates ?? []).entries()) {
-        if (typeof gate.metric === "string" &&
-            gate.metric.startsWith("judge:") &&
-            !judgeMetricIds.has(gate.metric.slice("judge:".length))) {
-            pushIssue(issues, suitePath, `compare.gates[${index}].metric`, "judge gate must name a compare.judgeMetrics id", scenario.name);
-        }
-    }
-}
-function validateNamedCompareArms(issues, suitePath, scenarioName, compare) {
-    if (!Array.isArray(compare.arms) || compare.arms.length < 2) {
-        pushIssue(issues, suitePath, "compare.arms", "compare.arms must be an array of two or more arms", scenarioName);
-        return;
-    }
-    const seen = new Set();
-    for (const [index, arm] of compare.arms.entries()) {
-        const fieldPrefix = `compare.arms[${index}]`;
-        if (typeof arm !== "object" || arm === null) {
-            pushIssue(issues, suitePath, fieldPrefix, "arm must be an object", scenarioName);
-            continue;
-        }
-        const id = parseCompareArmId(arm.id);
-        if (!id) {
-            pushIssue(issues, suitePath, `${fieldPrefix}.id`, "id must be a lowercase slug such as skel-clean", scenarioName);
-        }
-        else if (seen.has(id)) {
-            pushIssue(issues, suitePath, `${fieldPrefix}.id`, `duplicate arm id ${id}`, scenarioName);
-        }
-        else {
-            seen.add(id);
-        }
-        validateCompareArmFields(issues, suitePath, scenarioName, fieldPrefix, arm);
-    }
-}
-function validateCompareGates(issues, suitePath, scenarioName, value, armIds) {
-    if (value === undefined)
-        return;
-    if (!Array.isArray(value) || value.length === 0) {
-        pushIssue(issues, suitePath, "compare.gates", "gates must be a non-empty array", scenarioName);
-        return;
-    }
-    const metrics = new Set(["outcome", "turns", "tokens", "tools", "durationMs"]);
-    const operators = new Set(["equal", "lessThan", "atMost", "atLeast", "greaterThan"]);
-    for (const [index, item] of value.entries()) {
-        const field = `compare.gates[${index}]`;
-        if (!item || typeof item !== "object") {
-            pushIssue(issues, suitePath, field, "gate must be an object", scenarioName);
-            continue;
-        }
-        const gate = item;
-        if (typeof gate.metric !== "string" ||
-            (!metrics.has(gate.metric) && !gate.metric.startsWith("judge:")))
-            pushIssue(issues, suitePath, `${field}.metric`, "metric must name a supported comparison value", scenarioName);
-        if ("winner" in gate) {
-            if (!armIds.includes(gate.winner))
-                pushIssue(issues, suitePath, `${field}.winner`, `must be one of ${armIds.join(", ")}`, scenarioName);
-            if (!armIds.includes(gate.loser))
-                pushIssue(issues, suitePath, `${field}.loser`, `must be one of ${armIds.join(", ")}`, scenarioName);
-            if (gate.winner === gate.loser)
-                pushIssue(issues, suitePath, field, "winner and loser must be different arms", scenarioName);
-        }
-        else if ("arm" in gate) {
-            if (!armIds.includes(gate.arm))
-                pushIssue(issues, suitePath, `${field}.arm`, `must be one of ${armIds.join(", ")}`, scenarioName);
-            if (!operators.has(gate.operator))
-                pushIssue(issues, suitePath, `${field}.operator`, "operator is not supported", scenarioName);
-            if (typeof gate.value !== "number" && gate.value !== "pass" && gate.value !== "fail")
-                pushIssue(issues, suitePath, `${field}.value`, "value must be a number, pass, or fail", scenarioName);
-        }
-        else
-            pushIssue(issues, suitePath, field, "gate needs winner and loser, or arm, operator, and value", scenarioName);
-    }
-}
-function validateHosts(issues, suitePath, suite) {
-    if (suite.hosts === undefined) {
-        return;
-    }
-    if (!Array.isArray(suite.hosts) || suite.hosts.length === 0) {
-        pushIssue(issues, suitePath, "hosts", "hosts must be a non-empty array of host slugs");
-        return;
-    }
-    const seen = new Set();
-    for (const host of suite.hosts) {
-        if (!isKnownHostId(host)) {
-            pushIssue(issues, suitePath, "hosts", unknownHostMessage(host));
-            continue;
-        }
-        if (seen.has(host)) {
-            pushIssue(issues, suitePath, "hosts", `duplicate host ${host}`);
-        }
-        seen.add(host);
-    }
-    if (suite.defaults?.host !== undefined &&
-        isKnownHostId(suite.defaults.host) &&
-        !suite.hosts.includes(suite.defaults.host)) {
-        pushIssue(issues, suitePath, "defaults.host", `defaults.host ${suite.defaults.host} must be listed in hosts`);
-    }
-}
-function validateDefaults(issues, suitePath, suite) {
-    const defaults = suite.defaults;
-    if (!defaults) {
-        return;
-    }
-    if (defaults.host !== undefined && !isKnownHostId(defaults.host)) {
-        pushIssue(issues, suitePath, "defaults.host", unknownHostMessage(defaults.host));
-    }
-    if (defaults.contextMode !== undefined && !VALID_CONTEXT_MODES.has(defaults.contextMode)) {
-        pushIssue(issues, suitePath, "defaults.contextMode", `contextMode must be host-native|harness-preamble, got ${JSON.stringify(defaults.contextMode)}`);
-    }
-    if (defaults.profile !== undefined && !VALID_PROFILES.has(defaults.profile)) {
-        pushIssue(issues, suitePath, "defaults.profile", `profile must be shared|cursor|claude|skeleton, got ${JSON.stringify(defaults.profile)}`);
-    }
-    if (defaults.contextSources !== undefined && !Array.isArray(defaults.contextSources)) {
-        pushIssue(issues, suitePath, "defaults.contextSources", "contextSources must be an array of strings");
-    }
-    if (defaults.workspace !== undefined) {
-        const parsed = parseScenarioWorkspace(defaults.workspace);
-        if (!parsed.ok) {
-            pushIssue(issues, suitePath, "defaults.workspace", parsed.message);
-        }
-    }
-    if (defaults.skills !== undefined && !isValidSkillSetting(defaults.skills)) {
-        pushIssue(issues, suitePath, "defaults.skills", `skills must be "none" or repo-relative SKILL.md / skill-folder paths, got ${JSON.stringify(defaults.skills)}`);
-    }
-    if (defaults.allowUserSkills !== undefined && typeof defaults.allowUserSkills !== "boolean") {
-        pushIssue(issues, suitePath, "defaults.allowUserSkills", `allowUserSkills must be a boolean, got ${JSON.stringify(defaults.allowUserSkills)}`);
-    }
-}
-function mcpScriptPaths(suite, scenario) {
-    const servers = { ...suite.defaults?.mcpServers, ...scenario.mcpServers };
-    const paths = [];
-    for (const config of Object.values(servers)) {
-        if (!("command" in config) || !config.args) {
-            continue;
-        }
-        for (const arg of config.args) {
-            if (arg.startsWith("-")) {
-                continue;
-            }
-            if (/\.(mjs|cjs|js)$/.test(arg) || arg.includes("/")) {
-                paths.push(arg);
-            }
-        }
-    }
-    return paths;
-}
-/** Semantic validation beyond structural loadSuiteFile checks. */
-export function validateSuiteFile(suitePath, suite) {
-    const issues = [];
-    validateHosts(issues, suitePath, suite);
-    validateDefaults(issues, suitePath, suite);
-    for (const scenario of suite.scenarios) {
-        validateScenario(issues, suitePath, scenario);
-        const effectiveScenarios = scenario.compare
-            ? resolveCompareArms(scenario.compare).map((entry) => applyCompareArm(scenario, entry.id))
-            : [scenario];
-        for (const effective of effectiveScenarios) {
-            const mode = effective.contextMode ?? suite.defaults?.contextMode ?? "harness-preamble";
-            if (mode !== "host-native")
-                continue;
-            const contextSources = [
-                ...(suite.defaults?.contextSources ?? []),
-                ...(effective.contextSources ?? []),
-            ].filter((source) => typeof source === "string" && source.trim().length > 0);
-            if (contextSources.length > 0) {
-                pushIssue(issues, suitePath, "contextMode", "host-native contextMode cannot use contextSources because they are injected into the prompt", scenario.name);
-            }
-            const skills = effective.skills ?? suite.defaults?.skills;
-            if (injectsSkillCatalog(skills)) {
-                pushIssue(issues, suitePath, "contextMode", "host-native contextMode cannot use skills because agent-test injects a synthetic catalog", scenario.name);
-            }
-        }
-    }
-    return issues;
-}
-export async function validateSuitePaths(suitePaths, options) {
-    const issues = [];
-    let scenariosChecked = 0;
-    const repoRoot = options?.repoRoot;
-    for (const suitePath of suitePaths) {
-        let suite;
-        try {
-            suite = await loadSuiteFile(suitePath, { rubricsDir: options?.rubricsDir });
-        }
-        catch (error) {
-            pushIssue(issues, suitePath, "file", error instanceof Error ? error.message : "failed to load suite file");
-            continue;
-        }
-        issues.push(...validateSuiteFile(suitePath, suite));
-        scenariosChecked += suite.scenarios.length;
-        if (options?.validatePaths && repoRoot) {
-            for (const scenario of suite.scenarios) {
-                const workspaceChecks = scenario.compare
-                    ? resolveCompareArms(scenario.compare).map((entry) => ({
-                        field: `compare.${entry.id}.workspace`,
-                        rel: resolveScenarioWorkspaceRel(suite, applyCompareArm(scenario, entry.id)),
-                    }))
-                    : [{ field: "workspace", rel: resolveScenarioWorkspaceRel(suite, scenario) }];
-                for (const check of workspaceChecks) {
-                    const workspaceRel = check.rel;
-                    if (!workspaceRel) {
-                        continue;
-                    }
-                    const workspacePath = resolve(repoRoot, workspaceRel);
-                    try {
-                        const info = await stat(workspacePath);
-                        if (!info.isDirectory()) {
-                            pushIssue(issues, suitePath, check.field, `workspace must be a directory: ${workspaceRel}`, scenario.name);
-                        }
-                    }
-                    catch {
-                        pushIssue(issues, suitePath, check.field, `workspace not found: ${workspaceRel}`, scenario.name);
-                    }
-                }
-                const skillTargets = scenario.compare
-                    ? resolveCompareArms(scenario.compare).map((entry) => ({
-                        field: `compare.${entry.id}.skills`,
-                        scenario: applyCompareArm(scenario, entry.id),
-                    }))
-                    : [{ field: "skills", scenario }];
-                for (const target of skillTargets) {
-                    const skills = target.scenario.skills ?? suite.defaults?.skills;
-                    if (skills === undefined || !isValidSkillSetting(skills)) {
-                        continue;
-                    }
-                    const skillWorkspaceRel = resolveScenarioWorkspaceRel(suite, target.scenario);
-                    const skillRoot = skillWorkspaceRel ? resolve(repoRoot, skillWorkspaceRel) : repoRoot;
-                    for (const rel of skillPathsFromSetting(skills)) {
-                        const manifest = skillManifestRelPath(rel);
-                        try {
-                            await access(resolve(skillRoot, manifest));
-                        }
-                        catch {
-                            pushIssue(issues, suitePath, target.field, `skill path not found: ${manifest}`, scenario.name);
-                        }
-                    }
-                }
-                for (const scriptPath of mcpScriptPaths(suite, scenario)) {
-                    try {
-                        await access(resolve(repoRoot, scriptPath));
-                    }
-                    catch {
-                        pushIssue(issues, suitePath, "mcpServers", `MCP script not found: ${scriptPath}`, scenario.name);
-                    }
-                }
-                if (scenario.seedPatch) {
-                    const patchPath = resolve(repoRoot, scenario.seedPatch);
-                    try {
-                        await access(patchPath);
-                    }
-                    catch {
-                        pushIssue(issues, suitePath, "seedPatch", `seed patch not found: ${scenario.seedPatch}`, scenario.name);
-                    }
-                }
-            }
-        }
-    }
-    return {
-        ok: issues.length === 0,
-        issues,
-        suitesChecked: suitePaths.length,
-        scenariosChecked,
-    };
-}
-export function formatValidationReport(report) {
-    if (report.ok) {
-        return `Validated ${report.suitesChecked} suite(s), ${report.scenariosChecked} scenario(s): OK`;
-    }
-    const lines = report.issues.map((issue) => {
-        const where = issue.scenario ? `${issue.suitePath} · ${issue.scenario}` : issue.suitePath;
-        return `${where} · ${issue.field}: ${issue.message}`;
-    });
-    return [
-        `Validation failed (${report.issues.length} issue(s)):`,
-        ...lines.map((l) => `  - ${l}`),
-    ].join("\n");
-}
-/** Scenario workspace wins over suite defaults. Caller HEAD when omit or `"."`. */
-export function resolveScenarioWorkspaceRel(suite, scenario) {
-    const raw = scenario.workspace !== undefined ? scenario.workspace : suite.defaults?.workspace;
-    const parsed = parseScenarioWorkspace(raw);
-    return parsed.ok ? parsed.rel : undefined;
-}
-//# sourceMappingURL=validate-suite.js.map