@post-print/agent-test 0.1.18 → 0.2.1

package/dist/validate-suite.js

@@ -0,0 +1,149 @@
+import { access } from "node:fs/promises";
+import { resolve } from "node:path";
+import { loadSuiteFile } from "./load-suite.js";
+const VALID_HOSTS = new Set(["cursor", "claude", "replay"]);
+const VALID_PROFILES = new Set(["shared", "cursor", "claude"]);
+const VALID_SKILLS = new Set(["none", "catalog", "full"]);
+const VALID_TIERS = new Set(["low", "medium", "high"]);
+const VALID_REVIEW_DEPTHS = new Set([
+    "quick",
+    "standard",
+    "thorough",
+    "full",
+]);
+function pushIssue(issues, suitePath, field, message, scenario) {
+    issues.push({ suitePath, scenario, field, message });
+}
+function validateRubric(issues, suitePath, scenarioName, rubric) {
+    if (rubric.tier !== undefined && !VALID_TIERS.has(rubric.tier)) {
+        pushIssue(issues, suitePath, "rubric.tier", `tier must be low|medium|high, got ${JSON.stringify(rubric.tier)}`, scenarioName);
+    }
+    if (rubric.reviewDepth !== undefined && !VALID_REVIEW_DEPTHS.has(rubric.reviewDepth)) {
+        pushIssue(issues, suitePath, "rubric.reviewDepth", `reviewDepth must be quick|standard|thorough|full, got ${JSON.stringify(rubric.reviewDepth)}`, scenarioName);
+    }
+    for (const key of ["must", "mustNot", "mustRun", "mustCallTool", "mustNotCallTool"]) {
+        const value = rubric[key];
+        if (value !== undefined && !Array.isArray(value)) {
+            pushIssue(issues, suitePath, `rubric.${key}`, `${key} must be an array of strings`, scenarioName);
+        }
+    }
+    for (const key of ["mustInvokeSkill", "mustNotInvokeSkill"]) {
+        const value = rubric[key];
+        if (value !== undefined && !Array.isArray(value)) {
+            pushIssue(issues, suitePath, `rubric.${key}`, `${key} must be an array of skill folder names`, scenarioName);
+        }
+    }
+    if (rubric.judge !== undefined) {
+        if (!Array.isArray(rubric.judge)) {
+            pushIssue(issues, suitePath, "rubric.judge", "judge must be an array of strings or { question } objects", scenarioName);
+        }
+        else {
+            for (const [index, item] of rubric.judge.entries()) {
+                if (typeof item === "string") {
+                    continue;
+                }
+                if (typeof item !== "object" || item === null || typeof item.question !== "string") {
+                    pushIssue(issues, suitePath, `rubric.judge[${index}]`, "judge item must be a string or { question: string }", scenarioName);
+                }
+            }
+        }
+    }
+}
+function validateScenario(issues, suitePath, scenario) {
+    if (scenario.host !== undefined && !VALID_HOSTS.has(scenario.host)) {
+        pushIssue(issues, suitePath, "host", `host must be cursor|claude|replay, got ${JSON.stringify(scenario.host)}`, scenario.name);
+    }
+    if (scenario.profile !== undefined && !VALID_PROFILES.has(scenario.profile)) {
+        pushIssue(issues, suitePath, "profile", `profile must be shared|cursor|claude, got ${JSON.stringify(scenario.profile)}`, scenario.name);
+    }
+    if (scenario.skills !== undefined && !VALID_SKILLS.has(scenario.skills)) {
+        pushIssue(issues, suitePath, "skills", `skills must be none|catalog|full, got ${JSON.stringify(scenario.skills)}`, scenario.name);
+    }
+    if (scenario.seedStageOnly && !scenario.seedPatch) {
+        pushIssue(issues, suitePath, "seedStageOnly", "seedStageOnly requires seedPatch", scenario.name);
+    }
+    validateRubric(issues, suitePath, scenario.name, scenario.rubric);
+}
+function validateDefaults(issues, suitePath, suite) {
+    const defaults = suite.defaults;
+    if (!defaults) {
+        return;
+    }
+    if (defaults.host !== undefined && !VALID_HOSTS.has(defaults.host)) {
+        pushIssue(issues, suitePath, "defaults.host", `host must be cursor|claude|replay, got ${JSON.stringify(defaults.host)}`);
+    }
+    if (defaults.profile !== undefined && !VALID_PROFILES.has(defaults.profile)) {
+        pushIssue(issues, suitePath, "defaults.profile", `profile must be shared|cursor|claude, got ${JSON.stringify(defaults.profile)}`);
+    }
+    if (defaults.skills !== undefined && !VALID_SKILLS.has(defaults.skills)) {
+        pushIssue(issues, suitePath, "defaults.skills", `skills must be none|catalog|full, got ${JSON.stringify(defaults.skills)}`);
+    }
+}
+/** Semantic validation beyond structural loadSuiteFile checks. */
+export function validateSuiteFile(suitePath, suite) {
+    const issues = [];
+    validateDefaults(issues, suitePath, suite);
+    for (const scenario of suite.scenarios) {
+        validateScenario(issues, suitePath, scenario);
+    }
+    return issues;
+}
+export async function validateSuitePaths(suitePaths, options) {
+    const issues = [];
+    let scenariosChecked = 0;
+    const repoRoot = options?.repoRoot;
+    for (const suitePath of suitePaths) {
+        let suite;
+        try {
+            suite = await loadSuiteFile(suitePath);
+        }
+        catch (error) {
+            pushIssue(issues, suitePath, "file", error instanceof Error ? error.message : "failed to load suite file");
+            continue;
+        }
+        issues.push(...validateSuiteFile(suitePath, suite));
+        scenariosChecked += suite.scenarios.length;
+        if (options?.validatePaths && repoRoot) {
+            for (const scenario of suite.scenarios) {
+                if (scenario.replayTrace) {
+                    const replayPath = resolve(repoRoot, scenario.replayTrace);
+                    try {
+                        await access(replayPath);
+                    }
+                    catch {
+                        pushIssue(issues, suitePath, "replayTrace", `replay trace not found: ${scenario.replayTrace}`, scenario.name);
+                    }
+                }
+                if (scenario.seedPatch) {
+                    const patchPath = resolve(repoRoot, scenario.seedPatch);
+                    try {
+                        await access(patchPath);
+                    }
+                    catch {
+                        pushIssue(issues, suitePath, "seedPatch", `seed patch not found: ${scenario.seedPatch}`, scenario.name);
+                    }
+                }
+            }
+        }
+    }
+    return {
+        ok: issues.length === 0,
+        issues,
+        suitesChecked: suitePaths.length,
+        scenariosChecked,
+    };
+}
+export function formatValidationReport(report) {
+    if (report.ok) {
+        return `Validated ${report.suitesChecked} suite(s), ${report.scenariosChecked} scenario(s): OK`;
+    }
+    const lines = report.issues.map((issue) => {
+        const where = issue.scenario ? `${issue.suitePath} · ${issue.scenario}` : issue.suitePath;
+        return `${where} · ${issue.field}: ${issue.message}`;
+    });
+    return [
+        `Validation failed (${report.issues.length} issue(s)):`,
+        ...lines.map((l) => `  - ${l}`),
+    ].join("\n");
+}
+//# sourceMappingURL=validate-suite.js.map

package/dist/validate-suite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate-suite.js","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACvE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAiB,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAC/E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAsB,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;AAC/E,MAAM,WAAW,GAAG,IAAI,GAAG,CAAsC,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5F,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAA6C;IAC/E,OAAO;IACP,UAAU;IACV,UAAU;IACV,MAAM;CACN,CAAC,CAAC;AAgBH,SAAS,SAAS,CACjB,MAA8B,EAC9B,SAAiB,EACjB,KAAa,EACb,OAAe,EACf,QAAiB;IAEjB,MAAM,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,cAAc,CACtB,MAA8B,EAC9B,SAAiB,EACjB,YAAoB,EACpB,MAAsB;IAEtB,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAChE,SAAS,CACR,MAAM,EACN,SAAS,EACT,aAAa,EACb,qCAAqC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAClE,YAAY,CACZ,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QACtF,SAAS,CACR,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,yDAAyD,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAC7F,YAAY,CACZ,CAAC;IACH,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,cAAc,EAAE,iBAAiB,CAAU,EAAE,CAAC;QAC9F,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,SAAS,CACR,MAAM,EACN,SAAS,EACT,UAAU,GAAG,EAAE,EACf,GAAG,GAAG,8BAA8B,EACpC,YAAY,CACZ,CAAC;QACH,CAAC;IACF,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,CAAC,iBAAiB,EAAE,oBAAoB,CAAU,EAAE,CAAC;QACtE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,SAAS,CACR,MAAM,EACN,SAAS,EACT,UAAU,GAAG,EAAE,EACf,GAAG,GAAG,yCAAyC,EAC/C,YAAY,CACZ,CAAC;QACH,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAChC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,SAAS,CACR,MAAM,EACN,SAAS,EACT,cAAc,EACd,2DAA2D,EAC3D,YAAY,CACZ,CAAC;QACH,CAAC;aAAM,CAAC;YACP,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;gBACpD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC9B,SAAS;gBACV,CAAC;gBACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;oBACpF,SAAS,CACR,MAAM,EACN,SAAS,EACT,gBAAgB,KAAK,GAAG,EACxB,qDAAqD,EACrD,YAAY,CACZ,CAAC;gBACH,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED,SAAS,gBAAgB,CACxB,MAA8B,EAC9B,SAAiB,EACjB,QAAuB;IAEvB,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACpE,SAAS,CACR,MAAM,EACN,SAAS,EACT,MAAM,EACN,0CAA0C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EACzE,QAAQ,CAAC,IAAI,CACb,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7E,SAAS,CACR,MAAM,EACN,SAAS,EACT,SAAS,EACT,6CAA6C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,EAC/E,QAAQ,CAAC,IAAI,CACb,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACzE,SAAS,CACR,MAAM,EACN,SAAS,EACT,QAAQ,EACR,yCAAyC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAC1E,QAAQ,CAAC,IAAI,CACb,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,aAAa,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QACnD,SAAS,CACR,MAAM,EACN,SAAS,EACT,eAAe,EACf,kCAAkC,EAClC,QAAQ,CAAC,IAAI,CACb,CAAC;IACH,CAAC;IACD,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,gBAAgB,CACxB,MAA8B,EAC9B,SAAiB,EACjB,KAAqB;IAErB,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAChC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,OAAO;IACR,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACpE,SAAS,CACR,MAAM,EACN,SAAS,EACT,eAAe,EACf,0CAA0C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CACzE,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7E,SAAS,CACR,MAAM,EACN,SAAS,EACT,kBAAkB,EAClB,6CAA6C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAC/E,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACzE,SAAS,CACR,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,yCAAyC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAC1E,CAAC;IACH,CAAC;AACF,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,iBAAiB,CAChC,SAAiB,EACjB,KAAqB;IAErB,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACxC,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,UAAoB,EACpB,OAAwD;IAExD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IAEnC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,IAAI,KAAqB,CAAC;QAC1B,IAAI,CAAC;YACJ,KAAK,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,SAAS,CACR,MAAM,EACN,SAAS,EACT,MAAM,EACN,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,CACpE,CAAC;YACF,SAAS;QACV,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;QACpD,gBAAgB,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC;QAE3C,IAAI,OAAO,EAAE,aAAa,IAAI,QAAQ,EAAE,CAAC;YACxC,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACxC,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;oBAC1B,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;oBAC3D,IAAI,CAAC;wBACJ,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;oBAC1B,CAAC;oBAAC,MAAM,CAAC;wBACR,SAAS,CACR,MAAM,EACN,SAAS,EACT,aAAa,EACb,2BAA2B,QAAQ,CAAC,WAAW,EAAE,EACjD,QAAQ,CAAC,IAAI,CACb,CAAC;oBACH,CAAC;gBACF,CAAC;gBACD,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;oBACxB,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;oBACxD,IAAI,CAAC;wBACJ,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;oBACzB,CAAC;oBAAC,MAAM,CAAC;wBACR,SAAS,CACR,MAAM,EACN,SAAS,EACT,WAAW,EACX,yBAAyB,QAAQ,CAAC,SAAS,EAAE,EAC7C,QAAQ,CAAC,IAAI,CACb,CAAC;oBACH,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO;QACN,EAAE,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;QACvB,MAAM;QACN,aAAa,EAAE,UAAU,CAAC,MAAM;QAChC,gBAAgB;KAChB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAA6B;IACnE,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,aAAa,MAAM,CAAC,aAAa,cAAc,MAAM,CAAC,gBAAgB,kBAAkB,CAAC;IACjG,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACzC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,SAAS,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;QAC1F,OAAO,GAAG,KAAK,MAAM,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;IACtD,CAAC,CAAC,CAAC;IACH,OAAO;QACN,sBAAsB,MAAM,CAAC,MAAM,CAAC,MAAM,aAAa;QACvD,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;KAC/B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@post-print/agent-test",
-	"version": "0.1.18",
+	"version": "0.2.1",
 	"type": "module",
 	"bin": {
 		"agent-test": "./dist/cli.js"
@@ -25,7 +25,7 @@
 		"clean": "rm -rf dist tsconfig.tsbuildinfo"
 	},
 	"dependencies": {
-		"@post-print/agent-harness": "0.1.18",
+		"@post-print/agent-harness": "0.2.1",
 		"chalk": "5"
 	},
 	"devDependencies": {