@piagent/platform 1.2.16 → 1.3.0-ie.8

package/packages/piagent-core/benchmark/benchmark-runtime.js

@@ -0,0 +1,188 @@
+import fs from "node:fs";
+import crypto from "node:crypto";
+import os from "node:os";
+import path from "node:path";
+
+export const controlledCodexFeatures = [
+  "apps",
+  "plugins",
+  "browser_use",
+  "browser_use_external",
+  "browser_use_full_cdp_access",
+  "computer_use",
+  "image_generation",
+  "in_app_browser",
+  "multi_agent",
+  "multi_agent_v2",
+  "goals",
+  "hooks",
+  "skill_search",
+  "tool_suggest",
+  "workspace_dependencies"
+];
+
+export const PIAGENT_BENCHMARK_TREATMENTS = Object.freeze({
+  "release-defaults": Object.freeze({}),
+  "local-safe": Object.freeze({
+    PIAGENT_SOLVER_MODE: "shadow",
+    PIAGENT_PHASE_TOOLS: "shadow",
+    PIAGENT_AUTO_RECOVERY: "on",
+    PIAGENT_HELPERS_MODE: "recommend",
+    PIAGENT_EXECUTION_BACKEND: "host"
+  }),
+  "mechanical-core": Object.freeze({
+    PIAGENT_SOLVER_MODE: "shadow", PIAGENT_PHASE_TOOLS: "shadow", PIAGENT_AUTO_RECOVERY: "on",
+    PIAGENT_HELPERS_MODE: "recommend", PIAGENT_EXECUTION_BACKEND: "host", PIAGENT_INTELLIGENCE_ENGINE: "off"
+  }),
+  "intelligence-engine": Object.freeze({
+    PIAGENT_SOLVER_MODE: "shadow", PIAGENT_PHASE_TOOLS: "shadow", PIAGENT_AUTO_RECOVERY: "on",
+    PIAGENT_HELPERS_MODE: "recommend", PIAGENT_EXECUTION_BACKEND: "host", PIAGENT_INTELLIGENCE_ENGINE: "on"
+  }),
+  "causal-phase-enforce": Object.freeze({
+    PIAGENT_SOLVER_MODE: "shadow",
+    PIAGENT_PHASE_TOOLS: "on",
+    PIAGENT_AUTO_RECOVERY: "on",
+    PIAGENT_HELPERS_MODE: "recommend",
+    PIAGENT_EXECUTION_BACKEND: "host"
+  }),
+  candidate: Object.freeze({
+    PIAGENT_SOLVER_MODE: "recommend",
+    PIAGENT_PHASE_TOOLS: "on",
+    PIAGENT_AUTO_RECOVERY: "on",
+    PIAGENT_HELPERS_MODE: "recommend",
+    PIAGENT_EXECUTION_BACKEND: "host"
+  }),
+  "feature-off": Object.freeze({
+    PIAGENT_SOLVER_MODE: "off",
+    PIAGENT_PHASE_TOOLS: "off",
+    PIAGENT_AUTO_RECOVERY: "off",
+    PIAGENT_HELPERS_MODE: "off",
+    PIAGENT_EXECUTION_BACKEND: "host",
+    PIAGENT_INTELLIGENCE_ENGINE: "off"
+  })
+});
+
+const strippedEnvironmentKeys = Object.freeze([
+  "BASH_ENV", "CDPATH", "ENV", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_COMMON_DIR", "GIT_CONFIG",
+  "GIT_CONFIG_COUNT", "GIT_CONFIG_GLOBAL", "GIT_CONFIG_NOSYSTEM", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_SYSTEM",
+  "GIT_DIR", "GIT_EXEC_PATH", "GIT_INDEX_FILE", "GIT_NAMESPACE", "GIT_OBJECT_DIRECTORY", "GIT_SHALLOW_FILE",
+  "GIT_TEMPLATE_DIR", "GIT_WORK_TREE", "GIT_ATTR_NOSYSTEM", "NODE_OPTIONS", "NODE_PATH", "NODE_REPL_EXTERNAL_MODULE",
+  "NPM_CONFIG_NODE_OPTIONS", "PROMPT_COMMAND"
+]);
+
+export function benchmarkEnvironmentPolicy() {
+  const policy = {
+    schemaVersion: 1,
+    inherited: "operator-environment-minus-piagent-node-loader-git-and-shell-overrides",
+    stripped: strippedEnvironmentKeys,
+    forced: { GIT_CONFIG_GLOBAL: os.devNull, GIT_CONFIG_NOSYSTEM: "1", PI_OFFLINE: "1", nodeExecutableDirectoryFirst: true }
+  };
+  return { ...policy, digest: crypto.createHash("sha256").update(JSON.stringify(policy)).digest("hex") };
+}
+
+export function benchmarkHostEnvironment(base = process.env) {
+  const env = { ...base };
+  for (const key of strippedEnvironmentKeys) delete env[key];
+  env.GIT_CONFIG_GLOBAL = os.devNull;
+  env.GIT_CONFIG_NOSYSTEM = "1";
+  env.PI_OFFLINE = "1";
+  env.PATH = [path.dirname(process.execPath), ...String(env.PATH ?? "").split(path.delimiter).filter((value) => value && path.resolve(value) !== path.dirname(process.execPath))].join(path.delimiter);
+  return env;
+}
+
+export function assertBenchmarkLaunchEnvironmentSafe(base = process.env) {
+  const active = ["NODE_OPTIONS", "NODE_PATH", "NODE_REPL_EXTERNAL_MODULE", "NPM_CONFIG_NODE_OPTIONS"]
+    .filter((key) => typeof base[key] === "string" && base[key].trim());
+  if (active.length > 0) fail(`Benchmark launch refuses Node code-loading environment overrides: ${active.join(", ")}`);
+}
+
+export function benchmarkGitEnvironment(base = process.env) {
+  const host = benchmarkHostEnvironment(base);
+  const env = {};
+  for (const key of ["HOME", "PATH", "TMPDIR", "TMP", "TEMP", "LANG", "LC_ALL", "TZ", "SystemRoot", "GIT_CONFIG_GLOBAL", "GIT_CONFIG_NOSYSTEM"]) {
+    if (host[key] !== undefined) env[key] = host[key];
+  }
+  return env;
+}
+
+function fail(message, code = 1) {
+  const error = new Error(message);
+  error.exitCode = code;
+  throw error;
+}
+
+export function benchmarkEnvironment(extra = {}) {
+  const env = benchmarkHostEnvironment();
+  for (const key of Object.keys(env)) {
+    if (key.startsWith("PIAGENT_")) delete env[key];
+  }
+  if (process.env.PIAGENT_BENCHMARK_PI_COMMAND && process.env.PIAGENT_BENCHMARK_TASK_FIXTURE) {
+    env.PIAGENT_BENCHMARK_TASK_FIXTURE = process.env.PIAGENT_BENCHMARK_TASK_FIXTURE;
+  }
+  return { ...env, ...extra, PI_OFFLINE: "1" };
+}
+
+export function piagentTreatment(id = "release-defaults") {
+  const values = PIAGENT_BENCHMARK_TREATMENTS[id];
+  if (!values) throw new Error(`Unknown Piagent benchmark treatment: ${id}`);
+  return { id, explicit: id !== "release-defaults", environment: { ...values } };
+}
+
+export function piagentProcessEnvironment(treatmentId, extra = {}) {
+  const treatment = piagentTreatment(treatmentId);
+  return benchmarkEnvironment({ ...treatment.environment, ...extra });
+}
+
+function operatorCodexHome() {
+  return path.resolve(process.env.CODEX_HOME || path.join(os.homedir(), ".codex"));
+}
+
+export function createCodexRuntime(options) {
+  if (!options.surfaces.includes("codex-cli")) {
+    return { mode: null, home: null, credentialBridge: null, cleanup() {} };
+  }
+  if (options.codexMode === "native") {
+    return { mode: "native", home: null, credentialBridge: "operator-home", cleanup() {} };
+  }
+
+  const home = fs.mkdtempSync(path.join(os.tmpdir(), "piagent-benchmark-codex-home-"));
+  try { fs.chmodSync(home, 0o700); } catch { /* Non-POSIX filesystem. */ }
+  let credentialBridge = "environment-only";
+  const sourceAuth = process.env.PIAGENT_BENCHMARK_CODEX_AUTH_SNAPSHOT || path.join(operatorCodexHome(), "auth.json");
+  try {
+    const resolvedAuth = fs.realpathSync(sourceAuth);
+    if (!fs.statSync(resolvedAuth).isFile()) fail(`Codex credential path is not a file: ${sourceAuth}`, 1);
+    fs.copyFileSync(resolvedAuth, path.join(home, "auth.json"));
+    fs.chmodSync(path.join(home, "auth.json"), 0o600);
+    credentialBridge = process.env.PIAGENT_BENCHMARK_CODEX_AUTH_SNAPSHOT ? "frozen-auth-json-copy" : "auth-json-copy";
+  } catch (error) {
+    if (error?.code !== "ENOENT") {
+      fs.rmSync(home, { recursive: true, force: true });
+      throw error;
+    }
+  }
+  return {
+    mode: "controlled",
+    home,
+    credentialBridge,
+    cleanup() { fs.rmSync(home, { recursive: true, force: true }); }
+  };
+}
+
+export function codexProcessEnvironment(runtime, extra = {}) {
+  const env = benchmarkEnvironment(extra);
+  if (runtime.mode !== "controlled") return env;
+  for (const key of Object.keys(env)) {
+    if (key.startsWith("CODEX_") && key !== "CODEX_ACCESS_TOKEN") delete env[key];
+  }
+  for (const key of ["OPENAI_BASE_URL", "OPENAI_API_BASE"]) delete env[key];
+  env.CODEX_HOME = runtime.home;
+  return env;
+}
+
+export function comparisonSurfaces(options) {
+  return {
+    baselineSurface: options.surfaces.find((surface) => surface !== "piagent"),
+    candidateSurface: "piagent"
+  };
+}

package/packages/piagent-core/benchmark/benchmark-stop-policy.js

@@ -0,0 +1,36 @@
+function samePair(left, right) {
+  return left?.scenario?.id === right?.scenario?.id && left?.repeat === right?.repeat;
+}
+
+function outcomeFailure(run, floor) {
+  if (run?.surface !== "piagent") return null;
+  if (run?.resolved !== true) return "unresolved-outcome";
+  if (run.scenarioKind !== "safety-refusal" && (!Number.isFinite(run.grade?.score) || run.grade.score <= floor)) {
+    return "quality-outcome-floor";
+  }
+  if (run.scenarioKind !== "safety-refusal"
+    && (!Number.isFinite(run.workflow?.score) || run.workflow.score <= floor)) return "workflow-outcome-floor";
+  return null;
+}
+
+export function pairedOutcomeFloorStop({ enabled, suite, runs, current, next }) {
+  const floor = suite?.releaseGate?.minimumOutcomeScoreExclusive;
+  if (!enabled || !Number.isFinite(floor) || samePair(current, next)) return null;
+  const pair = runs.filter((run) => run.scenarioId === current?.scenario?.id && run.repeat === current?.repeat);
+  if (pair.length !== 2 || new Set(pair.map((run) => run.surface)).size !== 2) return null;
+  const failed = pair.map((run) => ({
+    surface: run.surface,
+    reason: outcomeFailure(run, floor),
+    gradeScore: Number.isFinite(run.grade?.score) ? run.grade.score : null,
+    workflowScore: Number.isFinite(run.workflow?.score) ? run.workflow.score : null
+  })).filter((item) => item.reason);
+  return failed.length ? {
+    schemaVersion: 1,
+    reason: "paired-outcome-floor-failed",
+    scope: "candidate-only",
+    scenarioId: current.scenario.id,
+    repeat: current.repeat,
+    minimumOutcomeScoreExclusive: floor,
+    failed
+  } : null;
+}

package/packages/piagent-core/benchmark/benchmark-suite-assets.js

@@ -0,0 +1,56 @@
+import fs from "node:fs";
+import path from "node:path";
+
+function fail(message) {
+  const error = new Error(message);
+  error.exitCode = 1;
+  throw error;
+}
+
+function inside(root, target) {
+  const relative = path.relative(root, target);
+  return relative === "" || (relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));
+}
+
+function moduleSpecifiers(source, label) {
+  const values = new Set();
+  const patterns = [
+    /\b(?:import|export)\s+(?:[^"';]*?\s+from\s*)?["']([^"']+)["']/g,
+    /\bimport\s*\(\s*["']([^"']+)["']\s*\)/g,
+    /\brequire\s*\(\s*["']([^"']+)["']\s*\)/g
+  ];
+  for (const pattern of patterns) for (const match of source.matchAll(pattern)) values.add(match[1]);
+  return [...values];
+}
+
+function resolveRelativeModule(from, specifier, label) {
+  const lexical = path.resolve(path.dirname(from), specifier);
+  const candidates = [lexical, `${lexical}.js`, `${lexical}.mjs`, `${lexical}.cjs`, `${lexical}.json`];
+  const selected = candidates.find((candidate) => fs.existsSync(candidate) && fs.statSync(candidate).isFile());
+  if (!selected) fail(`Suite ${label} imports a missing relative module: ${specifier}`);
+  return fs.realpathSync(selected);
+}
+
+export function assertBenchmarkModuleGraphBound(entryFile, suiteRoot, label) {
+  const canonicalRoot = fs.realpathSync(suiteRoot);
+  const pending = [fs.realpathSync(entryFile)];
+  const seen = new Set();
+  while (pending.length > 0) {
+    const file = pending.pop();
+    if (seen.has(file)) continue;
+    seen.add(file);
+    if (!inside(canonicalRoot, file)) fail(`Suite ${label} module escapes the frozen suite root: ${file}`);
+    const extension = path.extname(file);
+    if (extension === ".json") continue;
+    const source = fs.readFileSync(file, "utf8");
+    for (const specifier of moduleSpecifiers(source, label)) {
+      if (specifier.startsWith("node:")) continue;
+      if (!specifier.startsWith("./") && !specifier.startsWith("../")) {
+        fail(`Suite ${label} imports an unbound external module: ${specifier}`);
+      }
+      const dependency = resolveRelativeModule(file, specifier, label);
+      if (!inside(canonicalRoot, dependency)) fail(`Suite ${label} relative module escapes the frozen suite root: ${specifier}`);
+      pending.push(dependency);
+    }
+  }
+}

package/packages/piagent-core/benchmark/benchmark-suite-runtime.js

@@ -0,0 +1,115 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+import { benchmarkAssuranceEvidenceValidationErrors } from "./benchmark-core.js";
+import { validateBenchmarkSuite } from "./benchmark-suite.js";
+import { assertBenchmarkModuleGraphBound } from "./benchmark-suite-assets.js";
+
+function fail(message) {
+  const error = new Error(message);
+  error.exitCode = 1;
+  throw error;
+}
+
+function inside(parent, target) {
+  const relative = path.relative(parent, target);
+  return relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
+}
+
+export function loadBenchmarkSuite(input, packageRoot) {
+  const builtIn = new Map([
+    ["core-v1", path.join(packageRoot, "benchmarks", "core-v1", "suite.json")],
+    ["capability-v1", path.join(packageRoot, "benchmarks", "capability-v1", "suite.json")],
+    ["e2-framework-v1", path.join(packageRoot, "benchmarks", "e2-framework-v1", "suite.json")],
+    ["production-v1", path.join(packageRoot, "benchmarks", "production-v1", "suite.json")]
+  ]);
+  const candidate = builtIn.get(input) ?? path.resolve(input);
+  const manifestPath = fs.existsSync(candidate) && fs.statSync(candidate).isDirectory()
+    ? path.join(candidate, "suite.json")
+    : candidate;
+  let raw;
+  try { raw = JSON.parse(fs.readFileSync(manifestPath, "utf8")); }
+  catch (error) { fail(`Cannot read benchmark suite ${manifestPath}: ${error.message}`); }
+  const suite = validateBenchmarkSuite(raw);
+  return { suite, manifestPath: fs.realpathSync(manifestPath), suiteRoot: fs.realpathSync(path.dirname(manifestPath)) };
+}
+
+export function resolveBenchmarkSuiteEntry(suiteRoot, relativePath, kind) {
+  const lexical = path.resolve(suiteRoot, relativePath);
+  if (!inside(suiteRoot, lexical)) fail(`Suite ${kind} escapes its root: ${relativePath}`);
+  let resolved;
+  try { resolved = fs.realpathSync(lexical); }
+  catch { fail(`Suite ${kind} does not exist: ${relativePath}`); }
+  if (!inside(suiteRoot, resolved)) fail(`Suite ${kind} resolves outside its root: ${relativePath}`);
+  const stat = fs.statSync(resolved);
+  if (kind === "fixture" ? !stat.isDirectory() : !stat.isFile()) fail(`Suite ${kind} has the wrong file type: ${relativePath}`);
+  return resolved;
+}
+
+function rejectSymlinks(root) {
+  const pending = [root];
+  while (pending.length > 0) {
+    const current = pending.pop();
+    for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
+      const target = path.join(current, entry.name);
+      const stat = fs.lstatSync(target);
+      if (stat.isSymbolicLink()) fail(`Benchmark fixture must not contain symbolic links: ${path.relative(root, target)}`);
+      if (stat.isDirectory()) pending.push(target);
+    }
+  }
+}
+
+export function validateBenchmarkSuiteFiles(suite, suiteRoot) {
+  rejectSymlinks(suiteRoot);
+  for (const scenario of suite.scenarios) {
+    const fixture = resolveBenchmarkSuiteEntry(suiteRoot, scenario.fixture, "fixture");
+    const prompt = resolveBenchmarkSuiteEntry(suiteRoot, scenario.prompt, "prompt");
+    const grader = resolveBenchmarkSuiteEntry(suiteRoot, scenario.grader, "grader");
+    const generator = scenario.variantGenerator
+      ? resolveBenchmarkSuiteEntry(suiteRoot, scenario.variantGenerator, "variant generator")
+      : null;
+    assertBenchmarkModuleGraphBound(grader, suiteRoot, `${scenario.id} grader`);
+    if (generator) assertBenchmarkModuleGraphBound(generator, suiteRoot, `${scenario.id} variant generator`);
+    if (inside(fixture, grader) || inside(fixture, prompt)) fail(`Suite prompt and grader must stay outside the agent fixture: ${scenario.id}`);
+    if (generator && inside(fixture, generator)) fail(`Suite variant generator must stay outside the agent fixture: ${scenario.id}`);
+  }
+}
+
+export function loadBenchmarkAssuranceEvidence(suite, suiteRoot) {
+  const relativePath = suite.assurance?.evidenceManifest;
+  if (!relativePath) return { verified: false, reason: "not-declared" };
+  const target = resolveBenchmarkSuiteEntry(suiteRoot, relativePath, "assurance evidence");
+  const buffer = fs.readFileSync(target);
+  let value;
+  try { value = JSON.parse(buffer.toString("utf8")); }
+  catch { fail(`Suite assurance evidence is not valid JSON: ${relativePath}`); }
+  const errors = benchmarkAssuranceEvidenceValidationErrors(value);
+  if (errors.length > 0) fail(`Suite assurance evidence is invalid: ${errors.join("; ")}`);
+  const now = Date.now();
+  const accessReceiptCurrent = value.schemaVersion === 2
+    && now >= Date.parse(value.accessControl.issuedAt)
+    && now < Date.parse(value.accessControl.expiresAt);
+  if (value.schemaVersion === 2 && !accessReceiptCurrent) fail("Suite assurance access receipt is not currently valid");
+  const matchingFields = [
+    "claimTier", "visibility", "familyDisjointSplit", "repositoryDisjointSplit", "holdoutManifestDigest",
+    "referenceSolutionDigest", "mutationReportDigest", "calibrationReportDigest", "accessPolicyDigest",
+    "disjointnessReportDigest", "humanRubricDigest", "disagreementReportDigest"
+  ];
+  const mismatches = matchingFields.filter((field) => suite.assurance?.[field] !== value[field]);
+  if (mismatches.length > 0) fail(`Suite assurance evidence does not match suite assurance: ${mismatches.join(", ")}`);
+  return {
+    verified: true,
+    manifestPath: relativePath,
+    manifestDigest: crypto.createHash("sha256").update(buffer).digest("hex"),
+    schemaVersion: value.schemaVersion,
+    accessReceiptCurrent,
+    claimTier: value.claimTier,
+    ...Object.fromEntries(matchingFields.slice(3).map((field) => [field, value[field]])),
+    accessControl: value.accessControl,
+    disjointness: value.disjointness,
+    referenceSolutions: value.referenceSolutions,
+    mutationChecks: value.mutationChecks,
+    calibration: value.calibration
+  };
+}

package/packages/piagent-core/benchmark/benchmark-suite.js

@@ -0,0 +1,174 @@
+import { benchmarkAssuranceValidationErrors } from "./benchmark-assurance.js";
+
+const SUITE_FIELDS = new Set([
+  "schemaVersion", "id", "title", "description", "profile", "defaultRepeats", "timeoutSeconds",
+  "assurance", "releaseGate", "scenarios"
+]);
+const SCENARIO_FIELDS = new Set([
+  "id", "title", "description", "kind", "fixture", "prompt", "grader", "allowedChanges",
+  "setupFiles", "forbiddenOutputSubstrings", "requiredOutputSubstrings", "category", "difficulty",
+  "profile", "lifecycle", "variantGenerator"
+]);
+const RELEASE_GATE_FIELDS = new Set([
+  "minimumQualityScore", "minimumSafetyScore", "minimumReliabilityScore", "minimumWorkflowScore",
+  "minimumCategoryScore", "minimumOutcomeScoreExclusive", "minimumPairedScenarios", "minimumRepeats",
+  "minimumComparableEfficiencyScenarios", "maximumFreshTokenRatioUpper95", "requireEfficiencyClaim"
+]);
+const SCENARIO_KINDS = new Set(["source-change", "read-only", "safety-refusal"]);
+const SCENARIO_DIFFICULTIES = new Set(["small", "medium", "large"]);
+const SCENARIO_LIFECYCLES = new Set(["steady-state", "cold-start"]);
+const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
+
+function plainObject(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function requiredString(value, label, errors) {
+  if (typeof value !== "string" || !value.trim()) errors.push(`${label} must be a non-empty string`);
+}
+
+function safeRelativePath(value) {
+  if (typeof value !== "string" || !value.trim() || value.includes("\0")) return false;
+  const normalized = value.replaceAll("\\", "/");
+  return !normalized.startsWith("/")
+    && !/^[A-Za-z]:\//.test(normalized)
+    && !normalized.split("/").includes("..");
+}
+
+export function benchmarkSuiteValidationErrors(input) {
+  if (!plainObject(input)) return ["suite must be an object"];
+  const errors = [];
+  for (const field of Object.keys(input)) {
+    if (!SUITE_FIELDS.has(field)) errors.push(`unsupported suite field ${field}`);
+  }
+  if (![1, 2].includes(input.schemaVersion)) errors.push("schemaVersion must be 1 or 2");
+  requiredString(input.id, "id", errors);
+  if (typeof input.id === "string" && !ID_PATTERN.test(input.id)) errors.push("id must use lowercase kebab-case");
+  requiredString(input.title, "title", errors);
+  requiredString(input.profile, "profile", errors);
+  if (!Number.isInteger(input.defaultRepeats) || input.defaultRepeats < 1 || input.defaultRepeats > 10) {
+    errors.push("defaultRepeats must be between 1 and 10");
+  }
+  if (!Number.isInteger(input.timeoutSeconds) || input.timeoutSeconds < 30 || input.timeoutSeconds > 3600) {
+    errors.push("timeoutSeconds must be between 30 and 3600");
+  }
+  if (input.assurance !== undefined) {
+    errors.push(...benchmarkAssuranceValidationErrors(input.assurance, input.schemaVersion));
+  }
+  if (input.releaseGate !== undefined) {
+    if (!plainObject(input.releaseGate)) errors.push("releaseGate must be an object");
+    else {
+      for (const field of Object.keys(input.releaseGate)) {
+        if (!RELEASE_GATE_FIELDS.has(field)) errors.push(`releaseGate has unsupported field ${field}`);
+      }
+      for (const field of [
+        "minimumQualityScore", "minimumSafetyScore", "minimumReliabilityScore", "minimumWorkflowScore",
+        "minimumCategoryScore"
+      ]) {
+        const value = input.releaseGate[field];
+        if (value !== undefined && (!Number.isFinite(value) || value < 0 || value > 10)) {
+          errors.push(`releaseGate.${field} must be between 0 and 10`);
+        }
+      }
+      const minimumOutcomeScoreExclusive = input.releaseGate.minimumOutcomeScoreExclusive;
+      if (minimumOutcomeScoreExclusive !== undefined && (!Number.isFinite(minimumOutcomeScoreExclusive) || minimumOutcomeScoreExclusive < 0 || minimumOutcomeScoreExclusive >= 10)) {
+        errors.push("releaseGate.minimumOutcomeScoreExclusive must be at least 0 and below 10");
+      }
+      const minimumPairedScenarios = input.releaseGate.minimumPairedScenarios;
+      if (minimumPairedScenarios !== undefined && (!Number.isInteger(minimumPairedScenarios) || minimumPairedScenarios < 1 || minimumPairedScenarios > 50)) {
+        errors.push("releaseGate.minimumPairedScenarios must be between 1 and 50");
+      }
+      const minimumComparableEfficiencyScenarios = input.releaseGate.minimumComparableEfficiencyScenarios;
+      if (minimumComparableEfficiencyScenarios !== undefined && (!Number.isInteger(minimumComparableEfficiencyScenarios) || minimumComparableEfficiencyScenarios < 1 || minimumComparableEfficiencyScenarios > 50)) {
+        errors.push("releaseGate.minimumComparableEfficiencyScenarios must be between 1 and 50");
+      }
+      const minimumRepeats = input.releaseGate.minimumRepeats;
+      if (minimumRepeats !== undefined && (!Number.isInteger(minimumRepeats) || minimumRepeats < 1 || minimumRepeats > 10)) {
+        errors.push("releaseGate.minimumRepeats must be between 1 and 10");
+      }
+      const maximumRatio = input.releaseGate.maximumFreshTokenRatioUpper95;
+      if (maximumRatio !== undefined && (!Number.isFinite(maximumRatio) || maximumRatio <= 0 || maximumRatio > 10)) {
+        errors.push("releaseGate.maximumFreshTokenRatioUpper95 must be greater than 0 and at most 10");
+      }
+      if (input.releaseGate.requireEfficiencyClaim !== undefined && typeof input.releaseGate.requireEfficiencyClaim !== "boolean") {
+        errors.push("releaseGate.requireEfficiencyClaim must be a boolean");
+      }
+    }
+  }
+  if (input.schemaVersion === 2 && (!plainObject(input.assurance) || !plainObject(input.releaseGate))) {
+    errors.push("schemaVersion 2 requires assurance and releaseGate objects");
+  }
+  if (input.schemaVersion === 2 && input.releaseGate?.requireEfficiencyClaim === true && !Number.isInteger(input.releaseGate?.minimumComparableEfficiencyScenarios)) {
+    errors.push("schemaVersion 2 efficiency claims require releaseGate.minimumComparableEfficiencyScenarios");
+  }
+  if (!Array.isArray(input.scenarios) || input.scenarios.length === 0 || input.scenarios.length > 50) {
+    errors.push("scenarios must contain between 1 and 50 entries");
+    return errors;
+  }
+
+  const ids = new Set();
+  for (const [index, scenario] of input.scenarios.entries()) {
+    const label = `scenarios[${index}]`;
+    if (!plainObject(scenario)) {
+      errors.push(`${label} must be an object`);
+      continue;
+    }
+    for (const field of Object.keys(scenario)) {
+      if (!SCENARIO_FIELDS.has(field)) errors.push(`${label} has unsupported field ${field}`);
+    }
+    requiredString(scenario.id, `${label}.id`, errors);
+    if (typeof scenario.id === "string" && !ID_PATTERN.test(scenario.id)) errors.push(`${label}.id must use lowercase kebab-case`);
+    if (ids.has(scenario.id)) errors.push(`duplicate scenario id ${scenario.id}`);
+    ids.add(scenario.id);
+    requiredString(scenario.title, `${label}.title`, errors);
+    if (!SCENARIO_KINDS.has(scenario.kind)) errors.push(`${label}.kind is invalid`);
+    if (scenario.category !== undefined) {
+      requiredString(scenario.category, `${label}.category`, errors);
+      if (typeof scenario.category === "string" && !ID_PATTERN.test(scenario.category)) errors.push(`${label}.category must use lowercase kebab-case`);
+    } else if (input.schemaVersion === 2) errors.push(`${label}.category is required by schemaVersion 2`);
+    if (scenario.difficulty !== undefined && !SCENARIO_DIFFICULTIES.has(scenario.difficulty)) errors.push(`${label}.difficulty is invalid`);
+    else if (input.schemaVersion === 2 && scenario.difficulty === undefined) errors.push(`${label}.difficulty is required by schemaVersion 2`);
+    if (scenario.profile !== undefined) requiredString(scenario.profile, `${label}.profile`, errors);
+    if (scenario.lifecycle !== undefined && !SCENARIO_LIFECYCLES.has(scenario.lifecycle)) errors.push(`${label}.lifecycle is invalid`);
+    else if (input.schemaVersion === 2 && scenario.lifecycle === undefined) errors.push(`${label}.lifecycle is required by schemaVersion 2`);
+    for (const field of ["fixture", "prompt", "grader"]) {
+      if (!safeRelativePath(scenario[field])) errors.push(`${label}.${field} must stay inside the suite directory`);
+    }
+    if (scenario.variantGenerator !== undefined && !safeRelativePath(scenario.variantGenerator)) {
+      errors.push(`${label}.variantGenerator must stay inside the suite directory`);
+    } else if (input.schemaVersion === 2 && input.assurance?.generatedVariants === true && !scenario.variantGenerator) {
+      errors.push(`${label}.variantGenerator is required when assurance.generatedVariants is true`);
+    }
+    if (!Array.isArray(scenario.allowedChanges) || scenario.allowedChanges.some((item) => typeof item !== "string" || !item.trim())) {
+      errors.push(`${label}.allowedChanges must be an array of non-empty patterns`);
+    } else if (scenario.kind === "source-change" && scenario.allowedChanges.length === 0) {
+      errors.push(`${label}.allowedChanges must not be empty for a source-change task`);
+    }
+    if (scenario.setupFiles !== undefined) {
+      if (!plainObject(scenario.setupFiles)) errors.push(`${label}.setupFiles must be an object`);
+      else {
+        for (const [file, content] of Object.entries(scenario.setupFiles)) {
+          if (!safeRelativePath(file) || typeof content !== "string" || content.length > 100_000) {
+            errors.push(`${label}.setupFiles contains an invalid path or oversized non-string value`);
+            break;
+          }
+        }
+      }
+    }
+    if (scenario.forbiddenOutputSubstrings !== undefined && (
+      !Array.isArray(scenario.forbiddenOutputSubstrings)
+      || scenario.forbiddenOutputSubstrings.some((item) => typeof item !== "string" || !item)
+    )) errors.push(`${label}.forbiddenOutputSubstrings must contain non-empty strings`);
+    if (scenario.requiredOutputSubstrings !== undefined && (
+      !Array.isArray(scenario.requiredOutputSubstrings)
+      || scenario.requiredOutputSubstrings.some((item) => typeof item !== "string" || !item)
+    )) errors.push(`${label}.requiredOutputSubstrings must contain non-empty strings`);
+  }
+  return errors;
+}
+
+export function validateBenchmarkSuite(input) {
+  const errors = benchmarkSuiteValidationErrors(input);
+  if (errors.length > 0) throw new Error(`Benchmark suite is invalid: ${errors.join("; ")}`);
+  return input;
+}