@dungle-scrubs/skillval 0.1.0 → 0.3.0

package/dist/cli.js

@@ -4,8 +4,8 @@
 import { Command } from "commander";
 
 // src/config.ts
-import { existsSync, readFileSync as readFileSync3 } from "fs";
-import { homedir as homedir2 } from "os";
+import { existsSync as existsSync2, readFileSync as readFileSync5 } from "fs";
+import { homedir as homedir4 } from "os";
 import { resolve } from "path";
 import { Check as checkSchema, Errors as schemaErrors } from "typebox/value";
 import { parse as parseYaml } from "yaml";
@@ -13,9 +13,9 @@ import { parse as parseYaml } from "yaml";
 // src/config-contract.ts
 import Type from "typebox";
 
-// src/executors/codex.ts
+// src/executors/claude.ts
 import { spawnSync } from "child_process";
-import { mkdirSync, readFileSync as readFileSync2, symlinkSync } from "fs";
+import { copyFileSync, existsSync, mkdirSync, readFileSync as readFileSync2, symlinkSync } from "fs";
 import { homedir } from "os";
 import { join as join2 } from "path";
 
@@ -51,30 +51,205 @@ function isRecord(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
-// src/executors/codex.ts
+// src/executors/types.ts
+function assertEffortSupported(executor, effort, allowed) {
+  if (effort !== void 0 && !allowed.includes(effort)) {
+    throw new Error(
+      `${executor} does not support effort "${effort}"; valid levels: ${allowed.join(", ")}`
+    );
+  }
+}
+
+// src/executors/claude.ts
 var TRIAL_TIMEOUT_MS = 15 * 60 * 1e3;
+var CLAUDE_EFFORT_LEVELS = ["low", "medium", "high", "xhigh", "max"];
+var CLAUDE_INVOCATION_DETECTION = "structured";
+var ClaudeExecutor = class {
+  metadata;
+  #overrides;
+  #realConfigDirectory;
+  constructor(overrides = {}, realConfigDirectory = defaultConfigDirectory()) {
+    assertEffortSupported("claude", overrides.effort, CLAUDE_EFFORT_LEVELS);
+    this.#overrides = overrides;
+    this.#realConfigDirectory = realConfigDirectory;
+    const detected = detectClaude(realConfigDirectory);
+    this.metadata = {
+      ...detected,
+      model: overrides.model ?? detected.model,
+      thinking: overrides.effort ?? detected.thinking
+    };
+  }
+  runTrial(request) {
+    if (request.arm === "skill") seedSkill(request);
+    const selection = [];
+    if (this.#overrides.model !== void 0) selection.push("--model", this.#overrides.model);
+    if (this.#overrides.effort !== void 0) selection.push("--effort", this.#overrides.effort);
+    const permissions = request.evalCase.mode === "generation" ? ["--permission-mode", "acceptEdits"] : ["--permission-mode", "dontAsk", "--allowedTools", "Read,Glob,Grep,Skill"];
+    const environment = request.arm === "baseline" ? {
+      ...process.env,
+      CLAUDE_CONFIG_DIR: prepareBaselineConfig(request.home, this.#realConfigDirectory)
+    } : { ...process.env };
+    const result = spawnSync(
+      "claude",
+      [
+        "-p",
+        request.evalCase.prompt,
+        "--output-format",
+        "stream-json",
+        "--verbose",
+        "--no-session-persistence",
+        ...selection,
+        ...permissions
+      ],
+      {
+        cwd: request.workspace,
+        encoding: "utf8",
+        env: environment,
+        maxBuffer: 64 * 1024 * 1024,
+        timeout: TRIAL_TIMEOUT_MS
+      }
+    );
+    if (result.status !== 0) {
+      throw new Error(`claude -p exited ${result.status}: ${result.stderr?.slice(-500)}`);
+    }
+    return parseClaudeTrace(result.stdout, request.skillName);
+  }
+};
+function defaultConfigDirectory() {
+  return process.env.CLAUDE_CONFIG_DIR ?? join2(homedir(), ".claude");
+}
+function seedSkill(request) {
+  const skillsRoot = join2(request.workspace, ".claude/skills");
+  mkdirSync(skillsRoot, { recursive: true });
+  symlinkSync(request.skillDirectory, join2(skillsRoot, request.skillName));
+}
+function prepareBaselineConfig(home, realConfigDirectory) {
+  const configDirectory = join2(home, "claude-config");
+  mkdirSync(configDirectory, { recursive: true });
+  const credentials = join2(realConfigDirectory, ".credentials.json");
+  if (existsSync(credentials)) {
+    copyFileSync(credentials, join2(configDirectory, ".credentials.json"));
+  }
+  return configDirectory;
+}
+function detectClaude(realConfigDirectory = defaultConfigDirectory()) {
+  const version = spawnSync("claude", ["--version"], { encoding: "utf8" }).stdout?.trim() ?? "";
+  if (version === "") throw new Error("claude CLI not found on PATH");
+  let model = "default";
+  let thinking = "default";
+  try {
+    const settings = JSON.parse(
+      readFileSync2(join2(realConfigDirectory, "settings.json"), "utf8")
+    );
+    if (isRecord(settings)) {
+      if (typeof settings.model === "string") model = settings.model;
+      if (typeof settings.effort === "string") thinking = settings.effort;
+    }
+  } catch {
+  }
+  return {
+    invocationDetection: CLAUDE_INVOCATION_DETECTION,
+    model,
+    name: "claude",
+    thinking,
+    version
+  };
+}
+function parseClaudeTrace(stdout, skillName) {
+  let completed = false;
+  let invoked = false;
+  let invocationEvidence = null;
+  let resultText = "";
+  const texts = [];
+  let usage;
+  for (const line of stdout.split("\n")) {
+    if (line.trim() === "") continue;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (!isRecord(event)) continue;
+    if (event.type === "assistant" && isRecord(event.message) && Array.isArray(event.message.content)) {
+      for (const block of event.message.content) {
+        if (!isRecord(block)) continue;
+        if (block.type === "text" && typeof block.text === "string") texts.push(block.text);
+        if (block.type === "tool_use" && block.name === "Skill" && JSON.stringify(block.input ?? "").includes(skillName)) {
+          invoked = true;
+          invocationEvidence ??= `Skill tool_use: ${JSON.stringify(block.input ?? null)}`;
+        }
+      }
+    }
+    if (event.type === "result") {
+      completed = event.is_error === false;
+      usage = event.usage;
+      if (typeof event.result === "string") resultText = event.result;
+    }
+  }
+  return {
+    agentText: texts.length > 0 ? texts.join("\n") : resultText,
+    completed,
+    invocationEvidence,
+    invoked,
+    usage
+  };
+}
+
+// src/executors/codex.ts
+import { spawnSync as spawnSync2 } from "child_process";
+import { mkdirSync as mkdirSync2, readFileSync as readFileSync3, symlinkSync as symlinkSync2 } from "fs";
+import { homedir as homedir2 } from "os";
+import { join as join3 } from "path";
+var TRIAL_TIMEOUT_MS2 = 15 * 60 * 1e3;
+var CODEX_EFFORT_LEVELS = [
+  "none",
+  "minimal",
+  "low",
+  "medium",
+  "high",
+  "xhigh",
+  "max"
+];
+var CODEX_INVOCATION_DETECTION = "heuristic";
 var CodexExecutor = class {
   metadata;
+  #overrides;
   #realHome;
-  constructor(realHome = homedir()) {
+  constructor(overrides = {}, realHome = homedir2()) {
+    assertEffortSupported("codex", overrides.effort, CODEX_EFFORT_LEVELS);
+    this.#overrides = overrides;
     this.#realHome = realHome;
-    this.metadata = detectCodex(realHome);
+    const detected = detectCodex(realHome);
+    this.metadata = {
+      ...detected,
+      model: overrides.model ?? detected.model,
+      thinking: overrides.effort ?? detected.thinking
+    };
   }
   runTrial(request) {
-    if (request.arm === "skill") seedSkill(request);
+    if (request.arm === "skill") seedSkill2(request);
     const sandbox = request.evalCase.mode === "generation" ? "workspace-write" : "read-only";
+    const selection = [];
+    if (this.#overrides.model !== void 0) {
+      selection.push("-c", `model=${JSON.stringify(this.#overrides.model)}`);
+    }
+    if (this.#overrides.effort !== void 0) {
+      selection.push("-c", `model_reasoning_effort=${JSON.stringify(this.#overrides.effort)}`);
+    }
     const environment = request.arm === "baseline" ? {
       ...process.env,
-      CODEX_HOME: join2(this.#realHome, ".codex"),
+      CODEX_HOME: join3(this.#realHome, ".codex"),
       HOME: request.home
     } : { ...process.env };
-    const result = spawnSync(
+    const result = spawnSync2(
       "codex",
       [
         "exec",
         "--json",
         "--skip-git-repo-check",
         "--ephemeral",
+        ...selection,
         "-s",
         sandbox,
         "-C",
@@ -85,7 +260,7 @@ var CodexExecutor = class {
         encoding: "utf8",
         env: environment,
         maxBuffer: 64 * 1024 * 1024,
-        timeout: TRIAL_TIMEOUT_MS
+        timeout: TRIAL_TIMEOUT_MS2
       }
     );
     if (result.status !== 0) {
@@ -94,22 +269,29 @@ var CodexExecutor = class {
     return parseCodexTrace(result.stdout, request.skillName);
   }
 };
-function seedSkill(request) {
-  const skillsRoot = join2(request.workspace, ".agents/skills");
-  mkdirSync(skillsRoot, { recursive: true });
-  symlinkSync(request.skillDirectory, join2(skillsRoot, request.skillName));
+function seedSkill2(request) {
+  const skillsRoot = join3(request.workspace, ".agents/skills");
+  mkdirSync2(skillsRoot, { recursive: true });
+  symlinkSync2(request.skillDirectory, join3(skillsRoot, request.skillName));
 }
-function detectCodex(realHome = homedir()) {
-  const version = spawnSync("codex", ["--version"], { encoding: "utf8" }).stdout?.trim() ?? "";
+function detectCodex(realHome = homedir2()) {
+  const version = spawnSync2("codex", ["--version"], { encoding: "utf8" }).stdout?.trim() ?? "";
   if (version === "") throw new Error("codex CLI not found on PATH");
-  const model = readFileSync2(join2(realHome, ".codex/config.toml"), "utf8").match(
-    /^model\s*=\s*"([^"]+)"/m
-  )?.[1] ?? "default";
-  return { model, name: "codex", version };
+  const configuration = readFileSync3(join3(realHome, ".codex/config.toml"), "utf8");
+  const model = configuration.match(/^model\s*=\s*"([^"]+)"/m)?.[1] ?? "default";
+  const thinking = configuration.match(/^model_reasoning_effort\s*=\s*"([^"]+)"/m)?.[1] ?? "default";
+  return {
+    invocationDetection: CODEX_INVOCATION_DETECTION,
+    model,
+    name: "codex",
+    thinking,
+    version
+  };
 }
 function parseCodexTrace(stdout, skillName) {
   let completed = false;
   let invoked = false;
+  let invocationEvidence = null;
   const texts = [];
   let usage;
   for (const line of stdout.split("\n")) {
@@ -124,6 +306,7 @@ function parseCodexTrace(stdout, skillName) {
     const item = isRecord(event.item) ? event.item : void 0;
     if (item?.type === "command_execution" && typeof item.command === "string" && item.command.includes(`${skillName}/SKILL.md`)) {
       invoked = true;
+      invocationEvidence ??= `command_execution: ${item.command}`;
     }
     if (event.type === "item.completed" && item?.type === "agent_message" && typeof item.text === "string") {
       texts.push(item.text);
@@ -133,16 +316,146 @@ function parseCodexTrace(stdout, skillName) {
       usage = event.usage;
     }
   }
-  return { agentText: texts.join("\n"), completed, invoked, usage };
+  return { agentText: texts.join("\n"), completed, invocationEvidence, invoked, usage };
+}
+
+// src/executors/pi.ts
+import { spawnSync as spawnSync3 } from "child_process";
+import { readFileSync as readFileSync4 } from "fs";
+import { homedir as homedir3 } from "os";
+import { join as join4 } from "path";
+var TRIAL_TIMEOUT_MS3 = 15 * 60 * 1e3;
+var PI_EFFORT_LEVELS = [
+  "off",
+  "minimal",
+  "low",
+  "medium",
+  "high",
+  "xhigh"
+];
+var PI_INVOCATION_DETECTION = "heuristic";
+var PiExecutor = class {
+  metadata;
+  #overrides;
+  constructor(overrides = {}, settingsDirectory = join4(homedir3(), ".pi")) {
+    assertEffortSupported("pi", overrides.effort, PI_EFFORT_LEVELS);
+    this.#overrides = overrides;
+    const detected = detectPi(settingsDirectory);
+    this.metadata = {
+      ...detected,
+      model: overrides.model ?? detected.model,
+      thinking: overrides.effort ?? detected.thinking
+    };
+  }
+  runTrial(request) {
+    const arm = request.arm === "skill" ? ["--skill", request.skillDirectory] : ["--no-skills"];
+    const selection = [];
+    if (this.#overrides.model !== void 0) selection.push("--model", this.#overrides.model);
+    if (this.#overrides.effort !== void 0) selection.push("--thinking", this.#overrides.effort);
+    const tools = request.evalCase.mode === "generation" ? [] : ["-t", "read"];
+    const result = spawnSync3(
+      "pi",
+      [
+        "-p",
+        "--mode",
+        "json",
+        "--no-session",
+        ...selection,
+        ...arm,
+        ...tools,
+        request.evalCase.prompt
+      ],
+      {
+        cwd: request.workspace,
+        encoding: "utf8",
+        env: { ...process.env },
+        maxBuffer: 64 * 1024 * 1024,
+        timeout: TRIAL_TIMEOUT_MS3
+      }
+    );
+    if (result.status !== 0) {
+      throw new Error(`pi -p exited ${result.status}: ${result.stderr?.slice(-500)}`);
+    }
+    const trace = parsePiTrace(result.stdout, request.skillName);
+    if (!trace.completed && /no api key/i.test(result.stdout)) {
+      throw new Error(
+        "pi found no API key for its configured provider; export the provider key (e.g. ZAI_API_KEY) in the environment running skillval"
+      );
+    }
+    return trace;
+  }
+};
+function detectPi(settingsDirectory = join4(homedir3(), ".pi")) {
+  const version = spawnSync3("pi", ["--version"], { encoding: "utf8" }).stdout?.trim() ?? "";
+  if (version === "") throw new Error("pi CLI not found on PATH");
+  let model = "default";
+  let thinking = "default";
+  try {
+    const settings = JSON.parse(
+      readFileSync4(join4(settingsDirectory, "settings.json"), "utf8")
+    );
+    if (isRecord(settings)) {
+      if (typeof settings.defaultModel === "string") {
+        model = typeof settings.defaultProvider === "string" ? `${settings.defaultProvider}/${settings.defaultModel}` : settings.defaultModel;
+      }
+      if (typeof settings.defaultThinkingLevel === "string") {
+        thinking = settings.defaultThinkingLevel;
+      }
+    }
+  } catch {
+  }
+  return {
+    invocationDetection: PI_INVOCATION_DETECTION,
+    model,
+    name: "pi",
+    thinking,
+    version
+  };
+}
+function parsePiTrace(stdout, skillName) {
+  let completed = false;
+  let invoked = false;
+  let invocationEvidence = null;
+  const texts = [];
+  let usage;
+  for (const line of stdout.split("\n")) {
+    if (line.trim() === "") continue;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (!isRecord(event)) continue;
+    if (event.type !== "agent_end" || !Array.isArray(event.messages)) continue;
+    completed = true;
+    for (const message of event.messages) {
+      if (!isRecord(message) || message.role !== "assistant") continue;
+      if (!Array.isArray(message.content)) continue;
+      for (const block of message.content) {
+        if (!isRecord(block)) continue;
+        if (block.type === "text" && typeof block.text === "string") texts.push(block.text);
+        if (block.type === "toolCall" && JSON.stringify(block.arguments ?? "").includes(`${skillName}/SKILL.md`)) {
+          invoked = true;
+          const name = typeof block.name === "string" ? block.name : "toolCall";
+          invocationEvidence ??= `${name} toolCall: ${JSON.stringify(block.arguments ?? null)}`;
+        }
+      }
+      usage = message.usage ?? usage;
+    }
+  }
+  return { agentText: texts.join("\n"), completed, invocationEvidence, invoked, usage };
 }
 
 // src/executors/index.ts
 var executorFactories = {
-  codex: () => new CodexExecutor()
+  claude: (overrides) => new ClaudeExecutor(overrides),
+  codex: (overrides) => new CodexExecutor(overrides),
+  pi: (overrides) => new PiExecutor(overrides)
 };
 var EXECUTOR_NAMES = Object.keys(executorFactories);
-function createExecutor(name) {
-  return executorFactories[name]();
+function createExecutor(name, overrides = {}) {
+  return executorFactories[name](overrides);
 }
 
 // src/config-contract.ts
@@ -177,20 +490,20 @@ var ConfigError = class extends Error {
 };
 function resolveConfigPath(options = {}) {
   const environment = options.environment ?? process.env;
-  const home = options.home ?? homedir2();
+  const home = options.home ?? homedir4();
   const selected = options.cliPath ?? environment.SKILLVAL_CONFIG ?? (environment.XDG_CONFIG_HOME ? resolve(environment.XDG_CONFIG_HOME, "skillval/config.yml") : resolve(home, ".config/skillval/config.yml"));
   return resolve(expandHome(selected, home));
 }
-function expandRoot(root, home = homedir2()) {
+function expandRoot(root, home = homedir4()) {
   return resolve(expandHome(root, home));
 }
-function loadConfig(path, home = homedir2()) {
-  if (!existsSync(path)) {
+function loadConfig(path, home = homedir4()) {
+  if (!existsSync2(path)) {
     throw new ConfigError(`config file not found: ${path}`, "CONFIG_NOT_FOUND");
   }
   let parsed;
   try {
-    parsed = parseYaml(readFileSync3(path, "utf8"));
+    parsed = parseYaml(readFileSync5(path, "utf8"));
   } catch (error) {
     const detail = error instanceof Error ? error.message : String(error);
     throw new ConfigError(`invalid YAML in ${path}: ${detail}`, "CONFIG_YAML_INVALID");
@@ -206,7 +519,7 @@ function loadConfig(path, home = homedir2()) {
     roots: parsed.roots.map((root) => expandRoot(root, home))
   };
 }
-function resolveStateDirectory(environment = process.env, home = homedir2()) {
+function resolveStateDirectory(environment = process.env, home = homedir4()) {
   return environment.XDG_STATE_HOME ? resolve(environment.XDG_STATE_HOME, "skillval") : resolve(home, ".local/state/skillval");
 }
 function expandHome(value, home) {
@@ -215,23 +528,64 @@ function expandHome(value, home) {
 }
 
 // src/discovery.ts
-import { existsSync as existsSync3, readdirSync as readdirSync3 } from "fs";
-import { join as join6 } from "path";
+import { existsSync as existsSync4, readdirSync as readdirSync3 } from "fs";
+import { join as join8 } from "path";
 
 // src/case-file.ts
-import { readFileSync as readFileSync5, statSync as statSync2 } from "fs";
-import { dirname as dirname2, join as join5 } from "path";
+import { readFileSync as readFileSync8, statSync as statSync2 } from "fs";
+import { dirname as dirname2, join as join7 } from "path";
 import { parse as parseYaml2 } from "yaml";
 
 // src/case-contract.ts
-import Type2 from "typebox";
+import { isAbsolute, normalize } from "path";
+import Type3 from "typebox";
 import { Check as checkSchema2, Errors as schemaErrors2 } from "typebox/value";
 
 // src/graders.ts
-import { spawnSync as spawnSync2 } from "child_process";
-import { existsSync as existsSync2, writeFileSync } from "fs";
+import { spawnSync as spawnSync4 } from "child_process";
+import {
+  existsSync as existsSync3,
+  lstatSync,
+  readFileSync as readFileSync6,
+  realpathSync,
+  writeFileSync
+} from "fs";
 import { createRequire } from "module";
-import { dirname, join as join3 } from "path";
+import { dirname, join as join5, resolve as resolve2, sep } from "path";
+import { Ajv2020 } from "ajv/dist/2020.js";
+import Type2 from "typebox";
+var jsonSchemaGraderSchema = Type2.ReadonlyObject(
+  Type2.Object({
+    file: Type2.String({
+      description: "Produced file, relative to the workspace, parsed as JSON and validated.",
+      minLength: 1,
+      pattern: String.raw`\S`
+    }),
+    schema: Type2.Union([Type2.Record(Type2.String(), Type2.Unknown()), Type2.Boolean()], {
+      description: "JSON Schema (draft 2020-12) the produced file must satisfy; an object or a boolean schema. Omit $schema, or set it to 2020-12; other declared dialects are rejected."
+    })
+  }),
+  { additionalProperties: false }
+);
+var JSON_SCHEMA_GRADER_MODES = ["generation"];
+var commandExitGraderSchema = Type2.ReadonlyObject(
+  Type2.Object({
+    command: Type2.String({
+      description: "Shell command run in the workspace; the grader passes when it exits as expected. Trusted case input: this is arbitrary shell executed on the grading machine (see the README's Trust model section).",
+      minLength: 1,
+      pattern: String.raw`\S`
+    }),
+    expect: Type2.Optional(
+      Type2.Integer({
+        description: "Exit code the command must produce to pass. Defaults to 0.",
+        maximum: 255,
+        minimum: 0
+      })
+    )
+  }),
+  { additionalProperties: false }
+);
+var COMMAND_EXIT_GRADER_MODES = ["generation"];
 var packageRequire = createRequire(import.meta.url);
 var graders = {
   tsc: {
@@ -245,15 +599,135 @@ function graderSupportsMode(name, mode) {
   return modes.includes(mode);
 }
 function runGraders(evalCase, workspace) {
-  return (evalCase.assert?.graders ?? []).map((name) => graders[name].run(workspace));
+  const checks = [];
+  if (evalCase.assert?.json_schema !== void 0) {
+    checks.push(gradeJsonSchema(workspace, evalCase.assert.json_schema));
+  }
+  if (evalCase.assert?.command_exit !== void 0) {
+    checks.push(gradeCommandExit(workspace, evalCase.assert.command_exit));
+  }
+  for (const name of evalCase.assert?.graders ?? []) {
+    checks.push(graders[name].run(workspace));
+  }
+  return checks;
+}
+var COMMAND_EXIT_TIMEOUT_MS = 12e4;
+function gradeCommandExit(workspace, config) {
+  const expected = config.expect ?? 0;
+  const outcome = spawnSync4(config.command, {
+    cwd: workspace,
+    encoding: "utf8",
+    env: { HOME: workspace, PATH: process.env.PATH ?? "" },
+    killSignal: "SIGKILL",
+    maxBuffer: 64 * 1024 * 1024,
+    shell: true,
+    timeout: COMMAND_EXIT_TIMEOUT_MS
+  });
+  if (outcome.error !== void 0) {
+    const timedOut = outcome.error.code === "ETIMEDOUT";
+    const reason = timedOut ? `timed out after ${COMMAND_EXIT_TIMEOUT_MS / 1e3}s` : outcome.error.message;
+    return {
+      detail: `command "${config.command}" failed to run: ${reason}`,
+      name: "command_exit",
+      pass: false
+    };
+  }
+  if (outcome.signal !== null) {
+    return {
+      detail: `command "${config.command}" terminated by ${outcome.signal}`,
+      name: "command_exit",
+      pass: false
+    };
+  }
+  if (outcome.status === expected) {
+    return {
+      detail: `command "${config.command}" exited ${expected}`,
+      name: "command_exit",
+      pass: true
+    };
+  }
+  const stderr = outcome.stderr === "" ? "" : `: ${outcome.stderr.slice(0, 300)}`;
+  return {
+    detail: `command "${config.command}" exited ${outcome.status}, expected ${expected}${stderr}`,
+    name: "command_exit",
+    pass: false
+  };
+}
+function compileSchema(schema) {
+  try {
+    const ajv = new Ajv2020({ allErrors: false, strict: false });
+    return { ok: true, validate: ajv.compile(schema) };
+  } catch (error) {
+    return { message: error instanceof Error ? error.message : String(error), ok: false };
+  }
+}
+function jsonSchemaCompileError(schema) {
+  const result = compileSchema(schema);
+  return result.ok ? null : result.message;
+}
+function safeRealpath(target) {
+  try {
+    return realpathSync(target);
+  } catch {
+    return null;
+  }
+}
+function safeLstat(target) {
+  try {
+    return lstatSync(target);
+  } catch {
+    return null;
+  }
+}
+function gradeJsonSchema(workspace, config) {
+  const workspaceRoot = safeRealpath(resolve2(workspace));
+  if (workspaceRoot === null) {
+    return { detail: "workspace not found", name: "json_schema", pass: false };
+  }
+  const target = safeRealpath(resolve2(workspaceRoot, config.file));
+  if (target === null) {
+    return { detail: `file not found: ${config.file}`, name: "json_schema", pass: false };
+  }
+  if (target !== workspaceRoot && !target.startsWith(workspaceRoot + sep)) {
+    return { detail: `file escapes workspace: ${config.file}`, name: "json_schema", pass: false };
+  }
+  const stats = safeLstat(target);
+  if (stats === null || !stats.isFile()) {
+    return { detail: `not a regular file: ${config.file}`, name: "json_schema", pass: false };
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(readFileSync6(target, "utf8"));
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      detail: `invalid JSON in ${config.file}: ${message}`,
+      name: "json_schema",
+      pass: false
+    };
+  }
+  const compiled = compileSchema(config.schema);
+  if (!compiled.ok) {
+    return { detail: `invalid json_schema: ${compiled.message}`, name: "json_schema", pass: false };
+  }
+  if (compiled.validate(parsed)) {
+    return { detail: `${config.file} matches schema`, name: "json_schema", pass: true };
+  }
+  const first = compiled.validate.errors?.[0];
+  const location = first?.instancePath === void 0 || first.instancePath === "" ? "(root)" : first.instancePath;
+  return {
+    detail: `${config.file} ${location} ${first?.message ?? "does not match schema"}`,
+    name: "json_schema",
+    pass: false
+  };
 }
 function gradeTsc(workspace) {
-  if (!existsSync2(join3(workspace, "package.json"))) {
-    writeFileSync(join3(workspace, "package.json"), '{ "type": "module" }\n');
+  if (!existsSync3(join5(workspace, "package.json"))) {
+    writeFileSync(join5(workspace, "package.json"), '{ "type": "module" }\n');
   }
   const nodeTypesDirectory = dirname(packageRequire.resolve("@types/node/package.json"));
   writeFileSync(
-    join3(workspace, "tsconfig.json"),
+    join5(workspace, "tsconfig.json"),
     JSON.stringify({
       compilerOptions: {
         lib: ["es2023"],
@@ -268,8 +742,12 @@ function gradeTsc(workspace) {
       }
     })
   );
-  const typescriptBinary = packageRequire.resolve("typescript/bin/tsc");
-  const result = spawnSync2(typescriptBinary, ["-p", workspace], {
+  const typescriptBinary = join5(
+    dirname(packageRequire.resolve("typescript/package.json")),
+    "bin",
+    "tsc"
+  );
+  const result = spawnSync4(typescriptBinary, ["-p", workspace], {
     encoding: "utf8",
     timeout: 12e4
   });
@@ -281,65 +759,67 @@ function gradeTsc(workspace) {
 }
 
 // src/case-contract.ts
-var classificationSchema = Type2.Enum(["capability", "preference"]);
-var nonEmptyStringSchema = Type2.String({ minLength: 1, pattern: String.raw`\S` });
-var stringArraySchema = Type2.Readonly(Type2.Array(Type2.String()));
-var armSchema = Type2.Enum(["baseline", "skill"]);
-var fixtureSchema = Type2.ReadonlyObject(
-  Type2.Object({
-    path: Type2.Optional(
-      Type2.String({
+var classificationSchema = Type3.Enum(["capability", "preference"]);
+var nonEmptyStringSchema = Type3.String({ minLength: 1, pattern: String.raw`\S` });
+var stringArraySchema = Type3.Readonly(Type3.Array(Type3.String()));
+var armSchema = Type3.Enum(["baseline", "skill"]);
+var fixtureSchema = Type3.ReadonlyObject(
+  Type3.Object({
+    path: Type3.Optional(
+      Type3.String({
         description: "Directory relative to skillval.yml, copied into the workspace.",
         minLength: 1,
         pattern: String.raw`\S`
       })
     ),
-    setup: Type2.Optional(
-      Type2.Readonly(
-        Type2.Array(nonEmptyStringSchema, {
-          description: "Shell commands run sequentially inside the workspace after the copy."
+    setup: Type3.Optional(
+      Type3.Readonly(
+        Type3.Array(nonEmptyStringSchema, {
+          description: "Shell commands run sequentially inside the workspace after the copy. Trusted case input: arbitrary shell executed on the grading machine (see the README's Trust model section)."
         })
       )
     )
   }),
   { additionalProperties: false, minProperties: 1 }
 );
-var caseAssertSchema = Type2.ReadonlyObject(
-  Type2.Object({
-    graders: Type2.Optional(
-      Type2.Readonly(Type2.Array(Type2.Enum(GRADER_NAMES), { uniqueItems: true }))
+var caseAssertSchema = Type3.ReadonlyObject(
+  Type3.Object({
+    command_exit: Type3.Optional(commandExitGraderSchema),
+    graders: Type3.Optional(
+      Type3.Readonly(Type3.Array(Type3.Enum(GRADER_NAMES), { uniqueItems: true }))
     ),
-    must_match: Type2.Optional(stringArraySchema),
-    must_not_match: Type2.Optional(stringArraySchema)
+    json_schema: Type3.Optional(jsonSchemaGraderSchema),
+    must_match: Type3.Optional(stringArraySchema),
+    must_not_match: Type3.Optional(stringArraySchema)
   }),
   { additionalProperties: false }
 );
-var evalCaseSchema = Type2.ReadonlyObject(
-  Type2.Object({
-    arms: Type2.Optional(
-      Type2.Readonly(
-        Type2.Array(armSchema, {
+var evalCaseSchema = Type3.ReadonlyObject(
+  Type3.Object({
+    arms: Type3.Optional(
+      Type3.Readonly(
+        Type3.Array(armSchema, {
           uniqueItems: true
         })
       )
     ),
-    assert: Type2.Optional(caseAssertSchema),
-    fixture: Type2.Optional(fixtureSchema),
+    assert: Type3.Optional(caseAssertSchema),
+    fixture: Type3.Optional(fixtureSchema),
     id: nonEmptyStringSchema,
-    mode: Type2.Enum(["generation", "trigger"]),
+    mode: Type3.Enum(["generation", "trigger"]),
     prompt: nonEmptyStringSchema,
-    rule: Type2.Optional(Type2.String()),
-    should_trigger: Type2.Optional(Type2.Boolean()),
-    trials: Type2.Optional(Type2.Integer({ maximum: 5, minimum: 1 })),
-    type: Type2.Optional(classificationSchema)
+    rule: Type3.Optional(Type3.String()),
+    should_trigger: Type3.Optional(Type3.Boolean()),
+    trials: Type3.Optional(Type3.Integer({ maximum: 5, minimum: 1 })),
+    type: Type3.Optional(classificationSchema)
   }),
   { additionalProperties: false }
 );
-var skillEvalsSchema = Type2.ReadonlyObject(
-  Type2.Object({
-    cases: Type2.Readonly(Type2.Array(evalCaseSchema)),
+var skillEvalsSchema = Type3.ReadonlyObject(
+  Type3.Object({
+    cases: Type3.Readonly(Type3.Array(evalCaseSchema)),
     class: classificationSchema,
-    fixture: Type2.Optional(fixtureSchema),
+    fixture: Type3.Optional(fixtureSchema),
     skill: nonEmptyStringSchema
   }),
   {
@@ -384,9 +864,40 @@ function parseCaseValue(value, path, expectedSkill) {
     ids.add(evalCase.id);
     validatePatterns(evalCase, path);
     validateGraders(evalCase, path);
+    validateJsonSchemaGrader(evalCase, path);
+    validateCommandExitGrader(evalCase, path);
   }
   return value;
 }
+function validateCommandExitGrader(evalCase, path) {
+  if (evalCase.assert?.command_exit === void 0) return;
+  if (!COMMAND_EXIT_GRADER_MODES.includes(evalCase.mode)) {
+    throw new CaseContractError(
+      `${path} case "${evalCase.id}" grader "command_exit" does not support ${evalCase.mode} mode`
+    );
+  }
+}
+function validateJsonSchemaGrader(evalCase, path) {
+  const config = evalCase.assert?.json_schema;
+  if (config === void 0) return;
+  if (!JSON_SCHEMA_GRADER_MODES.includes(evalCase.mode)) {
+    throw new CaseContractError(
+      `${path} case "${evalCase.id}" grader "json_schema" does not support ${evalCase.mode} mode`
+    );
+  }
+  const segments = normalize(config.file).split(/[/\\]/);
+  if (isAbsolute(config.file) || segments.includes("..")) {
+    throw new CaseContractError(
+      `${path} case "${evalCase.id}" json_schema file "${config.file}" must be a path inside the workspace`
+    );
+  }
+  const schemaError = jsonSchemaCompileError(config.schema);
+  if (schemaError !== null) {
+    throw new CaseContractError(
+      `${path} case "${evalCase.id}" has an invalid json_schema: ${schemaError}`
+    );
+  }
+}
 function validateGraders(evalCase, path) {
   for (const grader of evalCase.assert?.graders ?? []) {
     if (!graderSupportsMode(grader, evalCase.mode)) {
@@ -413,10 +924,10 @@ function validatePatterns(evalCase, path) {
 }
 
 // src/fixture.ts
-import { spawnSync as spawnSync3 } from "child_process";
+import { spawnSync as spawnSync5 } from "child_process";
 import { createHash as createHash2 } from "crypto";
-import { cpSync, readdirSync as readdirSync2, readFileSync as readFileSync4, statSync } from "fs";
-import { basename, join as join4 } from "path";
+import { cpSync, readdirSync as readdirSync2, readFileSync as readFileSync7, statSync } from "fs";
+import { basename, join as join6 } from "path";
 var SETUP_COMMAND_TIMEOUT_MS = 6e4;
 var FixtureSetupError = class extends Error {
   results;
@@ -431,7 +942,7 @@ function selectFixture(caseFixture, suiteFixture) {
 }
 function resolveFixture(fixture, baseDirectory) {
   if (fixture === void 0) return void 0;
-  const directory = fixture.path === void 0 ? void 0 : join4(baseDirectory, fixture.path);
+  const directory = fixture.path === void 0 ? void 0 : join6(baseDirectory, fixture.path);
   const setup = fixture.setup ?? [];
   return { directory, hash: fixtureIdentityHash(directory, setup), setup };
 }
@@ -443,7 +954,7 @@ function walkFixtureEntries(directory) {
     );
     for (const dirent of dirents) {
       if (SKIPPED_DIRECTORIES.has(dirent.name)) continue;
-      const absolutePath = join4(current, dirent.name);
+      const absolutePath = join6(current, dirent.name);
       const path = prefix === "" ? dirent.name : `${prefix}/${dirent.name}`;
       if (dirent.isSymbolicLink()) {
         throw new Error(
@@ -472,7 +983,7 @@ function fixtureIdentityHash(directory, setup) {
         hash.update(`D\0${entry.path}\0`);
         continue;
       }
-      const content = readFileSync4(entry.absolutePath);
+      const content = readFileSync7(entry.absolutePath);
       hash.update(`F\0${entry.path}\0${entry.executable ? "x" : "-"}\0${content.byteLength}\0`);
       hash.update(content);
     }
@@ -490,7 +1001,7 @@ function applyFixture(fixture, workspace, home) {
   }
   const results = [];
   for (const command of fixture.setup) {
-    const outcome = spawnSync3(command, {
+    const outcome = spawnSync5(command, {
       cwd: workspace,
       encoding: "utf8",
       env: { HOME: home, PATH: process.env.PATH ?? "" },
@@ -522,7 +1033,7 @@ var CaseFileError = class extends CaseContractError {
   }
 };
 function readCaseFile(path, expectedSkill) {
-  const evals = parseCaseFile(readFileSync5(path, "utf8"), path, expectedSkill);
+  const evals = parseCaseFile(readFileSync8(path, "utf8"), path, expectedSkill);
   const baseDirectory = dirname2(path);
   assertFixtureDirectory(evals.fixture, `${path} suite fixture`, baseDirectory);
   for (const evalCase of evals.cases) {
@@ -536,7 +1047,7 @@ function readCaseFile(path, expectedSkill) {
 }
 function assertFixtureDirectory(fixture, subject, baseDirectory) {
   if (fixture?.path === void 0) return;
-  const directory = join5(baseDirectory, fixture.path);
+  const directory = join7(baseDirectory, fixture.path);
   const stats = statSync2(directory, { throwIfNoEntry: false });
   if (stats === void 0) {
     throw new CaseFileError(
@@ -580,14 +1091,14 @@ function discoverSkills(roots) {
   const missingRoots = [];
   const skills = [];
   for (const root of roots) {
-    if (!existsSync3(root)) {
+    if (!existsSync4(root)) {
       missingRoots.push(root);
       continue;
     }
     const entries = readdirSync3(root, { withFileTypes: true }).filter((entry) => entry.isDirectory()).sort((left, right) => left.name.localeCompare(right.name));
     for (const entry of entries) {
-      const skillDirectory = join6(root, entry.name);
-      if (!existsSync3(join6(skillDirectory, "SKILL.md"))) continue;
+      const skillDirectory = join8(root, entry.name);
+      if (!existsSync4(join8(skillDirectory, "SKILL.md"))) continue;
       skills.push(describeSkill(entry.name, root, skillDirectory));
     }
   }
@@ -636,8 +1147,8 @@ function selectSkills(discovery, requestedNames) {
   });
 }
 function describeSkill(name, root, skillDirectory) {
-  const caseFilePath = join6(skillDirectory, "skillval.yml");
-  if (!existsSync3(caseFilePath)) {
+  const caseFilePath = join8(skillDirectory, "skillval.yml");
+  if (!existsSync4(caseFilePath)) {
     return {
       caseCount: 0,
       class: void 0,
@@ -678,14 +1189,14 @@ function describeSkill(name, root, skillDirectory) {
 }
 
 // src/runner.ts
-import { mkdirSync as mkdirSync3, mkdtempSync, rmSync, writeFileSync as writeFileSync3 } from "fs";
+import { mkdirSync as mkdirSync4, mkdtempSync, rmSync, writeFileSync as writeFileSync3 } from "fs";
 import { tmpdir } from "os";
-import { join as join8 } from "path";
+import { join as join10 } from "path";
 
 // src/cache.ts
-import { existsSync as existsSync4, mkdirSync as mkdirSync2, readFileSync as readFileSync6, writeFileSync as writeFileSync2 } from "fs";
-import { dirname as dirname3, join as join7 } from "path";
-var RUNNER_VERSION = 6;
+import { existsSync as existsSync5, mkdirSync as mkdirSync3, readFileSync as readFileSync9, writeFileSync as writeFileSync2 } from "fs";
+import { dirname as dirname3, join as join9 } from "path";
+var RUNNER_VERSION = 9;
 var ArmCache = class {
   #stateDirectory;
   constructor(stateDirectory = resolveStateDirectory()) {
@@ -693,32 +1204,34 @@ var ArmCache = class {
   }
   lookup(identity) {
     const path = this.#path(identity);
-    if (!existsSync4(path)) return void 0;
-    return { ...JSON.parse(readFileSync6(path, "utf8")), cached: true };
+    if (!existsSync5(path)) return void 0;
+    return { ...JSON.parse(readFileSync9(path, "utf8")), cached: true };
   }
   store(identity, result) {
     const path = this.#path(identity);
-    mkdirSync2(dirname3(path), { recursive: true });
+    mkdirSync3(dirname3(path), { recursive: true });
     writeFileSync2(path, JSON.stringify(result));
   }
   #path(identity) {
+    const skillComponent = identity.arm === "skill" ? identity.skillHash : "";
     const parts = [
       String(RUNNER_VERSION),
-      identity.skillHash,
+      skillComponent,
       JSON.stringify(identity.evalCase),
       identity.arm,
       identity.executor.name,
       identity.executor.version,
-      identity.executor.model
+      identity.executor.model,
+      identity.executor.thinking
     ];
     if (identity.fixtureHash !== void 0) parts.push(identity.fixtureHash);
     const key = sha256(parts.join("\0"));
-    return join7(this.#stateDirectory, "cache", `${key}.json`);
+    return join9(this.#stateDirectory, "cache", `${key}.json`);
   }
 };
 
 // src/grade.ts
-import { readFileSync as readFileSync7 } from "fs";
+import { readFileSync as readFileSync10 } from "fs";
 import { relative as relative2 } from "path";
 var INJECTED_FILES = /* @__PURE__ */ new Set(["package.json", "tsconfig.json"]);
 function gradeTrial(evalCase, arm, trace, workspace) {
@@ -729,14 +1242,15 @@ function gradeTrial(evalCase, arm, trace, workspace) {
     pass: trace.completed
   });
   if (evalCase.should_trigger !== void 0 && arm === "skill") {
+    const evidence = trace.invocationEvidence === null ? "none" : trace.invocationEvidence;
     checks.push({
-      detail: `invoked=${trace.invoked}, expected=${evalCase.should_trigger}`,
+      detail: `invoked=${trace.invoked}, expected=${evalCase.should_trigger}, evidence=${evidence}`,
       name: "trigger",
       pass: trace.invoked === evalCase.should_trigger
     });
   }
   const gradedText = evalCase.mode === "generation" ? walkFiles(workspace).filter((file) => !INJECTED_FILES.has(relative2(workspace, file))).map((file) => `=== ${relative2(workspace, file)} ===
-${readFileSync7(file, "utf8")}`).join("\n") : trace.agentText;
+${readFileSync10(file, "utf8")}`).join("\n") : trace.agentText;
   for (const pattern of evalCase.assert?.must_match ?? []) {
     const pass = new RegExp(pattern, "m").test(gradedText);
     checks.push({
@@ -772,7 +1286,19 @@ function shouldEscalate(trials) {
 function runEvaluation(config, options, log) {
   const discovery = discoverSkills(config.roots);
   const selectedSkills = selectSkills(discovery, options.requestedSkills);
-  const executor = createExecutor(config.executor);
+  assertPiGenerationAcknowledged(
+    config.executor,
+    selectedSkills,
+    options.caseFilter,
+    options.allowUnsandboxedPi
+  );
+  const executor = createExecutor(config.executor, {
+    effort: options.effort,
+    model: options.model
+  });
+  log(
+    `executor: ${executor.metadata.name} ${executor.metadata.version} (model ${executor.metadata.model}, thinking ${executor.metadata.thinking}, invocation detection ${executor.metadata.invocationDetection})`
+  );
   const stateDirectory = resolveStateDirectory();
   const cache = new ArmCache(stateDirectory);
   const skillInputs = selectedSkills.map((skill) => ({
@@ -811,12 +1337,25 @@ function runEvaluation(config, options, log) {
     runHash,
     skills: skillReports
   };
-  const reportDirectory = join8(stateDirectory, "reports");
-  const reportPath = join8(reportDirectory, `${runHash}.json`);
-  mkdirSync3(reportDirectory, { recursive: true });
+  const reportDirectory = join10(stateDirectory, "reports");
+  const reportPath = join10(reportDirectory, `${runHash}.json`);
+  mkdirSync4(reportDirectory, { recursive: true });
   writeFileSync3(reportPath, JSON.stringify(report, null, 2));
   return { failures, noops, report, reportPath };
 }
+function assertPiGenerationAcknowledged(executorName, skills, caseFilter, allow) {
+  if (executorName !== "pi" || allow) return;
+  for (const skill of skills) {
+    for (const evalCase of skill.evals.cases) {
+      if (caseFilter !== void 0 && evalCase.id !== caseFilter) continue;
+      if (evalCase.mode === "generation") {
+        throw new Error(
+          `pi has no OS sandbox, so generation case "${evalCase.id}" (skill "${skill.name}") would run agent writes without enforced isolation. Re-run with --allow-unsandboxed-pi to acknowledge, or use codex or claude for generation cases.`
+        );
+      }
+    }
+  }
+}
 function participatingSkillsHash(skills) {
   return sha256(
     [...skills].sort((left, right) => left.name.localeCompare(right.name)).map(({ contentHash, name }) => `${name}\0${contentHash}`).join("\0")
@@ -885,8 +1424,8 @@ function runArm(context, arm) {
   return result;
 }
 function runTrial(context, arm) {
-  const workspace = mkdtempSync(join8(tmpdir(), `skillval-${context.evalCase.id}-`));
-  const trialHome = mkdtempSync(join8(tmpdir(), "skillval-home-"));
+  const workspace = mkdtempSync(join10(tmpdir(), `skillval-${context.evalCase.id}-`));
+  const trialHome = mkdtempSync(join10(tmpdir(), "skillval-home-"));
   try {
     const fixtureSetup = context.fixture === void 0 ? void 0 : applyFixture(context.fixture, workspace, trialHome);
     const trace = context.executor.runTrial({
@@ -927,17 +1466,23 @@ function runTrial(context, arm) {
 
 // src/cli.ts
 var program = new Command();
-program.name("skillval").description("Evaluate agent skills with deterministic graders").configureHelp({ showGlobalOptions: true }).version("0.1.0").option("--config <path>", "read configuration from this path");
+program.name("skillval").description("Evaluate agent skills with deterministic graders").configureHelp({ showGlobalOptions: true }).version("0.3.0").option("--config <path>", "read configuration from this path");
 program.command("run").description(
   "Run selected cases and return a report whose exit status fails when any skill arm fails"
-).argument("[skill...]", "skill names; omit to run every skill with skillval.yml").option("--case <id>", "run only the case with this id").option("--no-cache", "ignore cached arm results").option("--skip-baseline", "do not run baseline arms").option("--json", "return the complete report as JSON").action((skills, options, command) => {
+).argument("[skill...]", "skill names; omit to run every skill with skillval.yml").option("--case <id>", "run only the case with this id").option("--model <model>", "model for the executor to use this run").option("--effort <level>", "effort/thinking level for the executor to use this run").option("--no-cache", "ignore cached arm results").option("--skip-baseline", "do not run baseline arms").option(
+  "--allow-unsandboxed-pi",
+  "acknowledge that pi generation trials run without an OS sandbox"
+).option("--json", "return the complete report as JSON").action((skills, options, command) => {
   const globalOptions = command.optsWithGlobals();
   const configPath = resolveConfigPath({ cliPath: globalOptions.config });
   const config = loadConfig(configPath);
   const outcome = runEvaluation(
     config,
     {
+      allowUnsandboxedPi: options.allowUnsandboxedPi === true,
       caseFilter: options.case,
+      effort: options.effort,
+      model: options.model,
       requestedSkills: skills,
       skipBaseline: options.skipBaseline === true,
       useCache: options.cache