@dungle-scrubs/skillval 0.1.0

package/dist/cli.js

@@ -0,0 +1,1009 @@
+#!/usr/bin/env node
+
+// src/cli.ts
+import { Command } from "commander";
+
+// src/config.ts
+import { existsSync, readFileSync as readFileSync3 } from "fs";
+import { homedir as homedir2 } from "os";
+import { resolve } from "path";
+import { Check as checkSchema, Errors as schemaErrors } from "typebox/value";
+import { parse as parseYaml } from "yaml";
+
+// src/config-contract.ts
+import Type from "typebox";
+
+// src/executors/codex.ts
+import { spawnSync } from "child_process";
+import { mkdirSync, readFileSync as readFileSync2, symlinkSync } from "fs";
+import { homedir } from "os";
+import { join as join2 } from "path";
+
+// src/utils.ts
+import { createHash } from "crypto";
+import { readdirSync, readFileSync } from "fs";
+import { join, relative } from "path";
+function sha256(input) {
+  return createHash("sha256").update(input).digest("hex");
+}
+var SKIPPED_DIRECTORIES = /* @__PURE__ */ new Set([".git", "node_modules"]);
+function walkFiles(root) {
+  const files = [];
+  function visit(directory) {
+    for (const entry of readdirSync(directory, { withFileTypes: true })) {
+      if (SKIPPED_DIRECTORIES.has(entry.name)) continue;
+      const path = join(directory, entry.name);
+      if (entry.isDirectory()) visit(path);
+      else if (entry.isFile()) files.push(path);
+    }
+  }
+  visit(root);
+  return files.sort();
+}
+function skillContentHash(skillDirectory) {
+  const parts = walkFiles(skillDirectory).map(
+    (file) => `${relative(skillDirectory, file)}
+${readFileSync(file, "utf8")}`
+  );
+  return sha256(parts.join("\0"));
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+// src/executors/codex.ts
+var TRIAL_TIMEOUT_MS = 15 * 60 * 1e3;
+var CodexExecutor = class {
+  metadata;
+  #realHome;
+  constructor(realHome = homedir()) {
+    this.#realHome = realHome;
+    this.metadata = detectCodex(realHome);
+  }
+  runTrial(request) {
+    if (request.arm === "skill") seedSkill(request);
+    const sandbox = request.evalCase.mode === "generation" ? "workspace-write" : "read-only";
+    const environment = request.arm === "baseline" ? {
+      ...process.env,
+      CODEX_HOME: join2(this.#realHome, ".codex"),
+      HOME: request.home
+    } : { ...process.env };
+    const result = spawnSync(
+      "codex",
+      [
+        "exec",
+        "--json",
+        "--skip-git-repo-check",
+        "--ephemeral",
+        "-s",
+        sandbox,
+        "-C",
+        request.workspace,
+        request.evalCase.prompt
+      ],
+      {
+        encoding: "utf8",
+        env: environment,
+        maxBuffer: 64 * 1024 * 1024,
+        timeout: TRIAL_TIMEOUT_MS
+      }
+    );
+    if (result.status !== 0) {
+      throw new Error(`codex exec exited ${result.status}: ${result.stderr?.slice(-500)}`);
+    }
+    return parseCodexTrace(result.stdout, request.skillName);
+  }
+};
+function seedSkill(request) {
+  const skillsRoot = join2(request.workspace, ".agents/skills");
+  mkdirSync(skillsRoot, { recursive: true });
+  symlinkSync(request.skillDirectory, join2(skillsRoot, request.skillName));
+}
+function detectCodex(realHome = homedir()) {
+  const version = spawnSync("codex", ["--version"], { encoding: "utf8" }).stdout?.trim() ?? "";
+  if (version === "") throw new Error("codex CLI not found on PATH");
+  const model = readFileSync2(join2(realHome, ".codex/config.toml"), "utf8").match(
+    /^model\s*=\s*"([^"]+)"/m
+  )?.[1] ?? "default";
+  return { model, name: "codex", version };
+}
+function parseCodexTrace(stdout, skillName) {
+  let completed = false;
+  let invoked = false;
+  const texts = [];
+  let usage;
+  for (const line of stdout.split("\n")) {
+    if (line.trim() === "") continue;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (!isRecord(event)) continue;
+    const item = isRecord(event.item) ? event.item : void 0;
+    if (item?.type === "command_execution" && typeof item.command === "string" && item.command.includes(`${skillName}/SKILL.md`)) {
+      invoked = true;
+    }
+    if (event.type === "item.completed" && item?.type === "agent_message" && typeof item.text === "string") {
+      texts.push(item.text);
+    }
+    if (event.type === "turn.completed") {
+      completed = true;
+      usage = event.usage;
+    }
+  }
+  return { agentText: texts.join("\n"), completed, invoked, usage };
+}
+
+// src/executors/index.ts
+var executorFactories = {
+  codex: () => new CodexExecutor()
+};
+var EXECUTOR_NAMES = Object.keys(executorFactories);
+function createExecutor(name) {
+  return executorFactories[name]();
+}
+
+// src/config-contract.ts
+var configFileSchema = Type.ReadonlyObject(
+  Type.Object({
+    // Names come from the executor registry so configuration cannot advertise a missing adapter.
+    executor: Type.Enum(EXECUTOR_NAMES, {
+      description: "Trial executor."
+    }),
+    roots: Type.Readonly(
+      Type.Array(Type.String({ minLength: 1, pattern: String.raw`\S` }), {
+        description: "Directories whose immediate children are agent skill directories."
+      })
+    )
+  }),
+  {
+    $id: "https://raw.githubusercontent.com/dungle-scrubs/skillval/main/schemas/config.schema.json",
+    $schema: "https://json-schema.org/draft/2020-12/schema",
+    additionalProperties: false,
+    title: "skillval configuration"
+  }
+);
+
+// src/config.ts
+var ConfigError = class extends Error {
+  code;
+  constructor(message, code = "CONFIG_ERROR") {
+    super(message);
+    this.code = code;
+    this.name = "ConfigError";
+  }
+};
+function resolveConfigPath(options = {}) {
+  const environment = options.environment ?? process.env;
+  const home = options.home ?? homedir2();
+  const selected = options.cliPath ?? environment.SKILLVAL_CONFIG ?? (environment.XDG_CONFIG_HOME ? resolve(environment.XDG_CONFIG_HOME, "skillval/config.yml") : resolve(home, ".config/skillval/config.yml"));
+  return resolve(expandHome(selected, home));
+}
+function expandRoot(root, home = homedir2()) {
+  return resolve(expandHome(root, home));
+}
+function loadConfig(path, home = homedir2()) {
+  if (!existsSync(path)) {
+    throw new ConfigError(`config file not found: ${path}`, "CONFIG_NOT_FOUND");
+  }
+  let parsed;
+  try {
+    parsed = parseYaml(readFileSync3(path, "utf8"));
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new ConfigError(`invalid YAML in ${path}: ${detail}`, "CONFIG_YAML_INVALID");
+  }
+  if (!checkSchema(configFileSchema, parsed)) {
+    const [firstError] = schemaErrors(configFileSchema, parsed);
+    const location = firstError?.instancePath.replaceAll("/", ".").replace(/^\./, "");
+    const subject = location === void 0 || location === "" ? path : `${path} ${location}`;
+    throw new ConfigError(`${subject} ${firstError?.message ?? "is invalid"}`);
+  }
+  return {
+    executor: parsed.executor,
+    roots: parsed.roots.map((root) => expandRoot(root, home))
+  };
+}
+function resolveStateDirectory(environment = process.env, home = homedir2()) {
+  return environment.XDG_STATE_HOME ? resolve(environment.XDG_STATE_HOME, "skillval") : resolve(home, ".local/state/skillval");
+}
+function expandHome(value, home) {
+  const withTilde = value === "~" ? home : value.startsWith("~/") ? `${home}${value.slice(1)}` : value;
+  return withTilde.replaceAll(String.raw`\${HOME}`, home).replaceAll("$HOME", home);
+}
+
+// src/discovery.ts
+import { existsSync as existsSync3, readdirSync as readdirSync3 } from "fs";
+import { join as join6 } from "path";
+
+// src/case-file.ts
+import { readFileSync as readFileSync5, statSync as statSync2 } from "fs";
+import { dirname as dirname2, join as join5 } from "path";
+import { parse as parseYaml2 } from "yaml";
+
+// src/case-contract.ts
+import Type2 from "typebox";
+import { Check as checkSchema2, Errors as schemaErrors2 } from "typebox/value";
+
+// src/graders.ts
+import { spawnSync as spawnSync2 } from "child_process";
+import { existsSync as existsSync2, writeFileSync } from "fs";
+import { createRequire } from "module";
+import { dirname, join as join3 } from "path";
+var packageRequire = createRequire(import.meta.url);
+var graders = {
+  tsc: {
+    modes: ["generation"],
+    run: gradeTsc
+  }
+};
+var GRADER_NAMES = Object.keys(graders);
+function graderSupportsMode(name, mode) {
+  const modes = graders[name].modes;
+  return modes.includes(mode);
+}
+function runGraders(evalCase, workspace) {
+  return (evalCase.assert?.graders ?? []).map((name) => graders[name].run(workspace));
+}
+function gradeTsc(workspace) {
+  if (!existsSync2(join3(workspace, "package.json"))) {
+    writeFileSync(join3(workspace, "package.json"), '{ "type": "module" }\n');
+  }
+  const nodeTypesDirectory = dirname(packageRequire.resolve("@types/node/package.json"));
+  writeFileSync(
+    join3(workspace, "tsconfig.json"),
+    JSON.stringify({
+      compilerOptions: {
+        lib: ["es2023"],
+        module: "esnext",
+        moduleResolution: "bundler",
+        noEmit: true,
+        noUncheckedIndexedAccess: true,
+        strict: true,
+        target: "es2023",
+        typeRoots: [dirname(nodeTypesDirectory)],
+        types: ["node"]
+      }
+    })
+  );
+  const typescriptBinary = packageRequire.resolve("typescript/bin/tsc");
+  const result = spawnSync2(typescriptBinary, ["-p", workspace], {
+    encoding: "utf8",
+    timeout: 12e4
+  });
+  return {
+    detail: result.status === 0 ? "compiles strict" : (result.stdout ?? "").slice(0, 500),
+    name: "tsc",
+    pass: result.status === 0
+  };
+}
+
+// src/case-contract.ts
+var classificationSchema = Type2.Enum(["capability", "preference"]);
+var nonEmptyStringSchema = Type2.String({ minLength: 1, pattern: String.raw`\S` });
+var stringArraySchema = Type2.Readonly(Type2.Array(Type2.String()));
+var armSchema = Type2.Enum(["baseline", "skill"]);
+var fixtureSchema = Type2.ReadonlyObject(
+  Type2.Object({
+    path: Type2.Optional(
+      Type2.String({
+        description: "Directory relative to skillval.yml, copied into the workspace.",
+        minLength: 1,
+        pattern: String.raw`\S`
+      })
+    ),
+    setup: Type2.Optional(
+      Type2.Readonly(
+        Type2.Array(nonEmptyStringSchema, {
+          description: "Shell commands run sequentially inside the workspace after the copy."
+        })
+      )
+    )
+  }),
+  { additionalProperties: false, minProperties: 1 }
+);
+var caseAssertSchema = Type2.ReadonlyObject(
+  Type2.Object({
+    graders: Type2.Optional(
+      Type2.Readonly(Type2.Array(Type2.Enum(GRADER_NAMES), { uniqueItems: true }))
+    ),
+    must_match: Type2.Optional(stringArraySchema),
+    must_not_match: Type2.Optional(stringArraySchema)
+  }),
+  { additionalProperties: false }
+);
+var evalCaseSchema = Type2.ReadonlyObject(
+  Type2.Object({
+    arms: Type2.Optional(
+      Type2.Readonly(
+        Type2.Array(armSchema, {
+          uniqueItems: true
+        })
+      )
+    ),
+    assert: Type2.Optional(caseAssertSchema),
+    fixture: Type2.Optional(fixtureSchema),
+    id: nonEmptyStringSchema,
+    mode: Type2.Enum(["generation", "trigger"]),
+    prompt: nonEmptyStringSchema,
+    rule: Type2.Optional(Type2.String()),
+    should_trigger: Type2.Optional(Type2.Boolean()),
+    trials: Type2.Optional(Type2.Integer({ maximum: 5, minimum: 1 })),
+    type: Type2.Optional(classificationSchema)
+  }),
+  { additionalProperties: false }
+);
+var skillEvalsSchema = Type2.ReadonlyObject(
+  Type2.Object({
+    cases: Type2.Readonly(Type2.Array(evalCaseSchema)),
+    class: classificationSchema,
+    fixture: Type2.Optional(fixtureSchema),
+    skill: nonEmptyStringSchema
+  }),
+  {
+    $id: "https://raw.githubusercontent.com/dungle-scrubs/skillval/main/schemas/skillval.schema.json",
+    $schema: "https://json-schema.org/draft/2020-12/schema",
+    additionalProperties: false,
+    title: "skillval case file"
+  }
+);
+var CaseContractError = class extends Error {
+  code;
+  constructor(message, code = "CASE_FILE_INVALID") {
+    super(message);
+    this.code = code;
+    this.name = "CaseContractError";
+  }
+};
+function parseCaseValue(value, path, expectedSkill) {
+  if (!checkSchema2(skillEvalsSchema, value)) {
+    const [firstError] = schemaErrors2(skillEvalsSchema, value);
+    const location = firstError?.instancePath.replaceAll("/", ".").replace(/^\./, "");
+    if (location?.endsWith(".trials")) {
+      const match = /cases\.(\d+)\.trials$/.exec(location);
+      const caseIndex = match?.[1] ?? "unknown";
+      throw new CaseContractError(
+        `${path} case at index ${caseIndex} trials must be an integer from 1 through 5`
+      );
+    }
+    const subject = location === void 0 || location === "" ? path : `${path} ${location}`;
+    throw new CaseContractError(`${subject} ${firstError?.message ?? "is invalid"}`);
+  }
+  if (expectedSkill !== void 0 && value.skill !== expectedSkill) {
+    throw new CaseContractError(
+      `${path} declares skill "${value.skill}", expected "${expectedSkill}"`
+    );
+  }
+  const ids = /* @__PURE__ */ new Set();
+  for (const evalCase of value.cases) {
+    if (ids.has(evalCase.id)) {
+      throw new CaseContractError(`${path} case id "${evalCase.id}" is duplicated`);
+    }
+    ids.add(evalCase.id);
+    validatePatterns(evalCase, path);
+    validateGraders(evalCase, path);
+  }
+  return value;
+}
+function validateGraders(evalCase, path) {
+  for (const grader of evalCase.assert?.graders ?? []) {
+    if (!graderSupportsMode(grader, evalCase.mode)) {
+      throw new CaseContractError(
+        `${path} case "${evalCase.id}" grader "${grader}" does not support ${evalCase.mode} mode`
+      );
+    }
+  }
+}
+function validatePatterns(evalCase, path) {
+  const fields = ["must_match", "must_not_match"];
+  for (const field of fields) {
+    for (const pattern of evalCase.assert?.[field] ?? []) {
+      try {
+        new RegExp(pattern, "m");
+      } catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new CaseContractError(
+          `${path} case "${evalCase.id}" has invalid ${field} regex "${pattern}": ${detail}`
+        );
+      }
+    }
+  }
+}
+
+// src/fixture.ts
+import { spawnSync as spawnSync3 } from "child_process";
+import { createHash as createHash2 } from "crypto";
+import { cpSync, readdirSync as readdirSync2, readFileSync as readFileSync4, statSync } from "fs";
+import { basename, join as join4 } from "path";
+var SETUP_COMMAND_TIMEOUT_MS = 6e4;
+var FixtureSetupError = class extends Error {
+  results;
+  constructor(message, results) {
+    super(message);
+    this.name = "FixtureSetupError";
+    this.results = results;
+  }
+};
+function selectFixture(caseFixture, suiteFixture) {
+  return caseFixture ?? suiteFixture;
+}
+function resolveFixture(fixture, baseDirectory) {
+  if (fixture === void 0) return void 0;
+  const directory = fixture.path === void 0 ? void 0 : join4(baseDirectory, fixture.path);
+  const setup = fixture.setup ?? [];
+  return { directory, hash: fixtureIdentityHash(directory, setup), setup };
+}
+function walkFixtureEntries(directory) {
+  const entries = [];
+  function visit(current, prefix) {
+    const dirents = [...readdirSync2(current, { withFileTypes: true })].sort(
+      (left, right) => left.name.localeCompare(right.name)
+    );
+    for (const dirent of dirents) {
+      if (SKIPPED_DIRECTORIES.has(dirent.name)) continue;
+      const absolutePath = join4(current, dirent.name);
+      const path = prefix === "" ? dirent.name : `${prefix}/${dirent.name}`;
+      if (dirent.isSymbolicLink()) {
+        throw new Error(
+          `contains unsupported symlink "${path}"; create links with setup commands instead`
+        );
+      }
+      if (dirent.isDirectory()) {
+        entries.push({ absolutePath, executable: false, path, type: "directory" });
+        visit(absolutePath, path);
+      } else if (dirent.isFile()) {
+        const executable = (statSync(absolutePath).mode & 73) !== 0;
+        entries.push({ absolutePath, executable, path, type: "file" });
+      } else {
+        throw new Error(`contains unsupported special file "${path}"`);
+      }
+    }
+  }
+  visit(directory, "");
+  return entries;
+}
+function fixtureIdentityHash(directory, setup) {
+  const hash = createHash2("sha256");
+  if (directory !== void 0) {
+    for (const entry of walkFixtureEntries(directory)) {
+      if (entry.type === "directory") {
+        hash.update(`D\0${entry.path}\0`);
+        continue;
+      }
+      const content = readFileSync4(entry.absolutePath);
+      hash.update(`F\0${entry.path}\0${entry.executable ? "x" : "-"}\0${content.byteLength}\0`);
+      hash.update(content);
+    }
+  }
+  hash.update(`S\0${JSON.stringify(setup)}`);
+  return hash.digest("hex");
+}
+function applyFixture(fixture, workspace, home) {
+  if (fixture.directory !== void 0) {
+    cpSync(fixture.directory, workspace, {
+      filter: (source) => !SKIPPED_DIRECTORIES.has(basename(source)),
+      recursive: true,
+      verbatimSymlinks: true
+    });
+  }
+  const results = [];
+  for (const command of fixture.setup) {
+    const outcome = spawnSync3(command, {
+      cwd: workspace,
+      encoding: "utf8",
+      env: { HOME: home, PATH: process.env.PATH ?? "" },
+      killSignal: "SIGKILL",
+      shell: true,
+      timeout: SETUP_COMMAND_TIMEOUT_MS
+    });
+    results.push({
+      command,
+      exitCode: outcome.status,
+      signal: outcome.signal ?? null,
+      stderr: outcome.stderr ?? "",
+      stdout: outcome.stdout ?? ""
+    });
+    if (outcome.error !== void 0 || outcome.status !== 0) {
+      const timedOut = outcome.error?.code === "ETIMEDOUT";
+      const reason = timedOut ? `timed out after ${SETUP_COMMAND_TIMEOUT_MS / 1e3}s` : outcome.error?.message ?? (outcome.signal !== null ? `terminated by ${outcome.signal}` : `exit code ${outcome.status}`);
+      throw new FixtureSetupError(`fixture setup failed: "${command}" (${reason})`, results);
+    }
+  }
+  return results;
+}
+
+// src/case-file.ts
+var CaseFileError = class extends CaseContractError {
+  constructor(message, code = "CASE_FILE_INVALID") {
+    super(message, code);
+    this.name = "CaseFileError";
+  }
+};
+function readCaseFile(path, expectedSkill) {
+  const evals = parseCaseFile(readFileSync5(path, "utf8"), path, expectedSkill);
+  const baseDirectory = dirname2(path);
+  assertFixtureDirectory(evals.fixture, `${path} suite fixture`, baseDirectory);
+  for (const evalCase of evals.cases) {
+    assertFixtureDirectory(
+      evalCase.fixture,
+      `${path} case "${evalCase.id}" fixture`,
+      baseDirectory
+    );
+  }
+  return evals;
+}
+function assertFixtureDirectory(fixture, subject, baseDirectory) {
+  if (fixture?.path === void 0) return;
+  const directory = join5(baseDirectory, fixture.path);
+  const stats = statSync2(directory, { throwIfNoEntry: false });
+  if (stats === void 0) {
+    throw new CaseFileError(
+      `${subject} path "${fixture.path}" does not exist`,
+      "CASE_FIXTURE_INVALID"
+    );
+  }
+  if (!stats.isDirectory()) {
+    throw new CaseFileError(
+      `${subject} path "${fixture.path}" is not a directory`,
+      "CASE_FIXTURE_INVALID"
+    );
+  }
+  try {
+    walkFixtureEntries(directory);
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new CaseFileError(`${subject} path "${fixture.path}" ${detail}`, "CASE_FIXTURE_INVALID");
+  }
+}
+function parseCaseFile(source, path = "skillval.yml", expectedSkill) {
+  let parsed;
+  try {
+    parsed = parseYaml2(source);
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new CaseFileError(`${path} YAML is invalid: ${detail}`, "CASE_FILE_YAML_INVALID");
+  }
+  try {
+    return parseCaseValue(parsed, path, expectedSkill);
+  } catch (error) {
+    if (error instanceof CaseContractError) {
+      throw new CaseFileError(error.message, error.code);
+    }
+    throw error;
+  }
+}
+
+// src/discovery.ts
+function discoverSkills(roots) {
+  const missingRoots = [];
+  const skills = [];
+  for (const root of roots) {
+    if (!existsSync3(root)) {
+      missingRoots.push(root);
+      continue;
+    }
+    const entries = readdirSync3(root, { withFileTypes: true }).filter((entry) => entry.isDirectory()).sort((left, right) => left.name.localeCompare(right.name));
+    for (const entry of entries) {
+      const skillDirectory = join6(root, entry.name);
+      if (!existsSync3(join6(skillDirectory, "SKILL.md"))) continue;
+      skills.push(describeSkill(entry.name, root, skillDirectory));
+    }
+  }
+  return { missingRoots, skills };
+}
+function discoveryReport(discovery) {
+  return {
+    missingRoots: discovery.missingRoots,
+    skills: discovery.skills.map((skill) => {
+      if (skill.status !== "ready") return skill;
+      return {
+        caseCount: skill.caseCount,
+        class: skill.class,
+        hasSkillval: skill.hasSkillval,
+        name: skill.name,
+        root: skill.root,
+        skillDirectory: skill.skillDirectory,
+        status: skill.status,
+        validationError: skill.validationError
+      };
+    })
+  };
+}
+function selectSkills(discovery, requestedNames) {
+  const byName = /* @__PURE__ */ new Map();
+  for (const skill of discovery.skills) {
+    if (!byName.has(skill.name)) byName.set(skill.name, skill);
+  }
+  if (requestedNames.length === 0) {
+    return [...byName.values()].filter(
+      (skill) => skill.status === "ready"
+    );
+  }
+  return requestedNames.map((name) => {
+    const skill = byName.get(name);
+    if (skill === void 0) {
+      throw new Error(`skill "${name}" not found under configured roots`);
+    }
+    if (skill.status === "missing") {
+      throw new Error(`skill "${name}" has no skillval.yml`);
+    }
+    if (skill.status === "invalid") {
+      throw new Error(skill.validationError);
+    }
+    return skill;
+  });
+}
+function describeSkill(name, root, skillDirectory) {
+  const caseFilePath = join6(skillDirectory, "skillval.yml");
+  if (!existsSync3(caseFilePath)) {
+    return {
+      caseCount: 0,
+      class: void 0,
+      hasSkillval: false,
+      name,
+      root,
+      skillDirectory,
+      status: "missing",
+      validationError: void 0
+    };
+  }
+  let evals;
+  try {
+    evals = readCaseFile(caseFilePath, name);
+  } catch (error) {
+    return {
+      caseCount: 0,
+      class: "invalid",
+      hasSkillval: true,
+      name,
+      root,
+      skillDirectory,
+      status: "invalid",
+      validationError: error instanceof Error ? error.message : String(error)
+    };
+  }
+  return {
+    caseCount: evals.cases.length,
+    class: evals.class,
+    evals,
+    hasSkillval: true,
+    name,
+    root,
+    skillDirectory,
+    status: "ready",
+    validationError: void 0
+  };
+}
+
+// src/runner.ts
+import { mkdirSync as mkdirSync3, mkdtempSync, rmSync, writeFileSync as writeFileSync3 } from "fs";
+import { tmpdir } from "os";
+import { join as join8 } from "path";
+
+// src/cache.ts
+import { existsSync as existsSync4, mkdirSync as mkdirSync2, readFileSync as readFileSync6, writeFileSync as writeFileSync2 } from "fs";
+import { dirname as dirname3, join as join7 } from "path";
+var RUNNER_VERSION = 6;
+var ArmCache = class {
+  #stateDirectory;
+  constructor(stateDirectory = resolveStateDirectory()) {
+    this.#stateDirectory = stateDirectory;
+  }
+  lookup(identity) {
+    const path = this.#path(identity);
+    if (!existsSync4(path)) return void 0;
+    return { ...JSON.parse(readFileSync6(path, "utf8")), cached: true };
+  }
+  store(identity, result) {
+    const path = this.#path(identity);
+    mkdirSync2(dirname3(path), { recursive: true });
+    writeFileSync2(path, JSON.stringify(result));
+  }
+  #path(identity) {
+    const parts = [
+      String(RUNNER_VERSION),
+      identity.skillHash,
+      JSON.stringify(identity.evalCase),
+      identity.arm,
+      identity.executor.name,
+      identity.executor.version,
+      identity.executor.model
+    ];
+    if (identity.fixtureHash !== void 0) parts.push(identity.fixtureHash);
+    const key = sha256(parts.join("\0"));
+    return join7(this.#stateDirectory, "cache", `${key}.json`);
+  }
+};
+
+// src/grade.ts
+import { readFileSync as readFileSync7 } from "fs";
+import { relative as relative2 } from "path";
+var INJECTED_FILES = /* @__PURE__ */ new Set(["package.json", "tsconfig.json"]);
+function gradeTrial(evalCase, arm, trace, workspace) {
+  const checks = [];
+  checks.push({
+    detail: trace.completed ? "turn.completed seen" : "no turn.completed in trace",
+    name: "trace",
+    pass: trace.completed
+  });
+  if (evalCase.should_trigger !== void 0 && arm === "skill") {
+    checks.push({
+      detail: `invoked=${trace.invoked}, expected=${evalCase.should_trigger}`,
+      name: "trigger",
+      pass: trace.invoked === evalCase.should_trigger
+    });
+  }
+  const gradedText = evalCase.mode === "generation" ? walkFiles(workspace).filter((file) => !INJECTED_FILES.has(relative2(workspace, file))).map((file) => `=== ${relative2(workspace, file)} ===
+${readFileSync7(file, "utf8")}`).join("\n") : trace.agentText;
+  for (const pattern of evalCase.assert?.must_match ?? []) {
+    const pass = new RegExp(pattern, "m").test(gradedText);
+    checks.push({
+      detail: pass ? pattern : `${pattern} | got: ${gradedText.slice(0, 400)}`,
+      name: "must_match",
+      pass
+    });
+  }
+  for (const pattern of evalCase.assert?.must_not_match ?? []) {
+    checks.push({
+      detail: pattern,
+      name: "must_not_match",
+      pass: !new RegExp(pattern, "m").test(gradedText)
+    });
+  }
+  checks.push(...runGraders(evalCase, workspace));
+  return checks;
+}
+
+// src/vote.ts
+function clampedTrialCount(configured) {
+  return Math.min(5, Math.max(1, configured ?? 1));
+}
+function hasMajority(trials) {
+  const passes = trials.filter((trial) => trial.pass).length;
+  return passes * 2 > trials.length;
+}
+function shouldEscalate(trials) {
+  return new Set(trials.map((trial) => trial.pass)).size > 1 && trials.length < 5;
+}
+
+// src/runner.ts
+function runEvaluation(config, options, log) {
+  const discovery = discoverSkills(config.roots);
+  const selectedSkills = selectSkills(discovery, options.requestedSkills);
+  const executor = createExecutor(config.executor);
+  const stateDirectory = resolveStateDirectory();
+  const cache = new ArmCache(stateDirectory);
+  const skillInputs = selectedSkills.map((skill) => ({
+    contentHash: skillContentHash(skill.skillDirectory),
+    skill
+  }));
+  const runHash = participatingSkillsHash(
+    skillInputs.map(({ contentHash, skill }) => ({ contentHash, name: skill.name }))
+  );
+  const skillReports = {};
+  let failures = 0;
+  let noops = 0;
+  for (const { contentHash, skill } of skillInputs) {
+    const evals = skill.evals;
+    log(`${skill.name} (${evals.class}, ${contentHash.slice(0, 12)}):`);
+    const cases = evals.cases.filter((evalCase) => options.caseFilter === void 0 || evalCase.id === options.caseFilter).map(
+      (evalCase) => runCase(
+        {
+          cache,
+          executor,
+          skill,
+          skillHash: contentHash,
+          skipBaseline: options.skipBaseline,
+          useCache: options.useCache
+        },
+        evalCase,
+        log
+      )
+    );
+    skillReports[skill.name] = { cases, class: evals.class, contentHash };
+    failures += cases.filter((result) => !result.pass).length;
+    noops += cases.filter((result) => result.noop).length;
+  }
+  const report = {
+    executor: executor.metadata,
+    runHash,
+    skills: skillReports
+  };
+  const reportDirectory = join8(stateDirectory, "reports");
+  const reportPath = join8(reportDirectory, `${runHash}.json`);
+  mkdirSync3(reportDirectory, { recursive: true });
+  writeFileSync3(reportPath, JSON.stringify(report, null, 2));
+  return { failures, noops, report, reportPath };
+}
+function participatingSkillsHash(skills) {
+  return sha256(
+    [...skills].sort((left, right) => left.name.localeCompare(right.name)).map(({ contentHash, name }) => `${name}\0${contentHash}`).join("\0")
+  );
+}
+function runCase(context, evalCase, log) {
+  const arms = (evalCase.arms ?? ["skill"]).filter(
+    (arm) => arm === "skill" || !context.skipBaseline
+  );
+  const fixture = resolveFixture(
+    selectFixture(evalCase.fixture, context.skill.evals.fixture),
+    context.skill.skillDirectory
+  );
+  const results = arms.map((arm) => {
+    log(`  ${evalCase.id} [${arm}] ...`);
+    const result = runArm(
+      {
+        cache: context.cache,
+        evalCase,
+        executor: context.executor,
+        fixture,
+        skill: context.skill,
+        skillHash: context.skillHash,
+        useCache: context.useCache
+      },
+      arm
+    );
+    log(
+      `  ${evalCase.id} [${arm}] ${result.pass ? "pass" : "FAIL"}${result.cached ? " (cached)" : ""}`
+    );
+    return result;
+  });
+  return {
+    arms: results,
+    id: evalCase.id,
+    noop: results.find((result) => result.arm === "baseline")?.pass === true,
+    pass: results.find((result) => result.arm === "skill")?.pass === true,
+    rule: evalCase.rule
+  };
+}
+function runArm(context, arm) {
+  const identity = {
+    arm,
+    evalCase: context.evalCase,
+    executor: context.executor.metadata,
+    fixtureHash: context.fixture?.hash,
+    skillHash: context.skillHash
+  };
+  if (context.useCache) {
+    const hit = context.cache.lookup(identity);
+    if (hit !== void 0) return hit;
+  }
+  const trials = [];
+  const wanted = clampedTrialCount(context.evalCase.trials);
+  for (let index = 0; index < wanted; index += 1) {
+    trials.push(runTrial(context, arm));
+  }
+  while (shouldEscalate(trials)) trials.push(runTrial(context, arm));
+  const result = {
+    arm,
+    cached: false,
+    pass: hasMajority(trials),
+    trials
+  };
+  context.cache.store(identity, result);
+  return result;
+}
+function runTrial(context, arm) {
+  const workspace = mkdtempSync(join8(tmpdir(), `skillval-${context.evalCase.id}-`));
+  const trialHome = mkdtempSync(join8(tmpdir(), "skillval-home-"));
+  try {
+    const fixtureSetup = context.fixture === void 0 ? void 0 : applyFixture(context.fixture, workspace, trialHome);
+    const trace = context.executor.runTrial({
+      arm,
+      evalCase: context.evalCase,
+      home: trialHome,
+      skillDirectory: context.skill.skillDirectory,
+      skillName: context.skill.name,
+      workspace
+    });
+    const checks = gradeTrial(context.evalCase, arm, trace, workspace);
+    return {
+      checks,
+      fixtureSetup,
+      pass: checks.every((check) => check.pass),
+      usage: trace.usage
+    };
+  } catch (error) {
+    if (error instanceof FixtureSetupError) {
+      return {
+        checks: [{ detail: error.message, name: "fixture-setup", pass: false }],
+        fixtureSetup: error.results,
+        pass: false,
+        usage: void 0
+      };
+    }
+    const detail = error instanceof Error ? error.message : String(error);
+    return {
+      checks: [{ detail, name: "run", pass: false }],
+      pass: false,
+      usage: void 0
+    };
+  } finally {
+    rmSync(workspace, { force: true, recursive: true });
+    rmSync(trialHome, { force: true, recursive: true });
+  }
+}
+
+// src/cli.ts
+var program = new Command();
+program.name("skillval").description("Evaluate agent skills with deterministic graders").configureHelp({ showGlobalOptions: true }).version("0.1.0").option("--config <path>", "read configuration from this path");
+program.command("run").description(
+  "Run selected cases and return a report whose exit status fails when any skill arm fails"
+).argument("[skill...]", "skill names; omit to run every skill with skillval.yml").option("--case <id>", "run only the case with this id").option("--no-cache", "ignore cached arm results").option("--skip-baseline", "do not run baseline arms").option("--json", "return the complete report as JSON").action((skills, options, command) => {
+  const globalOptions = command.optsWithGlobals();
+  const configPath = resolveConfigPath({ cliPath: globalOptions.config });
+  const config = loadConfig(configPath);
+  const outcome = runEvaluation(
+    config,
+    {
+      caseFilter: options.case,
+      requestedSkills: skills,
+      skipBaseline: options.skipBaseline === true,
+      useCache: options.cache
+    },
+    options.json === true ? () => void 0 : (message) => console.log(message)
+  );
+  if (options.json === true) {
+    console.log(JSON.stringify(outcome.report, null, 2));
+  } else {
+    console.log(`report: ${outcome.reportPath}`);
+    if (outcome.noops > 0) {
+      console.log(`no-op alert: ${outcome.noops} case(s) pass at baseline - prune candidates`);
+    }
+    console.log(
+      outcome.failures === 0 ? "all cases passed" : `${outcome.failures} case(s) FAILED`
+    );
+  }
+  process.exitCode = outcome.failures > 0 ? 1 : 0;
+});
+program.command("list").description(
+  "Return discovered skills, evaluation metadata, and configured roots that are missing"
+).option("--json", "return the complete discovery result as JSON").action((options, command) => {
+  const globalOptions = command.optsWithGlobals();
+  const configPath = resolveConfigPath({ cliPath: globalOptions.config });
+  const config = loadConfig(configPath);
+  const discovery = discoverSkills(config.roots);
+  if (options.json === true) {
+    console.log(JSON.stringify(discoveryReport(discovery), null, 2));
+    return;
+  }
+  printSkillTable(discovery.skills);
+  for (const skill of discovery.skills) {
+    if (skill.status === "invalid") console.log(`invalid skill: ${skill.validationError}`);
+  }
+  for (const root of discovery.missingRoots) console.log(`missing root: ${root}`);
+});
+async function main() {
+  try {
+    await program.parseAsync(process.argv);
+  } catch (error) {
+    console.error(`error: ${error instanceof Error ? error.message : String(error)}`);
+    process.exitCode = 1;
+  }
+}
+function printSkillTable(skills) {
+  const rows = [
+    ["NAME", "ROOT", "CLASS", "CASES", "HAS-SKILLVAL.YML"],
+    ...skills.map((skill) => [
+      skill.name,
+      skill.root,
+      skill.class ?? "-",
+      String(skill.caseCount),
+      skill.hasSkillval ? "true" : "false"
+    ])
+  ];
+  const widths = rows[0]?.map(
+    (_, column) => Math.max(...rows.map((row) => row[column]?.length ?? 0))
+  );
+  for (const row of rows) {
+    console.log(
+      row.map((cell, column) => cell.padEnd(widths?.[column] ?? cell.length)).join("  ").trimEnd()
+    );
+  }
+}
+await main();
+export {
+  main
+};
+//# sourceMappingURL=cli.js.map