@dungle-scrubs/skillval 0.1.0

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cli.ts","../src/config.ts","../src/config-contract.ts","../src/executors/codex.ts","../src/utils.ts","../src/executors/index.ts","../src/discovery.ts","../src/case-file.ts","../src/case-contract.ts","../src/graders.ts","../src/fixture.ts","../src/runner.ts","../src/cache.ts","../src/grade.ts","../src/vote.ts"],"sourcesContent":["#!/usr/bin/env node\n\n/** Defines the command-line transport and renders runner and discovery results. */\nimport { Command } from \"commander\";\nimport { loadConfig, resolveConfigPath } from \"./config.js\";\nimport type { DiscoveredSkill } from \"./discovery.js\";\nimport { discoverSkills, discoveryReport } from \"./discovery.js\";\nimport { runEvaluation } from \"./runner.js\";\n\ninterface GlobalOptions {\n  readonly config?: string;\n}\n\ninterface ListOptions {\n  readonly json?: boolean;\n}\n\ninterface RunCommandOptions {\n  readonly cache: boolean;\n  readonly case?: string;\n  readonly json?: boolean;\n  readonly skipBaseline?: boolean;\n}\n\nconst program = new Command();\n\nprogram\n  .name(\"skillval\")\n  .description(\"Evaluate agent skills with deterministic graders\")\n  .configureHelp({ showGlobalOptions: true })\n  .version(\"0.1.0\") // x-release-please-version\n  .option(\"--config <path>\", \"read configuration from this path\");\n\nprogram\n  .command(\"run\")\n  .description(\n    \"Run selected cases and return a report whose exit status fails when any skill arm fails\",\n  )\n  .argument(\"[skill...]\", \"skill names; omit to run every skill with skillval.yml\")\n  .option(\"--case <id>\", \"run only the case with this id\")\n  .option(\"--no-cache\", \"ignore cached arm results\")\n  .option(\"--skip-baseline\", \"do not run baseline arms\")\n  .option(\"--json\", \"return the complete report as JSON\")\n  .action((skills: string[], options: RunCommandOptions, command: Command): void => {\n    const globalOptions = command.optsWithGlobals() as GlobalOptions & RunCommandOptions;\n    const configPath = resolveConfigPath({ cliPath: globalOptions.config });\n    const config = loadConfig(configPath);\n    const outcome = runEvaluation(\n      config,\n      {\n        caseFilter: options.case,\n        requestedSkills: skills,\n        skipBaseline: options.skipBaseline === true,\n        useCache: options.cache,\n      },\n      options.json === true ? () => undefined : (message) => console.log(message),\n    );\n\n    if (options.json === true) {\n      console.log(JSON.stringify(outcome.report, null, 2));\n    } else {\n      console.log(`report: ${outcome.reportPath}`);\n      if (outcome.noops > 0) {\n        console.log(`no-op alert: ${outcome.noops} case(s) pass at baseline - prune candidates`);\n      }\n      console.log(\n        outcome.failures === 0 ? \"all cases passed\" : `${outcome.failures} case(s) FAILED`,\n      );\n    }\n    process.exitCode = outcome.failures > 0 ? 1 : 0;\n  });\n\nprogram\n  .command(\"list\")\n  .description(\n    \"Return discovered skills, evaluation metadata, and configured roots that are missing\",\n  )\n  .option(\"--json\", \"return the complete discovery result as JSON\")\n  .action((options: ListOptions, command: Command): void => {\n    const globalOptions = command.optsWithGlobals() as GlobalOptions & ListOptions;\n    const configPath = resolveConfigPath({ cliPath: globalOptions.config });\n    const config = loadConfig(configPath);\n    const discovery = discoverSkills(config.roots);\n    if (options.json === true) {\n      console.log(JSON.stringify(discoveryReport(discovery), null, 2));\n      return;\n    }\n\n    printSkillTable(discovery.skills);\n    for (const skill of discovery.skills) {\n      if (skill.status === \"invalid\") console.log(`invalid skill: ${skill.validationError}`);\n    }\n    for (const root of discovery.missingRoots) console.log(`missing root: ${root}`);\n  });\n\nexport async function main(): Promise<void> {\n  try {\n    await program.parseAsync(process.argv);\n  } catch (error) {\n    console.error(`error: ${error instanceof Error ? error.message : String(error)}`);\n    process.exitCode = 1;\n  }\n}\n\nfunction printSkillTable(skills: readonly DiscoveredSkill[]): void {\n  const rows = [\n    [\"NAME\", \"ROOT\", \"CLASS\", \"CASES\", \"HAS-SKILLVAL.YML\"],\n    ...skills.map((skill) => [\n      skill.name,\n      skill.root,\n      skill.class ?? \"-\",\n      String(skill.caseCount),\n      skill.hasSkillval ? \"true\" : \"false\",\n    ]),\n  ];\n  const widths = rows[0]?.map((_, column) =>\n    Math.max(...rows.map((row) => row[column]?.length ?? 0)),\n  );\n  for (const row of rows) {\n    console.log(\n      row\n        .map((cell, column) => cell.padEnd(widths?.[column] ?? cell.length))\n        .join(\"  \")\n        .trimEnd(),\n    );\n  }\n}\n\nawait main();\n","/** Resolves configuration and state paths, then loads validated configuration values. */\nimport { existsSync, readFileSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { resolve } from \"node:path\";\nimport { Check as checkSchema, Errors as schemaErrors } from \"typebox/value\";\nimport { parse as parseYaml } from \"yaml\";\nimport type { ConfigFile } from \"./config-contract.js\";\nimport { configFileSchema } from \"./config-contract.js\";\n\nexport type SkillvalConfig = ConfigFile;\n\nexport interface ConfigPathOptions {\n  readonly cliPath?: string;\n  readonly environment?: NodeJS.ProcessEnv;\n  readonly home?: string;\n}\n\nexport class ConfigError extends Error {\n  public readonly code: string;\n\n  public constructor(message: string, code = \"CONFIG_ERROR\") {\n    super(message);\n    this.code = code;\n    this.name = \"ConfigError\";\n  }\n}\n\nexport function resolveConfigPath(options: ConfigPathOptions = {}): string {\n  const environment = options.environment ?? process.env;\n  const home = options.home ?? homedir();\n  const selected =\n    options.cliPath ??\n    environment.SKILLVAL_CONFIG ??\n    (environment.XDG_CONFIG_HOME\n      ? resolve(environment.XDG_CONFIG_HOME, \"skillval/config.yml\")\n      : resolve(home, \".config/skillval/config.yml\"));\n  return resolve(expandHome(selected, home));\n}\n\nexport function expandRoot(root: string, home = homedir()): string {\n  return resolve(expandHome(root, home));\n}\n\nexport function loadConfig(path: string, home = homedir()): SkillvalConfig {\n  if (!existsSync(path)) {\n    throw new ConfigError(`config file not found: ${path}`, \"CONFIG_NOT_FOUND\");\n  }\n\n  let parsed: unknown;\n  try {\n    parsed = parseYaml(readFileSync(path, \"utf8\"));\n  } catch (error) {\n    const detail = error instanceof Error ? error.message : String(error);\n    throw new ConfigError(`invalid YAML in ${path}: ${detail}`, \"CONFIG_YAML_INVALID\");\n  }\n\n  if (!checkSchema(configFileSchema, parsed)) {\n    const [firstError] = schemaErrors(configFileSchema, parsed);\n    const location = firstError?.instancePath.replaceAll(\"/\", \".\").replace(/^\\./, \"\");\n    const subject = location === undefined || location === \"\" ? path : `${path} ${location}`;\n    throw new ConfigError(`${subject} ${firstError?.message ?? \"is invalid\"}`);\n  }\n\n  return {\n    executor: parsed.executor,\n    roots: parsed.roots.map((root) => expandRoot(root, home)),\n  };\n}\n\nexport function resolveStateDirectory(\n  environment: NodeJS.ProcessEnv = process.env,\n  home = homedir(),\n): string {\n  return environment.XDG_STATE_HOME\n    ? resolve(environment.XDG_STATE_HOME, \"skillval\")\n    : resolve(home, \".local/state/skillval\");\n}\n\nfunction expandHome(value: string, home: string): string {\n  const withTilde =\n    value === \"~\" ? home : value.startsWith(\"~/\") ? `${home}${value.slice(1)}` : value;\n  return withTilde.replaceAll(String.raw`\\${HOME}`, home).replaceAll(\"$HOME\", home);\n}\n","/** Defines the configuration contract shared by runtime validation and generated JSON Schema. */\nimport type { Static } from \"typebox\";\nimport Type from \"typebox\";\nimport { EXECUTOR_NAMES } from \"./executors/index.js\";\n\nexport const configFileSchema = Type.ReadonlyObject(\n  Type.Object({\n    // Names come from the executor registry so configuration cannot advertise a missing adapter.\n    executor: Type.Enum(EXECUTOR_NAMES, {\n      description: \"Trial executor.\",\n    }),\n    roots: Type.Readonly(\n      Type.Array(Type.String({ minLength: 1, pattern: String.raw`\\S` }), {\n        description: \"Directories whose immediate children are agent skill directories.\",\n      }),\n    ),\n  }),\n  {\n    $id: \"https://raw.githubusercontent.com/dungle-scrubs/skillval/main/schemas/config.schema.json\",\n    $schema: \"https://json-schema.org/draft/2020-12/schema\",\n    additionalProperties: false,\n    title: \"skillval configuration\",\n  },\n);\n\nexport type ConfigFile = Static<typeof configFileSchema>;\n","/** Implements Codex-specific skill seeding, process isolation, invocation, and trace parsing. */\nimport { spawnSync } from \"node:child_process\";\nimport { mkdirSync, readFileSync, symlinkSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport type { Trace } from \"../types.js\";\nimport { isRecord } from \"../utils.js\";\nimport type { Executor, ExecutorMetadata, TrialRequest } from \"./types.js\";\n\nconst TRIAL_TIMEOUT_MS = 15 * 60 * 1000;\n\nexport class CodexExecutor implements Executor {\n  public readonly metadata: ExecutorMetadata;\n  readonly #realHome: string;\n\n  public constructor(realHome = homedir()) {\n    this.#realHome = realHome;\n    this.metadata = detectCodex(realHome);\n  }\n\n  public runTrial(request: TrialRequest): Trace {\n    if (request.arm === \"skill\") seedSkill(request);\n    const sandbox = request.evalCase.mode === \"generation\" ? \"workspace-write\" : \"read-only\";\n    // Baselines need the real Codex configuration for authentication and model selection, but an\n    // empty HOME prevents globally installed skills from influencing the comparison arm.\n    const environment =\n      request.arm === \"baseline\"\n        ? {\n            ...process.env,\n            CODEX_HOME: join(this.#realHome, \".codex\"),\n            HOME: request.home,\n          }\n        : { ...process.env };\n    const result = spawnSync(\n      \"codex\",\n      [\n        \"exec\",\n        \"--json\",\n        \"--skip-git-repo-check\",\n        \"--ephemeral\",\n        \"-s\",\n        sandbox,\n        \"-C\",\n        request.workspace,\n        request.evalCase.prompt,\n      ],\n      {\n        encoding: \"utf8\",\n        env: environment,\n        maxBuffer: 64 * 1024 * 1024,\n        timeout: TRIAL_TIMEOUT_MS,\n      },\n    );\n    if (result.status !== 0) {\n      throw new Error(`codex exec exited ${result.status}: ${result.stderr?.slice(-500)}`);\n    }\n\n    return parseCodexTrace(result.stdout, request.skillName);\n  }\n}\n\nfunction seedSkill(request: TrialRequest): void {\n  // Skill installation paths are provider knowledge and intentionally stay inside this adapter.\n  const skillsRoot = join(request.workspace, \".agents/skills\");\n  mkdirSync(skillsRoot, { recursive: true });\n  symlinkSync(request.skillDirectory, join(skillsRoot, request.skillName));\n}\n\nexport function detectCodex(realHome = homedir()): ExecutorMetadata {\n  const version = spawnSync(\"codex\", [\"--version\"], { encoding: \"utf8\" }).stdout?.trim() ?? \"\";\n  if (version === \"\") throw new Error(\"codex CLI not found on PATH\");\n  const model =\n    readFileSync(join(realHome, \".codex/config.toml\"), \"utf8\").match(\n      /^model\\s*=\\s*\"([^\"]+)\"/m,\n    )?.[1] ?? \"default\";\n  return { model, name: \"codex\", version };\n}\n\nexport function parseCodexTrace(stdout: string, skillName: string): Trace {\n  let completed = false;\n  let invoked = false;\n  const texts: string[] = [];\n  let usage: unknown;\n\n  for (const line of stdout.split(\"\\n\")) {\n    if (line.trim() === \"\") continue;\n    let event: unknown;\n    try {\n      event = JSON.parse(line);\n    } catch {\n      continue;\n    }\n    if (!isRecord(event)) continue;\n    const item = isRecord(event.item) ? event.item : undefined;\n    if (\n      item?.type === \"command_execution\" &&\n      typeof item.command === \"string\" &&\n      item.command.includes(`${skillName}/SKILL.md`)\n    ) {\n      invoked = true;\n    }\n    if (\n      event.type === \"item.completed\" &&\n      item?.type === \"agent_message\" &&\n      typeof item.text === \"string\"\n    ) {\n      texts.push(item.text);\n    }\n    if (event.type === \"turn.completed\") {\n      completed = true;\n      usage = event.usage;\n    }\n  }\n\n  return { agentText: texts.join(\"\\n\"), completed, invoked, usage };\n}\n","import { createHash } from \"node:crypto\";\nimport { readdirSync, readFileSync } from \"node:fs\";\nimport { join, relative } from \"node:path\";\n\nexport function sha256(input: string): string {\n  return createHash(\"sha256\").update(input).digest(\"hex\");\n}\n\n// Directories that never contribute to content identity or workspace materialization.\nexport const SKIPPED_DIRECTORIES: ReadonlySet<string> = new Set([\".git\", \"node_modules\"]);\n\nexport function walkFiles(root: string): string[] {\n  const files: string[] = [];\n\n  function visit(directory: string): void {\n    for (const entry of readdirSync(directory, { withFileTypes: true })) {\n      if (SKIPPED_DIRECTORIES.has(entry.name)) continue;\n      const path = join(directory, entry.name);\n      if (entry.isDirectory()) visit(path);\n      else if (entry.isFile()) files.push(path);\n    }\n  }\n\n  visit(root);\n  return files.sort();\n}\n\nexport function skillContentHash(skillDirectory: string): string {\n  const parts = walkFiles(skillDirectory).map(\n    (file) => `${relative(skillDirectory, file)}\\n${readFileSync(file, \"utf8\")}`,\n  );\n  return sha256(parts.join(\"\\0\"));\n}\n\nexport function isRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n","/** Registers executor adapters and creates them from validated configuration names. */\nimport { CodexExecutor } from \"./codex.js\";\nimport type { Executor } from \"./types.js\";\n\nconst executorFactories = {\n  codex: (): Executor => new CodexExecutor(),\n};\n\n// Both the configuration schema and factory derive from this registry.\nexport type ExecutorName = keyof typeof executorFactories;\nexport const EXECUTOR_NAMES = Object.keys(executorFactories) as ExecutorName[];\n\nexport function createExecutor(name: ExecutorName): Executor {\n  return executorFactories[name]();\n}\n","/** Discovers skills and carries validated case suites into evaluation without reparsing them. */\nimport { existsSync, readdirSync } from \"node:fs\";\nimport { join } from \"node:path\";\nimport { readCaseFile } from \"./case-file.js\";\nimport type { SkillEvals } from \"./types.js\";\n\ninterface SkillLocation {\n  readonly name: string;\n  readonly root: string;\n  readonly skillDirectory: string;\n}\n\nexport interface MissingDiscoveredSkill extends SkillLocation {\n  readonly caseCount: 0;\n  readonly class: undefined;\n  readonly hasSkillval: false;\n  readonly status: \"missing\";\n  readonly validationError: undefined;\n}\n\nexport interface InvalidDiscoveredSkill extends SkillLocation {\n  readonly caseCount: 0;\n  readonly class: \"invalid\";\n  readonly hasSkillval: true;\n  readonly status: \"invalid\";\n  readonly validationError: string;\n}\n\nexport interface ReadyDiscoveredSkill extends SkillLocation {\n  readonly caseCount: number;\n  readonly class: \"capability\" | \"preference\";\n  readonly evals: SkillEvals;\n  readonly hasSkillval: true;\n  readonly status: \"ready\";\n  readonly validationError: undefined;\n}\n\nexport type DiscoveredSkill =\n  | InvalidDiscoveredSkill\n  | MissingDiscoveredSkill\n  | ReadyDiscoveredSkill;\n\nexport interface DiscoveryResult {\n  readonly missingRoots: readonly string[];\n  readonly skills: readonly DiscoveredSkill[];\n}\n\nexport interface DiscoveryReport {\n  readonly missingRoots: readonly string[];\n  readonly skills: readonly DiscoverySkillSummary[];\n}\n\nexport type DiscoverySkillSummary =\n  | Omit<ReadyDiscoveredSkill, \"evals\">\n  | Exclude<DiscoveredSkill, ReadyDiscoveredSkill>;\n\nexport function discoverSkills(roots: readonly string[]): DiscoveryResult {\n  const missingRoots: string[] = [];\n  const skills: DiscoveredSkill[] = [];\n\n  for (const root of roots) {\n    if (!existsSync(root)) {\n      missingRoots.push(root);\n      continue;\n    }\n\n    const entries = readdirSync(root, { withFileTypes: true })\n      .filter((entry) => entry.isDirectory())\n      .sort((left, right) => left.name.localeCompare(right.name));\n    for (const entry of entries) {\n      const skillDirectory = join(root, entry.name);\n      if (!existsSync(join(skillDirectory, \"SKILL.md\"))) continue;\n      skills.push(describeSkill(entry.name, root, skillDirectory));\n    }\n  }\n\n  return { missingRoots, skills };\n}\n\nexport function discoveryReport(discovery: DiscoveryResult): DiscoveryReport {\n  return {\n    missingRoots: discovery.missingRoots,\n    skills: discovery.skills.map((skill) => {\n      if (skill.status !== \"ready\") return skill;\n      // Parsed prompts are execution input, not listing metadata. Keep list --json compact and\n      // avoid exposing prompt contents when callers only asked what skills are available.\n      return {\n        caseCount: skill.caseCount,\n        class: skill.class,\n        hasSkillval: skill.hasSkillval,\n        name: skill.name,\n        root: skill.root,\n        skillDirectory: skill.skillDirectory,\n        status: skill.status,\n        validationError: skill.validationError,\n      };\n    }),\n  };\n}\n\nexport function selectSkills(\n  discovery: DiscoveryResult,\n  requestedNames: readonly string[],\n): readonly ReadyDiscoveredSkill[] {\n  const byName = new Map<string, DiscoveredSkill>();\n  for (const skill of discovery.skills) {\n    if (!byName.has(skill.name)) byName.set(skill.name, skill);\n  }\n\n  if (requestedNames.length === 0) {\n    return [...byName.values()].filter(\n      (skill): skill is ReadyDiscoveredSkill => skill.status === \"ready\",\n    );\n  }\n\n  return requestedNames.map((name) => {\n    const skill = byName.get(name);\n    if (skill === undefined) {\n      throw new Error(`skill \"${name}\" not found under configured roots`);\n    }\n    if (skill.status === \"missing\") {\n      throw new Error(`skill \"${name}\" has no skillval.yml`);\n    }\n    if (skill.status === \"invalid\") {\n      throw new Error(skill.validationError);\n    }\n    return skill;\n  });\n}\n\nfunction describeSkill(name: string, root: string, skillDirectory: string): DiscoveredSkill {\n  const caseFilePath = join(skillDirectory, \"skillval.yml\");\n  if (!existsSync(caseFilePath)) {\n    return {\n      caseCount: 0,\n      class: undefined,\n      hasSkillval: false,\n      name,\n      root,\n      skillDirectory,\n      status: \"missing\",\n      validationError: undefined,\n    };\n  }\n\n  let evals: SkillEvals;\n  try {\n    evals = readCaseFile(caseFilePath, name);\n  } catch (error) {\n    return {\n      caseCount: 0,\n      class: \"invalid\",\n      hasSkillval: true,\n      name,\n      root,\n      skillDirectory,\n      status: \"invalid\",\n      validationError: error instanceof Error ? error.message : String(error),\n    };\n  }\n\n  return {\n    caseCount: evals.cases.length,\n    class: evals.class,\n    evals,\n    hasSkillval: true,\n    name,\n    root,\n    skillDirectory,\n    status: \"ready\",\n    validationError: undefined,\n  };\n}\n","/** Adapts YAML case files into values validated by the executable case contract. */\nimport { readFileSync, statSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { parse as parseYaml } from \"yaml\";\nimport { CaseContractError, parseCaseValue } from \"./case-contract.js\";\nimport { walkFixtureEntries } from \"./fixture.js\";\nimport type { Fixture, SkillEvals } from \"./types.js\";\n\nexport class CaseFileError extends CaseContractError {\n  public constructor(message: string, code = \"CASE_FILE_INVALID\") {\n    super(message, code);\n    this.name = \"CaseFileError\";\n  }\n}\n\nexport function readCaseFile(path: string, expectedSkill?: string): SkillEvals {\n  const evals = parseCaseFile(readFileSync(path, \"utf8\"), path, expectedSkill);\n  // Fixture directories are validated here, not in the pure contract, because only the reader\n  // knows where the case file actually lives on disk.\n  const baseDirectory = dirname(path);\n  assertFixtureDirectory(evals.fixture, `${path} suite fixture`, baseDirectory);\n  for (const evalCase of evals.cases) {\n    assertFixtureDirectory(\n      evalCase.fixture,\n      `${path} case \"${evalCase.id}\" fixture`,\n      baseDirectory,\n    );\n  }\n  return evals;\n}\n\nfunction assertFixtureDirectory(\n  fixture: Fixture | undefined,\n  subject: string,\n  baseDirectory: string,\n): void {\n  if (fixture?.path === undefined) return;\n  const directory = join(baseDirectory, fixture.path);\n  const stats = statSync(directory, { throwIfNoEntry: false });\n  if (stats === undefined) {\n    throw new CaseFileError(\n      `${subject} path \"${fixture.path}\" does not exist`,\n      \"CASE_FIXTURE_INVALID\",\n    );\n  }\n  if (!stats.isDirectory()) {\n    throw new CaseFileError(\n      `${subject} path \"${fixture.path}\" is not a directory`,\n      \"CASE_FIXTURE_INVALID\",\n    );\n  }\n  try {\n    walkFixtureEntries(directory);\n  } catch (error) {\n    const detail = error instanceof Error ? error.message : String(error);\n    throw new CaseFileError(`${subject} path \"${fixture.path}\" ${detail}`, \"CASE_FIXTURE_INVALID\");\n  }\n}\n\nexport function parseCaseFile(\n  source: string,\n  path = \"skillval.yml\",\n  expectedSkill?: string,\n): SkillEvals {\n  let parsed: unknown;\n  try {\n    parsed = parseYaml(source);\n  } catch (error) {\n    const detail = error instanceof Error ? error.message : String(error);\n    throw new CaseFileError(`${path} YAML is invalid: ${detail}`, \"CASE_FILE_YAML_INVALID\");\n  }\n\n  try {\n    return parseCaseValue(parsed, path, expectedSkill);\n  } catch (error) {\n    if (error instanceof CaseContractError) {\n      throw new CaseFileError(error.message, error.code);\n    }\n    throw error;\n  }\n}\n","/** Defines the case-file contract used for static types, runtime validation, and JSON Schema. */\nimport type { Static } from \"typebox\";\nimport Type from \"typebox\";\nimport { Check as checkSchema, Errors as schemaErrors } from \"typebox/value\";\nimport { GRADER_NAMES, graderSupportsMode } from \"./graders.js\";\n\nconst classificationSchema = Type.Enum([\"capability\", \"preference\"]);\nconst nonEmptyStringSchema = Type.String({ minLength: 1, pattern: String.raw`\\S` });\nconst stringArraySchema = Type.Readonly(Type.Array(Type.String()));\nexport const armSchema = Type.Enum([\"baseline\", \"skill\"]);\n\n// These schemas are executable at runtime and are also serialized into schemas/ for editor tooling.\nexport const fixtureSchema = Type.ReadonlyObject(\n  Type.Object({\n    path: Type.Optional(\n      Type.String({\n        description: \"Directory relative to skillval.yml, copied into the workspace.\",\n        minLength: 1,\n        pattern: String.raw`\\S`,\n      }),\n    ),\n    setup: Type.Optional(\n      Type.Readonly(\n        Type.Array(nonEmptyStringSchema, {\n          description: \"Shell commands run sequentially inside the workspace after the copy.\",\n        }),\n      ),\n    ),\n  }),\n  { additionalProperties: false, minProperties: 1 },\n);\n\nexport const caseAssertSchema = Type.ReadonlyObject(\n  Type.Object({\n    graders: Type.Optional(\n      Type.Readonly(Type.Array(Type.Enum(GRADER_NAMES), { uniqueItems: true })),\n    ),\n    must_match: Type.Optional(stringArraySchema),\n    must_not_match: Type.Optional(stringArraySchema),\n  }),\n  { additionalProperties: false },\n);\n\nexport const evalCaseSchema = Type.ReadonlyObject(\n  Type.Object({\n    arms: Type.Optional(\n      Type.Readonly(\n        Type.Array(armSchema, {\n          uniqueItems: true,\n        }),\n      ),\n    ),\n    assert: Type.Optional(caseAssertSchema),\n    fixture: Type.Optional(fixtureSchema),\n    id: nonEmptyStringSchema,\n    mode: Type.Enum([\"generation\", \"trigger\"]),\n    prompt: nonEmptyStringSchema,\n    rule: Type.Optional(Type.String()),\n    should_trigger: Type.Optional(Type.Boolean()),\n    trials: Type.Optional(Type.Integer({ maximum: 5, minimum: 1 })),\n    type: Type.Optional(classificationSchema),\n  }),\n  { additionalProperties: false },\n);\n\nexport const skillEvalsSchema = Type.ReadonlyObject(\n  Type.Object({\n    cases: Type.Readonly(Type.Array(evalCaseSchema)),\n    class: classificationSchema,\n    fixture: Type.Optional(fixtureSchema),\n    skill: nonEmptyStringSchema,\n  }),\n  {\n    $id: \"https://raw.githubusercontent.com/dungle-scrubs/skillval/main/schemas/skillval.schema.json\",\n    $schema: \"https://json-schema.org/draft/2020-12/schema\",\n    additionalProperties: false,\n    title: \"skillval case file\",\n  },\n);\n\nexport type Arm = Static<typeof armSchema>;\nexport type CaseAssert = Static<typeof caseAssertSchema>;\nexport type EvalCase = Static<typeof evalCaseSchema>;\nexport type Fixture = Static<typeof fixtureSchema>;\nexport type SkillEvals = Static<typeof skillEvalsSchema>;\n\nexport class CaseContractError extends Error {\n  public readonly code: string;\n\n  public constructor(message: string, code = \"CASE_FILE_INVALID\") {\n    super(message);\n    this.code = code;\n    this.name = \"CaseContractError\";\n  }\n}\n\nexport function parseCaseValue(value: unknown, path: string, expectedSkill?: string): SkillEvals {\n  if (!checkSchema(skillEvalsSchema, value)) {\n    const [firstError] = schemaErrors(skillEvalsSchema, value);\n    const location = firstError?.instancePath.replaceAll(\"/\", \".\").replace(/^\\./, \"\");\n    if (location?.endsWith(\".trials\")) {\n      const match = /cases\\.(\\d+)\\.trials$/.exec(location);\n      const caseIndex = match?.[1] ?? \"unknown\";\n      throw new CaseContractError(\n        `${path} case at index ${caseIndex} trials must be an integer from 1 through 5`,\n      );\n    }\n    const subject = location === undefined || location === \"\" ? path : `${path} ${location}`;\n    throw new CaseContractError(`${subject} ${firstError?.message ?? \"is invalid\"}`);\n  }\n\n  if (expectedSkill !== undefined && value.skill !== expectedSkill) {\n    throw new CaseContractError(\n      `${path} declares skill \"${value.skill}\", expected \"${expectedSkill}\"`,\n    );\n  }\n\n  const ids = new Set<string>();\n  for (const evalCase of value.cases) {\n    // Semantic rules that depend on sibling cases, JavaScript regex compilation, or runtime\n    // registries remain here. Keeping them beside the structural schema preserves one owner.\n    if (ids.has(evalCase.id)) {\n      throw new CaseContractError(`${path} case id \"${evalCase.id}\" is duplicated`);\n    }\n    ids.add(evalCase.id);\n    validatePatterns(evalCase, path);\n    validateGraders(evalCase, path);\n  }\n  return value;\n}\n\nfunction validateGraders(evalCase: EvalCase, path: string): void {\n  for (const grader of evalCase.assert?.graders ?? []) {\n    if (!graderSupportsMode(grader, evalCase.mode)) {\n      throw new CaseContractError(\n        `${path} case \"${evalCase.id}\" grader \"${grader}\" does not support ${evalCase.mode} mode`,\n      );\n    }\n  }\n}\n\nfunction validatePatterns(evalCase: EvalCase, path: string): void {\n  const fields = [\"must_match\", \"must_not_match\"] as const;\n  for (const field of fields) {\n    for (const pattern of evalCase.assert?.[field] ?? []) {\n      try {\n        new RegExp(pattern, \"m\");\n      } catch (error) {\n        const detail = error instanceof Error ? error.message : String(error);\n        throw new CaseContractError(\n          `${path} case \"${evalCase.id}\" has invalid ${field} regex \"${pattern}\": ${detail}`,\n        );\n      }\n    }\n  }\n}\n","/** Owns the deterministic grader catalog, mode support, and grader implementations. */\nimport { spawnSync } from \"node:child_process\";\nimport { existsSync, writeFileSync } from \"node:fs\";\nimport { createRequire } from \"node:module\";\nimport { dirname, join } from \"node:path\";\n\ntype CaseMode = \"generation\" | \"trigger\";\n\ninterface GradableCase {\n  readonly assert?: {\n    readonly graders?: readonly GraderName[];\n  };\n}\n\ninterface GraderCheck {\n  readonly detail: string;\n  readonly name: string;\n  readonly pass: boolean;\n}\n\ninterface Grader {\n  readonly modes: readonly CaseMode[];\n  run(workspace: string): GraderCheck;\n}\n\nconst packageRequire = createRequire(import.meta.url);\n// This registry is the sole source of grader names, supported modes, and implementations.\nconst graders = {\n  tsc: {\n    modes: [\"generation\"],\n    run: gradeTsc,\n  },\n} satisfies Readonly<Record<string, Grader>>;\n\nexport type GraderName = keyof typeof graders;\n// The case contract derives its allowed grader values from the registry keys.\nexport const GRADER_NAMES = Object.keys(graders) as GraderName[];\n\nexport function graderSupportsMode(name: GraderName, mode: CaseMode): boolean {\n  const modes: readonly CaseMode[] = graders[name].modes;\n  return modes.includes(mode);\n}\n\nexport function runGraders(evalCase: GradableCase, workspace: string): readonly GraderCheck[] {\n  return (evalCase.assert?.graders ?? []).map((name) => graders[name].run(workspace));\n}\n\nfunction gradeTsc(workspace: string): GraderCheck {\n  if (!existsSync(join(workspace, \"package.json\"))) {\n    writeFileSync(join(workspace, \"package.json\"), '{ \"type\": \"module\" }\\n');\n  }\n  const nodeTypesDirectory = dirname(packageRequire.resolve(\"@types/node/package.json\"));\n  writeFileSync(\n    join(workspace, \"tsconfig.json\"),\n    JSON.stringify({\n      compilerOptions: {\n        lib: [\"es2023\"],\n        module: \"esnext\",\n        moduleResolution: \"bundler\",\n        noEmit: true,\n        noUncheckedIndexedAccess: true,\n        strict: true,\n        target: \"es2023\",\n        typeRoots: [dirname(nodeTypesDirectory)],\n        types: [\"node\"],\n      },\n    }),\n  );\n  const typescriptBinary = packageRequire.resolve(\"typescript/bin/tsc\");\n  const result = spawnSync(typescriptBinary, [\"-p\", workspace], {\n    encoding: \"utf8\",\n    timeout: 120_000,\n  });\n  return {\n    detail: result.status === 0 ? \"compiles strict\" : (result.stdout ?? \"\").slice(0, 500),\n    name: \"tsc\",\n    pass: result.status === 0,\n  };\n}\n","/** Prepares trial workspaces from case fixtures: content copy, setup commands, identity hashing. */\nimport { spawnSync } from \"node:child_process\";\nimport { createHash } from \"node:crypto\";\nimport { cpSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { basename, join } from \"node:path\";\nimport type { Fixture } from \"./case-contract.js\";\nimport type { FixtureCommandResult } from \"./types.js\";\nimport { SKIPPED_DIRECTORIES } from \"./utils.js\";\n\n// Setup commands are workspace staging (git init, file moves), not agent work; a minute is ample.\nconst SETUP_COMMAND_TIMEOUT_MS = 60_000;\n\nexport interface ResolvedFixture {\n  readonly directory: string | undefined;\n  readonly hash: string;\n  readonly setup: readonly string[];\n}\n\nexport class FixtureSetupError extends Error {\n  public readonly results: readonly FixtureCommandResult[];\n\n  public constructor(message: string, results: readonly FixtureCommandResult[]) {\n    super(message);\n    this.name = \"FixtureSetupError\";\n    this.results = results;\n  }\n}\n\nexport function selectFixture(\n  caseFixture: Fixture | undefined,\n  suiteFixture: Fixture | undefined,\n): Fixture | undefined {\n  // A per-case fixture replaces the suite default entirely; path and setup never merge.\n  return caseFixture ?? suiteFixture;\n}\n\nexport function resolveFixture(\n  fixture: Fixture | undefined,\n  baseDirectory: string,\n): ResolvedFixture | undefined {\n  if (fixture === undefined) return undefined;\n  const directory = fixture.path === undefined ? undefined : join(baseDirectory, fixture.path);\n  const setup = fixture.setup ?? [];\n  return { directory, hash: fixtureIdentityHash(directory, setup), setup };\n}\n\ninterface FixtureEntry {\n  readonly absolutePath: string;\n  readonly executable: boolean;\n  readonly path: string;\n  readonly type: \"directory\" | \"file\";\n}\n\n/**\n * Walks a fixture directory into a deterministic manifest, rejecting symlinks and special files.\n * Symlinks are refused because a copied link can resolve back into the source fixture and let\n * setup commands mutate it; links belong in setup commands instead.\n */\nexport function walkFixtureEntries(directory: string): FixtureEntry[] {\n  const entries: FixtureEntry[] = [];\n\n  function visit(current: string, prefix: string): void {\n    const dirents = [...readdirSync(current, { withFileTypes: true })].sort((left, right) =>\n      left.name.localeCompare(right.name),\n    );\n    for (const dirent of dirents) {\n      if (SKIPPED_DIRECTORIES.has(dirent.name)) continue;\n      const absolutePath = join(current, dirent.name);\n      const path = prefix === \"\" ? dirent.name : `${prefix}/${dirent.name}`;\n      if (dirent.isSymbolicLink()) {\n        throw new Error(\n          `contains unsupported symlink \"${path}\"; create links with setup commands instead`,\n        );\n      }\n      if (dirent.isDirectory()) {\n        entries.push({ absolutePath, executable: false, path, type: \"directory\" });\n        visit(absolutePath, path);\n      } else if (dirent.isFile()) {\n        const executable = (statSync(absolutePath).mode & 0o111) !== 0;\n        entries.push({ absolutePath, executable, path, type: \"file\" });\n      } else {\n        throw new Error(`contains unsupported special file \"${path}\"`);\n      }\n    }\n  }\n\n  visit(directory, \"\");\n  return entries;\n}\n\nexport function fixtureIdentityHash(\n  directory: string | undefined,\n  setup: readonly string[],\n): string {\n  // Each entry is framed with its type, path, mode, and byte length so file bytes can never\n  // masquerade as entry boundaries, and raw bytes are hashed so binary files cannot collide\n  // through lossy text decoding.\n  const hash = createHash(\"sha256\");\n  if (directory !== undefined) {\n    for (const entry of walkFixtureEntries(directory)) {\n      if (entry.type === \"directory\") {\n        hash.update(`D\\0${entry.path}\\0`);\n        continue;\n      }\n      const content = readFileSync(entry.absolutePath);\n      hash.update(`F\\0${entry.path}\\0${entry.executable ? \"x\" : \"-\"}\\0${content.byteLength}\\0`);\n      hash.update(content);\n    }\n  }\n  hash.update(`S\\0${JSON.stringify(setup)}`);\n  return hash.digest(\"hex\");\n}\n\nexport function applyFixture(\n  fixture: ResolvedFixture,\n  workspace: string,\n  home: string,\n): readonly FixtureCommandResult[] {\n  if (fixture.directory !== undefined) {\n    // verbatimSymlinks guards the window between hashing (which rejects symlinks) and copying:\n    // a link that appears mid-run is copied inert instead of resolving into the source fixture.\n    cpSync(fixture.directory, workspace, {\n      filter: (source) => !SKIPPED_DIRECTORIES.has(basename(source)),\n      recursive: true,\n      verbatimSymlinks: true,\n    });\n  }\n\n  const results: FixtureCommandResult[] = [];\n  for (const command of fixture.setup) {\n    // Minimal environment: PATH for tool lookup plus the throwaway trial home, nothing inherited.\n    // SIGKILL on timeout because SIGTERM is trappable and would let a hung command run forever.\n    const outcome = spawnSync(command, {\n      cwd: workspace,\n      encoding: \"utf8\",\n      env: { HOME: home, PATH: process.env.PATH ?? \"\" },\n      killSignal: \"SIGKILL\",\n      shell: true,\n      timeout: SETUP_COMMAND_TIMEOUT_MS,\n    });\n    results.push({\n      command,\n      exitCode: outcome.status,\n      signal: outcome.signal ?? null,\n      stderr: outcome.stderr ?? \"\",\n      stdout: outcome.stdout ?? \"\",\n    });\n    if (outcome.error !== undefined || outcome.status !== 0) {\n      const timedOut = (outcome.error as NodeJS.ErrnoException | undefined)?.code === \"ETIMEDOUT\";\n      const reason = timedOut\n        ? `timed out after ${SETUP_COMMAND_TIMEOUT_MS / 1000}s`\n        : (outcome.error?.message ??\n          (outcome.signal !== null\n            ? `terminated by ${outcome.signal}`\n            : `exit code ${outcome.status}`));\n      throw new FixtureSetupError(`fixture setup failed: \"${command}\" (${reason})`, results);\n    }\n  }\n  return results;\n}\n","/** Orchestrates discovery, trial arms, voting, caching, cleanup, and report persistence. */\nimport { mkdirSync, mkdtempSync, rmSync, writeFileSync } from \"node:fs\";\nimport { tmpdir } from \"node:os\";\nimport { join } from \"node:path\";\nimport { ArmCache } from \"./cache.js\";\nimport type { SkillvalConfig } from \"./config.js\";\nimport { resolveStateDirectory } from \"./config.js\";\nimport type { ReadyDiscoveredSkill } from \"./discovery.js\";\nimport { discoverSkills, selectSkills } from \"./discovery.js\";\nimport { createExecutor } from \"./executors/index.js\";\nimport type { Executor, ExecutorMetadata } from \"./executors/types.js\";\nimport type { ResolvedFixture } from \"./fixture.js\";\nimport { applyFixture, FixtureSetupError, resolveFixture, selectFixture } from \"./fixture.js\";\nimport { gradeTrial } from \"./grade.js\";\nimport type { Arm, ArmResult, CaseResult, EvalCase, TrialResult } from \"./types.js\";\nimport { sha256, skillContentHash } from \"./utils.js\";\nimport { clampedTrialCount, hasMajority, shouldEscalate } from \"./vote.js\";\n\nexport interface RunOptions {\n  readonly caseFilter: string | undefined;\n  readonly requestedSkills: readonly string[];\n  readonly skipBaseline: boolean;\n  readonly useCache: boolean;\n}\n\nexport interface SkillReport {\n  readonly cases: readonly CaseResult[];\n  readonly class: \"capability\" | \"preference\";\n  readonly contentHash: string;\n}\n\nexport interface RunReport {\n  readonly executor: ExecutorMetadata;\n  readonly runHash: string;\n  readonly skills: Readonly<Record<string, SkillReport>>;\n}\n\nexport interface RunOutcome {\n  readonly failures: number;\n  readonly noops: number;\n  readonly report: RunReport;\n  readonly reportPath: string;\n}\n\ninterface ArmContext {\n  readonly cache: ArmCache;\n  readonly evalCase: EvalCase;\n  readonly executor: Executor;\n  readonly fixture: ResolvedFixture | undefined;\n  readonly skill: ReadyDiscoveredSkill;\n  readonly skillHash: string;\n  readonly useCache: boolean;\n}\n\ninterface CaseContext {\n  readonly cache: ArmCache;\n  readonly executor: Executor;\n  readonly skill: ReadyDiscoveredSkill;\n  readonly skillHash: string;\n  readonly skipBaseline: boolean;\n  readonly useCache: boolean;\n}\n\nexport function runEvaluation(\n  config: SkillvalConfig,\n  options: RunOptions,\n  log: (message: string) => void,\n): RunOutcome {\n  const discovery = discoverSkills(config.roots);\n  const selectedSkills = selectSkills(discovery, options.requestedSkills);\n  const executor = createExecutor(config.executor);\n  const stateDirectory = resolveStateDirectory();\n  const cache = new ArmCache(stateDirectory);\n  const skillInputs = selectedSkills.map((skill) => ({\n    contentHash: skillContentHash(skill.skillDirectory),\n    skill,\n  }));\n  const runHash = participatingSkillsHash(\n    skillInputs.map(({ contentHash, skill }) => ({ contentHash, name: skill.name })),\n  );\n  const skillReports: Record<string, SkillReport> = {};\n  let failures = 0;\n  let noops = 0;\n\n  for (const { contentHash, skill } of skillInputs) {\n    const evals = skill.evals;\n    log(`${skill.name} (${evals.class}, ${contentHash.slice(0, 12)}):`);\n    const cases = evals.cases\n      .filter((evalCase) => options.caseFilter === undefined || evalCase.id === options.caseFilter)\n      .map((evalCase) =>\n        runCase(\n          {\n            cache,\n            executor,\n            skill,\n            skillHash: contentHash,\n            skipBaseline: options.skipBaseline,\n            useCache: options.useCache,\n          },\n          evalCase,\n          log,\n        ),\n      );\n    skillReports[skill.name] = { cases, class: evals.class, contentHash };\n    failures += cases.filter((result) => !result.pass).length;\n    noops += cases.filter((result) => result.noop).length;\n  }\n\n  const report: RunReport = {\n    executor: executor.metadata,\n    runHash,\n    skills: skillReports,\n  };\n  const reportDirectory = join(stateDirectory, \"reports\");\n  const reportPath = join(reportDirectory, `${runHash}.json`);\n  mkdirSync(reportDirectory, { recursive: true });\n  writeFileSync(reportPath, JSON.stringify(report, null, 2));\n  return { failures, noops, report, reportPath };\n}\n\nexport function participatingSkillsHash(\n  skills: readonly { readonly contentHash: string; readonly name: string }[],\n): string {\n  return sha256(\n    [...skills]\n      .sort((left, right) => left.name.localeCompare(right.name))\n      .map(({ contentHash, name }) => `${name}\\0${contentHash}`)\n      .join(\"\\0\"),\n  );\n}\n\nfunction runCase(\n  context: CaseContext,\n  evalCase: EvalCase,\n  log: (message: string) => void,\n): CaseResult {\n  const arms = (evalCase.arms ?? [\"skill\"]).filter(\n    (arm) => arm === \"skill\" || !context.skipBaseline,\n  );\n  // Fixture paths are relative to skillval.yml, which sits in the skill directory.\n  const fixture = resolveFixture(\n    selectFixture(evalCase.fixture, context.skill.evals.fixture),\n    context.skill.skillDirectory,\n  );\n  const results = arms.map((arm) => {\n    log(`  ${evalCase.id} [${arm}] ...`);\n    const result = runArm(\n      {\n        cache: context.cache,\n        evalCase,\n        executor: context.executor,\n        fixture,\n        skill: context.skill,\n        skillHash: context.skillHash,\n        useCache: context.useCache,\n      },\n      arm,\n    );\n    log(\n      `  ${evalCase.id} [${arm}] ${result.pass ? \"pass\" : \"FAIL\"}${\n        result.cached ? \" (cached)\" : \"\"\n      }`,\n    );\n    return result;\n  });\n  return {\n    arms: results,\n    id: evalCase.id,\n    noop: results.find((result) => result.arm === \"baseline\")?.pass === true,\n    pass: results.find((result) => result.arm === \"skill\")?.pass === true,\n    rule: evalCase.rule,\n  };\n}\n\nfunction runArm(context: ArmContext, arm: Arm): ArmResult {\n  const identity = {\n    arm,\n    evalCase: context.evalCase,\n    executor: context.executor.metadata,\n    fixtureHash: context.fixture?.hash,\n    skillHash: context.skillHash,\n  };\n  if (context.useCache) {\n    const hit = context.cache.lookup(identity);\n    if (hit !== undefined) return hit;\n  }\n\n  const trials: TrialResult[] = [];\n  const wanted = clampedTrialCount(context.evalCase.trials);\n  for (let index = 0; index < wanted; index += 1) {\n    trials.push(runTrial(context, arm));\n  }\n  // A disagreement makes the configured sample inconclusive, so collect the full five trials.\n  while (shouldEscalate(trials)) trials.push(runTrial(context, arm));\n\n  const result: ArmResult = {\n    arm,\n    cached: false,\n    pass: hasMajority(trials),\n    trials,\n  };\n  context.cache.store(identity, result);\n  return result;\n}\n\nfunction runTrial(context: ArmContext, arm: Arm): TrialResult {\n  // The runner owns generic resource lifecycle; adapters own provider-specific setup inside it.\n  const workspace = mkdtempSync(join(tmpdir(), `skillval-${context.evalCase.id}-`));\n  const trialHome = mkdtempSync(join(tmpdir(), \"skillval-home-\"));\n\n  try {\n    const fixtureSetup =\n      context.fixture === undefined\n        ? undefined\n        : applyFixture(context.fixture, workspace, trialHome);\n    const trace = context.executor.runTrial({\n      arm,\n      evalCase: context.evalCase,\n      home: trialHome,\n      skillDirectory: context.skill.skillDirectory,\n      skillName: context.skill.name,\n      workspace,\n    });\n    const checks = gradeTrial(context.evalCase, arm, trace, workspace);\n    return {\n      checks,\n      fixtureSetup,\n      pass: checks.every((check) => check.pass),\n      usage: trace.usage,\n    };\n  } catch (error) {\n    if (error instanceof FixtureSetupError) {\n      // Workspace staging failed before the agent ran; this is infrastructure, not grading.\n      return {\n        checks: [{ detail: error.message, name: \"fixture-setup\", pass: false }],\n        fixtureSetup: error.results,\n        pass: false,\n        usage: undefined,\n      };\n    }\n    const detail = error instanceof Error ? error.message : String(error);\n    return {\n      checks: [{ detail, name: \"run\", pass: false }],\n      pass: false,\n      usage: undefined,\n    };\n  } finally {\n    // Trials may contain generated source or credentials-related environment state. Always clean\n    // both directories, including executor and grader failure paths.\n    rmSync(workspace, { force: true, recursive: true });\n    rmSync(trialHome, { force: true, recursive: true });\n  }\n}\n","/** Owns arm-result persistence and keeps cache-key construction private from the runner. */\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { resolveStateDirectory } from \"./config.js\";\nimport type { ExecutorMetadata } from \"./executors/types.js\";\nimport type { Arm, ArmResult, EvalCase } from \"./types.js\";\nimport { sha256 } from \"./utils.js\";\n\n// Bump this whenever execution or grading semantics change so old results cannot be reused.\nexport const RUNNER_VERSION = 6;\n\nexport interface ArmCacheIdentity {\n  readonly arm: Arm;\n  readonly evalCase: EvalCase;\n  readonly executor: ExecutorMetadata;\n  readonly fixtureHash?: string;\n  readonly skillHash: string;\n}\n\nexport class ArmCache {\n  readonly #stateDirectory: string;\n\n  public constructor(stateDirectory = resolveStateDirectory()) {\n    this.#stateDirectory = stateDirectory;\n  }\n\n  public lookup(identity: ArmCacheIdentity): ArmResult | undefined {\n    const path = this.#path(identity);\n    if (!existsSync(path)) return undefined;\n    return { ...(JSON.parse(readFileSync(path, \"utf8\")) as ArmResult), cached: true };\n  }\n\n  public store(identity: ArmCacheIdentity, result: ArmResult): void {\n    const path = this.#path(identity);\n    mkdirSync(dirname(path), { recursive: true });\n    writeFileSync(path, JSON.stringify(result));\n  }\n\n  #path(identity: ArmCacheIdentity): string {\n    // Every input that can change an arm result belongs in this identity.\n    const parts = [\n      String(RUNNER_VERSION),\n      identity.skillHash,\n      JSON.stringify(identity.evalCase),\n      identity.arm,\n      identity.executor.name,\n      identity.executor.version,\n      identity.executor.model,\n    ];\n    // Appended conditionally so fixture-free identities keep their historical keys.\n    if (identity.fixtureHash !== undefined) parts.push(identity.fixtureHash);\n    const key = sha256(parts.join(\"\\0\"));\n    return join(this.#stateDirectory, \"cache\", `${key}.json`);\n  }\n}\n","/** Composes trace, trigger, regex, and registered deterministic checks for one trial. */\nimport { readFileSync } from \"node:fs\";\nimport { relative } from \"node:path\";\nimport { runGraders } from \"./graders.js\";\nimport type { Arm, Check, EvalCase, Trace } from \"./types.js\";\nimport { walkFiles } from \"./utils.js\";\n\nconst INJECTED_FILES = new Set([\"package.json\", \"tsconfig.json\"]);\nexport function gradeTrial(evalCase: EvalCase, arm: Arm, trace: Trace, workspace: string): Check[] {\n  const checks: Check[] = [];\n\n  checks.push({\n    detail: trace.completed ? \"turn.completed seen\" : \"no turn.completed in trace\",\n    name: \"trace\",\n    pass: trace.completed,\n  });\n\n  if (evalCase.should_trigger !== undefined && arm === \"skill\") {\n    checks.push({\n      detail: `invoked=${trace.invoked}, expected=${evalCase.should_trigger}`,\n      name: \"trigger\",\n      pass: trace.invoked === evalCase.should_trigger,\n    });\n  }\n\n  const gradedText =\n    evalCase.mode === \"generation\"\n      ? walkFiles(workspace)\n          .filter((file) => !INJECTED_FILES.has(relative(workspace, file)))\n          .map((file) => `=== ${relative(workspace, file)} ===\\n${readFileSync(file, \"utf8\")}`)\n          .join(\"\\n\")\n      : trace.agentText;\n\n  for (const pattern of evalCase.assert?.must_match ?? []) {\n    const pass = new RegExp(pattern, \"m\").test(gradedText);\n    checks.push({\n      detail: pass ? pattern : `${pattern} | got: ${gradedText.slice(0, 400)}`,\n      name: \"must_match\",\n      pass,\n    });\n  }\n  for (const pattern of evalCase.assert?.must_not_match ?? []) {\n    checks.push({\n      detail: pattern,\n      name: \"must_not_match\",\n      pass: !new RegExp(pattern, \"m\").test(gradedText),\n    });\n  }\n\n  checks.push(...runGraders(evalCase, workspace));\n  return checks;\n}\n","import type { TrialResult } from \"./types.js\";\n\nexport function clampedTrialCount(configured: number | undefined): number {\n  return Math.min(5, Math.max(1, configured ?? 1));\n}\n\nexport function hasMajority(trials: readonly Pick<TrialResult, \"pass\">[]): boolean {\n  const passes = trials.filter((trial) => trial.pass).length;\n  return passes * 2 > trials.length;\n}\n\nexport function shouldEscalate(trials: readonly Pick<TrialResult, \"pass\">[]): boolean {\n  return new Set(trials.map((trial) => trial.pass)).size > 1 && trials.length < 5;\n}\n"],"mappings":";;;AAGA,SAAS,eAAe;;;ACFxB,SAAS,YAAY,gBAAAA,qBAAoB;AACzC,SAAS,WAAAC,gBAAe;AACxB,SAAS,eAAe;AACxB,SAAS,SAAS,aAAa,UAAU,oBAAoB;AAC7D,SAAS,SAAS,iBAAiB;;;ACHnC,OAAO,UAAU;;;ACDjB,SAAS,iBAAiB;AAC1B,SAAS,WAAW,gBAAAC,eAAc,mBAAmB;AACrD,SAAS,eAAe;AACxB,SAAS,QAAAC,aAAY;;;ACJrB,SAAS,kBAAkB;AAC3B,SAAS,aAAa,oBAAoB;AAC1C,SAAS,MAAM,gBAAgB;AAExB,SAAS,OAAO,OAAuB;AAC5C,SAAO,WAAW,QAAQ,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK;AACxD;AAGO,IAAM,sBAA2C,oBAAI,IAAI,CAAC,QAAQ,cAAc,CAAC;AAEjF,SAAS,UAAU,MAAwB;AAChD,QAAM,QAAkB,CAAC;AAEzB,WAAS,MAAM,WAAyB;AACtC,eAAW,SAAS,YAAY,WAAW,EAAE,eAAe,KAAK,CAAC,GAAG;AACnE,UAAI,oBAAoB,IAAI,MAAM,IAAI,EAAG;AACzC,YAAM,OAAO,KAAK,WAAW,MAAM,IAAI;AACvC,UAAI,MAAM,YAAY,EAAG,OAAM,IAAI;AAAA,eAC1B,MAAM,OAAO,EAAG,OAAM,KAAK,IAAI;AAAA,IAC1C;AAAA,EACF;AAEA,QAAM,IAAI;AACV,SAAO,MAAM,KAAK;AACpB;AAEO,SAAS,iBAAiB,gBAAgC;AAC/D,QAAM,QAAQ,UAAU,cAAc,EAAE;AAAA,IACtC,CAAC,SAAS,GAAG,SAAS,gBAAgB,IAAI,CAAC;AAAA,EAAK,aAAa,MAAM,MAAM,CAAC;AAAA,EAC5E;AACA,SAAO,OAAO,MAAM,KAAK,IAAI,CAAC;AAChC;AAEO,SAAS,SAAS,OAAkD;AACzE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;;;AD3BA,IAAM,mBAAmB,KAAK,KAAK;AAE5B,IAAM,gBAAN,MAAwC;AAAA,EAC7B;AAAA,EACP;AAAA,EAEF,YAAY,WAAW,QAAQ,GAAG;AACvC,SAAK,YAAY;AACjB,SAAK,WAAW,YAAY,QAAQ;AAAA,EACtC;AAAA,EAEO,SAAS,SAA8B;AAC5C,QAAI,QAAQ,QAAQ,QAAS,WAAU,OAAO;AAC9C,UAAM,UAAU,QAAQ,SAAS,SAAS,eAAe,oBAAoB;AAG7E,UAAM,cACJ,QAAQ,QAAQ,aACZ;AAAA,MACE,GAAG,QAAQ;AAAA,MACX,YAAYC,MAAK,KAAK,WAAW,QAAQ;AAAA,MACzC,MAAM,QAAQ;AAAA,IAChB,IACA,EAAE,GAAG,QAAQ,IAAI;AACvB,UAAM,SAAS;AAAA,MACb;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ;AAAA,QACR,QAAQ,SAAS;AAAA,MACnB;AAAA,MACA;AAAA,QACE,UAAU;AAAA,QACV,KAAK;AAAA,QACL,WAAW,KAAK,OAAO;AAAA,QACvB,SAAS;AAAA,MACX;AAAA,IACF;AACA,QAAI,OAAO,WAAW,GAAG;AACvB,YAAM,IAAI,MAAM,qBAAqB,OAAO,MAAM,KAAK,OAAO,QAAQ,MAAM,IAAI,CAAC,EAAE;AAAA,IACrF;AAEA,WAAO,gBAAgB,OAAO,QAAQ,QAAQ,SAAS;AAAA,EACzD;AACF;AAEA,SAAS,UAAU,SAA6B;AAE9C,QAAM,aAAaA,MAAK,QAAQ,WAAW,gBAAgB;AAC3D,YAAU,YAAY,EAAE,WAAW,KAAK,CAAC;AACzC,cAAY,QAAQ,gBAAgBA,MAAK,YAAY,QAAQ,SAAS,CAAC;AACzE;AAEO,SAAS,YAAY,WAAW,QAAQ,GAAqB;AAClE,QAAM,UAAU,UAAU,SAAS,CAAC,WAAW,GAAG,EAAE,UAAU,OAAO,CAAC,EAAE,QAAQ,KAAK,KAAK;AAC1F,MAAI,YAAY,GAAI,OAAM,IAAI,MAAM,6BAA6B;AACjE,QAAM,QACJC,cAAaD,MAAK,UAAU,oBAAoB,GAAG,MAAM,EAAE;AAAA,IACzD;AAAA,EACF,IAAI,CAAC,KAAK;AACZ,SAAO,EAAE,OAAO,MAAM,SAAS,QAAQ;AACzC;AAEO,SAAS,gBAAgB,QAAgB,WAA0B;AACxE,MAAI,YAAY;AAChB,MAAI,UAAU;AACd,QAAM,QAAkB,CAAC;AACzB,MAAI;AAEJ,aAAW,QAAQ,OAAO,MAAM,IAAI,GAAG;AACrC,QAAI,KAAK,KAAK,MAAM,GAAI;AACxB,QAAI;AACJ,QAAI;AACF,cAAQ,KAAK,MAAM,IAAI;AAAA,IACzB,QAAQ;AACN;AAAA,IACF;AACA,QAAI,CAAC,SAAS,KAAK,EAAG;AACtB,UAAM,OAAO,SAAS,MAAM,IAAI,IAAI,MAAM,OAAO;AACjD,QACE,MAAM,SAAS,uBACf,OAAO,KAAK,YAAY,YACxB,KAAK,QAAQ,SAAS,GAAG,SAAS,WAAW,GAC7C;AACA,gBAAU;AAAA,IACZ;AACA,QACE,MAAM,SAAS,oBACf,MAAM,SAAS,mBACf,OAAO,KAAK,SAAS,UACrB;AACA,YAAM,KAAK,KAAK,IAAI;AAAA,IACtB;AACA,QAAI,MAAM,SAAS,kBAAkB;AACnC,kBAAY;AACZ,cAAQ,MAAM;AAAA,IAChB;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,MAAM,KAAK,IAAI,GAAG,WAAW,SAAS,MAAM;AAClE;;;AE/GA,IAAM,oBAAoB;AAAA,EACxB,OAAO,MAAgB,IAAI,cAAc;AAC3C;AAIO,IAAM,iBAAiB,OAAO,KAAK,iBAAiB;AAEpD,SAAS,eAAe,MAA8B;AAC3D,SAAO,kBAAkB,IAAI,EAAE;AACjC;;;AHTO,IAAM,mBAAmB,KAAK;AAAA,EACnC,KAAK,OAAO;AAAA;AAAA,IAEV,UAAU,KAAK,KAAK,gBAAgB;AAAA,MAClC,aAAa;AAAA,IACf,CAAC;AAAA,IACD,OAAO,KAAK;AAAA,MACV,KAAK,MAAM,KAAK,OAAO,EAAE,WAAW,GAAG,SAAS,OAAO,QAAQ,CAAC,GAAG;AAAA,QACjE,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AAAA,EACD;AAAA,IACE,KAAK;AAAA,IACL,SAAS;AAAA,IACT,sBAAsB;AAAA,IACtB,OAAO;AAAA,EACT;AACF;;;ADNO,IAAM,cAAN,cAA0B,MAAM;AAAA,EACrB;AAAA,EAET,YAAY,SAAiB,OAAO,gBAAgB;AACzD,UAAM,OAAO;AACb,SAAK,OAAO;AACZ,SAAK,OAAO;AAAA,EACd;AACF;AAEO,SAAS,kBAAkB,UAA6B,CAAC,GAAW;AACzE,QAAM,cAAc,QAAQ,eAAe,QAAQ;AACnD,QAAM,OAAO,QAAQ,QAAQE,SAAQ;AACrC,QAAM,WACJ,QAAQ,WACR,YAAY,oBACX,YAAY,kBACT,QAAQ,YAAY,iBAAiB,qBAAqB,IAC1D,QAAQ,MAAM,6BAA6B;AACjD,SAAO,QAAQ,WAAW,UAAU,IAAI,CAAC;AAC3C;AAEO,SAAS,WAAW,MAAc,OAAOA,SAAQ,GAAW;AACjE,SAAO,QAAQ,WAAW,MAAM,IAAI,CAAC;AACvC;AAEO,SAAS,WAAW,MAAc,OAAOA,SAAQ,GAAmB;AACzE,MAAI,CAAC,WAAW,IAAI,GAAG;AACrB,UAAM,IAAI,YAAY,0BAA0B,IAAI,IAAI,kBAAkB;AAAA,EAC5E;AAEA,MAAI;AACJ,MAAI;AACF,aAAS,UAAUC,cAAa,MAAM,MAAM,CAAC;AAAA,EAC/C,SAAS,OAAO;AACd,UAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACpE,UAAM,IAAI,YAAY,mBAAmB,IAAI,KAAK,MAAM,IAAI,qBAAqB;AAAA,EACnF;AAEA,MAAI,CAAC,YAAY,kBAAkB,MAAM,GAAG;AAC1C,UAAM,CAAC,UAAU,IAAI,aAAa,kBAAkB,MAAM;AAC1D,UAAM,WAAW,YAAY,aAAa,WAAW,KAAK,GAAG,EAAE,QAAQ,OAAO,EAAE;AAChF,UAAM,UAAU,aAAa,UAAa,aAAa,KAAK,OAAO,GAAG,IAAI,IAAI,QAAQ;AACtF,UAAM,IAAI,YAAY,GAAG,OAAO,IAAI,YAAY,WAAW,YAAY,EAAE;AAAA,EAC3E;AAEA,SAAO;AAAA,IACL,UAAU,OAAO;AAAA,IACjB,OAAO,OAAO,MAAM,IAAI,CAAC,SAAS,WAAW,MAAM,IAAI,CAAC;AAAA,EAC1D;AACF;AAEO,SAAS,sBACd,cAAiC,QAAQ,KACzC,OAAOD,SAAQ,GACP;AACR,SAAO,YAAY,iBACf,QAAQ,YAAY,gBAAgB,UAAU,IAC9C,QAAQ,MAAM,uBAAuB;AAC3C;AAEA,SAAS,WAAW,OAAe,MAAsB;AACvD,QAAM,YACJ,UAAU,MAAM,OAAO,MAAM,WAAW,IAAI,IAAI,GAAG,IAAI,GAAG,MAAM,MAAM,CAAC,CAAC,KAAK;AAC/E,SAAO,UAAU,WAAW,OAAO,eAAe,IAAI,EAAE,WAAW,SAAS,IAAI;AAClF;;;AKjFA,SAAS,cAAAE,aAAY,eAAAC,oBAAmB;AACxC,SAAS,QAAAC,aAAY;;;ACDrB,SAAS,gBAAAC,eAAc,YAAAC,iBAAgB;AACvC,SAAS,WAAAC,UAAS,QAAAC,aAAY;AAC9B,SAAS,SAASC,kBAAiB;;;ACDnC,OAAOC,WAAU;AACjB,SAAS,SAASC,cAAa,UAAUC,qBAAoB;;;ACF7D,SAAS,aAAAC,kBAAiB;AAC1B,SAAS,cAAAC,aAAY,qBAAqB;AAC1C,SAAS,qBAAqB;AAC9B,SAAS,SAAS,QAAAC,aAAY;AAqB9B,IAAM,iBAAiB,cAAc,YAAY,GAAG;AAEpD,IAAM,UAAU;AAAA,EACd,KAAK;AAAA,IACH,OAAO,CAAC,YAAY;AAAA,IACpB,KAAK;AAAA,EACP;AACF;AAIO,IAAM,eAAe,OAAO,KAAK,OAAO;AAExC,SAAS,mBAAmB,MAAkB,MAAyB;AAC5E,QAAM,QAA6B,QAAQ,IAAI,EAAE;AACjD,SAAO,MAAM,SAAS,IAAI;AAC5B;AAEO,SAAS,WAAW,UAAwB,WAA2C;AAC5F,UAAQ,SAAS,QAAQ,WAAW,CAAC,GAAG,IAAI,CAAC,SAAS,QAAQ,IAAI,EAAE,IAAI,SAAS,CAAC;AACpF;AAEA,SAAS,SAAS,WAAgC;AAChD,MAAI,CAACD,YAAWC,MAAK,WAAW,cAAc,CAAC,GAAG;AAChD,kBAAcA,MAAK,WAAW,cAAc,GAAG,wBAAwB;AAAA,EACzE;AACA,QAAM,qBAAqB,QAAQ,eAAe,QAAQ,0BAA0B,CAAC;AACrF;AAAA,IACEA,MAAK,WAAW,eAAe;AAAA,IAC/B,KAAK,UAAU;AAAA,MACb,iBAAiB;AAAA,QACf,KAAK,CAAC,QAAQ;AAAA,QACd,QAAQ;AAAA,QACR,kBAAkB;AAAA,QAClB,QAAQ;AAAA,QACR,0BAA0B;AAAA,QAC1B,QAAQ;AAAA,QACR,QAAQ;AAAA,QACR,WAAW,CAAC,QAAQ,kBAAkB,CAAC;AAAA,QACvC,OAAO,CAAC,MAAM;AAAA,MAChB;AAAA,IACF,CAAC;AAAA,EACH;AACA,QAAM,mBAAmB,eAAe,QAAQ,oBAAoB;AACpE,QAAM,SAASF,WAAU,kBAAkB,CAAC,MAAM,SAAS,GAAG;AAAA,IAC5D,UAAU;AAAA,IACV,SAAS;AAAA,EACX,CAAC;AACD,SAAO;AAAA,IACL,QAAQ,OAAO,WAAW,IAAI,qBAAqB,OAAO,UAAU,IAAI,MAAM,GAAG,GAAG;AAAA,IACpF,MAAM;AAAA,IACN,MAAM,OAAO,WAAW;AAAA,EAC1B;AACF;;;ADxEA,IAAM,uBAAuBG,MAAK,KAAK,CAAC,cAAc,YAAY,CAAC;AACnE,IAAM,uBAAuBA,MAAK,OAAO,EAAE,WAAW,GAAG,SAAS,OAAO,QAAQ,CAAC;AAClF,IAAM,oBAAoBA,MAAK,SAASA,MAAK,MAAMA,MAAK,OAAO,CAAC,CAAC;AAC1D,IAAM,YAAYA,MAAK,KAAK,CAAC,YAAY,OAAO,CAAC;AAGjD,IAAM,gBAAgBA,MAAK;AAAA,EAChCA,MAAK,OAAO;AAAA,IACV,MAAMA,MAAK;AAAA,MACTA,MAAK,OAAO;AAAA,QACV,aAAa;AAAA,QACb,WAAW;AAAA,QACX,SAAS,OAAO;AAAA,MAClB,CAAC;AAAA,IACH;AAAA,IACA,OAAOA,MAAK;AAAA,MACVA,MAAK;AAAA,QACHA,MAAK,MAAM,sBAAsB;AAAA,UAC/B,aAAa;AAAA,QACf,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF,CAAC;AAAA,EACD,EAAE,sBAAsB,OAAO,eAAe,EAAE;AAClD;AAEO,IAAM,mBAAmBA,MAAK;AAAA,EACnCA,MAAK,OAAO;AAAA,IACV,SAASA,MAAK;AAAA,MACZA,MAAK,SAASA,MAAK,MAAMA,MAAK,KAAK,YAAY,GAAG,EAAE,aAAa,KAAK,CAAC,CAAC;AAAA,IAC1E;AAAA,IACA,YAAYA,MAAK,SAAS,iBAAiB;AAAA,IAC3C,gBAAgBA,MAAK,SAAS,iBAAiB;AAAA,EACjD,CAAC;AAAA,EACD,EAAE,sBAAsB,MAAM;AAChC;AAEO,IAAM,iBAAiBA,MAAK;AAAA,EACjCA,MAAK,OAAO;AAAA,IACV,MAAMA,MAAK;AAAA,MACTA,MAAK;AAAA,QACHA,MAAK,MAAM,WAAW;AAAA,UACpB,aAAa;AAAA,QACf,CAAC;AAAA,MACH;AAAA,IACF;AAAA,IACA,QAAQA,MAAK,SAAS,gBAAgB;AAAA,IACtC,SAASA,MAAK,SAAS,aAAa;AAAA,IACpC,IAAI;AAAA,IACJ,MAAMA,MAAK,KAAK,CAAC,cAAc,SAAS,CAAC;AAAA,IACzC,QAAQ;AAAA,IACR,MAAMA,MAAK,SAASA,MAAK,OAAO,CAAC;AAAA,IACjC,gBAAgBA,MAAK,SAASA,MAAK,QAAQ,CAAC;AAAA,IAC5C,QAAQA,MAAK,SAASA,MAAK,QAAQ,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC,CAAC;AAAA,IAC9D,MAAMA,MAAK,SAAS,oBAAoB;AAAA,EAC1C,CAAC;AAAA,EACD,EAAE,sBAAsB,MAAM;AAChC;AAEO,IAAM,mBAAmBA,MAAK;AAAA,EACnCA,MAAK,OAAO;AAAA,IACV,OAAOA,MAAK,SAASA,MAAK,MAAM,cAAc,CAAC;AAAA,IAC/C,OAAO;AAAA,IACP,SAASA,MAAK,SAAS,aAAa;AAAA,IACpC,OAAO;AAAA,EACT,CAAC;AAAA,EACD;AAAA,IACE,KAAK;AAAA,IACL,SAAS;AAAA,IACT,sBAAsB;AAAA,IACtB,OAAO;AAAA,EACT;AACF;AAQO,IAAM,oBAAN,cAAgC,MAAM;AAAA,EAC3B;AAAA,EAET,YAAY,SAAiB,OAAO,qBAAqB;AAC9D,UAAM,OAAO;AACb,SAAK,OAAO;AACZ,SAAK,OAAO;AAAA,EACd;AACF;AAEO,SAAS,eAAe,OAAgB,MAAc,eAAoC;AAC/F,MAAI,CAACC,aAAY,kBAAkB,KAAK,GAAG;AACzC,UAAM,CAAC,UAAU,IAAIC,cAAa,kBAAkB,KAAK;AACzD,UAAM,WAAW,YAAY,aAAa,WAAW,KAAK,GAAG,EAAE,QAAQ,OAAO,EAAE;AAChF,QAAI,UAAU,SAAS,SAAS,GAAG;AACjC,YAAM,QAAQ,wBAAwB,KAAK,QAAQ;AACnD,YAAM,YAAY,QAAQ,CAAC,KAAK;AAChC,YAAM,IAAI;AAAA,QACR,GAAG,IAAI,kBAAkB,SAAS;AAAA,MACpC;AAAA,IACF;AACA,UAAM,UAAU,aAAa,UAAa,aAAa,KAAK,OAAO,GAAG,IAAI,IAAI,QAAQ;AACtF,UAAM,IAAI,kBAAkB,GAAG,OAAO,IAAI,YAAY,WAAW,YAAY,EAAE;AAAA,EACjF;AAEA,MAAI,kBAAkB,UAAa,MAAM,UAAU,eAAe;AAChE,UAAM,IAAI;AAAA,MACR,GAAG,IAAI,oBAAoB,MAAM,KAAK,gBAAgB,aAAa;AAAA,IACrE;AAAA,EACF;AAEA,QAAM,MAAM,oBAAI,IAAY;AAC5B,aAAW,YAAY,MAAM,OAAO;AAGlC,QAAI,IAAI,IAAI,SAAS,EAAE,GAAG;AACxB,YAAM,IAAI,kBAAkB,GAAG,IAAI,aAAa,SAAS,EAAE,iBAAiB;AAAA,IAC9E;AACA,QAAI,IAAI,SAAS,EAAE;AACnB,qBAAiB,UAAU,IAAI;AAC/B,oBAAgB,UAAU,IAAI;AAAA,EAChC;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,UAAoB,MAAoB;AAC/D,aAAW,UAAU,SAAS,QAAQ,WAAW,CAAC,GAAG;AACnD,QAAI,CAAC,mBAAmB,QAAQ,SAAS,IAAI,GAAG;AAC9C,YAAM,IAAI;AAAA,QACR,GAAG,IAAI,UAAU,SAAS,EAAE,aAAa,MAAM,sBAAsB,SAAS,IAAI;AAAA,MACpF;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,iBAAiB,UAAoB,MAAoB;AAChE,QAAM,SAAS,CAAC,cAAc,gBAAgB;AAC9C,aAAW,SAAS,QAAQ;AAC1B,eAAW,WAAW,SAAS,SAAS,KAAK,KAAK,CAAC,GAAG;AACpD,UAAI;AACF,YAAI,OAAO,SAAS,GAAG;AAAA,MACzB,SAAS,OAAO;AACd,cAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACpE,cAAM,IAAI;AAAA,UACR,GAAG,IAAI,UAAU,SAAS,EAAE,iBAAiB,KAAK,WAAW,OAAO,MAAM,MAAM;AAAA,QAClF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AE1JA,SAAS,aAAAC,kBAAiB;AAC1B,SAAS,cAAAC,mBAAkB;AAC3B,SAAS,QAAQ,eAAAC,cAAa,gBAAAC,eAAc,gBAAgB;AAC5D,SAAS,UAAU,QAAAC,aAAY;AAM/B,IAAM,2BAA2B;AAQ1B,IAAM,oBAAN,cAAgC,MAAM;AAAA,EAC3B;AAAA,EAET,YAAY,SAAiB,SAA0C;AAC5E,UAAM,OAAO;AACb,SAAK,OAAO;AACZ,SAAK,UAAU;AAAA,EACjB;AACF;AAEO,SAAS,cACd,aACA,cACqB;AAErB,SAAO,eAAe;AACxB;AAEO,SAAS,eACd,SACA,eAC6B;AAC7B,MAAI,YAAY,OAAW,QAAO;AAClC,QAAM,YAAY,QAAQ,SAAS,SAAY,SAAYC,MAAK,eAAe,QAAQ,IAAI;AAC3F,QAAM,QAAQ,QAAQ,SAAS,CAAC;AAChC,SAAO,EAAE,WAAW,MAAM,oBAAoB,WAAW,KAAK,GAAG,MAAM;AACzE;AAcO,SAAS,mBAAmB,WAAmC;AACpE,QAAM,UAA0B,CAAC;AAEjC,WAAS,MAAM,SAAiB,QAAsB;AACpD,UAAM,UAAU,CAAC,GAAGC,aAAY,SAAS,EAAE,eAAe,KAAK,CAAC,CAAC,EAAE;AAAA,MAAK,CAAC,MAAM,UAC7E,KAAK,KAAK,cAAc,MAAM,IAAI;AAAA,IACpC;AACA,eAAW,UAAU,SAAS;AAC5B,UAAI,oBAAoB,IAAI,OAAO,IAAI,EAAG;AAC1C,YAAM,eAAeD,MAAK,SAAS,OAAO,IAAI;AAC9C,YAAM,OAAO,WAAW,KAAK,OAAO,OAAO,GAAG,MAAM,IAAI,OAAO,IAAI;AACnE,UAAI,OAAO,eAAe,GAAG;AAC3B,cAAM,IAAI;AAAA,UACR,iCAAiC,IAAI;AAAA,QACvC;AAAA,MACF;AACA,UAAI,OAAO,YAAY,GAAG;AACxB,gBAAQ,KAAK,EAAE,cAAc,YAAY,OAAO,MAAM,MAAM,YAAY,CAAC;AACzE,cAAM,cAAc,IAAI;AAAA,MAC1B,WAAW,OAAO,OAAO,GAAG;AAC1B,cAAM,cAAc,SAAS,YAAY,EAAE,OAAO,QAAW;AAC7D,gBAAQ,KAAK,EAAE,cAAc,YAAY,MAAM,MAAM,OAAO,CAAC;AAAA,MAC/D,OAAO;AACL,cAAM,IAAI,MAAM,sCAAsC,IAAI,GAAG;AAAA,MAC/D;AAAA,IACF;AAAA,EACF;AAEA,QAAM,WAAW,EAAE;AACnB,SAAO;AACT;AAEO,SAAS,oBACd,WACA,OACQ;AAIR,QAAM,OAAOE,YAAW,QAAQ;AAChC,MAAI,cAAc,QAAW;AAC3B,eAAW,SAAS,mBAAmB,SAAS,GAAG;AACjD,UAAI,MAAM,SAAS,aAAa;AAC9B,aAAK,OAAO,MAAM,MAAM,IAAI,IAAI;AAChC;AAAA,MACF;AACA,YAAM,UAAUC,cAAa,MAAM,YAAY;AAC/C,WAAK,OAAO,MAAM,MAAM,IAAI,KAAK,MAAM,aAAa,MAAM,GAAG,KAAK,QAAQ,UAAU,IAAI;AACxF,WAAK,OAAO,OAAO;AAAA,IACrB;AAAA,EACF;AACA,OAAK,OAAO,MAAM,KAAK,UAAU,KAAK,CAAC,EAAE;AACzC,SAAO,KAAK,OAAO,KAAK;AAC1B;AAEO,SAAS,aACd,SACA,WACA,MACiC;AACjC,MAAI,QAAQ,cAAc,QAAW;AAGnC,WAAO,QAAQ,WAAW,WAAW;AAAA,MACnC,QAAQ,CAAC,WAAW,CAAC,oBAAoB,IAAI,SAAS,MAAM,CAAC;AAAA,MAC7D,WAAW;AAAA,MACX,kBAAkB;AAAA,IACpB,CAAC;AAAA,EACH;AAEA,QAAM,UAAkC,CAAC;AACzC,aAAW,WAAW,QAAQ,OAAO;AAGnC,UAAM,UAAUC,WAAU,SAAS;AAAA,MACjC,KAAK;AAAA,MACL,UAAU;AAAA,MACV,KAAK,EAAE,MAAM,MAAM,MAAM,QAAQ,IAAI,QAAQ,GAAG;AAAA,MAChD,YAAY;AAAA,MACZ,OAAO;AAAA,MACP,SAAS;AAAA,IACX,CAAC;AACD,YAAQ,KAAK;AAAA,MACX;AAAA,MACA,UAAU,QAAQ;AAAA,MAClB,QAAQ,QAAQ,UAAU;AAAA,MAC1B,QAAQ,QAAQ,UAAU;AAAA,MAC1B,QAAQ,QAAQ,UAAU;AAAA,IAC5B,CAAC;AACD,QAAI,QAAQ,UAAU,UAAa,QAAQ,WAAW,GAAG;AACvD,YAAM,WAAY,QAAQ,OAA6C,SAAS;AAChF,YAAM,SAAS,WACX,mBAAmB,2BAA2B,GAAI,MACjD,QAAQ,OAAO,YACf,QAAQ,WAAW,OAChB,iBAAiB,QAAQ,MAAM,KAC/B,aAAa,QAAQ,MAAM;AACnC,YAAM,IAAI,kBAAkB,0BAA0B,OAAO,MAAM,MAAM,KAAK,OAAO;AAAA,IACvF;AAAA,EACF;AACA,SAAO;AACT;;;AHvJO,IAAM,gBAAN,cAA4B,kBAAkB;AAAA,EAC5C,YAAY,SAAiB,OAAO,qBAAqB;AAC9D,UAAM,SAAS,IAAI;AACnB,SAAK,OAAO;AAAA,EACd;AACF;AAEO,SAAS,aAAa,MAAc,eAAoC;AAC7E,QAAM,QAAQ,cAAcC,cAAa,MAAM,MAAM,GAAG,MAAM,aAAa;AAG3E,QAAM,gBAAgBC,SAAQ,IAAI;AAClC,yBAAuB,MAAM,SAAS,GAAG,IAAI,kBAAkB,aAAa;AAC5E,aAAW,YAAY,MAAM,OAAO;AAClC;AAAA,MACE,SAAS;AAAA,MACT,GAAG,IAAI,UAAU,SAAS,EAAE;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,uBACP,SACA,SACA,eACM;AACN,MAAI,SAAS,SAAS,OAAW;AACjC,QAAM,YAAYC,MAAK,eAAe,QAAQ,IAAI;AAClD,QAAM,QAAQC,UAAS,WAAW,EAAE,gBAAgB,MAAM,CAAC;AAC3D,MAAI,UAAU,QAAW;AACvB,UAAM,IAAI;AAAA,MACR,GAAG,OAAO,UAAU,QAAQ,IAAI;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AACA,MAAI,CAAC,MAAM,YAAY,GAAG;AACxB,UAAM,IAAI;AAAA,MACR,GAAG,OAAO,UAAU,QAAQ,IAAI;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AACA,MAAI;AACF,uBAAmB,SAAS;AAAA,EAC9B,SAAS,OAAO;AACd,UAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACpE,UAAM,IAAI,cAAc,GAAG,OAAO,UAAU,QAAQ,IAAI,KAAK,MAAM,IAAI,sBAAsB;AAAA,EAC/F;AACF;AAEO,SAAS,cACd,QACA,OAAO,gBACP,eACY;AACZ,MAAI;AACJ,MAAI;AACF,aAASC,WAAU,MAAM;AAAA,EAC3B,SAAS,OAAO;AACd,UAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACpE,UAAM,IAAI,cAAc,GAAG,IAAI,qBAAqB,MAAM,IAAI,wBAAwB;AAAA,EACxF;AAEA,MAAI;AACF,WAAO,eAAe,QAAQ,MAAM,aAAa;AAAA,EACnD,SAAS,OAAO;AACd,QAAI,iBAAiB,mBAAmB;AACtC,YAAM,IAAI,cAAc,MAAM,SAAS,MAAM,IAAI;AAAA,IACnD;AACA,UAAM;AAAA,EACR;AACF;;;ADxBO,SAAS,eAAe,OAA2C;AACxE,QAAM,eAAyB,CAAC;AAChC,QAAM,SAA4B,CAAC;AAEnC,aAAW,QAAQ,OAAO;AACxB,QAAI,CAACC,YAAW,IAAI,GAAG;AACrB,mBAAa,KAAK,IAAI;AACtB;AAAA,IACF;AAEA,UAAM,UAAUC,aAAY,MAAM,EAAE,eAAe,KAAK,CAAC,EACtD,OAAO,CAAC,UAAU,MAAM,YAAY,CAAC,EACrC,KAAK,CAAC,MAAM,UAAU,KAAK,KAAK,cAAc,MAAM,IAAI,CAAC;AAC5D,eAAW,SAAS,SAAS;AAC3B,YAAM,iBAAiBC,MAAK,MAAM,MAAM,IAAI;AAC5C,UAAI,CAACF,YAAWE,MAAK,gBAAgB,UAAU,CAAC,EAAG;AACnD,aAAO,KAAK,cAAc,MAAM,MAAM,MAAM,cAAc,CAAC;AAAA,IAC7D;AAAA,EACF;AAEA,SAAO,EAAE,cAAc,OAAO;AAChC;AAEO,SAAS,gBAAgB,WAA6C;AAC3E,SAAO;AAAA,IACL,cAAc,UAAU;AAAA,IACxB,QAAQ,UAAU,OAAO,IAAI,CAAC,UAAU;AACtC,UAAI,MAAM,WAAW,QAAS,QAAO;AAGrC,aAAO;AAAA,QACL,WAAW,MAAM;AAAA,QACjB,OAAO,MAAM;AAAA,QACb,aAAa,MAAM;AAAA,QACnB,MAAM,MAAM;AAAA,QACZ,MAAM,MAAM;AAAA,QACZ,gBAAgB,MAAM;AAAA,QACtB,QAAQ,MAAM;AAAA,QACd,iBAAiB,MAAM;AAAA,MACzB;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEO,SAAS,aACd,WACA,gBACiC;AACjC,QAAM,SAAS,oBAAI,IAA6B;AAChD,aAAW,SAAS,UAAU,QAAQ;AACpC,QAAI,CAAC,OAAO,IAAI,MAAM,IAAI,EAAG,QAAO,IAAI,MAAM,MAAM,KAAK;AAAA,EAC3D;AAEA,MAAI,eAAe,WAAW,GAAG;AAC/B,WAAO,CAAC,GAAG,OAAO,OAAO,CAAC,EAAE;AAAA,MAC1B,CAAC,UAAyC,MAAM,WAAW;AAAA,IAC7D;AAAA,EACF;AAEA,SAAO,eAAe,IAAI,CAAC,SAAS;AAClC,UAAM,QAAQ,OAAO,IAAI,IAAI;AAC7B,QAAI,UAAU,QAAW;AACvB,YAAM,IAAI,MAAM,UAAU,IAAI,oCAAoC;AAAA,IACpE;AACA,QAAI,MAAM,WAAW,WAAW;AAC9B,YAAM,IAAI,MAAM,UAAU,IAAI,uBAAuB;AAAA,IACvD;AACA,QAAI,MAAM,WAAW,WAAW;AAC9B,YAAM,IAAI,MAAM,MAAM,eAAe;AAAA,IACvC;AACA,WAAO;AAAA,EACT,CAAC;AACH;AAEA,SAAS,cAAc,MAAc,MAAc,gBAAyC;AAC1F,QAAM,eAAeA,MAAK,gBAAgB,cAAc;AACxD,MAAI,CAACF,YAAW,YAAY,GAAG;AAC7B,WAAO;AAAA,MACL,WAAW;AAAA,MACX,OAAO;AAAA,MACP,aAAa;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,MACR,iBAAiB;AAAA,IACnB;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACF,YAAQ,aAAa,cAAc,IAAI;AAAA,EACzC,SAAS,OAAO;AACd,WAAO;AAAA,MACL,WAAW;AAAA,MACX,OAAO;AAAA,MACP,aAAa;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,MACR,iBAAiB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IACxE;AAAA,EACF;AAEA,SAAO;AAAA,IACL,WAAW,MAAM,MAAM;AAAA,IACvB,OAAO,MAAM;AAAA,IACb;AAAA,IACA,aAAa;AAAA,IACb;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,IACR,iBAAiB;AAAA,EACnB;AACF;;;AK3KA,SAAS,aAAAG,YAAW,aAAa,QAAQ,iBAAAC,sBAAqB;AAC9D,SAAS,cAAc;AACvB,SAAS,QAAAC,aAAY;;;ACFrB,SAAS,cAAAC,aAAY,aAAAC,YAAW,gBAAAC,eAAc,iBAAAC,sBAAqB;AACnE,SAAS,WAAAC,UAAS,QAAAC,aAAY;AAOvB,IAAM,iBAAiB;AAUvB,IAAM,WAAN,MAAe;AAAA,EACX;AAAA,EAEF,YAAY,iBAAiB,sBAAsB,GAAG;AAC3D,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEO,OAAO,UAAmD;AAC/D,UAAM,OAAO,KAAK,MAAM,QAAQ;AAChC,QAAI,CAACC,YAAW,IAAI,EAAG,QAAO;AAC9B,WAAO,EAAE,GAAI,KAAK,MAAMC,cAAa,MAAM,MAAM,CAAC,GAAiB,QAAQ,KAAK;AAAA,EAClF;AAAA,EAEO,MAAM,UAA4B,QAAyB;AAChE,UAAM,OAAO,KAAK,MAAM,QAAQ;AAChC,IAAAC,WAAUC,SAAQ,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AAC5C,IAAAC,eAAc,MAAM,KAAK,UAAU,MAAM,CAAC;AAAA,EAC5C;AAAA,EAEA,MAAM,UAAoC;AAExC,UAAM,QAAQ;AAAA,MACZ,OAAO,cAAc;AAAA,MACrB,SAAS;AAAA,MACT,KAAK,UAAU,SAAS,QAAQ;AAAA,MAChC,SAAS;AAAA,MACT,SAAS,SAAS;AAAA,MAClB,SAAS,SAAS;AAAA,MAClB,SAAS,SAAS;AAAA,IACpB;AAEA,QAAI,SAAS,gBAAgB,OAAW,OAAM,KAAK,SAAS,WAAW;AACvE,UAAM,MAAM,OAAO,MAAM,KAAK,IAAI,CAAC;AACnC,WAAOC,MAAK,KAAK,iBAAiB,SAAS,GAAG,GAAG,OAAO;AAAA,EAC1D;AACF;;;ACrDA,SAAS,gBAAAC,qBAAoB;AAC7B,SAAS,YAAAC,iBAAgB;AAKzB,IAAM,iBAAiB,oBAAI,IAAI,CAAC,gBAAgB,eAAe,CAAC;AACzD,SAAS,WAAW,UAAoB,KAAU,OAAc,WAA4B;AACjG,QAAM,SAAkB,CAAC;AAEzB,SAAO,KAAK;AAAA,IACV,QAAQ,MAAM,YAAY,wBAAwB;AAAA,IAClD,MAAM;AAAA,IACN,MAAM,MAAM;AAAA,EACd,CAAC;AAED,MAAI,SAAS,mBAAmB,UAAa,QAAQ,SAAS;AAC5D,WAAO,KAAK;AAAA,MACV,QAAQ,WAAW,MAAM,OAAO,cAAc,SAAS,cAAc;AAAA,MACrE,MAAM;AAAA,MACN,MAAM,MAAM,YAAY,SAAS;AAAA,IACnC,CAAC;AAAA,EACH;AAEA,QAAM,aACJ,SAAS,SAAS,eACd,UAAU,SAAS,EAChB,OAAO,CAAC,SAAS,CAAC,eAAe,IAAIC,UAAS,WAAW,IAAI,CAAC,CAAC,EAC/D,IAAI,CAAC,SAAS,OAAOA,UAAS,WAAW,IAAI,CAAC;AAAA,EAASC,cAAa,MAAM,MAAM,CAAC,EAAE,EACnF,KAAK,IAAI,IACZ,MAAM;AAEZ,aAAW,WAAW,SAAS,QAAQ,cAAc,CAAC,GAAG;AACvD,UAAM,OAAO,IAAI,OAAO,SAAS,GAAG,EAAE,KAAK,UAAU;AACrD,WAAO,KAAK;AAAA,MACV,QAAQ,OAAO,UAAU,GAAG,OAAO,WAAW,WAAW,MAAM,GAAG,GAAG,CAAC;AAAA,MACtE,MAAM;AAAA,MACN;AAAA,IACF,CAAC;AAAA,EACH;AACA,aAAW,WAAW,SAAS,QAAQ,kBAAkB,CAAC,GAAG;AAC3D,WAAO,KAAK;AAAA,MACV,QAAQ;AAAA,MACR,MAAM;AAAA,MACN,MAAM,CAAC,IAAI,OAAO,SAAS,GAAG,EAAE,KAAK,UAAU;AAAA,IACjD,CAAC;AAAA,EACH;AAEA,SAAO,KAAK,GAAG,WAAW,UAAU,SAAS,CAAC;AAC9C,SAAO;AACT;;;ACjDO,SAAS,kBAAkB,YAAwC;AACxE,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,cAAc,CAAC,CAAC;AACjD;AAEO,SAAS,YAAY,QAAuD;AACjF,QAAM,SAAS,OAAO,OAAO,CAAC,UAAU,MAAM,IAAI,EAAE;AACpD,SAAO,SAAS,IAAI,OAAO;AAC7B;AAEO,SAAS,eAAe,QAAuD;AACpF,SAAO,IAAI,IAAI,OAAO,IAAI,CAAC,UAAU,MAAM,IAAI,CAAC,EAAE,OAAO,KAAK,OAAO,SAAS;AAChF;;;AHkDO,SAAS,cACd,QACA,SACA,KACY;AACZ,QAAM,YAAY,eAAe,OAAO,KAAK;AAC7C,QAAM,iBAAiB,aAAa,WAAW,QAAQ,eAAe;AACtE,QAAM,WAAW,eAAe,OAAO,QAAQ;AAC/C,QAAM,iBAAiB,sBAAsB;AAC7C,QAAM,QAAQ,IAAI,SAAS,cAAc;AACzC,QAAM,cAAc,eAAe,IAAI,CAAC,WAAW;AAAA,IACjD,aAAa,iBAAiB,MAAM,cAAc;AAAA,IAClD;AAAA,EACF,EAAE;AACF,QAAM,UAAU;AAAA,IACd,YAAY,IAAI,CAAC,EAAE,aAAa,MAAM,OAAO,EAAE,aAAa,MAAM,MAAM,KAAK,EAAE;AAAA,EACjF;AACA,QAAM,eAA4C,CAAC;AACnD,MAAI,WAAW;AACf,MAAI,QAAQ;AAEZ,aAAW,EAAE,aAAa,MAAM,KAAK,aAAa;AAChD,UAAM,QAAQ,MAAM;AACpB,QAAI,GAAG,MAAM,IAAI,KAAK,MAAM,KAAK,KAAK,YAAY,MAAM,GAAG,EAAE,CAAC,IAAI;AAClE,UAAM,QAAQ,MAAM,MACjB,OAAO,CAAC,aAAa,QAAQ,eAAe,UAAa,SAAS,OAAO,QAAQ,UAAU,EAC3F;AAAA,MAAI,CAAC,aACJ;AAAA,QACE;AAAA,UACE;AAAA,UACA;AAAA,UACA;AAAA,UACA,WAAW;AAAA,UACX,cAAc,QAAQ;AAAA,UACtB,UAAU,QAAQ;AAAA,QACpB;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACF,iBAAa,MAAM,IAAI,IAAI,EAAE,OAAO,OAAO,MAAM,OAAO,YAAY;AACpE,gBAAY,MAAM,OAAO,CAAC,WAAW,CAAC,OAAO,IAAI,EAAE;AACnD,aAAS,MAAM,OAAO,CAAC,WAAW,OAAO,IAAI,EAAE;AAAA,EACjD;AAEA,QAAM,SAAoB;AAAA,IACxB,UAAU,SAAS;AAAA,IACnB;AAAA,IACA,QAAQ;AAAA,EACV;AACA,QAAM,kBAAkBC,MAAK,gBAAgB,SAAS;AACtD,QAAM,aAAaA,MAAK,iBAAiB,GAAG,OAAO,OAAO;AAC1D,EAAAC,WAAU,iBAAiB,EAAE,WAAW,KAAK,CAAC;AAC9C,EAAAC,eAAc,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AACzD,SAAO,EAAE,UAAU,OAAO,QAAQ,WAAW;AAC/C;AAEO,SAAS,wBACd,QACQ;AACR,SAAO;AAAA,IACL,CAAC,GAAG,MAAM,EACP,KAAK,CAAC,MAAM,UAAU,KAAK,KAAK,cAAc,MAAM,IAAI,CAAC,EACzD,IAAI,CAAC,EAAE,aAAa,KAAK,MAAM,GAAG,IAAI,KAAK,WAAW,EAAE,EACxD,KAAK,IAAI;AAAA,EACd;AACF;AAEA,SAAS,QACP,SACA,UACA,KACY;AACZ,QAAM,QAAQ,SAAS,QAAQ,CAAC,OAAO,GAAG;AAAA,IACxC,CAAC,QAAQ,QAAQ,WAAW,CAAC,QAAQ;AAAA,EACvC;AAEA,QAAM,UAAU;AAAA,IACd,cAAc,SAAS,SAAS,QAAQ,MAAM,MAAM,OAAO;AAAA,IAC3D,QAAQ,MAAM;AAAA,EAChB;AACA,QAAM,UAAU,KAAK,IAAI,CAAC,QAAQ;AAChC,QAAI,KAAK,SAAS,EAAE,KAAK,GAAG,OAAO;AACnC,UAAM,SAAS;AAAA,MACb;AAAA,QACE,OAAO,QAAQ;AAAA,QACf;AAAA,QACA,UAAU,QAAQ;AAAA,QAClB;AAAA,QACA,OAAO,QAAQ;AAAA,QACf,WAAW,QAAQ;AAAA,QACnB,UAAU,QAAQ;AAAA,MACpB;AAAA,MACA;AAAA,IACF;AACA;AAAA,MACE,KAAK,SAAS,EAAE,KAAK,GAAG,KAAK,OAAO,OAAO,SAAS,MAAM,GACxD,OAAO,SAAS,cAAc,EAChC;AAAA,IACF;AACA,WAAO;AAAA,EACT,CAAC;AACD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,IAAI,SAAS;AAAA,IACb,MAAM,QAAQ,KAAK,CAAC,WAAW,OAAO,QAAQ,UAAU,GAAG,SAAS;AAAA,IACpE,MAAM,QAAQ,KAAK,CAAC,WAAW,OAAO,QAAQ,OAAO,GAAG,SAAS;AAAA,IACjE,MAAM,SAAS;AAAA,EACjB;AACF;AAEA,SAAS,OAAO,SAAqB,KAAqB;AACxD,QAAM,WAAW;AAAA,IACf;AAAA,IACA,UAAU,QAAQ;AAAA,IAClB,UAAU,QAAQ,SAAS;AAAA,IAC3B,aAAa,QAAQ,SAAS;AAAA,IAC9B,WAAW,QAAQ;AAAA,EACrB;AACA,MAAI,QAAQ,UAAU;AACpB,UAAM,MAAM,QAAQ,MAAM,OAAO,QAAQ;AACzC,QAAI,QAAQ,OAAW,QAAO;AAAA,EAChC;AAEA,QAAM,SAAwB,CAAC;AAC/B,QAAM,SAAS,kBAAkB,QAAQ,SAAS,MAAM;AACxD,WAAS,QAAQ,GAAG,QAAQ,QAAQ,SAAS,GAAG;AAC9C,WAAO,KAAK,SAAS,SAAS,GAAG,CAAC;AAAA,EACpC;AAEA,SAAO,eAAe,MAAM,EAAG,QAAO,KAAK,SAAS,SAAS,GAAG,CAAC;AAEjE,QAAM,SAAoB;AAAA,IACxB;AAAA,IACA,QAAQ;AAAA,IACR,MAAM,YAAY,MAAM;AAAA,IACxB;AAAA,EACF;AACA,UAAQ,MAAM,MAAM,UAAU,MAAM;AACpC,SAAO;AACT;AAEA,SAAS,SAAS,SAAqB,KAAuB;AAE5D,QAAM,YAAY,YAAYF,MAAK,OAAO,GAAG,YAAY,QAAQ,SAAS,EAAE,GAAG,CAAC;AAChF,QAAM,YAAY,YAAYA,MAAK,OAAO,GAAG,gBAAgB,CAAC;AAE9D,MAAI;AACF,UAAM,eACJ,QAAQ,YAAY,SAChB,SACA,aAAa,QAAQ,SAAS,WAAW,SAAS;AACxD,UAAM,QAAQ,QAAQ,SAAS,SAAS;AAAA,MACtC;AAAA,MACA,UAAU,QAAQ;AAAA,MAClB,MAAM;AAAA,MACN,gBAAgB,QAAQ,MAAM;AAAA,MAC9B,WAAW,QAAQ,MAAM;AAAA,MACzB;AAAA,IACF,CAAC;AACD,UAAM,SAAS,WAAW,QAAQ,UAAU,KAAK,OAAO,SAAS;AACjE,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,MAAM,OAAO,MAAM,CAAC,UAAU,MAAM,IAAI;AAAA,MACxC,OAAO,MAAM;AAAA,IACf;AAAA,EACF,SAAS,OAAO;AACd,QAAI,iBAAiB,mBAAmB;AAEtC,aAAO;AAAA,QACL,QAAQ,CAAC,EAAE,QAAQ,MAAM,SAAS,MAAM,iBAAiB,MAAM,MAAM,CAAC;AAAA,QACtE,cAAc,MAAM;AAAA,QACpB,MAAM;AAAA,QACN,OAAO;AAAA,MACT;AAAA,IACF;AACA,UAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACpE,WAAO;AAAA,MACL,QAAQ,CAAC,EAAE,QAAQ,MAAM,OAAO,MAAM,MAAM,CAAC;AAAA,MAC7C,MAAM;AAAA,MACN,OAAO;AAAA,IACT;AAAA,EACF,UAAE;AAGA,WAAO,WAAW,EAAE,OAAO,MAAM,WAAW,KAAK,CAAC;AAClD,WAAO,WAAW,EAAE,OAAO,MAAM,WAAW,KAAK,CAAC;AAAA,EACpD;AACF;;;AXpOA,IAAM,UAAU,IAAI,QAAQ;AAE5B,QACG,KAAK,UAAU,EACf,YAAY,kDAAkD,EAC9D,cAAc,EAAE,mBAAmB,KAAK,CAAC,EACzC,QAAQ,OAAO,EACf,OAAO,mBAAmB,mCAAmC;AAEhE,QACG,QAAQ,KAAK,EACb;AAAA,EACC;AACF,EACC,SAAS,cAAc,wDAAwD,EAC/E,OAAO,eAAe,gCAAgC,EACtD,OAAO,cAAc,2BAA2B,EAChD,OAAO,mBAAmB,0BAA0B,EACpD,OAAO,UAAU,oCAAoC,EACrD,OAAO,CAAC,QAAkB,SAA4B,YAA2B;AAChF,QAAM,gBAAgB,QAAQ,gBAAgB;AAC9C,QAAM,aAAa,kBAAkB,EAAE,SAAS,cAAc,OAAO,CAAC;AACtE,QAAM,SAAS,WAAW,UAAU;AACpC,QAAM,UAAU;AAAA,IACd;AAAA,IACA;AAAA,MACE,YAAY,QAAQ;AAAA,MACpB,iBAAiB;AAAA,MACjB,cAAc,QAAQ,iBAAiB;AAAA,MACvC,UAAU,QAAQ;AAAA,IACpB;AAAA,IACA,QAAQ,SAAS,OAAO,MAAM,SAAY,CAAC,YAAY,QAAQ,IAAI,OAAO;AAAA,EAC5E;AAEA,MAAI,QAAQ,SAAS,MAAM;AACzB,YAAQ,IAAI,KAAK,UAAU,QAAQ,QAAQ,MAAM,CAAC,CAAC;AAAA,EACrD,OAAO;AACL,YAAQ,IAAI,WAAW,QAAQ,UAAU,EAAE;AAC3C,QAAI,QAAQ,QAAQ,GAAG;AACrB,cAAQ,IAAI,gBAAgB,QAAQ,KAAK,8CAA8C;AAAA,IACzF;AACA,YAAQ;AAAA,MACN,QAAQ,aAAa,IAAI,qBAAqB,GAAG,QAAQ,QAAQ;AAAA,IACnE;AAAA,EACF;AACA,UAAQ,WAAW,QAAQ,WAAW,IAAI,IAAI;AAChD,CAAC;AAEH,QACG,QAAQ,MAAM,EACd;AAAA,EACC;AACF,EACC,OAAO,UAAU,8CAA8C,EAC/D,OAAO,CAAC,SAAsB,YAA2B;AACxD,QAAM,gBAAgB,QAAQ,gBAAgB;AAC9C,QAAM,aAAa,kBAAkB,EAAE,SAAS,cAAc,OAAO,CAAC;AACtE,QAAM,SAAS,WAAW,UAAU;AACpC,QAAM,YAAY,eAAe,OAAO,KAAK;AAC7C,MAAI,QAAQ,SAAS,MAAM;AACzB,YAAQ,IAAI,KAAK,UAAU,gBAAgB,SAAS,GAAG,MAAM,CAAC,CAAC;AAC/D;AAAA,EACF;AAEA,kBAAgB,UAAU,MAAM;AAChC,aAAW,SAAS,UAAU,QAAQ;AACpC,QAAI,MAAM,WAAW,UAAW,SAAQ,IAAI,kBAAkB,MAAM,eAAe,EAAE;AAAA,EACvF;AACA,aAAW,QAAQ,UAAU,aAAc,SAAQ,IAAI,iBAAiB,IAAI,EAAE;AAChF,CAAC;AAEH,eAAsB,OAAsB;AAC1C,MAAI;AACF,UAAM,QAAQ,WAAW,QAAQ,IAAI;AAAA,EACvC,SAAS,OAAO;AACd,YAAQ,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC,EAAE;AAChF,YAAQ,WAAW;AAAA,EACrB;AACF;AAEA,SAAS,gBAAgB,QAA0C;AACjE,QAAM,OAAO;AAAA,IACX,CAAC,QAAQ,QAAQ,SAAS,SAAS,kBAAkB;AAAA,IACrD,GAAG,OAAO,IAAI,CAAC,UAAU;AAAA,MACvB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,MAAM,SAAS;AAAA,MACf,OAAO,MAAM,SAAS;AAAA,MACtB,MAAM,cAAc,SAAS;AAAA,IAC/B,CAAC;AAAA,EACH;AACA,QAAM,SAAS,KAAK,CAAC,GAAG;AAAA,IAAI,CAAC,GAAG,WAC9B,KAAK,IAAI,GAAG,KAAK,IAAI,CAAC,QAAQ,IAAI,MAAM,GAAG,UAAU,CAAC,CAAC;AAAA,EACzD;AACA,aAAW,OAAO,MAAM;AACtB,YAAQ;AAAA,MACN,IACG,IAAI,CAAC,MAAM,WAAW,KAAK,OAAO,SAAS,MAAM,KAAK,KAAK,MAAM,CAAC,EAClE,KAAK,IAAI,EACT,QAAQ;AAAA,IACb;AAAA,EACF;AACF;AAEA,MAAM,KAAK;","names":["readFileSync","homedir","readFileSync","join","join","readFileSync","homedir","readFileSync","existsSync","readdirSync","join","readFileSync","statSync","dirname","join","parseYaml","Type","checkSchema","schemaErrors","spawnSync","existsSync","join","Type","checkSchema","schemaErrors","spawnSync","createHash","readdirSync","readFileSync","join","join","readdirSync","createHash","readFileSync","spawnSync","readFileSync","dirname","join","statSync","parseYaml","existsSync","readdirSync","join","mkdirSync","writeFileSync","join","existsSync","mkdirSync","readFileSync","writeFileSync","dirname","join","existsSync","readFileSync","mkdirSync","dirname","writeFileSync","join","readFileSync","relative","relative","readFileSync","join","mkdirSync","writeFileSync"]}

package/package.json

@@ -0,0 +1,70 @@
+{
+  "name": "@dungle-scrubs/skillval",
+  "version": "0.1.0",
+  "description": "Deterministic evaluation for agent skills",
+  "keywords": [
+    "agent-skills",
+    "agents",
+    "cli",
+    "codex",
+    "evaluation",
+    "skill-md",
+    "testing"
+  ],
+  "type": "module",
+  "bin": {
+    "skillval": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "schemas",
+    "LICENSE",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup src/cli.ts --clean --format esm --platform node --sourcemap --target node22",
+    "format": "biome format --write .",
+    "lint": "biome check .",
+    "prepare": "pnpm build",
+    "schema": "tsx scripts/generate-schemas.ts",
+    "schema:check": "tsx scripts/generate-schemas.ts --check",
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/dungle-scrubs/skillval.git"
+  },
+  "bugs": {
+    "url": "https://github.com/dungle-scrubs/skillval/issues"
+  },
+  "homepage": "https://github.com/dungle-scrubs/skillval#readme",
+  "license": "MIT",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "packageManager": "pnpm@10.28.2",
+  "dependencies": {
+    "@types/node": "^26.0.1",
+    "commander": "^15.0.0",
+    "typebox": "^1.3.6",
+    "typescript": "^6.0.3",
+    "yaml": "^2.9.0"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "2.5.5",
+    "lefthook": "^2.1.10",
+    "tsup": "^8.5.0",
+    "tsx": "^4.23.1",
+    "vitest": "^4.1.9"
+  },
+  "pnpm": {
+    "overrides": {
+      "esbuild": ">=0.28.1"
+    }
+  }
+}

package/schemas/config.schema.json

@@ -0,0 +1,23 @@
+{
+  "type": "object",
+  "required": ["executor", "roots"],
+  "properties": {
+    "executor": {
+      "enum": ["codex"],
+      "description": "Trial executor."
+    },
+    "roots": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1,
+        "pattern": "\\S"
+      },
+      "description": "Directories whose immediate children are agent skill directories."
+    }
+  },
+  "$id": "https://raw.githubusercontent.com/dungle-scrubs/skillval/main/schemas/config.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "additionalProperties": false,
+  "title": "skillval configuration"
+}

package/schemas/skillval.schema.json

@@ -0,0 +1,131 @@
+{
+  "type": "object",
+  "required": ["cases", "class", "skill"],
+  "properties": {
+    "cases": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["id", "mode", "prompt"],
+        "properties": {
+          "arms": {
+            "type": "array",
+            "items": {
+              "enum": ["baseline", "skill"]
+            },
+            "uniqueItems": true
+          },
+          "assert": {
+            "type": "object",
+            "properties": {
+              "graders": {
+                "type": "array",
+                "items": {
+                  "enum": ["tsc"]
+                },
+                "uniqueItems": true
+              },
+              "must_match": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "must_not_match": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              }
+            },
+            "additionalProperties": false
+          },
+          "fixture": {
+            "type": "object",
+            "properties": {
+              "path": {
+                "type": "string",
+                "description": "Directory relative to skillval.yml, copied into the workspace.",
+                "minLength": 1,
+                "pattern": "\\S"
+              },
+              "setup": {
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "minLength": 1,
+                  "pattern": "\\S"
+                },
+                "description": "Shell commands run sequentially inside the workspace after the copy."
+              }
+            },
+            "additionalProperties": false,
+            "minProperties": 1
+          },
+          "id": {
+            "type": "string",
+            "minLength": 1,
+            "pattern": "\\S"
+          },
+          "mode": {
+            "enum": ["generation", "trigger"]
+          },
+          "prompt": {
+            "type": "string",
+            "minLength": 1,
+            "pattern": "\\S"
+          },
+          "rule": {
+            "type": "string"
+          },
+          "should_trigger": {
+            "type": "boolean"
+          },
+          "trials": {
+            "type": "integer",
+            "maximum": 5,
+            "minimum": 1
+          },
+          "type": {
+            "enum": ["capability", "preference"]
+          }
+        },
+        "additionalProperties": false
+      }
+    },
+    "class": {
+      "enum": ["capability", "preference"]
+    },
+    "fixture": {
+      "type": "object",
+      "properties": {
+        "path": {
+          "type": "string",
+          "description": "Directory relative to skillval.yml, copied into the workspace.",
+          "minLength": 1,
+          "pattern": "\\S"
+        },
+        "setup": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "minLength": 1,
+            "pattern": "\\S"
+          },
+          "description": "Shell commands run sequentially inside the workspace after the copy."
+        }
+      },
+      "additionalProperties": false,
+      "minProperties": 1
+    },
+    "skill": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "\\S"
+    }
+  },
+  "$id": "https://raw.githubusercontent.com/dungle-scrubs/skillval/main/schemas/skillval.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "additionalProperties": false,
+  "title": "skillval case file"
+}