@iowarp/clio-coder 0.4.6 → 0.4.8

package/dist/skills-eval-J5TCKSJA.js

@@ -1,1034 +0,0 @@
-import { createRequire as __clioCreateRequire } from "node:module"; const require = __clioCreateRequire(import.meta.url);
-import {
-  parseSkillEvals
-} from "./chunk-QCOV5DFY.js";
-import "./chunk-XWMSPJYN.js";
-import "./chunk-VAHLPXPY.js";
-import "./chunk-3UW3KU4A.js";
-import {
-  discoverMarketplaceSkills,
-  loadSkills
-} from "./chunk-536HXRFE.js";
-import "./chunk-DNCE23JJ.js";
-import "./chunk-U7H5WRG5.js";
-import "./chunk-5VTJP36Q.js";
-import {
-  buildEvalEvidence,
-  summarizeEvalResults
-} from "./chunk-24T2YZ7A.js";
-import "./chunk-PECBKCDK.js";
-import "./chunk-ODUTYZIU.js";
-import "./chunk-P5SC4DLJ.js";
-import "./chunk-VHMX7SXF.js";
-import "./chunk-ZS43Y2FT.js";
-import {
-  evalClioProvenance,
-  evalEnvironmentProvenance
-} from "./chunk-HZ2T2YED.js";
-import "./chunk-DXLUVUD4.js";
-import {
-  ZERO_EVAL_HARNESS_METRICS,
-  writeEvalArtifact
-} from "./chunk-U44YYLVV.js";
-import "./chunk-UQIVHSE4.js";
-import "./chunk-USZA7GWL.js";
-import "./chunk-LVFANMOH.js";
-import "./chunk-WTQGRBHV.js";
-import "./chunk-7WUZEB2M.js";
-import "./chunk-O5F25NIE.js";
-import "./chunk-IGLVVNEG.js";
-import "./chunk-3WBALRQH.js";
-import "./chunk-CXQ37WJH.js";
-import "./chunk-6OII3LOK.js";
-import "./chunk-7YJWEF2L.js";
-import "./chunk-QH3ZTUOA.js";
-import "./chunk-LQTKDAUO.js";
-import "./chunk-22PQ4TX4.js";
-import "./chunk-DZK63TZJ.js";
-import "./chunk-TJFDL26W.js";
-import "./chunk-SD7SHNDW.js";
-import "./chunk-T52TPHYC.js";
-import "./chunk-DNHVRBH4.js";
-import {
-  formatColumns,
-  printError
-} from "./chunk-3T4DXWP6.js";
-import {
-  HEADLESS_PERMISSION_DENIED_MARKER
-} from "./chunk-P5JMBPDJ.js";
-import "./chunk-K2HL3GBL.js";
-import "./chunk-YPRSNUTC.js";
-import "./chunk-2GPGAE5W.js";
-import {
-  NO_NETWORK_TOOLS_ENV
-} from "./chunk-SL2RDHKO.js";
-import "./chunk-YGDMBK7O.js";
-import "./chunk-V552WL53.js";
-import "./chunk-7JSSKXG2.js";
-import "./chunk-KECNXMDE.js";
-import "./chunk-WXA2CFJO.js";
-import "./chunk-CBSKDFWL.js";
-import "./chunk-LY4N4TTL.js";
-import "./chunk-OQM5IVJY.js";
-import "./chunk-L4N3DWNW.js";
-import "./chunk-WJIMKGD6.js";
-import "./chunk-GH5ZR6JP.js";
-import "./chunk-VQD7NH7Z.js";
-import "./chunk-O3VV6IVK.js";
-import "./chunk-KAFMW7WM.js";
-import "./chunk-I2K2NCZQ.js";
-import "./chunk-IKRWLEDA.js";
-import "./chunk-NTK3WTHM.js";
-import "./chunk-4U4TKZFR.js";
-import "./chunk-MCJKEGHO.js";
-import "./chunk-77CZRFTE.js";
-import "./chunk-LUL5Z66S.js";
-import "./chunk-M6TAYK66.js";
-import "./chunk-KQ6IDQIB.js";
-import "./chunk-PZN2LQY6.js";
-import "./chunk-R2P2U3ZS.js";
-import "./chunk-NTTS3O53.js";
-import "./chunk-M4KFUXS7.js";
-import "./chunk-J2FYEQ5L.js";
-import "./chunk-EM2WNQS2.js";
-import "./chunk-WNGF5AJX.js";
-import "./chunk-DVNAPPXU.js";
-import "./chunk-5CY4Q6QI.js";
-import "./chunk-FQOOGXBH.js";
-import "./chunk-UHW5V7HC.js";
-import "./chunk-B6UM3OZC.js";
-import "./chunk-MYI32N4R.js";
-import "./chunk-EHOSNVVM.js";
-import "./chunk-KMF54VH3.js";
-import {
-  combineBashOutput,
-  runBashCommand
-} from "./chunk-5A6XR3RJ.js";
-import "./chunk-EFSEIZE6.js";
-import "./chunk-B5ZTR6KZ.js";
-import "./chunk-HL26ADLH.js";
-import "./chunk-VHKTOLEW.js";
-import "./chunk-KPRLAMP4.js";
-import "./chunk-W2RJEH6Q.js";
-import "./chunk-CVDAFGPW.js";
-import "./chunk-PPEZJWUG.js";
-import "./chunk-WDV62RKY.js";
-import "./chunk-GD55RTIW.js";
-import "./chunk-E3LEPLQ2.js";
-import "./chunk-WV6TDAPN.js";
-import "./chunk-3BJGMT2D.js";
-import "./chunk-U2MSJ57E.js";
-import "./chunk-C27U2TYG.js";
-import "./chunk-GJ24ODAX.js";
-import "./chunk-LTR5QCLS.js";
-import "./chunk-6EXUUOSN.js";
-import "./chunk-ZTNFYMIO.js";
-import "./chunk-HHVJNVUJ.js";
-import "./chunk-HXHFDHB6.js";
-import "./chunk-VNBICSS2.js";
-import "./chunk-2QPI3TR3.js";
-import "./chunk-TK2BMCBZ.js";
-import "./chunk-B5GGDIHT.js";
-import "./chunk-ZZFWWYLX.js";
-import "./chunk-SRCEQKQS.js";
-import "./chunk-TN5FW4XY.js";
-import "./chunk-5CMARGPF.js";
-import "./chunk-DXVMLM3Z.js";
-import "./chunk-CKK25WN3.js";
-import {
-  clioDataDir,
-  clioStateDir
-} from "./chunk-CTTI7G3T.js";
-import {
-  init_esm_shims
-} from "./chunk-ECBHH7LU.js";
-
-// src/cli/skills-eval.ts
-init_esm_shims();
-import { spawn } from "node:child_process";
-import { createHash, randomBytes } from "node:crypto";
-import { existsSync } from "node:fs";
-import { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join, resolve } from "node:path";
-import { performance } from "node:perf_hooks";
-var DEFAULT_RUN_TIMEOUT_MS = 6e5;
-var ARM_AUTONOMY = "full-auto";
-var CHILD_OUTPUT_LIMIT = 4e6;
-var PREVIEW_MAX_CHARS = 300;
-var TRANSCRIPT_HEAD_CHARS = 9e3;
-var TRANSCRIPT_TAIL_CHARS = 4e3;
-var SKILL_EVAL_SIDECAR = "skill-eval.json";
-async function runSkillsEvalCommand(nameOrPath, options) {
-  const resolved = resolveSkillBaseDir(nameOrPath, process.cwd());
-  if (resolved.baseDir === null) {
-    printError(resolved.error ?? `unknown skill: ${nameOrPath}`);
-    return 2;
-  }
-  const list = loadSkills({ disableDiscovery: true, explicitSkillPaths: [resolved.baseDir] });
-  const skill = list.items[0];
-  if (skill === void 0) {
-    printError(`skill did not load from ${resolved.baseDir}: ${list.diagnostics.map((d) => d.message).join("; ")}`);
-    return 2;
-  }
-  process.stderr.write(
-    `clio-coder skills eval: ${skill.name} from ${resolved.origin ?? "unknown"} at ${resolved.baseDir} (sha256 ${skill.normalizedHash.slice(0, 12)}\u2026)
-`
-  );
-  const evalsPath = join(resolved.baseDir, "evals.md");
-  let evalsRaw;
-  try {
-    evalsRaw = await readFile(evalsPath, "utf8");
-  } catch {
-    printError(`skill ${skill.name} has no evals.md at ${evalsPath}`);
-    return 2;
-  }
-  const parsed = parseSkillEvals(evalsRaw);
-  for (const diagnostic of parsed.diagnostics) {
-    process.stderr.write(`clio-coder skills eval: ${diagnostic}
-`);
-  }
-  const matcher = options.scenario === void 0 ? null : scenarioMatcher(options.scenario);
-  if (options.scenario !== void 0 && matcher === null) {
-    printError(`invalid --scenario "${options.scenario}": use a scenario id like S1 or a bare number`);
-    return 2;
-  }
-  const scenarios = matcher === null ? parsed.scenarios : parsed.scenarios.filter(matcher);
-  if (scenarios.length === 0) {
-    printError(
-      matcher === null ? `no parseable scenarios in ${evalsPath}` : `scenario ${options.scenario} not found in ${evalsPath} (have: ${parsed.scenarios.map((s) => s.id).join(", ") || "none"})`
-    );
-    return 2;
-  }
-  process.stderr.write(`clio-coder skills eval: ${describeArmPolicy(options.allowNetwork)}
-`);
-  const childEnv = evalChildEnv(options.allowNetwork);
-  const timeoutMs = options.timeoutSeconds !== void 0 ? options.timeoutSeconds * 1e3 : DEFAULT_RUN_TIMEOUT_MS;
-  const workspaceOverride = await resolveWorkspaceOverride(options.workspace);
-  if (typeof workspaceOverride !== "string" && workspaceOverride !== null) {
-    printError(workspaceOverride.error);
-    return 2;
-  }
-  const startedAt = (/* @__PURE__ */ new Date()).toISOString();
-  const outcomes = [];
-  for (const scenario of scenarios) {
-    process.stderr.write(`clio-coder skills eval: ${skill.name} ${scenario.id} baseline/treatment/judge...
-`);
-    outcomes.push(
-      await runScenario(
-        skill.name,
-        resolved.baseDir,
-        scenario,
-        options.target,
-        timeoutMs,
-        workspaceOverride,
-        options.trustFixtures,
-        childEnv
-      )
-    );
-  }
-  const endedAt = (/* @__PURE__ */ new Date()).toISOString();
-  const artifact = synthesizeArtifact(skill.name, evalsPath, evalsRaw, startedAt, endedAt, outcomes, options.target);
-  let evidenceId = null;
-  let evidenceDirectory = null;
-  const evidenceErrors = [];
-  try {
-    await writeEvalArtifact(clioDataDir(), artifact);
-    const built = await buildEvalEvidence({
-      dataDir: clioDataDir(),
-      stateDir: clioStateDir(),
-      artifact,
-      sidecars: [SKILL_EVAL_SIDECAR]
-    });
-    evidenceId = built.evidenceId;
-    evidenceDirectory = built.directory;
-    await writeFile(
-      join(built.directory, SKILL_EVAL_SIDECAR),
-      `${JSON.stringify(sidecar(skill.name, artifact.evalId, outcomes, options.allowNetwork), null, 2)}
-`,
-      "utf8"
-    );
-  } catch (error) {
-    evidenceErrors.push(error instanceof Error ? error.message : String(error));
-  }
-  for (const message of evidenceErrors) {
-    process.stderr.write(`clio-coder skills eval: evidence build failed: ${message}
-`);
-  }
-  if (options.json) {
-    for (const outcome of outcomes) {
-      for (const bullet of outcome.bullets) {
-        process.stdout.write(
-          `${JSON.stringify({
-            schema: "experimental",
-            kind: "skill-eval-bullet",
-            skill: skill.name,
-            scenario: outcome.scenario.id,
-            title: outcome.scenario.title,
-            bullet: bullet.index,
-            expected: bullet.text,
-            verdict: bullet.verdict,
-            reason: bullet.reason,
-            network: networkPolicyLabel(options.allowNetwork),
-            autonomy: ARM_AUTONOMY,
-            baselineSessionId: outcome.baseline?.sessionId ?? null,
-            treatmentSessionId: outcome.treatment?.sessionId ?? null,
-            judgeSessionId: outcome.judge?.sessionId ?? null,
-            evalId: artifact.evalId,
-            evidenceId
-          })}
-`
-        );
-      }
-    }
-  } else {
-    printHumanReport(skill.name, outcomes, artifact.evalId, evidenceId, evidenceDirectory, options.allowNetwork);
-  }
-  const anyFailure = outcomes.some(
-    (outcome) => outcome.bullets.some((bullet) => bullet.verdict === "fail" || bullet.verdict === "error")
-  );
-  const anyUnmeasured = outcomes.some((outcome) => outcome.bullets.some((bullet) => bullet.verdict === "unmeasured"));
-  if (anyFailure) return 1;
-  return anyUnmeasured || evidenceErrors.length > 0 ? 3 : 0;
-}
-function networkPolicyLabel(allowNetwork) {
-  return allowNetwork ? "allowed" : "retrieve-disabled";
-}
-function describeArmPolicy(allowNetwork) {
-  const network = allowNetwork ? "network allowed (--allow-network), so arm runs keep the web tools" : "retrieval tools disabled in every arm run; bash networking requires OS isolation (pass --allow-network to keep web tools)";
-  return `policy: autonomy ${ARM_AUTONOMY} for the baseline and treatment arms in per-run disposable workspaces; ${network}`;
-}
-function describeArmPolicyOutcome(allowNetwork) {
-  const network = allowNetwork ? "network: allowed (--allow-network); arm runs kept the web tools" : "retrieval: disabled; bash networking was not constrained";
-  return `policy: the baseline and treatment arms ran at autonomy ${ARM_AUTONOMY}; ${network}`;
-}
-function armRunArgs(arm, prompt, options = {}) {
-  const args = ["run", "--json", "--json-events", "full", "--no-skills"];
-  if (arm !== "judge") args.push("--autonomy", ARM_AUTONOMY);
-  if (options.skillBaseDir !== void 0) args.push("--skill", options.skillBaseDir);
-  if (options.target !== void 0) args.push("--target", options.target);
-  args.push(prompt);
-  return args;
-}
-function permissionWallReason(arm, run) {
-  const haystack = `${run.transcript}
-${run.finalText}`;
-  let blocked = 0;
-  let at = haystack.indexOf(HEADLESS_PERMISSION_DENIED_MARKER);
-  while (at >= 0) {
-    blocked += 1;
-    at = haystack.indexOf(HEADLESS_PERMISSION_DENIED_MARKER, at + HEADLESS_PERMISSION_DENIED_MARKER.length);
-  }
-  if (blocked === 0) return null;
-  return `${arm} arm transcript carries the headless permission wall ${blocked} time(s) ("${HEADLESS_PERMISSION_DENIED_MARKER}"): this run measured the harness's own autonomy gate, not the skill`;
-}
-function unmeasuredBullets(scenario, reason) {
-  return scenario.expected.map((text, index) => ({ index: index + 1, text, verdict: "unmeasured", reason }));
-}
-function evalChildEnv(allowNetwork, env = process.env) {
-  const childEnv = { ...env };
-  if (allowNetwork) {
-    Reflect.deleteProperty(childEnv, NO_NETWORK_TOOLS_ENV);
-    Reflect.deleteProperty(childEnv, "CLIO_CODER_NO_NETWORK_TOOLS");
-  } else childEnv[NO_NETWORK_TOOLS_ENV] = "1";
-  return childEnv;
-}
-function resolveSkillBaseDir(nameOrPath, cwd) {
-  const asPath = resolve(nameOrPath);
-  if (existsSync(join(asPath, "SKILL.md"))) return { baseDir: asPath, origin: "path" };
-  const discovered = loadSkills({ cwd }).items.find((skill) => skill.name === nameOrPath);
-  if (discovered !== void 0) {
-    return { baseDir: discovered.baseDir, origin: `${discovered.source}/${discovered.scope}` };
-  }
-  const marketplace = discoverMarketplaceSkills({ cwd });
-  const entry = marketplace.skills.find((skill) => skill.name === nameOrPath && skill.origin === "catalog");
-  if (entry !== void 0) return { baseDir: entry.sourceUrl, origin: "catalog" };
-  return {
-    baseDir: null,
-    error: `skill not found: ${nameOrPath} (expected an installed skill, a catalog skill name, or a directory containing SKILL.md)`
-  };
-}
-function scenarioMatcher(value) {
-  const trimmed = value.trim();
-  const full = /^([A-Za-z])(\d+)$/.exec(trimmed);
-  if (full !== null) {
-    const id = `${(full[1] ?? "").toUpperCase()}${Number.parseInt(full[2] ?? "", 10)}`;
-    return (scenario) => scenario.id === id;
-  }
-  const bare = /^(\d+)$/.exec(trimmed);
-  if (bare !== null) {
-    const wanted = Number.parseInt(bare[1] ?? "", 10);
-    return (scenario) => scenario.number === wanted;
-  }
-  return null;
-}
-async function resolveWorkspaceOverride(workspace) {
-  if (workspace === void 0) return null;
-  const resolved = resolve(workspace);
-  try {
-    const info = await stat(resolved);
-    if (!info.isDirectory()) return { error: `--workspace is not a directory: ${resolved}` };
-    return resolved;
-  } catch (error) {
-    const detail = error instanceof Error ? error.message : String(error);
-    return { error: `--workspace is not readable: ${resolved}: ${detail}` };
-  }
-}
-async function runScenario(skillName, skillBaseDir, scenario, target, timeoutMs, workspaceOverride, trustFixtures, childEnv) {
-  const scenarioStart = performance.now();
-  const workspace = await mkdtemp(join(tmpdir(), "clio-coder-skill-eval-seed-"));
-  let runWorkspaces = null;
-  try {
-    if (workspaceOverride !== null) await copyWorkspace(workspaceOverride, workspace);
-    const fixtureError = await runFixtureCommands(scenario, workspace, timeoutMs, trustFixtures);
-    if (fixtureError !== null) {
-      return await completeScenarioOutcome({
-        scenario,
-        bullets: errorBullets(scenario, fixtureError),
-        baseline: null,
-        treatment: null,
-        judge: null,
-        workspace,
-        wallTimeMs: Math.round(performance.now() - scenarioStart),
-        infraError: fixtureError
-      });
-    }
-    runWorkspaces = await materializeSkillEvalWorkspaces(workspace);
-    const baseline = await captureHeadlessRun(
-      armRunArgs("baseline", scenario.setup, { target }),
-      runWorkspaces.baseline,
-      timeoutMs,
-      childEnv
-    );
-    const treatment = await captureHeadlessRun(
-      armRunArgs("treatment", `/skill ${skillName} ${scenario.setup}`, { target, skillBaseDir }),
-      runWorkspaces.treatment,
-      timeoutMs,
-      childEnv
-    );
-    const infra = runInfraError("baseline", baseline) ?? runInfraError("treatment", treatment);
-    if (infra !== null) {
-      return await completeScenarioOutcome({
-        scenario,
-        bullets: errorBullets(scenario, infra),
-        baseline,
-        treatment,
-        judge: null,
-        workspace,
-        wallTimeMs: Math.round(performance.now() - scenarioStart),
-        infraError: infra
-      });
-    }
-    const wall = permissionWallReason("baseline", baseline) ?? permissionWallReason("treatment", treatment);
-    if (wall !== null) {
-      return await completeScenarioOutcome({
-        scenario,
-        bullets: unmeasuredBullets(scenario, wall),
-        baseline,
-        treatment,
-        judge: null,
-        workspace,
-        wallTimeMs: Math.round(performance.now() - scenarioStart),
-        infraError: wall
-      });
-    }
-    const judge = await captureHeadlessRun(
-      armRunArgs("judge", judgePrompt(scenario, baseline.transcript, treatment.transcript), { target }),
-      runWorkspaces.judge,
-      timeoutMs,
-      childEnv
-    );
-    const judgeInfra = runInfraError("judge", judge);
-    if (judgeInfra !== null) {
-      return await completeScenarioOutcome({
-        scenario,
-        bullets: errorBullets(scenario, judgeInfra),
-        baseline,
-        treatment,
-        judge,
-        workspace,
-        wallTimeMs: Math.round(performance.now() - scenarioStart),
-        infraError: judgeInfra
-      });
-    }
-    const bullets = parseJudgeVerdicts(scenario, judge);
-    return await completeScenarioOutcome({
-      scenario,
-      bullets,
-      baseline,
-      treatment,
-      judge,
-      workspace,
-      wallTimeMs: Math.round(performance.now() - scenarioStart),
-      infraError: null
-    });
-  } finally {
-    try {
-      if (runWorkspaces !== null) await runWorkspaces.cleanup();
-    } finally {
-      await rm(workspace, { recursive: true, force: true });
-    }
-  }
-}
-async function armWorkspace(created) {
-  const root = await mkdtemp(join(tmpdir(), "clio-coder-skill-eval-"));
-  created.push(root);
-  const workspace = join(root, "workspace");
-  await mkdir(workspace, { recursive: true });
-  return workspace;
-}
-async function materializeSkillEvalWorkspaces(seedWorkspace) {
-  const created = [];
-  try {
-    const baseline = await armWorkspace(created);
-    const treatment = await armWorkspace(created);
-    const judge = await armWorkspace(created);
-    await Promise.all([copyWorkspace(seedWorkspace, baseline), copyWorkspace(seedWorkspace, treatment)]);
-    return {
-      baseline,
-      treatment,
-      judge,
-      cleanup: async () => {
-        await Promise.all(created.map((path) => rm(path, { recursive: true, force: true })));
-      }
-    };
-  } catch (error) {
-    await Promise.all(created.map((path) => rm(path, { recursive: true, force: true })));
-    throw error;
-  }
-}
-async function copyWorkspace(source, destination) {
-  await cp(source, destination, {
-    recursive: true,
-    preserveTimestamps: true,
-    verbatimSymlinks: true
-  });
-}
-async function completeScenarioOutcome(outcome) {
-  return {
-    ...outcome,
-    usage: await usageForCapturedRuns([outcome.baseline, outcome.treatment, outcome.judge])
-  };
-}
-async function runFixtureCommands(scenario, workspace, timeoutMs, trustFixtures) {
-  const commands = scenario.fixtureCommands?.trim();
-  if (commands === void 0 || commands.length === 0) return null;
-  if (!trustFixtures) {
-    return "scenario declares fixture commands, which are real shell from the skill's evals.md; review them and rerun with --trust-fixtures";
-  }
-  const validationError = validateFixtureCommands(commands);
-  if (validationError !== null) return `fixture setup rejected: ${validationError}`;
-  const result = await runBashCommand(commands, {
-    cwd: workspace,
-    timeoutMs: Math.min(timeoutMs, 12e4)
-  });
-  const output = combineBashOutput(result).trim();
-  if (result.timedOut) return `fixture setup timed out${output.length > 0 ? `: ${truncate(output, 300)}` : ""}`;
-  if (result.outputCapped) {
-    return `fixture setup output exceeded limit${output.length > 0 ? `: ${truncate(output, 300)}` : ""}`;
-  }
-  if (result.error !== null) {
-    return `fixture setup failed${output.length > 0 ? `: ${truncate(output, 300)}` : `: ${result.error.message}`}`;
-  }
-  return null;
-}
-function validateFixtureCommands(commands) {
-  const checks = [
-    {
-      pattern: /(^|[\s;&|(<>='"`])\/(?!dev\/null(?:\s|$))(?=\S)/,
-      reason: "absolute paths are not allowed in fixture commands"
-    },
-    {
-      pattern: /(^|[\s;&|()'"`/=])\.\.(?=$|[/\s;&|()'"`])/,
-      reason: "parent-directory path segments are not allowed in fixture commands"
-    },
-    {
-      pattern: /(^|[\s;&|()])~(?=$|[/\s;&|()])/,
-      reason: "home-directory expansion is not allowed in fixture commands"
-    },
-    {
-      pattern: /\$(?:\{(?:HOME|CLIO_CODER_HOME|CLIO_CODER_CONFIG_DIR|CLIO_CODER_DATA_DIR|CLIO_CODER_STATE_DIR|CLIO_CODER_CACHE_DIR)\}|(?:HOME|CLIO_CODER_HOME|CLIO_CODER_CONFIG_DIR|CLIO_CODER_DATA_DIR|CLIO_CODER_STATE_DIR|CLIO_CODER_CACHE_DIR)\b)/,
-      reason: "home and Clio directory environment variables are not allowed in fixture commands"
-    },
-    {
-      pattern: /(^|[\s;&|()])(?:cd|pushd|popd)\b/,
-      reason: "directory-changing commands are not allowed in fixture commands"
-    },
-    {
-      pattern: /(^|[\s;&|()])(?:sudo|su|ssh|scp|rsync)\b/,
-      reason: "privilege-changing or remote commands are not allowed in fixture commands"
-    }
-  ];
-  for (const check of checks) {
-    if (check.pattern.test(commands)) return check.reason;
-  }
-  return null;
-}
-async function usageForCapturedRuns(runs) {
-  const sessionIds = new Set(
-    runs.flatMap((run) => run?.sessionId !== null && run?.sessionId !== void 0 ? [run.sessionId] : [])
-  );
-  const usage = {
-    tokens: 0,
-    costUsd: 0,
-    harness: { ...ZERO_EVAL_HARNESS_METRICS }
-  };
-  if (sessionIds.size === 0) return usage;
-  const receiptRoot = join(clioStateDir(), "receipts");
-  let files;
-  try {
-    files = (await readdir(receiptRoot)).filter((name) => name.endsWith(".json"));
-  } catch {
-    return usage;
-  }
-  for (const file of files) {
-    let parsed;
-    try {
-      parsed = JSON.parse(await readFile(join(receiptRoot, file), "utf8"));
-    } catch {
-      continue;
-    }
-    if (!isRecord(parsed) || typeof parsed.sessionId !== "string" || !sessionIds.has(parsed.sessionId)) continue;
-    usage.tokens += readNumber(parsed.tokenCount);
-    usage.costUsd += readNumber(parsed.costUsd);
-    usage.harness.receiptCount += 1;
-    usage.harness.toolCalls += readNumber(parsed.toolCalls);
-    if (isRecord(parsed.safety) && isRecord(parsed.safety.decisions)) {
-      usage.harness.safetyBlocks += readNumber(parsed.safety.decisions.blocked);
-    }
-  }
-  return usage;
-}
-function runInfraError(label, run) {
-  if (run.timedOut) return `${label} run timed out`;
-  if (run.exitCode !== 0) {
-    const detail = run.stderr.trim().split("\n").at(-1) ?? "";
-    return `${label} run exited ${run.exitCode}${detail.length > 0 ? `: ${detail}` : ""}`;
-  }
-  if (run.transcript.trim().length === 0 && run.finalText.trim().length === 0) {
-    return `${label} run produced no transcript`;
-  }
-  return null;
-}
-function errorBullets(scenario, reason) {
-  return scenario.expected.map((text, index) => ({ index: index + 1, text, verdict: "error", reason }));
-}
-function captureHeadlessRun(args, cwd, timeoutMs, env) {
-  const startedMs = performance.now();
-  return new Promise((resolvePromise) => {
-    let stdout = "";
-    let pendingStdout = "";
-    let stderr = "";
-    let timedOut = false;
-    let settled = false;
-    const keepLine = (line) => {
-      if (line.length === 0) return;
-      if (stdout.length < CHILD_OUTPUT_LIMIT) stdout += `${line}
-`;
-    };
-    const child = spawn(process.execPath, [process.argv[1] ?? "", ...args], {
-      cwd,
-      stdio: ["ignore", "pipe", "pipe"],
-      env
-    });
-    child.stdout.setEncoding("utf8");
-    child.stderr.setEncoding("utf8");
-    child.stdout.on("data", (chunk) => {
-      pendingStdout += chunk;
-      const lines = pendingStdout.split("\n");
-      pendingStdout = lines.pop() ?? "";
-      for (const line of lines) keepLine(line);
-    });
-    child.stderr.on("data", (chunk) => {
-      if (stderr.length < CHILD_OUTPUT_LIMIT) stderr += chunk;
-    });
-    let killTimer;
-    const timer = setTimeout(() => {
-      timedOut = true;
-      child.kill("SIGTERM");
-      killTimer = setTimeout(() => child.kill("SIGKILL"), 5e3);
-    }, timeoutMs);
-    const finish = (exitCode) => {
-      if (settled) return;
-      settled = true;
-      clearTimeout(timer);
-      if (killTimer !== void 0) clearTimeout(killTimer);
-      keepLine(pendingStdout);
-      pendingStdout = "";
-      const parsedRun = parseRunStdout(stdout);
-      resolvePromise({
-        ...parsedRun,
-        exitCode,
-        timedOut,
-        wallTimeMs: Math.round(performance.now() - startedMs),
-        stderr
-      });
-    };
-    child.on("error", (error) => {
-      stderr += `
-${error.message}`;
-      finish(1);
-    });
-    child.on("close", (code) => finish(typeof code === "number" ? code : timedOut ? 124 : 1));
-  });
-}
-function loadsSkillBody(tool, args) {
-  if (tool !== "context" || !isRecord(args)) return false;
-  return args.scope === "skills" && typeof args.name === "string" && args.name.trim().length > 0;
-}
-function parseRunStdout(stdout) {
-  let sessionId = null;
-  const lines = [];
-  let finalText = "";
-  let sawJson = false;
-  const streamedText = /* @__PURE__ */ new Map();
-  const skillBodyCallIds = /* @__PURE__ */ new Set();
-  for (const line of stdout.split("\n")) {
-    if (line.trim().length === 0) continue;
-    let event;
-    try {
-      event = JSON.parse(line);
-    } catch {
-      continue;
-    }
-    if (!isRecord(event)) continue;
-    sawJson = true;
-    if (event.type === "session" && typeof event.id === "string") {
-      sessionId = event.id;
-      continue;
-    }
-    if (event.type === "text_delta" && typeof event.delta === "string") {
-      const contentIndex = typeof event.contentIndex === "number" ? event.contentIndex : 0;
-      streamedText.set(contentIndex, `${streamedText.get(contentIndex) ?? ""}${event.delta}`);
-      continue;
-    }
-    if (event.type === "tool_execution_start") {
-      const tool = readString(event.toolName) ?? readString(event.tool) ?? "tool";
-      const args = event.args ?? event.arguments ?? event.input;
-      const callId = readString(event.toolCallId);
-      if (callId !== null && loadsSkillBody(tool, args)) skillBodyCallIds.add(callId);
-      lines.push(`TOOL ${tool} args=${preview(args)}`);
-      if (tool === "artifact" && isRecord(args) && typeof args.content === "string") {
-        lines.push(`TERMINAL ${tool} content:
-${args.content}`);
-      }
-      continue;
-    }
-    if (event.type === "tool_execution_end") {
-      const tool = readString(event.toolName) ?? readString(event.tool) ?? "tool";
-      const status = event.isError === true ? "error" : "ok";
-      const callId = readString(event.toolCallId);
-      if (callId !== null && skillBodyCallIds.has(callId)) {
-        lines.push(`RESULT ${tool} ${status}: <skill body withheld from the judge>`);
-        continue;
-      }
-      lines.push(`RESULT ${tool} ${status}: ${preview(event.result)}`);
-      continue;
-    }
-    if (event.type === "message_end" && isRecord(event.message) && event.message.role === "assistant") {
-      const content = Array.isArray(event.message.content) ? event.message.content : [];
-      const completedText = content.filter((item) => {
-        return isRecord(item) && item.type === "text" && typeof item.text === "string";
-      }).map((item) => item.text).join("").trim();
-      const streamed = [...streamedText.entries()].sort(([left], [right]) => left - right).map(([, text2]) => text2).join("").trim();
-      streamedText.clear();
-      const text = completedText.length > 0 ? completedText : streamed;
-      if (text.length > 0) {
-        lines.push(`ASSISTANT: ${text}`);
-        finalText = text;
-      }
-    }
-  }
-  if (!sawJson) {
-    const text = stdout.trim();
-    return { sessionId: null, transcript: text, finalText: text };
-  }
-  return { sessionId, transcript: elide(lines.join("\n")), finalText };
-}
-function judgePrompt(scenario, baselineTranscript, treatmentTranscript) {
-  const bullets = scenario.expected.map((text, index) => `${index + 1}. ${text}`).join("\n");
-  return [
-    "You are scoring a skill evaluation. Two transcripts follow: BASELINE ran without the skill, TREATMENT ran with the skill loaded.",
-    "Score each EXPECTED bullet strictly against the TREATMENT transcript; the baseline exists only for gap context.",
-    "A bullet passes only if the treatment transcript observably satisfies it; anything unverifiable from the transcript fails.",
-    'Reply with STRICT JSON only, no prose and no code fences, exactly: {"bullets":[{"index":1,"pass":true,"reason":"<= 25 words"}]}',
-    `Include one entry per bullet, indexes 1 through ${scenario.expected.length} in order.`,
-    "Do not use any tools. Respond with the JSON verdict directly.",
-    "",
-    `SCENARIO ${scenario.id} - ${scenario.title}`,
-    `SETUP: ${scenario.setup}`,
-    "",
-    "EXPECTED BULLETS:",
-    bullets,
-    "",
-    "BASELINE TRANSCRIPT:",
-    baselineTranscript.length > 0 ? baselineTranscript : "(empty)",
-    "",
-    "TREATMENT TRANSCRIPT:",
-    treatmentTranscript.length > 0 ? treatmentTranscript : "(empty)"
-  ].join("\n");
-}
-function judgeVerdictAbsenceReason(judge) {
-  const sawBulletsKey = `${judge.finalText}
-${judge.transcript}`.includes('"bullets"');
-  return sawBulletsKey ? "judge response truncated mid-verdict (no complete bullets object): bullet not scored" : "judge response carried no parseable verdict object: bullet not scored";
-}
-function parseJudgeVerdicts(scenario, judge) {
-  const parsed = extractBulletsObject(judge.finalText) ?? extractBulletsObject(judge.transcript);
-  if (parsed === null) {
-    const reason = judgeVerdictAbsenceReason(judge);
-    return scenario.expected.map((text, i) => ({ index: i + 1, text, verdict: "unmeasured", reason }));
-  }
-  const entries = /* @__PURE__ */ new Map();
-  if (Array.isArray(parsed.bullets)) {
-    for (const item of parsed.bullets) {
-      if (!isRecord(item)) continue;
-      const index = typeof item.index === "number" ? item.index : Number.parseInt(String(item.index), 10);
-      if (!Number.isInteger(index)) continue;
-      entries.set(index, {
-        pass: item.pass === true,
-        reason: readString(item.reason) ?? ""
-      });
-    }
-  }
-  return scenario.expected.map((text, i) => {
-    const index = i + 1;
-    const entry = entries.get(index);
-    if (entry === void 0) {
-      return {
-        index,
-        text,
-        verdict: "unmeasured",
-        reason: "judge output omitted this bullet: bullet not scored"
-      };
-    }
-    return { index, text, verdict: entry.pass ? "pass" : "fail", reason: entry.reason };
-  });
-}
-function extractBulletsObject(text) {
-  const stripped = text.replace(/```(?:json)?/g, "");
-  const bulletsAt = stripped.indexOf('"bullets"');
-  if (bulletsAt < 0) return null;
-  let start = stripped.lastIndexOf("{", bulletsAt);
-  while (start >= 0) {
-    const candidate = balancedJsonSlice(stripped, start);
-    if (candidate !== null) {
-      try {
-        const parsed = JSON.parse(candidate);
-        if (isRecord(parsed) && Array.isArray(parsed.bullets)) return parsed;
-      } catch {
-      }
-    }
-    if (start === 0) break;
-    start = stripped.lastIndexOf("{", start - 1);
-  }
-  return null;
-}
-function balancedJsonSlice(text, start) {
-  let depth = 0;
-  let inString = false;
-  let escaped = false;
-  for (let i = start; i < text.length; i += 1) {
-    const ch = text[i];
-    if (escaped) {
-      escaped = false;
-      continue;
-    }
-    if (ch === "\\") {
-      if (inString) escaped = true;
-      continue;
-    }
-    if (ch === '"') {
-      inString = !inString;
-      continue;
-    }
-    if (inString) continue;
-    if (ch === "{") depth += 1;
-    else if (ch === "}") {
-      depth -= 1;
-      if (depth === 0) return text.slice(start, i + 1);
-    }
-  }
-  return null;
-}
-function synthesizeArtifact(skillName, evalsPath, evalsRaw, startedAt, endedAt, outcomes, target) {
-  const contentHash = createHash("sha256").update(evalsRaw, "utf8").digest("hex");
-  const stamp = startedAt.replace(/[-:.]/g, "");
-  const evalId = `skill-${skillName}-${stamp}-${contentHash.slice(0, 8)}-${randomBytes(6).toString("hex")}`;
-  const results = outcomes.map((outcome) => {
-    const failed = outcome.bullets.some((bullet) => bullet.verdict === "fail" || bullet.verdict === "error");
-    const unmeasured = outcome.bullets.some((bullet) => bullet.verdict === "unmeasured");
-    const pass = outcome.bullets.length > 0 && !failed && !unmeasured;
-    const record = {
-      taskId: outcome.scenario.id,
-      runId: `${evalId}-${outcome.scenario.id}`,
-      repeatIndex: 0,
-      cwd: outcome.workspace,
-      prompt: outcome.scenario.setup,
-      tags: [
-        "skill-eval",
-        `skill:${skillName}`,
-        ...unmeasured ? ["scenario:unmeasured"] : [],
-        ...outcome.bullets.map((bullet) => `bullet-${bullet.index}:${bullet.verdict}`)
-      ],
-      pass,
-      // 3, not 1, when the scenario carries an unscored bullet and no failed
-      // one: a reader who sees exitCode 1 with failureClass verifier_failed
-      // reads "this skill failed its rubric", and an unscored bullet is the
-      // judge's silence, not the skill's answer.
-      exitCode: pass ? 0 : failed ? 1 : 3,
-      tokens: outcome.usage.tokens,
-      costUsd: outcome.usage.costUsd,
-      wallTimeMs: outcome.wallTimeMs,
-      harness: outcome.usage.harness,
-      commands: []
-    };
-    if (failed) record.failureClass = "verifier_failed";
-    return record;
-  });
-  return {
-    version: 1,
-    evalId,
-    taskFile: evalsPath,
-    taskFileHash: contentHash,
-    clioCoder: evalClioProvenance(),
-    environment: evalEnvironmentProvenance(),
-    target: target ?? null,
-    model: null,
-    thinking: null,
-    paths: { taskFile: evalsPath, receipts: [], sessionLedgers: [] },
-    repeat: 1,
-    startedAt,
-    endedAt,
-    summary: summarizeEvalResults(results),
-    results
-  };
-}
-function sidecar(skillName, evalId, outcomes, allowNetwork) {
-  return {
-    version: 1,
-    schema: "experimental",
-    kind: "skill-eval",
-    skill: skillName,
-    evalId,
-    network: networkPolicyLabel(allowNetwork),
-    autonomy: ARM_AUTONOMY,
-    deltas: [
-      "bullet verdicts are judge-scored from run transcripts, not command exit codes; the evals-domain artifact carries scenario-level records with empty command lists",
-      "a bullet the judge never scored is recorded unmeasured, not failed: its scenario record is pass:false with exitCode 3 and no failureClass",
-      "an arm whose transcript carries the headless permission wall is recorded unmeasured with an infraError: the harness's own gate is not a verdict about the skill",
-      "tokens and cost are rolled up from headless main-agent receipts when those receipts are present",
-      "this sidecar is additive and is registered in overview.json files[]"
-    ],
-    scenarios: outcomes.map((outcome) => ({
-      id: outcome.scenario.id,
-      title: outcome.scenario.title,
-      setup: outcome.scenario.setup,
-      fixtureCommands: outcome.scenario.fixtureCommands ?? null,
-      infraError: outcome.infraError,
-      usage: outcome.usage,
-      bullets: outcome.bullets,
-      baseline: sidecarRun(outcome.baseline),
-      treatment: sidecarRun(outcome.treatment),
-      judge: sidecarRun(outcome.judge)
-    }))
-  };
-}
-function sidecarRun(run) {
-  if (run === null) return null;
-  return {
-    sessionId: run.sessionId,
-    exitCode: run.exitCode,
-    timedOut: run.timedOut,
-    wallTimeMs: run.wallTimeMs,
-    transcript: run.transcript,
-    stderrTail: run.stderr.slice(-500)
-  };
-}
-function printHumanReport(skillName, outcomes, evalId, evidenceId, evidenceDirectory, allowNetwork) {
-  const rows = [["scenario", "bullet", "verdict", "expected"]];
-  for (const outcome of outcomes) {
-    for (const bullet of outcome.bullets) {
-      rows.push([outcome.scenario.id, String(bullet.index), bullet.verdict, truncate(bullet.text, 76)]);
-    }
-  }
-  process.stdout.write(formatColumns(rows));
-  const total = outcomes.reduce((sum, outcome) => sum + outcome.bullets.length, 0);
-  const passed = outcomes.reduce(
-    (sum, outcome) => sum + outcome.bullets.filter((bullet) => bullet.verdict === "pass").length,
-    0
-  );
-  const unmeasured = outcomes.reduce(
-    (sum, outcome) => sum + outcome.bullets.filter((bullet) => bullet.verdict === "unmeasured").length,
-    0
-  );
-  const unmeasuredNote = unmeasured > 0 ? `, ${unmeasured} unmeasured` : "";
-  process.stdout.write(
-    `
-${skillName}: ${passed}/${total} bullets passed${unmeasuredNote} (judge-scored, experimental)
-`
-  );
-  if (unmeasured > 0) {
-    process.stdout.write(
-      unmeasured === total ? "no bullet was scored, so this run is not evidence about the skill either way\n" : `${unmeasured} of ${total} bullets were never scored; unmeasured is not a failed bullet
-`
-    );
-  }
-  for (const outcome of outcomes) {
-    if (outcome.infraError !== null) {
-      process.stdout.write(`${outcome.scenario.id}: infra error: ${outcome.infraError}
-`);
-      continue;
-    }
-    for (const bullet of outcome.bullets.filter((item) => item.verdict !== "pass")) {
-      process.stdout.write(
-        `${outcome.scenario.id} bullet ${bullet.index} ${bullet.verdict}${bullet.reason.length > 0 ? `: ${bullet.reason}` : ""}
-`
-      );
-    }
-  }
-  process.stdout.write(`${describeArmPolicyOutcome(allowNetwork)}
-`);
-  process.stdout.write(`eval artifact: ${evalId}
-`);
-  if (evidenceId !== null && evidenceDirectory !== null) {
-    process.stdout.write(`evidence: ${evidenceId} at ${evidenceDirectory} (per-bullet detail in skill-eval.json)
-`);
-  }
-}
-function preview(value) {
-  if (value === void 0) return "";
-  const text = typeof value === "string" ? value : safeStringify(value);
-  return truncate(text.replace(/\s+/g, " ").trim(), PREVIEW_MAX_CHARS);
-}
-function safeStringify(value) {
-  try {
-    return JSON.stringify(value) ?? "";
-  } catch {
-    return String(value);
-  }
-}
-function elide(text) {
-  if (text.length <= TRANSCRIPT_HEAD_CHARS + TRANSCRIPT_TAIL_CHARS) return text;
-  return `${text.slice(0, TRANSCRIPT_HEAD_CHARS)}
-[... transcript elided ...]
-${text.slice(-TRANSCRIPT_TAIL_CHARS)}`;
-}
-function truncate(text, maxChars) {
-  if (text.length <= maxChars) return text;
-  return `${text.slice(0, Math.max(0, maxChars - 3))}...`;
-}
-function readString(value) {
-  return typeof value === "string" && value.length > 0 ? value : null;
-}
-function readNumber(value) {
-  return typeof value === "number" && Number.isFinite(value) ? value : 0;
-}
-function isRecord(value) {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-export {
-  evalChildEnv,
-  runSkillsEvalCommand
-};
-//# sourceMappingURL=skills-eval-J5TCKSJA.js.map