@iowarp/clio-coder 0.3.0

package/dist/skills-eval-E2ZTW4PL.js

@@ -0,0 +1,932 @@
+import {
+  HEADLESS_PERMISSION_DENIED_MARKER,
+  NO_NETWORK_TOOLS_ENV,
+  buildEvalEvidence,
+  combineBashOutput,
+  runBashCommand,
+  summarizeEvalResults
+} from "./chunk-GNGDQYDU.js";
+import {
+  parseSkillEvals
+} from "./chunk-2J63S4SF.js";
+import "./chunk-RQDWMVRB.js";
+import {
+  ZERO_EVAL_HARNESS_METRICS,
+  evalClioProvenance,
+  evalEnvironmentProvenance,
+  writeEvalArtifact
+} from "./chunk-7BHLZB3A.js";
+import "./chunk-TPOCL34A.js";
+import "./chunk-E3NYWENM.js";
+import "./chunk-XV4ZJ6ZM.js";
+import {
+  discoverMarketplaceSkills,
+  loadSkills
+} from "./chunk-26BZQOAD.js";
+import "./chunk-WD6AJM35.js";
+import "./chunk-QV47YRF4.js";
+import "./chunk-CLJ5HLUD.js";
+import "./chunk-BBUH4VAA.js";
+import "./chunk-GOTUR54M.js";
+import "./chunk-PIDUD6M2.js";
+import {
+  formatColumns,
+  printError
+} from "./chunk-67MTHP2E.js";
+import "./chunk-TKHQ4DGZ.js";
+import "./chunk-D5YD55AR.js";
+import "./chunk-JWHCJDCI.js";
+import "./chunk-UHTSULZS.js";
+import "./chunk-5LDRNKX2.js";
+import "./chunk-UGYAX5YI.js";
+import "./chunk-5N2FG33Q.js";
+import "./chunk-VPAWTYLY.js";
+import {
+  clioDataDir,
+  clioStateDir
+} from "./chunk-X3NE4WVW.js";
+import "./chunk-DXQNI4PC.js";
+
+// src/cli/skills-eval.ts
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+var DEFAULT_RUN_TIMEOUT_MS = 6e5;
+var ARM_AUTONOMY = "full-auto";
+var CHILD_OUTPUT_LIMIT = 4e6;
+var PREVIEW_MAX_CHARS = 300;
+var TRANSCRIPT_HEAD_CHARS = 9e3;
+var TRANSCRIPT_TAIL_CHARS = 4e3;
+var SKILL_EVAL_SIDECAR = "skill-eval.json";
+async function runSkillsEvalCommand(nameOrPath, options) {
+  const resolved = resolveSkillBaseDir(nameOrPath, process.cwd());
+  if (resolved.baseDir === null) {
+    printError(resolved.error ?? `unknown skill: ${nameOrPath}`);
+    return 2;
+  }
+  const list = loadSkills({ disableDiscovery: true, explicitSkillPaths: [resolved.baseDir] });
+  const skill = list.items[0];
+  if (skill === void 0) {
+    printError(`skill did not load from ${resolved.baseDir}: ${list.diagnostics.map((d) => d.message).join("; ")}`);
+    return 2;
+  }
+  process.stderr.write(
+    `clio-coder skills eval: ${skill.name} from ${resolved.origin ?? "unknown"} at ${resolved.baseDir} (sha256 ${skill.normalizedHash.slice(0, 12)}\u2026)
+`
+  );
+  const evalsPath = join(resolved.baseDir, "evals.md");
+  let evalsRaw;
+  try {
+    evalsRaw = await readFile(evalsPath, "utf8");
+  } catch {
+    printError(`skill ${skill.name} has no evals.md at ${evalsPath}`);
+    return 2;
+  }
+  const parsed = parseSkillEvals(evalsRaw);
+  for (const diagnostic of parsed.diagnostics) {
+    process.stderr.write(`clio-coder skills eval: ${diagnostic}
+`);
+  }
+  const matcher = options.scenario === void 0 ? null : scenarioMatcher(options.scenario);
+  if (options.scenario !== void 0 && matcher === null) {
+    printError(`invalid --scenario "${options.scenario}": use a scenario id like S1 or a bare number`);
+    return 2;
+  }
+  const scenarios = matcher === null ? parsed.scenarios : parsed.scenarios.filter(matcher);
+  if (scenarios.length === 0) {
+    printError(
+      matcher === null ? `no parseable scenarios in ${evalsPath}` : `scenario ${options.scenario} not found in ${evalsPath} (have: ${parsed.scenarios.map((s) => s.id).join(", ") || "none"})`
+    );
+    return 2;
+  }
+  process.stderr.write(`clio-coder skills eval: ${describeArmPolicy(options.allowNetwork)}
+`);
+  const childEnv = evalChildEnv(options.allowNetwork);
+  const timeoutMs = options.timeoutSeconds !== void 0 ? options.timeoutSeconds * 1e3 : DEFAULT_RUN_TIMEOUT_MS;
+  const workspaceOverride = await resolveWorkspaceOverride(options.workspace);
+  if (typeof workspaceOverride !== "string" && workspaceOverride !== null) {
+    printError(workspaceOverride.error);
+    return 2;
+  }
+  const startedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const outcomes = [];
+  for (const scenario of scenarios) {
+    process.stderr.write(`clio-coder skills eval: ${skill.name} ${scenario.id} baseline/treatment/judge...
+`);
+    outcomes.push(
+      await runScenario(
+        skill.name,
+        resolved.baseDir,
+        scenario,
+        options.target,
+        timeoutMs,
+        workspaceOverride,
+        options.trustFixtures,
+        childEnv
+      )
+    );
+  }
+  const endedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const artifact = synthesizeArtifact(skill.name, evalsPath, evalsRaw, startedAt, endedAt, outcomes, options.target);
+  let evidenceId = null;
+  let evidenceDirectory = null;
+  const evidenceErrors = [];
+  try {
+    await writeEvalArtifact(clioDataDir(), artifact);
+    const built = await buildEvalEvidence({
+      dataDir: clioDataDir(),
+      stateDir: clioStateDir(),
+      artifact,
+      sidecars: [SKILL_EVAL_SIDECAR]
+    });
+    evidenceId = built.evidenceId;
+    evidenceDirectory = built.directory;
+    await writeFile(
+      join(built.directory, SKILL_EVAL_SIDECAR),
+      `${JSON.stringify(sidecar(skill.name, artifact.evalId, outcomes, options.allowNetwork), null, 2)}
+`,
+      "utf8"
+    );
+  } catch (error) {
+    evidenceErrors.push(error instanceof Error ? error.message : String(error));
+  }
+  for (const message of evidenceErrors) {
+    process.stderr.write(`clio-coder skills eval: evidence build failed: ${message}
+`);
+  }
+  if (options.json) {
+    for (const outcome of outcomes) {
+      for (const bullet of outcome.bullets) {
+        process.stdout.write(
+          `${JSON.stringify({
+            schema: "experimental",
+            kind: "skill-eval-bullet",
+            skill: skill.name,
+            scenario: outcome.scenario.id,
+            title: outcome.scenario.title,
+            bullet: bullet.index,
+            expected: bullet.text,
+            verdict: bullet.verdict,
+            reason: bullet.reason,
+            network: networkPolicyLabel(options.allowNetwork),
+            autonomy: ARM_AUTONOMY,
+            baselineSessionId: outcome.baseline?.sessionId ?? null,
+            treatmentSessionId: outcome.treatment?.sessionId ?? null,
+            judgeSessionId: outcome.judge?.sessionId ?? null,
+            evalId: artifact.evalId,
+            evidenceId
+          })}
+`
+        );
+      }
+    }
+  } else {
+    printHumanReport(skill.name, outcomes, artifact.evalId, evidenceId, evidenceDirectory, options.allowNetwork);
+  }
+  const anyFailure = outcomes.some(
+    (outcome) => outcome.bullets.some((bullet) => bullet.verdict === "fail" || bullet.verdict === "error")
+  );
+  const anyUnmeasured = outcomes.some((outcome) => outcome.bullets.some((bullet) => bullet.verdict === "unmeasured"));
+  if (anyFailure) return 1;
+  return anyUnmeasured || evidenceErrors.length > 0 ? 3 : 0;
+}
+function networkPolicyLabel(allowNetwork) {
+  return allowNetwork ? "allowed" : "hermetic";
+}
+function describeArmPolicy(allowNetwork) {
+  const network = allowNetwork ? "network allowed (--allow-network), so arm runs keep the web tools" : "network hermetic, so the web tools are stripped from every arm run (pass --allow-network to keep them)";
+  return `policy: autonomy ${ARM_AUTONOMY} for the baseline and treatment arms in per-run disposable workspaces; ${network}`;
+}
+function describeArmPolicyOutcome(allowNetwork) {
+  const network = allowNetwork ? "network: allowed (--allow-network); arm runs kept the web tools" : "network: hermetic; the web tools were stripped from every arm run";
+  return `policy: the baseline and treatment arms ran at autonomy ${ARM_AUTONOMY}; ${network}`;
+}
+function armRunArgs(arm, prompt, options = {}) {
+  const args = ["run", "--json", "--json-events", "terminal", "--no-skills"];
+  if (arm !== "judge") args.push("--autonomy", ARM_AUTONOMY);
+  if (options.skillBaseDir !== void 0) args.push("--skill", options.skillBaseDir);
+  if (options.target !== void 0) args.push("--target", options.target);
+  args.push(prompt);
+  return args;
+}
+function permissionWallReason(arm, run) {
+  const haystack = `${run.transcript}
+${run.finalText}`;
+  let blocked = 0;
+  let at = haystack.indexOf(HEADLESS_PERMISSION_DENIED_MARKER);
+  while (at >= 0) {
+    blocked += 1;
+    at = haystack.indexOf(HEADLESS_PERMISSION_DENIED_MARKER, at + HEADLESS_PERMISSION_DENIED_MARKER.length);
+  }
+  if (blocked === 0) return null;
+  return `${arm} arm transcript carries the headless permission wall ${blocked} time(s) ("${HEADLESS_PERMISSION_DENIED_MARKER}"): this run measured the harness's own autonomy gate, not the skill`;
+}
+function unmeasuredBullets(scenario, reason) {
+  return scenario.expected.map((text, index) => ({ index: index + 1, text, verdict: "unmeasured", reason }));
+}
+function evalChildEnv(allowNetwork, env = process.env) {
+  const childEnv = { ...env };
+  if (allowNetwork) Reflect.deleteProperty(childEnv, NO_NETWORK_TOOLS_ENV);
+  else childEnv[NO_NETWORK_TOOLS_ENV] = "1";
+  return childEnv;
+}
+function resolveSkillBaseDir(nameOrPath, cwd) {
+  const asPath = resolve(nameOrPath);
+  if (existsSync(join(asPath, "SKILL.md"))) return { baseDir: asPath, origin: "path" };
+  const discovered = loadSkills({ cwd }).items.find((skill) => skill.name === nameOrPath);
+  if (discovered !== void 0) {
+    return { baseDir: discovered.baseDir, origin: `${discovered.source}/${discovered.scope}` };
+  }
+  const marketplace = discoverMarketplaceSkills({ cwd });
+  const entry = marketplace.skills.find((skill) => skill.name === nameOrPath && skill.origin === "catalog");
+  if (entry !== void 0) return { baseDir: entry.sourceUrl, origin: "catalog" };
+  return {
+    baseDir: null,
+    error: `skill not found: ${nameOrPath} (expected an installed skill, a catalog skill name, or a directory containing SKILL.md)`
+  };
+}
+function scenarioMatcher(value) {
+  const trimmed = value.trim();
+  const full = /^([A-Za-z])(\d+)$/.exec(trimmed);
+  if (full !== null) {
+    const id = `${(full[1] ?? "").toUpperCase()}${Number.parseInt(full[2] ?? "", 10)}`;
+    return (scenario) => scenario.id === id;
+  }
+  const bare = /^(\d+)$/.exec(trimmed);
+  if (bare !== null) {
+    const wanted = Number.parseInt(bare[1] ?? "", 10);
+    return (scenario) => scenario.number === wanted;
+  }
+  return null;
+}
+async function resolveWorkspaceOverride(workspace) {
+  if (workspace === void 0) return null;
+  const resolved = resolve(workspace);
+  try {
+    const info = await stat(resolved);
+    if (!info.isDirectory()) return { error: `--workspace is not a directory: ${resolved}` };
+    return resolved;
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    return { error: `--workspace is not readable: ${resolved}: ${detail}` };
+  }
+}
+async function runScenario(skillName, skillBaseDir, scenario, target, timeoutMs, workspaceOverride, trustFixtures, childEnv) {
+  const scenarioStart = Date.now();
+  const workspace = await mkdtemp(join(tmpdir(), "clio-skill-eval-seed-"));
+  let runWorkspaces = null;
+  try {
+    if (workspaceOverride !== null) await copyWorkspace(workspaceOverride, workspace);
+    const fixtureError = await runFixtureCommands(scenario, workspace, timeoutMs, trustFixtures);
+    if (fixtureError !== null) {
+      return await completeScenarioOutcome({
+        scenario,
+        bullets: errorBullets(scenario, fixtureError),
+        baseline: null,
+        treatment: null,
+        judge: null,
+        workspace,
+        wallTimeMs: Date.now() - scenarioStart,
+        infraError: fixtureError
+      });
+    }
+    runWorkspaces = await materializeSkillEvalWorkspaces(workspace);
+    const baseline = await captureHeadlessRun(
+      armRunArgs("baseline", scenario.setup, { target }),
+      runWorkspaces.baseline,
+      timeoutMs,
+      childEnv
+    );
+    const treatment = await captureHeadlessRun(
+      armRunArgs("treatment", `/skill:${skillName} ${scenario.setup}`, { target, skillBaseDir }),
+      runWorkspaces.treatment,
+      timeoutMs,
+      childEnv
+    );
+    const infra = runInfraError("baseline", baseline) ?? runInfraError("treatment", treatment);
+    if (infra !== null) {
+      return await completeScenarioOutcome({
+        scenario,
+        bullets: errorBullets(scenario, infra),
+        baseline,
+        treatment,
+        judge: null,
+        workspace,
+        wallTimeMs: Date.now() - scenarioStart,
+        infraError: infra
+      });
+    }
+    const wall = permissionWallReason("baseline", baseline) ?? permissionWallReason("treatment", treatment);
+    if (wall !== null) {
+      return await completeScenarioOutcome({
+        scenario,
+        bullets: unmeasuredBullets(scenario, wall),
+        baseline,
+        treatment,
+        judge: null,
+        workspace,
+        wallTimeMs: Date.now() - scenarioStart,
+        infraError: wall
+      });
+    }
+    const judge = await captureHeadlessRun(
+      armRunArgs("judge", judgePrompt(scenario, baseline.transcript, treatment.transcript), { target }),
+      runWorkspaces.judge,
+      timeoutMs,
+      childEnv
+    );
+    const judgeInfra = runInfraError("judge", judge);
+    if (judgeInfra !== null) {
+      return await completeScenarioOutcome({
+        scenario,
+        bullets: errorBullets(scenario, judgeInfra),
+        baseline,
+        treatment,
+        judge,
+        workspace,
+        wallTimeMs: Date.now() - scenarioStart,
+        infraError: judgeInfra
+      });
+    }
+    const bullets = parseJudgeVerdicts(scenario, judge);
+    return await completeScenarioOutcome({
+      scenario,
+      bullets,
+      baseline,
+      treatment,
+      judge,
+      workspace,
+      wallTimeMs: Date.now() - scenarioStart,
+      infraError: null
+    });
+  } finally {
+    try {
+      if (runWorkspaces !== null) await runWorkspaces.cleanup();
+    } finally {
+      await rm(workspace, { recursive: true, force: true });
+    }
+  }
+}
+async function armWorkspace(created) {
+  const root = await mkdtemp(join(tmpdir(), "clio-skill-eval-"));
+  created.push(root);
+  const workspace = join(root, "workspace");
+  await mkdir(workspace, { recursive: true });
+  return workspace;
+}
+async function materializeSkillEvalWorkspaces(seedWorkspace) {
+  const created = [];
+  try {
+    const baseline = await armWorkspace(created);
+    const treatment = await armWorkspace(created);
+    const judge = await armWorkspace(created);
+    await Promise.all([copyWorkspace(seedWorkspace, baseline), copyWorkspace(seedWorkspace, treatment)]);
+    return {
+      baseline,
+      treatment,
+      judge,
+      cleanup: async () => {
+        await Promise.all(created.map((path) => rm(path, { recursive: true, force: true })));
+      }
+    };
+  } catch (error) {
+    await Promise.all(created.map((path) => rm(path, { recursive: true, force: true })));
+    throw error;
+  }
+}
+async function copyWorkspace(source, destination) {
+  await cp(source, destination, {
+    recursive: true,
+    preserveTimestamps: true,
+    verbatimSymlinks: true
+  });
+}
+async function completeScenarioOutcome(outcome) {
+  return {
+    ...outcome,
+    usage: await usageForCapturedRuns([outcome.baseline, outcome.treatment, outcome.judge])
+  };
+}
+async function runFixtureCommands(scenario, workspace, timeoutMs, trustFixtures) {
+  const commands = scenario.fixtureCommands?.trim();
+  if (commands === void 0 || commands.length === 0) return null;
+  if (!trustFixtures) {
+    return "scenario declares fixture commands, which are real shell from the skill's evals.md; review them and rerun with --trust-fixtures";
+  }
+  const validationError = validateFixtureCommands(commands);
+  if (validationError !== null) return `fixture setup rejected: ${validationError}`;
+  const result = await runBashCommand(commands, {
+    cwd: workspace,
+    timeoutMs: Math.min(timeoutMs, 12e4)
+  });
+  const output = combineBashOutput(result).trim();
+  if (result.timedOut) return `fixture setup timed out${output.length > 0 ? `: ${truncate(output, 300)}` : ""}`;
+  if (result.outputCapped) {
+    return `fixture setup output exceeded limit${output.length > 0 ? `: ${truncate(output, 300)}` : ""}`;
+  }
+  if (result.error !== null) {
+    return `fixture setup failed${output.length > 0 ? `: ${truncate(output, 300)}` : `: ${result.error.message}`}`;
+  }
+  return null;
+}
+function validateFixtureCommands(commands) {
+  const checks = [
+    {
+      pattern: /(^|[\s;&|(<>='"`])\/(?!dev\/null(?:\s|$))(?=\S)/,
+      reason: "absolute paths are not allowed in fixture commands"
+    },
+    {
+      pattern: /(^|[\s;&|()'"`/=])\.\.(?=$|[/\s;&|()'"`])/,
+      reason: "parent-directory path segments are not allowed in fixture commands"
+    },
+    {
+      pattern: /(^|[\s;&|()])~(?=$|[/\s;&|()])/,
+      reason: "home-directory expansion is not allowed in fixture commands"
+    },
+    {
+      pattern: /\$(?:\{(?:HOME|CLIO_CODER_HOME|CLIO_CODER_CONFIG_DIR|CLIO_CODER_DATA_DIR|CLIO_CODER_STATE_DIR|CLIO_CODER_CACHE_DIR)\}|(?:HOME|CLIO_CODER_HOME|CLIO_CODER_CONFIG_DIR|CLIO_CODER_DATA_DIR|CLIO_CODER_STATE_DIR|CLIO_CODER_CACHE_DIR)\b)/,
+      reason: "home and Clio directory environment variables are not allowed in fixture commands"
+    },
+    {
+      pattern: /(^|[\s;&|()])(?:cd|pushd|popd)\b/,
+      reason: "directory-changing commands are not allowed in fixture commands"
+    },
+    {
+      pattern: /(^|[\s;&|()])(?:sudo|su|ssh|scp|rsync)\b/,
+      reason: "privilege-changing or remote commands are not allowed in fixture commands"
+    }
+  ];
+  for (const check of checks) {
+    if (check.pattern.test(commands)) return check.reason;
+  }
+  return null;
+}
+async function usageForCapturedRuns(runs) {
+  const sessionIds = new Set(
+    runs.flatMap((run) => run?.sessionId !== null && run?.sessionId !== void 0 ? [run.sessionId] : [])
+  );
+  const usage = {
+    tokens: 0,
+    costUsd: 0,
+    harness: { ...ZERO_EVAL_HARNESS_METRICS }
+  };
+  if (sessionIds.size === 0) return usage;
+  const receiptRoot = join(clioStateDir(), "receipts");
+  let files;
+  try {
+    files = (await readdir(receiptRoot)).filter((name) => name.endsWith(".json"));
+  } catch {
+    return usage;
+  }
+  for (const file of files) {
+    let parsed;
+    try {
+      parsed = JSON.parse(await readFile(join(receiptRoot, file), "utf8"));
+    } catch {
+      continue;
+    }
+    if (!isRecord(parsed) || typeof parsed.sessionId !== "string" || !sessionIds.has(parsed.sessionId)) continue;
+    usage.tokens += readNumber(parsed.tokenCount);
+    usage.costUsd += readNumber(parsed.costUsd);
+    usage.harness.receiptCount += 1;
+    usage.harness.toolCalls += readNumber(parsed.toolCalls);
+    if (isRecord(parsed.safety) && isRecord(parsed.safety.decisions)) {
+      usage.harness.safetyBlocks += readNumber(parsed.safety.decisions.blocked);
+    }
+  }
+  return usage;
+}
+function runInfraError(label, run) {
+  if (run.timedOut) return `${label} run timed out`;
+  if (run.exitCode !== 0) {
+    const detail = run.stderr.trim().split("\n").at(-1) ?? "";
+    return `${label} run exited ${run.exitCode}${detail.length > 0 ? `: ${detail}` : ""}`;
+  }
+  if (run.transcript.trim().length === 0 && run.finalText.trim().length === 0) {
+    return `${label} run produced no transcript`;
+  }
+  return null;
+}
+function errorBullets(scenario, reason) {
+  return scenario.expected.map((text, index) => ({ index: index + 1, text, verdict: "error", reason }));
+}
+function captureHeadlessRun(args, cwd, timeoutMs, env) {
+  const startedMs = Date.now();
+  return new Promise((resolvePromise) => {
+    let stdout = "";
+    let pendingStdout = "";
+    let stderr = "";
+    let timedOut = false;
+    let settled = false;
+    const keepLine = (line) => {
+      if (line.length === 0) return;
+      if (stdout.length < CHILD_OUTPUT_LIMIT) stdout += `${line}
+`;
+    };
+    const child = spawn(process.execPath, [process.argv[1] ?? "", ...args], {
+      cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+      env
+    });
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+      pendingStdout += chunk;
+      const lines = pendingStdout.split("\n");
+      pendingStdout = lines.pop() ?? "";
+      for (const line of lines) keepLine(line);
+    });
+    child.stderr.on("data", (chunk) => {
+      if (stderr.length < CHILD_OUTPUT_LIMIT) stderr += chunk;
+    });
+    let killTimer;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      child.kill("SIGTERM");
+      killTimer = setTimeout(() => child.kill("SIGKILL"), 5e3);
+    }, timeoutMs);
+    const finish = (exitCode) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (killTimer !== void 0) clearTimeout(killTimer);
+      keepLine(pendingStdout);
+      pendingStdout = "";
+      const parsedRun = parseRunStdout(stdout);
+      resolvePromise({
+        ...parsedRun,
+        exitCode,
+        timedOut,
+        wallTimeMs: Date.now() - startedMs,
+        stderr
+      });
+    };
+    child.on("error", (error) => {
+      stderr += `
+${error.message}`;
+      finish(1);
+    });
+    child.on("close", (code) => finish(typeof code === "number" ? code : timedOut ? 124 : 1));
+  });
+}
+function loadsSkillBody(tool, args) {
+  if (tool !== "context" || !isRecord(args)) return false;
+  return args.scope === "skills" && typeof args.name === "string" && args.name.trim().length > 0;
+}
+function parseRunStdout(stdout) {
+  let sessionId = null;
+  const lines = [];
+  let finalText = "";
+  let sawJson = false;
+  const skillBodyCallIds = /* @__PURE__ */ new Set();
+  for (const line of stdout.split("\n")) {
+    if (line.trim().length === 0) continue;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (!isRecord(event)) continue;
+    sawJson = true;
+    if (event.type === "session" && typeof event.id === "string") {
+      sessionId = event.id;
+      continue;
+    }
+    if (event.type === "tool_execution_start") {
+      const tool = readString(event.toolName) ?? readString(event.tool) ?? "tool";
+      const args = event.args ?? event.arguments ?? event.input;
+      const callId = readString(event.toolCallId);
+      if (callId !== null && loadsSkillBody(tool, args)) skillBodyCallIds.add(callId);
+      lines.push(`TOOL ${tool} args=${preview(args)}`);
+      if (tool === "artifact" && isRecord(args) && typeof args.content === "string") {
+        lines.push(`TERMINAL ${tool} content:
+${args.content}`);
+      }
+      continue;
+    }
+    if (event.type === "tool_execution_end") {
+      const tool = readString(event.toolName) ?? readString(event.tool) ?? "tool";
+      const status = event.isError === true ? "error" : "ok";
+      const callId = readString(event.toolCallId);
+      if (callId !== null && skillBodyCallIds.has(callId)) {
+        lines.push(`RESULT ${tool} ${status}: <skill body withheld from the judge>`);
+        continue;
+      }
+      lines.push(`RESULT ${tool} ${status}: ${preview(event.result)}`);
+      continue;
+    }
+    if (event.type === "message_end" && isRecord(event.message) && event.message.role === "assistant") {
+      const content = Array.isArray(event.message.content) ? event.message.content : [];
+      const text = content.filter((item) => {
+        return isRecord(item) && item.type === "text" && typeof item.text === "string";
+      }).map((item) => item.text).join("").trim();
+      if (text.length > 0) {
+        lines.push(`ASSISTANT: ${text}`);
+        finalText = text;
+      }
+    }
+  }
+  if (!sawJson) {
+    const text = stdout.trim();
+    return { sessionId: null, transcript: text, finalText: text };
+  }
+  return { sessionId, transcript: elide(lines.join("\n")), finalText };
+}
+function judgePrompt(scenario, baselineTranscript, treatmentTranscript) {
+  const bullets = scenario.expected.map((text, index) => `${index + 1}. ${text}`).join("\n");
+  return [
+    "You are scoring a skill evaluation. Two transcripts follow: BASELINE ran without the skill, TREATMENT ran with the skill loaded.",
+    "Score each EXPECTED bullet strictly against the TREATMENT transcript; the baseline exists only for gap context.",
+    "A bullet passes only if the treatment transcript observably satisfies it; anything unverifiable from the transcript fails.",
+    'Reply with STRICT JSON only, no prose and no code fences, exactly: {"bullets":[{"index":1,"pass":true,"reason":"<= 25 words"}]}',
+    `Include one entry per bullet, indexes 1 through ${scenario.expected.length} in order.`,
+    "Do not use any tools. Respond with the JSON verdict directly.",
+    "",
+    `SCENARIO ${scenario.id} - ${scenario.title}`,
+    `SETUP: ${scenario.setup}`,
+    "",
+    "EXPECTED BULLETS:",
+    bullets,
+    "",
+    "BASELINE TRANSCRIPT:",
+    baselineTranscript.length > 0 ? baselineTranscript : "(empty)",
+    "",
+    "TREATMENT TRANSCRIPT:",
+    treatmentTranscript.length > 0 ? treatmentTranscript : "(empty)"
+  ].join("\n");
+}
+function judgeVerdictAbsenceReason(judge) {
+  const sawBulletsKey = `${judge.finalText}
+${judge.transcript}`.includes('"bullets"');
+  return sawBulletsKey ? "judge response truncated mid-verdict (no complete bullets object): bullet not scored" : "judge response carried no parseable verdict object: bullet not scored";
+}
+function parseJudgeVerdicts(scenario, judge) {
+  const parsed = extractBulletsObject(judge.finalText) ?? extractBulletsObject(judge.transcript);
+  if (parsed === null) {
+    const reason = judgeVerdictAbsenceReason(judge);
+    return scenario.expected.map((text, i) => ({ index: i + 1, text, verdict: "unmeasured", reason }));
+  }
+  const entries = /* @__PURE__ */ new Map();
+  if (Array.isArray(parsed.bullets)) {
+    for (const item of parsed.bullets) {
+      if (!isRecord(item)) continue;
+      const index = typeof item.index === "number" ? item.index : Number.parseInt(String(item.index), 10);
+      if (!Number.isInteger(index)) continue;
+      entries.set(index, {
+        pass: item.pass === true,
+        reason: readString(item.reason) ?? ""
+      });
+    }
+  }
+  return scenario.expected.map((text, i) => {
+    const index = i + 1;
+    const entry = entries.get(index);
+    if (entry === void 0) {
+      return {
+        index,
+        text,
+        verdict: "unmeasured",
+        reason: "judge output omitted this bullet: bullet not scored"
+      };
+    }
+    return { index, text, verdict: entry.pass ? "pass" : "fail", reason: entry.reason };
+  });
+}
+function extractBulletsObject(text) {
+  const stripped = text.replace(/```(?:json)?/g, "");
+  const bulletsAt = stripped.indexOf('"bullets"');
+  if (bulletsAt < 0) return null;
+  let start = stripped.lastIndexOf("{", bulletsAt);
+  while (start >= 0) {
+    const candidate = balancedJsonSlice(stripped, start);
+    if (candidate !== null) {
+      try {
+        const parsed = JSON.parse(candidate);
+        if (isRecord(parsed) && Array.isArray(parsed.bullets)) return parsed;
+      } catch {
+      }
+    }
+    if (start === 0) break;
+    start = stripped.lastIndexOf("{", start - 1);
+  }
+  return null;
+}
+function balancedJsonSlice(text, start) {
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+  for (let i = start; i < text.length; i += 1) {
+    const ch = text[i];
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+    if (ch === "\\") {
+      if (inString) escaped = true;
+      continue;
+    }
+    if (ch === '"') {
+      inString = !inString;
+      continue;
+    }
+    if (inString) continue;
+    if (ch === "{") depth += 1;
+    else if (ch === "}") {
+      depth -= 1;
+      if (depth === 0) return text.slice(start, i + 1);
+    }
+  }
+  return null;
+}
+function synthesizeArtifact(skillName, evalsPath, evalsRaw, startedAt, endedAt, outcomes, target) {
+  const contentHash = createHash("sha256").update(evalsRaw, "utf8").digest("hex");
+  const stamp = startedAt.replace(/[-:.]/g, "");
+  const evalId = `skill-${skillName}-${stamp}-${contentHash.slice(0, 8)}`;
+  const results = outcomes.map((outcome) => {
+    const failed = outcome.bullets.some((bullet) => bullet.verdict === "fail" || bullet.verdict === "error");
+    const unmeasured = outcome.bullets.some((bullet) => bullet.verdict === "unmeasured");
+    const pass = outcome.bullets.length > 0 && !failed && !unmeasured;
+    const record = {
+      taskId: outcome.scenario.id,
+      runId: `${evalId}-${outcome.scenario.id}`,
+      repeatIndex: 0,
+      cwd: outcome.workspace,
+      prompt: outcome.scenario.setup,
+      tags: [
+        "skill-eval",
+        `skill:${skillName}`,
+        ...unmeasured ? ["scenario:unmeasured"] : [],
+        ...outcome.bullets.map((bullet) => `bullet-${bullet.index}:${bullet.verdict}`)
+      ],
+      pass,
+      // 3, not 1, when the scenario carries an unscored bullet and no failed
+      // one: a reader who sees exitCode 1 with failureClass verifier_failed
+      // reads "this skill failed its rubric", and an unscored bullet is the
+      // judge's silence, not the skill's answer.
+      exitCode: pass ? 0 : failed ? 1 : 3,
+      tokens: outcome.usage.tokens,
+      costUsd: outcome.usage.costUsd,
+      wallTimeMs: outcome.wallTimeMs,
+      harness: outcome.usage.harness,
+      commands: []
+    };
+    if (failed) record.failureClass = "verifier_failed";
+    return record;
+  });
+  return {
+    version: 1,
+    evalId,
+    taskFile: evalsPath,
+    taskFileHash: contentHash,
+    clio: evalClioProvenance(),
+    environment: evalEnvironmentProvenance(),
+    target: target ?? null,
+    model: null,
+    thinking: null,
+    paths: { taskFile: evalsPath, receipts: [], sessionLedgers: [] },
+    repeat: 1,
+    startedAt,
+    endedAt,
+    summary: summarizeEvalResults(results),
+    results
+  };
+}
+function sidecar(skillName, evalId, outcomes, allowNetwork) {
+  return {
+    version: 1,
+    schema: "experimental",
+    kind: "skill-eval",
+    skill: skillName,
+    evalId,
+    network: networkPolicyLabel(allowNetwork),
+    autonomy: ARM_AUTONOMY,
+    deltas: [
+      "bullet verdicts are judge-scored from run transcripts, not command exit codes; the evals-domain artifact carries scenario-level records with empty command lists",
+      "a bullet the judge never scored is recorded unmeasured, not failed: its scenario record is pass:false with exitCode 3 and no failureClass",
+      "an arm whose transcript carries the headless permission wall is recorded unmeasured with an infraError: the harness's own gate is not a verdict about the skill",
+      "tokens and cost are rolled up from headless main-agent receipts when those receipts are present",
+      "this sidecar is additive and is registered in overview.json files[]"
+    ],
+    scenarios: outcomes.map((outcome) => ({
+      id: outcome.scenario.id,
+      title: outcome.scenario.title,
+      setup: outcome.scenario.setup,
+      fixtureCommands: outcome.scenario.fixtureCommands ?? null,
+      infraError: outcome.infraError,
+      usage: outcome.usage,
+      bullets: outcome.bullets,
+      baseline: sidecarRun(outcome.baseline),
+      treatment: sidecarRun(outcome.treatment),
+      judge: sidecarRun(outcome.judge)
+    }))
+  };
+}
+function sidecarRun(run) {
+  if (run === null) return null;
+  return {
+    sessionId: run.sessionId,
+    exitCode: run.exitCode,
+    timedOut: run.timedOut,
+    wallTimeMs: run.wallTimeMs,
+    transcript: run.transcript,
+    stderrTail: run.stderr.slice(-500)
+  };
+}
+function printHumanReport(skillName, outcomes, evalId, evidenceId, evidenceDirectory, allowNetwork) {
+  const rows = [["scenario", "bullet", "verdict", "expected"]];
+  for (const outcome of outcomes) {
+    for (const bullet of outcome.bullets) {
+      rows.push([outcome.scenario.id, String(bullet.index), bullet.verdict, truncate(bullet.text, 76)]);
+    }
+  }
+  process.stdout.write(formatColumns(rows));
+  const total = outcomes.reduce((sum, outcome) => sum + outcome.bullets.length, 0);
+  const passed = outcomes.reduce(
+    (sum, outcome) => sum + outcome.bullets.filter((bullet) => bullet.verdict === "pass").length,
+    0
+  );
+  const unmeasured = outcomes.reduce(
+    (sum, outcome) => sum + outcome.bullets.filter((bullet) => bullet.verdict === "unmeasured").length,
+    0
+  );
+  const unmeasuredNote = unmeasured > 0 ? `, ${unmeasured} unmeasured` : "";
+  process.stdout.write(
+    `
+${skillName}: ${passed}/${total} bullets passed${unmeasuredNote} (judge-scored, experimental)
+`
+  );
+  if (unmeasured > 0) {
+    process.stdout.write(
+      unmeasured === total ? "no bullet was scored, so this run is not evidence about the skill either way\n" : `${unmeasured} of ${total} bullets were never scored; unmeasured is not a failed bullet
+`
+    );
+  }
+  for (const outcome of outcomes) {
+    if (outcome.infraError !== null) {
+      process.stdout.write(`${outcome.scenario.id}: infra error: ${outcome.infraError}
+`);
+      continue;
+    }
+    for (const bullet of outcome.bullets.filter((item) => item.verdict !== "pass")) {
+      process.stdout.write(
+        `${outcome.scenario.id} bullet ${bullet.index} ${bullet.verdict}${bullet.reason.length > 0 ? `: ${bullet.reason}` : ""}
+`
+      );
+    }
+  }
+  process.stdout.write(`${describeArmPolicyOutcome(allowNetwork)}
+`);
+  process.stdout.write(`eval artifact: ${evalId}
+`);
+  if (evidenceId !== null && evidenceDirectory !== null) {
+    process.stdout.write(`evidence: ${evidenceId} at ${evidenceDirectory} (per-bullet detail in skill-eval.json)
+`);
+  }
+}
+function preview(value) {
+  if (value === void 0) return "";
+  const text = typeof value === "string" ? value : safeStringify(value);
+  return truncate(text.replace(/\s+/g, " ").trim(), PREVIEW_MAX_CHARS);
+}
+function safeStringify(value) {
+  try {
+    return JSON.stringify(value) ?? "";
+  } catch {
+    return String(value);
+  }
+}
+function elide(text) {
+  if (text.length <= TRANSCRIPT_HEAD_CHARS + TRANSCRIPT_TAIL_CHARS) return text;
+  return `${text.slice(0, TRANSCRIPT_HEAD_CHARS)}
+[... transcript elided ...]
+${text.slice(-TRANSCRIPT_TAIL_CHARS)}`;
+}
+function truncate(text, maxChars) {
+  if (text.length <= maxChars) return text;
+  return `${text.slice(0, Math.max(0, maxChars - 3))}...`;
+}
+function readString(value) {
+  return typeof value === "string" && value.length > 0 ? value : null;
+}
+function readNumber(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+export {
+  armRunArgs,
+  evalChildEnv,
+  extractBulletsObject,
+  materializeSkillEvalWorkspaces,
+  parseJudgeVerdicts,
+  parseRunStdout,
+  permissionWallReason,
+  resolveSkillBaseDir,
+  runSkillsEvalCommand
+};
+//# sourceMappingURL=skills-eval-E2ZTW4PL.js.map