@hassangameryt/freeflow 0.7.2 → 0.8.0

package/skills/evaluate-skill/scripts/lib/review.mjs

@@ -0,0 +1,196 @@
+import { randomInt } from "node:crypto";
+import { mkdir, readFile, rm } from "node:fs/promises";
+import path from "node:path";
+
+import { sha256, writeJson, writeText } from "./evidence.mjs";
+import { baseProcessEnvironment, runPiProcess } from "./pi.mjs";
+import { resolveResult } from "./view.mjs";
+
+const ANSWERS = new Set(["yes", "no", "unclear"]);
+const RESPONSE_LIMIT = 8000;
+const ARGUMENT_LIMIT = 300;
+
+/**
+ * Answers each group's review questions with a separate model that sees two unlabeled runs.
+ * The result is advisory: it is written beside, never into, canonical run and grade evidence.
+ *
+ * @param {string} target
+ * @param {{group: string | null}} selectors
+ * @param {{root?: string, signal?: AbortSignal, model: string, thinking?: string | null}} options
+ */
+export async function reviewResult(target, selectors, { root = process.cwd(), signal, model, thinking = null }) {
+  const resultDirectory = await resolveResult(target, root);
+  const summary = JSON.parse(await readFile(path.join(resultDirectory, "summary.json"), "utf8"));
+  const groups = selectGroups(summary, selectors.group);
+  const outcomes = [];
+  for (const group of groups) {
+    if (signal?.aborted) break;
+    outcomes.push(
+      await reviewGroup(path.join(resultDirectory, "groups", group.id), group.id, { model, thinking, signal }),
+    );
+  }
+  return { resultDirectory, groups: outcomes };
+}
+
+function selectGroups(summary, selector) {
+  const groups = Array.isArray(summary.groups) ? summary.groups : [];
+  if (selector === null) return groups;
+  if (summary.definitionKind === "group") throw new Error("--group cannot be used with a direct group result");
+  const position = /^\d+$/.test(selector) ? Number(selector) : null;
+  const selected = groups.filter((group) => (position === null ? group.id === selector : group.position === position));
+  if (selected.length === 0) throw new Error(`Unknown group selector: ${selector}`);
+  return selected;
+}
+
+async function reviewGroup(groupDirectory, id, { model, thinking, signal }) {
+  const definition = JSON.parse(await readFile(path.join(groupDirectory, "definition.json"), "utf8"));
+  const questions = Array.isArray(definition.review_questions) ? definition.review_questions : [];
+  if (questions.length === 0) return { id, state: "skipped", reason: "no review questions" };
+  const runs = {};
+  for (const variant of ["baseline", "candidate"]) {
+    runs[variant] = JSON.parse(await readFile(path.join(groupDirectory, variant, "run.json"), "utf8"));
+    if (runs[variant].state !== "complete")
+      return { id, state: "skipped", reason: `${variant} run is ${runs[variant].state}` };
+  }
+
+  const labels = randomInt(2) === 0 ? { A: "baseline", B: "candidate" } : { A: "candidate", B: "baseline" };
+  const prompt = reviewPrompt(definition, questions, { A: runs[labels.A], B: runs[labels.B] });
+  const reviewDirectory = path.join(groupDirectory, "review");
+  await rm(reviewDirectory, { recursive: true, force: true });
+  await mkdir(reviewDirectory, { recursive: true });
+  await writeText(path.join(reviewDirectory, "prompt.md"), prompt);
+
+  const args = [
+    "--mode",
+    "json",
+    "-p",
+    "--no-session",
+    "--no-extensions",
+    "--no-skills",
+    "--no-prompt-templates",
+    "--no-themes",
+    "--no-context-files",
+    "--no-approve",
+    "--no-tools",
+    "--model",
+    model,
+  ];
+  if (thinking !== null) args.push("--thinking", thinking);
+  args.push(prompt);
+  const observation = await runPiProcess({
+    command: "pi",
+    args,
+    cwd: reviewDirectory,
+    eventsFile: path.join(reviewDirectory, "events.jsonl"),
+    stderrFile: path.join(reviewDirectory, "stderr.log"),
+    signal,
+    environment: { ...baseProcessEnvironment(), PWD: reviewDirectory, PI_SKIP_VERSION_CHECK: "1", PI_TELEMETRY: "0" },
+  });
+  await writeText(path.join(reviewDirectory, "final.md"), `${observation.response}\n`);
+
+  const errors = [];
+  if (observation.exitCode !== 0 || !observation.settled || observation.assistantError !== null) {
+    errors.push(
+      `reviewer did not complete: exit=${observation.exitCode} settled=${observation.settled} error=${observation.assistantError}`,
+    );
+  }
+  const answers = errors.length === 0 ? parseAnswers(observation.response, questions, labels, errors) : [];
+  const state = errors.length === 0 ? "complete" : "review-error";
+  await writeJson(path.join(groupDirectory, "semantic-grade.json"), {
+    schema_version: 1,
+    kind: "advisory-review",
+    advisory: true,
+    state,
+    model,
+    thinking,
+    labels,
+    prompt: { file: "review/prompt.md", sha256: sha256(prompt) },
+    answers,
+    errors,
+    usage: observation.usage,
+    artifacts: { events: "review/events.jsonl", final: "review/final.md", stderr: "review/stderr.log" },
+    completedAt: new Date().toISOString(),
+  });
+  return { id, state, artifact: "semantic-grade.json" };
+}
+
+function reviewPrompt(definition, questions, runs) {
+  const task =
+    typeof definition.input?.prompt === "string" ? [definition.input.prompt] : (definition.input?.turns ?? []);
+  const lines = [
+    "You are reviewing two anonymous runs, Run A and Run B, of the same agent task.",
+    "Judge only the evidence below. Answer every question for both runs with yes, no, or unclear.",
+    "",
+    "Task turns:",
+    ...task.map((turn, index) => `${index + 1}. ${turn}`),
+    "",
+    "Questions:",
+    ...questions.map((question, index) => `${index + 1}. ${question}`),
+  ];
+  for (const label of ["A", "B"]) lines.push("", `## Run ${label}`, ...runEvidence(runs[label]));
+  lines.push(
+    "",
+    "Reply with only JSON in this shape:",
+    '{"answers":[{"question":1,"A":"yes|no|unclear","B":"yes|no|unclear","rationale":"one sentence"}]}',
+  );
+  return lines.join("\n");
+}
+
+function runEvidence(run) {
+  const turns = Array.isArray(run.turns)
+    ? run.turns
+    : [{ turn: 1, response: run.response, toolActivity: run.toolActivity }];
+  const lines = [];
+  for (const turn of turns) {
+    lines.push(`Turn ${turn.turn ?? 1} response:`, truncate(String(turn.response ?? ""), RESPONSE_LIMIT));
+    for (const call of turn.toolActivity ?? []) {
+      const status = call.completed ? (call.isError ? "failed" : "succeeded") : "incomplete";
+      lines.push(`- tool ${call.toolName} ${status}: ${truncate(JSON.stringify(call.args ?? null), ARGUMENT_LIMIT)}`);
+    }
+  }
+  const changes = run.effects?.changes;
+  if (changes) {
+    for (const kind of ["created", "modified", "deleted"]) {
+      lines.push(`Files ${kind}: ${(changes[kind] ?? []).join(", ") || "none"}`);
+    }
+  }
+  return lines;
+}
+
+function truncate(value, limit) {
+  return value.length <= limit ? value : `${value.slice(0, limit)}… [truncated ${value.length - limit} characters]`;
+}
+
+function parseAnswers(response, questions, labels, errors) {
+  const text = response
+    .trim()
+    .replace(/^```(?:json)?\s*/i, "")
+    .replace(/\s*```$/, "");
+  let parsed;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    errors.push("reviewer response is not JSON");
+    return [];
+  }
+  const answers = Array.isArray(parsed?.answers) ? parsed.answers : null;
+  if (answers === null || answers.length !== questions.length) {
+    errors.push(`reviewer must answer exactly ${questions.length} question(s)`);
+    return [];
+  }
+  const mapped = [];
+  for (const [index, question] of questions.entries()) {
+    const answer = answers.find((entry) => entry?.question === index + 1);
+    if (!answer || !ANSWERS.has(answer.A) || !ANSWERS.has(answer.B) || typeof answer.rationale !== "string") {
+      errors.push(`reviewer answer for question ${index + 1} is missing or invalid`);
+      continue;
+    }
+    mapped.push({
+      question,
+      [labels.A]: answer.A,
+      [labels.B]: answer.B,
+      rationale: answer.rationale,
+    });
+  }
+  return errors.length === 0 ? mapped : [];
+}

package/skills/evaluate-skill/scripts/lib/runner.mjs

@@ -1,7 +1,7 @@
 import { mkdir, realpath } from "node:fs/promises";
 import path from "node:path";
 
-import { loadDefinition, selectDefinition } from "./definitions.mjs";
+import { COMMAND_TOOL, loadDefinition, selectDefinition } from "./definitions.mjs";
 import { createInvocationId, fileIdentity, writeJson, writeText } from "./evidence.mjs";
 import { gradeDeterministic } from "./grade.mjs";
 import { runBody, runDescription, VariantSetupError } from "./pi.mjs";
@@ -12,14 +12,24 @@ const DESCRIPTION_TOOLS = new Set(["read"]);
 const BODY_TOOLS = new Set(["read", "write", "edit"]);
 // Native tools must be separated from extension tools before CLI tool selection is built.
 const NATIVE_TOOLS = new Set(["read", "bash", "powershell", "edit", "write", "grep", "find", "ls"]);
-const DEFAULT_RUNTIME = { host: "pi", session: false, extensions: [], environment: { literal: {}, inherit: [] } };
+const DEFAULT_RUNTIME = {
+  host: "pi",
+  session: false,
+  prompt: "isolated",
+  extensions: [],
+  environment: { literal: {}, inherit: [] },
+};
 
 /**
  * @param {string} definitionFile
  * @param {any} selectors
- * @param {{root?: string, signal?: AbortSignal}} options
+ * @param {{root?: string, signal?: AbortSignal, budget?: {maxCost: number, spent: number, exhausted: boolean} | null}} options
  */
-export async function runEvaluation(definitionFile, selectors = {}, { root = process.cwd(), signal } = {}) {
+export async function runEvaluation(
+  definitionFile,
+  selectors = {},
+  { root = process.cwd(), signal, budget = null } = {},
+) {
   const definitionPath = await realpath(path.resolve(root, definitionFile));
   const definition = await loadDefinition(definitionPath, { root });
   const selection = selectDefinition(definition, selectors);
@@ -45,9 +55,27 @@ export async function runEvaluation(definitionFile, selectors = {}, { root = pro
     startedAt: new Date().toISOString(),
   });
 
+  // Budget exhaustion cancels only queued subjects; a subject already running completes and is charged.
+  const controller = new AbortController();
+  const onParentAbort = () => controller.abort();
+  if (signal?.aborted) controller.abort();
+  else signal?.addEventListener("abort", onParentAbort, { once: true });
+  const charge = (run) => {
+    if (budget === null) return;
+    budget.spent += run.usage?.cost?.total ?? 0;
+    if (budget.spent >= budget.maxCost) {
+      budget.exhausted = true;
+      controller.abort();
+    }
+  };
+
   const groups = [];
-  for (const selected of selection.groups) {
-    groups.push(await runGroup({ selected, root, resultDirectory, signal }));
+  try {
+    for (const selected of selection.groups) {
+      groups.push(await runGroup({ selected, root, resultDirectory, signal: controller.signal, charge }));
+    }
+  } finally {
+    signal?.removeEventListener("abort", onParentAbort);
   }
   const state = batchState(groups);
   const summary = {
@@ -58,6 +86,7 @@ export async function runEvaluation(definitionFile, selectors = {}, { root = pro
     groups,
     completedAt: new Date().toISOString(),
   };
+  if (budget !== null) summary.budget = budgetEvidence(budget);
   await writeJson(path.join(resultDirectory, "summary.json"), summary);
   return { id: invocationId, path: resultDirectory, state, summary };
 }
@@ -76,7 +105,10 @@ function assertSupportedSelection(selection) {
     if (unsupportedNativeTools.length > 0) {
       throw new Error(`${group.type} execution does not support native tools: ${unsupportedNativeTools.join(", ")}`);
     }
-    const customTools = group.tools.filter((tool) => !NATIVE_TOOLS.has(tool) && !supportedTools.has(tool));
+    // The evaluator supplies run_command itself when the definition declares commands.
+    const customTools = group.tools.filter(
+      (tool) => !NATIVE_TOOLS.has(tool) && !supportedTools.has(tool) && tool !== COMMAND_TOOL,
+    );
     if (group.type === "description" && customTools.length > 0) {
       throw new Error(`${group.type} execution does not support custom tools: ${customTools.join(", ")}`);
     }
@@ -88,7 +120,11 @@ function assertSupportedSelection(selection) {
   }
 }
 
-async function runGroup({ selected, root, resultDirectory, signal }) {
+export function budgetEvidence(budget) {
+  return { maxCost: budget.maxCost, spent: Number(budget.spent.toFixed(10)), exhausted: budget.exhausted };
+}
+
+async function runGroup({ selected, root, resultDirectory, signal, charge }) {
   const { group } = selected;
   const groupDirectory = path.join(resultDirectory, "groups", group.id);
   await mkdir(groupDirectory, { recursive: true });
@@ -121,6 +157,7 @@ async function runGroup({ selected, root, resultDirectory, signal }) {
       runs[variant] = cancelledRun(group, variant);
     } else if (resourceError === null) {
       runs[variant] = await executeVariant({ group, variant, root, variantDirectory, fixture, runtime, signal });
+      charge(runs[variant]);
     } else {
       runs[variant] = invalidRun(group, variant, resourceError);
     }

package/skills/evaluate-skill/scripts/lib/sandbox.mjs

@@ -110,6 +110,7 @@ export async function prepareRuntime({ runtime, root, groupDirectory }) {
   return {
     host: runtime.host,
     session: runtime.session,
+    prompt: runtime.prompt ?? "isolated",
     environment: { ...runtime.environment },
     extensions: bundles,
   };
@@ -145,6 +146,7 @@ export async function materializeRuntime({ runtime, variantDirectory }) {
   return {
     host: runtime.host,
     session: runtime.session,
+    prompt: runtime.prompt ?? "isolated",
     environment: { ...runtime.environment },
     extensions: bundles,
   };

package/skills/evaluate-skill/scripts/lib/trials.mjs

@@ -0,0 +1,103 @@
+import { mkdir, readFile } from "node:fs/promises";
+import path from "node:path";
+
+import { createInvocationId, writeJson } from "./evidence.mjs";
+import { budgetEvidence, runEvaluation } from "./runner.mjs";
+
+const CHECK_STATES = ["pass", "fail", "unavailable"];
+
+/**
+ * Runs the selection as independent complete invocations and records an aggregate beside them.
+ * Each trial remains an ordinary result; the aggregate only counts their deterministic grades.
+ *
+ * @param {string} definitionFile
+ * @param {any} selectors
+ * @param {{root?: string, signal?: AbortSignal, trials: number, maxCost: number | null, onTrial?: (result: any) => void}} options
+ */
+export async function runTrials(definitionFile, selectors, { root = process.cwd(), signal, trials, maxCost, onTrial }) {
+  const budget = maxCost === null ? null : { maxCost, spent: 0, exhausted: false };
+  const results = [];
+  for (let trial = 0; trial < trials; trial += 1) {
+    if (signal?.aborted || budget?.exhausted) break;
+    const result = await runEvaluation(definitionFile, selectors, { root, signal, budget });
+    results.push(result);
+    onTrial?.(result);
+  }
+
+  const id = createInvocationId();
+  const directory = path.join(root, ".skill-eval", "runs", id);
+  await mkdir(directory, { recursive: true });
+  const aggregate = {
+    schema_version: 1,
+    kind: "aggregate",
+    id,
+    state: aggregateState(results, trials),
+    requestedTrials: trials,
+    trials: results.map((result) => ({ id: result.id, state: result.state })),
+    groups: await aggregateGroups(results),
+    completedAt: new Date().toISOString(),
+  };
+  if (budget !== null) aggregate.budget = budgetEvidence(budget);
+  await writeJson(path.join(directory, "aggregate.json"), aggregate);
+  return { id, path: directory, state: aggregate.state, aggregate };
+}
+
+function aggregateState(results, requested) {
+  if (results.length < requested || results.some((result) => result.state === "cancelled")) return "cancelled";
+  if (results.every((result) => result.state === "complete")) return "complete";
+  return "partially-complete";
+}
+
+async function aggregateGroups(results) {
+  /** @type {Map<string, any>} */
+  const groups = new Map();
+  for (const result of results) {
+    for (const group of result.summary.groups) {
+      const entry = groups.get(group.id) ?? { id: group.id, trials: 0, comparisons: new Map(), checks: new Map() };
+      groups.set(group.id, entry);
+      const gradeFile = path.join(
+        result.path,
+        "groups",
+        group.id,
+        group.artifacts?.grade ?? "deterministic-grade.json",
+      );
+      const grade = JSON.parse(await readFile(gradeFile, "utf8"));
+      entry.trials += 1;
+      for (const check of grade.checks ?? []) {
+        const counts = entry.checks.get(check.id) ?? {
+          id: check.id,
+          variant: check.variant,
+          kind: check.kind,
+          ...zeroCounts(),
+        };
+        entry.checks.set(check.id, counts);
+        if (CHECK_STATES.includes(check.state)) counts[check.state] += 1;
+      }
+      for (const comparison of grade.comparisons ?? []) {
+        const counts = entry.comparisons.get(comparison.id) ?? {
+          id: comparison.id,
+          kind: comparison.kind,
+          baseline: zeroCounts(),
+          candidate: zeroCounts(),
+          transitions: {},
+        };
+        entry.comparisons.set(comparison.id, counts);
+        for (const variant of ["baseline", "candidate"]) {
+          const state = comparison[variant]?.state;
+          if (CHECK_STATES.includes(state)) counts[variant][state] += 1;
+        }
+        counts.transitions[comparison.transition] = (counts.transitions[comparison.transition] ?? 0) + 1;
+      }
+    }
+  }
+  return [...groups.values()].map((entry) => ({
+    id: entry.id,
+    trials: entry.trials,
+    comparisons: [...entry.comparisons.values()],
+    checks: [...entry.checks.values()],
+  }));
+}
+
+function zeroCounts() {
+  return { pass: 0, fail: 0, unavailable: 0 };
+}

package/skills/evaluate-skill/scripts/lib/view.mjs

@@ -22,6 +22,13 @@ class ViewError extends Error {
 
 export async function renderResult(target, selectors = DEFAULT_SELECTORS, { root = process.cwd() } = {}) {
   const resultDirectory = await resolveResult(target, root);
+  const aggregateFile = path.join(resultDirectory, "aggregate.json");
+  if (await stat(aggregateFile).catch(() => null)) {
+    if (selectors.group !== null || selectors.variant !== null) {
+      throw new ViewError("selectors are not supported for an aggregate; view an individual trial instead");
+    }
+    return renderAggregate(resultDirectory, await readJson(aggregateFile, "aggregate"));
+  }
   const summary = await readJson(path.join(resultDirectory, "summary.json"), "result summary");
   let groups = Array.isArray(summary.groups) ? summary.groups : [];
   if (summary.definitionKind === "group" && selectors.group !== null) {
@@ -56,6 +63,7 @@ async function renderGroup(resultDirectory, group, selectedVariant) {
 
   appendSharedEvidence(lines, definition);
   appendGradeEvidence(lines, grade, group, definition, selectedVariant);
+  await appendAdvisoryReview(lines, groupDirectory, selectedVariant);
   for (const variant of variants) {
     await appendVariantEvidence(lines, resultDirectory, groupDirectory, group.id, variant);
   }
@@ -135,6 +143,64 @@ function appendGradeEvidence(lines, grade, group, definition, selectedVariant) {
   }
 }
 
+// Advisory answers stay visually separate from deterministic grades; they are not canonical evidence.
+async function appendAdvisoryReview(lines, groupDirectory, selectedVariant) {
+  const file = path.join(groupDirectory, "semantic-grade.json");
+  if (!(await stat(file).catch(() => null))) return;
+  const review = await readJson(file, "advisory review");
+  lines.push(`Advisory review [${review.state}] model=${review.model} (not canonical evidence)`);
+  const variants = selectedVariant === null ? VARIANTS : [selectedVariant];
+  for (const [index, answer] of (review.answers ?? []).entries()) {
+    lines.push(
+      compactRow("review", [
+        `Q${index + 1}`,
+        ...variants.map((variant) => `${variant}=${answer[variant]}`),
+        answer.rationale,
+      ]),
+    );
+  }
+  for (const error of review.errors ?? []) lines.push(compactRow("error", ["review", error]));
+}
+
+function renderAggregate(resultDirectory, aggregate) {
+  const lines = [
+    `Aggregate ${aggregate.id} [${aggregate.state}] trials=${aggregate.trials.length}/${aggregate.requestedTrials}`,
+    `Path: ${escapeCell(resultDirectory)}`,
+  ];
+  if (aggregate.budget) {
+    lines.push(
+      compactRow("budget", [
+        `max-cost=${formatNumber(aggregate.budget.maxCost)}`,
+        `spent=${formatNumber(aggregate.budget.spent)}`,
+        `exhausted=${aggregate.budget.exhausted}`,
+      ]),
+    );
+  }
+  for (const trial of aggregate.trials) lines.push(compactRow("trial", [trial.id, trial.state]));
+  for (const group of aggregate.groups) {
+    lines.push("", `Group ${group.id} trials=${group.trials}`);
+    for (const comparison of group.comparisons) {
+      lines.push(
+        compactRow("comparison", [
+          comparison.id,
+          comparison.kind,
+          `baseline=${passRate(comparison.baseline)}`,
+          `candidate=${passRate(comparison.candidate)}`,
+          ...Object.entries(comparison.transitions).map(([transition, count]) => `${transition}=${count}`),
+        ]),
+      );
+    }
+    for (const check of group.checks) {
+      lines.push(compactRow("check", [check.id, check.variant, check.kind, `pass=${passRate(check)}`]));
+    }
+  }
+  return `${lines.join("\n")}\n`;
+}
+
+function passRate(counts) {
+  return `${counts.pass}/${counts.pass + counts.fail + counts.unavailable}`;
+}
+
 function selectedGradeErrors(errors, definition, selectedVariant) {
   if (!Array.isArray(errors)) return [];
   if (selectedVariant === null) return errors;
@@ -185,7 +251,7 @@ async function appendVariantEvidence(lines, resultDirectory, groupDirectory, gro
   );
 }
 
-async function resolveResult(target, root) {
+export async function resolveResult(target, root) {
   const explicit = path.resolve(root, target);
   const stored = path.join(root, ".skill-eval", "runs", target);
   const isPath = path.isAbsolute(target) || target.includes("/") || target.includes("\\") || target.startsWith(".");

package/skills/evaluate-skill/scripts/pi-command-tool.mjs

@@ -0,0 +1,144 @@
+import { spawn } from "node:child_process";
+
+import { COMMAND_TOOL } from "./lib/definitions.mjs";
+
+const DEFAULT_TIMEOUT_MS = 120 * 1000;
+const OUTPUT_LIMIT_BYTES = 32 * 1024;
+const TERMINATION_GRACE_MS = 2000;
+
+function parseCommands(value) {
+  const parsed = JSON.parse(value ?? "[]");
+  if (!Array.isArray(parsed) || parsed.length === 0) throw new Error("SKILL_EVAL_COMMANDS must declare commands");
+  return parsed;
+}
+
+function commandEnvironment() {
+  return Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("SKILL_EVAL_")));
+}
+
+function captureStream(stream) {
+  const chunks = [];
+  const capture = { bytes: 0, kept: 0, truncated: false };
+  stream.on("data", (chunk) => {
+    capture.bytes += chunk.length;
+    const room = OUTPUT_LIMIT_BYTES - capture.kept;
+    if (room <= 0) {
+      capture.truncated = true;
+      return;
+    }
+    const kept = chunk.length > room ? chunk.subarray(0, room) : chunk;
+    if (kept.length < chunk.length) capture.truncated = true;
+    chunks.push(kept);
+    capture.kept += kept.length;
+  });
+  capture.text = () => Buffer.concat(chunks).toString("utf8");
+  return capture;
+}
+
+function signalGroup(child, signal) {
+  try {
+    if (process.platform === "win32") child.kill(signal);
+    else process.kill(-child.pid, signal);
+  } catch {
+    // The process group already exited.
+  }
+}
+
+// Runs one author-declared argv in the workspace. A nonzero exit or timeout is ordinary evidence, not a tool error.
+function runCommand(command, cwd, signal) {
+  const timeoutMs = command.timeout_ms ?? DEFAULT_TIMEOUT_MS;
+  return new Promise((resolve, reject) => {
+    const child = spawn(command.argv[0], command.argv.slice(1), {
+      cwd,
+      env: commandEnvironment(),
+      shell: false,
+      detached: process.platform !== "win32",
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    const stdout = captureStream(child.stdout);
+    const stderr = captureStream(child.stderr);
+    let timedOut = false;
+    let forced;
+    const stop = () => {
+      signalGroup(child, "SIGTERM");
+      forced = setTimeout(() => signalGroup(child, "SIGKILL"), TERMINATION_GRACE_MS);
+    };
+    const timer = setTimeout(() => {
+      timedOut = true;
+      stop();
+    }, timeoutMs);
+    const onAbort = () => stop();
+    signal?.addEventListener("abort", onAbort, { once: true });
+    child.once("error", (error) => {
+      clearTimeout(timer);
+      clearTimeout(forced);
+      signal?.removeEventListener("abort", onAbort);
+      reject(error);
+    });
+    child.once("close", (exitCode, exitSignal) => {
+      clearTimeout(timer);
+      clearTimeout(forced);
+      signal?.removeEventListener("abort", onAbort);
+      resolve({ exitCode, signal: exitSignal, timedOut, timeoutMs, stdout, stderr });
+    });
+  });
+}
+
+function renderOutput(label, capture) {
+  const note = capture.truncated ? ` (truncated: showing ${capture.kept} of ${capture.bytes} bytes)` : "";
+  return `--- ${label}${note} ---\n${capture.text()}`;
+}
+
+export default function registerCommandTool(pi) {
+  const commands = parseCommands(process.env.SKILL_EVAL_COMMANDS);
+  const workspace = process.env.SKILL_EVAL_WRITABLE_ROOT;
+  if (!workspace) throw new Error("SKILL_EVAL_WRITABLE_ROOT is required");
+  const byId = new Map(commands.map((command) => [command.id, command]));
+
+  pi.registerTool({
+    name: COMMAND_TOOL,
+    label: "Run command",
+    description: `Run one declared command in the workspace. Available commands: ${commands
+      .map((command) => `${command.id} (${command.argv.join(" ")})`)
+      .join("; ")}.`,
+    parameters: {
+      type: "object",
+      additionalProperties: false,
+      required: ["command"],
+      properties: {
+        command: { type: "string", enum: commands.map((command) => command.id), description: "Declared command ID" },
+      },
+    },
+    executionMode: "sequential",
+    async execute(_toolCallId, input, signal) {
+      const command = byId.get(input?.command);
+      if (!command) throw new Error(`Command is not declared for this evaluation: ${String(input?.command)}`);
+      const result = await runCommand(command, workspace, signal);
+      const status = result.timedOut
+        ? `timed out after ${result.timeoutMs} ms`
+        : `exit code: ${result.exitCode ?? `signal ${result.signal}`}`;
+      return {
+        content: [
+          {
+            type: "text",
+            text: [
+              `$ ${command.argv.join(" ")}`,
+              status,
+              renderOutput("stdout", result.stdout),
+              renderOutput("stderr", result.stderr),
+            ].join("\n"),
+          },
+        ],
+        details: {
+          id: command.id,
+          argv: command.argv,
+          exitCode: result.exitCode,
+          signal: result.signal,
+          timedOut: result.timedOut,
+          stdout: { bytes: result.stdout.bytes, truncated: result.stdout.truncated },
+          stderr: { bytes: result.stderr.bytes, truncated: result.stderr.truncated },
+        },
+      };
+    },
+  });
+}