@hassangameryt/freeflow 0.8.0 → 0.9.0

package/skills/{evaluate-skill → evaluate-behavior}/scripts/lib/view.mjs

@@ -1,7 +1,13 @@
 import { readFile, realpath, stat } from "node:fs/promises";
 import path from "node:path";
 
-const VARIANTS = ["baseline", "candidate"];
+import { ARM_NAME, comparisonArms, storedArms } from "./arms.mjs";
+import { renderClaim, trialCounts } from "./decision.mjs";
+import { judgedAggregateLines, judgedLines } from "./judge.mjs";
+import { aggregateMetricsLines, runMetricsLines } from "./metrics.mjs";
+import { participantLines } from "./participants.mjs";
+import { readClearances } from "./scan.mjs";
+
 const DEFAULT_SELECTORS = /** @type {{group: string | null, variant: string | null}} */ ({
   group: null,
   variant: null,
@@ -20,8 +26,12 @@ class ViewError extends Error {
  * @param {{root?: string}} [options]
  */
 
-export async function renderResult(target, selectors = DEFAULT_SELECTORS, { root = process.cwd() } = {}) {
-  const resultDirectory = await resolveResult(target, root);
+export async function renderResult(
+  target,
+  selectors = DEFAULT_SELECTORS,
+  { root = process.cwd(), results = null } = {},
+) {
+  const resultDirectory = await resolveResult(target, root, results);
   const aggregateFile = path.join(resultDirectory, "aggregate.json");
   if (await stat(aggregateFile).catch(() => null)) {
     if (selectors.group !== null || selectors.variant !== null) {
@@ -42,11 +52,31 @@ export async function renderResult(target, selectors = DEFAULT_SELECTORS, { root
         : groups.filter((group) => group.position === position);
     if (groups.length === 0) throw new ViewError(`Unknown group selector: ${selectors.group}`);
   }
-  if (selectors.variant !== null && !VARIANTS.includes(selectors.variant)) {
-    throw new ViewError("variant must be baseline or candidate");
+  if (selectors.variant !== null && !ARM_NAME.test(selectors.variant)) {
+    throw new ViewError("variant must be an arm name, such as baseline or candidate");
   }
 
   const lines = [`Result ${summary.id} [${summary.state}]`, `Path: ${escapeCell(resultDirectory)}`];
+  // This invocation's own spend; in a trial, the budget's total is the whole run's so far.
+  if (summary.cost) {
+    const unmeasured = summary.cost.unmeasured ? [`unmeasured=${summary.cost.unmeasured}`] : [];
+    lines.push(
+      compactRow("cost", [
+        `spent=${formatNumber(summary.cost.spent)}`,
+        `subjects=${summary.cost.subjects}`,
+        ...unmeasured,
+      ]),
+    );
+  }
+  if (summary.budget) {
+    lines.push(
+      compactRow("budget", [
+        `max-cost=${formatNumber(summary.budget.maxCost)}`,
+        `run-total=${formatNumber(summary.budget.runTotal ?? summary.budget.spent)}`,
+        `exhausted=${summary.budget.exhausted}`,
+      ]),
+    );
+  }
   for (const group of groups) {
     lines.push("", ...(await renderGroup(resultDirectory, group, selectors.variant)));
   }
@@ -58,12 +88,18 @@ async function renderGroup(resultDirectory, group, selectedVariant) {
   const definition = await readJson(path.join(groupDirectory, "definition.json"), `definition for ${group.id}`);
   const gradeFile = path.join(groupDirectory, group.artifacts?.grade ?? "deterministic-grade.json");
   const grade = await readJson(gradeFile, `grade for ${group.id}`);
-  const variants = selectedVariant === null ? VARIANTS : [selectedVariant];
+  const arms = storedArms(group, definition);
+  if (selectedVariant !== null && !arms.includes(selectedVariant)) {
+    throw new ViewError(`variant ${selectedVariant} is not an arm of group ${group.id}`);
+  }
+  const variants = selectedVariant === null ? arms : [selectedVariant];
   const lines = [`Group ${group.id} [${group.state}] type=${definition.type}`];
 
   appendSharedEvidence(lines, definition);
   appendGradeEvidence(lines, grade, group, definition, selectedVariant);
-  await appendAdvisoryReview(lines, groupDirectory, selectedVariant);
+  await appendAdvisoryReview(lines, groupDirectory, variants);
+  lines.push(...(await judgedLines(groupDirectory, variants)));
+  lines.push(...(await participantLines(groupDirectory, variants)));
   for (const variant of variants) {
     await appendVariantEvidence(lines, resultDirectory, groupDirectory, group.id, variant);
   }
@@ -130,13 +166,26 @@ function appendGradeEvidence(lines, grade, group, definition, selectedVariant) {
   if (selectedVariant === null && Array.isArray(grade.comparisons) && grade.comparisons.length > 0) {
     lines.push("Comparisons:");
     for (const comparison of grade.comparisons) {
+      if (comparison.arms === undefined) {
+        lines.push(
+          compactRow("comparison", [
+            comparison.id,
+            comparison.kind,
+            comparison.transition,
+            `baseline=${comparison.baseline.check}:${comparison.baseline.state}`,
+            `candidate=${comparison.candidate.check}:${comparison.candidate.state}`,
+          ]),
+        );
+        continue;
+      }
+      // Each non-reference arm's transition from the reference, then every arm's check and state.
+      const { arms, transitions } = comparisonArms(comparison);
       lines.push(
         compactRow("comparison", [
           comparison.id,
           comparison.kind,
-          comparison.transition,
-          `baseline=${comparison.baseline.check}:${comparison.baseline.state}`,
-          `candidate=${comparison.candidate.check}:${comparison.candidate.state}`,
+          ...Object.entries(transitions).map(([arm, transition]) => `${arm}:${transition}`),
+          ...Object.entries(arms).map(([arm, outcome]) => `${arm}=${outcome?.check}:${outcome?.state}`),
         ]),
       );
     }
@@ -144,12 +193,11 @@ function appendGradeEvidence(lines, grade, group, definition, selectedVariant) {
 }
 
 // Advisory answers stay visually separate from deterministic grades; they are not canonical evidence.
-async function appendAdvisoryReview(lines, groupDirectory, selectedVariant) {
+async function appendAdvisoryReview(lines, groupDirectory, variants) {
   const file = path.join(groupDirectory, "semantic-grade.json");
   if (!(await stat(file).catch(() => null))) return;
   const review = await readJson(file, "advisory review");
   lines.push(`Advisory review [${review.state}] model=${review.model} (not canonical evidence)`);
-  const variants = selectedVariant === null ? VARIANTS : [selectedVariant];
   for (const [index, answer] of (review.answers ?? []).entries()) {
     lines.push(
       compactRow("review", [
@@ -159,12 +207,26 @@ async function appendAdvisoryReview(lines, groupDirectory, selectedVariant) {
       ]),
     );
   }
+  if (review.subject?.familyRelation) lines.push(compactRow("judge", [`family=${review.subject.familyRelation}`]));
+  if (review.swap?.agreement) {
+    const { agreed, questions } = review.swap.agreement;
+    lines.push(compactRow("judge", [`order swap agreed on ${agreed}/${questions}`]));
+    const byArm = Object.entries(review.swap.agreement.arms ?? {});
+    if (byArm.length > 0) {
+      lines.push(
+        compactRow("judge", [
+          `order swap agreed per arm: ${byArm.map(([arm, counts]) => `${arm}=${counts.agreed}/${counts.questions}`).join(" ")}`,
+        ]),
+      );
+    }
+  }
+  for (const gap of review.contractGaps ?? []) lines.push(compactRow("judge", [`contract gap: ${gap}`]));
   for (const error of review.errors ?? []) lines.push(compactRow("error", ["review", error]));
 }
 
 function renderAggregate(resultDirectory, aggregate) {
   const lines = [
-    `Aggregate ${aggregate.id} [${aggregate.state}] trials=${aggregate.trials.length}/${aggregate.requestedTrials}`,
+    `Aggregate ${aggregate.id} [${aggregate.state}] trials=${trialCounts(aggregate)}`,
     `Path: ${escapeCell(resultDirectory)}`,
   ];
   if (aggregate.budget) {
@@ -176,10 +238,36 @@ function renderAggregate(resultDirectory, aggregate) {
       ]),
     );
   }
-  for (const trial of aggregate.trials) lines.push(compactRow("trial", [trial.id, trial.state]));
+  for (const entry of aggregate.held ?? []) {
+    lines.push(compactRow("held", [entry.trial, entry.group, entry.arm, `flags=${entry.flags.join(",")}`]));
+  }
+  for (const trial of aggregate.trials) {
+    lines.push(
+      compactRow("trial", [
+        trial.id,
+        trial.state,
+        ...(trial.cost === undefined ? [] : [`cost=${formatNumber(trial.cost)}`]),
+      ]),
+    );
+  }
+  lines.push(...renderClaim(aggregate));
   for (const group of aggregate.groups) {
-    lines.push("", `Group ${group.id} trials=${group.trials}`);
+    const unreadable = group.unreadableGrades ? ` unreadable-grades=${group.unreadableGrades}` : "";
+    const gradeErrors = group.gradeErrors ? ` grade-errors=${group.gradeErrors}` : "";
+    lines.push("", `Group ${group.id} trials=${group.trials}${unreadable}${gradeErrors}`);
+    for (const [variant, states] of Object.entries(group.runStates ?? {})) {
+      const notComplete = Object.entries(states).filter(([state]) => state !== "complete");
+      if (notComplete.length > 0) {
+        lines.push(
+          compactRow("runs", [variant, ...Object.entries(states).map(([state, count]) => `${state}=${count}`)]),
+        );
+      }
+    }
     for (const comparison of group.comparisons) {
+      if (comparison.arms !== undefined) {
+        appendArmComparison(lines, comparison);
+        continue;
+      }
       lines.push(
         compactRow("comparison", [
           comparison.id,
@@ -189,14 +277,56 @@ function renderAggregate(resultDirectory, aggregate) {
           ...Object.entries(comparison.transitions).map(([transition, count]) => `${transition}=${count}`),
         ]),
       );
+      if (comparison.pairs) {
+        const pairs = Object.entries(comparison.pairs).filter(([, count]) => count > 0);
+        lines.push(
+          compactRow("comparison", [
+            comparison.id,
+            `pairs: ${pairs.map(([pair, count]) => `${pair}=${count}`).join(" ")}`,
+          ]),
+        );
+      }
     }
     for (const check of group.checks) {
-      lines.push(compactRow("check", [check.id, check.variant, check.kind, `pass=${passRate(check)}`]));
+      const counts =
+        typeof check.passAll === "boolean"
+          ? [
+              `pass=${passRate(check)}`,
+              `fail=${check.fail}`,
+              `unavailable=${check.unavailable}`,
+              `pass^${check.trials}=${check.passAll ? "yes" : "no"}`,
+            ]
+          : [`pass=${passRate(check)}`];
+      lines.push(compactRow("check", [check.id, check.variant, check.kind, ...counts]));
     }
+    lines.push(...aggregateMetricsLines(group.metrics));
+    lines.push(...judgedAggregateLines(group.judged));
   }
   return `${lines.join("\n")}\n`;
 }
 
+/** An arm-keyed aggregate comparison: every arm's pass rate, then each arm's transitions and pairs against the reference. */
+function appendArmComparison(lines, comparison) {
+  lines.push(
+    compactRow("comparison", [
+      comparison.id,
+      comparison.kind,
+      ...Object.entries(comparison.arms).map(([arm, counts]) => `${arm}=${passRate(counts)}`),
+    ]),
+  );
+  for (const [arm, versus] of Object.entries(comparison.outcomes ?? {})) {
+    const pairs = Object.entries(versus.pairs ?? {}).filter(([, count]) => count > 0);
+    lines.push(
+      compactRow("comparison", [
+        comparison.id,
+        `${arm} against ${comparison.reference}`,
+        ...Object.entries(versus.transitions ?? {}).map(([transition, count]) => `${transition}=${count}`),
+        `pairs: ${pairs.map(([pair, count]) => `${pair}=${count}`).join(" ")}`,
+      ]),
+    );
+  }
+}
+
 function passRate(counts) {
   return `${counts.pass}/${counts.pass + counts.fail + counts.unavailable}`;
 }
@@ -231,6 +361,12 @@ async function appendVariantEvidence(lines, resultDirectory, groupDirectory, gro
   lines.push("", `${title(variant)} [${run.state}]`);
   appendRunEvidence(lines, run);
   const variantDirectory = path.dirname(runFile);
+  // Transcript-scan flags, each with whether a person cleared it and why; an uncleared one holds the run.
+  const cleared = new Map((await readClearances(variantDirectory)).map((entry) => [entry.flag, entry.reason]));
+  for (const flag of run.postRun?.scan?.flags ?? []) {
+    const status = cleared.has(flag.id) ? `cleared: ${cleared.get(flag.id)}` : "not cleared";
+    lines.push(compactRow("  flag", [flag.id, flag.detail, status]));
+  }
   const artifacts = [["run", relativeArtifact(resultDirectory, runFile)]];
   if (typeof run.workspace === "string") {
     artifacts.push(["workspace", relativeArtifact(resultDirectory, run.workspace)]);
@@ -251,9 +387,13 @@ async function appendVariantEvidence(lines, resultDirectory, groupDirectory, gro
   );
 }
 
-export async function resolveResult(target, root) {
+/** A result or aggregate directory: an explicit path, or a stored ID under `results` (default `<root>/.skill-eval/runs`). */
+export async function resolveResult(target, root, results = null) {
   const explicit = path.resolve(root, target);
-  const stored = path.join(root, ".skill-eval", "runs", target);
+  const stored = path.join(
+    results === null ? path.join(root, ".skill-eval", "runs") : path.resolve(root, results),
+    target,
+  );
   const isPath = path.isAbsolute(target) || target.includes("/") || target.includes("\\") || target.startsWith(".");
   const candidates = isPath ? [explicit] : [stored, explicit];
   for (const candidate of candidates) {
@@ -325,6 +465,7 @@ function appendRunEvidence(lines, run) {
     if (typeof run.usage.cost?.total === "number") usage.push(`cost=${formatNumber(run.usage.cost.total)}`);
     lines.push(compactRow("  usage", usage));
   }
+  lines.push(...runMetricsLines(run));
   if (run.process?.assistantError) lines.push(`  assistant-error: ${run.process.assistantError}`);
   if (run.process?.terminationReason) {
     lines.push(`  termination: ${run.process.terminationReason}`);

package/skills/{evaluate-skill → evaluate-behavior}/scripts/pi-command-tool.mjs

@@ -1,6 +1,8 @@
 import { spawn } from "node:child_process";
 
+import { containerExecArgv, dockerClientEnvironment, parseContainerTarget } from "./lib/container.mjs";
 import { COMMAND_TOOL } from "./lib/definitions.mjs";
+import { LOGIN_VARIABLES } from "./lib/hosts/claude-code/isolation.mjs";
 
 const DEFAULT_TIMEOUT_MS = 120 * 1000;
 const OUTPUT_LIMIT_BYTES = 32 * 1024;
@@ -12,8 +14,16 @@ function parseCommands(value) {
   return parsed;
 }
 
+// A declared command runs the workspace's own code, so it gets neither the evaluator's control variables nor a
+// pointer to the login: Pi's agent directory, which holds the copied provider login, and any login variable. On the
+// host this keeps the login out of easy reach, not out of reach (the run's network "host"); in a container the
+// command gets none of this environment at all.
+const WITHHELD = new Set(["PI_CODING_AGENT_DIR", ...LOGIN_VARIABLES]);
+
 function commandEnvironment() {
-  return Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("SKILL_EVAL_")));
+  return Object.fromEntries(
+    Object.entries(process.env).filter(([key]) => !key.startsWith("SKILL_EVAL_") && !WITHHELD.has(key)),
+  );
 }
 
 function captureStream(stream) {
@@ -44,13 +54,16 @@ function signalGroup(child, signal) {
   }
 }
 
-// Runs one author-declared argv in the workspace. A nonzero exit or timeout is ordinary evidence, not a tool error.
-function runCommand(command, cwd, signal) {
+// Runs one author-declared argv in the workspace, or, in a container environment, through `docker exec` in the
+// subject's container, where bash runs too, so both use the image's toolchain. A nonzero exit or timeout is
+// ordinary evidence, not a tool error.
+function runCommand(command, cwd, signal, container) {
   const timeoutMs = command.timeout_ms ?? DEFAULT_TIMEOUT_MS;
+  const argv = container === null ? command.argv : containerExecArgv(container, command.argv);
   return new Promise((resolve, reject) => {
-    const child = spawn(command.argv[0], command.argv.slice(1), {
+    const child = spawn(argv[0], argv.slice(1), {
       cwd,
-      env: commandEnvironment(),
+      env: container === null ? commandEnvironment() : dockerClientEnvironment(),
       shell: false,
       detached: process.platform !== "win32",
       stdio: ["ignore", "pipe", "pipe"],
@@ -94,6 +107,8 @@ export default function registerCommandTool(pi) {
   const workspace = process.env.SKILL_EVAL_WRITABLE_ROOT;
   if (!workspace) throw new Error("SKILL_EVAL_WRITABLE_ROOT is required");
   const byId = new Map(commands.map((command) => [command.id, command]));
+  // Set by the evaluator only for a container environment; the guard routes bash by the same value.
+  const container = parseContainerTarget(process.env.SKILL_EVAL_CONTAINER);
 
   pi.registerTool({
     name: COMMAND_TOOL,
@@ -113,7 +128,7 @@ export default function registerCommandTool(pi) {
     async execute(_toolCallId, input, signal) {
       const command = byId.get(input?.command);
       if (!command) throw new Error(`Command is not declared for this evaluation: ${String(input?.command)}`);
-      const result = await runCommand(command, workspace, signal);
+      const result = await runCommand(command, workspace, signal, container);
       const status = result.timedOut
         ? `timed out after ${result.timeoutMs} ms`
         : `exit code: ${result.exitCode ?? `signal ${result.signal}`}`;

package/skills/evaluate-behavior/scripts/pi-container-route.mjs

@@ -0,0 +1,23 @@
+// Pi's last extension in a subject with a container environment: it moves each shell tool's command (bash, and
+// Freeflow's bash_background and monitor) into the subject's container. The guard (pi-guard.mjs, loaded first) has
+// already decided the call; this runs after every other extension's tool_call handler, so those handlers see the
+// command the model wrote, and only then does it become the docker client command Pi's shell runs. Pi runs every
+// handler in load order and stops only on a refusal, so a call that runs has passed through here. The guard refuses
+// container shell calls when this extension is not loaded.
+
+import { CONTAINER_ROUTE, decideToolCall, guardState, SHELL_TOOLS } from "./guard-rules.mjs";
+
+export default function registerSkillEvalContainerRoute(pi) {
+  const state = guardState(process.env);
+  globalThis[CONTAINER_ROUTE] = true;
+  pi.on("tool_call", (event, ctx) => {
+    if (!SHELL_TOOLS.has(event.toolName)) return undefined;
+    const decision = decideToolCall(state, event.toolName, event.input, ctx.cwd, process.env);
+    if (decision?.block) return decision;
+    if (decision?.command === undefined) {
+      return { block: true, reason: `${event.toolName} has no container to run in for this evaluation` };
+    }
+    event.input.command = decision.command;
+    return undefined;
+  });
+}

package/skills/evaluate-behavior/scripts/pi-guard.mjs

@@ -0,0 +1,77 @@
+import {
+  CONTAINER_ROUTE,
+  decideToolCall,
+  escapeXml,
+  guardState,
+  loadDeclaredContext,
+  renderDeclaredContext,
+} from "./guard-rules.mjs";
+
+function buildSubjectSystemPrompt(options, declaredContext) {
+  const tools = options.selectedTools ?? [];
+  const toolLines = tools.map((tool) => `- ${tool}: ${options.toolSnippets?.[tool] ?? "Declared evaluation tool"}`);
+  const lines = [
+    "You are an expert coding assistant operating inside pi, a coding agent harness.",
+    "Help the user with the task using only the tools and resources listed below.",
+    "",
+    "Available tools:",
+    ...(toolLines.length > 0 ? toolLines : ["(none)"]),
+    "",
+    "Guidelines:",
+    "- Use read to load a skill's SKILL.md when the task matches its description.",
+    "- Resolve relative skill resources from the skill directory.",
+    "- Be concise in your response.",
+  ];
+  const skills = (options.skills ?? []).filter((skill) => !skill.disableModelInvocation);
+  if (skills.length > 0) {
+    lines.push("", "<available_skills>");
+    for (const skill of skills) {
+      lines.push(
+        "  <skill>",
+        `    <name>${escapeXml(skill.name)}</name>`,
+        `    <description>${escapeXml(skill.description)}</description>`,
+        `    <location>${escapeXml(skill.filePath)}</location>`,
+        "  </skill>",
+      );
+    }
+    lines.push("</available_skills>");
+  }
+  if (declaredContext.length > 0) lines.push("", renderDeclaredContext(declaredContext));
+  lines.push(`Current working directory: ${options.cwd}`);
+  return lines.join("\n");
+}
+
+// "isolated" replaces the host prompt with an evaluator-owned one; "host" keeps the host's own prompt
+// (and anything declared extensions compose onto it) so evidence reflects the production prompt.
+function beforeAgentStartHandler(promptMode, declaredContext) {
+  if (promptMode === "isolated") {
+    return (event) => ({ systemPrompt: buildSubjectSystemPrompt(event.systemPromptOptions, declaredContext) });
+  }
+  return (event) =>
+    declaredContext.length === 0
+      ? undefined
+      : { systemPrompt: `${event.systemPrompt}\n\n${renderDeclaredContext(declaredContext)}` };
+}
+
+export default function registerSkillEvalGuard(pi) {
+  const state = guardState(process.env);
+  const declaredContext = loadDeclaredContext(process.env.SKILL_EVAL_CONTEXT_MANIFEST);
+  const promptMode = process.env.SKILL_EVAL_PROMPT_MODE || "isolated";
+  if (promptMode !== "isolated" && promptMode !== "host") {
+    throw new Error(`SKILL_EVAL_PROMPT_MODE must be isolated or host: ${promptMode}`);
+  }
+
+  pi.on("before_agent_start", beforeAgentStartHandler(promptMode, declaredContext));
+
+  pi.on("tool_call", (event, ctx) => {
+    const decision = decideToolCall(state, event.toolName, event.input, ctx.cwd, process.env);
+    if (decision?.command === undefined) return decision;
+    // A shell tool's command runs in the subject's container. The route extension, loaded after every other one,
+    // rewrites it there, so extensions that read the command (Freeflow refuses a trailing `&` in bash) see the one
+    // the model wrote. Without the route nothing would move it into the container, so the call is refused.
+    if (globalThis[CONTAINER_ROUTE] !== true) {
+      return { block: true, reason: "the evaluator's container route is not loaded, so this command cannot run" };
+    }
+    return undefined;
+  });
+}

package/skills/evaluate-behavior/scripts/pi-login-refresh.mjs

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+
+// Asks the installed Pi to make one provider's stored OAuth login valid for at least the given minutes,
+// refreshing it if needed. Pi does the refresh itself, under its own lock on auth.json, exactly as it does in
+// ordinary use. Usage: pi-login-refresh.mjs <auth.json path> <provider> <minutes>
+// Prints the login's new expiry as epoch milliseconds.
+
+import { execFileSync } from "node:child_process";
+import { realpathSync } from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+const [authPath, provider, minutes] = process.argv.slice(2);
+if (!authPath || !provider || !/^\d+$/.test(minutes ?? "")) {
+  process.stderr.write("usage: pi-login-refresh.mjs <auth.json path> <provider> <minutes>\n");
+  process.exit(2);
+}
+
+// The installed Pi's package root, found from the `pi` command so the refresh uses the same Pi the runs use.
+const binary = realpathSync(execFileSync("which", ["pi"], { encoding: "utf8" }).trim());
+let packageRoot = path.dirname(binary);
+while (path.basename(packageRoot) !== "pi-coding-agent" && packageRoot !== path.dirname(packageRoot)) {
+  packageRoot = path.dirname(packageRoot);
+}
+const { ModelRuntime } = await import(pathToFileURL(path.join(packageRoot, "dist", "index.js")).href);
+const runtime = await ModelRuntime.create({ authPath });
+await runtime.getAuth(provider, { minOAuthValidityMs: Number(minutes) * 60_000 });
+
+const { readFileSync } = await import("node:fs");
+const stored = JSON.parse(readFileSync(authPath, "utf8"))[provider];
+process.stdout.write(`${stored?.expires ?? ""}\n`);