@hassangameryt/freeflow 0.8.0 → 0.9.0

package/skills/evaluate-behavior/scripts/lib/hosts/claude-code/skills.mjs

@@ -0,0 +1,81 @@
+// Declared skills on Claude Code. Its own skill loader would also find bundled and user skills, so the evaluator
+// keeps it off and delivers declared skills the way Pi's evaluated subjects see them: a listing of each skill's
+// name, description and SKILL.md location in the system prompt, and, for a body run's target, Pi's exact skill
+// command expansion as the first prompt.
+
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+
+import { escapeXml } from "../../../guard-rules.mjs";
+
+const FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/;
+
+function scalar(value) {
+  const trimmed = value.trim();
+  if (trimmed.startsWith('"')) return JSON.parse(trimmed);
+  if (trimmed.startsWith("'") && trimmed.endsWith("'")) return trimmed.slice(1, -1).replaceAll("''", "'");
+  return trimmed;
+}
+
+/**
+ * A declared skill's SKILL.md: its frontmatter `name`, `description` and `disable-model-invocation`, and its body.
+ * Only one-line scalars are read; a skill whose name or description this cannot read is refused, never guessed.
+ *
+ * @param {string} directory the materialized skill directory
+ */
+export async function readSkill(directory) {
+  const filePath = path.join(directory, "SKILL.md");
+  const contents = await readFile(filePath, "utf8");
+  const match = FRONTMATTER.exec(contents);
+  if (match === null) throw new Error(`skill has no frontmatter: ${filePath}`);
+  const fields = {};
+  for (const line of match[1].split(/\r?\n/)) {
+    const field = /^([A-Za-z][\w-]*):(.*)$/.exec(line);
+    if (field !== null) fields[field[1]] = field[2];
+  }
+  const read = (key) => {
+    if (fields[key] === undefined) return undefined;
+    try {
+      return scalar(fields[key]);
+    } catch {
+      return undefined;
+    }
+  };
+  const name = read("name");
+  const description = read("description");
+  if (!name || !description || /^[>|]/.test(name) || /^[>|]/.test(description)) {
+    throw new Error(`skill needs a one-line name and description the Claude Code adapter can read: ${filePath}`);
+  }
+  return {
+    name,
+    description,
+    filePath,
+    baseDir: directory,
+    disableModelInvocation: read("disable-model-invocation") === "true",
+    body: contents.slice(match[0].length).trim(),
+  };
+}
+
+/** The `<available_skills>` listing for the system prompt, as Pi writes it; empty when no skill may be invoked. */
+export function skillListing(skills) {
+  const listed = skills.filter((skill) => !skill.disableModelInvocation);
+  if (listed.length === 0) return "";
+  const lines = ["<available_skills>"];
+  for (const skill of listed) {
+    lines.push(
+      "  <skill>",
+      `    <name>${escapeXml(skill.name)}</name>`,
+      `    <description>${escapeXml(skill.description)}</description>`,
+      `    <location>${escapeXml(skill.filePath)}</location>`,
+      "  </skill>",
+    );
+  }
+  lines.push("</available_skills>");
+  return lines.join("\n");
+}
+
+/** The prompt Pi delivers for `/skill:<name> <prompt>`, so an explicit invocation reads the same on both hosts. */
+export function skillCommandPrompt(skill, prompt) {
+  const block = `<skill name="${skill.name}" location="${skill.filePath}">\nReferences are relative to ${skill.baseDir}.\n\n${skill.body}\n</skill>`;
+  return prompt ? `${block}\n\n${prompt}` : block;
+}

package/skills/evaluate-behavior/scripts/lib/hosts/index.mjs

@@ -0,0 +1,31 @@
+// Evaluation hosts. A host is the agent program that runs a subject's model and tools: Pi or Claude Code. The evaluator's
+// core is host-neutral; each host's adapter implements the same interface:
+//
+// - name, label, command; version(environment): the installed host's version.
+// - resolveExtensions(declared, { deferSources }): declared host extensions, found among the installed ones.
+// - refreshLogins(providers): make every login a batch uses valid before it starts.
+// - effectiveSettings(declared): the evaluator's host defaults with declared settings winning.
+// - prepareIsolation(directory, model, { settings }) -> { environment, evidence, loginDirectory }: a private home,
+//   temp and host configuration for one process, with only the declared provider's login, and the directory that
+//   login lives in (the shell probes plant a marker there); removeCredentials(directory) after.
+// - prepareLaunch(context) -> { args, environment }: one subject's arguments and the host's own variables.
+// - runOneShot(options), runSession(options): run a subject and return the normalized observation (see subject.mjs).
+// - modelCall(directory, prompt, options): one tool-less call for a judge, reviewer or simulated user.
+// - probeShell(options): run the shell tier's confinement probes through the host (see shell.mjs).
+//
+// Names a definition may use are in names.mjs, which definitions.mjs reads without loading any adapter.
+
+import { claudeCodeHost } from "./claude-code/index.mjs";
+import { piHost } from "./pi/index.mjs";
+
+const ADAPTERS = new Map([
+  [piHost.name, piHost],
+  [claudeCodeHost.name, claudeCodeHost],
+]);
+
+/** The adapter for a host name; an unknown name is an error, never a fallback to Pi. */
+export function hostAdapter(name) {
+  const adapter = ADAPTERS.get(name);
+  if (adapter === undefined) throw new Error(`unknown evaluation host: ${name}`);
+  return adapter;
+}

package/skills/evaluate-behavior/scripts/lib/hosts/names.mjs

@@ -0,0 +1,3 @@
+// The hosts a definition may name in `host.name`. Each has an adapter in hosts/; definitions.mjs reads this list
+// without loading the adapters.
+export const EVALUATION_HOSTS = new Set(["pi", "claude-code"]);

package/skills/evaluate-behavior/scripts/lib/hosts/pi/confinement.mjs

@@ -0,0 +1,173 @@
+// Pi's part of the shell tier's confinement probes (shell.mjs): the arm's own Pi, with its extensions and the
+// evaluator's guard, runs every probe as a bash call from a local scripted model, and reports what each returned.
+
+import { readFile, writeFile } from "node:fs/promises";
+import http from "node:http";
+import path from "node:path";
+
+import { baseProcessEnvironment } from "../../process.mjs";
+import { containerRouteExtensionPath, guardExtensionPath, runPiProcess } from "./process.mjs";
+
+const PROBE_PROVIDER = "skill-eval-probe";
+const PROBE_MODEL = "confinement";
+
+/**
+ * Runs the probes through Pi in JSON mode. `guard` is the guard's environment for the probe workspace; `isolation`
+ * is the probe's prepared isolation, whose agent directory gets the scripted model.
+ *
+ * @returns {Promise<{observation: any, launchError: string | null, results: Map<string, {ran: boolean, isError: boolean, output: string}>}>}
+ */
+export async function probeShell({ probes, workspace, isolation, extensions, guard, eventsFile, stderrFile, signal }) {
+  const model = await startModelStub(probes);
+  try {
+    const agent = path.join(isolation.evidence.root, "agent");
+    await writeFile(
+      path.join(agent, "models.json"),
+      `${JSON.stringify({ providers: { [PROBE_PROVIDER]: { baseUrl: model.baseUrl, api: "openai-completions", apiKey: "probe", models: [{ id: PROBE_MODEL }] } } }, null, 2)}\n`,
+    );
+    const environment = {
+      ...baseProcessEnvironment(),
+      PI_SKIP_VERSION_CHECK: "1",
+      PI_TELEMETRY: "0",
+      ...guard,
+      ...isolation.environment,
+    };
+    const args = ["--mode", "json", "-p", "--no-session", "--no-extensions", "--extension", guardExtensionPath()];
+    for (const extension of extensions) args.push("--extension", extension.path);
+    // A container shell's commands reach the container through the route extension, last, as in a subject.
+    if (guard.SKILL_EVAL_CONTAINER) args.push("--extension", containerRouteExtensionPath());
+    args.push(
+      "--no-skills",
+      "--no-prompt-templates",
+      "--no-themes",
+      "--no-context-files",
+      "--no-approve",
+      "--tools",
+      "bash",
+      "--model",
+      `${PROBE_PROVIDER}/${PROBE_MODEL}`,
+      "Run each confinement probe with bash, exactly as given.",
+    );
+    let observation = null;
+    let launchError = null;
+    try {
+      observation = await runPiProcess({
+        command: "pi",
+        args,
+        cwd: workspace,
+        eventsFile,
+        stderrFile,
+        signal,
+        environment,
+      });
+    } catch (error) {
+      launchError = error instanceof Error ? error.message : String(error);
+    }
+    // Whatever Pi reported before a launch error still counts.
+    return { observation, launchError, results: await toolResults(eventsFile) };
+  } finally {
+    await model.close();
+  }
+}
+
+/** Each probe's tool result from Pi's JSON events, by tool call ID. */
+async function toolResults(eventsFile) {
+  const results = new Map();
+  const contents = await readFile(eventsFile, "utf8").catch(() => "");
+  for (const line of contents.split("\n")) {
+    if (line.trim() === "") continue;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (event.type !== "tool_execution_end" || typeof event.toolCallId !== "string") continue;
+    const content = Array.isArray(event.result?.content) ? event.result.content : [];
+    results.set(event.toolCallId, {
+      ran: true,
+      isError: event.isError === true,
+      output: content
+        .filter((part) => part?.type === "text")
+        .map((part) => part.text)
+        .join(""),
+    });
+  }
+  return results;
+}
+
+/**
+ * A local OpenAI-compatible chat-completions endpoint that answers the first request with every probe as a bash
+ * tool call, and any request carrying tool results with a short final answer. No model is called.
+ */
+function startModelStub(probes) {
+  return new Promise((resolve, reject) => {
+    const server = http.createServer((request, response) => {
+      let body = "";
+      request.setEncoding("utf8");
+      request.on("data", (chunk) => {
+        body += chunk;
+      });
+      request.on("end", () => {
+        let parsed = {};
+        try {
+          parsed = JSON.parse(body || "{}");
+        } catch {
+          // Answered as a first request.
+        }
+        const followUp = Array.isArray(parsed.messages) && parsed.messages.some((message) => message?.role === "tool");
+        const message = followUp
+          ? { role: "assistant", content: "Probes finished." }
+          : {
+              role: "assistant",
+              content: null,
+              tool_calls: probes.map((probe, index) => ({
+                index,
+                id: probe.id,
+                type: "function",
+                function: { name: "bash", arguments: JSON.stringify({ command: probe.command }) },
+              })),
+            };
+        const finish = followUp ? "stop" : "tool_calls";
+        const base = { id: "probe", created: 0, model: parsed.model ?? PROBE_MODEL };
+        const usage = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 };
+        if (parsed.stream !== true) {
+          response.writeHead(200, { "content-type": "application/json" });
+          response.end(
+            JSON.stringify({
+              ...base,
+              object: "chat.completion",
+              choices: [
+                {
+                  index: 0,
+                  message: { ...message, tool_calls: message.tool_calls?.map(({ index: _index, ...call }) => call) },
+                  finish_reason: finish,
+                },
+              ],
+              usage,
+            }),
+          );
+          return;
+        }
+        response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" });
+        const send = (chunk) =>
+          response.write(`data: ${JSON.stringify({ ...base, object: "chat.completion.chunk", ...chunk })}\n\n`);
+        send({ choices: [{ index: 0, delta: message, finish_reason: null }] });
+        send({ choices: [{ index: 0, delta: {}, finish_reason: finish }] });
+        send({ choices: [], usage });
+        response.end("data: [DONE]\n\n");
+      });
+    });
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", () => {
+      resolve({
+        baseUrl: `http://127.0.0.1:${server.address().port}/v1`,
+        close: () =>
+          new Promise((done) => {
+            server.closeAllConnections?.();
+            server.close(() => done());
+          }),
+      });
+    });
+  });
+}

package/skills/evaluate-behavior/scripts/lib/hosts/pi/host.mjs

@@ -0,0 +1,148 @@
+// Pi-specific knowledge the evaluator needs outside a subject process: where the user's Pi agent directory
+// is, how installed Pi packages are found, and how a stored OAuth login is kept fresh. Kept in one module so a
+// later host adapter has one place to grow from (D-007).
+
+import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdir, readFile, rm, stat } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+
+const run = promisify(execFile);
+const REFRESH_HELPER = fileURLToPath(new URL("../../../pi-login-refresh.mjs", import.meta.url));
+const LOCK_STALE_MS = 2 * 60 * 1000;
+const LOCK_WAIT_MS = 90 * 1000;
+
+/**
+ * Pi settings the evaluator uses unless a configuration's host.settings declares otherwise: automatic retry and
+ * compaction stay off, so a provider error or a compaction never happens silently inside a measured run.
+ */
+export const EVALUATOR_PI_SETTINGS = Object.freeze({ retry: { enabled: false }, compaction: { enabled: false } });
+
+/** The settings a subject actually runs with: the evaluator's defaults, with every declared setting winning. */
+export function effectivePiSettings(declared = {}) {
+  const merge = (base, overlay) => {
+    const result = structuredClone(base);
+    for (const [key, value] of Object.entries(overlay)) {
+      const isObject = (entry) => entry !== null && typeof entry === "object" && !Array.isArray(entry);
+      result[key] = isObject(result[key]) && isObject(value) ? merge(result[key], value) : structuredClone(value);
+    }
+    return result;
+  };
+  return merge(EVALUATOR_PI_SETTINGS, declared);
+}
+
+/** The user's Pi agent directory: an explicit override for the evaluator, then Pi's own variable, then its default. */
+export function piAgentDirectory() {
+  return (
+    process.env.SKILL_EVAL_CREDENTIALS_DIR ||
+    process.env.PI_CODING_AGENT_DIR ||
+    path.join(process.env.HOME || os.homedir(), ".pi", "agent")
+  );
+}
+
+export async function readJsonObject(file) {
+  try {
+    const value = JSON.parse(await readFile(file, "utf8"));
+    return value && typeof value === "object" && !Array.isArray(value) ? value : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Finds each declared package extension among the user's installed Pi packages (`<agent dir>/npm/node_modules`),
+ * by exact name and version. A missing package or another version refuses the run. An extension built from source
+ * (`source` or `path`, see build-source.mjs) is built per group into its resources, so with `deferSources` it stays
+ * in its declared position as `{ pending: <declared entry> }`; without it, it is refused.
+ *
+ * @param {Array<any>} declared
+ * @param {{deferSources?: boolean}} [options]
+ */
+export async function resolveHostExtensions(declared, { deferSources = false } = {}) {
+  const root = path.join(piAgentDirectory(), "npm", "node_modules");
+  const resolved = [];
+  for (const extension of declared) {
+    if (!Object.hasOwn(extension, "package")) {
+      if (!deferSources) throw new Error("host.extensions built from source are supported only in arm configurations");
+      resolved.push({ pending: extension });
+      continue;
+    }
+    const directory = path.join(root, ...extension.package.split("/"));
+    const manifest = await readJsonObject(path.join(directory, "package.json"));
+    if (manifest === null) {
+      throw new Error(`host.extensions: ${extension.package} is not installed in Pi (looked in ${root})`);
+    }
+    if (manifest.version !== extension.version) {
+      throw new Error(
+        `host.extensions: ${extension.package} is ${manifest.version} in Pi, but the definition declares ${extension.version}`,
+      );
+    }
+    resolved.push({ path: directory, package: `${extension.package}@${extension.version}` });
+  }
+  return resolved;
+}
+
+/** Minutes left on a provider's stored OAuth login, or null when the login is not OAuth or not stored. */
+export async function loginMinutesLeft(provider) {
+  const login = (await readJsonObject(path.join(piAgentDirectory(), "auth.json")))?.[provider];
+  if (!login || login.type !== "oauth" || typeof login.expires !== "number") return null;
+  return Math.floor((login.expires - Date.now()) / 60000);
+}
+
+/**
+ * Makes a provider's stored OAuth login valid for at least `target` minutes when it has fewer than `threshold`
+ * left, by asking the installed Pi to refresh it in the user's own agent directory. One evaluator process
+ * refreshes at a time; the others wait and then reread. API-key and absent logins are left alone.
+ *
+ * @returns {Promise<number | null>} minutes left afterwards, or null when the login is not OAuth
+ */
+export async function ensureLogin(provider, { threshold, target }) {
+  const before = await loginMinutesLeft(provider);
+  if (before === null || before >= threshold) return before;
+  const authPath = path.join(piAgentDirectory(), "auth.json");
+  await withLock(authPath, async () => {
+    const current = await loginMinutesLeft(provider);
+    if (current !== null && current >= threshold) return;
+    const helper = process.env.SKILL_EVAL_LOGIN_REFRESH || REFRESH_HELPER;
+    try {
+      await run(process.execPath, [helper, authPath, provider, String(target)], { timeout: 60000 });
+    } catch (error) {
+      const detail = error?.stderr?.trim() || (error instanceof Error ? error.message : String(error));
+      throw new Error(`refreshing the ${provider} login failed (log in again in Pi): ${detail}`);
+    }
+  });
+  const after = await loginMinutesLeft(provider);
+  if (after === null || after < threshold) {
+    throw new Error(`the ${provider} login has ${after ?? 0} minutes left after a refresh; log in again in Pi`);
+  }
+  return after;
+}
+
+async function withLock(authPath, work) {
+  const lock = path.join(
+    os.tmpdir(),
+    `skill-eval-login-${createHash("sha256").update(authPath).digest("hex").slice(0, 16)}.lock`,
+  );
+  await mkdir(path.dirname(lock), { recursive: true });
+  const deadline = Date.now() + LOCK_WAIT_MS;
+  for (;;) {
+    try {
+      await mkdir(lock);
+      break;
+    } catch (error) {
+      if (error?.code !== "EEXIST") throw error;
+      const info = await stat(lock).catch(() => null);
+      if (info && Date.now() - info.mtimeMs > LOCK_STALE_MS) await rm(lock, { recursive: true, force: true });
+      if (Date.now() > deadline) throw new Error(`timed out waiting for another login refresh (${lock})`);
+      await new Promise((resolve) => setTimeout(resolve, 200));
+    }
+  }
+  try {
+    return await work();
+  } finally {
+    await rm(lock, { recursive: true, force: true });
+  }
+}

package/skills/evaluate-behavior/scripts/lib/hosts/pi/index.mjs

@@ -0,0 +1,70 @@
+// The Pi host adapter: everything the evaluator does that depends on Pi, behind the interface in ../index.mjs.
+
+import { installedHostVersion } from "../../identity.mjs";
+import { baseProcessEnvironment } from "../../process.mjs";
+import { probeShell } from "./confinement.mjs";
+import { effectivePiSettings, ensureLogin, resolveHostExtensions } from "./host.mjs";
+import { prepareIsolation, removeCopiedCredentials, TARGET_MINUTES } from "./isolation.mjs";
+import { launchModel } from "./model-call.mjs";
+import { piArguments, runPiProcess } from "./process.mjs";
+import { runRpcBodySession } from "./rpc.mjs";
+
+export const piHost = Object.freeze({
+  name: "pi",
+  label: "Pi",
+  command: "pi",
+
+  /** The installed Pi's version, as `pi --version` prints it. */
+  version: (environment = baseProcessEnvironment()) => installedHostVersion("pi", environment),
+
+  resolveExtensions: resolveHostExtensions,
+
+  /** Makes each provider's stored OAuth login valid for the target window, once, before a batch. */
+  async refreshLogins(providers) {
+    for (const provider of providers)
+      await ensureLogin(provider, { threshold: TARGET_MINUTES, target: TARGET_MINUTES });
+  },
+
+  effectiveSettings: effectivePiSettings,
+  prepareIsolation,
+  removeCredentials: removeCopiedCredentials,
+
+  /** Pi's arguments and its own environment switches for one subject. */
+  async prepareLaunch({
+    group,
+    variant,
+    environment,
+    runtime,
+    mode,
+    observeContext,
+    sessionDirectory,
+    model,
+    freeflowEntry,
+    hostExtensions,
+  }) {
+    return {
+      args: piArguments(
+        group,
+        environment,
+        runtime,
+        mode,
+        observeContext,
+        sessionDirectory,
+        model,
+        freeflowEntry,
+        hostExtensions,
+        group.variants[variant]?.shell?.container !== undefined,
+      ),
+      environment: { PI_SKIP_VERSION_CHECK: "1", PI_TELEMETRY: "0" },
+    };
+  },
+
+  /** One prompt in JSON mode; the prompt is the last argument. */
+  runOneShot: runPiProcess,
+
+  /** A persistent RPC session over the declared prompts, then any `nextPrompt` replies. */
+  runSession: runRpcBodySession,
+
+  modelCall: launchModel,
+  probeShell,
+});

package/skills/evaluate-behavior/scripts/lib/hosts/pi/isolation.mjs

@@ -0,0 +1,81 @@
+import { mkdir, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+import { ensureLogin, piAgentDirectory, readJsonObject } from "./host.mjs";
+
+// A copied OAuth login must outlast the process using it: a refresh inside the copy would rotate the token
+// there and leave the user's own login holding a revoked one. So the login is refreshed centrally, in the
+// user's agent directory, whenever it has less than SUBJECT_MINUTES left, to at least TARGET_MINUTES.
+export const SUBJECT_MINUTES = 30;
+export const TARGET_MINUTES = 120;
+
+/**
+ * Creates a private home, temp directory and Pi agent directory for one host process, and the environment
+ * that points the process at them. Only the declared provider's login is copied in, never another's, after
+ * a central refresh if it was close to expiring; the declared Pi settings become the agent directory's
+ * settings.json. The parent's HOME, TMPDIR, XDG directories and Pi settings are not visible.
+ *
+ * @param {string} directory where the isolation root is created
+ * @param {string | null} model the declared `provider/model`, or null
+ * @param {{settings?: Record<string, unknown>}} [options]
+ * @returns {Promise<{environment: Record<string, string>, evidence: any, loginDirectory: string}>}
+ */
+export async function prepareIsolation(directory, model, { settings = {} } = {}) {
+  const root = path.join(directory, "isolation");
+  const home = path.join(root, "home");
+  const agent = path.join(root, "agent");
+  const tmp = path.join(root, "tmp");
+  for (const entry of [home, agent, tmp]) await mkdir(entry, { recursive: true });
+  if (Object.keys(settings).length > 0) {
+    await writeFile(path.join(agent, "settings.json"), `${JSON.stringify(settings, null, 2)}\n`);
+  }
+
+  const provider = typeof model === "string" && model.includes("/") ? model.split("/")[0] : null;
+  const source = piAgentDirectory();
+  const credentials = [];
+  if (provider !== null) {
+    // The caller reports a refresh failure as an invalid run, before any host process starts.
+    await ensureLogin(provider, { threshold: SUBJECT_MINUTES, target: TARGET_MINUTES });
+    const login = (await readJsonObject(path.join(source, "auth.json")))?.[provider];
+    if (login && typeof login === "object") {
+      await writeFile(path.join(agent, "auth.json"), `${JSON.stringify({ [provider]: login })}\n`, { mode: 0o600 });
+      credentials.push({ file: "auth.json", provider, type: typeof login.type === "string" ? login.type : null });
+    }
+    const models = await readJsonObject(path.join(source, "models.json"));
+    const custom = models?.providers?.[provider];
+    if (custom && typeof custom === "object") {
+      await writeFile(path.join(agent, "models.json"), `${JSON.stringify({ providers: { [provider]: custom } })}\n`, {
+        mode: 0o600,
+      });
+      credentials.push({ file: "models.json", provider, type: "provider" });
+    }
+  }
+
+  return {
+    environment: {
+      HOME: home,
+      TMPDIR: tmp,
+      TMP: tmp,
+      TEMP: tmp,
+      XDG_CONFIG_HOME: path.join(home, ".config"),
+      XDG_DATA_HOME: path.join(home, ".local", "share"),
+      XDG_CACHE_HOME: path.join(home, ".cache"),
+      PI_CODING_AGENT_DIR: agent,
+    },
+    evidence: { root, provider, credentials, settings },
+    // Where the host keeps its login: the confinement probe plants a marker here that the shell must not read.
+    loginDirectory: agent,
+  };
+}
+
+/**
+ * Deletes the copied login and model entries once the process has exited, so no copy of a credential
+ * stays in the run's evidence. Safe to call when nothing was copied.
+ *
+ * @param {string} directory the directory passed to prepareIsolation
+ */
+export async function removeCopiedCredentials(directory) {
+  for (const file of ["auth.json", "models.json"]) {
+    await rm(path.join(directory, "isolation", "agent", file), { force: true });
+  }
+}

package/skills/evaluate-behavior/scripts/lib/hosts/pi/model-call.mjs

@@ -0,0 +1,79 @@
+// One model call through Pi that is not a subject: a judge, a reviewer, or a simulated user. It runs without tools,
+// skills, context files or session, in its own isolated home and agent directory, with the provider's login copied in
+// and removed afterward.
+
+import { mkdir, rm } from "node:fs/promises";
+import path from "node:path";
+
+import { writeText } from "../../evidence.mjs";
+import { baseProcessEnvironment } from "../../process.mjs";
+import { effectivePiSettings } from "./host.mjs";
+import { prepareIsolation, removeCopiedCredentials } from "./isolation.mjs";
+import { runPiProcess } from "./process.mjs";
+
+/**
+ * Starts one model call on a prompt in its own directory, which keeps the prompt, events, final response and stderr.
+ *
+ * @param {string} directory replaced on every call
+ * @param {string} prompt
+ * @param {{model: string, thinking: string | null, extensions: Array<{path: string}>, settings: object, signal?: AbortSignal, role?: string}} options
+ * @returns {Promise<{response: string | null, errors: string[], usage: any, isolation: any}>}
+ */
+export async function launchModel(directory, prompt, options) {
+  const { model, thinking, extensions, settings, signal, role = "reviewer" } = options;
+  await rm(directory, { recursive: true, force: true });
+  await mkdir(directory, { recursive: true });
+  await writeText(path.join(directory, "prompt.md"), prompt);
+
+  const args = [
+    "--mode",
+    "json",
+    "-p",
+    "--no-session",
+    "--no-extensions",
+    ...extensions.flatMap((extension) => ["--extension", extension.path]),
+    "--no-skills",
+    "--no-prompt-templates",
+    "--no-themes",
+    "--no-context-files",
+    "--no-approve",
+    "--no-tools",
+    "--model",
+    model,
+  ];
+  if (thinking !== null) args.push("--thinking", thinking);
+  args.push(prompt);
+  let isolation;
+  try {
+    // The evaluator's Pi defaults (retry and compaction off) apply here as for subjects; declared settings win.
+    isolation = await prepareIsolation(directory, model, { settings: effectivePiSettings(settings) });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { response: null, errors: [message], usage: null, isolation: null };
+  }
+  const observation = await runPiProcess({
+    command: "pi",
+    args,
+    cwd: directory,
+    eventsFile: path.join(directory, "events.jsonl"),
+    stderrFile: path.join(directory, "stderr.log"),
+    signal,
+    environment: {
+      ...baseProcessEnvironment(),
+      PWD: directory,
+      PI_SKIP_VERSION_CHECK: "1",
+      PI_TELEMETRY: "0",
+      ...isolation.environment,
+    },
+  });
+  await removeCopiedCredentials(directory);
+  await writeText(path.join(directory, "final.md"), `${observation.response}\n`);
+
+  const errors = [];
+  if (observation.exitCode !== 0 || !observation.settled || observation.assistantError !== null) {
+    errors.push(
+      `${role} did not complete: exit=${observation.exitCode} settled=${observation.settled} error=${observation.assistantError}`,
+    );
+  }
+  return { response: observation.response, errors, usage: observation.usage, isolation: isolation.evidence };
+}