@hassangameryt/freeflow 0.8.0 → 0.9.0

package/skills/evaluate-behavior/scripts/lib/hosts/pi/process.mjs

@@ -0,0 +1,241 @@
+// How the evaluator launches Pi itself: the command-line arguments for a subject and the one-shot JSON-mode process
+// that reports Pi's events. RPC sessions are in rpc.mjs.
+
+import { spawn } from "node:child_process";
+import { once } from "node:events";
+import { createWriteStream } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+import {
+  mergeUsage,
+  messageText,
+  NO_PROGRESS_MS,
+  pipeStream,
+  processGroupExists,
+  signalProcessTree,
+  TERMINATION_GRACE_MS,
+  waitForForcedKill,
+  writeStream,
+} from "../../process.mjs";
+
+const guardExtension = fileURLToPath(new URL("../../../pi-guard.mjs", import.meta.url));
+
+/** The evaluator's guard extension, which every subject process (and a shell's confinement probe) loads first. */
+export function guardExtensionPath() {
+  return guardExtension;
+}
+const commandToolExtension = fileURLToPath(new URL("../../../pi-command-tool.mjs", import.meta.url));
+const containerRouteExtension = fileURLToPath(new URL("../../../pi-container-route.mjs", import.meta.url));
+
+/**
+ * The extension that moves shell commands into a subject's container. It loads after every other extension, so
+ * theirs see each command as the model wrote it; the guard refuses container shell calls without it.
+ */
+export function containerRouteExtensionPath() {
+  return containerRouteExtension;
+}
+const observerExtension = fileURLToPath(new URL("../../../pi-observer.mjs", import.meta.url));
+
+export function piArguments(
+  group,
+  environment,
+  runtime,
+  mode,
+  observeContext,
+  sessionDirectory,
+  model,
+  freeflowEntry,
+  hostExtensions = [],
+  containerRoute = false,
+) {
+  const args = ["--mode", mode];
+  if (mode === "json") args.push("-p");
+  if (runtime.session) args.push("--session-dir", sessionDirectory);
+  else args.push("--no-session");
+  args.push("--no-extensions", "--extension", guardExtension);
+  // Provider extensions (such as a subscription's auth extension) load right after the guard.
+  for (const extension of hostExtensions) args.push("--extension", extension.path);
+  if (group.commands !== undefined) args.push("--extension", commandToolExtension);
+  if (freeflowEntry !== null) args.push("--extension", freeflowEntry);
+  for (const bundle of runtime.extensions) args.push("--extension", bundle.entry);
+  if (observeContext) args.push("--extension", observerExtension);
+  if (containerRoute) args.push("--extension", containerRouteExtension);
+  args.push("--no-skills", "--no-prompt-templates", "--no-themes", "--no-context-files", "--no-approve");
+  if (group.tools.length === 0) args.push("--no-tools");
+  else args.push("--tools", group.tools.join(","));
+  for (const skill of environment.skills) args.push("--skill", skill.path);
+  if (model?.model) args.push("--model", model.model);
+  if (model?.thinking) args.push("--thinking", model.thinking);
+  if (mode === "json") args.push(group.input.prompt);
+  return args;
+}
+
+export async function runPiProcess({ command, args, cwd, eventsFile, stderrFile, signal, environment }) {
+  const child = spawn(command, args, {
+    cwd,
+    detached: process.platform !== "win32",
+    env: environment,
+    shell: false,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  const events = createWriteStream(eventsFile, { encoding: "utf8" });
+  const stderr = createWriteStream(stderrFile, { encoding: "utf8" });
+  const transcript = [];
+  const calls = new Map();
+  const toolActivity = [];
+  const successfulReads = [];
+  const parseErrors = [];
+  const protocolErrors = [];
+  let settled = false;
+  let response = "";
+  let usage = null;
+  let assistantError = null;
+  let observedModel = null;
+  let buffer = "";
+  let eventLine = 0;
+  const processGroupId = child.pid;
+  let watchdog;
+  let forcedKill;
+  /** @type {Promise<void> | null} */
+  let forcedKillPromise = null;
+  let terminationReason = null;
+
+  const resetWatchdog = () => {
+    clearTimeout(watchdog);
+    watchdog = setTimeout(() => requestTermination("no-progress-watchdog"), NO_PROGRESS_MS);
+    watchdog.unref();
+  };
+  const requestTermination = (reason) => {
+    if (terminationReason !== null) return;
+    terminationReason = reason;
+    signalProcessTree(processGroupId, child, "SIGTERM");
+    if (process.platform !== "win32") {
+      forcedKillPromise = new Promise((resolve) => {
+        forcedKill = setTimeout(() => {
+          signalProcessTree(processGroupId, child, "SIGKILL");
+          resolve();
+        }, TERMINATION_GRACE_MS);
+      });
+    }
+  };
+  const onAbort = () => requestTermination("cancelled");
+  if (signal?.aborted) onAbort();
+  else signal?.addEventListener("abort", onAbort, { once: true });
+  resetWatchdog();
+
+  const stderrTask = pipeStream(child.stderr, stderr, resetWatchdog);
+  const stdoutTask = (async () => {
+    for await (const chunk of child.stdout) {
+      resetWatchdog();
+      buffer += chunk.toString("utf8");
+      while (true) {
+        const newline = buffer.indexOf("\n");
+        if (newline === -1) break;
+        const line = buffer.slice(0, newline);
+        buffer = buffer.slice(newline + 1);
+        await writeStream(events, `${line}\n`);
+        observeLine(line.endsWith("\r") ? line.slice(0, -1) : line);
+      }
+    }
+    if (buffer.length > 0) {
+      await writeStream(events, buffer);
+      observeLine(buffer.endsWith("\r") ? buffer.slice(0, -1) : buffer);
+    }
+    events.end();
+    await once(events, "finish");
+  })();
+
+  function observeLine(line) {
+    eventLine += 1;
+    if (line.trim() === "") return;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      parseErrors.push({ line: eventLine, reason: "invalid-json" });
+      return;
+    }
+    if (event.type === "extension_error") {
+      protocolErrors.push({
+        reason: "extension-error",
+        extensionPath: event.extensionPath ?? null,
+        event: event.event ?? null,
+        error: event.error ?? null,
+      });
+      requestTermination("protocol-error");
+    }
+    if (event.type === "agent_settled") settled = true;
+    if (event.type === "tool_execution_start") {
+      // Same shape as RPC turns record, so graders and the advisory reviewer see one-shot tool use too.
+      const activity = {
+        toolCallId: event.toolCallId ?? null,
+        toolName: event.toolName ?? null,
+        args: event.args ?? null,
+        completed: false,
+        isError: null,
+      };
+      toolActivity.push(activity);
+      calls.set(event.toolCallId, activity);
+    }
+    if (event.type === "tool_execution_end") {
+      const call = calls.get(event.toolCallId);
+      if (call) {
+        call.completed = true;
+        call.isError = event.isError === true;
+      }
+      if (!event.isError && call?.toolName === "read" && typeof call.args?.path === "string") {
+        successfulReads.push(call.args.path);
+      }
+    }
+    if (event.type === "message_end" && event.message) {
+      transcript.push(event.message);
+      if (event.message.role === "assistant") {
+        response = messageText(event.message);
+        usage = mergeUsage(usage, event.message.usage);
+        observedModel = {
+          provider: event.message.provider ?? null,
+          model: event.message.model ?? null,
+        };
+        assistantError =
+          event.message.stopReason === "error" || event.message.stopReason === "aborted"
+            ? (event.message.errorMessage ?? `assistant stopped with ${event.message.stopReason}`)
+            : null;
+      }
+    }
+  }
+
+  let exitCode = null;
+  let exitSignal = null;
+  try {
+    const [code, closedSignal] = await once(child, "close");
+    exitCode = code;
+    exitSignal = closedSignal;
+    await Promise.all([stdoutTask, stderrTask]);
+  } catch (error) {
+    requestTermination("process-error");
+    throw error;
+  } finally {
+    clearTimeout(watchdog);
+    if (forcedKillPromise !== null) {
+      if (processGroupExists(processGroupId)) await waitForForcedKill(forcedKillPromise);
+      else clearTimeout(forcedKill);
+    }
+    signal?.removeEventListener("abort", onAbort);
+  }
+
+  return {
+    exitCode,
+    signal: exitSignal,
+    settled,
+    assistantError,
+    model: observedModel,
+    terminationReason,
+    parseErrors,
+    protocolErrors,
+    successfulReads,
+    toolActivity,
+    transcript,
+    response,
+    usage,
+  };
+}

package/skills/{evaluate-skill/scripts/lib → evaluate-behavior/scripts/lib/hosts/pi}/rpc.mjs

@@ -15,7 +15,7 @@ import {
   waitForForcedKill,
   waitForProcessTreeExit,
   writeStream,
-} from "./process.mjs";
+} from "../../process.mjs";
 
 const MAX_INCOMPLETE_FRAME_BYTES = 16 * 1024 * 1024;
 
@@ -28,7 +28,25 @@ export async function runRpcBodySession(options) {
 }
 
 class RpcSession {
-  constructor({ command = "pi", args, cwd, eventsFile, stderrFile, signal, environment, afterTurn = null }) {
+  constructor({
+    command = "pi",
+    args,
+    cwd,
+    eventsFile,
+    stderrFile,
+    signal,
+    environment,
+    afterTurn = null,
+    nextPrompt = null,
+    settings = {},
+    resumeBefore = [],
+  }) {
+    this.settings = settings;
+    // Turn numbers before which the session is closed and resumed from its own file, as `pi --resume` would.
+    this.resumeBefore = new Set(resumeBefore);
+    // After the last declared prompt settles, `nextPrompt(turns)` may return one more prompt (a simulated user's
+    // reply), or null to end the session.
+    this.nextPrompt = nextPrompt;
     this.child = spawn(command, args, {
       cwd,
       detached: process.platform !== "win32",
@@ -47,6 +65,10 @@ class RpcSession {
     this.parseErrors = [];
     this.protocolErrors = [];
     this.activeTurn = null;
+    // A run Pi starts on its own while no prompt is in flight: an extension's message that starts a turn, such as
+    // Freeflow's notice that a background command exited or a monitor printed a line. Its activity joins the latest
+    // turn when it settles; the next prompt waits for it.
+    this.unprompted = null;
     this.decoder = new StringDecoder("utf8");
     this.buffer = "";
     this.incompleteFrameBytes = 0;
@@ -77,7 +99,7 @@ class RpcSession {
   }
 
   async run(prompts, targetPath = null) {
-    let deliveredPrompts = prompts;
+    let deliveredPrompts = [...prompts];
     /** @type {any} */
     let delivery = { kind: "unavailable", turn: 1, targetPath };
     try {
@@ -88,12 +110,23 @@ class RpcSession {
         delivery = { kind: "explicit-skill-command", turn: 1, targetPath, command };
       }
       for (const [index, prompt] of deliveredPrompts.entries()) {
+        await this.unpromptedSettled();
         if (this.signal?.aborted || this.terminationReason !== null) break;
         if (delivery.kind === "unavailable") delivery = { kind: "natural-prompt", turn: 1, targetPath: null };
+        const resumed = this.resumeBefore.has(index + 1) ? await this.resume() : null;
         const turn = await this.runTurn(index + 1, prompt);
+        if (resumed !== null) turn.resumed = resumed;
         this.turns.push(turn);
         if (this.afterTurn !== null) turn.workspace = await this.afterTurn(turn.turn);
         if (turn.assistantError !== null) break;
+        if (this.nextPrompt !== null && index === deliveredPrompts.length - 1 && this.terminationReason === null) {
+          // A run a notice started right after the turn is part of what the user sees before replying.
+          await this.unpromptedSettled();
+          const next = await this.nextPrompt(this.turns);
+          // The simulated user's time is not the subject's lack of progress.
+          this.resetWatchdog();
+          if (typeof next === "string") deliveredPrompts.push(next);
+        }
       }
     } catch (error) {
       this.operationError = error;
@@ -104,6 +137,7 @@ class RpcSession {
         });
         this.requestTermination("protocol-error");
       }
+      await this.unpromptedSettled();
     } finally {
       if (!this.child.stdin.destroyed) this.child.stdin.end();
     }
@@ -114,9 +148,25 @@ class RpcSession {
     return { ...this.observation(deliveredPrompts, exitCode, exitSignal), delivery };
   }
 
+  // The subject's effective settings decide retry and compaction; the evaluator's defaults keep both off.
   async configure() {
-    await this.sendCommand({ type: "set_auto_retry", enabled: false });
-    await this.sendCommand({ type: "set_auto_compaction", enabled: false });
+    await this.sendCommand({ type: "set_auto_retry", enabled: this.settings.retry?.enabled === true });
+    await this.sendCommand({ type: "set_auto_compaction", enabled: this.settings.compaction?.enabled === true });
+  }
+
+  /**
+   * Closes the session and opens it again from its file: Pi tears the session down (`session_shutdown`) and starts
+   * it anew with `session_start` reason `resume`, as a user resuming it would.
+   */
+  async resume() {
+    /** @type {any} */
+    const state = await this.sendCommand({ type: "get_state" });
+    const sessionFile = state.data?.sessionFile;
+    if (typeof sessionFile !== "string" || sessionFile === "") throw new Error("Pi RPC session has no file to resume");
+    /** @type {any} */
+    const result = await this.sendCommand({ type: "switch_session", sessionPath: sessionFile });
+    if (result.data?.cancelled === true) throw new Error("Pi cancelled resuming the session");
+    return { sessionFile };
   }
 
   async resolveSkillCommand(targetPath) {
@@ -145,22 +195,31 @@ class RpcSession {
       resolveSettled = resolve;
       rejectSettled = reject;
     });
+    // An unprompted run still going when the prompt is sent becomes part of this turn.
+    const joined = this.unprompted;
+    this.unprompted = null;
     this.activeTurn = {
       turn: turnNumber,
       prompt,
-      response: "",
-      transcript: [],
-      successfulReads: [],
-      toolActivity: [],
-      calls: new Map(),
+      response: joined?.response ?? "",
+      transcript: joined?.transcript ?? [],
+      successfulReads: joined?.successfulReads ?? [],
+      toolActivity: joined?.toolActivity ?? [],
+      calls: joined?.calls ?? new Map(),
       promptAccepted: false,
-      usage: null,
-      assistantError: null,
+      usage: joined?.usage ?? null,
+      assistantError: joined?.assistantError ?? null,
       settled: false,
       resolveSettled,
       rejectSettled,
     };
-    await Promise.all([this.sendCommand({ type: "prompt", message: prompt }), settlement]);
+    joined?.resolveSettled();
+    // Pi refuses a prompt while it runs, unless told how to queue it. A run a notice starts as the prompt is sent
+    // takes the prompt as its follow-up, and the turn records both.
+    await Promise.all([
+      this.sendCommand({ type: "prompt", message: prompt, streamingBehavior: "followUp" }),
+      settlement,
+    ]);
     const result = finishTurn(this.activeTurn);
     this.activeTurn = null;
     return result;
@@ -238,6 +297,61 @@ class RpcSession {
     } else if (event.type === "response") this.observeResponse(event);
     else if (event.type === "agent_settled") this.observeSettlement();
     else if (this.activeTurn !== null) this.observeTurnEvent(event);
+    else if (event.type === "agent_start" || this.unprompted !== null) this.observeUnpromptedEvent(event);
+  }
+
+  /** Events of a run Pi started without a prompt; it opens with agent_start. */
+  observeUnpromptedEvent(event) {
+    if (this.unprompted === null) {
+      let resolveSettled;
+      const settled = new Promise((resolve) => {
+        resolveSettled = resolve;
+      });
+      this.unprompted = {
+        response: "",
+        transcript: [],
+        successfulReads: [],
+        toolActivity: [],
+        calls: new Map(),
+        usage: null,
+        assistantError: null,
+        settled,
+        resolveSettled,
+      };
+    }
+    this.observeTurnEvent(event, this.unprompted);
+  }
+
+  /** Resolves once no unprompted run is in progress. */
+  async unpromptedSettled() {
+    while (this.unprompted !== null && this.terminationReason === null) {
+      const { settled } = this.unprompted;
+      await Promise.race([settled, this.closed.catch(() => {})]);
+      if (this.unprompted?.settled === settled && this.terminationReason === null) {
+        // The process closed while the run was in progress; nothing more will arrive.
+        break;
+      }
+    }
+  }
+
+  /** An unprompted run settled: its activity joins the latest turn, as if that turn had continued. */
+  settleUnprompted() {
+    const run = this.unprompted;
+    this.unprompted = null;
+    const turn = this.turns.at(-1);
+    if (turn === undefined) {
+      run.resolveSettled();
+      this.failProtocol("unprompted-run-before-first-turn", { line: this.eventLine });
+      return;
+    }
+    turn.transcript.push(...run.transcript);
+    turn.successfulReads.push(...run.successfulReads);
+    turn.toolActivity.push(...run.toolActivity);
+    if (run.response !== "") turn.response = run.response;
+    turn.usage = mergeUsage(turn.usage, run.usage);
+    turn.assistantError ??= run.assistantError;
+    turn.unpromptedRuns = (turn.unpromptedRuns ?? 0) + 1;
+    run.resolveSettled();
   }
 
   observeResponse(event) {
@@ -268,6 +382,10 @@ class RpcSession {
   }
 
   observeSettlement() {
+    if (this.activeTurn === null && this.unprompted !== null) {
+      this.settleUnprompted();
+      return;
+    }
     if (this.activeTurn === null || this.activeTurn.settled) {
       this.failProtocol("unexpected-agent-settled", { line: this.eventLine });
       return;
@@ -276,7 +394,7 @@ class RpcSession {
     this.activeTurn.resolveSettled();
   }
 
-  observeTurnEvent(event) {
+  observeTurnEvent(event, turn = this.activeTurn) {
     if (event.type === "tool_execution_start") {
       const activity = {
         toolCallId: event.toolCallId ?? null,
@@ -285,39 +403,39 @@ class RpcSession {
         completed: false,
         isError: null,
       };
-      this.activeTurn.toolActivity.push(activity);
-      if (typeof event.toolCallId === "string") this.activeTurn.calls.set(event.toolCallId, activity);
+      turn.toolActivity.push(activity);
+      if (typeof event.toolCallId === "string") turn.calls.set(event.toolCallId, activity);
       return;
     }
     if (event.type === "tool_execution_end") {
-      this.observeToolEnd(event);
+      this.observeToolEnd(event, turn);
       return;
     }
-    if (event.type === "message_end" && event.message) this.observeMessage(event.message);
+    if (event.type === "message_end" && event.message) this.observeMessage(event.message, turn);
   }
 
-  observeToolEnd(event) {
-    const activity = this.activeTurn.calls.get(event.toolCallId);
+  observeToolEnd(event, turn) {
+    const activity = turn.calls.get(event.toolCallId);
     if (!activity) return;
     activity.completed = true;
     activity.isError = event.isError === true;
     if (event.isError !== true && activity.toolName === "read" && typeof activity.args?.path === "string") {
-      this.activeTurn.successfulReads.push(activity.args.path);
+      turn.successfulReads.push(activity.args.path);
     }
   }
 
-  observeMessage(message) {
-    this.activeTurn.transcript.push(message);
+  observeMessage(message, turn) {
+    turn.transcript.push(message);
     this.transcript.push(message);
     if (message.role !== "assistant") return;
-    this.activeTurn.response = messageText(message);
-    this.activeTurn.usage = mergeUsage(this.activeTurn.usage, message.usage);
+    turn.response = messageText(message);
+    turn.usage = mergeUsage(turn.usage, message.usage);
     this.observedModel = {
       provider: message.provider ?? null,
       model: message.model ?? null,
     };
     if (message.stopReason === "error" || message.stopReason === "aborted") {
-      this.activeTurn.assistantError = message.errorMessage ?? `assistant stopped with ${message.stopReason}`;
+      turn.assistantError = message.errorMessage ?? `assistant stopped with ${message.stopReason}`;
     }
   }
 

package/skills/evaluate-behavior/scripts/lib/identity.mjs

@@ -0,0 +1,82 @@
+import { execFile } from "node:child_process";
+import { realpath } from "node:fs/promises";
+import path from "node:path";
+import { promisify } from "node:util";
+
+const run = promisify(execFile);
+
+/**
+ * The identity of the working-tree inputs a run reads: the definition root's HEAD commit, and which of the
+ * inputs differ from it (modified, untracked, or ignored). A run is canonical only when nothing differs, so
+ * every input it read can be recovered from that commit.
+ *
+ * @param {string} root the definition root
+ * @param {string[]} inputs absolute paths of files and directories the run reads from the working tree
+ * @returns {Promise<{commit: string | null, uncommitted: string[], repository: boolean}>}
+ */
+export async function inputIdentity(root, inputs) {
+  const canonicalRoot = await realpath(root);
+  let toplevel;
+  try {
+    toplevel = await realpath((await git(canonicalRoot, ["rev-parse", "--show-toplevel"])).trim());
+  } catch {
+    return { commit: null, uncommitted: [], repository: false };
+  }
+  const commit = await git(toplevel, ["rev-parse", "--verify", "HEAD"])
+    .then((output) => output.trim())
+    .catch(() => null);
+  const paths = [];
+  for (const input of inputs) {
+    const resolved = await realpath(input).catch(() => path.resolve(input));
+    paths.push(path.relative(toplevel, resolved) || ".");
+  }
+  if (paths.length === 0) return { commit, uncommitted: [], repository: true };
+  const status = await git(toplevel, [
+    "status",
+    "--porcelain=v1",
+    "-z",
+    "--untracked-files=all",
+    "--ignored=matching",
+    "--",
+    ...paths,
+  ]);
+  const uncommitted = new Set();
+  const entries = status.split("\0").filter(Boolean);
+  for (let index = 0; index < entries.length; index += 1) {
+    const entry = entries[index];
+    const code = entry.slice(0, 2);
+    uncommitted.add(
+      path
+        .relative(canonicalRoot, path.join(toplevel, entry.slice(3)))
+        .split(path.sep)
+        .join("/"),
+    );
+    if (code[0] === "R" || code[0] === "C") index += 1; // a rename or copy carries its source path next
+  }
+  if (commit === null) {
+    for (const entry of paths) uncommitted.add(path.relative(canonicalRoot, path.join(toplevel, entry)));
+  }
+  return { commit, uncommitted: [...uncommitted].sort(), repository: true };
+}
+
+/** The message that refuses a canonical run whose inputs cannot be recovered from a commit. */
+export function identityRefusal(identity) {
+  if (!identity.repository) {
+    return "the evaluation inputs are not in a Git repository, so no commit identifies them; run with --dry-run";
+  }
+  if (identity.uncommitted.length === 0) return null;
+  const shown = identity.uncommitted.slice(0, 10).join(", ");
+  const more = identity.uncommitted.length > 10 ? ` and ${identity.uncommitted.length - 10} more` : "";
+  return `uncommitted inputs: ${shown}${more}; commit them, or run with --dry-run`;
+}
+
+/** The installed host's version, as `pi --version` prints it. */
+export async function installedHostVersion(command, environment) {
+  const { stdout } = await run(command, ["--version"], { env: environment, timeout: 30000 });
+  return stdout.trim().split(/\s+/).pop() ?? "";
+}
+
+async function git(cwd, args) {
+  const { stdout } = await run("git", args, { cwd, maxBuffer: 16 * 1024 * 1024 });
+  return stdout;
+}